From c82c88658089279ea0cf381026cd4db723507113 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 14 Oct 2026 16:53:56 -0400 Subject: [PATCH 1/6] Validation: only the programs the reform changes Baseline comparisons listed every scorecard claim whose program variable the reform reached within six formula steps, so a CDCC change (report 4332) also showed the refundable CTC, the saver's credit and the Maryland and South Carolina EITCs. A comparison now shows only when it measures a program the reform changes (named in a changed parameter's path, or in a variable that reads that parameter directly) and only in the report's jurisdiction: a nationwide report keeps national comparisons and a state's only when the reform changes that state's rules; a state report keeps that state's and the national ones. When none applies, one plain line says so. The rule is one pure function, relevantComparisons. Co-Authored-By: Claude Opus 5.5 --- .../components/flagship/ValidationPanel.tsx | 79 +++++- .../flagship/report/ReportValidation.tsx | 14 +- .../components/flagship/report/ReportView.tsx | 1 + app/src/libs/flagship/comparisonRelevance.ts | 205 ++++++++++++++ .../libs/flagship/comparisonRelevanceMocks.ts | 253 ++++++++++++++++++ .../libs/flagship/scorecardComparisonMocks.ts | 2 + .../flagship/ValidationPanel.test.tsx | 77 +++++- .../flagship/report/ReportValidation.test.tsx | 92 +++++-- .../libs/flagship/comparisonRelevance.test.ts | 152 +++++++++++ 9 files changed, 824 insertions(+), 51 deletions(-) create mode 100644 app/src/libs/flagship/comparisonRelevance.ts create mode 100644 app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts create mode 100644 app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts diff --git a/app/src/components/flagship/ValidationPanel.tsx b/app/src/components/flagship/ValidationPanel.tsx index 2866d1771..f226ab099 100644 --- a/app/src/components/flagship/ValidationPanel.tsx +++ b/app/src/components/flagship/ValidationPanel.tsx @@ -3,6 +3,7 @@ import { BillValidation } from '@/api/billFeed'; import { describeDepth, RING_LABELS } from '@/components/flagship/CalibrationMatches'; import { Stack } from '@/components/ui'; import { colors, spacing, typography } from '@/designTokens'; +import { relevantComparisons, type ReformScope } from '@/libs/flagship/comparisonRelevance'; import { claimsFromBillValidation, METRIC_LABELS, @@ -12,6 +13,7 @@ import { ScorecardMatches, scorecardMatchesForPaths, } from '@/libs/flagship/modelValidation'; +import { loadParameterDependencies, readersOfPath } from '@/libs/flagship/parameterDependencies'; import type { RelatedScorecardClaim } from '@/libs/flagship/scorecardComparisons'; import { ScorecardClaimTable, type ScorecardClaimEntry } from './ScorecardComparisons'; import { @@ -34,7 +36,7 @@ import { * bill (official fiscal note, third-party analyses, verdict). * - BaselineComparisons: the PolicyEngine scorecard's comparisons of the * model's baseline with published measures, for the programs the - * reform touches. + * reform changes in the report's jurisdiction. */ function compactMoney(value: number): string { @@ -262,6 +264,7 @@ export function trackRecordClaims(trackRecord: ScorecardMatches): ScorecardClaim claim_id: claimId, country: 'US', geography: 'US', + program: row.program, name: metricLabel(row), source: row.source, url: row.sourceUrl ?? undefined, @@ -303,15 +306,58 @@ export function trackRecordClaims(trackRecord: ScorecardMatches): ScorecardClaim }); } +const NO_READERS: Record = {}; + +/** + * The variables that read each reform path directly, from the traced + * dependency map: `undefined` while it loads. The map is US-only, so + * another country, or a map that fails to load, matches on the paths alone. + */ +export function useReformReaders( + countryId: string, + paths: string[] +): Record | undefined { + const requestKey = JSON.stringify({ countryId, paths }); + const needsMap = countryId === 'us' && paths.length > 0; + const [state, setState] = useState<{ key: string; readers: Record }>(); + useEffect(() => { + if (!needsMap) { + return; + } + const request = JSON.parse(requestKey) as { paths: string[] }; + let cancelled = false; + loadParameterDependencies() + .then((map) => + Object.fromEntries(request.paths.map((path) => [path, readersOfPath(path, map)])) + ) + .catch(() => NO_READERS) + .then((readers) => { + if (!cancelled) { + setState({ key: requestKey, readers }); + } + }); + return () => { + cancelled = true; + }; + }, [requestKey, needsMap]); + if (!needsMap) { + return NO_READERS; + } + return state?.key === requestKey ? state.readers : undefined; +} + /** * How the model's baseline compares with published measures of the - * programs the reform touches: the scorecard's track record for those - * programs, then its other baseline claims linked to the reform. + * programs the reform changes: the scorecard's track record for those + * programs, then its other baseline claims linked to the reform. Only + * comparisons of a program the reform changes, in the report's + * jurisdiction, show; a loose link through the dependency graph does not. */ export function BaselineComparisons({ trackRecord, scorecardRows, baselineLabel, + reform, }: { trackRecord: ModelTrackRecord; /** The published scorecard's baseline claims for the reform; undefined while loading. */ @@ -321,11 +367,22 @@ export function BaselineComparisons({ * measures current law, so its comparisons do not apply. */ baselineLabel?: string | null; + /** The reform and the report's region, which pick the comparisons; undefined while loading. */ + reform: ReformScope | undefined; }) { - const entries = [ + const candidates = [ ...(trackRecord ? trackRecordClaims(trackRecord) : []), ...(scorecardRows ?? []).map((row) => ({ row })), ]; + const relevant = new Set( + reform + ? relevantComparisons( + candidates.map((entry) => entry.row), + reform + ) + : [] + ); + const entries = candidates.filter((entry) => relevant.has(entry.row)); let content; if (baselineLabel) { content = ( @@ -335,21 +392,17 @@ export function BaselineComparisons({ ); } else if (entries.length > 0) { content = ; - } else if (trackRecord === undefined || scorecardRows === undefined) { + } else if (trackRecord === undefined || scorecardRows === undefined || reform === undefined) { content = ; - } else if (trackRecord === null) { + } else if (trackRecord === null || scorecardRows === null) { content = Baseline comparisons are temporarily unavailable.; - } else if (trackRecord.reachedCount > 0) { - // Silence would read as "fine". State credits are the usual case: the - // scorecard measures federal programs and poverty, not state credits. + } else { + // Silence would read as "fine"; loose matches would read as evidence. content = ( - None of the {trackRecord.reachedCount} variables this reform moves is measured by the - scorecard. + No scorecard comparison measures the programs this reform changes. ); - } else { - content = No baseline comparison covers this reform.; } return ( ); diff --git a/app/src/components/flagship/report/ReportView.tsx b/app/src/components/flagship/report/ReportView.tsx index 6c9f72a98..fb54e6b8d 100644 --- a/app/src/components/flagship/report/ReportView.tsx +++ b/app/src/components/flagship/report/ReportView.tsx @@ -415,6 +415,7 @@ export default function ReportView({ p.path)} + region={region} calibration={calibration} calibrationPin={calibrationPin} trackRecord={trackRecord} diff --git a/app/src/libs/flagship/comparisonRelevance.ts b/app/src/libs/flagship/comparisonRelevance.ts new file mode 100644 index 000000000..f8b8afbbe --- /dev/null +++ b/app/src/libs/flagship/comparisonRelevance.ts @@ -0,0 +1,205 @@ +/** + * Which baseline comparisons describe a report: a comparison counts only + * when it measures a program the reform changes, in the report's + * jurisdiction. + * + * A program counts as changed when its name appears in a changed + * parameter's path (gov.irs.credits.cdcc.… changes the CDCC) or in the + * name of a variable that reads a changed parameter directly + * (capped_count_cdcc_eligible; sc_cdcc_potential is South Carolina's + * CDCC). Programs further down the dependency graph, such as a state + * EITC that a federal credit reaches through the tax it reduces, are the + * tax-benefit system echoing the change, not programs the reform changes. + * + * Jurisdiction, US only: a nationwide report keeps the national + * comparisons of the federal programs the reform changes, and a state's + * comparisons only when the reform changes that state's own rules. A + * state report keeps that state's comparisons (its own programs, and + * federal programs measured there) and the national comparisons of the + * federal programs the reform changes. Other states never show. + */ +import { normalizeParameterPath } from './parameterDependencies'; + +/** What the relevance check needs to know about a comparison. */ +export interface ComparisonScope { + /** The program the comparison measures, as the scorecard names it: "cdcc", "md_eitc". */ + program?: string | null; + /** "US" for a national comparison, else a state code. */ + geography?: string | null; + /** "parameter": the comparison's receipt names a parameter the reform changes. */ + matchBasis?: string; +} + +/** The reform behind a report, and where the report looks. */ +export interface ReformScope { + countryId: string; + /** The reform's parameter paths. */ + paths: string[]; + /** The variables that read each path directly, from the traced dependency map. */ + readers: Record; + /** The report's region: "us", "state/md", "congressional_district/MD-05". */ + region?: string | null; +} + +const NATIONAL = 'national'; + +const US_STATES = new Set( + ( + 'AL AK AZ AR CA CO CT DE DC FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO MT NE NV NH ' + + 'NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY' + ).split(' ') +); + +/** Words that qualify a program, not name it: refundable_ctc and non_refundable_ctc are the CTC. */ +const QUALIFIERS = new Set([ + 'refundable', + 'nonrefundable', + 'non', + 'refund', + 'credit', + 'credits', + 'tax', + 'deduction', +]); + +/** Scorecard program ids that are not model names, and the model names they stand for. */ +const PROGRAM_ALIASES: Record = { + co_fac: ['co_family_affordability_credit'], + housing: ['hud'], + ma_cftc: ['ma_child_and_family_credit'], + ny_escc: ['ny_ctc'], + or_kids_credit: ['or_ctc'], + va_eitc_cli: ['va_eitc', 'va_low_income_credit'], + wa_wftc: ['wa_working_families_tax_credit'], + wi_eic: ['wi_earned_income_credit'], +}; + +/** A program the reform changes: its name, whose program it is, and where the change applies. */ +interface ProgramSignal { + tokens: string[]; + /** National, a state code, or null for a local or territory program. */ + owner: string | null; + /** National or a state code; null for a local or territory parameter. */ + applies: string | null; +} + +function stateCode(code: string | null | undefined): string | null { + const upper = code?.toUpperCase(); + return upper && US_STATES.has(upper) ? upper : null; +} + +/** Whose parameter a path is: gov.states.md.… is Maryland's, gov.local.… no state's. */ +function pathScope(segments: string[]): string | null { + const rest = segments[1] === 'contrib' ? segments.slice(2) : segments.slice(1); + if (rest[0] === 'states') { + return stateCode(rest[1]); + } + if (rest[0] === 'local' || rest[0] === 'territories') { + return null; + } + return NATIONAL; +} + +function reformSignals(reform: ReformScope, us: boolean): ProgramSignal[] { + return reform.paths.flatMap((path) => { + const segments = normalizeParameterPath(path).split('.'); + const scope = pathScope(segments); + // The last segment names a setting (max, rate, min_children_for_ss_taxes_minus_eitc). + const own: ProgramSignal = { + tokens: segments.slice(0, -1).flatMap((segment) => segment.split('_')), + owner: scope, + applies: scope, + }; + const readers = (us ? (reform.readers[path] ?? []) : []).map((variable): ProgramSignal => { + const [first, ...rest] = variable.split('_'); + const state = rest.length > 0 ? stateCode(first) : null; + // An unprefixed variable reading a state's parameter is a federal + // program the state runs, such as a SNAP option. + return state + ? { tokens: rest, owner: state, applies: scope } + : { tokens: variable.split('_'), owner: scope === null ? null : NATIONAL, applies: scope }; + }); + return [own, ...readers]; + }); +} + +/** The program names a comparison measures, each with whose program it is. */ +function comparisonPrograms( + program: string, + geography: string +): Array<{ owner: string; family: string[] }> { + return program + .split('+') + .flatMap((id) => PROGRAM_ALIASES[id] ?? [id]) + .map((name) => { + const tokens = name.split('_'); + // "md_eitc" measured in Maryland is Maryland's program; "eitc" there is the federal one. + const statesOwn = + geography !== NATIONAL && tokens.length > 1 && stateCode(tokens[0]) === geography; + const named = statesOwn ? tokens.slice(1) : tokens; + const family = named.filter((token) => !QUALIFIERS.has(token)); + return { + owner: statesOwn ? geography : NATIONAL, + family: family.length > 0 ? family : named, + }; + }); +} + +function containsRun(tokens: string[], run: string[]): boolean { + for (let start = 0; start + run.length <= tokens.length; start++) { + if (run.every((token, offset) => tokens[start + offset] === token)) { + return true; + } + } + return false; +} + +/** Whether a change that applies in `applies` belongs in a report on `report`, for a comparison in `geography`. */ +function reaches(applies: string | null, geography: string, report: string): boolean { + if (geography === NATIONAL) { + return true; + } + // A nationwide report shows a state's comparison only when the reform + // changes that state's own rules, not a federal rule the state reads. + return report === NATIONAL + ? applies === geography + : applies === NATIONAL || applies === geography; +} + +/** + * The comparisons that describe this reform in this report's + * jurisdiction, in their original order. + */ +export function relevantComparisons( + comparisons: T[], + reform: ReformScope +): T[] { + const us = reform.countryId.toLowerCase() === 'us'; + const signals = reformSignals(reform, us); + const reportState = us + ? /^(?:state\/([a-z]{2})|congressional_district\/([a-z]{2})-\d+)$/i.exec(reform.region ?? '') + : null; + const report = (reportState && stateCode(reportState[1] ?? reportState[2])) || NATIONAL; + return comparisons.filter((comparison) => { + const geography = (us && stateCode(comparison.geography)) || NATIONAL; + // Another state's comparison never describes a state report. + if (report !== NATIONAL && geography !== NATIONAL && geography !== report) { + return false; + } + // A receipt that names a changed parameter scores the program itself. + if (comparison.matchBasis === 'parameter') { + return signals.some((signal) => reaches(signal.applies, geography, report)); + } + if (!comparison.program) { + return false; + } + return comparisonPrograms(comparison.program, geography).some(({ owner, family }) => + signals.some( + (signal) => + signal.owner === owner && + containsRun(signal.tokens, family) && + reaches(signal.applies, geography, report) + ) + ); + }); +} diff --git a/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts b/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts new file mode 100644 index 000000000..32ad63a83 --- /dev/null +++ b/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts @@ -0,0 +1,253 @@ +import type { ReformScope } from '@/libs/flagship/comparisonRelevance'; +import type { ParameterDependencyMap } from '@/libs/flagship/parameterDependencies'; +import type { RelatedScorecardClaim } from '@/libs/flagship/scorecardComparisons'; + +/** + * Report 4332: nationwide, 2026, one change — the CDCC's limit on + * qualifying people (gov.irs.credits.cdcc.eligibility.max) from 2 to 4. + * Its direct readers are the traced map's, so the state CDCCs that read + * the federal limit are among them. + */ +export const CDCC_ELIGIBILITY_MAX_PATH = 'gov.irs.credits.cdcc.eligibility.max'; +export const CDCC_DIRECT_READERS = [ + 'ar_federal_cdcc', + 'ca_cdcc_relevant_expenses', + 'capped_count_cdcc_eligible', + 'md_dependent_care_subtraction', + 'sc_cdcc_potential', +]; +export const CTC_BASE_PATH = 'gov.irs.credits.ctc.amount.base[0].amount'; +export const CTC_DIRECT_READERS = [ + 'co_federal_ctc_child_individual_maximum', + 'ctc_child_individual_maximum', + 'ctc_qualifying_child', +]; +/** A CTC rule whose own name mentions the EITC. */ +export const CTC_PHASE_IN_EITC_PATH = + 'gov.irs.credits.ctc.refundable.phase_in.min_children_for_ss_taxes_minus_eitc'; +export const MD_EITC_MATCH_PATH = + 'gov.states.md.tax.income.credits.eitc.refundable.married_or_has_child.match'; +export const EITC_MAX_PATH = 'gov.irs.credits.eitc.max[0].amount'; +export const DC_SNAP_MINIMUM_PATH = 'gov.states.dc.dhs.snap.min_allotment.amount'; +export const NYC_EITC_PATH = 'gov.local.ny.nyc.tax.income.credits.eitc.percent'; +export const WA_WFTC_PATH = + 'gov.states.wa.tax.income.credits.working_families_tax_credit.max_amount'; +export const STANDARD_DEDUCTION_PATH = 'gov.irs.deductions.standard.amount.SINGLE'; +export const SNAP_DEDUCTION_PATH = 'gov.usda.snap.income.deductions.standard'; +export const REFUNDABLE_CTC_MAX_PATH = 'gov.irs.credits.ctc.refundable.individual_max'; +export const UT_CTC_AMOUNT_PATH = 'gov.states.ut.tax.income.credits.ctc.amount'; + +export const NATIONWIDE = 'us'; +export const MARYLAND = 'state/md'; +export const SOUTH_CAROLINA = 'state/sc'; +export const UTAH = 'state/ut'; +export const MARYLAND_DISTRICT = 'congressional_district/MD-05'; + +/** A US reform scope for these paths; readers default to none. */ +export const usReform = ( + paths: string[], + region: string, + readers: Record = {} +): ReformScope => ({ countryId: 'us', paths, readers, region }); + +export const REPORT_4332_REFORM = usReform([CDCC_ELIGIBILITY_MAX_PATH], NATIONWIDE, { + [CDCC_ELIGIBILITY_MAX_PATH]: CDCC_DIRECT_READERS, +}); + +/** The traced map's slice for report 4332's path, for page-level tests. */ +export const cdccDependencyMap: ParameterDependencyMap = { + generatedAt: '2026-09-10T13:20:23+00:00', + model: { + package: 'policyengine-us', + version: '1.824.8', + coreVersion: '3.32.5', + dataset: 'us_multi_year_2024_2035', + households: 2000, + year: 2026, + }, + readers: { [CDCC_ELIGIBILITY_MAX_PATH]: CDCC_DIRECT_READERS }, + consumers: {}, +}; + +const baselineClaim = ( + overrides: Partial & Pick +): RelatedScorecardClaim => ({ + country: 'US', + source: 'irs_soi', + geography: 'US', + metric: 'tax_credits_applied', + unit_concept: 'usd', + period: 2023, + window: 'TY2023', + time_basis: 'annual', + reform_framework: 'baseline', + reform_key: '03395a10966bb849', + external_value: 0, + claim_baseline: 'current_law', + matchBasis: 'program', + latest: { + value: 0, + status: 'constructed', + engine_version: '2.2.1', + data_bundle: 'populace-us-2024-spm-20260915', + construction: 'level:cdcc', + policyengine_variables: [], + annotations: [], + baseline: 'current_law', + }, + ...overrides, +}); + +/** The six claims the scorecard matched for report 4332, as its feed records them. */ +export const ACTC_CLAIM = baselineClaim({ + claim_id: 'dfc756e1df8119360e71', + name: 'Additional child tax credit (refundable)', + program: 'refundable_ctc', + url: 'https://www.irs.gov/pub/irs-soi/23in33ar.xls', + external_value: 34_533_251_000, + latest: { value: 37_566_830_646, status: 'constructed', construction: 'level:refundable_ctc' }, +}); +export const TREASURY_CDCC_CLAIM = baselineClaim({ + claim_id: 'd5d06d635b1bba810311', + source: 'treasury', + name: 'Child & Dependent Care Credit', + program: 'cdcc', + metric: 'tax_expenditure', + url: 'https://home.treasury.gov/policy-issues/tax-policy/tax-expenditures', + period: 2024, + window: 'FY2024', + time_basis: 'fiscal_year', + external_value: 3_690_000_000, + latest: { value: 3_999_831_868, status: 'constructed', construction: 'repeal_delta:income_tax' }, +}); +export const SOI_CDCC_CLAIM = baselineClaim({ + claim_id: 'a30038fadf850d62d6f5', + name: 'Child and dependent care credit', + program: 'cdcc', + url: 'https://www.irs.gov/pub/irs-soi/23in33ar.xls', + external_value: 3_468_910_000, + latest: { value: 4_348_964_473, status: 'constructed', construction: 'level:cdcc' }, +}); +export const MD_EITC_CLAIM = baselineClaim({ + claim_id: 'a0fc12d9e7bbddfdc301', + source: 'md_admin', + name: 'Maryland EITC (refundable + nonrefundable)', + program: 'md_eitc', + geography: 'MD', + metric: 'benefit_cost', + period: 2024, + window: undefined, + external_value: 550_881_492, + latest: { value: 489_503_006, status: 'comparable', construction: 'level:md_eitc' }, +}); +export const SAVERS_CREDIT_CLAIM = baselineClaim({ + claim_id: '1dba5bf211130e2ed566', + name: 'Retirement savings contribution (saver’s) credit', + program: 'savers_credit', + external_value: 2_000_000_000, + latest: { value: 2_036_000_000, status: 'constructed', construction: 'level:savers_credit' }, +}); +export const SC_EITC_CLAIM = baselineClaim({ + claim_id: '4ed16ed1e0fce7e35aac', + source: 'sc_admin', + name: 'South Carolina EITC (125% nonrefundable)', + program: 'sc_eitc', + geography: 'SC', + metric: 'benefit_cost', + period: 2025, + window: 'FY2024-25', + time_basis: 'fiscal_year', + external_value: 23_200_000, + latest: { value: 14_700_000, status: 'constructed', construction: 'level:sc_eitc' }, +}); +export const REPORT_4332_CLAIMS = [ + ACTC_CLAIM, + TREASURY_CDCC_CLAIM, + SOI_CDCC_CLAIM, + MD_EITC_CLAIM, + SAVERS_CREDIT_CLAIM, + SC_EITC_CLAIM, +]; + +/** Other claims the rules have to place. */ +export const NONREFUNDABLE_CTC_CLAIM = baselineClaim({ + claim_id: 'soi-ctc-nonrefundable', + name: 'Child tax credit + ODC (nonrefundable)', + program: 'non_refundable_ctc', +}); +export const JCT_CTC_CLAIM = baselineClaim({ + claim_id: 'jct-te-ctc', + source: 'jct', + name: 'Child Tax Credit (incl. ODC)', + program: 'ctc', + metric: 'tax_expenditure', +}); +export const COLORADO_CTC_CLAIM = baselineClaim({ + claim_id: 'co-ctc', + source: 'co_admin', + name: 'Colorado Child Tax Credit', + program: 'co_ctc', + geography: 'CO', +}); +export const SC_CDCC_CLAIM = baselineClaim({ + claim_id: 'sc-cdcc', + source: 'sc_admin', + name: 'South Carolina child and dependent care credit', + program: 'sc_cdcc', + geography: 'SC', +}); +export const US_EITC_CLAIM = baselineClaim({ + claim_id: 'eitc-participation-us', + source: 'irs_eitc_participation', + name: 'EITC participation rate, United States', + program: 'eitc', + matchBasis: 'variable', +}); +export const MARYLAND_FEDERAL_EITC_CLAIM = baselineClaim({ + claim_id: 'eitc-participation-md', + source: 'irs_eitc_participation', + name: 'EITC participation rate, Maryland', + program: 'eitc', + geography: 'MD', + matchBasis: 'variable', +}); +export const US_SNAP_CLAIM = baselineClaim({ + claim_id: 'snap-eligible-us', + source: 'fns_snap_rates', + name: 'People eligible for SNAP, United States', + program: 'snap', + matchBasis: 'variable', +}); +export const DC_SNAP_CLAIM = baselineClaim({ + claim_id: 'snap-eligible-dc', + source: 'fns_snap_rates', + name: 'People eligible for SNAP, District of Columbia', + program: 'snap', + geography: 'DC', + matchBasis: 'variable', +}); +export const WA_WFTC_CLAIM = baselineClaim({ + claim_id: 'wa-wftc', + source: 'wa_admin', + name: 'Washington Working Families Tax Credit', + program: 'wa_wftc', + geography: 'WA', +}); +/** A claim that records no program, linked only through a reached variable. */ +export const UNNAMED_PROGRAM_CLAIM = baselineClaim({ + claim_id: 'census-spm-child-poverty', + source: 'census', + name: 'United States SPM child poverty rate', + program: null, + matchBasis: 'variable', +}); +/** A UK claim whose receipt names the changed parameter. */ +export const UK_PARAMETER_CLAIM = baselineClaim({ + claim_id: 'uk-uc-child-element', + country: 'UK', + source: 'dwp', + name: 'Universal Credit child element', + program: null, + geography: 'UK', + matchBasis: 'parameter', +}); diff --git a/app/src/tests/fixtures/libs/flagship/scorecardComparisonMocks.ts b/app/src/tests/fixtures/libs/flagship/scorecardComparisonMocks.ts index 51b36a577..06f4e3f5f 100644 --- a/app/src/tests/fixtures/libs/flagship/scorecardComparisonMocks.ts +++ b/app/src/tests/fixtures/libs/flagship/scorecardComparisonMocks.ts @@ -52,4 +52,6 @@ export const utahBaselineClaim = scorecardClaim({ reform_framework: 'baseline', name: 'Utah Child Tax Credit', metric: 'benefit_cost', + program: 'ut_ctc', + geography: 'UT', }); diff --git a/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx b/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx index 046a9eff2..d04940f63 100644 --- a/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx +++ b/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx @@ -6,9 +6,30 @@ import { trackRecordClaims, } from '@/components/flagship/ValidationPanel'; import type { ScorecardMatches } from '@/libs/flagship/modelValidation'; +import { + NATIONWIDE, + REFUNDABLE_CTC_MAX_PATH, + REPORT_4332_CLAIMS, + REPORT_4332_REFORM, + SNAP_DEDUCTION_PATH, + SOI_CDCC_CLAIM, + STANDARD_DEDUCTION_PATH, + TREASURY_CDCC_CLAIM, + usReform, + UT_CTC_AMOUNT_PATH, + UTAH, +} from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; import { utahBaselineClaim } from '@/tests/fixtures/libs/flagship/scorecardComparisonMocks'; import { TRACKED_BILL } from '@/tests/fixtures/libs/flagship/trackedBillMocks'; +const NO_COMPARISON = 'No scorecard comparison measures the programs this reform changes.'; +const EMPTY_TRACK_RECORD: ScorecardMatches = { + modelVersion: '1.808.0', + reachedCount: 0, + programs: [], + rows: [], +}; + const SNAP_AND_CTC_TRACK_RECORD: ScorecardMatches = { modelVersion: '1.808.0', reachedCount: 12, @@ -101,42 +122,80 @@ describe('BillExternalEstimates', () => { describe('BaselineComparisons', () => { test('given the scorecard is unreachable then an honest note renders, not a blank card', () => { - render(); + render( + + ); expect(screen.getByText(/temporarily unavailable/i)).toBeInTheDocument(); expect(screen.getByRole('link', { name: /view scorecard/i })).toBeInTheDocument(); }); test('given matches are loading then the loading state renders', () => { - render(); + render( + + ); expect( screen.getByRole('status', { name: 'Loading baseline comparisons' }) ).toBeInTheDocument(); }); + test('given the reform is still being traced then rows wait instead of showing unchecked', () => { + render( + + ); + + expect( + screen.getByRole('status', { name: 'Loading baseline comparisons' }) + ).toBeInTheDocument(); + expect(screen.queryByRole('table')).not.toBeInTheDocument(); + }); + test('given the reform reaches nothing then a plain line says no comparison covers it', () => { render( ); - expect(screen.getByText('No baseline comparison covers this reform.')).toBeInTheDocument(); + expect(screen.getByText(NO_COMPARISON)).toBeInTheDocument(); }); - test('given reached variables the scorecard does not measure then the note says so', () => { + test('given only loose matches then one plain line says so instead of listing them', () => { render( + ); + + expect(screen.getByText(NO_COMPARISON)).toBeInTheDocument(); + expect(screen.queryByRole('table')).not.toBeInTheDocument(); + }); + + test('given report 4332 then only the CDCC rows show', () => { + render( + ); + const table = screen.getByRole('table', { name: 'Baseline comparisons' }); + expect(within(table).getAllByRole('row')).toHaveLength(3); expect( - screen.getByText(/none of the 30 variables this reform moves is measured/i) + within(table).getByRole('button', { name: TREASURY_CDCC_CLAIM.name }) ).toBeInTheDocument(); + expect(within(table).getByRole('button', { name: SOI_CDCC_CLAIM.name })).toBeInTheDocument(); + expect(within(table).queryByText(/Maryland|South Carolina|saver/i)).not.toBeInTheDocument(); }); test('given track record and scorecard rows then they share one table', () => { @@ -144,6 +203,7 @@ describe('BaselineComparisons', () => { ); @@ -169,6 +229,7 @@ describe('trackRecordClaims', () => { expect(snap.row.reform_framework).toBe('baseline'); expect(snap.row.calibration_relationship).toBe('held_out'); + expect(snap.row.program).toBe('snap'); expect(snap.presentation?.title).toBe('SNAP — Eligible people'); expect(snap.presentation?.externalValue).toBe('69.1M'); }); diff --git a/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx b/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx index 586cd20ea..3afa4029e 100644 --- a/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx +++ b/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx @@ -1,9 +1,18 @@ -import { render, screen, within } from '@test-utils'; +import { act, render, screen, within } from '@test-utils'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import ReportValidation from '@/components/flagship/report/ReportValidation'; import type { ScorecardComparisonsState } from '@/components/flagship/ScorecardComparisons'; import { BillExternalEstimates } from '@/components/flagship/ValidationPanel'; import { mockCalibrationMatches } from '@/tests/fixtures/libs/flagship/calibrationMatchingMocks'; +import { + CDCC_ELIGIBILITY_MAX_PATH, + cdccDependencyMap, + NATIONWIDE, + REPORT_4332_CLAIMS, + SOI_CDCC_CLAIM, + STANDARD_DEDUCTION_PATH, + TREASURY_CDCC_CLAIM, +} from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; import { CTC_PATH, scorecardClaim, @@ -14,6 +23,12 @@ import { TRACKED_BILL } from '@/tests/fixtures/libs/flagship/trackedBillMocks'; const mockUseScorecardComparisons = vi.fn<(countryId: string, paths: string[]) => ScorecardComparisonsState>(); +// The traced map's slice for report 4332, so the relevance check runs on known readers. +vi.mock('@/libs/flagship/parameterDependencies', async (importOriginal) => ({ + ...(await importOriginal()), + loadParameterDependencies: () => Promise.resolve(cdccDependencyMap), +})); + vi.mock('@/components/flagship/ScorecardComparisons', async (importOriginal) => ({ ...(await importOriginal()), useScorecardComparisons: (countryId: string, paths: string[]) => @@ -26,8 +41,9 @@ const UTAH_BASELINE = { ...utahBaselineClaim, matchBasis: 'program' as const }; const EMPTY_TRACK_RECORD = { modelVersion: '1.808.0', reachedCount: 12, programs: [], rows: [] }; const SAVED_REFORM_BASELINE = 'Smaller CTC'; -function renderValidation(props: Partial[0]> = {}) { - return render( +/** Renders the section and lets the dependency map resolve, as the page would. */ +async function renderValidation(props: Partial[0]> = {}) { + render( [0]> {...props} /> ); + await act(async () => {}); } const card = (name: string) => @@ -47,16 +64,16 @@ describe('ReportValidation', () => { mockUseScorecardComparisons.mockReturnValue(NO_SCORECARD_ROWS); }); - test('given a custom report then the external estimates, calibration and baseline cards render in order', () => { - renderValidation(); + test('given a custom report then the external estimates, calibration and baseline cards render in order', async () => { + await renderValidation(); expect( screen.getAllByRole('heading', { level: 3 }).map((heading) => heading.textContent) ).toEqual(['External estimates', 'Data calibration', 'Baseline comparisons']); }); - test('given a custom report without external estimates then one plain line says so', () => { - renderValidation(); + test('given a custom report without external estimates then one plain line says so', async () => { + await renderValidation(); expect( within(card('External estimates')).getByText( @@ -70,12 +87,12 @@ describe('ReportValidation', () => { ).toBeInTheDocument(); }); - test('given a custom report with scorer estimates then they show instead of the plain line', () => { + test('given a custom report with scorer estimates then they show instead of the plain line', async () => { mockUseScorecardComparisons.mockReturnValue({ ...NO_SCORECARD_ROWS, rows: [RELATED_REFORM], }); - renderValidation(); + await renderValidation(); const estimates = card('External estimates'); expect(within(estimates).getByRole('table', { name: 'Related reforms' })).toBeInTheDocument(); @@ -86,8 +103,8 @@ describe('ReportValidation', () => { ).not.toBeInTheDocument(); }); - test('given a bill report then its fiscal note leads the external estimates', () => { - renderValidation({ + test('given a bill report then its fiscal note leads the external estimates', async () => { + await renderValidation({ externalEstimates: ( ), @@ -107,16 +124,20 @@ describe('ReportValidation', () => { ).not.toBeInTheDocument(); }); - test('given scorecard baseline claims then they join the baseline comparisons, not the estimates', () => { + test('given report 4332 then the baseline card lists only the CDCC comparisons', async () => { mockUseScorecardComparisons.mockReturnValue({ ...NO_SCORECARD_ROWS, - rows: [UTAH_BASELINE], + rows: REPORT_4332_CLAIMS, }); - renderValidation(); + await renderValidation({ paths: [CDCC_ELIGIBILITY_MAX_PATH], region: NATIONWIDE }); + const baseline = card('Baseline comparisons'); + const table = await within(baseline).findByRole('table', { name: 'Baseline comparisons' }); expect( - within(card('Baseline comparisons')).getByRole('button', { name: 'Utah Child Tax Credit' }) - ).toBeInTheDocument(); + within(table) + .getAllByRole('button') + .map((button) => button.textContent) + ).toEqual([TREASURY_CDCC_CLAIM.name, SOI_CDCC_CLAIM.name]); expect( within(card('External estimates')).getByText( 'No external estimate is available for this reform.' @@ -124,18 +145,33 @@ describe('ReportValidation', () => { ).toBeInTheDocument(); }); - test('given the scorecard is still loading then the estimates card says it is loading', () => { + test('given baseline claims of other programs then one plain line says none applies', async () => { + mockUseScorecardComparisons.mockReturnValue({ + ...NO_SCORECARD_ROWS, + rows: REPORT_4332_CLAIMS, + }); + await renderValidation({ paths: [STANDARD_DEDUCTION_PATH], region: NATIONWIDE }); + + expect( + await within(card('Baseline comparisons')).findByText( + 'No scorecard comparison measures the programs this reform changes.' + ) + ).toBeInTheDocument(); + expect(within(card('Baseline comparisons')).queryByRole('table')).not.toBeInTheDocument(); + }); + + test('given the scorecard is still loading then the estimates card says it is loading', async () => { mockUseScorecardComparisons.mockReturnValue(undefined); - renderValidation(); + await renderValidation(); expect( within(card('External estimates')).getByRole('status', { name: 'Loading external estimates' }) ).toBeInTheDocument(); }); - test('given the scorecard is unavailable then the estimates card says so, not that none exists', () => { + test('given the scorecard is unavailable then the estimates card says so, not that none exists', async () => { mockUseScorecardComparisons.mockReturnValue(null); - renderValidation(); + await renderValidation(); expect( within(card('External estimates')).getByText( @@ -147,8 +183,8 @@ describe('ReportValidation', () => { ).not.toBeInTheDocument(); }); - test('given the coverage of the scorecard match then it is collapsed under the estimates', () => { - renderValidation(); + test('given the coverage of the scorecard match then it is collapsed under the estimates', async () => { + await renderValidation(); expect(within(card('External estimates')).getByText('Coverage')).toBeInTheDocument(); expect( @@ -156,12 +192,12 @@ describe('ReportValidation', () => { ).not.toBeVisible(); }); - test('given a custom baseline then both scorecard cards are marked not comparable', () => { + test('given a custom baseline then both scorecard cards are marked not comparable', async () => { mockUseScorecardComparisons.mockReturnValue({ ...NO_SCORECARD_ROWS, rows: [RELATED_REFORM, UTAH_BASELINE], }); - renderValidation({ baselineLabel: SAVED_REFORM_BASELINE }); + await renderValidation({ baselineLabel: SAVED_REFORM_BASELINE }); for (const name of ['External estimates', 'Baseline comparisons']) { expect(within(card(name)).getByText('Not comparable')).toBeInTheDocument(); @@ -179,8 +215,8 @@ describe('ReportValidation', () => { ).toBeInTheDocument(); }); - test('given a custom baseline then the scorecard is not asked and calibration still shows', () => { - renderValidation({ baselineLabel: SAVED_REFORM_BASELINE }); + test('given a custom baseline then the scorecard is not asked and calibration still shows', async () => { + await renderValidation({ baselineLabel: SAVED_REFORM_BASELINE }); expect(mockUseScorecardComparisons).toHaveBeenCalledWith('us', []); expect( @@ -188,8 +224,8 @@ describe('ReportValidation', () => { ).toBeInTheDocument(); }); - test('given current law then the scorecard is asked for the reform paths and nothing is marked', () => { - renderValidation(); + test('given current law then the scorecard is asked for the reform paths and nothing is marked', async () => { + await renderValidation(); expect(mockUseScorecardComparisons).toHaveBeenCalledWith('us', [CTC_PATH]); expect(screen.queryByText('Not comparable')).not.toBeInTheDocument(); diff --git a/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts b/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts new file mode 100644 index 000000000..8cd7bf4b3 --- /dev/null +++ b/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, test } from 'vitest'; +import { relevantComparisons } from '@/libs/flagship/comparisonRelevance'; +import { + ACTC_CLAIM, + CDCC_DIRECT_READERS, + CDCC_ELIGIBILITY_MAX_PATH, + COLORADO_CTC_CLAIM, + CTC_BASE_PATH, + CTC_DIRECT_READERS, + CTC_PHASE_IN_EITC_PATH, + DC_SNAP_CLAIM, + DC_SNAP_MINIMUM_PATH, + EITC_MAX_PATH, + JCT_CTC_CLAIM, + MARYLAND, + MARYLAND_DISTRICT, + MARYLAND_FEDERAL_EITC_CLAIM, + MD_EITC_CLAIM, + MD_EITC_MATCH_PATH, + NATIONWIDE, + NONREFUNDABLE_CTC_CLAIM, + NYC_EITC_PATH, + REPORT_4332_CLAIMS, + REPORT_4332_REFORM, + SAVERS_CREDIT_CLAIM, + SC_CDCC_CLAIM, + SC_EITC_CLAIM, + SOI_CDCC_CLAIM, + SOUTH_CAROLINA, + STANDARD_DEDUCTION_PATH, + TREASURY_CDCC_CLAIM, + UK_PARAMETER_CLAIM, + UNNAMED_PROGRAM_CLAIM, + US_EITC_CLAIM, + US_SNAP_CLAIM, + usReform, + WA_WFTC_CLAIM, + WA_WFTC_PATH, +} from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; + +const names = (claims: Array<{ name: string }>) => claims.map((claim) => claim.name); + +describe('relevantComparisons', () => { + test('given report 4332 then only the national CDCC comparisons remain', () => { + expect(names(relevantComparisons(REPORT_4332_CLAIMS, REPORT_4332_REFORM))).toEqual([ + TREASURY_CDCC_CLAIM.name, + SOI_CDCC_CLAIM.name, + ]); + }); + + test('given a federal CTC reform nationwide then only the national CTC comparisons remain', () => { + const reform = usReform([CTC_BASE_PATH], NATIONWIDE, { [CTC_BASE_PATH]: CTC_DIRECT_READERS }); + + expect( + relevantComparisons( + [...REPORT_4332_CLAIMS, NONREFUNDABLE_CTC_CLAIM, JCT_CTC_CLAIM, COLORADO_CTC_CLAIM], + reform + ) + ).toEqual([ACTC_CLAIM, NONREFUNDABLE_CTC_CLAIM, JCT_CTC_CLAIM]); + }); + + test('given a Maryland EITC reform then the Maryland EITC comparison remains, nationwide or in Maryland', () => { + const claims = [...REPORT_4332_CLAIMS, US_EITC_CLAIM, MARYLAND_FEDERAL_EITC_CLAIM]; + + for (const region of [NATIONWIDE, MARYLAND, MARYLAND_DISTRICT]) { + expect(relevantComparisons(claims, usReform([MD_EITC_MATCH_PATH], region))).toEqual([ + MD_EITC_CLAIM, + ]); + } + }); + + test('given a reform no comparison measures then none remain', () => { + expect( + relevantComparisons(REPORT_4332_CLAIMS, usReform([STANDARD_DEDUCTION_PATH], NATIONWIDE)) + ).toEqual([]); + }); + + test('given a federal rule a state reads then the state comparison shows in that state only', () => { + const readers = { [CDCC_ELIGIBILITY_MAX_PATH]: CDCC_DIRECT_READERS }; + const claims = [SOI_CDCC_CLAIM, SC_CDCC_CLAIM, MD_EITC_CLAIM]; + + expect( + relevantComparisons(claims, usReform([CDCC_ELIGIBILITY_MAX_PATH], SOUTH_CAROLINA, readers)) + ).toEqual([SOI_CDCC_CLAIM, SC_CDCC_CLAIM]); + expect( + relevantComparisons(claims, usReform([CDCC_ELIGIBILITY_MAX_PATH], NATIONWIDE, readers)) + ).toEqual([SOI_CDCC_CLAIM]); + }); + + test('given a federal reform then its comparison in a state shows only in that state report', () => { + const claims = [US_EITC_CLAIM, MARYLAND_FEDERAL_EITC_CLAIM, SC_EITC_CLAIM]; + + expect(relevantComparisons(claims, usReform([EITC_MAX_PATH], NATIONWIDE))).toEqual([ + US_EITC_CLAIM, + ]); + expect(relevantComparisons(claims, usReform([EITC_MAX_PATH], MARYLAND))).toEqual([ + US_EITC_CLAIM, + MARYLAND_FEDERAL_EITC_CLAIM, + ]); + }); + + test('given a state option of a federal program then that state and the nation show', () => { + const reform = usReform([DC_SNAP_MINIMUM_PATH], NATIONWIDE, { + [DC_SNAP_MINIMUM_PATH]: ['snap_min_allotment'], + }); + + expect(relevantComparisons([US_SNAP_CLAIM, DC_SNAP_CLAIM], reform)).toEqual([ + US_SNAP_CLAIM, + DC_SNAP_CLAIM, + ]); + }); + + test('given a setting named after another program then that program does not match', () => { + expect( + relevantComparisons( + [US_EITC_CLAIM, ACTC_CLAIM], + usReform([CTC_PHASE_IN_EITC_PATH], NATIONWIDE, { + [CTC_PHASE_IN_EITC_PATH]: ['co_refundable_ctc', 'ctc_phase_in'], + }) + ) + ).toEqual([ACTC_CLAIM]); + }); + + test('given a city credit then no federal or state comparison matches', () => { + const reform = usReform([NYC_EITC_PATH], NATIONWIDE, { [NYC_EITC_PATH]: ['nyc_eitc'] }); + + expect(relevantComparisons([US_EITC_CLAIM, MD_EITC_CLAIM], reform)).toEqual([]); + }); + + test('given a scorecard abbreviation then it matches the model name it stands for', () => { + expect(relevantComparisons([WA_WFTC_CLAIM], usReform([WA_WFTC_PATH], 'state/wa'))).toEqual([ + WA_WFTC_CLAIM, + ]); + }); + + test('given no recorded program then only a receipt naming the parameter keeps it', () => { + expect( + relevantComparisons( + [UNNAMED_PROGRAM_CLAIM, SAVERS_CREDIT_CLAIM], + usReform([CTC_BASE_PATH], NATIONWIDE) + ) + ).toEqual([]); + expect( + relevantComparisons([UK_PARAMETER_CLAIM], { + countryId: 'uk', + paths: ['gov.dwp.universal_credit.elements.child.amount'], + readers: {}, + region: 'uk', + }) + ).toEqual([UK_PARAMETER_CLAIM]); + }); +}); From 85388a125680a63dbbcd93e2937f917e7092b7f7 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 14 Oct 2026 16:54:57 -0400 Subject: [PATCH 2/6] Validation: a period shows only when it is recorded Every baseline row read "Period not recorded separately" under the PolicyEngine value, and claims without a time basis read "time basis not recorded". A period now shows only when the scorecard records it; an unknown one leaves the cell with just its value. Co-Authored-By: Claude Opus 5.5 --- .../flagship/ScorecardComparisons.tsx | 26 ++++++++++++------- .../components/flagship/ValidationPanel.tsx | 5 ++-- app/src/libs/flagship/scorecardComparisons.ts | 3 ++- .../flagship/ScorecardComparisons.test.tsx | 24 +++++++++++++++++ .../flagship/ValidationPanel.test.tsx | 8 ++++++ .../flagship/report/ReportValidation.test.tsx | 1 + 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/app/src/components/flagship/ScorecardComparisons.tsx b/app/src/components/flagship/ScorecardComparisons.tsx index 8be7bdfb7..c50371b6d 100644 --- a/app/src/components/flagship/ScorecardComparisons.tsx +++ b/app/src/components/flagship/ScorecardComparisons.tsx @@ -72,10 +72,11 @@ function displayTitle(title: string): string { .trim(); return cleaned ? cleaned[0].toUpperCase() + cleaned.slice(1) : title; } -function modelPeriod(row: RelatedScorecardClaim): string { +/** The model's period when the receipt records one; null says nothing, so nothing shows. */ +function modelPeriod(row: RelatedScorecardClaim): string | null { const receipt = [row.latest?.construction, ...(row.latest?.annotations ?? [])].join(' '); const years = [...new Set([...receipt.matchAll(/\bCY\s*(20\d{2})/gi)].map((match) => match[1]))]; - return years.length === 1 ? `Calendar year ${years[0]}` : 'Period not recorded separately'; + return years.length === 1 ? `Calendar year ${years[0]}` : null; } function comparisonReason( @@ -96,11 +97,12 @@ function comparisonReason( ); const calendar = pePeriod ?? modelPeriod(row); const fiscal = /fiscal|FY/i.test(externalPeriod ?? claimPeriod(row)); - if (baselineDiffers && fiscal && calendar.startsWith('Calendar year')) { + const calendarYear = calendar?.startsWith('Calendar year') ? calendar : null; + if (baselineDiffers && fiscal && calendarYear) { return 'Different starting policies and timing: the source uses a fiscal year; PolicyEngine uses a calendar year.'; } - if (fiscal && calendar.startsWith('Calendar year')) { - return `Different timing: the source uses a fiscal year; PolicyEngine uses ${calendar.toLowerCase()}.`; + if (fiscal && calendarYear) { + return `Different timing: the source uses a fiscal year; PolicyEngine uses ${calendarYear.toLowerCase()}.`; } if (baselineDiffers) { return 'The source and PolicyEngine start from different baseline policies.'; @@ -164,6 +166,7 @@ function ComparisonContext({ row }: { row: RelatedScorecardClaim }) { ], ]); if (isJctCtc(row)) { + const period = modelPeriod(row); const reversed = row.latest?.construction?.includes('scored as expiry reversal'); const stacked = row.latest?.baseline === 'obbba_stack_below__jcx_producer__obbba_child_tax_credit'; @@ -238,7 +241,7 @@ function ComparisonContext({ row }: { row: RelatedScorecardClaim }) { {reversed ? 'This run keeps other enacted tax changes in place while reversing the CTC expiry. JCT starts with only the bill’s tax-rate, standard-deduction and personal-exemption changes applied. Those different tax rules affect how much CTC families can receive.' : stacked - ? `This run adds the CTC changes after the bill’s tax-rate, standard-deduction and personal-exemption changes. Model period: ${modelPeriod(row)}; check the source year below.` + ? `This run adds the CTC changes after the bill’s tax-rate, standard-deduction and personal-exemption changes.${period ? ` Model period: ${period}; check the source year below.` : ''}` : 'The stored calculation does not provide enough detail to confirm the starting tax rules.'} @@ -380,7 +383,7 @@ export function ScorecardClaimDetails({ row, presentation }: ScorecardClaimEntry left={{ label: 'Published estimate', value: presentation?.externalValue ?? claimValue(row.external_value, row.unit_concept), - note: presentation?.externalPeriod ?? claimPeriod(row), + note: presentation?.externalPeriod ?? (claimPeriod(row) || undefined), }} right={{ label: @@ -388,7 +391,7 @@ export function ScorecardClaimDetails({ row, presentation }: ScorecardClaimEntry ? 'PolicyEngine estimate' : 'PolicyEngine scorecard estimate', value: presentation?.peValue ?? claimValue(latest?.value, row.unit_concept), - note: presentation?.modelPeriod ?? modelPeriod(row), + note: presentation?.modelPeriod ?? modelPeriod(row) ?? undefined, }} /> {status === 'constructed' && @@ -477,6 +480,9 @@ export function ScorecardClaimDetails({ row, presentation }: ScorecardClaimEntry /** One claim as a table row: what is compared, both values, and the gap. */ function ScorecardClaimRow({ row, presentation }: ScorecardClaimEntry) { const sourceUrl = safeUrl(row.url); + // Only a period that is recorded; "not recorded" under every value says nothing. + const publishedNote = presentation?.externalPeriod ?? claimPeriod(row); + const peNote = presentation?.modelPeriod ?? modelPeriod(row); return ( @@ -512,11 +518,11 @@ function ScorecardClaimRow({ row, presentation }: ScorecardClaimEntry) { {presentation?.externalValue ?? claimValue(row.external_value, row.unit_concept)} - {presentation?.externalPeriod ?? claimPeriod(row)} + {publishedNote && {publishedNote}} {presentation?.peValue ?? claimValue(row.latest?.value, row.unit_concept)} - {presentation?.modelPeriod ?? modelPeriod(row)} + {peNote && {peNote}} {gapText(row.latest?.value, row.external_value)} diff --git a/app/src/components/flagship/ValidationPanel.tsx b/app/src/components/flagship/ValidationPanel.tsx index f226ab099..88f4b9a87 100644 --- a/app/src/components/flagship/ValidationPanel.tsx +++ b/app/src/components/flagship/ValidationPanel.tsx @@ -297,8 +297,9 @@ export function trackRecordClaims(trackRecord: ScorecardMatches): ScorecardClaim presentation: { title: `${PROGRAM_LABELS[row.program] ?? row.program} — ${metricLabel(row)}`, sourceName: row.sourceName ?? row.source, - externalPeriod: row.period ?? 'Period not recorded', - modelPeriod: row.pePeriod ?? 'Period not recorded', + // A period shows only when the scorecard records it. + externalPeriod: row.period ?? undefined, + modelPeriod: row.pePeriod ?? undefined, externalValue: metricValue(row, row.externalValue), peValue: metricValue(row, row.peValue), }, diff --git a/app/src/libs/flagship/scorecardComparisons.ts b/app/src/libs/flagship/scorecardComparisons.ts index 24e48681b..77846627c 100644 --- a/app/src/libs/flagship/scorecardComparisons.ts +++ b/app/src/libs/flagship/scorecardComparisons.ts @@ -124,7 +124,8 @@ export function claimPeriod(row: ScorecardClaim): string { row.period_start != null && row.period_end != null ? `${row.period_start}–${row.period_end}` : String(row.period); - return `${period} · ${(row.time_basis ?? 'time basis not recorded').replaceAll('_', ' ')}`; + // An unrecorded time basis says nothing; the period alone stands. + return row.time_basis ? `${period} · ${row.time_basis.replaceAll('_', ' ')}` : period; } export function claimValue(value: number | null | undefined, unit: string): string { diff --git a/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx b/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx index 1c257b74f..98eac2f53 100644 --- a/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx +++ b/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx @@ -6,6 +6,7 @@ import { splitScorecardRows, useScorecardComparisons, } from '@/components/flagship/ScorecardComparisons'; +import { SOI_CDCC_CLAIM } from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; import { CTC_PATH, jctCtcClaim, @@ -88,6 +89,29 @@ describe('scorecard evidence presentation', () => { expect(screen.getByText(/Modeled Utah credit actually usable/)).toBeInTheDocument(); }); + test('given a claim without a recorded model period then none shows under its value', () => { + render( + + ); + + const cells = within(screen.getByRole('row', { name: new RegExp(`^${SOI_CDCC_CLAIM.name}`) })) + .getAllByRole('cell') + .map((cell) => cell.textContent); + expect(cells).toEqual(['$3.5BTY2023', '$4.3B', '25.4%']); + expect(screen.queryByText(/not recorded/i)).not.toBeInTheDocument(); + }); + + test('given a receipt that records the calendar year then the period shows under the value', () => { + render( + + ); + + expect(screen.getByText('Calendar year 2026')).toBeInTheDocument(); + }); + test('given an undocumented reform then explicitly identifies missing before-and-after rules', () => { render( { ).toBeInTheDocument(); expect(within(table).getByRole('button', { name: SOI_CDCC_CLAIM.name })).toBeInTheDocument(); expect(within(table).queryByText(/Maryland|South Carolina|saver/i)).not.toBeInTheDocument(); + expect(within(table).queryByText(/not recorded/i)).not.toBeInTheDocument(); }); test('given track record and scorecard rows then they share one table', () => { @@ -233,4 +234,11 @@ describe('trackRecordClaims', () => { expect(snap.presentation?.title).toBe('SNAP — Eligible people'); expect(snap.presentation?.externalValue).toBe('69.1M'); }); + + test('given a row without a model period then none is invented', () => { + const [snap] = trackRecordClaims(SNAP_AND_CTC_TRACK_RECORD); + + expect(snap.presentation?.modelPeriod).toBeUndefined(); + expect(snap.presentation?.externalPeriod).toBe('2023 average month'); + }); }); diff --git a/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx b/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx index 3afa4029e..7b869fb8d 100644 --- a/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx +++ b/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx @@ -138,6 +138,7 @@ describe('ReportValidation', () => { .getAllByRole('button') .map((button) => button.textContent) ).toEqual([TREASURY_CDCC_CLAIM.name, SOI_CDCC_CLAIM.name]); + expect(within(table).queryByText('Period not recorded separately')).not.toBeInTheDocument(); expect( within(card('External estimates')).getByText( 'No external estimate is available for this reform.' From c9c566e6edf01bebf87b518035519c9753e2c5a7 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 14 Oct 2026 16:56:20 -0400 Subject: [PATCH 3/6] Validation: each comparison links to its place on the scorecard Only the card header linked to the scorecard, and it opened the comparison table, while most baseline claims are listed in the scorecard's claims registry. Each row now links to the view that lists it: published claims to ?view=validation (with &country= outside the US), comparison-table rows to ?view=scorecard. The scorecard's URL has no address for a single entry, so this is the closest link it supports. The Treasury source also reads by name. Co-Authored-By: Claude Opus 5.5 --- .../flagship/ScorecardComparisons.tsx | 28 ++++++++++++--- .../components/flagship/ValidationPanel.tsx | 3 ++ app/src/libs/flagship/modelValidation.ts | 22 ++++++++++++ .../flagship/ScorecardComparisons.test.tsx | 36 ++++++++++++++++++- .../flagship/ValidationPanel.test.tsx | 29 +++++++++++++-- .../flagship/report/ReportValidation.test.tsx | 1 + .../libs/flagship/modelValidation.test.ts | 18 ++++++++++ 7 files changed, 129 insertions(+), 8 deletions(-) diff --git a/app/src/components/flagship/ScorecardComparisons.tsx b/app/src/components/flagship/ScorecardComparisons.tsx index c50371b6d..a89ce45e9 100644 --- a/app/src/components/flagship/ScorecardComparisons.tsx +++ b/app/src/components/flagship/ScorecardComparisons.tsx @@ -3,6 +3,7 @@ import { IconExternalLink } from '@tabler/icons-react'; import { Stack, Text } from '@/components/ui'; import { Dialog, DialogContent, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; import { colors, spacing, typography } from '@/designTokens'; +import { scorecardEntryUrl } from '@/libs/flagship/modelValidation'; import { loadParameterDependencies, variablesReachedByPaths, @@ -47,6 +48,7 @@ const sourceNames: Record = { tpc: 'Tax Policy Center', cpsp: 'Center on Poverty and Social Policy', pwbm: 'Penn Wharton Budget Model', + treasury: 'US Treasury', }; const metricNames: Record = { revenue_change: 'Change in tax revenue', @@ -319,6 +321,8 @@ export interface ClaimPresentation { modelPeriod?: string; externalValue?: string; peValue?: string; + /** Where the claim sits on the scorecard, when not its registry of published claims. */ + scorecardUrl?: string; } /** A scorecard claim, and how to present it. */ @@ -334,6 +338,11 @@ function claimTitle(row: RelatedScorecardClaim, presentation?: ClaimPresentation ); } +/** The claim's own place on the scorecard, as close as the scorecard's links reach. */ +function claimScorecardUrl(row: RelatedScorecardClaim, presentation?: ClaimPresentation): string { + return presentation?.scorecardUrl ?? scorecardEntryUrl('validation', row.country); +} + /** The source, and the geography when the claim records one: "Joint Committee on Taxation · US". */ function claimWhere(row: RelatedScorecardClaim, presentation?: ClaimPresentation): string { const sourceName = @@ -480,6 +489,7 @@ export function ScorecardClaimDetails({ row, presentation }: ScorecardClaimEntry /** One claim as a table row: what is compared, both values, and the gap. */ function ScorecardClaimRow({ row, presentation }: ScorecardClaimEntry) { const sourceUrl = safeUrl(row.url); + const smallLink = { ...linkStyle, fontSize: typography.fontSize.xs }; // Only a period that is recorded; "not recorded" under every value says nothing. const publishedNote = presentation?.externalPeriod ?? claimPeriod(row); const peNote = presentation?.modelPeriod ?? modelPeriod(row); @@ -505,16 +515,24 @@ function ScorecardClaimRow({ row, presentation }: ScorecardClaimEntry) { {claimWhere(row, presentation)} - {sourceUrl && ( + + {sourceUrl && ( + <> + + Read source + + {' · '} + + )} - Read source + View in scorecard - )} + {presentation?.externalValue ?? claimValue(row.external_value, row.unit_concept)} diff --git a/app/src/components/flagship/ValidationPanel.tsx b/app/src/components/flagship/ValidationPanel.tsx index 88f4b9a87..1e24efaa9 100644 --- a/app/src/components/flagship/ValidationPanel.tsx +++ b/app/src/components/flagship/ValidationPanel.tsx @@ -10,6 +10,7 @@ import { ModelValidationRow, PROGRAM_LABELS, SCORECARD_COMPARISON_URL, + scorecardEntryUrl, ScorecardMatches, scorecardMatchesForPaths, } from '@/libs/flagship/modelValidation'; @@ -302,6 +303,8 @@ export function trackRecordClaims(trackRecord: ScorecardMatches): ScorecardClaim modelPeriod: row.pePeriod ?? undefined, externalValue: metricValue(row, row.externalValue), peValue: metricValue(row, row.peValue), + // Comparison rows live in the scorecard's comparison table. + scorecardUrl: scorecardEntryUrl('scorecard', 'US'), }, }; }); diff --git a/app/src/libs/flagship/modelValidation.ts b/app/src/libs/flagship/modelValidation.ts index 1acb08a5c..8ce2cf971 100644 --- a/app/src/libs/flagship/modelValidation.ts +++ b/app/src/libs/flagship/modelValidation.ts @@ -52,6 +52,28 @@ export const SCORECARD_URL = 'https://www.policyengine.org/scorecard'; export const SCORECARD_COMPARISON_URL = `${SCORECARD_URL}?view=scorecard`; export const SCORECARD_METHOD_URL = `${SCORECARD_URL}?view=about`; +/** + * Where an entry sits on the scorecard: its comparison table + * ("scorecard", the comparison.json rows) or its registry of published + * claims ("validation", the populations feed, baseline claims included). + */ +export type ScorecardView = 'scorecard' | 'validation'; + +/** + * The closest link the scorecard has to one entry: the view that lists + * it, in the entry's country. The scorecard's URL carries only `view` and + * `country` (US is its default and stays out); it has no address for a + * single entry, so the reader finds the row in that view. + */ +export function scorecardEntryUrl(view: ScorecardView, country?: string | null): string { + const params = new URLSearchParams({ view }); + const code = country?.toLowerCase(); + if (code && code !== 'us') { + params.set('country', code); + } + return `${SCORECARD_URL}?${params}`; +} + export const PROGRAM_LABELS: Record = { snap: 'SNAP', wic: 'WIC', diff --git a/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx b/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx index 98eac2f53..3a3b69d23 100644 --- a/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx +++ b/app/src/tests/unit/components/flagship/ScorecardComparisons.test.tsx @@ -6,7 +6,11 @@ import { splitScorecardRows, useScorecardComparisons, } from '@/components/flagship/ScorecardComparisons'; -import { SOI_CDCC_CLAIM } from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; +import { SCORECARD_COMPARISON_URL } from '@/libs/flagship/modelValidation'; +import { + SOI_CDCC_CLAIM, + UK_PARAMETER_CLAIM, +} from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; import { CTC_PATH, jctCtcClaim, @@ -89,6 +93,36 @@ describe('scorecard evidence presentation', () => { expect(screen.getByText(/Modeled Utah credit actually usable/)).toBeInTheDocument(); }); + test('given a claim then its row links to the scorecard view that lists it', () => { + render( + + ); + + const scorecardLink = (title: string) => + within(screen.getByRole('row', { name: new RegExp(`^${title}`) })).getByRole('link', { + name: 'View in scorecard', + }); + expect(scorecardLink(SOI_CDCC_CLAIM.name)).toHaveAttribute( + 'href', + 'https://www.policyengine.org/scorecard?view=validation' + ); + expect(scorecardLink(UK_PARAMETER_CLAIM.name)).toHaveAttribute( + 'href', + 'https://www.policyengine.org/scorecard?view=validation&country=uk' + ); + expect(scorecardLink('Comparison row')).toHaveAttribute('href', SCORECARD_COMPARISON_URL); + }); + test('given a claim without a recorded model period then none shows under its value', () => { render( diff --git a/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx b/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx index 6262f5142..990e3e827 100644 --- a/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx +++ b/app/src/tests/unit/components/flagship/ValidationPanel.test.tsx @@ -5,7 +5,7 @@ import { BillExternalEstimates, trackRecordClaims, } from '@/components/flagship/ValidationPanel'; -import type { ScorecardMatches } from '@/libs/flagship/modelValidation'; +import { SCORECARD_COMPARISON_URL, type ScorecardMatches } from '@/libs/flagship/modelValidation'; import { NATIONWIDE, REFUNDABLE_CTC_MAX_PATH, @@ -22,6 +22,8 @@ import { import { utahBaselineClaim } from '@/tests/fixtures/libs/flagship/scorecardComparisonMocks'; import { TRACKED_BILL } from '@/tests/fixtures/libs/flagship/trackedBillMocks'; +/** The scorecard's registry of published claims, where baseline claims are listed. */ +const SCORECARD_CLAIMS_URL = 'https://www.policyengine.org/scorecard?view=validation'; const NO_COMPARISON = 'No scorecard comparison measures the programs this reform changes.'; const EMPTY_TRACK_RECORD: ScorecardMatches = { modelVersion: '1.808.0', @@ -180,7 +182,7 @@ describe('BaselineComparisons', () => { expect(screen.queryByRole('table')).not.toBeInTheDocument(); }); - test('given report 4332 then only the CDCC rows show', () => { + test('given report 4332 then only the CDCC rows show, each linked to the scorecard', () => { render( { ).toBeInTheDocument(); expect(within(table).getByRole('button', { name: SOI_CDCC_CLAIM.name })).toBeInTheDocument(); expect(within(table).queryByText(/Maryland|South Carolina|saver/i)).not.toBeInTheDocument(); + for (const link of within(table).getAllByRole('link', { name: 'View in scorecard' })) { + expect(link).toHaveAttribute('href', SCORECARD_CLAIMS_URL); + } expect(within(table).queryByText(/not recorded/i)).not.toBeInTheDocument(); }); @@ -222,6 +227,26 @@ describe('BaselineComparisons', () => { 'https://apps.urban.org/features/state-safety-net/' ); }); + + test('given both kinds of row then each links to the scorecard view that lists it', () => { + render( + + ); + + const scorecardLink = (title: string) => + within(screen.getByRole('row', { name: new RegExp(`^${title}`) })).getByRole('link', { + name: 'View in scorecard', + }); + expect(scorecardLink('SNAP — Eligible people')).toHaveAttribute( + 'href', + SCORECARD_COMPARISON_URL + ); + expect(scorecardLink('Utah Child Tax Credit')).toHaveAttribute('href', SCORECARD_CLAIMS_URL); + }); }); describe('trackRecordClaims', () => { diff --git a/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx b/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx index 7b869fb8d..b8cf8efc1 100644 --- a/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx +++ b/app/src/tests/unit/components/flagship/report/ReportValidation.test.tsx @@ -138,6 +138,7 @@ describe('ReportValidation', () => { .getAllByRole('button') .map((button) => button.textContent) ).toEqual([TREASURY_CDCC_CLAIM.name, SOI_CDCC_CLAIM.name]); + expect(within(table).getAllByRole('link', { name: 'View in scorecard' })).toHaveLength(2); expect(within(table).queryByText('Period not recorded separately')).not.toBeInTheDocument(); expect( within(card('External estimates')).getByText( diff --git a/app/src/tests/unit/libs/flagship/modelValidation.test.ts b/app/src/tests/unit/libs/flagship/modelValidation.test.ts index 900824d44..13e3fd40d 100644 --- a/app/src/tests/unit/libs/flagship/modelValidation.test.ts +++ b/app/src/tests/unit/libs/flagship/modelValidation.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { claimsFromBillValidation, fetchModelValidation, + scorecardEntryUrl, scorecardMatchesForPaths, summarizeScorecardPrograms, } from '@/libs/flagship/modelValidation'; @@ -204,3 +205,20 @@ describe('claimsFromBillValidation', () => { expect(claimsFromBillValidation('x', { peEstimate: -5 })).toEqual([]); }); }); + +describe('scorecardEntryUrl', () => { + test('given a US entry then the link opens its view with the default country left out', () => { + expect(scorecardEntryUrl('validation', 'US')).toBe( + 'https://www.policyengine.org/scorecard?view=validation' + ); + expect(scorecardEntryUrl('scorecard')).toBe( + 'https://www.policyengine.org/scorecard?view=scorecard' + ); + }); + + test('given another country then the link selects it', () => { + expect(scorecardEntryUrl('validation', 'UK')).toBe( + 'https://www.policyengine.org/scorecard?view=validation&country=uk' + ); + }); +}); From 42d3680b48e8e5841bfade21b287e4cd686a8793 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 14 Oct 2026 17:10:22 -0400 Subject: [PATCH 4/6] Validation: data calibration keeps to the programs the reform changes The data calibration card listed every target whose variables the reform reached within six formula steps, so a CDCC change (report 4332) showed the CTC and refundable CTC targets. Calibration matches now pass the same relevantComparisons rule as baseline comparisons: a target counts when the model variables it measures are a program the reform changes, in the report's geography. A match left with no target drops, and an empty card says so in one line. Names now compare without the words that only qualify a program (refundable, enrolled, eligible, the tax_unit entity) on both sides, so medicaid_enrolled is Medicaid and income_tax stays income tax rather than any path with "income" in it. ACA, education credit and LIHEAP targets get aliases to the names their parameters use. Co-Authored-By: Claude Opus 5.5 --- .../flagship/CalibrationMatches.tsx | 8 +-- app/src/libs/flagship/calibrationMatching.ts | 48 ++++++++++++- app/src/libs/flagship/comparisonRelevance.ts | 44 +++++++++--- .../libs/flagship/calibrationMatchingMocks.ts | 20 ++++++ .../libs/flagship/comparisonRelevanceMocks.ts | 4 ++ .../flagship/CalibrationMatches.test.tsx | 34 +++++----- .../libs/flagship/calibrationMatching.test.ts | 67 +++++++++++++++++++ .../libs/flagship/comparisonRelevance.test.ts | 25 +++++++ 8 files changed, 214 insertions(+), 36 deletions(-) diff --git a/app/src/components/flagship/CalibrationMatches.tsx b/app/src/components/flagship/CalibrationMatches.tsx index 124a026f9..4c0870086 100644 --- a/app/src/components/flagship/CalibrationMatches.tsx +++ b/app/src/components/flagship/CalibrationMatches.tsx @@ -214,14 +214,10 @@ export function CalibrationMatchSection({ return card(); } if (matches.matches.length === 0) { - const where = matches.geography === 'US' ? 'nationally' : `in ${matches.geography}`; - // Silence would read as "fine". State credits are the usual case: the - // dashboard's state income tax targets carry no model-variable mapping. + // Silence would read as "fine"; loose matches would read as evidence. return card( - {matches.reachedCount > 0 - ? `None of the ${matches.reachedCount} variables this reform moves is a calibration target ${where}.` - : 'No calibration target covers this reform.'} + No calibration target measures the programs this reform changes. ); } diff --git a/app/src/libs/flagship/calibrationMatching.ts b/app/src/libs/flagship/calibrationMatching.ts index f9fca81d9..8af81b323 100644 --- a/app/src/libs/flagship/calibrationMatching.ts +++ b/app/src/libs/flagship/calibrationMatching.ts @@ -14,10 +14,12 @@ * mechanism is usually the interacted quantity that bounds the data-side * bias of the estimate, so ordering is by depth, nearest first. */ +import { relevantComparisons, type ReformScope } from '@/libs/flagship/comparisonRelevance'; import { DEFAULT_MAX_DEPTH, loadParameterDependencies, ReachedVariable, + readersOfPath, variablesReachedByPaths, } from '@/libs/flagship/parameterDependencies'; @@ -166,6 +168,41 @@ export function summarizeMatches( ); } +/** + * The matches whose targets measure a program the reform changes, in the + * report's jurisdiction (the rule baseline comparisons use). A target's + * program is the model variables it was computed from; targets of a + * program the change only echoes into drop, and so does a match left + * with none. + */ +export function relevantCalibrationMatches( + matches: CalibrationVariableMatch[], + reform: ReformScope +): CalibrationVariableMatch[] { + return matches.flatMap((match) => { + const targets = relevantComparisons( + match.targets.map((target) => ({ + program: target.policyengineVariables.join('+'), + geography: target.geography, + target, + })), + reform + ).map((entry) => entry.target); + if (targets.length === 0) { + return []; + } + return [ + { + ...match, + targets, + targetCount: targets.length, + meanAbsRelativeError: weightedMeanAbsError(targets), + worst: targets[0], + }, + ]; + }); +} + /** * Fetches the targets that measure any of `variables` in `geography`. * Returns null when the route is unavailable (Vite build, dashboard @@ -200,7 +237,8 @@ export async function fetchCalibrationTargets( /** * The calibration matches for a reform: the variables its paths reach * (to `maxDepth`) that are calibration targets in `geography`, with - * their fit. Null when either the map or the dashboard is unavailable. + * their fit, kept to the programs the reform changes. Null when either + * the map or the dashboard is unavailable. */ export async function calibrationMatchesForPaths( paths: string[], @@ -221,11 +259,17 @@ export async function calibrationMatchesForPaths( if (fetched === null) { return null; } + const reform: ReformScope = { + countryId: 'us', + paths, + readers: Object.fromEntries(paths.map((path) => [path, readersOfPath(path, map)])), + region: geography === 'US' ? 'us' : `state/${geography.toLowerCase()}`, + }; return { releaseId: fetched.releaseId, geography, modelVersion: map.model.version, reachedCount: reached.length, - matches: summarizeMatches(reached, fetched.rows), + matches: relevantCalibrationMatches(summarizeMatches(reached, fetched.rows), reform), }; } diff --git a/app/src/libs/flagship/comparisonRelevance.ts b/app/src/libs/flagship/comparisonRelevance.ts index f8b8afbbe..ce5e784de 100644 --- a/app/src/libs/flagship/comparisonRelevance.ts +++ b/app/src/libs/flagship/comparisonRelevance.ts @@ -10,6 +10,10 @@ * CDCC). Programs further down the dependency graph, such as a state * EITC that a federal credit reaches through the tax it reduces, are the * tax-benefit system echoing the change, not programs the reform changes. + * Names are compared without the words that only qualify a program + * (refundable, enrolled, the tax_unit entity), so refundable_ctc is the + * CTC and medicaid_enrolled is Medicaid. Baseline comparisons and + * calibration targets use the same rule. * * Jurisdiction, US only: a nationwide report keeps the national * comparisons of the federal programs the reform changes, and a state's @@ -50,7 +54,11 @@ const US_STATES = new Set( ).split(' ') ); -/** Words that qualify a program, not name it: refundable_ctc and non_refundable_ctc are the CTC. */ +/** + * Words that qualify or state a program, not name it, left out of every + * name before comparing: refundable_ctc and non_refundable_ctc are the + * CTC, medicaid_enrolled and is_medicaid_eligible are Medicaid. + */ const QUALIFIERS = new Set([ 'refundable', 'nonrefundable', @@ -58,22 +66,42 @@ const QUALIFIERS = new Set([ 'refund', 'credit', 'credits', - 'tax', 'deduction', + 'enrolled', + 'eligible', + 'is', + 'has', ]); -/** Scorecard program ids that are not model names, and the model names they stand for. */ +/** The entity a model variable belongs to, which names no program: deductible_mortgage_interest_tax_unit. */ +const ENTITY = /(?:^|_)(?:tax_unit|spm_unit)(?=_|$)/g; + +/** + * Program ids whose name is not the one their parameters and readers + * use, and the names they stand for: scorecard abbreviations, and model + * variables named for a quantity rather than the program. + */ const PROGRAM_ALIASES: Record = { + assigned_aca_ptc: ['aca'], co_fac: ['co_family_affordability_credit'], + education_tax_credits: ['education'], housing: ['hud'], ma_cftc: ['ma_child_and_family_credit'], ny_escc: ['ny_ctc'], or_kids_credit: ['or_ctc'], + spm_unit_energy_subsidy: ['liheap'], va_eitc_cli: ['va_eitc', 'va_low_income_credit'], wa_wftc: ['wa_working_families_tax_credit'], wi_eic: ['wi_earned_income_credit'], }; +/** A name's words, without its entity and qualifiers; all of them when nothing else is left. */ +function nameTokens(name: string): string[] { + const tokens = name.replace(ENTITY, '').split('_').filter(Boolean); + const named = tokens.filter((token) => !QUALIFIERS.has(token)); + return named.length > 0 ? named : tokens; +} + /** A program the reform changes: its name, whose program it is, and where the change applies. */ interface ProgramSignal { tokens: string[]; @@ -106,7 +134,7 @@ function reformSignals(reform: ReformScope, us: boolean): ProgramSignal[] { const scope = pathScope(segments); // The last segment names a setting (max, rate, min_children_for_ss_taxes_minus_eitc). const own: ProgramSignal = { - tokens: segments.slice(0, -1).flatMap((segment) => segment.split('_')), + tokens: segments.slice(0, -1).flatMap(nameTokens), owner: scope, applies: scope, }; @@ -116,8 +144,8 @@ function reformSignals(reform: ReformScope, us: boolean): ProgramSignal[] { // An unprefixed variable reading a state's parameter is a federal // program the state runs, such as a SNAP option. return state - ? { tokens: rest, owner: state, applies: scope } - : { tokens: variable.split('_'), owner: scope === null ? null : NATIONAL, applies: scope }; + ? { tokens: nameTokens(rest.join('_')), owner: state, applies: scope } + : { tokens: nameTokens(variable), owner: scope === null ? null : NATIONAL, applies: scope }; }); return [own, ...readers]; }); @@ -136,11 +164,9 @@ function comparisonPrograms( // "md_eitc" measured in Maryland is Maryland's program; "eitc" there is the federal one. const statesOwn = geography !== NATIONAL && tokens.length > 1 && stateCode(tokens[0]) === geography; - const named = statesOwn ? tokens.slice(1) : tokens; - const family = named.filter((token) => !QUALIFIERS.has(token)); return { owner: statesOwn ? geography : NATIONAL, - family: family.length > 0 ? family : named, + family: nameTokens((statesOwn ? tokens.slice(1) : tokens).join('_')), }; }); } diff --git a/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts b/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts index 396d3a798..bbfcc53df 100644 --- a/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts +++ b/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts @@ -48,6 +48,26 @@ export const mockCalibrationRows: CalibrationTargetRow[] = [ ), ]; +/** A national EITC target, which an EITC reform changes directly. */ +export const mockEitcCalibrationRow: CalibrationTargetRow = soiRow( + 'irs_soi.ty2023.table_2_5.eitc_by_agi_children.no_qualifying_children.18k_to_19k.eitc_returns@2024', + 'eitc', + ['eitc'], + 0.08 +); + +/** A group named for a raw model variable, as the CTC targets reach a CDCC reform. */ +export const mockCtcLimitingMatch = { + variable: 'ctc_limiting_tax_liability', + depth: 5, + via: 'non_refundable_credits', + ring: 'downstream' as const, + targets: [mockCalibrationRows[2]], + targetCount: 1, + meanAbsRelativeError: 0.3, + worst: mockCalibrationRows[2], +}; + export const mockCalibrationMatches: CalibrationMatches = { releaseId: POPULACE_RELEASE_ID, geography: 'US', diff --git a/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts b/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts index 32ad63a83..9a15eff7a 100644 --- a/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts +++ b/app/src/tests/fixtures/libs/flagship/comparisonRelevanceMocks.ts @@ -36,6 +36,10 @@ export const STANDARD_DEDUCTION_PATH = 'gov.irs.deductions.standard.amount.SINGL export const SNAP_DEDUCTION_PATH = 'gov.usda.snap.income.deductions.standard'; export const REFUNDABLE_CTC_MAX_PATH = 'gov.irs.credits.ctc.refundable.individual_max'; export const UT_CTC_AMOUNT_PATH = 'gov.states.ut.tax.income.credits.ctc.amount'; +export const MEDICAID_CATEGORIES_PATH = 'gov.hhs.medicaid.eligibility.categories.covered'; +export const MORTGAGE_CAP_PATH = 'gov.irs.deductions.itemized.interest.mortgage.cap'; +export const ACA_CONTRIBUTION_PATH = 'gov.aca.required_contribution_percentage.final'; +export const BRACKET_RATE_PATH = 'gov.irs.income.bracket.rates.1'; export const NATIONWIDE = 'us'; export const MARYLAND = 'state/md'; diff --git a/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx b/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx index 2af77e229..e029d5d31 100644 --- a/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx +++ b/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx @@ -29,25 +29,21 @@ describe('CalibrationMatchSection', () => { ).toBeInTheDocument(); }); - test('given reached variables but none calibrated then the gap is stated, not hidden', () => { - render( - - ); - - expect(screen.getByText(/none of the 33 variables .* in CA/i)).toBeInTheDocument(); - }); - - test('given no provisions at all then a plain line says no target covers the reform', () => { - render( - - ); - - expect(screen.getByRole('heading', { name: 'Data calibration' })).toBeInTheDocument(); - expect(screen.getByText('No calibration target covers this reform.')).toBeInTheDocument(); + test('given no target of a changed program then one plain line says so', () => { + for (const reachedCount of [0, 33]) { + const { unmount } = render( + + ); + + expect(screen.getByRole('heading', { name: 'Data calibration' })).toBeInTheDocument(); + expect( + screen.getByText('No calibration target measures the programs this reform changes.') + ).toBeInTheDocument(); + expect(screen.queryByRole('table')).not.toBeInTheDocument(); + unmount(); + } }); test('shows each target with its period, values, gap and source link', () => { diff --git a/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts b/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts index 59829d0f1..e8e6ec0ef 100644 --- a/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts +++ b/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts @@ -4,16 +4,26 @@ import { dashboardTargetsUrl, fetchCalibrationTargets, geographyForRegion, + relevantCalibrationMatches, ringForDepth, summarizeMatches, weightedMeanAbsError, } from '@/libs/flagship/calibrationMatching'; import { + mockCalibrationMatches, mockCalibrationRows, + mockCtcLimitingMatch, + mockEitcCalibrationRow, POPULACE_RELEASE_ID, } from '@/tests/fixtures/libs/flagship/calibrationMatchingMocks'; +import { + NATIONWIDE, + REPORT_4332_REFORM, + usReform, +} from '@/tests/fixtures/libs/flagship/comparisonRelevanceMocks'; import { CTC_BASE_AMOUNT_PATH, + EITC_MAX_PATH, mockParameterDependencyMap, STANDARD_DEDUCTION_PATH, } from '@/tests/fixtures/libs/flagship/parameterDependenciesMocks'; @@ -179,4 +189,61 @@ describe('calibrationMatchesForPaths', () => { expect(result!.matches).toEqual([]); expect(fetchMock).not.toHaveBeenCalled(); }); + + test('given an EITC path echoing into the CTC then only the EITC targets remain', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => + okResponse({ + releaseId: POPULACE_RELEASE_ID, + rows: [...mockCalibrationRows, mockEitcCalibrationRow], + }) + ) + ); + + const result = await calibrationMatchesForPaths([EITC_MAX_PATH], 'US'); + + expect(result!.reachedCount).toBeGreaterThan(1); + expect(result!.matches.map((match) => match.variable)).toEqual(['eitc']); + }); +}); + +describe('relevantCalibrationMatches', () => { + test('given report 4332 then its CTC targets drop and no match is left', () => { + expect( + relevantCalibrationMatches( + [...mockCalibrationMatches.matches, mockCtcLimitingMatch], + REPORT_4332_REFORM + ) + ).toEqual([]); + }); + + test('given a CTC reform then its CTC targets stay with their fit', () => { + const [refundable] = relevantCalibrationMatches( + mockCalibrationMatches.matches, + usReform([CTC_BASE_AMOUNT_PATH], NATIONWIDE) + ); + + expect(refundable.variable).toBe('refundable_ctc'); + expect(refundable.targetCount).toBe(2); + expect(refundable.worst).toBe(mockCalibrationRows[1]); + }); + + test('given a group with targets of another program then only the changed program stays', () => { + const [match] = relevantCalibrationMatches( + [ + { + ...mockCalibrationMatches.matches[0], + targets: [mockCalibrationRows[2], mockEitcCalibrationRow], + targetCount: 2, + worst: mockCalibrationRows[2], + }, + ], + usReform([EITC_MAX_PATH], NATIONWIDE) + ); + + expect(match.targets).toEqual([mockEitcCalibrationRow]); + expect(match.worst).toBe(mockEitcCalibrationRow); + expect(match.meanAbsRelativeError).toBeCloseTo(0.08); + }); }); diff --git a/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts b/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts index 8cd7bf4b3..e8d7c250a 100644 --- a/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts +++ b/app/src/tests/unit/libs/flagship/comparisonRelevance.test.ts @@ -1,7 +1,9 @@ import { describe, expect, test } from 'vitest'; import { relevantComparisons } from '@/libs/flagship/comparisonRelevance'; import { + ACA_CONTRIBUTION_PATH, ACTC_CLAIM, + BRACKET_RATE_PATH, CDCC_DIRECT_READERS, CDCC_ELIGIBILITY_MAX_PATH, COLORADO_CTC_CLAIM, @@ -17,6 +19,8 @@ import { MARYLAND_FEDERAL_EITC_CLAIM, MD_EITC_CLAIM, MD_EITC_MATCH_PATH, + MEDICAID_CATEGORIES_PATH, + MORTGAGE_CAP_PATH, NATIONWIDE, NONREFUNDABLE_CTC_CLAIM, NYC_EITC_PATH, @@ -25,6 +29,7 @@ import { SAVERS_CREDIT_CLAIM, SC_CDCC_CLAIM, SC_EITC_CLAIM, + SNAP_DEDUCTION_PATH, SOI_CDCC_CLAIM, SOUTH_CAROLINA, STANDARD_DEDUCTION_PATH, @@ -149,4 +154,24 @@ describe('relevantComparisons', () => { }) ).toEqual([UK_PARAMETER_CLAIM]); }); + + test('given model variable names then entity and status words do not hide the program', () => { + const target = (program: string) => ({ program, geography: 'US' }); + const kept = (program: string, path: string, readers: string[] = []) => + relevantComparisons([target(program)], usReform([path], NATIONWIDE, { [path]: readers })) + .length > 0; + + expect(kept('medicaid_enrolled', MEDICAID_CATEGORIES_PATH)).toBe(true); + expect( + kept('deductible_mortgage_interest_tax_unit', MORTGAGE_CAP_PATH, [ + 'deductible_mortgage_interest', + ]) + ).toBe(true); + expect( + kept('assigned_aca_ptc', ACA_CONTRIBUTION_PATH, ['aca_required_contribution_percentage']) + ).toBe(true); + // Income tax is a program of its own, not every rule with "income" in its path. + expect(kept('income_tax', BRACKET_RATE_PATH, ['income_tax_main_rates'])).toBe(true); + expect(kept('income_tax', SNAP_DEDUCTION_PATH, ['snap_standard_deduction'])).toBe(false); + }); }); From 449db235199f179b708c682c37739a7a6d088359 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 14 Oct 2026 17:11:01 -0400 Subject: [PATCH 5/6] Validation: calibration targets read by name, not dashboard id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Target column showed dashboard ids such as "irs_soi.ty2022.historic_table_2.us.all.ctc_amount@2024", and groups read as raw variable names ("Ctc limiting tax liability"). A target now reads as the quantity it measures and the slice of its table ("Amount", "Claims", "Returns · 18k to 19k · no qualifying children"); the id stays as the button's tooltip and as "Target id" in its details. A group reads as our copy, else the model's own variable label, else its name in words with acronyms ("CTC limiting tax liability"). The calibration route now passes the dashboard's measure id and breakdown for this. Co-Authored-By: Claude Opus 5.5 --- .../flagship/CalibrationMatches.tsx | 48 ++++++------ app/src/libs/flagship/calibrationMatching.ts | 77 +++++++++++++++++++ .../libs/flagship/calibrationMatchingMocks.ts | 18 +++-- .../flagship/CalibrationMatches.test.tsx | 36 ++++++--- .../libs/flagship/calibrationMatching.test.ts | 57 ++++++++++++++ .../src/app/api/calibration-targets/route.ts | 6 ++ 6 files changed, 203 insertions(+), 39 deletions(-) diff --git a/app/src/components/flagship/CalibrationMatches.tsx b/app/src/components/flagship/CalibrationMatches.tsx index 4c0870086..f757d1aaa 100644 --- a/app/src/components/flagship/CalibrationMatches.tsx +++ b/app/src/components/flagship/CalibrationMatches.tsx @@ -1,4 +1,5 @@ import { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; import { Text } from '@/components/ui'; import { Dialog, DialogContent, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; import { colors, spacing, typography } from '@/designTokens'; @@ -6,11 +7,14 @@ import { CalibrationMatches, calibrationMatchesForPaths, CalibrationRing, + calibrationTargetLabel, CalibrationTargetRow, + calibrationVariableLabel, dashboardTargetsUrl, geographyForRegion, } from '@/libs/flagship/calibrationMatching'; import type { ReportValidationSnapshot } from '@/libs/flagship/reportValidation'; +import type { RootState } from '@/store'; import { CellNote, detailStyle, @@ -55,17 +59,6 @@ export const ATTENTION_ERROR = 0.25; const CALIBRATION_COLUMNS = ['Target', 'Period', 'Official', 'PolicyEngine', 'Gap']; -function humanize(variable: string): string { - const labels: Record = { - refundable_ctc: 'Refundable Child Tax Credit', - ctc: 'Child Tax Credit', - income_tax: 'Income tax', - assigned_aca_ptc: 'Assigned ACA premium tax credit', - }; - const label = labels[variable] ?? variable.replaceAll('_', ' '); - return label.charAt(0).toUpperCase() + label.slice(1); -} - function compactNumber(value: number | null): string { return value === null ? '—' @@ -134,18 +127,21 @@ function PinNote({ pin }: { pin: CalibrationPin }) { ); } -/** A target's name, opening its exact values. */ -function TargetDetails({ target }: { target: CalibrationTargetRow }) { +/** A target's name, opening its exact values; the dashboard id rides along as a tooltip. */ +function TargetDetails({ target, group }: { target: CalibrationTargetRow; group: string }) { + const label = calibrationTargetLabel(target); return ( - Calibration target - {target.name} + + {group} · {label} +
{[ + ['Target id', target.name], ['Source', target.sourceLabel ?? target.source], ['Geography', target.geography], ['Period', target.period ?? '—'], @@ -164,10 +161,10 @@ function TargetDetails({ target }: { target: CalibrationTargetRow }) { ['Official', exactNumber(target.target)], ['PolicyEngine', exactNumber(target.estimate)], ['Gap', gapOf(target)], - ].map(([label, value]) => ( -
-
{label}
-
{value}
+ ].map(([field, value]) => ( +
+
{field}
+
{value}
))}
@@ -196,6 +193,7 @@ export function CalibrationMatchSection({ matches: CalibrationMatches | null | undefined; pin?: CalibrationPin; }) { + const variables = useSelector((state: RootState) => state.metadata.variables); const level = matches ? (matches.geography === 'US' ? 'national' : 'state') : null; const card = (content: React.ReactNode) => ( target.sourceLabel ?? target.source)), ].join(', '); + const group = calibrationVariableLabel( + match.variable, + variables?.[match.variable]?.label + ); return ( @@ -237,9 +239,7 @@ export function CalibrationMatchSection({ colSpan={CALIBRATION_COLUMNS.length} style={{ ...labelCellStyle, paddingBottom: 0 }} > - - {humanize(match.variable)} - + {group} {sources} · {matches.geography} @@ -259,7 +259,7 @@ export function CalibrationMatchSection({ {match.targets.map((target) => ( - + {target.period ?? '—'} {compactNumber(target.target)} diff --git a/app/src/libs/flagship/calibrationMatching.ts b/app/src/libs/flagship/calibrationMatching.ts index 8af81b323..090b2609c 100644 --- a/app/src/libs/flagship/calibrationMatching.ts +++ b/app/src/libs/flagship/calibrationMatching.ts @@ -31,6 +31,10 @@ export interface CalibrationTargetRow { variable: string; policyengineVariables: string[]; measure: string | null; + /** The dashboard's id for the measured quantity, e.g. "ctc_amount". */ + sourceMeasureId?: string | null; + /** The target's slice of the source table, e.g. "18k to 19k · no qualifying children · All". */ + breakdown?: string | null; level: 'national' | 'state'; /** "US" for national targets, else the state code. */ geography: string; @@ -203,6 +207,79 @@ export function relevantCalibrationMatches( }); } +/** Words that read as acronyms in a model variable's name. */ +const ACRONYMS: Record = { + aca: 'ACA', + agi: 'AGI', + cdcc: 'CDCC', + chip: 'CHIP', + ctc: 'CTC', + eitc: 'EITC', + ira: 'IRA', + liheap: 'LIHEAP', + ptc: 'PTC', + salt: 'SALT', + snap: 'SNAP', + spm: 'SPM', + ssi: 'SSI', + tanf: 'TANF', + wic: 'WIC', +}; + +/** Copy for the variables whose model label reads badly on its own. */ +const VARIABLE_LABELS: Record = { + refundable_ctc: 'Refundable Child Tax Credit', + ctc: 'Child Tax Credit', + income_tax: 'Income tax', + assigned_aca_ptc: 'Assigned ACA premium tax credit', +}; + +/** + * A model variable as a reader names it: our copy, else the model's own + * label, else its name in words ("ctc_limiting_tax_liability" reads + * "CTC limiting tax liability"). + */ +export function calibrationVariableLabel(variable: string, modelLabel?: string | null): string { + const label = + VARIABLE_LABELS[variable] ?? + (modelLabel?.trim() || + variable + .split('_') + .filter(Boolean) + .map((word) => ACRONYMS[word] ?? word) + .join(' ')); + return label.charAt(0).toUpperCase() + label.slice(1); +} + +/** The quantity words a dashboard measure id ends with: ctc_amount, actc_claims, eitc_returns. */ +const MEASURE_WORDS = new Set(['amount', 'claims', 'returns', 'total', 'count']); +/** Breakdown parts that name no slice. */ +const WHOLE_SLICES = new Set(['', 'all', 'total']); + +/** + * A target as a reader names it, under its variable's heading: the + * quantity it measures, then the slice of the table it covers, e.g. + * "Amount", "Claims", "Returns · 18k to 19k · no qualifying children". + * The dashboard id stays in the target's details. + */ +export function calibrationTargetLabel(target: CalibrationTargetRow): string { + const id = target.sourceMeasureId ?? ''; + const last = id.split('_').pop() ?? ''; + const quantity = MEASURE_WORDS.has(last) + ? last + : id + ? id.replaceAll('_', ' ') + : target.measure === 'count' + ? 'count' + : 'total'; + const slices = (target.breakdown ?? '') + .split('·') + .map((part) => part.trim()) + .filter((part) => !WHOLE_SLICES.has(part.toLowerCase())); + const label = [quantity, ...slices].join(' · '); + return label.charAt(0).toUpperCase() + label.slice(1); +} + /** * Fetches the targets that measure any of `variables` in `geography`. * Returns null when the route is unavailable (Vite build, dashboard diff --git a/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts b/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts index bbfcc53df..cd05cbcbd 100644 --- a/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts +++ b/app/src/tests/fixtures/libs/flagship/calibrationMatchingMocks.ts @@ -16,6 +16,9 @@ const soiRow = ( variable, policyengineVariables, measure: 'total', + // The dashboard's measure id is the id's last segment: "…actc_amount@2024" → "actc_amount". + sourceMeasureId: name.split('.').pop()!.split('@')[0], + breakdown: 'All · All', level: geography === 'US' ? 'national' : 'state', geography, period: 2024, @@ -49,12 +52,15 @@ export const mockCalibrationRows: CalibrationTargetRow[] = [ ]; /** A national EITC target, which an EITC reform changes directly. */ -export const mockEitcCalibrationRow: CalibrationTargetRow = soiRow( - 'irs_soi.ty2023.table_2_5.eitc_by_agi_children.no_qualifying_children.18k_to_19k.eitc_returns@2024', - 'eitc', - ['eitc'], - 0.08 -); +export const mockEitcCalibrationRow: CalibrationTargetRow = { + ...soiRow( + 'irs_soi.ty2023.table_2_5.eitc_by_agi_children.no_qualifying_children.18k_to_19k.eitc_returns@2024', + 'eitc', + ['eitc'], + 0.08 + ), + breakdown: '18k to 19k · no qualifying children · All', +}; /** A group named for a raw model variable, as the CTC targets reach a CDCC reform. */ export const mockCtcLimitingMatch = { diff --git a/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx b/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx index e029d5d31..984fd02fd 100644 --- a/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx +++ b/app/src/tests/unit/components/flagship/CalibrationMatches.test.tsx @@ -3,6 +3,7 @@ import { describe, expect, test } from 'vitest'; import { CalibrationMatchSection } from '@/components/flagship/CalibrationMatches'; import { mockCalibrationMatches, + mockCtcLimitingMatch, POPULACE_RELEASE_ID, } from '@/tests/fixtures/libs/flagship/calibrationMatchingMocks'; import { @@ -62,9 +63,11 @@ describe('CalibrationMatchSection', () => { expect(screen.getAllByText('1M')[0]).toBeInTheDocument(); expect(screen.getByText('-6.0%')).toBeInTheDocument(); expect(screen.getAllByText('2024')).toHaveLength(3); - expect( - screen.getByRole('button', { name: 'irs_soi.ty2023.table_1_4.all.actc_returns@2024' }) - ).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Returns' })).toHaveAttribute( + 'title', + 'irs_soi.ty2023.table_1_4.all.actc_returns@2024' + ); + expect(screen.queryByText(/irs_soi\.ty2023/)).not.toBeInTheDocument(); expect( screen.queryByText(/formula steps|How to interpret|Worth a look/i) ).not.toBeInTheDocument(); @@ -74,14 +77,29 @@ describe('CalibrationMatchSection', () => { ); }); - test('clicking a target opens its exact values', () => { + test('clicking a target opens its exact values and its dashboard id', () => { render(); - fireEvent.click( - screen.getByRole('button', { name: 'irs_soi.ty2023.table_1_4.all.actc_returns@2024' }) + fireEvent.click(screen.getByRole('button', { name: 'Returns' })); + + const dialog = screen.getByRole('dialog'); + expect(within(dialog).getByText('Refundable Child Tax Credit · Returns')).toBeInTheDocument(); + expect( + within(dialog).getByText('irs_soi.ty2023.table_1_4.all.actc_returns@2024') + ).toBeInTheDocument(); + expect(within(dialog).getByText('940,000')).toBeInTheDocument(); + expect(within(dialog).getByText('1,000,000')).toBeInTheDocument(); + }); + + test('given a group named for a raw model variable then it reads as words', () => { + render( + ); - expect(screen.getByRole('dialog')).toBeInTheDocument(); - expect(screen.getByText('940,000')).toBeInTheDocument(); - expect(screen.getByText('1,000,000')).toBeInTheDocument(); + + expect(screen.getByText('CTC limiting tax liability')).toBeInTheDocument(); + expect(screen.queryByText(/ctc_limiting|Ctc limiting/)).not.toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Amount' })).toBeInTheDocument(); }); test('given a pin that is still current then the card says when it was pinned', () => { diff --git a/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts b/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts index e8e6ec0ef..fd3f18f6d 100644 --- a/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts +++ b/app/src/tests/unit/libs/flagship/calibrationMatching.test.ts @@ -1,6 +1,8 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { calibrationMatchesForPaths, + calibrationTargetLabel, + calibrationVariableLabel, dashboardTargetsUrl, fetchCalibrationTargets, geographyForRegion, @@ -247,3 +249,58 @@ describe('relevantCalibrationMatches', () => { expect(match.meanAbsRelativeError).toBeCloseTo(0.08); }); }); + +describe('calibrationTargetLabel', () => { + test('given a measure id and a whole-table slice then the quantity names the target', () => { + expect(calibrationTargetLabel(mockCalibrationRows[0])).toBe('Amount'); + expect(calibrationTargetLabel(mockCalibrationRows[1])).toBe('Returns'); + }); + + test('given a slice of the table then the slice follows the quantity', () => { + expect(calibrationTargetLabel(mockEitcCalibrationRow)).toBe( + 'Returns · 18k to 19k · no qualifying children' + ); + }); + + test('given a measure id without a quantity word then it reads as words', () => { + expect( + calibrationTargetLabel({ + ...mockCalibrationRows[0], + sourceMeasureId: 'average_monthly_households', + breakdown: '', + }) + ).toBe('Average monthly households'); + }); + + test('given no measure id then the measure names the target', () => { + expect( + calibrationTargetLabel({ + ...mockCalibrationRows[0], + sourceMeasureId: null, + breakdown: null, + measure: 'count', + }) + ).toBe('Count'); + }); +}); + +describe('calibrationVariableLabel', () => { + test('given our own copy then it wins over the model label', () => { + expect(calibrationVariableLabel('refundable_ctc', 'refundable CTC')).toBe( + 'Refundable Child Tax Credit' + ); + }); + + test('given the model label then it names the variable', () => { + expect( + calibrationVariableLabel('ctc_limiting_tax_liability', 'CTC-limiting tax liability') + ).toBe('CTC-limiting tax liability'); + expect(calibrationVariableLabel('snap', 'SNAP allotment')).toBe('SNAP allotment'); + }); + + test('given no label then the name reads as words with its acronyms', () => { + expect(calibrationVariableLabel('ctc_limiting_tax_liability')).toBe( + 'CTC limiting tax liability' + ); + }); +}); diff --git a/calculator-app/src/app/api/calibration-targets/route.ts b/calculator-app/src/app/api/calibration-targets/route.ts index 3f3aa6ade..afdb0751f 100644 --- a/calculator-app/src/app/api/calibration-targets/route.ts +++ b/calculator-app/src/app/api/calibration-targets/route.ts @@ -32,6 +32,8 @@ interface TargetRow { variable: string; policyengineVariables: string[]; measure: string | null; + sourceMeasureId: string | null; + breakdown: string | null; level: "national" | "state"; geography: string; period: number | string | null; @@ -61,6 +63,10 @@ function normalize(raw: any): TargetRow | null { ? raw.policyengine_variables.filter((v: unknown) => typeof v === "string") : [], measure: typeof raw.measure === "string" ? raw.measure : null, + // What the target counts and which slice of its table, for a readable name. + sourceMeasureId: + typeof raw.source_measure_id === "string" ? raw.source_measure_id : null, + breakdown: typeof raw.breakdown === "string" ? raw.breakdown : null, level, geography: level === "national" ? "US" : String(raw.state ?? raw.geography ?? ""), From 76c9d0007e4c9aa1c633b90511505c5d6d0ee6d4 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Wed, 14 Oct 2026 17:13:11 -0400 Subject: [PATCH 6/6] Validation: the pin and run stages count only the programs the reform changes The scorecard track record kept every program whose comparison rows were computed from a reached variable, so the validation pin and the run's "scorecard programs found" stage counted programs the change only echoes into (an EITC reform also pinned the refundable CTC). The track record now applies relevantComparisons at its source, so the pin, the stages and the baseline card agree. Its rows are all national, so the report's region does not narrow them. Co-Authored-By: Claude Opus 5.5 --- app/src/libs/flagship/modelValidation.ts | 19 ++++++++++++++++--- .../libs/flagship/modelValidation.test.ts | 5 +++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/src/libs/flagship/modelValidation.ts b/app/src/libs/flagship/modelValidation.ts index 8ce2cf971..9cbcbedc6 100644 --- a/app/src/libs/flagship/modelValidation.ts +++ b/app/src/libs/flagship/modelValidation.ts @@ -14,10 +14,12 @@ */ import { CalibrationRing, ringForDepth } from '@/libs/flagship/calibrationMatching'; +import { relevantComparisons } from '@/libs/flagship/comparisonRelevance'; import { DEFAULT_MAX_DEPTH, loadParameterDependencies, ReachedVariable, + readersOfPath, variablesReachedByPaths, } from '@/libs/flagship/parameterDependencies'; @@ -190,8 +192,10 @@ export async function fetchModelValidation( /** * The scorecard matches for a reform: the programs whose comparison rows - * were computed from variables its paths reach (to `maxDepth`), with the - * rows. Null when either the map or the scorecard is unavailable. + * were computed from variables its paths reach (to `maxDepth`), kept to + * the programs the reform changes, with the rows. The rows are all + * national, so the report's region does not narrow them. Null when either + * the map or the scorecard is unavailable. */ export async function scorecardMatchesForPaths( paths: string[], @@ -208,10 +212,19 @@ export async function scorecardMatchesForPaths( if (fetched === null) { return null; } + const changed = relevantComparisons( + fetched.map((row) => ({ program: row.program, geography: 'US', row })), + { + countryId: 'us', + paths, + readers: Object.fromEntries(paths.map((path) => [path, readersOfPath(path, map)])), + region: 'us', + } + ).map((entry) => entry.row); return { modelVersion: map.model.version, reachedCount: reached.length, - ...summarizeScorecardPrograms(reached, fetched), + ...summarizeScorecardPrograms(reached, changed), }; } diff --git a/app/src/tests/unit/libs/flagship/modelValidation.test.ts b/app/src/tests/unit/libs/flagship/modelValidation.test.ts index 13e3fd40d..a23126175 100644 --- a/app/src/tests/unit/libs/flagship/modelValidation.test.ts +++ b/app/src/tests/unit/libs/flagship/modelValidation.test.ts @@ -132,7 +132,7 @@ describe('scorecardMatchesForPaths', () => { expect(matches?.rows).toHaveLength(2); }); - test('given an EITC path echoing into the refundable CTC then both show, EITC nearest', async () => { + test('given an EITC path echoing into the refundable CTC then only the EITC is its program', async () => { vi.stubGlobal( 'fetch', vi.fn(async () => okResponse({ rows: mockEitcScorecardRows })) @@ -140,10 +140,11 @@ describe('scorecardMatchesForPaths', () => { const matches = await scorecardMatchesForPaths([EITC_MAX_PATH]); + // The CTC is reached through the tax the EITC reduces; the reform does not change it. expect(matches?.programs.map((match) => [match.program, match.ring])).toEqual([ ['eitc', 'primary'], - ['ctc_refund', 'mechanism'], ]); + expect(matches?.rows.every((row) => row.program === 'eitc')).toBe(true); }); test('given a state credit then no federal program is borrowed', async () => {