diff --git a/changelog.d/2773.fixed.md b/changelog.d/2773.fixed.md new file mode 100644 index 0000000000..0692a444a9 --- /dev/null +++ b/changelog.d/2773.fixed.md @@ -0,0 +1 @@ +The aggregate checker reports non-finite baseline values as blocking failures and continues checking readable households. diff --git a/policyengine_uk/tests/aggregate_impact/counted_once_base_reference.py b/policyengine_uk/tests/aggregate_impact/counted_once_base_reference.py new file mode 100644 index 0000000000..89f3cdfa88 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/counted_once_base_reference.py @@ -0,0 +1,1073 @@ +"""The counted-once check as it stood at 89bd11fec, without commentary. + +Keep this independent of the production tool: it is the differential oracle +for #2773, whose fix must publish exactly what this did for every model whose +heads and totals are finite at baseline. It holds the check (``guard.py``), +how ``measure`` records and publishes its result (``record_guard_checks``, +``publishable_guard``) and how ``render`` describes it (``_guard``). +""" + +from __future__ import annotations + +import math +from typing import Iterable, Mapping, Optional + +import numpy as np + +from policyengine_uk.utils.aggregate_impact.partition import Partition +from policyengine_uk.utils.aggregate_impact.privacy import MIN_CELL, count_text, small +from policyengine_uk.utils.aggregate_impact.spec import fiscal_year_label + + +def _raw(sim, variable, period): + try: + result = sim.calculate(variable, period, unweighted=True) + except TypeError: + result = sim.calculate(variable, period) + return np.asarray(getattr(result, "values", result)) + + +def discard_formula_results(sim) -> bool: + reset = getattr(sim, "_invalidate_all_caches", None) + if reset is None: + return False + reset() + return True + + +class _InputSnapshot(dict): + def __init__(self, values, *, owner, origins): + super().__init__(values) + self.owner = owner + self.origins = origins + + +def _visible_branches(sim): + branches = [] + branch = sim + while branch is not None: + if branch.branch_name not in branches: + branches.append(branch.branch_name) + branch = getattr(branch, "parent_branch", None) + if "default" not in branches: + branches.append("default") + return branches + + +def _capture_input_origins(sim, variables): + from policyengine_uk.utils.supplied_inputs import SUPPLIED_INPUT_VARIABLES + + names = set(variables) & SUPPLIED_INPUT_VARIABLES + if not names: + return {} + branches = set(_visible_branches(sim)) + keys = set(getattr(sim, "_user_input_keys", None) or ()) + model = getattr(sim, "_model_values", None) or {} + origins = {} + for name in names: + if name not in sim.tax_benefit_system.variables: + continue + holder = sim.get_holder(name) + slots = [] + for key in keys | set(model): + variable, branch, period = key + if variable != name or branch not in branches: + continue + value = holder._get_array_from_storage(period, branch) + if value is None: + continue + marker = model.get(key) + is_model = marker is not None and np.array_equal(marker, value) + if key in keys or is_model: + slots.append((branch, period, np.array(value, copy=True), is_model)) + origins[name] = tuple(slots) + return origins + + +def snapshot_inputs(sim, variables) -> dict: + variables = tuple(variables) + discard_formula_results(sim) + origins = _capture_input_origins(sim, variables) + snapshot = _InputSnapshot({}, owner=sim, origins=origins) + for variable in variables: + if variable not in sim.tax_benefit_system.variables: + continue + holder = sim.get_holder(variable) + periods = holder.get_known_periods() + if periods: + snapshot[variable] = [ + (period, np.array(holder.get_array(period), copy=True)) + for period in periods + ] + return snapshot + + +def _guard_snapshot(sim, snapshot, variables): + if snapshot is None: + return snapshot_inputs(sim, variables) + if isinstance(snapshot, _InputSnapshot): + if snapshot.owner is not sim: + raise ValueError("Input snapshot belongs to a different simulation") + return snapshot + from policyengine_uk.utils.supplied_inputs import SUPPLIED_INPUT_VARIABLES + + origins = _capture_input_origins(sim, variables) + for name in set(snapshot) & SUPPLIED_INPUT_VARIABLES: + for period, value in snapshot[name]: + if not any( + ( + saved_period == period and np.array_equal(saved_value, value) + for _, saved_period, saved_value, _ in origins.get(name, ()) + ) + ): + raise ValueError("Plain input snapshot has no matching original slot") + return _InputSnapshot(snapshot, owner=sim, origins=origins) + + +def _set_guard_value(sim, variable, period, value, touched=None): + from policyengine_uk.utils.supplied_inputs import ( + SUPPLIED_INPUT_VARIABLES, + set_model_value, + ) + + if touched is not None: + touched.add(variable) + if variable in SUPPLIED_INPUT_VARIABLES: + set_model_value(sim, variable, period, value) + else: + sim.set_input(variable, period, value) + + +def _restore(sim, variable, snapshot): + from policyengine_uk.utils.supplied_inputs import ( + SUPPLIED_INPUT_VARIABLES, + drop_missing_model_values, + drop_missing_supplied_inputs, + set_model_value, + ) + + if variable not in SUPPLIED_INPUT_VARIABLES: + sim.delete_arrays(variable) + for period, values in snapshot.get(variable, []): + sim.set_input(variable, period, values) + return + if not isinstance(snapshot, _InputSnapshot) or snapshot.owner is not sim: + raise ValueError("Registered restoration needs its original input snapshot") + if variable not in snapshot.origins: + raise ValueError("Registered variable is absent from original origin capture") + holder = sim.get_holder(variable) + holder.delete_arrays(branch_name=sim.branch_name) + drop_missing_supplied_inputs(sim, variable) + drop_missing_model_values(sim, variable) + for branch, period, value, is_model in snapshot.origins[variable]: + if branch == sim.branch_name: + if is_model: + set_model_value(sim, variable, period, value) + else: + sim.set_input(variable, period, value) + + +def _months(year): + from policyengine_core.periods import period as make_period + + return make_period(str(year)).get_subperiods("month") + + +def _household(sim, values, entity): + values = np.asarray(values, dtype=np.float64) + if entity == "household": + return values + return np.asarray(sim.map_result(values, entity, "household"), dtype=np.float64) + + +def _head_arrays(sim, year, variables) -> dict: + definitions = sim.tax_benefit_system.variables + return { + v: _household(sim, _raw(sim, v, year), definitions[v].entity.key) + for v in variables + } + + +ABSOLUTE_SLACK_GBP = 0.02 +RELATIVE_SLACK = 2e-06 + + +def _slack(scale: np.ndarray) -> np.ndarray: + return ABSOLUTE_SLACK_GBP + RELATIVE_SLACK * np.abs(scale) + + +def _response_slack(response: np.ndarray, expected: np.ndarray) -> np.ndarray: + return ABSOLUTE_SLACK_GBP * np.abs(expected) + RELATIVE_SLACK * np.abs(response) + + +def _moves( + response: np.ndarray, expected: np.ndarray, allowances: int = 1 +) -> np.ndarray: + beyond = np.abs(response) > allowances * _response_slack(response, expected) + return beyond | ~np.isfinite(response) + + +def _contains( + response: np.ndarray, expected: np.ndarray, allowances: int = 1 +) -> np.ndarray: + close = np.abs(response - expected) <= allowances * _response_slack( + response, expected + ) + return close | ~np.isfinite(response) + + +def _rises(response: np.ndarray, slack: np.ndarray) -> np.ndarray: + return (response > slack) | ~np.isfinite(response) + + +def _not_finite_units(values) -> np.ndarray: + if isinstance(values, list): + return np.logical_or.reduce([~np.isfinite(v) for _, v in values]) + return ~np.isfinite(values) + + +def _held_values(at_first, baseline, units): + if isinstance(at_first, list): + return [ + (month, np.where(units, start, value)) + for (month, value), (_, start) in zip(at_first, baseline) + ] + return np.where(units, baseline, at_first) + + +def _households_text(households) -> str: + if households is None: + return f"fewer than {MIN_CELL} households" + return count_text(households, "households") + + +def problem_text(kind, head, component, households, not_finite=False, weighted=True): + clause = ", or is not finite," if not_finite and weighted else "" + if kind == "own_head": + text = f"{head} does not move by exactly what was added to {component}{clause}" + elif kind == "contains": + text = f"{head} rises one for one with {component}" + ( + clause or ", so it contains it," + ) + elif kind == "unconfirmed": + text = f"{head} responds to {component}{clause}" + elif kind == "rises": + text = f"{head} rises{clause} when {component} does" + elif kind == "moves": + text = f"{head} moves{clause} when {component} does" + elif kind == "mediated": + text = f"{head} responds only through other heads{clause}" + else: + text = f"the model's {head} moves by other than its heads{clause} when {component} does" + text += f" in {_households_text(households)}" + if not_finite and (not weighted): + text += ", and is not finite in households with no weight" + return text + + +def _unheld_where(households, of_them=False) -> str: + if households == 0: + return "households with no weight" + what = "of them" if of_them else "households" + if households is None or small(households): + return f"fewer than {MIN_CELL} {what}" + return f"{households:,} {what}" + + +def _unheld_words(heads, component, name=str) -> str: + names = [name(v) for v in heads] + if len(names) == 1: + listed, which = (names[0], "which is") + else: + listed = f"{', '.join(names[:-1])} and {names[-1]}" + which = "one or more of which is" + return f"the check could not hold {listed}, {which} not finite there once {name(component)} rises" + + +def not_held_note(component, head, heads, households, name=str) -> str: + return f"in {_unheld_where(households)} {_unheld_words(heads, component, name)}, so it measured {name(head)} again there only with the other heads at their baseline values" + + +def _problem( + total, + head, + kind, + component, + where, + weights, + perturbed, + nan, + not_held=(), + unjudged=None, +): + weighted = weights > 0 + count = int(np.count_nonzero(where & weighted)) + base = float(np.sum(weights * perturbed)) + share = float(np.sum(weights * where)) / base if base else 0.0 + not_finite = bool((where & nan).any()) + not_finite_weighted = bool((where & nan & weighted).any()) + unjudged = np.zeros_like(where) if unjudged is None else unjudged & where + unheld = int(np.count_nonzero(unjudged & weighted)) if not_held else 0 + text = problem_text(kind, head, component, count, not_finite, not_finite_weighted) + message = f"{total}: {text}" + if kind == "unconfirmed": + message += f" where {_unheld_words(list(not_held), component)}, so it could not confirm whether {head} contains {component}" + elif not_held: + message += f" (in {_unheld_where(unheld, of_them=True)} {_unheld_words(list(not_held), component)}, so it could not show that {head} responds only through other heads)" + return { + "total": total, + "head": head, + "kind": kind, + "households": count, + "share": share, + "message": message, + "not_finite": not_finite, + "not_finite_weighted": not_finite_weighted, + "heads_not_held": list(not_held), + "households_not_held": unheld, + "where": where, + "nan": nan, + "unjudged": unjudged, + } + + +def judge_total( + *, + total: str, + is_tax: bool, + component: str, + own_variable: str, + declared_double_count: bool, + zero_baseline: bool, + weights: np.ndarray, + expected: np.ndarray, + head_changes: Mapping[str, np.ndarray], + head_scales: Mapping[str, np.ndarray], + model_change: Optional[np.ndarray], + model_scale: Optional[np.ndarray], + allowed_readers: Iterable[str], + second_changes: Optional[Mapping[str, np.ndarray]] = None, +) -> dict: + added = float(np.sum(weights * expected)) + allowed = set(allowed_readers) + perturbed = expected > 0 + responses, problems, screens, own_problem = ({}, [], [], None) + + def problem(head, kind, where, nan): + return _problem(total, head, kind, component, where, weights, perturbed, nan) + + for variable, change in head_changes.items(): + responses[variable] = float(np.sum(weights * change)) / added if added else 0.0 + slack = _slack(head_scales[variable]) + if variable == own_variable: + off = (np.abs(change - expected) > slack) | ~np.isfinite(change) + if off.any(): + own_problem = problem(variable, "own_head", off, ~np.isfinite(change)) + continue + reader = variable in allowed or (zero_baseline and (not is_tax)) + if reader: + if second_changes is not None: + slope = second_changes[variable] - change + response_slack = _response_slack(slope, expected) + containing = perturbed & _contains(slope, expected, 2) + linear = perturbed & (np.abs(slope) > 2 * response_slack) & ~containing + if linear.any(): + screens.append( + problem(variable, "contains", linear, ~np.isfinite(slope)) + ) + else: + slope = change + containing = perturbed & _contains(change, expected) + containing = containing | ~np.isfinite(slope) + if containing.any(): + problems.append( + problem(variable, "contains", containing, ~np.isfinite(slope)) + ) + elif is_tax: + moved = _moves(change, expected) + if moved.any(): + problems.append(problem(variable, "moves", moved, ~np.isfinite(change))) + else: + rises = _rises(change, slack) + if rises.any(): + problems.append(problem(variable, "rises", rises, ~np.isfinite(change))) + partition_coefficient = float(sum(responses.values())) + model_coefficient = None + if model_change is not None: + model_coefficient = ( + float(np.sum(weights * model_change)) / added if added else 0.0 + ) + heads_change = sum(head_changes.values()) + if declared_double_count: + heads_change = heads_change + expected + not_finite = ~np.isfinite(model_change) | ~np.isfinite(heads_change) + beyond = np.abs(model_change - heads_change) > _slack(model_scale) + mismatch = beyond | not_finite + if mismatch.any(): + problems.append(problem(total, "model_total", mismatch, not_finite)) + return { + "responses": responses, + "coefficient": partition_coefficient, + "model_coefficient": model_coefficient, + "own_problem": own_problem, + "problems": problems, + "screens": screens, + } + + +def _perturb( + sim, variable, year, baseline, added, monthly, hold=(), held_at=None, touched=None +): + discard_formula_results(sim) + if monthly: + for month, values in baseline[variable]: + _set_guard_value(sim, variable, month, values + added / 12, touched=touched) + else: + _set_guard_value( + sim, variable, year, baseline[variable] + added, touched=touched + ) + held_at = baseline if held_at is None else held_at + for held in hold: + if isinstance(held_at[held], list): + for month, values in held_at[held]: + _set_guard_value(sim, held, month, values, touched=touched) + else: + _set_guard_value(sim, held, year, held_at[held], touched=touched) + + +def _capture(sim, variables, year, layout) -> dict: + out = {} + for v in variables: + if isinstance(layout[v], list): + out[v] = [ + (month, _raw(sim, v, month).astype(np.float64)) + for month, _ in layout[v] + ] + else: + out[v] = _raw(sim, v, year).astype(np.float64) + return out + + +def breaks_on_slope(kind, where, expected, slack, slope): + if kind == "contains": + return where & _contains(slope, expected, allowances=2) + if kind == "rises": + return where & _rises(slope, 2 * slack) + if kind == "moves": + return where & _moves(slope, expected, allowances=2) + return where + + +def still_breaks(kind, where, expected, slack, first, second=None): + if kind == "contains": + if second is not None: + return where & _contains(second - first, expected, allowances=2) + return where & _contains(first, expected) + if kind == "rises": + return where & _rises(first, slack) + if kind == "moves": + return where & _moves(first, expected) + return where + + +def _held_breaks(kind, where, expected, slack, first, second, slope_held, unjudged): + held_at_baseline = still_breaks(kind, where, expected, slack, first, second) + judged = where & ~unjudged + return held_at_baseline | breaks_on_slope(kind, judged, expected, slack, slope_held) + + +def _held_nan(kind, first, second, slope_held, unjudged): + nan = ~np.isfinite(first) | ~np.isfinite(slope_held) & ~unjudged + if kind == "contains" and second is not None: + nan = nan | ~np.isfinite(second) + return nan + + +DIRECT_KINDS = ("contains", "rises", "moves") + + +def counted_once( + sim, + partition: Partition, + year: int, + tolerance: float, + snapshot: Optional[Mapping] = None, + active: Iterable[str] = (), + variables_to_check: Optional[Iterable[str]] = None, +) -> dict: + if not discard_formula_results(sim): + return { + "status": "skipped", + "reason": "this policyengine-core has no way to discard formula results", + } + snapshot = _guard_snapshot(sim, snapshot, partition.component_variables()) + active = set(active) + definitions = sim.tax_benefit_system.variables + totals = [t for t in partition.totals if partition.heads_in(t, active)] + heads_of = {t: [h.variable for h in partition.heads_in(t, active)] for t in totals} + all_heads = list(dict.fromkeys((v for t in totals for v in heads_of[t]))) + model_totals = [t for t in totals if t in definitions] + hw = _raw(sim, "household_weight", year).astype(np.float64) + heads0 = _head_arrays(sim, year, all_heads) + model0 = {t: _raw(sim, t, year).astype(np.float64) for t in model_totals} + baseline = {} + for variable in partition.component_variables(): + if definitions[variable].definition_period == "month": + baseline[variable] = [ + (month, _raw(sim, variable, month).astype(np.float64)) + for month in _months(year) + ] + else: + baseline[variable] = _raw(sim, variable, year).astype(np.float64) + declared = {dc.variable: dc for dc in partition.double_counts} + head_by_variable = {h.variable: h for h in partition.heads} + selected = None if variables_to_check is None else set(variables_to_check) + results, double_count_results = ([], []) + touched = set() + try: + for variable in partition.component_variables(): + if selected is not None and variable not in selected: + continue + definition = definitions[variable] + entity = definition.entity.key + in_totals = [ + t for t in totals if variable in partition.components(t, active) + ] + record = { + "variable": variable, + "totals": in_totals, + "coefficients": {}, + "model_coefficients": {}, + "responses": {}, + "problems": [], + "mediated": [], + "heads_held_skipped": False, + "heads_not_held": {}, + "households_not_held": {}, + "units": 0, + "status": "pass", + "detail": "", + } + if definition.value_type is not float: + record.update(status="skipped", detail="not a money variable") + results.append(record) + continue + monthly = definition.definition_period == "month" + year_values = ( + sum((values for _, values in baseline[variable])) + if monthly + else baseline[variable] + ) + nonzero = year_values != 0 + zero_baseline = not bool(nonzero.any()) + mask = np.ones_like(nonzero) if zero_baseline else nonzero + added = mask.astype(np.float64) + expected = _household(sim, added, entity) + record["units"] = int(round(float(np.sum(expected[hw > 0])))) + if not float(np.sum(hw * expected)) > 0: + record.update( + status="skipped", + detail="non-zero only in households with no weight", + ) + results.append(record) + continue + own = declared[variable].contained_in if variable in declared else variable + own_head = head_by_variable[own] + needed = list(dict.fromkeys((v for t in in_totals for v in heads_of[t]))) + readers_of = { + total: [ + v + for v in heads_of[total] + if own_head.id in head_by_variable[v].reads + ] + for total in in_totals + } + _perturb(sim, variable, year, baseline, added, monthly, touched=touched) + heads1 = _head_arrays(sim, year, needed) + model1 = { + t: _raw(sim, t, year).astype(np.float64) + for t in in_totals + if t in model0 + } + may_respond = { + v + for total in in_totals + for v in heads_of[total] + if v != own + and ( + v in readers_of[total] + or (zero_baseline and (not partition.is_tax_total(total))) + ) + } + heads2 = None + if may_respond: + _perturb( + sim, variable, year, baseline, 2 * added, monthly, touched=touched + ) + heads2 = _head_arrays(sim, year, sorted(may_respond)) + problems, screens = ([], []) + for total in in_totals: + verdict = judge_total( + total=total, + is_tax=partition.is_tax_total(total), + component=variable, + own_variable=own, + declared_double_count=variable in declared + and total in declared[variable].totals, + zero_baseline=zero_baseline, + weights=hw, + expected=expected, + head_changes={v: heads1[v] - heads0[v] for v in heads_of[total]}, + head_scales={v: heads0[v] for v in heads_of[total]}, + model_change=model1[total] - model0[total] + if total in model1 + else None, + model_scale=model0.get(total), + allowed_readers=readers_of[total], + second_changes=None + if heads2 is None + else {v: heads2[v] - heads0[v] for v in heads2}, + ) + record["coefficients"][total] = verdict["coefficient"] + if verdict["model_coefficient"] is not None: + record["model_coefficients"][total] = verdict["model_coefficient"] + record["responses"][total] = { + v: r + for v, r in verdict["responses"].items() + if not abs(r) <= tolerance + } + problems += verdict["problems"] + screens += verdict["screens"] + if verdict["own_problem"] and variable not in declared: + problems.append(verdict["own_problem"]) + mediated = [] + perturbed_units = expected > 0 + for head in dict.fromkeys( + [p["head"] for p in problems if p["kind"] in DIRECT_KINDS] + + [screen["head"] for screen in screens] + ): + held = [v for v in all_heads if v not in (variable, own, head)] + _perturb( + sim, + variable, + year, + baseline, + added, + monthly, + hold=held, + touched=touched, + ) + first = _head_arrays(sim, year, [head])[head] - heads0[head] + second = None + if any( + ( + p["head"] == head and p["kind"] == "contains" + for p in problems + screens + ) + ): + _perturb( + sim, + variable, + year, + baseline, + 2 * added, + monthly, + hold=held, + touched=touched, + ) + second = _head_arrays(sim, year, [head])[head] - heads0[head] + for v in held: + _restore(sim, v, snapshot) + _perturb(sim, variable, year, baseline, added, monthly, touched=touched) + at_first = _capture(sim, held, year, baseline) + head_first = _head_arrays(sim, year, [head])[head] + held_at, unheld_in = ({}, {}) + for v in held: + units = _not_finite_units(at_first[v]) + held_at[v] = at_first[v] + if units.any(): + held_at[v] = _held_values(at_first[v], baseline[v], units) + entity_key = definitions[v].entity.key + unheld_in[v] = _household(sim, units, entity_key) > 0 + _perturb( + sim, + variable, + year, + baseline, + 2 * added, + monthly, + hold=held, + held_at=held_at, + touched=touched, + ) + slope_held = _head_arrays(sim, year, [head])[head] - head_first + for v in held: + _restore(sim, v, snapshot) + not_held = list(unheld_in) + unjudged = np.zeros(len(hw), dtype=bool) + for v in not_held: + unjudged = unjudged | unheld_in[v] + if not_held: + record["heads_not_held"][head] = not_held + record["households_not_held"][head] = int( + np.count_nonzero(unjudged & (hw > 0)) + ) + + def unheld(where): + return [v for v in not_held if bool((unheld_in[v] & where).any())] + + slack = _slack(heads0[head]) + kept = [] + for p in problems: + if p["head"] != head or p["kind"] not in DIRECT_KINDS: + kept.append(p) + continue + breaks = _held_breaks( + p["kind"], + p["where"], + expected, + slack, + first, + second, + slope_held, + unjudged, + ) + rests = p["where"] & unjudged & ~breaks + direct = breaks | rests + cleared = p["where"] & ~direct + if cleared.any(): + via = [ + v + for v in needed + if v not in (variable, own, head) + and bool( + ( + np.abs(heads1[v] - heads0[v])[cleared] + > _slack(heads0[v])[cleared] + ).any() + ) + ] + nan = cleared & p["nan"] + mediated.append( + { + "total": p["total"], + "head": head, + "kind": p["kind"], + "households": int(np.count_nonzero(cleared & (hw > 0))), + "via": via, + "not_finite": bool(nan.any()), + "not_finite_weighted": bool((nan & (hw > 0)).any()), + } + ) + if direct.any(): + kept.append( + _problem( + p["total"], + head, + p["kind"], + variable, + direct, + hw, + perturbed_units, + p["nan"] + | _held_nan( + p["kind"], first, second, slope_held, unjudged + ), + unheld(rests), + rests, + ) + ) + for screen in screens: + if screen["head"] != head: + continue + confirmed = _held_breaks( + "contains", + screen["where"], + expected, + slack, + first, + second, + slope_held, + unjudged, + ) + unconfirmed = screen["where"] & unjudged & ~confirmed + nan = _held_nan("contains", first, second, slope_held, unjudged) + if confirmed.any(): + not_held_there, rests = ((), None) + same = next( + ( + k + for k in kept + if (k["total"], k["head"], k["kind"]) + == (screen["total"], head, "contains") + ), + None, + ) + if same is not None: + kept.remove(same) + confirmed = confirmed | same["where"] + not_held_there, rests = ( + same["heads_not_held"], + same["unjudged"], + ) + kept.append( + _problem( + screen["total"], + head, + "contains", + variable, + confirmed, + hw, + perturbed_units, + nan if same is None else nan | same["nan"], + not_held_there, + rests, + ) + ) + if unconfirmed.any(): + kept.append( + _problem( + screen["total"], + head, + "unconfirmed", + variable, + unconfirmed, + hw, + perturbed_units, + nan, + unheld(unconfirmed), + unconfirmed, + ) + ) + problems = kept + problems = [ + {k: v for k, v in p.items() if k not in ("where", "nan", "unjudged")} + for p in problems + ] + record["mediated"] = mediated + record["heads_held_skipped"] = bool(record["heads_not_held"]) + if variable in declared: + dc = declared[variable] + container_change = heads1[own] - heads0[own] + k = float(np.sum(hw * container_change)) / float(np.sum(hw * expected)) + finite = bool(np.isfinite(container_change).all()) + present = finite and ( + not bool( + ( + np.abs(container_change - expected) > _slack(heads0[own]) + ).any() + ) + ) + row = { + "variable": variable, + "contained_in": dc.contained_in, + "coefficient": k, + "issue": dc.issue, + "status": "pass" if present else "fail", + "not_finite": not finite, + "detail": "" + if present + else f"{dc.contained_in} moves by {k:.4f} per unit of {variable}: the model no longer counts it twice, so remove the declaration and give it its own head", + } + if not finite: + row["detail"] = ( + f"{dc.contained_in} is not finite when {variable} rises, so the check cannot show that {dc.contained_in} still contains {variable}" + ) + double_count_results.append(row) + if problems: + record.update( + status="fail", + problems=problems, + detail="; ".join((p["message"] for p in problems)), + ) + else: + record["detail"] = "; ".join( + ( + not_held_note( + variable, head, heads, record["households_not_held"][head] + ) + for head, heads in record["heads_not_held"].items() + ) + ) + results.append(record) + _restore(sim, variable, snapshot) + finally: + for variable in sorted(touched): + _restore(sim, variable, snapshot) + discard_formula_results(sim) + heads_again = _head_arrays(sim, year, all_heads) + reproduces = all((np.array_equal(heads_again[v], heads0[v]) for v in all_heads)) + return { + "status": "ran", + "year": year, + "tolerance": tolerance, + "components": results, + "double_counts": double_count_results, + "reset_reproduces_baseline": bool(reproduces), + } + + +def publishable_guard(guard: dict) -> dict: + out = dict(guard) + components = [] + for row in guard.get("components") or []: + row = dict(row) + few_units = small(row.get("units") or 0) + if few_units: + row["units"] = None + for key in ("responses", "coefficients", "model_coefficients"): + if key in row: + row[key] = {} + for key in ("problems", "mediated"): + cleaned = [] + for item in row.get(key) or []: + item = dict(item) + if few_units or small(item.get("households") or 0): + item.pop("share", None) + if small(item.get("households") or 0): + item["households"] = None + if small(item.get("households_not_held") or 0): + item["households_not_held"] = None + cleaned.append(item) + row[key] = cleaned + if "households_not_held" in row: + row["households_not_held"] = { + head: None if small(count or 0) else count + for head, count in row["households_not_held"].items() + } + components.append(row) + if "components" in guard: + out["components"] = components + units = { + row.get("variable"): row.get("units") for row in guard.get("components") or [] + } + double_counts = [] + for row in guard.get("double_counts") or []: + row = dict(row) + if small(units.get(row.get("variable")) or 0): + row["coefficient"] = None + if row.get("status") != "pass" and (not row.get("not_finite")): + row["detail"] = ( + f"{row['contained_in']} no longer moves one for one with {row['variable']} (fewer than {MIN_CELL} units): the model no longer counts it twice, so remove the declaration and give it its own head" + ) + double_counts.append(row) + if "double_counts" in guard: + out["double_counts"] = double_counts + return out + + +def record_guard_checks(rec, guard: dict) -> None: + for result in guard.get("components", []): + check_id = f"counted_once:{result['variable']}" + label = f"Raising {result['variable']} by £1 raises its own head by £1, moves no other tax head and raises no other benefit" + if result["status"] == "skipped": + rec.check( + check_id, + label, + None, + result["detail"], + section="counted_once", + scope="", + ) + elif result["status"] == "pass": + rec.check( + check_id, + label, + True, + section="counted_once", + scope="", + note=result["detail"], + ) + for item in result.get("problems") or []: + rec.check( + check_id, + label, + False, + item["message"], + section="counted_once", + scope=f"{item['total']}:{item['head']}:{item['kind']}", + ) + for result in publishable_guard(guard).get("double_counts", []): + rec.check( + f"double_count_present:{result['variable']}", + f"{result['variable']} is still contained in {result['contained_in']}", + result["status"] == "pass", + result["detail"], + section="counted_once", + ) + if "reset_reproduces_baseline" in guard: + rec.check( + "counted_once:reset_reproduces_baseline", + "The counted-once check leaves the baseline unchanged", + bool(guard["reset_reproduces_baseline"]), + "recomputed baseline totals differ from the first run", + section="counted_once", + ) + rec.check( + "counted_once:ran", + "The counted-once check ran", + guard.get("status") == "ran", + guard.get("reason", ""), + severity="warn", + section="counted_once", + ) + + +def _finite(value) -> bool: + return isinstance(value, (int, float)) and math.isfinite(value) + + +def _guard(summary) -> str: + guard = (summary.get("head") or {}).get("guard") or {} + if guard.get("status") != "ran": + return f"Counted-once check not run: {guard.get('reason', 'unknown reason')}." + components = guard.get("components", []) + tested = [c for c in components if c["status"] != "skipped"] + failed = [c for c in tested if c["status"] == "fail"] + skipped = [c for c in components if c["status"] == "skipped"] + lines = [ + f"The check raised each of {len(tested)} components by £1 per unit in {fiscal_year_label(guard['year'])} and read every head of its totals household by household: its own head must rise by exactly £1, no other tax head may move, no other benefit may rise, and the model's own total must move by what its heads moved. A response that is not finite breaks every rule. The check measures a head that breaks a rule twice more: once holding every other head at its baseline value, and once holding every other head at its value after the first £1 while the component rises again. The head fails where either measurement still shows the response. The check cannot hold a head at a value that is not finite. In each unit where another head is not finite after the first £1, it holds that head at its baseline value, and the second measurement judges no household the unit belongs to. There a head fails wherever it first broke a rule, or wherever the check could not confirm whether its response contains the component. {len(failed)} failed and the check skipped {len(skipped)}." + ] + for c in failed: + problems = c.get("problems") or [{"message": c.get("detail", "")}] + lines.append( + f"- `{c['variable']}`: " + "; ".join((p["message"] for p in problems)) + ) + for c in skipped: + lines.append(f"- `{c['variable']}` skipped: {c.get('detail', '')}") + code = "`{}`".format + for c in tested: + counts = c.get("households_not_held") or {} + for head, heads in (c.get("heads_not_held") or {}).items(): + note = not_held_note(c["variable"], head, heads, counts.get(head), code) + lines.append(f"- `{c['variable']}`: {note}") + for c in tested: + for m in c.get("mediated") or []: + moved = ", ".join((code(v) for v in m["via"])) or "none" + text = problem_text( + "mediated", + code(m["head"]), + code(c["variable"]), + m.get("households"), + m.get("not_finite", False), + m.get("not_finite_weighted", True), + ) + lines.append( + f"- `{c['variable']}`: {text} (other heads that moved there: {moved}; {m['total']}; not a failure)" + ) + for dc in guard.get("double_counts", []): + if dc.get("not_finite"): + lines.append( + f"- declared double count `{dc['variable']}`: `{dc['contained_in']}` is NOT FINITE when `{dc['variable']}` rises, so the check cannot show that `{dc['contained_in']}` still contains `{dc['variable']}` ({dc['issue']})" + ) + continue + state = "still contained" if dc["status"] == "pass" else "NO LONGER contained" + coefficient = ( + f"coefficient {dc['coefficient']:.3f}" + if _finite(dc.get("coefficient")) + else "coefficient not shown, fewer than 10 units" + ) + lines.append( + f"- declared double count `{dc['variable']}` is {state} in `{dc['contained_in']}` ({coefficient}; {dc['issue']})" + ) + return "\n".join(lines) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py index d33dc14140..db5fec11e5 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py @@ -776,12 +776,13 @@ def test_two_heads_that_turn_not_finite_fail_by_name(response, route): assert result["reset_reproduces_baseline"] -def core_counted_once(formulas, part, monthly): +def core_counted_once(formulas, part, monthly, **options): """counted_once on a policyengine-core simulation of one household. ``formulas`` maps each head to None (an input, 5 in every period) or a function of a reader by name and the period. Heads are monthly or - yearly; gov_tax and household_weight are yearly. + yearly; gov_tax and household_weight are yearly. ``options`` go to + counted_once. """ from policyengine_core.country_template import CountryTaxBenefitSystem from policyengine_core.country_template.entities import Household @@ -829,7 +830,7 @@ def household_variable(name, formula=None, default=0.0, definition_period=None): }, ) with np.errstate(invalid="ignore"): - return sim, counted_once(sim, part, YEAR, 1e-6) + return sim, counted_once(sim, part, YEAR, 1e-6, **options) @pytest.mark.parametrize("monthly", [False, True], ids=["yearly", "monthly"]) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard_baseline.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard_baseline.py new file mode 100644 index 0000000000..01027184cf --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard_baseline.py @@ -0,0 +1,884 @@ +"""The counted-once check where a head, a model total or a component is not +finite at baseline (#2773), on the fake simulations of +``test_aggregate_impact_guard`` and on policyengine-core. + +Such a value has no change to read. The check names each variable it could +not read once, as a failure of a ``finite:`` check; it fails no component for +it, never gives the engine NaN, and judges every other reading as before. +``counted_once_base_reference`` holds the check as it stood before: every +model whose heads, totals and checked components are finite at baseline must +publish exactly what it published. +""" + +import json +from types import SimpleNamespace + +import counted_once_base_reference as reference +import numpy as np +import pytest +from hypothesis import example, given, settings +from hypothesis import strategies as st +from test_aggregate_impact_guard import ( + GATED, + HEADS_OF, + MEDIATED_NAN, + MEDIATED_NAN_READS, + OVERLAP, + READS, + SHAPES, + YEAR, + EntitySim, + FakeSim, + by_variable, + core_counted_once, + declared_reads, + gated_taxes, + household_by_household, + kinds, + mediated_in_two_totals, + messages, + model, + overlap_model, + partition, + run_not_finite, + shapes_formulas, + two_tax_totals, +) + +from policyengine_uk.utils.aggregate_impact import guard, measure, render +from policyengine_uk.utils.aggregate_impact.compare import ( + ALWAYS_BLOCKING_PREFIXES, + classify_check, +) + +NOT_FINITE_VALUES = [np.nan, np.inf, -np.inf] + + +def named(result): + """{variable: head id} of what the result says is not finite at baseline.""" + return { + item["variable"]: item["head"] + for item in result.get("not_finite_at_baseline", []) + } + + +def note(variable, where="fewer than 10 households"): + return ( + f"{variable} is not finite at baseline in {where}, so the counted-once " + "check could not confirm it there" + ) + + +def recorded(result): + rec = measure.Recorder() + measure.record_guard_checks(rec, result) + return rec.checks + + +# tax_a is sqrt(-1): NaN at baseline, whatever tax_b does. +NAN_TAX_A = { + "tax_b": None, + "tax_a": lambda f, period: np.sqrt(-1 + 0 * f("tax_b")), +} + + +@pytest.mark.parametrize("monthly", [False, True], ids=["yearly", "monthly"]) +def test_a_component_nan_at_baseline_is_named_and_never_given_to_the_engine( + monthly, +): + """Issue #2773, case 1, on policyengine-core. Raising tax_a, NaN at + baseline, gave the engine NaN, and the check stopped with "ValueError: + Unable to set value for variable "tax_a", as the input contains NaN + values." Now the check raises no unit where tax_a is not finite, so it + skips tax_a, which has no other unit, and passes tax_b. It names tax_a, + and gov_tax, which adds it, once each.""" + sim, result = core_counted_once(NAN_TAX_A, partition(["tax_a", "tax_b"]), monthly) + rows = by_variable(result) + assert list(rows) == ["tax_a", "tax_b"] + assert rows["tax_a"]["status"] == "skipped" + assert rows["tax_a"]["detail"] == ( + "the check could raise it only in households with no weight or where it " + "is not finite at baseline" + ) + assert rows["tax_b"]["status"] == "pass" and rows["tax_b"]["detail"] == "" + assert named(result) == {"tax_a": "tax_a", "gov_tax": None} + assert [item["message"] for item in result["not_finite_at_baseline"]] == [ + note("tax_a"), + note("gov_tax"), + ] + assert result["reset_reproduces_baseline"] + assert np.isnan(sim.calculate("tax_a", YEAR)).all() + + +@pytest.mark.parametrize("monthly", [False, True], ids=["yearly", "monthly"]) +def test_holding_a_head_nan_at_baseline_raises_nothing(monthly): + """Issue #2773, case 2: tax_c is tax_b, so it moves with tax_b. To measure + tax_c again, the check held tax_a at its baseline, NaN, and the engine + raised the same ValueError. Now it leaves tax_a to its formula, which + gives it back NaN in every period, so tax_a is held at its baseline and + the check finds tax_c moving with tax_b, as it does.""" + formulas = {**NAN_TAX_A, "tax_c": lambda f, period: f("tax_b")} + _, result = core_counted_once( + formulas, + partition(["tax_a", "tax_b", "tax_c"]), + monthly, + variables_to_check=["tax_b"], + ) + row = by_variable(result)["tax_b"] + assert messages(row) == { + ("tax_c", "moves"): ( + "gov_tax: tax_c moves when tax_b does in fewer than 10 households" + ) + } + (problem,) = row["problems"] + assert problem["heads_not_held"] == [] and problem["households_not_held"] == 0 + assert "heads_not_held_at_baseline" not in problem + assert not problem["not_finite"] and not row["heads_held_skipped"] + assert named(result) == {"tax_a": "tax_a", "gov_tax": None} + assert result["reset_reproduces_baseline"] + + +def test_a_head_nan_at_baseline_fails_no_component_it_ignores(): + """Issue #2773, cases 3 and 4. Checking tax_b alone, the check read tax_a's + change as NaN - NaN and failed tax_b with "gov_tax: tax_a moves, or is + not finite, when tax_b does in fewer than 10 households" and "gov_tax: + the model's gov_tax moves by other than its heads, or is not finite, when + tax_b does in fewer than 10 households", and its reset check returned + False. Now no rule reads tax_a, or gov_tax, which adds it, in that + household: tax_b passes, its own head rising by exactly 1. No response or + coefficient comes from a household that cannot be read, and the reset + check finds tax_a NaN where it started.""" + _, result = core_counted_once( + NAN_TAX_A, + partition(["tax_a", "tax_b"]), + False, + variables_to_check=["tax_b"], + ) + (row,) = result["components"] + assert row["status"] == "pass" and row["problems"] == [] + assert row["responses"] == {"gov_tax": {"tax_b": 1.0}} + assert row["coefficients"] == {"gov_tax": 1.0} + assert row["model_coefficients"] == {} + assert result["reset_reproduces_baseline"] is True + assert named(result) == {"tax_a": "tax_a", "gov_tax": None} + + +class LastComputationSim(FakeSim): + """FakeSim whose ``watched`` variable takes ``after`` the ``at``-th time + it is computed: with ``at`` the count of a run that changed nothing, the + final recomputation, which the reset check reads.""" + + def __init__(self, formulas, inputs, weights, watched, after=None, at=None): + super().__init__(formulas, inputs, weights) + self.watched, self.after, self.at, self.computations = watched, after, at, 0 + + def calculate(self, variable, period, unweighted=True, map_to=None): + key = (variable, str(period)) + if variable == self.watched and key not in self.inputs | self.cache: + self.computations += 1 + if self.computations == self.at: + self.cache[key] = np.asarray(self.after, dtype=float) + return super().calculate(variable, period, unweighted, map_to) + + +@pytest.mark.parametrize( + "before, after, same", + [ + ([np.nan, 1.0], [np.nan, 1.0], True), + ([np.inf, 1.0], [np.inf, 1.0], True), + ([1.0, 1.0], [np.nan, 1.0], False), + ([np.nan, 1.0], [1.0, np.nan], False), + ([np.nan, 1.0], [0.0, 1.0], False), + ([np.inf, 1.0], [-np.inf, 1.0], False), + ([np.nan, 1.0], [np.nan, 1.5], False), + ], +) +def test_the_reset_check_matches_nan_only_with_nan_in_the_same_place( + before, after, same +): + """The reset check treats NaN as equal to NaN in the same household and to + nothing else, so a value that turns NaN after the reset, or a NaN that + moves, still counts as a change.""" + formulas = { + "tax_b": lambda f: 5.0 + 0 * f("x"), + "tax_a": lambda f: np.array(before) + 0 * f("x"), + "gov_tax": lambda f: f("tax_a") + f("tax_b"), + } + part = partition(["tax_a", "tax_b"]) + + def check(sim): + snapshot = guard.snapshot_inputs(sim, part.component_variables()) + with np.errstate(invalid="ignore"): + return guard.counted_once(sim, part, YEAR, 1e-6, snapshot) + + probe = LastComputationSim(formulas, {"x": (1, 1)}, (1, 1), "tax_a") + assert check(probe)["reset_reproduces_baseline"] + sim = LastComputationSim( + formulas, {"x": (1, 1)}, (1, 1), "tax_a", after, probe.computations + ) + assert check(sim)["reset_reproduces_baseline"] is same + + +# Three households of weights 1.5, 2 and 0.5; ``FIRST`` marks the first. +WEIGHTS = (1.5, 2.0, 0.5) +FIRST = np.array([True, False, False]) + + +def run_three(formulas, part, **options): + sim = FakeSim(formulas, {"x": (1000, 2000, 3000)}, WEIGHTS) + snapshot = guard.snapshot_inputs(sim, part.component_variables()) + with np.errstate(divide="ignore", invalid="ignore"): + return guard.counted_once(sim, part, YEAR, 1e-6, snapshot, **options) + + +@pytest.mark.parametrize("bad", NOT_FINITE_VALUES) +def test_a_head_not_finite_in_one_household_is_judged_in_the_others(bad): + """tax_a contains tax_b, and is not finite at baseline in the first + household. The check fails tax_b for tax_a in the other two, with their + share of the weight, and reads neither tax_a nor gov_tax in the first.""" + formulas = model(contains=True) + formulas["tax_a"] = lambda f: np.where(FIRST, bad, f("tax_b") + 0 * f("x")) + result = run_three(formulas, partition(["tax_a", "tax_b"])) + row = by_variable(result)["tax_b"] + assert messages(row) == { + ("tax_a", "moves"): ( + "gov_tax: tax_a moves when tax_b does in fewer than 10 households" + ) + } + (problem,) = row["problems"] + assert problem["households"] == 2 and problem["share"] == pytest.approx(2.5 / 4) + assert row["responses"]["gov_tax"] == {"tax_a": 1.0, "tax_b": 1.0} + assert row["model_coefficients"] == {"gov_tax": 2.0} + assert named(result) == {"tax_a": "tax_a", "gov_tax": None} + assert result["not_finite_at_baseline"][0]["households"] == 1 + assert result["reset_reproduces_baseline"] + + +@pytest.mark.parametrize("bad", NOT_FINITE_VALUES) +def test_a_component_not_finite_in_one_household_is_checked_in_the_others(bad): + """tax_a is not finite at baseline in the first household, and tax_c + holds tax_a, or 3 where tax_a is not finite. Raising tax_a, the check + sets the first household's tax_a to 0, which moves tax_c there by -3; + it judges nothing there, and fails tax_a for tax_c in the other two.""" + formulas = model(contains=False) + formulas["tax_a"] = lambda f: np.where(FIRST, bad, 0.1 * f("x")) + formulas["tax_c"] = lambda f: np.where(np.isfinite(f("tax_a")), f("tax_a"), 3.0) + formulas["gov_tax"] = lambda f: f("tax_a") + f("tax_b") + f("tax_c") + result = run_three( + formulas, partition(["tax_a", "tax_b", "tax_c"]), variables_to_check=["tax_a"] + ) + (row,) = result["components"] + assert row["units"] == 2 + assert kinds(row) == {("tax_c", "moves")} + (problem,) = row["problems"] + assert problem["households"] == 2 and problem["share"] == 1.0 + assert named(result) == {"tax_a": "tax_a", "gov_tax": None} + assert result["reset_reproduces_baseline"] + + +def mediated_with(tax_n): + """tax_a responds to tax_b only through tax_d, which declares its read of + tax_b; tax_n, a tax no head reads, is ``tax_n``.""" + return { + "tax_b": lambda f: 5.0 + 0 * f("x"), + "tax_d": lambda f: 0.2 * f("tax_b"), + "tax_a": lambda f: 0.5 * f("tax_d"), + "tax_n": tax_n, + "gov_tax": lambda f: f("tax_a") + f("tax_b") + f("tax_d") + f("tax_n"), + } + + +MEDIATED = partition(["tax_b", "tax_d", "tax_a", "tax_n"], reads={"tax_d": ["tax_b"]}) + + +@pytest.mark.parametrize("bad", NOT_FINITE_VALUES) +def test_a_held_head_not_finite_at_baseline_is_held_by_its_formula(bad): + """tax_n, a tax no head reads, is not finite at baseline in the first + household. Its formula gives it back its baseline in every held + measurement, so the check holds it exactly and clears tax_a, which + responds only through tax_d, in all three households: tax_b's result is + the one it gets with tax_n finite.""" + result = run_three( + mediated_with(lambda f: np.where(FIRST, bad, 7.0)), + MEDIATED, + variables_to_check=["tax_b"], + ) + finite = run_three( + mediated_with(lambda f: 7.0 + 0 * f("x")), + MEDIATED, + variables_to_check=["tax_b"], + ) + (row,) = result["components"] + assert row["status"] == "pass" and row["mediated"][0]["households"] == 3 + assert row == finite["components"][0] + assert named(result) == {"tax_n": "tax_n", "gov_tax": None} + assert "not_finite_at_baseline" not in finite + + +MOVES_WITH_TAX_B = ( + "the check could not hold tax_n, which is not finite at baseline in some " + "households and moves there with tax_b" +) + + +def test_a_held_head_that_moves_leaves_only_those_households_unjudged(): + """tax_n is NaN at baseline in the first household and tax_b in the + others, so it moves with tax_b there. Left to its formula in the held + measurements, it gives back its baseline only in the first household, + which the check judges: tax_a, carried by tax_d, is mediated there. In + the other two, tax_a's failure stays, and the check could not confirm + tax_d, a declared reader with a slope of 0.2; both name tax_n. tax_n + fails as it contains tax_b.""" + result = run_three( + mediated_with(lambda f: np.where(FIRST, np.nan, f("tax_b"))), + MEDIATED, + variables_to_check=["tax_b"], + ) + (row,) = result["components"] + assert messages(row) == { + ("tax_n", "moves"): ( + "gov_tax: tax_n moves when tax_b does in fewer than 10 households" + ), + ("tax_a", "moves"): ( + "gov_tax: tax_a moves when tax_b does in fewer than 10 households (in " + f"fewer than 10 of them {MOVES_WITH_TAX_B}, so it could not show that " + "tax_a responds only through other heads)" + ), + ("tax_d", "unconfirmed"): ( + "gov_tax: tax_d responds to tax_b in fewer than 10 households where " + f"{MOVES_WITH_TAX_B}, so it could not confirm whether tax_d contains " + "tax_b" + ), + } + assert {p["head"]: p["households"] for p in row["problems"]} == { + "tax_n": 2, + "tax_a": 2, + "tax_d": 2, + } + for problem in row["problems"]: + if problem["head"] != "tax_n": + assert problem["heads_not_held_at_baseline"] == ["tax_n"] + assert problem["households_not_held"] == 2 + (mediated,) = row["mediated"] + assert (mediated["head"], mediated["households"]) == ("tax_a", 1) + assert row["heads_not_held"] == {} and row["heads_held_skipped"] + + +@pytest.mark.parametrize( + "entity, people", + [ + ("person", [(0, 0), (1, 1), (1, 1)]), + ("benunit", [(0, 0), (1, 1), (2, 1)]), + ], +) +def test_units_that_cancel_within_a_household_still_count_as_moved(entity, people): + """tax_n, a head of people or benefit units, is NaN at baseline in the + first household. In the second it moves with tax_b, up in one unit and + down in the other, so the household's sum stays the same. The head still + moved, so the second household's failures stay and name tax_n.""" + formulas = { + "tax_b": lambda f: np.ones(2) * 5, + "tax_d": lambda f: 0.2 * f("tax_b"), + "tax_a": lambda f: 0.5 * f("tax_d"), + "tax_n": lambda f: np.array([np.nan, f("tax_b")[1], -f("tax_b")[1]]), + "gov_tax": lambda f: ( + f("tax_b") + + f("tax_d") + + f("tax_a") + + sim.map_result(f("tax_n"), entity, "household") + ), + } + sim = EntitySim(formulas, {"x": (1, 1)}, (1, 1), {"tax_n": entity}, people) + part = partition( + ["tax_b", "tax_d", "tax_a", "tax_n"], + reads={"tax_d": ["tax_b"]}, + entities={"tax_n": entity}, + ) + with np.errstate(all="ignore"): + result = guard.counted_once( + sim, + part, + YEAR, + 1e-6, + guard.snapshot_inputs(sim, part.component_variables()), + variables_to_check=("tax_b",), + ) + problems = by_variable(result)["tax_b"]["problems"] + assert {p["head"]: p["households"] for p in problems} == {"tax_a": 1, "tax_d": 1} + assert all(p["heads_not_held_at_baseline"] == ["tax_n"] for p in problems) + assert result["reset_reproduces_baseline"] + + +def test_a_problem_names_heads_the_check_could_not_hold_for_either_reason(): + """One household: ben_c turns NaN once tax_b rises, and tax_n, NaN at + baseline, turns 0. tax_a's failure names both, each with its reason.""" + formulas = mediated_with(lambda f: np.where(f("tax_b") > 5.5, 0.0, np.nan)) + formulas["ben_c"] = lambda f: np.sqrt(5 - f("tax_b")) + formulas["gov_spending"] = lambda f: f("ben_c") + part = partition( + ["tax_b", "tax_d", "tax_a", "tax_n"], ["ben_c"], reads={"tax_d": ["tax_b"]} + ) + _, result = run_not_finite(formulas, part, x=(1,), weights=(1,)) + row = by_variable(result)["tax_b"] + assert messages(row)[("tax_a", "moves")] == ( + "gov_tax: tax_a moves when tax_b does in fewer than 10 households (in " + "fewer than 10 of them the check could not hold ben_c, which is not " + "finite there once tax_b rises, and tax_n, which is not finite at " + "baseline in some households and moves there with tax_b, so it could " + "not show that tax_a responds only through other heads)" + ) + problem = next(p for p in row["problems"] if p["head"] == "tax_a") + assert problem["heads_not_held"] == ["ben_c"] + assert problem["heads_not_held_at_baseline"] == ["tax_n"] + assert row["heads_not_held"]["tax_a"] == ["ben_c"] + assert all(p["head"] != "tax_n" for p in row["problems"]) + + +def test_a_model_total_not_finite_at_baseline_is_named_and_fails_no_component(): + """gov_tax adds a NaN term in the first household that no head holds. The + check reads gov_tax in the other two only, fails no component for it, + names it, and records it under finite:metrics.""" + formulas = model(contains=False) + formulas["gov_tax"] = lambda f: f("tax_a") + f("tax_b") + np.where(FIRST, np.nan, 0) + result = run_three(formulas, partition(["tax_a", "tax_b"])) + assert all(row["status"] == "pass" for row in result["components"]) + assert named(result) == {"gov_tax": None} + checks = recorded(result) + assert checks["finite:metrics"]["failing"] == { + f"counted_once:{YEAR}:gov_tax": f"2026-27: {note('gov_tax')}" + } + assert checks["finite:metrics"]["label"] == "Every reported figure is finite" + + +def declared_double_count(container): + """tax_a contains tax_b, as the partition declares, and is ``container`` + of tax_b and x.""" + formulas = model(contains=True) + formulas["tax_a"] = container + return formulas, partition(["tax_a"], [], [("tax_b", "tax_a")]) + + +def test_a_container_not_finite_in_one_household_is_judged_in_the_others(): + formulas, part = declared_double_count( + lambda f: np.where(FIRST, np.nan, 0.1 * f("x") + f("tax_b")) + ) + result = run_three(formulas, part) + (row,) = result["double_counts"] + assert row["status"] == "pass" and row["coefficient"] == pytest.approx(1.0) + assert by_variable(result)["tax_b"]["status"] == "pass" + + +def test_a_container_not_finite_everywhere_skips_the_double_count(): + """The check cannot show that tax_a still contains tax_b, and says so; a + finite: check names tax_a.""" + formulas, part = declared_double_count(lambda f: np.nan + f("tax_b")) + result = run_three(formulas, part) + (row,) = result["double_counts"] + assert row["status"] == "skipped" and row["coefficient"] is None + assert row["detail"] == ( + "tax_a is not finite at baseline in every household with weight where " + "the check raised tax_b, so it cannot show that tax_a still contains tax_b" + ) + checks = recorded(result) + assert checks["double_count_present:tax_b"]["status"] == "skipped" + assert checks["finite:tax_a"]["status"] == "fail" + text = render._guard({"head": {"guard": measure.publishable_guard(result)}}) + assert ( + f"- declared double count `tax_b` not checked: {row['detail']} " + f"({row['issue']})" in text + ) + + +def test_a_double_count_not_finite_everywhere_is_skipped_and_named(): + """tax_b, the declared double count, is NaN at baseline everywhere: the + check skips it, says why, and names it under finite:metrics.""" + formulas, part = declared_double_count(lambda f: 0.1 * f("x") + f("tax_b")) + formulas["tax_b"] = lambda f: np.nan + 0 * f("x") + result = run_three(formulas, part) + assert by_variable(result)["tax_b"]["status"] == "skipped" + (row,) = result["double_counts"] + assert row["status"] == "skipped" and row["detail"] == ( + "the check could raise tax_b only in households with no weight or where " + "it is not finite at baseline, so it cannot show that tax_a still " + "contains tax_b" + ) + assert named(result) == {"tax_a": "tax_a", "gov_tax": None, "tax_b": None} + assert set(recorded(result)["finite:metrics"]["failing"]) == { + f"counted_once:{YEAR}:gov_tax", + f"counted_once:{YEAR}:tax_b", + } + + +def test_a_benefit_unit_not_finite_at_baseline_leaves_only_its_household(): + """ben_u, a benefit of each benefit unit, is NaN at baseline in the second + household's second unit; ben_w, a household benefit, adds each unit's + finite ben_u. Raising ben_u, the check judges neither the second + household nor ben_u's change there, and fails ben_u for ben_w in the + first and third households alone.""" + people = [(0, 0), (1, 1), (2, 1), (3, 2)] + bad = np.array([False, False, True, False]) + formulas = { + "tax_b": lambda f: 5.0 + 0 * f("x"), + "gov_tax": lambda f: f("tax_b"), + "ben_u": lambda f: np.where(bad, np.nan, 100.0), + "ben_w": lambda f: sim.map_result( + np.nan_to_num(f("ben_u")), "benunit", "household" + ), + "gov_spending": lambda f: ( + sim.map_result(f("ben_u"), "benunit", "household") + f("ben_w") + ), + } + sim = EntitySim(formulas, {"x": (1, 1, 1)}, (1, 2, 4), {"ben_u": "benunit"}, people) + part = partition(["tax_b"], ["ben_u", "ben_w"], entities={"ben_u": "benunit"}) + result = guard.counted_once( + sim, part, YEAR, 1e-6, guard.snapshot_inputs(sim, part.component_variables()) + ) + row = by_variable(result)["ben_u"] + assert row["units"] == 2 + (problem,) = row["problems"] + assert (problem["head"], problem["kind"], problem["households"]) == ( + "ben_w", + "rises", + 2, + ) + assert problem["share"] == 1.0 + assert result["not_finite_at_baseline"][0] == { + "variable": "ben_u", + "head": "ben_u", + "label": "ben_u", + "households": 1, + "message": note("ben_u"), + } + + +def test_the_named_result_blocks_whatever_the_base_did(): + """A head not finite at baseline fails its own finite: check in a scope + naming the year and the head, which a comparison blocks on even when + the base fails it too. No counted-once check fails for it.""" + _, result = core_counted_once(NAN_TAX_A, partition(["tax_a", "tax_b"]), False) + checks = recorded(result) + scope = f"counted_once:{YEAR}:tax_a" + assert checks["finite:tax_a"]["failing"] == {scope: f"2026-27: {note('tax_a')}"} + assert checks["finite:tax_a"]["label"] == "tax_a is finite for every unit" + assert checks["finite:tax_a"]["section"] == "finite" + assert all( + check["status"] != "fail" + for check_id, check in checks.items() + if check_id.startswith("counted_once:") + ) + for check_id, check in checks.items(): + if check["status"] == "fail": + assert check_id.startswith(ALWAYS_BLOCKING_PREFIXES) + assert ( + classify_check("fail", "fail", True, check["failing"], check["failing"]) + == "regression" + ) + + +@pytest.mark.parametrize("households", [3, 12]) +def test_the_published_guard_keeps_no_small_count_of_not_finite_households( + households, +): + """The published guard and the report keep a count of 10 or more + households, and say "fewer than 10" for fewer.""" + formulas = model(contains=False) + some = np.arange(12) < households + formulas["tax_a"] = lambda f: np.where(some, np.nan, 0.1 * f("x")) + sim = FakeSim(formulas, {"x": (1,) * 12}, (1,) * 12) + part = partition(["tax_a", "tax_b"]) + result = guard.counted_once( + sim, part, YEAR, 1e-6, guard.snapshot_inputs(sim, part.component_variables()) + ) + published = measure.publishable_guard(result) + shown = None if households < 10 else households + assert [item["households"] for item in published["not_finite_at_baseline"]] == [ + shown, + shown, + ] + where = "fewer than 10 households" if households < 10 else "12 households" + text = render._guard({"head": {"guard": published}}) + assert f"- {note('`tax_a`', where)}" in text.splitlines() + assert f"- {note('`gov_tax`', where)}" in text.splitlines() + + +# The property's extra heads: no head reads them, and each is not finite at +# baseline in any set of households (none, for a finite head). +EXTRA_TOTAL = {"tax_n": "gov_tax", "ben_n": "gov_spending"} + + +@st.composite +def extra_heads(draw): + shapes = draw(st.lists(SHAPES, min_size=1, max_size=3)) + extra = {} + for head in sorted(draw(st.sets(st.sampled_from(sorted(EXTRA_TOTAL))))): + where = draw( + st.lists(st.booleans(), min_size=len(shapes), max_size=len(shapes)) + ) + extra[head] = (np.array(where), draw(st.sampled_from(NOT_FINITE_VALUES))) + return shapes, extra + + +def with_extra_heads(shapes, extra, reads): + """The properties' model and partition, with ``extra`` heads.""" + formulas = shapes_formulas(shapes) + heads_of = {total: list(heads) for total, heads in HEADS_OF.items()} + for head, (where, value) in extra.items(): + formulas[head] = lambda f, where=where, value=value: np.where(where, value, 7.0) + heads_of[EXTRA_TOTAL[head]].append(head) + for total, heads in heads_of.items(): + formulas[total] = lambda f, heads=heads: sum(f(v) for v in heads) + part = partition(heads_of["gov_tax"], heads_of["gov_spending"], reads=reads) + return formulas, part + + +NAN_IN_FIRST = {"tax_n": (np.array([True, False]), np.nan)} + + +@settings(max_examples=40, deadline=None) +@given(case=extra_heads(), reads=READS) +@example(case=([MEDIATED_NAN, MEDIATED_NAN], NAN_IN_FIRST), reads=MEDIATED_NAN_READS) +@example( + case=( + [("half", "flat", "flat", "flat"), ("flat", "flat", "flat", "flat")], + {"ben_n": (np.array([True, True]), -np.inf)}, + ), + reads=[True, False, False, False, False, False], +) +def test_heads_not_finite_at_baseline_are_named_and_change_nothing_else(case, reads): + """For any heads, with any of the extra heads (which no head reads, and + which respond to nothing) not finite at baseline in any households, the + check returns, names exactly those heads, and the totals that add them, + in blocking finite: checks, and fails no component for them. In each + household it finds what it finds with those heads removed, except that + it reads no total that adds a head not finite there.""" + shapes, extra = case + households = len(shapes) + declared = declared_reads(reads) + bad = {head: where for head, (where, _) in extra.items() if where.any()} + formulas, part = with_extra_heads(shapes, extra, declared) + kept = {head: spec for head, spec in extra.items() if head not in bad} + formulas_without, part_without = with_extra_heads(shapes, kept, declared) + _, result = run_not_finite( + formulas, part, x=(1,) * households, weights=(1,) * households + ) + bad_totals = {EXTRA_TOTAL[head] for head in bad} + assert named(result) == { + **{head: head for head in bad}, + **{total: None for total in bad_totals}, + } + checks = recorded(result) + failing = { + c + for c, check in checks.items() + if check["status"] == "fail" and c.startswith(ALWAYS_BLOCKING_PREFIXES) + } + assert failing == {f"finite:{head}" for head in bad} | ( + {"finite:metrics"} if bad_totals else set() + ) + for row in result["components"]: + assert all(p["head"] not in bad for p in row["problems"]) + if row["variable"] in bad: + assert row["status"] != "fail" + assert row["heads_held_skipped"] == ( + bool(row["heads_not_held"]) + or any(p.get("heads_not_held_at_baseline") for p in row["problems"]) + ) + assert result["reset_reproduces_baseline"] + + found = household_by_household(formulas, part, households) + without = household_by_household(formulas_without, part_without, households) + for k in range(households): + here = {head for head, where in bad.items() if where[k]} + for component, before in without[k].items(): + after = found[k][component] + if not here: + assert after == before + continue + totals_here = {EXTRA_TOTAL[head] for head in here} + assert after == { + **before, + "problems": [ + p + for p in before["problems"] + if not (p[2] == "model_total" and p[0] in totals_here) + ], + } + + +class SetValuesSim(FakeSim): + """FakeSim that keeps every value given to ``set_input``. Like + policyengine-core, FakeSim refuses NaN but takes an infinite value.""" + + def __init__(self, formulas, inputs, weights): + super().__init__(formulas, inputs, weights) + self.set_values = [] + + def set_input(self, variable, period, values): + self.set_values.append(np.array(values, dtype=float)) + super().set_input(variable, period, values) + + +@settings(max_examples=40, deadline=None) +@given(case=extra_heads(), reads=READS) +def test_the_check_raises_and_holds_variables_only_at_finite_values(case, reads): + """Whatever is NaN or infinite at baseline, the check raises and holds + variables only at finite values. It also restores the supplied inputs, + which are finite here, so every value it gives the engine is finite.""" + shapes, extra = case + households = len(shapes) + formulas, part = with_extra_heads(shapes, extra, declared_reads(reads)) + sim = SetValuesSim(formulas, {"x": (1,) * households}, (1,) * households) + snapshot = guard.snapshot_inputs(sim, part.component_variables()) + with np.errstate(divide="ignore", invalid="ignore"): + guard.counted_once(sim, part, YEAR, 1e-6, snapshot) + assert sim.set_values + assert all(np.isfinite(values).all() for values in sim.set_values) + + +def published(check, record, publish, describe, formulas, part, x, weights): + """What one implementation publishes for a model, as JSON: the check's + result, the checks it records, the published guard and the report's + section.""" + sim = FakeSim(formulas, {"x": x}, weights) + snapshot = guard.snapshot_inputs(sim, part.component_variables()) + with np.errstate(divide="ignore", invalid="ignore"): + result = check(sim, part, YEAR, 1e-6, snapshot) + rec = measure.Recorder() + record(rec, result) + publishable = publish(result) + return json.dumps( + { + "result": result, + "checks": list(rec.checks.values()), + "published": publishable, + "report": describe({"head": {"guard": publishable}}), + } + ) + + +NOW = ( + guard.counted_once, + measure.record_guard_checks, + measure.publishable_guard, + render._guard, +) +BEFORE = ( + reference.counted_once, + reference.record_guard_checks, + reference.publishable_guard, + reference._guard, +) + + +def assert_publishes_as_before(formulas, part, x, weights): + assert published(*NOW, formulas, part, x, weights) == published( + *BEFORE, formulas, part, x, weights + ) + + +def stale_double_count(): + return declared_double_count(lambda f: 0.1 * f("x")) + + +# The guard tests' own models, on their three households. +MODELS = { + "containment": lambda: (model(contains=True), partition(["tax_a", "tax_b"])), + "means test": lambda: ( + model(contains=False), + partition(["tax_a"], ["ben_c", "ben_d"]), + ), + "rising benefit": lambda: ( + model(interaction=0.5), + partition(["tax_a"], ["ben_c", "ben_d"]), + ), + "declared double count": lambda: declared_double_count( + lambda f: 0.1 * f("x") + f("tax_b") + ), + "stale double count": stale_double_count, + "overlapping benefits": lambda: (overlap_model(), OVERLAP), + "overlap containing ib": lambda: (overlap_model(uc_contains_ib=1.0), OVERLAP), + "gated containment": lambda: ( + { + **gated_taxes(), + "ben_c": lambda f: 100 + 0 * f("x"), + "gov_spending": lambda f: f("ben_c"), + }, + GATED, + ), +} + + +@pytest.mark.parametrize("name", MODELS) +def test_every_finite_model_publishes_what_it_did_before(name): + """Differential against ``counted_once_base_reference``: with every head + and total finite at baseline, the check, the checks it records, the + published guard and the report say exactly what they said before, in + the same order.""" + formulas, part = MODELS[name]() + assert_publishes_as_before(formulas, part, (1000, 2000, 3000), (1.5, 2.0, 0.5)) + + +@pytest.mark.parametrize("nan_in", [(), (4,)], ids=["finite", "NaN once raised"]) +def test_twelve_households_in_two_totals_publish_what_they_did_before(nan_in): + assert_publishes_as_before( + mediated_in_two_totals(nan_in), two_tax_totals(), (1,) * 12, (1,) * 12 + ) + + +@settings(max_examples=60, deadline=None) +@given( + shapes=st.lists(SHAPES, min_size=1, max_size=3), + reads=READS, + weights=st.lists(st.sampled_from([0.0, 0.5, 1.0, 2.0]), min_size=3, max_size=3), +) +@example(shapes=[MEDIATED_NAN], reads=MEDIATED_NAN_READS, weights=[1.0, 1.0, 1.0]) +def test_finite_baselines_publish_what_they_did_before(shapes, reads, weights): + """Differential against ``counted_once_base_reference`` for the guard + tests' random heads, which are finite at baseline and may turn infinite + or NaN once raised, on up to three households of any weights.""" + households = len(shapes) + declared = declared_reads(reads) + part = partition(HEADS_OF["gov_tax"], HEADS_OF["gov_spending"], reads=declared) + assert_publishes_as_before( + shapes_formulas(shapes), + part, + (1,) * households, + tuple(weights[:households]), + ) + + +@settings(max_examples=100, deadline=None) +@given( + values=st.lists( + st.one_of( + st.floats(allow_nan=False, allow_infinity=False), + st.sampled_from(NOT_FINITE_VALUES), + ), + min_size=1, + max_size=6, + ), + monthly=st.booleans(), +) +def test_the_check_sets_finite_values_bit_for_bit_and_others_at_zero(values, monthly): + values = np.array(values) + layout = [("2026-01", values), ("2026-02", values[::-1])] if monthly else values + settable = guard._settable(layout) + pairs = zip(layout, settable) if monthly else [((None, values), (None, settable))] + for (_, before), (_, after) in pairs: + finite = np.isfinite(before) + assert np.isfinite(after).all() + assert (after[~finite] == 0).all() + assert before[finite].tobytes() == after[finite].tobytes() + + +def test_a_finite_held_head_keeps_the_words_it_had(): + assert guard._unheld_words(["ben_c"], "tax_b") == reference._unheld_words( + ["ben_c"], "tax_b" + ) + assert guard._unheld_words(["ben_c", "tax_g"], "tax_b") == ( + reference._unheld_words(["ben_c", "tax_g"], "tax_b") + ) + assert guard._unheld_words([], "tax_b", at_baseline=["tax_n", "tax_m"]) == ( + "the check could not hold tax_n and tax_m, one or more of which is not " + "finite at baseline in some households and moves there with tax_b" + ) diff --git a/policyengine_uk/utils/aggregate_impact/README.md b/policyengine_uk/utils/aggregate_impact/README.md index 942bc464c9..4569f14a3e 100644 --- a/policyengine_uk/utils/aggregate_impact/README.md +++ b/policyengine_uk/utils/aggregate_impact/README.md @@ -126,7 +126,12 @@ The checks are: every other head held at baseline, and fails only where it still breaks it; a response carried by another head (Incapacity Benefit lowers Carer's Allowance, which Universal Credit counts as income) is reported as - mediated. A declared double count must still be contained in its head; + mediated. A declared double count must still be contained in its head. The + check records a NaN or infinite baseline value under a blocking `finite:` + check (`finite:metrics` for a total or a double count) and continues + judging readable households. It holds such a head by leaving it to its + formula, and judges each household where the formula gives back the held + value; - **signs, bounds and finiteness**, and **determinism:** a rebuilt simulation reproduces the baseline exactly; - **inert heads:** a head that is zero for every unit is a warning, unless the diff --git a/policyengine_uk/utils/aggregate_impact/guard.py b/policyengine_uk/utils/aggregate_impact/guard.py index 4a460d37aa..d8357c5d87 100644 --- a/policyengine_uk/utils/aggregate_impact/guard.py +++ b/policyengine_uk/utils/aggregate_impact/guard.py @@ -62,7 +62,27 @@ (``households_not_held``); a problem does the same for the households that rest on the measurement the check could not make. A declared double count whose container's response is not finite fails: the check cannot - show that the container still contains it. + show that the container still contains it; +- a head, a model total or a component that is not finite at baseline has + no change to read where it is not finite, so the check cannot confirm it + there. It names each such variable once, with the households, in + ``not_finite_at_baseline``, which ``measure`` records as a failure of a + ``finite:`` check, and it fails no component for it. No rule reads a + head, or a model total, in the households where it, or one of its heads, + is not finite at baseline. The engine refuses NaN as an input, so the + check never sets such a value. To raise a component that is not finite + at baseline, it sets each such unit to 0 and raises the others. The 0s + change values only in the households that hold them, and it judges + nothing there for that component. To hold a head that is not + finite at baseline, it does not set the head at all: it leaves it to its + formula. Wherever the head does not respond to the component, its + formula gives it back the value the measurement holds it at (its + baseline, NaN where it was NaN, or its value after the first point), so + it is held there exactly and every other head reads what it would read + with the head held. The check compares the head with that value, unit by + unit and NaN matching only NaN, and judges the household only where they + match. A problem in a household it could not judge stays, and names the + head in ``heads_not_held_at_baseline``, beside ``heads_not_held``. What it cannot see: a head that adds a component back while tapering it away pound for pound (``max(0, a - c) + c``) does not move at the margin, so it is @@ -77,7 +97,8 @@ only after discarding formula results, so a value some formula computed earlier is never frozen as an input. Between perturbations it discards every formula result, and restores any input it overrode; at the end it recomputes -the baseline and requires it to match the first computation exactly. +the baseline and requires it to match the first computation exactly, with +NaN matching only NaN in the same place. """ from __future__ import annotations @@ -348,6 +369,15 @@ def _held_values(at_first, baseline, units): return np.where(units, baseline, at_first) +def _settable(values): + """``values`` in its layout (see ``_perturb``), with each value that is not + finite at 0: the engine refuses NaN as an input, and an infinite amount is + no level to hold. Finite values keep every bit.""" + if isinstance(values, list): + return [(month, np.where(np.isfinite(v), v, 0.0)) for month, v in values] + return np.where(np.isfinite(values), values, 0.0) + + def _households_text(households) -> str: """A count of households as messages give it (``count_text``); None is a small count that the published guard removed.""" @@ -404,18 +434,30 @@ def _unheld_where(households, of_them=False) -> str: return f"{households:,} {what}" -def _unheld_words(heads, component, name=str) -> str: - """The check could not hold ``heads``, which are not finite there.""" - names = [name(v) for v in heads] +def _listed(names) -> tuple: + """``names`` as a list in words, and how a relative clause refers to it.""" if len(names) == 1: - listed, which = names[0], "which is" - else: - listed = f"{', '.join(names[:-1])} and {names[-1]}" - which = "one or more of which is" - return ( - f"the check could not hold {listed}, {which} not finite there once " - f"{name(component)} rises" - ) + return names[0], "which is" + return f"{', '.join(names[:-1])} and {names[-1]}", "one or more of which is" + + +def _unheld_words(heads, component, name=str, at_baseline=()) -> str: + """The check could not hold ``heads``, which are not finite there once + ``component`` rises, or ``at_baseline``, which it left to their formulas + as they are not finite at baseline somewhere, and which moved there.""" + groups = [] + if heads: + listed, which = _listed([name(v) for v in heads]) + groups.append( + f"{listed}, {which} not finite there once {name(component)} rises" + ) + if at_baseline: + listed, which = _listed([name(v) for v in at_baseline]) + groups.append( + f"{listed}, {which} not finite at baseline in some households and " + f"moves there with {name(component)}" + ) + return "the check could not hold " + ", and ".join(groups) def not_held_note(component, head, heads, households, name=str) -> str: @@ -429,6 +471,17 @@ def not_held_note(component, head, heads, households, name=str) -> str: ) +def not_finite_note(variable, households, name=str) -> str: + """What the check says about ``variable``, not finite at baseline in + ``households`` households with weight (None: a small count that the + published guard removed).""" + return ( + f"{name(variable)} is not finite at baseline in " + f"{_unheld_where(households)}, so the counted-once check could not " + "confirm it there" + ) + + def _problem( total, head, @@ -440,15 +493,20 @@ def _problem( nan, not_held=(), unjudged=None, + at_baseline=(), ): """One failing (total, head, kind), with the households it fails in. ``nan`` marks the households where a measurement behind the problem saw a response that is not finite. ``unjudged`` marks the households of - ``where`` that rest on the second held measurement, which the check - could not make there because it could not hold ``not_held``. The + ``where`` that rest on a held measurement, which the check could not + make there because it could not hold ``not_held`` (not finite once the + component rises) or ``at_baseline`` (left to their formulas, as they are + not finite at baseline somewhere, and moved there). The direct-effect step reads ``where``, ``nan`` and ``unjudged``, and - ``counted_once`` drops them before it reports the result. + ``counted_once`` drops them before it reports the result. The problem + carries ``heads_not_held_at_baseline`` only when ``at_baseline`` names a + head. """ # Only households with weight count towards a published count. weighted = weights > 0 @@ -458,21 +516,23 @@ def _problem( not_finite = bool((where & nan).any()) not_finite_weighted = bool((where & nan & weighted).any()) unjudged = np.zeros_like(where) if unjudged is None else unjudged & where - unheld = int(np.count_nonzero(unjudged & weighted)) if not_held else 0 + unheld = ( + int(np.count_nonzero(unjudged & weighted)) if not_held or at_baseline else 0 + ) text = problem_text(kind, head, component, count, not_finite, not_finite_weighted) message = f"{total}: {text}" + words = _unheld_words(list(not_held), component, at_baseline=list(at_baseline)) if kind == "unconfirmed": message += ( - f" where {_unheld_words(list(not_held), component)}, so it could " - f"not confirm whether {head} contains {component}" + f" where {words}, so it could not confirm whether {head} contains " + f"{component}" ) - elif not_held: + elif not_held or at_baseline: message += ( - f" (in {_unheld_where(unheld, of_them=True)} " - f"{_unheld_words(list(not_held), component)}, so it could not show " - f"that {head} responds only through other heads)" + f" (in {_unheld_where(unheld, of_them=True)} {words}, so it could not " + f"show that {head} responds only through other heads)" ) - return { + out = { "total": total, "head": head, "kind": kind, @@ -482,11 +542,11 @@ def _problem( "not_finite": not_finite, "not_finite_weighted": not_finite_weighted, "heads_not_held": list(not_held), - "households_not_held": unheld, - "where": where, - "nan": nan, - "unjudged": unjudged, } + if at_baseline: + out["heads_not_held_at_baseline"] = list(at_baseline) + out.update(households_not_held=unheld, where=where, nan=nan, unjudged=unjudged) + return out def judge_total( @@ -505,13 +565,21 @@ def judge_total( model_scale: Optional[np.ndarray], allowed_readers: Iterable[str], second_changes: Optional[Mapping[str, np.ndarray]] = None, + unread: Optional[Mapping[str, np.ndarray]] = None, + model_unread: Optional[np.ndarray] = None, ) -> dict: """Apply the counted-once rules, household by household, to one total. ``head_changes`` maps each head variable of the total to its change by household when 1 per unit was added to ``component`` (``expected`` by household); ``second_changes`` gives the changes when 2 per unit were - added, where they were measured. + added, where they were measured. ``unread`` marks, for a head, the + households where its change cannot be read (it, or the component, is not + finite at baseline there), and ``model_unread`` the same for the model's + total. No rule judges a head there, nor the model's total where it or + any head of the total cannot be read. A head's response, and the model + total's coefficient, is its mean over the households where it can be + read, and is left out where no household with weight can be. - The component's own head must change by exactly ``expected`` in every household. @@ -534,19 +602,31 @@ def judge_total( mask ``nan`` of households whose judged response is not finite, and the mask ``unjudged`` (empty here). """ - added = float(np.sum(weights * expected)) allowed = set(allowed_readers) perturbed = expected > 0 responses, problems, screens, own_problem = {}, [], [], None + nowhere = np.zeros(len(expected), dtype=bool) + unread = unread or {} def problem(head, kind, where, nan): return _problem(total, head, kind, component, where, weights, perturbed, nan) + def mean_response(change, skip): + """The mean change per unit added, over households not in ``skip``; + None when ``skip`` leaves no household with weight to read.""" + added = float(np.sum(weights * np.where(skip, 0.0, expected))) + if not added: + return None if skip.any() else 0.0 + return float(np.sum(weights * np.where(skip, 0.0, change))) / added + for variable, change in head_changes.items(): - responses[variable] = float(np.sum(weights * change)) / added if added else 0.0 + skip = unread.get(variable, nowhere) + response = mean_response(change, skip) + if response is not None: + responses[variable] = response slack = _slack(head_scales[variable]) if variable == own_variable: - off = (np.abs(change - expected) > slack) | ~np.isfinite(change) + off = ((np.abs(change - expected) > slack) | ~np.isfinite(change)) & ~skip if off.any(): own_problem = problem(variable, "own_head", off, ~np.isfinite(change)) continue @@ -562,7 +642,12 @@ def problem(head, kind, where, nan): # Another head's response can add to a containing head's # slope, so any other linear response is measured again with # the other heads held. - linear = perturbed & (np.abs(slope) > 2 * response_slack) & ~containing + linear = ( + perturbed + & (np.abs(slope) > 2 * response_slack) + & ~containing + & ~skip + ) if linear.any(): screens.append( problem(variable, "contains", linear, ~np.isfinite(slope)) @@ -572,31 +657,33 @@ def problem(head, kind, where, nan): containing = perturbed & _contains(change, expected) # A response that is not finite fails wherever it is, as any # other head's does. - containing = containing | ~np.isfinite(slope) + containing = (containing | ~np.isfinite(slope)) & ~skip if containing.any(): problems.append( problem(variable, "contains", containing, ~np.isfinite(slope)) ) elif is_tax: - moved = _moves(change, expected) + moved = _moves(change, expected) & ~skip if moved.any(): problems.append(problem(variable, "moves", moved, ~np.isfinite(change))) else: - rises = _rises(change, slack) + rises = _rises(change, slack) & ~skip if rises.any(): problems.append(problem(variable, "rises", rises, ~np.isfinite(change))) partition_coefficient = float(sum(responses.values())) model_coefficient = None if model_change is not None: - model_coefficient = ( - float(np.sum(weights * model_change)) / added if added else 0.0 - ) + # The model's total cannot be read where it, or a head, cannot. + skip = nowhere if model_unread is None else model_unread + for variable in head_changes: + skip = skip | unread.get(variable, nowhere) + model_coefficient = mean_response(model_change, skip) heads_change = sum(head_changes.values()) if declared_double_count: heads_change = heads_change + expected not_finite = ~np.isfinite(model_change) | ~np.isfinite(heads_change) beyond = np.abs(model_change - heads_change) > _slack(model_scale) - mismatch = beyond | not_finite + mismatch = (beyond | not_finite) & ~skip if mismatch.any(): problems.append(problem(total, "model_total", mismatch, not_finite)) return { @@ -649,6 +736,21 @@ def _capture(sim, variables, year, layout) -> dict: return out +def _moved(sim, variable, year, layout) -> np.ndarray: + """Households holding a unit where ``variable`` now differs from its value + in ``layout`` (see ``_perturb``) in any period, NaN matching only NaN.""" + now, then = _capture(sim, [variable], year, layout)[variable], layout[variable] + if isinstance(now, list): + pairs = [(a, b) for (_, a), (_, b) in zip(now, then)] + else: + pairs = [(now, then)] + differs = np.logical_or.reduce( + [~((a == b) | (np.isnan(a) & np.isnan(b))) for a, b in pairs] + ) + entity = sim.tax_benefit_system.variables[variable].entity.key + return _household(sim, differs, entity) > 0 + + def breaks_on_slope(kind, where, expected, slack, slope): """Households of ``where`` whose head's slope, with the other heads held at their values after the first point, breaks ``kind``.""" @@ -679,25 +781,42 @@ def still_breaks(kind, where, expected, slack, first, second=None): return where -def _held_breaks(kind, where, expected, slack, first, second, slope_held, unjudged): +def _held_breaks( + kind, + where, + expected, + slack, + first, + second, + slope_held, + unjudged, + unjudged_at_baseline=None, +): """Households of ``where`` where a held measurement that judged them still breaks ``kind`` (see ``still_breaks`` and ``breaks_on_slope``). ``slope_held`` is the slope with the other heads held at their values - after the first point; it judges no household of ``unjudged``. + after the first point; it judges no household of ``unjudged``. ``first`` + and ``second``, with the other heads held at their baseline values, + judge no household of ``unjudged_at_baseline``. """ - held_at_baseline = still_breaks(kind, where, expected, slack, first, second) + at_baseline = where + if unjudged_at_baseline is not None: + at_baseline = where & ~unjudged_at_baseline + held_at_baseline = still_breaks(kind, at_baseline, expected, slack, first, second) judged = where & ~unjudged return held_at_baseline | breaks_on_slope(kind, judged, expected, slack, slope_held) -def _held_nan(kind, first, second, slope_held, unjudged): +def _held_nan(kind, first, second, slope_held, unjudged, unjudged_at_baseline=None): """Households where a held measurement that judges ``kind`` (see ``_held_breaks``) saw a response that is not finite.""" - nan = ~np.isfinite(first) | (~np.isfinite(slope_held) & ~unjudged) + nan = ~np.isfinite(first) if kind == "contains" and second is not None: nan = nan | ~np.isfinite(second) - return nan + if unjudged_at_baseline is not None: + nan = nan & ~unjudged_at_baseline + return nan | (~np.isfinite(slope_held) & ~unjudged) DIRECT_KINDS = ("contains", "rises", "moves") @@ -715,7 +834,11 @@ def counted_once( """Perturb each component (or those in ``variables_to_check``). ``tolerance`` is kept for reports of per-unit responses; the verdicts - themselves are household by household (see ``judge_total``). + themselves are household by household (see ``judge_total``). The result + carries ``not_finite_at_baseline`` only when a head, a model total or a + component it raised is not finite at baseline: one entry for each, with + the head's id and label (None for a variable that is no head), the + households with weight where it is not finite, and a message. """ if not discard_formula_results(sim): return { @@ -742,6 +865,16 @@ def counted_once( ] else: baseline[variable] = _raw(sim, variable, year).astype(np.float64) + # Where the check sets a variable, it sets these values: the baseline, + # with each value that is not finite at 0 (see the module docstring). + settable = {variable: _settable(values) for variable, values in baseline.items()} + # The households where each head, and each model total, has no change to + # read: it is not finite there at baseline. + unread = {v: ~np.isfinite(heads0[v]) for v in all_heads} + free_heads = {v for v in all_heads if unread[v].any()} + model_unread = {t: ~np.isfinite(model0[t]) for t in model_totals} + # The same for each component the check raises, by the units it holds. + blind_in = {} declared = {dc.variable: dc for dc in partition.double_counts} head_by_variable = {h.variable: h for h in partition.heads} @@ -782,19 +915,43 @@ def counted_once( if monthly else baseline[variable] ) - nonzero = year_values != 0 + # A unit that is not finite at baseline is never raised, and the + # households holding one are judged for nothing: their readings + # rest on the value the check sets there. + broken = _not_finite_units(baseline[variable]) + nonzero = (year_values != 0) & ~broken zero_baseline = not bool(nonzero.any()) - mask = np.ones_like(nonzero) if zero_baseline else nonzero + mask = ~broken if zero_baseline else nonzero added = mask.astype(np.float64) - expected = _household(sim, added, entity) + blind = _household(sim, broken, entity) > 0 + if blind.any(): + blind_in[variable] = blind + expected = np.where(blind, 0.0, _household(sim, added, entity)) # Perturbed units in households with weight: a zero-weight record # adds nothing to any published figure. record["units"] = int(round(float(np.sum(expected[hw > 0])))) if not float(np.sum(hw * expected)) > 0: + weighted_blind = bool((blind & (hw > 0)).any()) record.update( status="skipped", - detail="non-zero only in households with no weight", + detail=( + "the check could raise it only in households with no " + "weight or where it is not finite at baseline" + if weighted_blind + else "non-zero only in households with no weight" + ), ) + if weighted_blind and variable in declared: + dc = declared[variable] + double_count_results.append( + _unchecked_double_count( + dc, + f"the check could raise {variable} only in households " + "with no weight or where it is not finite at " + f"baseline, so it cannot show that {dc.contained_in} " + f"still contains {variable}", + ) + ) results.append(record) continue own = declared[variable].contained_in if variable in declared else variable @@ -809,7 +966,7 @@ def counted_once( for total in in_totals } - _perturb(sim, variable, year, baseline, added, monthly, touched=touched) + _perturb(sim, variable, year, settable, added, monthly, touched=touched) heads1 = _head_arrays(sim, year, needed) model1 = { t: _raw(sim, t, year).astype(np.float64) @@ -832,7 +989,7 @@ def counted_once( heads2 = None if may_respond: _perturb( - sim, variable, year, baseline, 2 * added, monthly, touched=touched + sim, variable, year, settable, 2 * added, monthly, touched=touched ) heads2 = _head_arrays(sim, year, sorted(may_respond)) problems, screens = [], [] @@ -857,6 +1014,10 @@ def counted_once( second_changes=None if heads2 is None else {v: heads2[v] - heads0[v] for v in heads2}, + unread={v: unread[v] | blind for v in heads_of[total]}, + model_unread=( + model_unread[total] | blind if total in model_unread else None + ), ) record["coefficients"][total] = verdict["coefficient"] if verdict["model_coefficient"] is not None: @@ -883,17 +1044,24 @@ def counted_once( + [screen["head"] for screen in screens] ): held = [v for v in all_heads if v not in (variable, own, head)] + # The engine refuses NaN as an input, so the check sets no head + # that is not finite at baseline: it leaves each to its formula, + # and the measurement judges a household only where each came + # out at the value it holds the heads at (see ``_moved``). + fixed = [v for v in held if v not in free_heads] + free = [v for v in held if v in free_heads] _perturb( sim, variable, year, - baseline, + settable, added, monthly, - hold=held, + hold=fixed, touched=touched, ) first = _head_arrays(sim, year, [head])[head] - heads0[head] + moved_in = {v: _moved(sim, v, year, baseline) for v in free} second = None if any( p["head"] == head and p["kind"] == "contains" @@ -903,14 +1071,16 @@ def counted_once( sim, variable, year, - baseline, + settable, 2 * added, monthly, - hold=held, + hold=fixed, touched=touched, ) second = _head_arrays(sim, year, [head])[head] - heads0[head] - for v in held: + for v in free: + moved_in[v] = moved_in[v] | _moved(sim, v, year, baseline) + for v in fixed: _restore(sim, v, snapshot) # Second test: hold the other heads at their values after the # first point, and move the component on to the second. A head @@ -921,11 +1091,11 @@ def counted_once( # head is not finite after the first point keeps that head at its # baseline value, and this test judges no household the unit # belongs to. - _perturb(sim, variable, year, baseline, added, monthly, touched=touched) + _perturb(sim, variable, year, settable, added, monthly, touched=touched) at_first = _capture(sim, held, year, baseline) head_first = _head_arrays(sim, year, [head])[head] held_at, unheld_in = {}, {} - for v in held: + for v in fixed: units = _not_finite_units(at_first[v]) held_at[v] = at_first[v] if units.any(): @@ -938,15 +1108,18 @@ def counted_once( sim, variable, year, - baseline, + settable, 2 * added, monthly, - hold=held, + hold=fixed, held_at=held_at, touched=touched, ) slope_held = _head_arrays(sim, year, [head])[head] - head_first - for v in held: + # A head left to its formula is held at its value after the + # first point only where it stays there. + moved_on = {v: _moved(sim, v, year, at_first) for v in free} + for v in fixed: _restore(sim, v, snapshot) not_held = list(unheld_in) unjudged = np.zeros(len(hw), dtype=bool) @@ -957,11 +1130,30 @@ def counted_once( record["households_not_held"][head] = int( np.count_nonzero(unjudged & (hw > 0)) ) + # The households each held measurement could not judge: the + # second where a head it holds is not finite after the first + # point or one left to its formula moved, the first (at the + # baseline) where one left to its formula moved. + unjudged_at_baseline = np.zeros(len(hw), dtype=bool) + free_moved = {} + for v in free: + unjudged_at_baseline = unjudged_at_baseline | moved_in[v] + unjudged = unjudged | moved_on[v] + if (moved_in[v] | moved_on[v]).any(): + free_moved[v] = moved_in[v] | moved_on[v] + either = unjudged | unjudged_at_baseline def unheld(where): - """The heads the check could not hold in ``where``.""" + """The heads the check could not hold in ``where`` once the + component rises.""" return [v for v in not_held if bool((unheld_in[v] & where).any())] + def unheld_free(where): + """The heads left to their formula that moved in ``where``.""" + return [ + v for v in free_moved if bool((free_moved[v] & where).any()) + ] + slack = _slack(heads0[head]) kept = [] for p in problems: @@ -977,10 +1169,11 @@ def unheld(where): second, slope_held, unjudged, + unjudged_at_baseline, ) - # The second measurement clears no household it could not + # A held measurement clears no household it could not # judge: there the problem rests on that measurement. - rests = p["where"] & unjudged & ~breaks + rests = p["where"] & either & ~breaks direct = breaks | rests cleared = p["where"] & ~direct if cleared.any(): @@ -1023,10 +1216,16 @@ def unheld(where): perturbed_units, p["nan"] | _held_nan( - p["kind"], first, second, slope_held, unjudged + p["kind"], + first, + second, + slope_held, + unjudged, + unjudged_at_baseline, ), unheld(rests), rests, + unheld_free(rests), ) ) # A linear response that was not one for one may hide a @@ -1043,13 +1242,21 @@ def unheld(where): second, slope_held, unjudged, + unjudged_at_baseline, + ) + # Where a held measurement judged no household, the check + # could not confirm the reader there either way. + unconfirmed = screen["where"] & either & ~confirmed + nan = _held_nan( + "contains", + first, + second, + slope_held, + unjudged, + unjudged_at_baseline, ) - # Where the second measurement judged no household, the - # check could not confirm the reader there either way. - unconfirmed = screen["where"] & unjudged & ~confirmed - nan = _held_nan("contains", first, second, slope_held, unjudged) if confirmed.any(): - not_held_there, rests = (), None + not_held_there, rests, at_baseline_there = (), None, () same = next( ( k @@ -1062,9 +1269,10 @@ def unheld(where): if same is not None: kept.remove(same) confirmed = confirmed | same["where"] - not_held_there, rests = ( + not_held_there, rests, at_baseline_there = ( same["heads_not_held"], same["unjudged"], + same.get("heads_not_held_at_baseline", ()), ) kept.append( _problem( @@ -1078,6 +1286,7 @@ def unheld(where): nan if same is None else nan | same["nan"], not_held_there, rests, + at_baseline_there, ) ) if unconfirmed.any(): @@ -1093,6 +1302,7 @@ def unheld(where): nan, unheld(unconfirmed), unconfirmed, + unheld_free(unconfirmed), ) ) problems = kept @@ -1101,38 +1311,21 @@ def unheld(where): for p in problems ] record["mediated"] = mediated - record["heads_held_skipped"] = bool(record["heads_not_held"]) + record["heads_held_skipped"] = bool(record["heads_not_held"]) or any( + "heads_not_held_at_baseline" in p for p in problems + ) if variable in declared: - dc = declared[variable] - container_change = heads1[own] - heads0[own] - k = float(np.sum(hw * container_change)) / float(np.sum(hw * expected)) - finite = bool(np.isfinite(container_change).all()) - present = finite and not bool( - (np.abs(container_change - expected) > _slack(heads0[own])).any() - ) - row = { - "variable": variable, - "contained_in": dc.contained_in, - "coefficient": k, - "issue": dc.issue, - "status": "pass" if present else "fail", - "not_finite": not finite, - "detail": "" - if present - else ( - f"{dc.contained_in} moves by {k:.4f} per unit of {variable}: " - "the model no longer counts it twice, so remove the " - "declaration and give it its own head" - ), - } - if not finite: - # Not a stale declaration: the check cannot tell. - row["detail"] = ( - f"{dc.contained_in} is not finite when {variable} rises, " - f"so the check cannot show that {dc.contained_in} still " - f"contains {variable}" + double_count_results.append( + _double_count_row( + declared[variable], + heads1[own] - heads0[own], + expected, + # The households where the container's change can be read. + ~(unread[own] | blind), + hw, + heads0[own], ) - double_count_results.append(row) + ) if problems: record.update( status="fail", @@ -1156,8 +1349,11 @@ def unheld(where): discard_formula_results(sim) heads_again = _head_arrays(sim, year, all_heads) - reproduces = all(np.array_equal(heads_again[v], heads0[v]) for v in all_heads) - return { + # NaN matches NaN in the same place, and nothing else. + reproduces = all( + np.array_equal(heads_again[v], heads0[v], equal_nan=True) for v in all_heads + ) + result = { "status": "ran", "year": year, "tolerance": tolerance, @@ -1165,3 +1361,83 @@ def unheld(where): "double_counts": double_count_results, "reset_reproduces_baseline": bool(reproduces), } + # Each variable the check could not read somewhere, named once: the heads, + # the model totals, then the other components it raised. + where_not_finite = { + **{v: unread[v] for v in all_heads if unread[v].any()}, + **{t: model_unread[t] for t in model_totals if model_unread[t].any()}, + } + for variable, blind in blind_in.items(): + where_not_finite.setdefault(variable, blind) + if where_not_finite: + result["not_finite_at_baseline"] = [] + for variable, households in where_not_finite.items(): + head = head_by_variable.get(variable) + count = int(np.count_nonzero(households & (hw > 0))) + result["not_finite_at_baseline"].append( + { + "variable": variable, + "head": None if head is None else head.id, + "label": None if head is None else head.label, + "households": count, + "message": not_finite_note(variable, count), + } + ) + return result + + +def _double_count_row(dc, change, expected, readable, weights, scale) -> dict: + """Whether the container of a declared double count still moves by exactly + what was added to it, in each household where its ``change`` can be read + (``readable``).""" + variable, container = dc.variable, dc.contained_in + base = float(np.sum(weights * np.where(readable, expected, 0.0))) + if not base > 0: + return _unchecked_double_count( + dc, + f"{container} is not finite at baseline in every household with " + f"weight where the check raised {variable}, so it cannot show that " + f"{container} still contains {variable}", + ) + k = float(np.sum(weights * np.where(readable, change, 0.0))) / base + finite = bool(np.isfinite(change[readable]).all()) + present = finite and not bool( + ((np.abs(change - expected) > _slack(scale)) & readable).any() + ) + row = { + "variable": variable, + "contained_in": container, + "coefficient": k, + "issue": dc.issue, + "status": "pass" if present else "fail", + "not_finite": not finite, + "detail": "" + if present + else ( + f"{container} moves by {k:.4f} per unit of {variable}: the model no " + "longer counts it twice, so remove the declaration and give it its " + "own head" + ), + } + if not finite: + # Not a stale declaration: the check cannot tell. + row["detail"] = ( + f"{container} is not finite when {variable} rises, so the check " + f"cannot show that {container} still contains {variable}" + ) + return row + + +def _unchecked_double_count(dc, detail) -> dict: + """A declared double count the check could not judge anywhere: a + container or component not finite at baseline wherever it would read it. + It is skipped, not passed, and a ``finite:`` check names the cause.""" + return { + "variable": dc.variable, + "contained_in": dc.contained_in, + "coefficient": None, + "issue": dc.issue, + "status": "skipped", + "not_finite": False, + "detail": detail, + } diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py index a4fb47e1cf..847b2fc4b3 100644 --- a/policyengine_uk/utils/aggregate_impact/measure.py +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -1180,8 +1180,10 @@ def publishable_guard(guard: dict) -> dict: no count and no share. A count of households where the check could not hold a head (``households_not_held``) below ``MIN_CELL`` goes too. Messages already say "fewer than 10". The heads the check could not hold - (``heads_not_held``) and the ``not_finite`` flags hold no figures, so - they stay. + (``heads_not_held``, ``heads_not_held_at_baseline``) and the + ``not_finite`` flags hold no figures, so they stay. A count below + ``MIN_CELL`` of households where a variable is not finite at baseline + (``not_finite_at_baseline``) goes too. """ out = dict(guard) components = [] @@ -1226,8 +1228,9 @@ def publishable_guard(guard: dict) -> dict: row = dict(row) if small(units.get(row.get("variable")) or 0): row["coefficient"] = None - # A container that is not finite has a detail with no figures. - if row.get("status") != "pass" and not row.get("not_finite"): + # A container that is not finite, or a double count the check + # skipped, has a detail with no figures. + if row.get("status") == "fail" and not row.get("not_finite"): row["detail"] = ( f"{row['contained_in']} no longer moves one for one with " f"{row['variable']} (fewer than {MIN_CELL} units): the model " @@ -1237,6 +1240,14 @@ def publishable_guard(guard: dict) -> dict: double_counts.append(row) if "double_counts" in guard: out["double_counts"] = double_counts + if "not_finite_at_baseline" in guard: + out["not_finite_at_baseline"] = [ + { + **item, + "households": None if small(item["households"]) else item["households"], + } + for item in guard["not_finite_at_baseline"] + ] return out @@ -1283,10 +1294,21 @@ def record_guard_checks(rec: Recorder, guard: dict) -> None: rec.check( f"double_count_present:{result['variable']}", f"{result['variable']} is still contained in {result['contained_in']}", - result["status"] == "pass", + None if result["status"] == "skipped" else result["status"] == "pass", result["detail"], section="counted_once", ) + # A variable not finite at baseline fails a finite: check, which blocks + # whatever the base did: its own for a head, else finite:metrics. + for item in guard.get("not_finite_at_baseline") or []: + detail = f"{fiscal_year_label(guard['year'])}: {item['message']}" + scope = f"counted_once:{guard['year']}:{item['variable']}" + if item["head"] is None: + check_id, label = "finite:metrics", "Every reported figure is finite" + else: + check_id = f"finite:{item['head']}" + label = f"{item['label']} is finite for every unit" + rec.check(check_id, label, False, detail, section="finite", scope=scope) if "reset_reproduces_baseline" in guard: rec.check( "counted_once:reset_reproduces_baseline", diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py index 8f398f3011..dff2799b13 100644 --- a/policyengine_uk/utils/aggregate_impact/render.py +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -6,7 +6,7 @@ from typing import Mapping, Optional from . import COMMENT_MARKER -from .guard import not_held_note, problem_text +from .guard import not_finite_note, not_held_note, problem_text from .privacy import withheld_text from .spec import Spec, fiscal_year_label @@ -330,6 +330,13 @@ def _guard(summary) -> str: f"response contains the component. {len(failed)} failed and the check " f"skipped {len(skipped)}." ] + code = "`{}`".format + # A variable not finite at baseline fails no component; a finite: check + # names it. + for item in guard.get("not_finite_at_baseline") or []: + lines.append( + "- " + not_finite_note(item["variable"], item.get("households"), code) + ) for c in failed: problems = c.get("problems") or [{"message": c.get("detail", "")}] lines.append( @@ -337,7 +344,6 @@ def _guard(summary) -> str: ) for c in skipped: lines.append(f"- `{c['variable']}` skipped: {c.get('detail', '')}") - code = "`{}`".format for c in tested: counts = c.get("households_not_held") or {} for head, heads in (c.get("heads_not_held") or {}).items(): @@ -361,6 +367,12 @@ def _guard(summary) -> str: f"{moved}; {m['total']}; not a failure)" ) for dc in guard.get("double_counts", []): + if dc.get("status") == "skipped": + lines.append( + f"- declared double count `{dc['variable']}` not checked: " + f"{dc['detail']} ({dc['issue']})" + ) + continue if dc.get("not_finite"): lines.append( f"- declared double count `{dc['variable']}`: `{dc['contained_in']}` "