From 456d984dc662fe3bb39601d31c19f684757da5ed Mon Sep 17 00:00:00 2001 From: Abhishek B R Date: Thu, 24 Sep 2026 03:20:10 +0530 Subject: [PATCH] fix: report receipt-less ok observations as unverifiable in receipts mode The as-published day files currently carry an empty receipts array on every observation, so `./reproduce --receipts` rebuilt zero passing sources and reported each printed value as a MISMATCH with exit 1. An empty receipts array is not evidence that a value is wrong; it means nothing was disclosed to recompute from. An ok observation with no receipts now gets its own unverifiable verdict. The CLI prints UNVERIFIABLE with the reason, adds an unverifiable count to the summary only when there is one, and exits 2 like the other could-not-verify outcomes. Without --version it also points at the versioned record, whose day files keep the receipts and verify. Validation: ruff passes and the offline suite passes; the two new tests fail on main. Signed-off-by: Abhishek B R --- README.md | 3 ++- scripts/verify_published_record.py | 35 ++++++++++++++++++++++--- src/gpu_index/published/verify.py | 23 +++++++++++++++- tests/unit/test_published_reader_cli.py | 19 ++++++++++++++ tests/unit/test_published_verify.py | 17 ++++++++++++ 5 files changed, 91 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8c273b9..ca15fb6 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,8 @@ hour. A full day takes a few minutes. `--full` explicitly selects the default. `--receipts` is the fast opt-in: it recomputes each value and band from that observation's own published receipts, without re-deriving attendance or weights. In receipts mode only, withheld -contributing receipts degrade to digest-only verification with a notice. +contributing receipts degrade to digest-only verification with a notice, and an +`ok` observation that carries no receipts is reported as unverifiable (exit 2). To verify one version's re-derivation throughout, pass `--version `: diff --git a/scripts/verify_published_record.py b/scripts/verify_published_record.py index 732e907..7c101d2 100644 --- a/scripts/verify_published_record.py +++ b/scripts/verify_published_record.py @@ -30,8 +30,9 @@ Exit codes: 0 every checked observation matched (digest OK; degraded observations are reported but do not fail), 1 any MISMATCH or digest FAIL, 2 could not verify (the record source is unreachable, no published -day file for the date, or no observation for the SKU/stamp) or usage -error. Never 0 without verifying something. +day file for the date, no observation for the SKU/stamp, or an "ok" +observation that carries no receipts) or usage error. Never 0 without +verifying something. """ from __future__ import annotations @@ -65,6 +66,7 @@ MIN_DISCLOSURE_WINDOW_DAYS, VERDICT_DEGRADED, VERDICT_MATCH, + VERDICT_UNVERIFIABLE, disclosure_window_warning, recompute_observation, select_observations, @@ -341,7 +343,7 @@ def main(argv=None) -> int: ) return 2 - matched = mismatched = degraded = 0 + matched = mismatched = degraded = unverifiable = 0 for observation in observations: try: check = recompute_observation(observation) @@ -356,6 +358,15 @@ def main(argv=None) -> int: f"{check.sku} {stamp_label} DEGRADED digest-only " f"(withheld: {', '.join(check.withheld_sources)}) digest OK {identity}" ) + elif check.verdict == VERDICT_UNVERIFIABLE: + unverifiable += 1 + published = _value_label( + check.published_value, check.published_band + ) + print( + f"{check.sku} {stamp_label} UNVERIFIABLE " + f"published {published} digest OK {identity}" + ) else: verdict = ( "MATCH" if check.verdict == VERDICT_MATCH else "MISMATCH" @@ -386,10 +397,11 @@ def main(argv=None) -> int: for message in check.messages: print(f" {message}") - total = matched + mismatched + degraded + total = matched + mismatched + degraded + unverifiable print( f"summary: {total} observation(s): {matched} MATCH, " f"{mismatched} MISMATCH, {degraded} degraded" + + (f", {unverifiable} unverifiable" if unverifiable else "") ) if stamp is None: window_note = _window_warning(reader, sku, date, version) @@ -404,6 +416,21 @@ def main(argv=None) -> int: "published disclosure policy, so the vote recompute cannot " "run for them (the file digest still verifies)" ) + if unverifiable: + print( + f"could not verify: {unverifiable} observation(s) carry no " + "receipts, so their published value and band cannot be " + "recomputed from the published record", + file=sys.stderr, + ) + if args.version is None: + print( + "the as-published history may omit receipts that the " + "versioned record keeps: rerun with --version using the " + "version shown on each line", + file=sys.stderr, + ) + return 2 return 0 diff --git a/src/gpu_index/published/verify.py b/src/gpu_index/published/verify.py index 652470e..22335d9 100644 --- a/src/gpu_index/published/verify.py +++ b/src/gpu_index/published/verify.py @@ -78,6 +78,9 @@ non-contributing receipt (rejected/excluded/never priced into the composite) does not impair the recompute and full verification proceeds. +An "ok" observation whose receipts array is empty discloses no votes at +all, so it is reported as unverifiable rather than as a mismatch. + No-print observations (value null) are checked for consistency instead: the passing set must be below ``calc_params.min_sources_to_publish`` (the same minimum-panel rule the panel applies), and — when every receipt is @@ -98,6 +101,7 @@ VERDICT_MATCH = "match" VERDICT_MISMATCH = "mismatch" VERDICT_DEGRADED = "degraded" +VERDICT_UNVERIFIABLE = "unverifiable" _SUPPORTED_AGGREGATIONS = frozenset( {"median_stddev_votes", "median_ci_votes"} @@ -167,7 +171,7 @@ class ObservationCheck: sku: str observed_at: str status: str # published status: "ok" | "no_print" - verdict: str # VERDICT_MATCH | VERDICT_MISMATCH | VERDICT_DEGRADED + verdict: str # VERDICT_MATCH | VERDICT_MISMATCH | VERDICT_DEGRADED | VERDICT_UNVERIFIABLE published_value: Optional[float] = None published_band: Optional[float] = None recomputed_value: Optional[float] = None @@ -267,6 +271,23 @@ def recompute_observation(observation: dict) -> ObservationCheck: raise PublishedRecordError( f"observation {sku} {observed_at} has no receipts array" ) + if status == "ok" and not receipts: + # A printed value with an empty receipts array discloses nothing + # to rebuild the votes from. That is not evidence the value is + # wrong, so it is never a MISMATCH: the observation could not be + # verified from this artifact at all. + return ObservationCheck( + sku=sku, + observed_at=observed_at, + status=status, + verdict=VERDICT_UNVERIFIABLE, + published_value=observation.get("value_usd_gpu_hr"), + published_band=observation.get("stability_band_usd_gpu_hr"), + messages=( + "observation carries no receipts: the published value and " + "band cannot be recomputed from this artifact", + ), + ) passing: List[Tuple[str, float, float]] = [] vote_stddevs: Dict[str, float] = {} diff --git a/tests/unit/test_published_reader_cli.py b/tests/unit/test_published_reader_cli.py index 41b5443..1ae7003 100644 --- a/tests/unit/test_published_reader_cli.py +++ b/tests/unit/test_published_reader_cli.py @@ -437,6 +437,25 @@ def test_cli_withheld_degrades_with_distinct_message_and_exit_zero( assert " MISMATCH digest OK" not in out +def test_cli_ok_observation_without_receipts_exits_two_not_mismatch( + tmp_path, monkeypatch, cli, capsys +): + def mutate(document): + document["data"]["observations"][0]["receipts"] = [] + + root = _tampered_record(tmp_path, "observations/2026/08/25.json", mutate) + monkeypatch.setenv("GPU_INDEX_DATA_DIR", str(root)) + monkeypatch.delenv("GPU_INDEX_PUBLIC_BASE_URL", raising=False) + assert _run(monkeypatch, cli, "--sku", "H100", "--date", "2026-08-25") == 2 + captured = capsys.readouterr() + assert "UNVERIFIABLE" in captured.out + assert "observation carries no receipts" in captured.out + assert " MISMATCH digest OK" not in captured.out + assert "1 MATCH, 0 MISMATCH, 0 degraded, 1 unverifiable" in captured.out + assert "could not verify" in captured.err + assert "--version " in captured.err + + def test_cli_unreachable_front_exits_two_with_one_actionable_line( record_env, monkeypatch, cli, capsys ): diff --git a/tests/unit/test_published_verify.py b/tests/unit/test_published_verify.py index 2dc3159..fc52d2a 100644 --- a/tests/unit/test_published_verify.py +++ b/tests/unit/test_published_verify.py @@ -37,6 +37,7 @@ VERDICT_DEGRADED, VERDICT_MATCH, VERDICT_MISMATCH, + VERDICT_UNVERIFIABLE, UnsupportedStatisticError, recompute_observation, select_observations, @@ -340,6 +341,22 @@ def test_withheld_contributing_source_degrades_to_digest_only(): assert any("digest" in m for m in check.messages) +def test_ok_observation_without_receipts_is_unverifiable_not_mismatch(): + def mutate(document): + document["data"]["observations"][0]["receipts"] = [] + + envelope = _tampered("observations/2026/08/25.json", mutate) + observation = select_observations(envelope, sku="H100")[0] + check = recompute_observation(observation) + assert check.verdict == VERDICT_UNVERIFIABLE + assert check.published_value == observation["value_usd_gpu_hr"] + assert check.recomputed_value is None + assert check.messages == ( + "observation carries no receipts: the published value and band " + "cannot be recomputed from this artifact", + ) + + def test_withheld_non_contributing_source_does_not_degrade(): # A withheld receipt that never contributed (rejected by the filter) # does not impair the vote rebuild: applyDisclosure nulls price+sd