Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,8 @@ hour. A full day takes a few minutes.
`--full` explicitly selects the default. `--receipts` is the fast opt-in: it
recomputes each value and band from that observation's own published receipts,
without re-deriving attendance or weights. In receipts mode only, withheld
contributing receipts degrade to digest-only verification with a notice.
contributing receipts degrade to digest-only verification with a notice, and an
`ok` observation that carries no receipts is reported as unverifiable (exit 2).

To verify one version's re-derivation throughout, pass `--version <n>`:

Expand Down
35 changes: 31 additions & 4 deletions scripts/verify_published_record.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@
Exit codes: 0 every checked observation matched (digest OK; degraded
observations are reported but do not fail), 1 any MISMATCH or digest
FAIL, 2 could not verify (the record source is unreachable, no published
day file for the date, or no observation for the SKU/stamp) or usage
error. Never 0 without verifying something.
day file for the date, no observation for the SKU/stamp, or an "ok"
observation that carries no receipts) or usage error. Never 0 without
verifying something.
"""

from __future__ import annotations
Expand Down Expand Up @@ -65,6 +66,7 @@
MIN_DISCLOSURE_WINDOW_DAYS,
VERDICT_DEGRADED,
VERDICT_MATCH,
VERDICT_UNVERIFIABLE,
disclosure_window_warning,
recompute_observation,
select_observations,
Expand Down Expand Up @@ -341,7 +343,7 @@ def main(argv=None) -> int:
)
return 2

matched = mismatched = degraded = 0
matched = mismatched = degraded = unverifiable = 0
for observation in observations:
try:
check = recompute_observation(observation)
Expand All @@ -356,6 +358,15 @@ def main(argv=None) -> int:
f"{check.sku} {stamp_label} DEGRADED digest-only "
f"(withheld: {', '.join(check.withheld_sources)}) digest OK {identity}"
)
elif check.verdict == VERDICT_UNVERIFIABLE:
unverifiable += 1
published = _value_label(
check.published_value, check.published_band
)
print(
f"{check.sku} {stamp_label} UNVERIFIABLE "
f"published {published} digest OK {identity}"
)
else:
verdict = (
"MATCH" if check.verdict == VERDICT_MATCH else "MISMATCH"
Expand Down Expand Up @@ -386,10 +397,11 @@ def main(argv=None) -> int:
for message in check.messages:
print(f" {message}")

total = matched + mismatched + degraded
total = matched + mismatched + degraded + unverifiable
print(
f"summary: {total} observation(s): {matched} MATCH, "
f"{mismatched} MISMATCH, {degraded} degraded"
+ (f", {unverifiable} unverifiable" if unverifiable else "")
)
if stamp is None:
window_note = _window_warning(reader, sku, date, version)
Expand All @@ -404,6 +416,21 @@ def main(argv=None) -> int:
"published disclosure policy, so the vote recompute cannot "
"run for them (the file digest still verifies)"
)
if unverifiable:
print(
f"could not verify: {unverifiable} observation(s) carry no "
"receipts, so their published value and band cannot be "
"recomputed from the published record",
file=sys.stderr,
)
if args.version is None:
print(
"the as-published history may omit receipts that the "
"versioned record keeps: rerun with --version <n> using the "
"version shown on each line",
file=sys.stderr,
)
return 2
return 0


Expand Down
23 changes: 22 additions & 1 deletion src/gpu_index/published/verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@
non-contributing receipt (rejected/excluded/never priced into the
composite) does not impair the recompute and full verification proceeds.

An "ok" observation whose receipts array is empty discloses no votes at
all, so it is reported as unverifiable rather than as a mismatch.

No-print observations (value null) are checked for consistency instead:
the passing set must be below ``calc_params.min_sources_to_publish``
(the same minimum-panel rule the panel applies), and — when every receipt is
Expand All @@ -98,6 +101,7 @@
VERDICT_MATCH = "match"
VERDICT_MISMATCH = "mismatch"
VERDICT_DEGRADED = "degraded"
VERDICT_UNVERIFIABLE = "unverifiable"

_SUPPORTED_AGGREGATIONS = frozenset(
{"median_stddev_votes", "median_ci_votes"}
Expand Down Expand Up @@ -167,7 +171,7 @@ class ObservationCheck:
sku: str
observed_at: str
status: str # published status: "ok" | "no_print"
verdict: str # VERDICT_MATCH | VERDICT_MISMATCH | VERDICT_DEGRADED
verdict: str # VERDICT_MATCH | VERDICT_MISMATCH | VERDICT_DEGRADED | VERDICT_UNVERIFIABLE
published_value: Optional[float] = None
published_band: Optional[float] = None
recomputed_value: Optional[float] = None
Expand Down Expand Up @@ -267,6 +271,23 @@ def recompute_observation(observation: dict) -> ObservationCheck:
raise PublishedRecordError(
f"observation {sku} {observed_at} has no receipts array"
)
if status == "ok" and not receipts:
# A printed value with an empty receipts array discloses nothing
# to rebuild the votes from. That is not evidence the value is
# wrong, so it is never a MISMATCH: the observation could not be
# verified from this artifact at all.
return ObservationCheck(
sku=sku,
observed_at=observed_at,
status=status,
verdict=VERDICT_UNVERIFIABLE,
published_value=observation.get("value_usd_gpu_hr"),
published_band=observation.get("stability_band_usd_gpu_hr"),
messages=(
"observation carries no receipts: the published value and "
"band cannot be recomputed from this artifact",
),
)

passing: List[Tuple[str, float, float]] = []
vote_stddevs: Dict[str, float] = {}
Expand Down
19 changes: 19 additions & 0 deletions tests/unit/test_published_reader_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,25 @@ def test_cli_withheld_degrades_with_distinct_message_and_exit_zero(
assert " MISMATCH digest OK" not in out


def test_cli_ok_observation_without_receipts_exits_two_not_mismatch(
tmp_path, monkeypatch, cli, capsys
):
def mutate(document):
document["data"]["observations"][0]["receipts"] = []

root = _tampered_record(tmp_path, "observations/2026/08/25.json", mutate)
monkeypatch.setenv("GPU_INDEX_DATA_DIR", str(root))
monkeypatch.delenv("GPU_INDEX_PUBLIC_BASE_URL", raising=False)
assert _run(monkeypatch, cli, "--sku", "H100", "--date", "2026-08-25") == 2
captured = capsys.readouterr()
assert "UNVERIFIABLE" in captured.out
assert "observation carries no receipts" in captured.out
assert " MISMATCH digest OK" not in captured.out
assert "1 MATCH, 0 MISMATCH, 0 degraded, 1 unverifiable" in captured.out
assert "could not verify" in captured.err
assert "--version <n>" in captured.err


def test_cli_unreachable_front_exits_two_with_one_actionable_line(
record_env, monkeypatch, cli, capsys
):
Expand Down
17 changes: 17 additions & 0 deletions tests/unit/test_published_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
VERDICT_DEGRADED,
VERDICT_MATCH,
VERDICT_MISMATCH,
VERDICT_UNVERIFIABLE,
UnsupportedStatisticError,
recompute_observation,
select_observations,
Expand Down Expand Up @@ -340,6 +341,22 @@ def test_withheld_contributing_source_degrades_to_digest_only():
assert any("digest" in m for m in check.messages)


def test_ok_observation_without_receipts_is_unverifiable_not_mismatch():
def mutate(document):
document["data"]["observations"][0]["receipts"] = []

envelope = _tampered("observations/2026/08/25.json", mutate)
observation = select_observations(envelope, sku="H100")[0]
check = recompute_observation(observation)
assert check.verdict == VERDICT_UNVERIFIABLE
assert check.published_value == observation["value_usd_gpu_hr"]
assert check.recomputed_value is None
assert check.messages == (
"observation carries no receipts: the published value and band "
"cannot be recomputed from this artifact",
)


def test_withheld_non_contributing_source_does_not_degrade():
# A withheld receipt that never contributed (rejected by the filter)
# does not impair the vote rebuild: applyDisclosure nulls price+sd
Expand Down