From 7ea9c8c4be8fcbffcc02e256908f7ef907ed73c8 Mon Sep 17 00:00:00 2001 From: Ray Apena <87909466+Ray3R3@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:47:39 +0000 Subject: [PATCH] Add JSON Lines results store and image-level evaluation Results are stored one InspectionResult per line, one file per run, and validated on load. Evaluation reports AUROC (Mann-Whitney U, ties as half) and precision/recall/F1 at a threshold chosen on held-out good images for a target false-positive rate, so the test set never sets the threshold. Co-authored-by: Sam Oteng <98087809+samoteng@users.noreply.github.com> --- README.md | 3 +- src/vista/evaluation.py | 120 +++++++++++++++++++++++++++++++++++++++ src/vista/store.py | 41 +++++++++++++ tests/test_evaluation.py | 91 +++++++++++++++++++++++++++++ tests/test_store.py | 42 ++++++++++++++ 5 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 src/vista/evaluation.py create mode 100644 src/vista/store.py create mode 100644 tests/test_evaluation.py create mode 100644 tests/test_store.py diff --git a/README.md b/README.md index ddd3573..e38b1d3 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ Every stage exchanges data as an `InspectionResult` ([`src/vista/schema.py`](src - [x] Inspection result schema - [x] Dataset preparation (MVTec AD) - [ ] Baseline anomaly model (PatchCore, see [ADR 0001](docs/decisions/0001-baseline-model.md)) -- [ ] Evaluation harness (image AUROC, pixel AUROC, PRO, latency) +- [x] Results store (JSON Lines) and image-level metrics (AUROC, precision/recall at a target false-positive rate) +- [ ] Pixel-level evaluation (pixel AUROC, PRO) and latency - [ ] Inference API - [ ] Web demo diff --git a/src/vista/evaluation.py b/src/vista/evaluation.py new file mode 100644 index 0000000..02ed510 --- /dev/null +++ b/src/vista/evaluation.py @@ -0,0 +1,120 @@ +"""Image-level evaluation for anomaly detection. + +Two things are kept apart on purpose: + +* **Ranking quality** (AUROC) does not depend on a threshold, so it measures + the model alone. +* **Operating point** (precision, recall, F1) depends on a threshold, which + must be chosen on held-out *good* images, never on the test set. We pick it + to hit a target false-positive rate on those images. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from dataclasses import asdict, dataclass + +from vista.schema import InspectionResult + + +def is_defective_from_id(image_id: str) -> bool: + """Ground truth from an MVTec-style id, e.g. ``bottle/test/broken_large/000.png``.""" + parts = image_id.replace("\\", "/").split("/") + if "test" not in parts: + raise ValueError(f"not a test image id: {image_id}") + defect_type = parts[parts.index("test") + 1] + return defect_type != "good" + + +def roc_auc(scores: Sequence[float], labels: Sequence[bool]) -> float: + """Area under the ROC curve via the Mann-Whitney U statistic. + + Equals the probability that a random defective image scores higher than a + random good one, with ties counted as half. O(n log n), no dependencies. + """ + if len(scores) != len(labels): + raise ValueError("scores and labels must be the same length") + n_pos = sum(labels) + n_neg = len(labels) - n_pos + if n_pos == 0 or n_neg == 0: + raise ValueError("AUROC needs at least one defective and one good image") + + # Average ranks (1-based) with ties sharing the mean rank. + order = sorted(range(len(scores)), key=lambda i: scores[i]) + ranks = [0.0] * len(scores) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and scores[order[j + 1]] == scores[order[i]]: + j += 1 + mean_rank = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[order[k]] = mean_rank + i = j + 1 + + rank_sum_pos = sum(r for r, y in zip(ranks, labels, strict=True) if y) + u = rank_sum_pos - n_pos * (n_pos + 1) / 2 + return u / (n_pos * n_neg) + + +def threshold_for_fpr(good_scores: Sequence[float], target_fpr: float) -> float: + """Smallest threshold whose false-positive rate on ``good_scores`` is <= ``target_fpr``. + + An image is flagged when ``score >= threshold``. + """ + if not good_scores: + raise ValueError("need at least one held-out good score") + if not 0.0 <= target_fpr < 1.0: + raise ValueError("target_fpr must be in [0, 1)") + ordered = sorted(good_scores) + allowed = math.floor(target_fpr * len(ordered)) # good images we may flag + if allowed == 0: + return math.nextafter(ordered[-1], math.inf) + candidate = ordered[len(ordered) - allowed] + # Ties at the cut would flag more than allowed; step above them. + if ordered.count(candidate) > allowed: + return math.nextafter(candidate, math.inf) + return candidate + + +@dataclass(frozen=True) +class ImageMetrics: + n_images: int + n_defective: int + auroc: float + threshold: float + precision: float + recall: float + f1: float + false_positive_rate: float + + def as_dict(self) -> dict: + return asdict(self) + + +def evaluate_images(results: Sequence[InspectionResult], threshold: float) -> ImageMetrics: + """Score a run of test-set results against MVTec ground truth at ``threshold``.""" + scores = [r.image_score for r in results] + labels = [is_defective_from_id(r.image_id) for r in results] + predicted = [s >= threshold for s in scores] + + tp = sum(p and y for p, y in zip(predicted, labels, strict=True)) + fp = sum(p and not y for p, y in zip(predicted, labels, strict=True)) + fn = sum(not p and y for p, y in zip(predicted, labels, strict=True)) + n_good = len(labels) - sum(labels) + + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + + return ImageMetrics( + n_images=len(results), + n_defective=sum(labels), + auroc=roc_auc(scores, labels), + threshold=threshold, + precision=precision, + recall=recall, + f1=f1, + false_positive_rate=fp / n_good if n_good else 0.0, + ) diff --git a/src/vista/store.py b/src/vista/store.py new file mode 100644 index 0000000..bed2d28 --- /dev/null +++ b/src/vista/store.py @@ -0,0 +1,41 @@ +"""Results storage: one JSON Lines file per run, one InspectionResult per line. + +Append-only and diffable, so offline evaluation runs are easy to reproduce +and compare. Everything goes through ``InspectionResult``, so a database +backend can replace this module later without touching callers. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from pathlib import Path + +from vista.schema import InspectionResult + + +def save_results(path: Path, results: Iterable[InspectionResult]) -> int: + """Append results to ``path``, creating parent folders. Returns the number written.""" + path.parent.mkdir(parents=True, exist_ok=True) + n = 0 + with path.open("a", encoding="utf-8") as f: + for result in results: + f.write(result.model_dump_json()) + f.write("\n") + n += 1 + return n + + +def iter_results(path: Path) -> Iterator[InspectionResult]: + """Yield results from ``path``, validating each line. Blank lines are skipped.""" + with path.open(encoding="utf-8") as f: + for lineno, line in enumerate(f, start=1): + if not line.strip(): + continue + try: + yield InspectionResult.model_validate_json(line) + except ValueError as exc: + raise ValueError(f"{path}:{lineno}: invalid result: {exc}") from exc + + +def load_results(path: Path) -> list[InspectionResult]: + return list(iter_results(path)) diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..18028d8 --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,91 @@ +import math + +import pytest + +from vista.evaluation import evaluate_images, is_defective_from_id, roc_auc, threshold_for_fpr +from vista.schema import InspectionResult + + +def r(image_id: str, score: float) -> InspectionResult: + return InspectionResult( + image_id=image_id, + image_width=10, + image_height=10, + model="m", + image_score=score, + threshold=0.5, + ) + + +@pytest.mark.parametrize( + ("image_id", "defective"), + [ + ("bottle/test/good/000.png", False), + ("bottle/test/broken_large/003.png", True), + ("data/raw/mvtec_ad/carpet/test/hole/010.png", True), + ], +) +def test_ground_truth_from_id(image_id, defective): + assert is_defective_from_id(image_id) is defective + + +def test_ground_truth_rejects_train_ids(): + with pytest.raises(ValueError): + is_defective_from_id("bottle/train/good/000.png") + + +def test_auroc_perfect_and_inverted(): + labels = [False, False, True, True] + assert roc_auc([0.1, 0.2, 0.8, 0.9], labels) == 1.0 + assert roc_auc([0.9, 0.8, 0.2, 0.1], labels) == 0.0 + + +def test_auroc_ties_count_half(): + assert roc_auc([0.5, 0.5], [False, True]) == 0.5 + + +def test_auroc_matches_pairwise_definition(): + scores = [0.3, 0.7, 0.2, 0.9, 0.7, 0.1, 0.6] + labels = [False, True, False, True, False, False, True] + pos = [s for s, y in zip(scores, labels, strict=True) if y] + neg = [s for s, y in zip(scores, labels, strict=True) if not y] + pairwise = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg) / (len(pos) * len(neg)) + assert math.isclose(roc_auc(scores, labels), pairwise) + + +def test_auroc_needs_both_classes(): + with pytest.raises(ValueError): + roc_auc([0.1, 0.2], [False, False]) + + +def test_threshold_respects_target_fpr(): + good = [i / 100 for i in range(100)] # 0.00 .. 0.99 + t = threshold_for_fpr(good, 0.05) + assert sum(s >= t for s in good) / len(good) <= 0.05 + assert sum(s >= t for s in good) == 5 + + +def test_threshold_zero_fpr_flags_no_good_images(): + good = [0.1, 0.4, 0.4] + t = threshold_for_fpr(good, 0.0) + assert not any(s >= t for s in good) + + +def test_threshold_steps_above_ties(): + good = [0.1, 0.2, 0.9, 0.9, 0.9] + t = threshold_for_fpr(good, 0.2) # may flag 1 image, but 0.9 appears 3 times + assert sum(s >= t for s in good) == 0 + + +def test_evaluate_images(): + results = [ + r("bottle/test/good/000.png", 0.1), + r("bottle/test/good/001.png", 0.6), + r("bottle/test/broken_large/000.png", 0.7), + r("bottle/test/contamination/000.png", 0.3), + ] + m = evaluate_images(results, threshold=0.5) + assert (m.n_images, m.n_defective) == (4, 2) + assert m.auroc == 0.75 + assert (m.precision, m.recall, m.false_positive_rate) == (0.5, 0.5, 0.5) + assert m.f1 == 0.5 diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..70cf0f2 --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,42 @@ +import pytest + +from vista.schema import BoundingBox, Defect, InspectionResult +from vista.store import iter_results, load_results, save_results + + +def result(image_id: str, score: float) -> InspectionResult: + return InspectionResult( + image_id=image_id, + image_width=900, + image_height=900, + model="patchcore-wrn50", + image_score=score, + threshold=0.5, + defects=[Defect(bbox=BoundingBox(x_min=1, y_min=1, x_max=9, y_max=9), score=score)], + ) + + +def test_round_trip_and_append(tmp_path): + path = tmp_path / "runs" / "run-001.jsonl" + first = [result("bottle/test/good/000.png", 0.1)] + second = [result("bottle/test/broken_large/000.png", 0.9)] + + assert save_results(path, first) == 1 + assert save_results(path, second) == 1 + assert load_results(path) == first + second + + +def test_blank_lines_skipped(tmp_path): + path = tmp_path / "run.jsonl" + save_results(path, [result("bottle/test/good/000.png", 0.2)]) + path.write_text(path.read_text() + "\n\n") + assert len(load_results(path)) == 1 + + +def test_invalid_line_reports_location(tmp_path): + path = tmp_path / "run.jsonl" + save_results(path, [result("bottle/test/good/000.png", 0.2)]) + with path.open("a") as f: + f.write('{"image_id": "x"}\n') + with pytest.raises(ValueError, match=r"run\.jsonl:2"): + list(iter_results(path))