Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ Every stage exchanges data as an `InspectionResult` ([`src/vista/schema.py`](src
- [x] Inspection result schema
- [x] Dataset preparation (MVTec AD)
- [ ] Baseline anomaly model (PatchCore, see [ADR 0001](docs/decisions/0001-baseline-model.md))
- [ ] Evaluation harness (image AUROC, pixel AUROC, PRO, latency)
- [x] Results store (JSON Lines) and image-level metrics (AUROC, precision/recall at a target false-positive rate)
- [ ] Pixel-level evaluation (pixel AUROC, PRO) and latency
- [ ] Inference API
- [ ] Web demo

Expand Down
120 changes: 120 additions & 0 deletions src/vista/evaluation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""Image-level evaluation for anomaly detection.

Two things are kept apart on purpose:

* **Ranking quality** (AUROC) does not depend on a threshold, so it measures
the model alone.
* **Operating point** (precision, recall, F1) depends on a threshold, which
must be chosen on held-out *good* images, never on the test set. We pick it
to hit a target false-positive rate on those images.
"""

from __future__ import annotations

import math
from collections.abc import Sequence
from dataclasses import asdict, dataclass

from vista.schema import InspectionResult


def is_defective_from_id(image_id: str) -> bool:
"""Ground truth from an MVTec-style id, e.g. ``bottle/test/broken_large/000.png``."""
parts = image_id.replace("\\", "/").split("/")
if "test" not in parts:
raise ValueError(f"not a test image id: {image_id}")
defect_type = parts[parts.index("test") + 1]
return defect_type != "good"


def roc_auc(scores: Sequence[float], labels: Sequence[bool]) -> float:
"""Area under the ROC curve via the Mann-Whitney U statistic.

Equals the probability that a random defective image scores higher than a
random good one, with ties counted as half. O(n log n), no dependencies.
"""
if len(scores) != len(labels):
raise ValueError("scores and labels must be the same length")
n_pos = sum(labels)
n_neg = len(labels) - n_pos
if n_pos == 0 or n_neg == 0:
raise ValueError("AUROC needs at least one defective and one good image")

# Average ranks (1-based) with ties sharing the mean rank.
order = sorted(range(len(scores)), key=lambda i: scores[i])
ranks = [0.0] * len(scores)
i = 0
while i < len(order):
j = i
while j + 1 < len(order) and scores[order[j + 1]] == scores[order[i]]:
j += 1
mean_rank = (i + j) / 2 + 1
for k in range(i, j + 1):
ranks[order[k]] = mean_rank
i = j + 1

rank_sum_pos = sum(r for r, y in zip(ranks, labels, strict=True) if y)
u = rank_sum_pos - n_pos * (n_pos + 1) / 2
return u / (n_pos * n_neg)


def threshold_for_fpr(good_scores: Sequence[float], target_fpr: float) -> float:
"""Smallest threshold whose false-positive rate on ``good_scores`` is <= ``target_fpr``.

An image is flagged when ``score >= threshold``.
"""
if not good_scores:
raise ValueError("need at least one held-out good score")
if not 0.0 <= target_fpr < 1.0:
raise ValueError("target_fpr must be in [0, 1)")
ordered = sorted(good_scores)
allowed = math.floor(target_fpr * len(ordered)) # good images we may flag
if allowed == 0:
return math.nextafter(ordered[-1], math.inf)
candidate = ordered[len(ordered) - allowed]
# Ties at the cut would flag more than allowed; step above them.
if ordered.count(candidate) > allowed:
return math.nextafter(candidate, math.inf)
return candidate


@dataclass(frozen=True)
class ImageMetrics:
n_images: int
n_defective: int
auroc: float
threshold: float
precision: float
recall: float
f1: float
false_positive_rate: float

def as_dict(self) -> dict:
return asdict(self)


def evaluate_images(results: Sequence[InspectionResult], threshold: float) -> ImageMetrics:
"""Score a run of test-set results against MVTec ground truth at ``threshold``."""
scores = [r.image_score for r in results]
labels = [is_defective_from_id(r.image_id) for r in results]
predicted = [s >= threshold for s in scores]

tp = sum(p and y for p, y in zip(predicted, labels, strict=True))
fp = sum(p and not y for p, y in zip(predicted, labels, strict=True))
fn = sum(not p and y for p, y in zip(predicted, labels, strict=True))
n_good = len(labels) - sum(labels)

precision = tp / (tp + fp) if tp + fp else 0.0
recall = tp / (tp + fn) if tp + fn else 0.0
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0

return ImageMetrics(
n_images=len(results),
n_defective=sum(labels),
auroc=roc_auc(scores, labels),
threshold=threshold,
precision=precision,
recall=recall,
f1=f1,
false_positive_rate=fp / n_good if n_good else 0.0,
)
41 changes: 41 additions & 0 deletions src/vista/store.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""Results storage: one JSON Lines file per run, one InspectionResult per line.

Append-only and diffable, so offline evaluation runs are easy to reproduce
and compare. Everything goes through ``InspectionResult``, so a database
backend can replace this module later without touching callers.
"""

from __future__ import annotations

from collections.abc import Iterable, Iterator
from pathlib import Path

from vista.schema import InspectionResult


def save_results(path: Path, results: Iterable[InspectionResult]) -> int:
"""Append results to ``path``, creating parent folders. Returns the number written."""
path.parent.mkdir(parents=True, exist_ok=True)
n = 0
with path.open("a", encoding="utf-8") as f:
for result in results:
f.write(result.model_dump_json())
f.write("\n")
n += 1
return n


def iter_results(path: Path) -> Iterator[InspectionResult]:
"""Yield results from ``path``, validating each line. Blank lines are skipped."""
with path.open(encoding="utf-8") as f:
for lineno, line in enumerate(f, start=1):
if not line.strip():
continue
try:
yield InspectionResult.model_validate_json(line)
except ValueError as exc:
raise ValueError(f"{path}:{lineno}: invalid result: {exc}") from exc


def load_results(path: Path) -> list[InspectionResult]:
return list(iter_results(path))
91 changes: 91 additions & 0 deletions tests/test_evaluation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import math

import pytest

from vista.evaluation import evaluate_images, is_defective_from_id, roc_auc, threshold_for_fpr
from vista.schema import InspectionResult


def r(image_id: str, score: float) -> InspectionResult:
return InspectionResult(
image_id=image_id,
image_width=10,
image_height=10,
model="m",
image_score=score,
threshold=0.5,
)


@pytest.mark.parametrize(
("image_id", "defective"),
[
("bottle/test/good/000.png", False),
("bottle/test/broken_large/003.png", True),
("data/raw/mvtec_ad/carpet/test/hole/010.png", True),
],
)
def test_ground_truth_from_id(image_id, defective):
assert is_defective_from_id(image_id) is defective


def test_ground_truth_rejects_train_ids():
with pytest.raises(ValueError):
is_defective_from_id("bottle/train/good/000.png")


def test_auroc_perfect_and_inverted():
labels = [False, False, True, True]
assert roc_auc([0.1, 0.2, 0.8, 0.9], labels) == 1.0
assert roc_auc([0.9, 0.8, 0.2, 0.1], labels) == 0.0


def test_auroc_ties_count_half():
assert roc_auc([0.5, 0.5], [False, True]) == 0.5


def test_auroc_matches_pairwise_definition():
scores = [0.3, 0.7, 0.2, 0.9, 0.7, 0.1, 0.6]
labels = [False, True, False, True, False, False, True]
pos = [s for s, y in zip(scores, labels, strict=True) if y]
neg = [s for s, y in zip(scores, labels, strict=True) if not y]
pairwise = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg) / (len(pos) * len(neg))
assert math.isclose(roc_auc(scores, labels), pairwise)


def test_auroc_needs_both_classes():
with pytest.raises(ValueError):
roc_auc([0.1, 0.2], [False, False])


def test_threshold_respects_target_fpr():
good = [i / 100 for i in range(100)] # 0.00 .. 0.99
t = threshold_for_fpr(good, 0.05)
assert sum(s >= t for s in good) / len(good) <= 0.05
assert sum(s >= t for s in good) == 5


def test_threshold_zero_fpr_flags_no_good_images():
good = [0.1, 0.4, 0.4]
t = threshold_for_fpr(good, 0.0)
assert not any(s >= t for s in good)


def test_threshold_steps_above_ties():
good = [0.1, 0.2, 0.9, 0.9, 0.9]
t = threshold_for_fpr(good, 0.2) # may flag 1 image, but 0.9 appears 3 times
assert sum(s >= t for s in good) == 0


def test_evaluate_images():
results = [
r("bottle/test/good/000.png", 0.1),
r("bottle/test/good/001.png", 0.6),
r("bottle/test/broken_large/000.png", 0.7),
r("bottle/test/contamination/000.png", 0.3),
]
m = evaluate_images(results, threshold=0.5)
assert (m.n_images, m.n_defective) == (4, 2)
assert m.auroc == 0.75
assert (m.precision, m.recall, m.false_positive_rate) == (0.5, 0.5, 0.5)
assert m.f1 == 0.5
42 changes: 42 additions & 0 deletions tests/test_store.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import pytest

from vista.schema import BoundingBox, Defect, InspectionResult
from vista.store import iter_results, load_results, save_results


def result(image_id: str, score: float) -> InspectionResult:
return InspectionResult(
image_id=image_id,
image_width=900,
image_height=900,
model="patchcore-wrn50",
image_score=score,
threshold=0.5,
defects=[Defect(bbox=BoundingBox(x_min=1, y_min=1, x_max=9, y_max=9), score=score)],
)


def test_round_trip_and_append(tmp_path):
path = tmp_path / "runs" / "run-001.jsonl"
first = [result("bottle/test/good/000.png", 0.1)]
second = [result("bottle/test/broken_large/000.png", 0.9)]

assert save_results(path, first) == 1
assert save_results(path, second) == 1
assert load_results(path) == first + second


def test_blank_lines_skipped(tmp_path):
path = tmp_path / "run.jsonl"
save_results(path, [result("bottle/test/good/000.png", 0.2)])
path.write_text(path.read_text() + "\n\n")
assert len(load_results(path)) == 1


def test_invalid_line_reports_location(tmp_path):
path = tmp_path / "run.jsonl"
save_results(path, [result("bottle/test/good/000.png", 0.2)])
with path.open("a") as f:
f.write('{"image_id": "x"}\n')
with pytest.raises(ValueError, match=r"run\.jsonl:2"):
list(iter_results(path))
Loading