diff --git a/README.md b/README.md index 339a833..8121819 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ Every stage exchanges data as an `InspectionResult` ([`src/vista/schema.py`](src - [x] Repo tooling: lint, tests, CI - [x] Inspection result schema -- [ ] Dataset preparation (MVTec AD) -- [ ] Baseline anomaly model +- [x] Dataset preparation (MVTec AD) +- [ ] Baseline anomaly model (PatchCore, see [ADR 0001](docs/decisions/0001-baseline-model.md)) - [ ] Evaluation harness (image AUROC, pixel AUROC, PRO, latency) - [ ] Inference API - [ ] Web demo diff --git a/data/datasets/mvtec_ad.yaml b/data/datasets/mvtec_ad.yaml new file mode 100644 index 0000000..c8c6d78 --- /dev/null +++ b/data/datasets/mvtec_ad.yaml @@ -0,0 +1,24 @@ +# MVTec AD: industrial anomaly detection benchmark. +# The data itself is never committed. Download the archive from the source +# page (accepting the licence there), then run scripts/prepare_mvtec.py. +name: mvtec_ad +source: https://www.mvtec.com/company/research/datasets/mvtec-ad +licence: CC BY-NC-SA 4.0 +commercial_use: false +citation: > + Bergmann, Batzner, Fauser, Sattlegger, Steger. "The MVTec Anomaly Detection + Dataset: A Comprehensive Real-World Dataset for Unsupervised Anomaly + Detection". International Journal of Computer Vision, 2021. + +layout: + train: train/good # defect-free images only + test: test/ # 'good' plus one folder per defect type + ground_truth: ground_truth/ # binary masks, *_mask.png + +categories: + objects: [bottle, cable, capsule, hazelnut, metal_nut, pill, screw, toothbrush, transistor, zipper] + textures: [carpet, grid, leather, tile, wood] + +# Categories we start with: one object, one texture, so the baseline is +# tested on both kinds of anomaly before we scale to all 15. +initial: [bottle, carpet] diff --git a/docs/decisions/0001-baseline-model.md b/docs/decisions/0001-baseline-model.md new file mode 100644 index 0000000..f187481 --- /dev/null +++ b/docs/decisions/0001-baseline-model.md @@ -0,0 +1,36 @@ +# 0001: Baseline anomaly model + +- Status: Accepted +- Date: 2026-11-16 +- Deciders: @Ray3R3, @samoteng + +## Context + +VISTA needs a first model to anchor the pipeline and the evaluation harness. Real inspection lines rarely have many labelled defect images; they have plenty of images of good parts. MVTec AD reflects this: training sets contain only defect-free images, and defects appear only at test time. + +So the baseline must: + +1. Train on normal images only (unsupervised / one-class). +2. Produce a pixel-level score map, not just an image label, so we can localise defects and evaluate segmentation (pixel AUROC, PRO). +3. Be reproducible on a laptop GPU or CPU in reasonable time. +4. Carry a licence compatible with an Apache-2.0 codebase. + +## Options considered + +| Option | Fits 1 | Fits 2 | Notes | +|---|---|---|---| +| **PatchCore** (Roth et al., CVPR 2022) | Yes | Yes | Memory bank of pretrained CNN patch features; no training loop, only feature extraction and coreset subsampling. Strong published results on MVTec AD. | +| PaDiM | Yes | Yes | Gaussian per patch position; simpler but generally weaker than PatchCore and assumes aligned images. | +| EfficientAD | Yes | Yes | Faster inference, but a student–teacher training loop adds tuning work we don't need for a baseline. | +| Supervised detector/segmenter (e.g. YOLO-seg) | No | Yes | Needs labelled defects for training, which contradicts the setting. Ultralytics models are AGPL-3.0. | + +## Decision + +Use **PatchCore via [anomalib](https://github.com/open-edge-platform/anomalib)** (Apache-2.0) with a WideResNet-50 backbone as the baseline, starting on the `bottle` and `carpet` categories (one object, one texture). + +## Consequences + +- No training loop to debug first: the evaluation harness can be built against a model that is known to work on this benchmark. +- Inference cost and memory grow with the memory bank. The coreset sampling ratio is the main knob and will be recorded in every experiment config. +- Scores are distances in feature space, not probabilities. `InspectionResult.image_score` requires `[0, 1]`, so we need a normalisation step fitted on held-out good images; the threshold is chosen there, not on the test set. +- Later models (EfficientAD, a supervised head once we have labels) are compared against this baseline using the same harness and data splits. diff --git a/pyproject.toml b/pyproject.toml index 94e0c83..70ea975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ license = "Apache-2.0" authors = [{ name = "V2GWorks" }] dependencies = [ "pydantic>=2.6", + "pyyaml>=6", ] [project.optional-dependencies] diff --git a/scripts/prepare_mvtec.py b/scripts/prepare_mvtec.py new file mode 100644 index 0000000..76b8298 --- /dev/null +++ b/scripts/prepare_mvtec.py @@ -0,0 +1,113 @@ +"""Extract and verify MVTec AD categories from a downloaded archive. + +MVTec AD has to be downloaded from MVTec's site after accepting its licence +(CC BY-NC-SA 4.0), so this script works from a local archive rather than +fetching anything itself. + + python scripts/prepare_mvtec.py --archive ~/Downloads/mvtec_anomaly_detection.tar.xz \ + --category bottle --category carpet + +Accepts the full archive or a single-category archive (e.g. bottle.tar.xz). +""" + +from __future__ import annotations + +import argparse +import json +import sys +import tarfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +MANIFEST = REPO_ROOT / "data" / "datasets" / "mvtec_ad.yaml" +DEFAULT_OUT = REPO_ROOT / "data" / "raw" / "mvtec_ad" +IMAGE_SUFFIX = ".png" + + +def known_categories(manifest: Path = MANIFEST) -> list[str]: + spec = yaml.safe_load(manifest.read_text()) + cats = spec["categories"] + return sorted(cats["objects"] + cats["textures"]) + + +def _safe_members(tar: tarfile.TarFile, category: str) -> list[tarfile.TarInfo]: + """Members belonging to `category`, rejecting any path that escapes the target.""" + members = [] + for m in tar.getmembers(): + parts = Path(m.name).parts + if category not in parts: + continue + if m.name.startswith("/") or ".." in parts or not (m.isfile() or m.isdir()): + raise ValueError(f"refusing unsafe archive member: {m.name}") + members.append(m) + return members + + +def extract_category(archive: Path, category: str, out_dir: Path) -> Path: + with tarfile.open(archive) as tar: + members = _safe_members(tar, category) + if not members: + raise ValueError(f"category '{category}' not found in {archive.name}") + tar.extractall(out_dir, members=members, filter="data") + # The full archive has no top-level folder, so the category may sit at + # out_dir/; locate it rather than assume. + matches = [p for p in out_dir.rglob(category) if p.is_dir() and (p / "train").exists()] + if not matches: + raise ValueError(f"extracted '{category}' but found no train/ folder") + return matches[0] + + +def verify_category(root: Path) -> dict: + """Check the expected layout and return image counts per split.""" + train_good = root / "train" / "good" + test_dir = root / "test" + gt_dir = root / "ground_truth" + for required in (train_good, test_dir / "good"): + if not required.is_dir(): + raise ValueError(f"missing {required.relative_to(root)}") + + def count(d: Path) -> int: + return sum(1 for p in d.iterdir() if p.suffix == IMAGE_SUFFIX) + + defect_types = sorted(d.name for d in test_dir.iterdir() if d.is_dir() and d.name != "good") + missing_masks = [] + for defect in defect_types: + for img in (test_dir / defect).glob(f"*{IMAGE_SUFFIX}"): + if not (gt_dir / defect / f"{img.stem}_mask.png").exists(): + missing_masks.append(f"{defect}/{img.name}") + if missing_masks: + raise ValueError(f"{len(missing_masks)} test images lack masks, e.g. {missing_masks[0]}") + + return { + "category": root.name, + "train_good": count(train_good), + "test_good": count(test_dir / "good"), + "test_defective": {d: count(test_dir / d) for d in defect_types}, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--archive", type=Path, required=True) + parser.add_argument("--category", action="append", required=True, dest="categories") + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args(argv) + + valid = known_categories() + unknown = [c for c in args.categories if c not in valid] + if unknown: + parser.error(f"unknown categories {unknown}; choose from {valid}") + + args.out.mkdir(parents=True, exist_ok=True) + summaries = [] + for category in args.categories: + root = extract_category(args.archive, category, args.out) + summaries.append(verify_category(root)) + print(json.dumps(summaries, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_prepare_mvtec.py b/tests/test_prepare_mvtec.py new file mode 100644 index 0000000..8c30828 --- /dev/null +++ b/tests/test_prepare_mvtec.py @@ -0,0 +1,70 @@ +import importlib.util +import io +import tarfile +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "prepare_mvtec.py" +spec = importlib.util.spec_from_file_location("prepare_mvtec", SCRIPT) +prepare = importlib.util.module_from_spec(spec) +spec.loader.exec_module(prepare) + + +def _add(tar: tarfile.TarFile, name: str, data: bytes = b"png") -> None: + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + + +def make_archive(path: Path, *, drop_mask: bool = False) -> Path: + """A miniature MVTec-style archive for one category.""" + with tarfile.open(path, "w:xz") as tar: + _add(tar, "bottle/train/good/000.png") + _add(tar, "bottle/train/good/001.png") + _add(tar, "bottle/test/good/000.png") + _add(tar, "bottle/test/broken_large/000.png") + if not drop_mask: + _add(tar, "bottle/ground_truth/broken_large/000_mask.png") + _add(tar, "carpet/train/good/000.png") + return path + + +def test_manifest_lists_all_fifteen_categories(): + cats = prepare.known_categories() + assert len(cats) == 15 + assert {"bottle", "carpet", "zipper"} <= set(cats) + + +def test_extracts_and_counts_only_requested_category(tmp_path): + archive = make_archive(tmp_path / "mvtec.tar.xz") + root = prepare.extract_category(archive, "bottle", tmp_path / "out") + summary = prepare.verify_category(root) + assert summary == { + "category": "bottle", + "train_good": 2, + "test_good": 1, + "test_defective": {"broken_large": 1}, + } + assert not (tmp_path / "out" / "carpet").exists() + + +def test_missing_mask_is_reported(tmp_path): + archive = make_archive(tmp_path / "mvtec.tar.xz", drop_mask=True) + root = prepare.extract_category(archive, "bottle", tmp_path / "out") + with pytest.raises(ValueError, match="lack masks"): + prepare.verify_category(root) + + +def test_unknown_category_in_archive(tmp_path): + archive = make_archive(tmp_path / "mvtec.tar.xz") + with pytest.raises(ValueError, match="not found"): + prepare.extract_category(archive, "screw", tmp_path / "out") + + +def test_path_traversal_rejected(tmp_path): + archive = tmp_path / "evil.tar.xz" + with tarfile.open(archive, "w:xz") as tar: + _add(tar, "bottle/../../escape.png") + with pytest.raises(ValueError, match="unsafe"): + prepare.extract_category(archive, "bottle", tmp_path / "out")