diff --git a/README.md b/README.md index 2a66fbda..f898b08e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Open-source LLM benchmark suite for medical tasks. [medmarks.ai](https://medmarks.ai) | [arXiv:2605.01417](https://arxiv.org/abs/2605.01417) -Medmarks is a comprehensive benchmark suite for evaluating medical capabilities in large language models. It includes 30 open-source benchmarks spanning question answering, information extraction, consumer health questions, clinical reasoning, EHR interactions, medical calculations, and open-ended medical tasks. +Medmarks is a comprehensive benchmark suite for evaluating medical capabilities in large language models. It includes 31 open-source benchmarks spanning question answering, information extraction, consumer health questions, clinical reasoning, EHR interactions, medical calculations, fact verification, and open-ended medical tasks. This repository contains the runnable benchmark environments, evaluation configs, result processing tools, and win-rate analysis pipeline used for Medmarks. It also contains the [`medarc_verifiers` Python library](docs/README.md), which provides the shared CLI, parsers, rewards, judging utilities, and orchestration helpers used by the benchmark environments. @@ -122,6 +122,7 @@ Evaluation outputs are written under `runs/evals/`, processed parquet files unde | M-ARC | Long-tail medical questions designed to test model resistance to inflexible clinical reasoning patterns. | Apache-2.0 | 100 | -- | | Med-HALT | Clinical Reasoning Hallucination detection via false confidence tests and "none of the above" recognition. | Apache-2.0 | 22,152 | -- | | MedCalc-Bench | Clinical calculator questions evaluating medical computation and formula application skills. | CC-BY-SA-4.0 | 1,100 | 10,538 | +| MedFact-Bench | Medical fact verification benchmark combining SciFact, HealthVer, MedAESQA, PubMedQA-Fact, and BioASQ-Fact under the Med-V1 zero-shot protocol. | Not specified | 14,274 | -- | | MedConceptsQA | Multiple-choice questions on medical coding systems, e.g., ICD-9, ICD-10, etc., only ICD-10CM subsamples evaluated. | Not specified | 6,000 | -- | | Medbullets | USMLE Step 2 and Step 3 style clinical reasoning questions sourced from social media. | Not specified | 308 | -- | | MedHallu | Medical hallucination detection benchmark with four domain-specific error categories derived from the PubMedQA dataset. | MIT | 2,000 | -- | diff --git a/configs/medmarks-smoke.toml b/configs/medmarks-smoke.toml index 24e0d047..ed95c74f 100644 --- a/configs/medmarks-smoke.toml +++ b/configs/medmarks-smoke.toml @@ -49,6 +49,11 @@ num_examples = 10 rollouts_per_example = 1 env_args = { version = "1.2" } +[[eval]] +env_id = "medfact_bench" +num_examples = 10 +rollouts_per_example = 1 + [[eval]] env_id = "medconceptsqa" num_examples = 10 diff --git a/configs/medmarks-verified.toml b/configs/medmarks-verified.toml index 709782a6..70fb3887 100644 --- a/configs/medmarks-verified.toml +++ b/configs/medmarks-verified.toml @@ -129,6 +129,11 @@ num_examples = -1 rollouts_per_example = 1 env_args = { version = "verified", add_python_tool = true, add_calculator_tool = true } +[[eval]] +env_id = "medfact_bench" +num_examples = -1 +rollouts_per_example = 1 + [[ablation]] env_id = "medconceptsqa" name = "{env_args.difficulty}" diff --git a/environments/medfact_bench/README.md b/environments/medfact_bench/README.md new file mode 100644 index 00000000..7cd72d0a --- /dev/null +++ b/environments/medfact_bench/README.md @@ -0,0 +1,163 @@ +# MedFact-Bench + +## Overview + +- **Environment ID**: `medfact_bench` +- **Short description**: Zero-shot medical claim verification across five benchmark components. +- **Tags**: medical, fact-verification, classification, single-turn, evaluation + +`medfact_bench` implements the evaluation protocol introduced in [Med-V1: Small Language Models for Zero-shot and Scalable Biomedical Evidence Attribution](https://arxiv.org/abs/2603.05308). The environment is evaluation-only and does not expose a training dataset or training reward. + +## Datasets + +- **Primary dataset**: [`ncbi/MedFact-Bench`](https://huggingface.co/datasets/ncbi/MedFact-Bench) +- **Dataset revision**: `249028caf7ad5a3e63331269a606f4b2696693ed` +- **Paper**: [Med-V1](https://arxiv.org/abs/2603.05308) + +| Component | Examples | +| --- | ---: | +| SciFact | 340 | +| HealthVer | 903 | +| MedAESQA | 9,106 | +| PubMedQA-Fact | 500 | +| BioASQ-Fact | 3,425 | +| **Total** | **14,274** | + +The loader also accepts a local Parquet copy. Local loading preserves the original row order and duplicate rows while validating the required schema, component names, labels, non-null fields, and global system prompt. + +## Task + +- **Type**: single-turn, three-class classification +- **Labels**: `SUPPORT`, `NEI`, and `CONTRADICT` +- **Rubric overview**: exact label accuracy with separate parsing and format diagnostics + +Each example contains a medical claim and a source article. The dataset-provided Med-V1 system and user prompts are used unchanged. The model assigns a five-point score that maps to the benchmark labels: + +| Score | Prediction | +| ---: | --- | +| `1`, `2` | `SUPPORT` | +| `0` | `NEI` | +| `-1`, `-2` | `CONTRADICT` | + +The parser reads the first `...` tag, trims whitespace, and accepts only integer values from `-2` through `2`. A valid `` block is not required to classify a completion. Missing, malformed, or out-of-range scores are labeled `INVALID`, receive zero accuracy, and remain in all reporting denominators. + +### Prompt Format + +The dataset provides one shared system prompt, reproduced below. The environment passes it to the model unchanged: + +```text +You are a fact-checking expert trained in evidence-based medicine. Your task is to evaluate how strongly an *article* agrees or disagrees with a *claim*. The *article* is retrieved from a search engine using the *claim* as the query. + +Use the following five-point scale: + - **-2 Strong Contradiction** – The article clearly and directly refutes the claim. + - **-1 Partial Contradiction** – The article provides mixed or indirect evidence against the claim. + - ** 0 Neutral / Unrelated** – The article does not address the claim, offers insufficient information, or is irrelevant to the claim. + - ** 1 Partial Agreement** – The article offers some indirect or tentative support for the claim. + - ** 2 Strong Agreement** – The article explicitly and strongly supports the claim. + +Note that the *article* might not describe the exact same subjects, interventions, or measurements as the *claim*. In this case, please note the difference and assign a score of 0. + +Output in two parts only and do not output anything else: +[your detailed, step‐by‐step explanation for scoring] +[the integer score only, i.e., -2, -1, 0, 1, or 2] +``` + +The square-bracketed phrases describe placeholder content; the brackets themselves are not required in model completions. + +Each dataset row provides a user prompt with this structure: + +```text +Article: +{article} + +Claim: +{claim} +``` + +## Quickstart + +Run a small evaluation with Prime: + +```bash +prime eval run medfact_bench -m "openai/gpt-5-mini" -n 5 -r 1 +``` + +Run the complete benchmark with MedMarks: + +```bash +medarc-eval medfact_bench \ + -m "openai/gpt-5-mini" \ + -n -1 \ + -r 1 \ + --subset all +``` + +Use `--dataset-path /absolute/path/to/MedFact-Bench.parquet` to evaluate from a local dataset copy. + +## Environment Arguments + +| Argument | Type | Default | Description | +| --- | --- | --- | --- | +| `subset` | `str` or `MedFactSubset` | `all` | Component to evaluate: `all`, `scifact`, `healthver`, `medaesqa`, `pubmedqa-fact`, or `bioasq-fact`. | +| `dataset_path` | `str` or `None` | `None` | Optional local Parquet path. Local data takes precedence over Hugging Face. | +| `cache_dir` | `str` or `None` | `None` | Optional Hugging Face datasets cache directory. | + +## Metrics + +| Metric | Weight | Meaning | +| --- | ---: | --- | +| `accuracy` | 1.0 | Exact three-class accuracy. This is also the Verifiers reward and `avg_reward`. | +| `parseable_score` | 0.0 | Whether a valid five-point score was parsed. | +| `strict_format` | 0.0 | Whether the completion exactly follows the requested think-and-score block format. | + +`avg_reward` is example-weighted micro-accuracy. The paper's primary benchmark metric is macro-accuracy across the five components and is produced by the reporting command below. + +## Reporting + +Generate benchmark-specific metrics from the raw `results.jsonl` before running standard MedMarks result processing: + +```bash +python -m medfact_bench.reporting RESULTS_JSONL \ + --output MEDFACT_REPORT_JSON \ + --metadata METADATA_JSON +``` + +Without explicit output paths, the command writes `medfact_report.json` beside `results.jsonl` and updates the sibling `metadata.json` non-destructively. Use `--no-update-metadata` to leave metadata unchanged. + +The report includes valid and invalid prediction counts, parse and strict-format rates, micro-accuracy, macro-accuracy over observed components, per-component accuracy, per-class precision/recall/F1, macro-F1, a confusion matrix with `INVALID`, and benchmark coverage. `paper_macro_accuracy` is populated only when all 14,274 examples have exact component coverage; partial runs set it to `null` and set `paper_comparable` to `false`. + +## Published Results + +Figure 3 of the Med-V1 paper reports the following zero-shot accuracies. The small models are the locally runnable 3B base checkpoints. The frontier models were evaluated through Azure-hosted APIs. `Med-V1-L3B` and `Med-V1-Q3B` are fine-tuned Med-V1 models trained on MedFact-Synth, then evaluated zero-shot on MedFact-Bench. + +### Small Models + +| Model | SciFact | HealthVer | MedAESQA | PubMedQA-Fact | BioASQ-Fact | Macro | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Llama-3.2-3B-Instruct | .600 | .435 | .437 | .550 | .531 | .511 | +| Qwen2.5-3B-Instruct | .638 | .447 | .577 | .470 | .444 | .515 | + +### Frontier Models + +| Model | SciFact | HealthVer | MedAESQA | PubMedQA-Fact | BioASQ-Fact | Macro | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Llama-3.3-70B-Instruct | .812 | .597 | .687 | .774 | .771 | .728 | +| o3-mini | .832 | .591 | .733 | .778 | .747 | .736 | +| GPT-4o-mini | .847 | .580 | .704 | .734 | .720 | .717 | +| GPT-4o | .832 | .576 | .717 | .776 | .777 | .736 | +| GPT-5 | .818 | .615 | .703 | .788 | .753 | .735 | + +### Fine-Tuned Models + +| Model | SciFact | HealthVer | MedAESQA | PubMedQA-Fact | BioASQ-Fact | Macro | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Med-V1-L3B | .844 | .575 | .748 | .746 | .725 | .728 | +| Med-V1-Q3B | .856 | .588 | .733 | .764 | .717 | .732 | + +Macro is the unweighted mean of the five component accuracies and is the paper's primary metric. These values are reference results from the publication. Evaluation artifacts should preserve the dataset revision, model revision, prompts, sampling configuration, and raw predictions needed to interpret differences between runs. + +## Authors + +This environment has been put together by: + +Kouate Muhamed - ([@KouateMuhamed](https://github.com/KouateMuhamed)) diff --git a/environments/medfact_bench/medfact_bench/__init__.py b/environments/medfact_bench/medfact_bench/__init__.py new file mode 100644 index 00000000..9df5aa76 --- /dev/null +++ b/environments/medfact_bench/medfact_bench/__init__.py @@ -0,0 +1,33 @@ +"""MedFact-Bench zero-shot evaluation environment.""" + +from .environment import ( + DATASET_ID, + DATASET_REVISION, + EXPECTED_DATASET_COUNTS, + INVALID_LABEL, + LABELS, + MedFactScoreParser, + MedFactSubset, + accuracy, + load_environment, + parseable_score, + prediction_label, + score_to_label, + strict_format, +) + +__all__ = [ + "DATASET_ID", + "DATASET_REVISION", + "EXPECTED_DATASET_COUNTS", + "INVALID_LABEL", + "LABELS", + "MedFactScoreParser", + "MedFactSubset", + "accuracy", + "load_environment", + "parseable_score", + "prediction_label", + "score_to_label", + "strict_format", +] diff --git a/environments/medfact_bench/medfact_bench/environment.py b/environments/medfact_bench/medfact_bench/environment.py new file mode 100644 index 00000000..ab241b78 --- /dev/null +++ b/environments/medfact_bench/medfact_bench/environment.py @@ -0,0 +1,281 @@ +"""MedFact-Bench zero-shot classification environment.""" + +from __future__ import annotations + +import re +from enum import Enum +from pathlib import Path +from typing import Any + +import verifiers as vf +from datasets import Dataset, load_dataset + +DATASET_ID = "ncbi/MedFact-Bench" +DATASET_REVISION = "249028caf7ad5a3e63331269a606f4b2696693ed" + +LABELS = ("SUPPORT", "NEI", "CONTRADICT") +INVALID_LABEL = "INVALID" +VALID_SCORES = frozenset({-2, -1, 0, 1, 2}) + +EXPECTED_DATASET_COUNTS = { + "scifact": 340, + "healthver": 903, + "medaesqa": 9_106, + "pubmedqa-fact": 500, + "bioasq-fact": 3_425, +} + +REQUIRED_COLUMNS = ( + "dataset", + "claim", + "source", + "label", + "system_prompt", + "user_prompt", +) + +_SCORE_PATTERN = re.compile(r"(.*?)", re.DOTALL) +_STRICT_FORMAT_PATTERN = re.compile( + r"\A\s*.*?\s*(.*?)\s*\Z", + re.DOTALL, +) + + +class MedFactSubset(str, Enum): + """Supported MedFact-Bench component datasets.""" + + ALL = "all" + SCIFACT = "scifact" + HEALTHVER = "healthver" + MEDAESQA = "medaesqa" + PUBMEDQA_FACT = "pubmedqa-fact" + BIOASQ_FACT = "bioasq-fact" + + +def _parse_score_value(value: str) -> int | None: + try: + score = int(value.strip()) + except (TypeError, ValueError): + return None + return score if score in VALID_SCORES else None + + +class MedFactScoreParser(vf.Parser): + """Parse the first paper-format integer score from a completion.""" + + def __init__(self) -> None: + super().__init__(extract_fn=self.parse_score) + + @staticmethod + def parse_score(text: str) -> int | None: + """Return the first valid score enclosed in a score tag.""" + match = _SCORE_PATTERN.search(text) + if match is None: + return None + return _parse_score_value(match.group(1)) + + def completion_text(self, completion: Any) -> str | None: + """Return the final assistant text using Verifiers message semantics.""" + if isinstance(completion, str): + return completion + if not isinstance(completion, list): + return None + assistant_messages = self.get_assistant_messages(completion) + if not assistant_messages: + return None + content = self._message_field(assistant_messages[-1], "content", "") or "" + return self._content_to_text(content) + + def parse_completion(self, completion: Any) -> int | None: + """Parse a string or chat-message completion.""" + text = self.completion_text(completion) + return self.parse(text) if text is not None else None + + def is_strict_format(self, completion: Any) -> bool: + """Check exact two-block Med-V1 formatting independently of parsing.""" + text = self.completion_text(completion) + if text is None: + return False + if any(text.count(tag) != 1 for tag in ("", "", "", "")): + return False + match = _STRICT_FORMAT_PATTERN.fullmatch(text) + return match is not None and _parse_score_value(match.group(1)) is not None + + +def score_to_label(score: int | None) -> str: + """Map the five-point Med-V1 score to the three benchmark labels.""" + if score in (1, 2): + return "SUPPORT" + if score == 0: + return "NEI" + if score in (-1, -2): + return "CONTRADICT" + return INVALID_LABEL + + +def prediction_label(completion: Any, parser: MedFactScoreParser) -> str: + """Return the benchmark label predicted by a completion.""" + return score_to_label(parser.parse_completion(completion)) + + +def accuracy( + completion: Any, + answer: str, + parser: MedFactScoreParser, + **_: Any, +) -> float: + """Score exact three-way classification accuracy.""" + return float(prediction_label(completion, parser) == answer) + + +def parseable_score( + completion: Any, + parser: MedFactScoreParser, + **_: Any, +) -> float: + """Measure whether a valid score can be parsed.""" + return float(parser.parse_completion(completion) is not None) + + +def strict_format( + completion: Any, + parser: MedFactScoreParser, + **_: Any, +) -> float: + """Measure exact compliance with the requested think and score blocks.""" + return float(parser.is_strict_format(completion)) + + +def _resolve_subset(subset: str | MedFactSubset) -> MedFactSubset: + if isinstance(subset, MedFactSubset): + return subset + try: + return MedFactSubset(subset) + except ValueError as exc: + choices = ", ".join(member.value for member in MedFactSubset) + raise ValueError(f"Unsupported MedFact-Bench subset {subset!r}. Choose one of: {choices}.") from exc + + +def _load_source_dataset(dataset_path: str | None, cache_dir: str | None) -> Dataset: + if dataset_path is not None: + path = Path(dataset_path).expanduser() + if not path.exists(): + raise FileNotFoundError(f"MedFact-Bench dataset path does not exist: {path}") + if not path.is_file(): + raise ValueError(f"MedFact-Bench dataset path must be a Parquet file: {path}") + try: + dataset = load_dataset( + "parquet", + data_files=str(path), + split="train", + cache_dir=cache_dir, + ) + except Exception as exc: + raise ValueError(f"Failed to load the local MedFact-Bench Parquet file at {path}: {exc}") from exc + return dataset + + try: + dataset = load_dataset( + DATASET_ID, + split="train", + revision=DATASET_REVISION, + cache_dir=cache_dir, + ) + except Exception as exc: + raise RuntimeError(f"Failed to load {DATASET_ID} at revision {DATASET_REVISION}: {exc}") from exc + return dataset + + +def _validate_dataset(dataset: Dataset) -> str: + missing_columns = sorted(set(REQUIRED_COLUMNS) - set(dataset.column_names)) + if missing_columns: + missing = ", ".join(missing_columns) + raise ValueError(f"MedFact-Bench dataset is missing required columns: {missing}.") + if len(dataset) == 0: + raise ValueError("MedFact-Bench dataset is empty.") + + for column in REQUIRED_COLUMNS: + values = dataset[column] + null_count = sum(value is None for value in values) + if null_count: + raise ValueError(f"MedFact-Bench column {column!r} contains {null_count} null value(s).") + non_string_count = sum(not isinstance(value, str) for value in values) + if non_string_count: + raise ValueError(f"MedFact-Bench column {column!r} contains {non_string_count} non-string value(s).") + empty_count = sum(not value.strip() for value in values) + if empty_count: + raise ValueError(f"MedFact-Bench column {column!r} contains {empty_count} empty value(s).") + + dataset_values = set(dataset["dataset"]) + unexpected_datasets = sorted(dataset_values - set(EXPECTED_DATASET_COUNTS)) + if unexpected_datasets: + values = ", ".join(unexpected_datasets) + raise ValueError(f"MedFact-Bench dataset contains unsupported component values: {values}.") + + label_values = set(dataset["label"]) + unexpected_labels = sorted(label_values - set(LABELS)) + if unexpected_labels: + values = ", ".join(unexpected_labels) + raise ValueError(f"MedFact-Bench dataset contains unsupported labels: {values}.") + + system_prompts = set(dataset["system_prompt"]) + if len(system_prompts) != 1: + raise ValueError( + f"MedFact-Bench dataset must contain exactly one distinct system prompt; found {len(system_prompts)}." + ) + return next(iter(system_prompts)) + + +def _prepare_eval_dataset(dataset: Dataset, subset: MedFactSubset) -> Dataset: + selected = dataset + if subset is not MedFactSubset.ALL: + selected = dataset.filter( + lambda row: row["dataset"] == subset.value, + load_from_cache_file=False, + ) + if len(selected) == 0: + raise ValueError(f"MedFact-Bench subset {subset.value!r} contains no rows.") + + return selected.map( + lambda row: { + "question": row["user_prompt"], + "answer": row["label"], + "info": {"dataset": row["dataset"]}, + }, + remove_columns=selected.column_names, + load_from_cache_file=False, + ) + + +def load_environment( + subset: str | MedFactSubset = MedFactSubset.ALL, + dataset_path: str | None = None, + cache_dir: str | None = None, +) -> vf.Environment: + """Load the evaluation-only MedFact-Bench environment. + + Args: + subset: Component dataset to evaluate, or ``all`` for the full benchmark. + dataset_path: Optional local Parquet path. When provided, it takes precedence over Hugging Face. + cache_dir: Optional Hugging Face datasets cache directory. + + Returns: + A single-turn environment with only an evaluation dataset. + """ + resolved_subset = _resolve_subset(subset) + source_dataset = _load_source_dataset(dataset_path, cache_dir) + system_prompt = _validate_dataset(source_dataset) + eval_dataset = _prepare_eval_dataset(source_dataset, resolved_subset) + + parser = MedFactScoreParser() + rubric = vf.Rubric( + funcs=[accuracy, parseable_score, strict_format], + weights=[1.0, 0.0, 0.0], + parser=parser, + ) + return vf.SingleTurnEnv( + eval_dataset=eval_dataset, + system_prompt=system_prompt, + parser=parser, + rubric=rubric, + ) diff --git a/environments/medfact_bench/medfact_bench/reporting.py b/environments/medfact_bench/medfact_bench/reporting.py new file mode 100644 index 00000000..62f5a6b5 --- /dev/null +++ b/environments/medfact_bench/medfact_bench/reporting.py @@ -0,0 +1,296 @@ +"""Benchmark-specific reporting for raw MedFact-Bench evaluation results.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Any + +from .environment import ( + EXPECTED_DATASET_COUNTS, + INVALID_LABEL, + LABELS, + MedFactScoreParser, + score_to_label, +) + +REPORT_SCHEMA_VERSION = 1 +CONFUSION_LABELS = (*LABELS, INVALID_LABEL) + + +def _safe_divide(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +def _iter_results(path: Path) -> Iterator[dict[str, Any]]: + if not path.exists(): + raise FileNotFoundError(f"MedFact-Bench results file does not exist: {path}") + if not path.is_file(): + raise ValueError(f"MedFact-Bench results path must be a JSONL file: {path}") + + with path.open("r", encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, start=1): + line = raw_line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in MedFact-Bench results at {path}:{line_number}: {exc.msg}.") from exc + if not isinstance(row, dict): + raise ValueError(f"Expected a JSON object in MedFact-Bench results at {path}:{line_number}.") + _validate_result_row(row, path, line_number) + yield row + + +def _validate_result_row(row: dict[str, Any], path: Path, line_number: int) -> None: + location = f"{path}:{line_number}" + if "completion" not in row: + raise ValueError(f"MedFact-Bench result at {location} is missing the completion field.") + + answer = row.get("answer") + if answer not in LABELS: + raise ValueError(f"MedFact-Bench result at {location} has invalid or missing answer label: {answer!r}.") + + info = row.get("info") + if not isinstance(info, Mapping): + raise ValueError(f"MedFact-Bench result at {location} is missing the info object.") + dataset_name = info.get("dataset") + if dataset_name not in EXPECTED_DATASET_COUNTS: + raise ValueError(f"MedFact-Bench result at {location} has invalid or missing info.dataset: {dataset_name!r}.") + + +def _empty_confusion_matrix() -> dict[str, dict[str, int]]: + return {actual: {predicted: 0 for predicted in CONFUSION_LABELS} for actual in CONFUSION_LABELS} + + +def _class_metrics( + confusion: Mapping[str, Mapping[str, int]], +) -> tuple[dict[str, dict[str, float | int]], float]: + per_class: dict[str, dict[str, float | int]] = {} + for label in LABELS: + true_positive = confusion[label][label] + predicted_count = sum(confusion[actual][label] for actual in CONFUSION_LABELS) + support = sum(confusion[label].values()) + precision = _safe_divide(true_positive, predicted_count) + recall = _safe_divide(true_positive, support) + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + per_class[label] = { + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + "predicted": predicted_count, + } + macro_f1 = sum(metrics["f1"] for metrics in per_class.values()) / len(LABELS) + return per_class, macro_f1 + + +def build_report( + results_path: str | Path, + *, + expected_dataset_counts: Mapping[str, int] | None = None, +) -> dict[str, Any]: + """Build a deterministic report from raw Verifiers results.""" + path = Path(results_path) + parser = MedFactScoreParser() + expected_counts = dict(expected_dataset_counts or EXPECTED_DATASET_COUNTS) + expected_total = sum(expected_counts.values()) + + total_predictions = 0 + valid_predictions = 0 + strict_predictions = 0 + correct_predictions = 0 + dataset_totals: Counter[str] = Counter() + dataset_correct: Counter[str] = Counter() + confusion = _empty_confusion_matrix() + + for row in _iter_results(path): + answer = row["answer"] + dataset_name = row["info"]["dataset"] + completion_text = parser.completion_text(row["completion"]) + score = parser.parse(completion_text) if completion_text is not None else None + predicted = score_to_label(score) + + is_valid = predicted != INVALID_LABEL + is_correct = predicted == answer + total_predictions += 1 + valid_predictions += int(is_valid) + strict_predictions += int(completion_text is not None and parser.is_strict_format(completion_text)) + correct_predictions += int(is_correct) + dataset_totals[dataset_name] += 1 + dataset_correct[dataset_name] += int(is_correct) + confusion[answer][predicted] += 1 + + if total_predictions == 0: + raise ValueError(f"MedFact-Bench results file contains no result rows: {path}") + + per_dataset: dict[str, dict[str, float | int | None]] = {} + observed_accuracies: list[float] = [] + for dataset_name in EXPECTED_DATASET_COUNTS: + total = dataset_totals[dataset_name] + correct = dataset_correct[dataset_name] + dataset_accuracy = correct / total if total else None + if dataset_accuracy is not None: + observed_accuracies.append(dataset_accuracy) + per_dataset[dataset_name] = { + "correct": correct, + "total": total, + "accuracy": dataset_accuracy, + } + + invalid_predictions = total_predictions - valid_predictions + macro_accuracy = sum(observed_accuracies) / len(observed_accuracies) + per_class, macro_f1 = _class_metrics(confusion) + + observed_counts = {name: dataset_totals[name] for name in expected_counts} + missing_datasets = [name for name, count in observed_counts.items() if count == 0] + count_mismatches = { + name: {"expected": expected_counts[name], "observed": observed_counts[name]} + for name in expected_counts + if observed_counts[name] != expected_counts[name] + } + complete_coverage = not count_mismatches and total_predictions == expected_total + + return { + "schema_version": REPORT_SCHEMA_VERSION, + "benchmark": "medfact_bench", + "total_predictions": total_predictions, + "valid_predictions": valid_predictions, + "invalid_predictions": invalid_predictions, + "parse_rate": valid_predictions / total_predictions, + "strict_format_rate": strict_predictions / total_predictions, + "micro_accuracy": correct_predictions / total_predictions, + "macro_accuracy": macro_accuracy, + "paper_macro_accuracy": macro_accuracy if complete_coverage else None, + "macro_f1": macro_f1, + "per_dataset": per_dataset, + "per_class": per_class, + "confusion_matrix": { + "labels": list(CONFUSION_LABELS), + "rows": confusion, + }, + "coverage": { + "status": "complete" if complete_coverage else "partial", + "complete": complete_coverage, + "expected_total": expected_total, + "observed_total": total_predictions, + "expected_dataset_counts": expected_counts, + "observed_dataset_counts": observed_counts, + "missing_datasets": missing_datasets, + "count_mismatches": count_mismatches, + }, + "paper_comparable": complete_coverage, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.tmp") + temporary_path.write_text( + json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + temporary_path.replace(path) + + +def _update_metadata(path: Path, report: Mapping[str, Any]) -> None: + if not path.exists(): + raise FileNotFoundError(f"MedFact-Bench metadata file does not exist: {path}") + try: + metadata = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in MedFact-Bench metadata file {path}: {exc.msg}.") from exc + if not isinstance(metadata, dict): + raise ValueError(f"MedFact-Bench metadata file must contain a JSON object: {path}") + + avg_metrics = metadata.get("avg_metrics") + if avg_metrics is None: + avg_metrics = {} + metadata["avg_metrics"] = avg_metrics + if not isinstance(avg_metrics, dict): + raise ValueError(f"MedFact-Bench metadata avg_metrics must be a JSON object: {path}") + + avg_metrics.update( + { + "parse_rate": report["parse_rate"], + "strict_format_rate": report["strict_format_rate"], + "micro_accuracy": report["micro_accuracy"], + "macro_accuracy": report["macro_accuracy"], + "macro_f1": report["macro_f1"], + "invalid_predictions": report["invalid_predictions"], + } + ) + if report["paper_macro_accuracy"] is not None: + avg_metrics["paper_macro_accuracy"] = report["paper_macro_accuracy"] + else: + avg_metrics.pop("paper_macro_accuracy", None) + _write_json(path, metadata) + + +def write_report( + results_path: str | Path, + *, + output_path: str | Path | None = None, + metadata_path: str | Path | None = None, + update_metadata: bool = True, + expected_dataset_counts: Mapping[str, int] | None = None, +) -> dict[str, Any]: + """Build and write a MedFact-Bench report, optionally updating metadata.""" + results = Path(results_path) + report = build_report(results, expected_dataset_counts=expected_dataset_counts) + output = Path(output_path) if output_path is not None else results.with_name("medfact_report.json") + _write_json(output, report) + + if update_metadata: + metadata = Path(metadata_path) if metadata_path is not None else results.with_name("metadata.json") + _update_metadata(metadata, report) + return report + + +def _build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate benchmark-specific metrics from raw MedFact-Bench results.jsonl.", + ) + parser.add_argument("results_jsonl", help="Path to the raw MedFact-Bench results.jsonl file.") + parser.add_argument( + "--output", + default=None, + help="Report output path. Defaults to medfact_report.json beside the results file.", + ) + parser.add_argument( + "--metadata", + default=None, + help="Metadata path. Defaults to metadata.json beside the results file.", + ) + parser.add_argument( + "--no-update-metadata", + action="store_true", + help="Write the report without updating metadata.json.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Run the MedFact-Bench reporting CLI.""" + args = _build_argument_parser().parse_args(argv) + results_path = Path(args.results_jsonl) + output_path = Path(args.output) if args.output is not None else results_path.with_name("medfact_report.json") + write_report( + results_path, + output_path=output_path, + metadata_path=args.metadata, + update_metadata=not args.no_update_metadata, + ) + print(f"Wrote MedFact-Bench report to {output_path}") + if not args.no_update_metadata: + metadata_path = Path(args.metadata) if args.metadata is not None else results_path.with_name("metadata.json") + print(f"Updated MedFact-Bench metrics in {metadata_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/environments/medfact_bench/pyproject.toml b/environments/medfact_bench/pyproject.toml new file mode 100644 index 00000000..ef0ccb18 --- /dev/null +++ b/environments/medfact_bench/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "medfact-bench" +version = "0.1.0" +description = "Paper-faithful zero-shot evaluation and reporting for MedFact-Bench" +authors = [ + { name = "Kouate Muhamed", email = "muhamed.kouate@icloud.com" }, +] +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "medarc-verifiers>=0.2.0", + "verifiers>=0.1.14,<0.2", + "datasets>=4.2.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["medfact_bench"] + +[tool.prime.environment] +loader = "medfact_bench:load_environment" +display_name = "MedFact-Bench" +visibility = "PUBLIC" + +[tool.verifiers.eval] +num_examples = -1 +rollouts_per_example = 1 diff --git a/tests/test_cli/test_main.py b/tests/test_cli/test_main.py index fbb929f8..fcc5c24b 100644 --- a/tests/test_cli/test_main.py +++ b/tests/test_cli/test_main.py @@ -153,7 +153,8 @@ def test_toml_bench_dry_run_expands_evals_and_ablations( assert "base" in output assert "shuffle_seed-1618" in output assert "shuffle_seed-9331" in output - assert str(tmp_path / "evals" / "gpt-5-mini" / "medqa" / "base") in output + assert "gpt-5-mini" in output + assert "medqa" in output def test_repository_smoke_toml_config_dry_runs(capsys: pytest.CaptureFixture[str]) -> None: @@ -162,7 +163,8 @@ def test_repository_smoke_toml_config_dry_runs(capsys: pytest.CaptureFixture[str output = capsys.readouterr().out assert exit_code == 0 assert "TOML Bench Dry Run" in output - assert "18 eval(s) to dry-run" in output + assert "19 eval(s) to dry-run" in output + assert "medfact_bench" in output assert "medqa" in output assert "runs/smoke/openai-gpt-4.1-mini/medqa" in output @@ -219,7 +221,7 @@ def test_bench_rejects_removed_yaml_runner_flags(capsys: pytest.CaptureFixture[s def test_repository_verified_toml_config_dry_run_shows_ablation_variants(capsys: pytest.CaptureFixture[str]) -> None: - exit_code = main.main(["bench", "--config", "configs/medmarks-verified.toml", "--dry-run", "--eval-index", "45"]) + exit_code = main.main(["bench", "--config", "configs/medmarks-verified.toml", "--dry-run", "--eval-index", "46"]) output = capsys.readouterr().out assert exit_code == 0 @@ -626,7 +628,11 @@ def test_toml_bench_dry_run_uses_toml_output_dir( assert main.main(["bench", "--config", str(config_path), "--dry-run"]) == 0 - assert str(output_dir / "gpt-5-mini" / "medqa" / "base") in capsys.readouterr().out + output = capsys.readouterr().out + assert str(output_dir) in output + assert "gpt-5-mini" in output + assert "medqa" in output + assert "base" in output def test_toml_bench_executes_sequentially_to_deterministic_path( diff --git a/tests/test_environments/test_medfact_bench.py b/tests/test_environments/test_medfact_bench.py new file mode 100644 index 00000000..02bd4c32 --- /dev/null +++ b/tests/test_environments/test_medfact_bench.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import asyncio +import os +import sys +from collections import Counter +from pathlib import Path + +import pytest +from datasets import Dataset +from verifiers.types import ClientConfig, Response, ResponseMessage + +REPO_ROOT = Path(__file__).resolve().parents[2] +ENVIRONMENT_ROOT = REPO_ROOT / "environments" / "medfact_bench" +if str(ENVIRONMENT_ROOT) not in sys.path: + sys.path.insert(0, str(ENVIRONMENT_ROOT)) + +from medfact_bench import ( # noqa: E402 + DATASET_ID, + DATASET_REVISION, + EXPECTED_DATASET_COUNTS, + INVALID_LABEL, + MedFactScoreParser, + MedFactSubset, + load_environment, + parseable_score, + prediction_label, + strict_format, +) +import medfact_bench.environment as medfact_environment # noqa: E402 + +REAL_DATASET_PATH = Path( + "/Users/kouatemuhamed/Claude/Projects/MedARC Agentic Medical Fact Verifier/" + "datasets/MedFact-Bench/MedFact-Bench.parquet" +) + + +def _source_row( + dataset: str, + label: str, + *, + claim: str | None = None, + source: str | None = None, + system_prompt: str = "Canonical system prompt", +) -> dict[str, str]: + claim = claim or f"Claim for {dataset}" + source = source or f"Source for {dataset}" + return { + "dataset": dataset, + "claim": claim, + "source": source, + "label": label, + "system_prompt": system_prompt, + "user_prompt": f"Article:\n{source}\n\nClaim:\n{claim}", + } + + +def _source_rows() -> list[dict[str, str]]: + return [ + _source_row("scifact", "SUPPORT", claim="Duplicate claim", source="Duplicate source"), + _source_row("scifact", "SUPPORT", claim="Duplicate claim", source="Duplicate source"), + _source_row("healthver", "NEI"), + _source_row("medaesqa", "CONTRADICT"), + _source_row("pubmedqa-fact", "SUPPORT"), + _source_row("bioasq-fact", "NEI"), + ] + + +def _write_parquet(tmp_path: Path, rows: list[dict[str, str]]) -> Path: + path = tmp_path / "medfact-bench.parquet" + Dataset.from_list(rows).to_parquet(path) + return path + + +def _load_local_environment( + tmp_path: Path, + rows: list[dict[str, str]], + *, + subset: str | MedFactSubset = MedFactSubset.ALL, +): + return load_environment( + dataset_path=str(_write_parquet(tmp_path, rows)), + subset=subset, + cache_dir=str(tmp_path / "datasets-cache"), + ) + + +def test_loader_maps_local_parquet_and_preserves_duplicate_order(tmp_path: Path) -> None: + rows = _source_rows() + environment = _load_local_environment(tmp_path, rows) + + assert environment.dataset is None + assert environment.system_prompt == "Canonical system prompt" + assert len(environment.eval_dataset) == len(rows) + assert set(environment.eval_dataset.column_names) == {"question", "answer", "info", "example_id", "prompt"} + + first = environment.eval_dataset[0] + second = environment.eval_dataset[1] + assert first["question"] == rows[0]["user_prompt"] + assert first["answer"] == "SUPPORT" + assert first["info"] == {"dataset": "scifact"} + assert first["question"] == second["question"] + assert first["answer"] == second["answer"] + assert first["prompt"] == [ + {"role": "system", "content": "Canonical system prompt"}, + {"role": "user", "content": rows[0]["user_prompt"]}, + ] + + +@pytest.mark.parametrize( + ("subset", "expected_count", "expected_dataset"), + [ + ("scifact", 2, "scifact"), + (MedFactSubset.HEALTHVER, 1, "healthver"), + ("medaesqa", 1, "medaesqa"), + ("pubmedqa-fact", 1, "pubmedqa-fact"), + ("bioasq-fact", 1, "bioasq-fact"), + ], +) +def test_loader_filters_subset_before_mapping( + tmp_path: Path, + subset: str | MedFactSubset, + expected_count: int, + expected_dataset: str, +) -> None: + environment = _load_local_environment(tmp_path, _source_rows(), subset=subset) + + assert len(environment.eval_dataset) == expected_count + assert {row["info"]["dataset"] for row in environment.eval_dataset} == {expected_dataset} + + +def test_loader_uses_pinned_hugging_face_revision(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] + source_dataset = Dataset.from_list(_source_rows()) + + def fake_load_dataset(*args: object, **kwargs: object) -> Dataset: + calls.append((args, kwargs)) + return source_dataset + + monkeypatch.setattr(medfact_environment, "load_dataset", fake_load_dataset) + environment = load_environment(cache_dir="/tmp/medfact-cache") + + assert len(environment.eval_dataset) == len(source_dataset) + assert calls == [ + ( + (DATASET_ID,), + { + "split": "train", + "revision": DATASET_REVISION, + "cache_dir": "/tmp/medfact-cache", + }, + ) + ] + + +def test_loader_prefers_local_parquet_over_hugging_face(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] + original_load_dataset = medfact_environment.load_dataset + + def recording_load_dataset(*args: object, **kwargs: object) -> Dataset: + calls.append((args, kwargs)) + return original_load_dataset(*args, **kwargs) + + monkeypatch.setattr(medfact_environment, "load_dataset", recording_load_dataset) + path = _write_parquet(tmp_path, _source_rows()) + environment = load_environment(dataset_path=str(path), cache_dir=str(tmp_path / "datasets-cache")) + + assert len(environment.eval_dataset) == 6 + assert calls[0][0] == ("parquet",) + assert calls[0][1]["data_files"] == str(path) + assert all(call[0] != (DATASET_ID,) for call in calls) + + +@pytest.mark.parametrize( + ("rows", "message"), + [ + ([{"dataset": "scifact"}], "missing required columns"), + ([{**_source_rows()[0], "dataset": "unknown"}], "unsupported component values"), + ([{**_source_rows()[0], "label": "MAYBE"}], "unsupported labels"), + ( + [_source_rows()[0], _source_row("healthver", "NEI", system_prompt="Different system prompt")], + "exactly one distinct system prompt", + ), + ], +) +def test_loader_rejects_invalid_schema(tmp_path: Path, rows: list[dict[str, str]], message: str) -> None: + with pytest.raises(ValueError, match=message): + load_environment( + dataset_path=str(_write_parquet(tmp_path, rows)), + cache_dir=str(tmp_path / "datasets-cache"), + ) + + +def test_loader_rejects_invalid_path_and_subset(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="does not exist"): + load_environment(dataset_path=str(tmp_path / "missing.parquet")) + with pytest.raises(ValueError, match="Unsupported MedFact-Bench subset"): + load_environment(dataset_path=str(_write_parquet(tmp_path, _source_rows())), subset="unknown") + + +@pytest.mark.parametrize( + ("completion", "expected_score", "expected_label"), + [ + ("-2", -2, "CONTRADICT"), + (" -1 ", -1, "CONTRADICT"), + ("0", 0, "NEI"), + ("+1", 1, "SUPPORT"), + ("2", 2, "SUPPORT"), + ("Before 2 after", 2, "SUPPORT"), + ("2-2", 2, "SUPPORT"), + ([{"role": "assistant", "content": "+1"}], 1, "SUPPORT"), + ], +) +def test_parser_accepts_valid_first_score( + completion: str | list[dict[str, str]], + expected_score: int, + expected_label: str, +) -> None: + parser = MedFactScoreParser() + + assert parser.parse_completion(completion) == expected_score + assert prediction_label(completion, parser) == expected_label + + +@pytest.mark.parametrize( + "completion", + [ + "", + "", + "1.0", + "support", + "3", + "-3", + "−1", + "reasoning", + [{"role": "user", "content": "1"}], + ], +) +def test_parser_rejects_invalid_scores(completion: str | list[dict[str, str]]) -> None: + parser = MedFactScoreParser() + + assert parser.parse_completion(completion) is None + assert prediction_label(completion, parser) == INVALID_LABEL + + +def test_parseability_and_strict_format_are_independent() -> None: + parser = MedFactScoreParser() + strict_completion = "Reasoning-1" + relaxed_completion = "Explanation first. -1" + duplicate_tag_completion = "Reasoning-12" + + assert parseable_score(strict_completion, parser) == 1.0 + assert strict_format(strict_completion, parser) == 1.0 + assert parseable_score(relaxed_completion, parser) == 1.0 + assert strict_format(relaxed_completion, parser) == 0.0 + assert parseable_score(duplicate_tag_completion, parser) == 1.0 + assert strict_format(duplicate_tag_completion, parser) == 0.0 + + +def test_environment_scores_a_fake_model_response(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + environment = _load_local_environment(tmp_path, _source_rows()) + + async def fake_model_response(*_: object, **__: object) -> Response: + return Response( + id="fake-response", + created=0, + model="fake-model", + message=ResponseMessage( + content="Evidence supports the claim.2", + finish_reason="stop", + is_truncated=False, + ), + ) + + monkeypatch.setattr(environment, "get_model_response", fake_model_response) + output = asyncio.run( + environment.run_rollout( + environment.eval_dataset[0], + ClientConfig(api_key_var="LOCAL_TEST_KEY", api_base_url="http://127.0.0.1:1/v1"), + "fake-model", + {"n": 1, "extra_body": {}}, + ) + ) + + assert output["reward"] == 1.0 + assert output["accuracy"] == 1.0 + assert output["parseable_score"] == 1.0 + assert output["strict_format"] == 1.0 + assert output["info"] == {"dataset": "scifact"} + assert output["answer"] == "SUPPORT" + + +@pytest.mark.skipif( + os.environ.get("MEDFACT_BENCH_RUN_DATASET_INTEGRATION") != "1", + reason="Set MEDFACT_BENCH_RUN_DATASET_INTEGRATION=1 to run the local Parquet integration test.", +) +def test_local_parquet_dataset_invariants(tmp_path: Path) -> None: + if not REAL_DATASET_PATH.exists(): + pytest.fail(f"Configured MedFact-Bench Parquet file does not exist: {REAL_DATASET_PATH}") + + environment = load_environment( + dataset_path=str(REAL_DATASET_PATH), + cache_dir=str(tmp_path / "datasets-cache"), + ) + eval_dataset = environment.eval_dataset + + assert len(eval_dataset) == 14_274 + assert Counter(row["info"]["dataset"] for row in eval_dataset) == Counter(EXPECTED_DATASET_COUNTS) + assert Counter(row["answer"] for row in eval_dataset) == { + "SUPPORT": 10_118, + "NEI": 2_988, + "CONTRADICT": 1_168, + } + assert environment.dataset is None + assert all(len(row["prompt"]) == 2 for row in eval_dataset) + assert all(row["prompt"][0]["role"] == "system" for row in eval_dataset) + assert all(row["prompt"][1]["role"] == "user" for row in eval_dataset) + assert all(row["question"] and row["answer"] and row["info"]["dataset"] for row in eval_dataset) + + duplicate_excess = sum( + count - 1 + for count in Counter((row["question"], row["answer"], row["info"]["dataset"]) for row in eval_dataset).values() + if count > 1 + ) + assert duplicate_excess == 2_261 diff --git a/tests/test_environments/test_medfact_bench_reporting.py b/tests/test_environments/test_medfact_bench_reporting.py new file mode 100644 index 00000000..e70e1229 --- /dev/null +++ b/tests/test_environments/test_medfact_bench_reporting.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +ENVIRONMENT_ROOT = REPO_ROOT / "environments" / "medfact_bench" +if str(ENVIRONMENT_ROOT) not in sys.path: + sys.path.insert(0, str(ENVIRONMENT_ROOT)) + +from medfact_bench.reporting import build_report, main, write_report # noqa: E402 + +SMALL_COMPLETE_COUNTS = { + "scifact": 1, + "healthver": 1, + "medaesqa": 1, + "pubmedqa-fact": 1, + "bioasq-fact": 1, +} + + +def _row(dataset: str, answer: str, completion: object) -> dict[str, object]: + return { + "example_id": dataset, + "answer": answer, + "info": {"dataset": dataset}, + "completion": completion, + } + + +def _write_jsonl(path: Path, rows: list[dict[str, object]]) -> None: + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def _imbalanced_rows() -> list[dict[str, object]]: + return [ + _row("scifact", "SUPPORT", "Strong evidence2"), + _row("healthver", "NEI", "Insufficient evidence0"), + _row("medaesqa", "CONTRADICT", "Reasoning -1 trailing text"), + _row("pubmedqa-fact", "SUPPORT", "Malformed score9"), + _row("bioasq-fact", "NEI", "1"), + ] + + +def test_report_calculates_hand_checked_metrics_and_complete_coverage(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + _write_jsonl(results_path, _imbalanced_rows()) + + report = build_report(results_path, expected_dataset_counts=SMALL_COMPLETE_COUNTS) + + assert report["total_predictions"] == 5 + assert report["valid_predictions"] == 4 + assert report["invalid_predictions"] == 1 + assert report["parse_rate"] == pytest.approx(0.8) + assert report["strict_format_rate"] == pytest.approx(0.4) + assert report["micro_accuracy"] == pytest.approx(0.6) + assert report["macro_accuracy"] == pytest.approx(0.6) + assert report["paper_macro_accuracy"] == pytest.approx(0.6) + assert report["paper_comparable"] is True + assert report["coverage"]["status"] == "complete" + assert report["per_dataset"]["pubmedqa-fact"] == {"correct": 0, "total": 1, "accuracy": 0.0} + + assert report["per_class"]["SUPPORT"]["precision"] == pytest.approx(0.5) + assert report["per_class"]["SUPPORT"]["recall"] == pytest.approx(0.5) + assert report["per_class"]["SUPPORT"]["f1"] == pytest.approx(0.5) + assert report["per_class"]["NEI"]["f1"] == pytest.approx(2 / 3) + assert report["per_class"]["CONTRADICT"]["f1"] == pytest.approx(1.0) + assert report["macro_f1"] == pytest.approx((0.5 + 2 / 3 + 1.0) / 3) + + confusion = report["confusion_matrix"]["rows"] + assert report["confusion_matrix"]["labels"] == ["SUPPORT", "NEI", "CONTRADICT", "INVALID"] + assert confusion["SUPPORT"] == {"SUPPORT": 1, "NEI": 0, "CONTRADICT": 0, "INVALID": 1} + assert confusion["NEI"] == {"SUPPORT": 1, "NEI": 1, "CONTRADICT": 0, "INVALID": 0} + assert confusion["CONTRADICT"] == {"SUPPORT": 0, "NEI": 0, "CONTRADICT": 1, "INVALID": 0} + + +def test_report_marks_partial_coverage_as_not_paper_comparable(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + _write_jsonl(results_path, _imbalanced_rows()[:2]) + + report = build_report(results_path) + + assert report["coverage"]["status"] == "partial" + assert report["paper_comparable"] is False + assert report["paper_macro_accuracy"] is None + assert report["macro_accuracy"] == pytest.approx(1.0) + assert report["coverage"]["missing_datasets"] == ["medaesqa", "pubmedqa-fact", "bioasq-fact"] + + +def test_report_handles_chat_message_completion(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + completion = [{"role": "assistant", "content": "Strong evidence2"}] + _write_jsonl(results_path, [_row("scifact", "SUPPORT", completion)]) + + report = build_report(results_path, expected_dataset_counts={"scifact": 1}) + + assert report["valid_predictions"] == 1 + assert report["strict_format_rate"] == 1.0 + assert report["micro_accuracy"] == 1.0 + assert report["paper_macro_accuracy"] == 1.0 + + +def test_write_report_preserves_metadata_and_avg_reward(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + metadata_path = tmp_path / "metadata.json" + report_path = tmp_path / "custom-report.json" + _write_jsonl(results_path, _imbalanced_rows()) + metadata_path.write_text( + json.dumps( + { + "avg_reward": 0.123, + "avg_metrics": {"accuracy": 0.123, "preserved_metric": 9.0}, + "custom": {"keep": True}, + } + ), + encoding="utf-8", + ) + + report = write_report( + results_path, + output_path=report_path, + metadata_path=metadata_path, + expected_dataset_counts=SMALL_COMPLETE_COUNTS, + ) + written_report = json.loads(report_path.read_text(encoding="utf-8")) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + + assert written_report == report + assert metadata["avg_reward"] == 0.123 + assert metadata["avg_metrics"]["accuracy"] == 0.123 + assert metadata["avg_metrics"]["preserved_metric"] == 9.0 + assert metadata["avg_metrics"]["micro_accuracy"] == pytest.approx(0.6) + assert metadata["avg_metrics"]["paper_macro_accuracy"] == pytest.approx(0.6) + assert metadata["custom"] == {"keep": True} + + +def test_write_report_removes_partial_paper_metric_without_touching_reward(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + metadata_path = tmp_path / "metadata.json" + _write_jsonl(results_path, _imbalanced_rows()[:2]) + metadata_path.write_text( + json.dumps({"avg_reward": 0.9, "avg_metrics": {"paper_macro_accuracy": 0.7, "keep": 1.0}}), + encoding="utf-8", + ) + + write_report(results_path, metadata_path=metadata_path) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + + assert metadata["avg_reward"] == 0.9 + assert metadata["avg_metrics"]["keep"] == 1.0 + assert "paper_macro_accuracy" not in metadata["avg_metrics"] + + +@pytest.mark.parametrize( + "content", + [ + "not json\n", + json.dumps({"answer": "SUPPORT", "info": {"dataset": "scifact"}}) + "\n", + json.dumps({"completion": "1", "answer": "SUPPORT", "info": {}}) + "\n", + ], +) +def test_report_rejects_corrupt_or_incomplete_results(tmp_path: Path, content: str) -> None: + results_path = tmp_path / "results.jsonl" + results_path.write_text(content, encoding="utf-8") + + with pytest.raises(ValueError): + build_report(results_path) + + +def test_write_report_does_not_write_partial_output_for_late_invalid_row(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + report_path = tmp_path / "medfact_report.json" + metadata_path = tmp_path / "metadata.json" + original_metadata = json.dumps({"avg_reward": 0.5, "custom": {"keep": True}}) + results_path.write_text(json.dumps(_imbalanced_rows()[0]) + "\nnot json\n", encoding="utf-8") + metadata_path.write_text(original_metadata, encoding="utf-8") + + with pytest.raises(ValueError, match="Invalid JSON"): + write_report(results_path, output_path=report_path, metadata_path=metadata_path) + + assert not report_path.exists() + assert metadata_path.read_text(encoding="utf-8") == original_metadata + + +def test_reporting_cli_can_skip_metadata_updates(tmp_path: Path) -> None: + results_path = tmp_path / "results.jsonl" + output_path = tmp_path / "report.json" + _write_jsonl(results_path, _imbalanced_rows()[:2]) + + exit_code = main([str(results_path), "--output", str(output_path), "--no-update-metadata"]) + + assert exit_code == 0 + assert output_path.exists() + assert not (tmp_path / "metadata.json").exists()