From 059fbd862bba0aef4d3aa223cfa60d1bc6756b7e Mon Sep 17 00:00:00 2001 From: Majd Abdallah Date: Sat, 19 Sep 2026 20:57:58 +0200 Subject: [PATCH 1/3] feat: add published result reproduction audit --- README.md | 20 +- docs/paper-reproduction.md | 124 +++++++ mkdocs.yml | 1 + pyproject.toml | 2 +- src/trialmatchai/cli/main.py | 7 + src/trialmatchai/cli/reproduce_paper.py | 117 +++++++ src/trialmatchai/trec/__init__.py | 16 +- src/trialmatchai/trec/paper_reproduction.py | 360 ++++++++++++++++++++ tests/test_paper_reproduction.py | 90 +++++ 9 files changed, 734 insertions(+), 3 deletions(-) create mode 100644 docs/paper-reproduction.md create mode 100644 src/trialmatchai/cli/reproduce_paper.py create mode 100644 src/trialmatchai/trec/paper_reproduction.py create mode 100644 tests/test_paper_reproduction.py diff --git a/README.md b/README.md index b2da9000..d562f680 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![Python 3.11](https://img.shields.io/badge/python-3.11-2563EB)](https://github.com/cbib/TrialMatchAI/blob/main/pyproject.toml) [![License: MIT](https://img.shields.io/badge/license-MIT-64748B)](https://github.com/cbib/TrialMatchAI/blob/main/LICENSE) -[Try the demo](#try-the-demo) · [Run your own data](#run-your-own-data) · [Architecture](#how-matching-works) · [CLI](#cli-reference) · [Development](#development-and-delivery) +[Try the demo](#try-the-demo) · [Run your own data](#run-your-own-data) · [Reproduce the paper](#reproduce-the-paper) · [Architecture](#how-matching-works) · [CLI](#cli-reference) · [Development](#development-and-delivery) @@ -234,6 +234,7 @@ Run `trialmatchai --help` for the current options. | `report` | Render a patient report or the multi-patient front page | | `update-registry` | Fetch and upsert ClinicalTrials.gov studies; optional watch mode | | `trec` | Run TREC Clinical Trials evaluation presets | +| `reproduce-paper` | Verify and recalculate the published TREC 2021/2022 result artifact | | `finetune` | Train supported reasoning, reranker, or NER components | | `healthcheck` | Inspect configured dependencies, paths, and services | | `artifacts` | Create or verify portable SHA-256 manifests | @@ -272,6 +273,23 @@ The current evaluator and retrieval pipeline have open audit findings; this release does not claim a newly measured ranking improvement or a validated comparison against another system. +## Reproduce the paper + +The published TREC result artifact can be audited on CPU with one command: + +```bash +trialmatchai reproduce-paper --workdir ./paper-reproduction +``` + +This verifies the pinned Zenodo archive, aggregates its stored per-topic metrics, +recalculates metrics from the archived rankings, and evaluates the same rankings +with the current metric implementation. It does not rerun model inference. The +archived summaries are internally consistent, but some archived rankings do not +regenerate their stored topic metrics, and the present tie-aware evaluator is not +the evaluator used for the paper values. See the [full reproduction record](https://github.com/cbib/TrialMatchAI/blob/main/docs/paper-reproduction.md) +for the measured differences, retrieval recall, offline usage, and the inputs +still required for an exact historical rerun. + The [research paper](https://doi.org/10.1038/s41467-026-70509-w) describes the published study. Its results should be distinguished from this release's software checks. The roadmap prioritizes retrieval recall, eligibility semantics, evidence diff --git a/docs/paper-reproduction.md b/docs/paper-reproduction.md new file mode 100644 index 00000000..983aadac --- /dev/null +++ b/docs/paper-reproduction.md @@ -0,0 +1,124 @@ +# Reproducing the published results + +The Nature Communications study evaluated TrialMatchAI on 75 TREC 2021 topics, +50 TREC 2022 topics, and a separate synthetic “ideal candidate” experiment. The +current repository can verify the immutable result artifact and recalculate its +TREC metrics without a GPU. It cannot yet perform an exact fresh inference rerun +of the historical pipeline. + +## Run the artifact audit + +From an installed release containing this command: + +```bash +trialmatchai reproduce-paper --workdir ./paper-reproduction +``` + +The command downloads the 124.7 MB `matching_results.zip` from the paper's +[Zenodo data record](https://zenodo.org/records/15045515), verifies its pinned +SHA-256 digest, extracts only the files needed for the audit, downloads the +official TREC qrels, and writes `paper-reproduction/reproduction-report.json`. +It reports three separate views: + +1. **Stored per-topic aggregation** checks whether the archived topic metrics + aggregate to the archived `average_metrics.json`. +2. **Recalculated from rankings** applies the paper artifact's metric convention + to `ranked_trials.json`: remove unjudged trials, use linear relevance gains, + retain file order within score ties, and compute graded precision as + `sum(grade) / (2*k)`. +3. **Current evaluator** applies the repository's present tie-aware, condensed + evaluator to the same rankings. These values answer a different metric + question and should not be substituted for the published values. + +For an offline or CI run, supply the already-downloaded official archive and +qrels: + +```bash +trialmatchai reproduce-paper \ + --workdir ./paper-reproduction \ + --archive /artifacts/matching_results.zip \ + --qrels-dir /artifacts/qrels \ + --no-download --json +``` + +The official archive accepted by the command has SHA-256 +`dbfd11f19ffbfd78acfeaeee634c3e7e7ca8cb4d72b526ff8d191f9e880312c8`. +The TREC 2021 and 2022 qrels are also pinned by SHA-256. Changing any input byte +causes the audit to stop. + +## What the published artifacts reproduce + +The following values were obtained with the command above on 19 September 2026. +Means and medians are macro-averages across all topics in each track. + +| Track and metric | Stored mean | Stored median | Recalculated mean | Current-evaluator mean | +| :--- | ---: | ---: | ---: | ---: | +| TREC 2021 nDCG@5 | 0.729721 | 0.746939 | 0.727659 | 0.669658 | +| TREC 2021 nDCG@10 | 0.716471 | 0.735478 | 0.706848 | 0.669746 | +| TREC 2021 nDCG@20 | 0.724413 | 0.733114 | 0.692088 | 0.688335 | +| TREC 2021 graded P@10 | 0.705333 | 0.750000 | 0.692667 | 0.692667 | +| TREC 2022 nDCG@5 | 0.722144 | 0.819318 | 0.717099 | 0.587426 | +| TREC 2022 nDCG@10 | 0.708484 | 0.762500 | 0.683137 | 0.598333 | +| TREC 2022 nDCG@20 | 0.719456 | 0.766262 | 0.656213 | 0.632342 | +| TREC 2022 graded P@10 | 0.663000 | 0.725000 | 0.661000 | 0.661000 | + +Across all 125 topics, weighting both tracks by topic count gives a stored mean +nDCG@10 of 0.713276 and stored mean graded P@10 of 0.688400. These do not exactly +regenerate the paper text's pooled values of 0.7232 and 0.6865 from the published +per-topic metric files. + +The stored per-topic files aggregate to the stored summaries. Recalculation from +the archived rankings finds at least one metric difference in 20 of 75 TREC 2021 +topics and 19 of 50 TREC 2022 topics. The report records every affected topic and +value. This means the aggregate tables are verifiable as an aggregation of stored +metrics, but some stored topic metrics cannot be regenerated from the ranking +files in the same archive. + +The archived first-stage cutoff files produce the following grade-1-or-higher +retrieval recall: + +| Track | Recall@500 mean | Recall@500 median | Recall@1000 mean | Recall@1000 median | +| :--- | ---: | ---: | ---: | ---: | +| TREC 2021 | 0.693768 | 0.738095 | 0.757988 | 0.817518 | +| TREC 2022 | 0.676977 | 0.719643 | 0.716526 | 0.769697 | + +These archived values do not establish the paper text's statement of greater +than 90% retrieval recall at roughly the top 3% of trials under this relevance +definition and macro-averaging. That statement may use a different run, +denominator, or aggregation, but the distinction is not recoverable from the +published artifact. + +The paper's source-data workbook supports the reported ideal-candidate result: +92 of 100 ground-truth trials are ranked first, 95 are in the top two, and all +are in the top nine. The matching archive's `IDEAL_CANDIDATES` directory contains +102 runs and does not regenerate that workbook table from its final ranking +files, so the current command does not present it as a ranking-file +recalculation. + +## Why a fresh exact rerun is not yet available + +The [paper](https://doi.org/10.1038/s41467-026-70509-w) describes Elasticsearch, +BGE-M3 retrieval, Gemma-2-2B reranking, and Phi-4 eligibility reasoning. The +current pipeline uses LanceDB and has changed ranking, tie, configuration, and +resume behavior. The [archived v0.01 code](https://doi.org/10.5281/zenodo.18329084) +does not include a single executable workflow that reconstructs all figures, and +its checked-in defaults do not fully match the methods text. The published +bundle also lacks enough lineage to bind every output to exact registry records, +model revisions, container images, prompts, random seeds, and hardware/runtime +versions. + +An exact inference reproduction therefore needs a new immutable experiment +manifest containing: + +- the historical full trial corpus and a content manifest; +- exact topics, qrels, preprocessing outputs, and candidate pools; +- model and adapter revisions with checksums; +- the Elasticsearch version, mappings, analyzers, and index snapshot; +- effective configuration and prompt templates for every stage; +- deterministic seeds, runtime/container lock, and GPU details; +- all per-topic run files, failures, timings, and figure-generation code. + +Until those inputs are recovered or rebuilt, call the command an **artifact +audit**, not an exact end-to-end replication. New experiments should use the +current evaluator and publish a complete run manifest; compare them with the +paper only after labeling the changed corpus, engine, models, and metrics. diff --git a/mkdocs.yml b/mkdocs.yml index d15fa17d..88c29cba 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,6 +61,7 @@ nav: - Patient interoperability: interoperability.md - Registry updater: registry-updater.md - Fine-tuning: finetuning.md + - Paper reproduction: paper-reproduction.md - API reference: api.md - Release process: release.md - Production roadmap: production-roadmap.md diff --git a/pyproject.toml b/pyproject.toml index e6d6281f..bf8c4209 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ finetune = [ [project.scripts] # Single entry point. Every capability is a subcommand: `trialmatchai ` # (pipeline, healthcheck, bootstrap-data, index, build-concepts, update-registry, -# import-patient, build, run, e2e, trec, report, finetune). +# import-patient, build, run, e2e, trec, reproduce-paper, report, finetune). trialmatchai = "trialmatchai.cli.main:main" [project.urls] diff --git a/src/trialmatchai/cli/main.py b/src/trialmatchai/cli/main.py index 0ab75287..d8c47eee 100644 --- a/src/trialmatchai/cli/main.py +++ b/src/trialmatchai/cli/main.py @@ -64,6 +64,11 @@ def main() -> int: help="End-to-end TREC CT evaluation (preset over e2e)", add_help=False, ) + subparsers.add_parser( + "reproduce-paper", + help="Verify and recalculate the published TREC 2021/2022 result artifact", + add_help=False, + ) subparsers.add_parser( "report", help="Render a self-contained HTML match report from a patient's results", @@ -102,6 +107,8 @@ def main() -> int: from trialmatchai.cli.e2e import main as command elif args.command == "trec": from trialmatchai.cli.trec import main as command + elif args.command == "reproduce-paper": + from trialmatchai.cli.reproduce_paper import main as command elif args.command == "report": from trialmatchai.cli.report import main as command elif args.command == "finetune": diff --git a/src/trialmatchai/cli/reproduce_paper.py b/src/trialmatchai/cli/reproduce_paper.py new file mode 100644 index 00000000..86798591 --- /dev/null +++ b/src/trialmatchai/cli/reproduce_paper.py @@ -0,0 +1,117 @@ +"""CLI for auditing the result artifact published with the TrialMatchAI paper.""" + +from __future__ import annotations + +import argparse +import json +import sys +import zipfile +from collections.abc import Sequence +from pathlib import Path + +import requests + +from trialmatchai.trec.paper_reproduction import ( + TRACKS, + extract_reproduction_files, + obtain_results_archive, + reproduce_paper_results, +) +from trialmatchai.trec.qrels import download_qrels +from trialmatchai.utils.file_utils import write_json_file + + +def _print_human(report: dict, output: Path) -> None: + print(f"Paper result audit: {report['status']}") + print(f"Verified result archive SHA-256: {report['artifact']['sha256']}") + print(f"Report: {output}") + pooled = report["all_125_topics_weighted_mean"]["stored_per_topic"] + print( + "All 125 topics, stored weighted means: " + f"nDCG@10={pooled['ndcg@10']:.6f}, graded P@10={pooled['p@10']:.6f}" + ) + for track in report["tracks"].values(): + checks = track["checks"] + stored = track["stored_per_topic"] + recalculated = track["recalculated_from_rankings"] + current = track["current_evaluator"] + print(f"\n{track['track']} ({track['topics']} topics)") + print( + " stored per-topic aggregation: " + + ("matches archived summary" if checks["archived_summary_matches_stored_topics"] else "MISMATCH") + ) + print(f" rankings differing from stored metrics: {checks['ranking_mismatch_count']}") + print( + " paper-method nDCG@10 mean: " + f"stored={stored['ndcg@10']['mean']:.6f}, " + f"recalculated={recalculated['ndcg@10']['mean']:.6f}" + ) + print( + " current-evaluator nDCG@10 mean: " + f"{current['ndcg@10']['mean']:.6f}" + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Verify and recalculate the published TrialMatchAI TREC 2021/2022 result " + "artifact. This audits released outputs; it does not rerun model inference." + ) + ) + parser.add_argument( + "--workdir", + type=Path, + default=Path("paper-reproduction"), + help="Cache and report directory (default: paper-reproduction)", + ) + parser.add_argument( + "--archive", + type=Path, + help="Use an existing official matching_results.zip instead of downloading it", + ) + parser.add_argument( + "--qrels-dir", + type=Path, + help="Directory containing qrels_21.txt and qrels_22.txt; missing files are downloaded", + ) + parser.add_argument( + "--no-download", + action="store_true", + help="Require already-cached result and qrels files (useful for offline CI)", + ) + parser.add_argument("--json", action="store_true", help="Print the full JSON report") + args = parser.parse_args(argv) + + workdir = args.workdir.resolve() + try: + if args.archive is None and not (workdir / "matching_results.zip").exists(): + print("Downloading and verifying the published result artifact...", file=sys.stderr) + archive = obtain_results_archive( + workdir, archive=args.archive, allow_download=not args.no_download + ) + results = extract_reproduction_files(archive, workdir) + qrels_dir = (args.qrels_dir or (workdir / "qrels")).resolve() + qrels_dir.mkdir(parents=True, exist_ok=True) + for track in TRACKS: + path = qrels_dir / f"qrels_{track}.txt" + if not path.exists(): + if args.no_download: + raise FileNotFoundError(f"Qrels file not found: {path}") + download_qrels(track, qrels_dir) + report = reproduce_paper_results(results, qrels_dir) + output = workdir / "reproduction-report.json" + write_json_file(report, str(output)) + except (OSError, ValueError, requests.RequestException, zipfile.BadZipFile) as exc: + print(f"reproduce-paper failed: {exc}", file=sys.stderr) + return 1 + + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + _print_human(report, output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/trialmatchai/trec/__init__.py b/src/trialmatchai/trec/__init__.py index 38eb67bd..da6c4ba5 100644 --- a/src/trialmatchai/trec/__init__.py +++ b/src/trialmatchai/trec/__init__.py @@ -8,7 +8,21 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any + from trialmatchai.trec.corpus import TRACK_KEYS, TrackSpec, resolve_tracks -from trialmatchai.trec.runner import run_tracks + + +def __getattr__(name: str) -> Any: + """Keep the public runner import without loading the full pipeline eagerly.""" + if name == "run_tracks": + from trialmatchai.trec.runner import run_tracks + + return run_tracks + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +if TYPE_CHECKING: + from trialmatchai.trec.runner import run_tracks as run_tracks __all__ = ["TRACK_KEYS", "TrackSpec", "resolve_tracks", "run_tracks"] diff --git a/src/trialmatchai/trec/paper_reproduction.py b/src/trialmatchai/trec/paper_reproduction.py new file mode 100644 index 00000000..b06fb011 --- /dev/null +++ b/src/trialmatchai/trec/paper_reproduction.py @@ -0,0 +1,360 @@ +"""Audit the immutable result bundle published with the TrialMatchAI paper. + +This module deliberately separates three questions that are easy to conflate: + +* do the archived per-topic metrics aggregate to the archived summary; +* can those metrics be recalculated from the archived rankings; and +* what does the current evaluator report for the same rankings. + +It does not claim to rerun model inference. A fresh inference reproduction also +needs the historical corpus, Elasticsearch setup, model revisions, adapters and +GPU runtime used by the study. +""" + +from __future__ import annotations + +import json +import math +import shutil +import stat +import tempfile +import uuid +import zipfile +from collections.abc import Mapping, Sequence +from pathlib import Path, PurePosixPath +from statistics import mean, median + +import requests +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential + +from trialmatchai.trec.qrels import evaluate, parse_qrels +from trialmatchai.utils.file_utils import write_json_file +from trialmatchai.utils.integrity import sha256_file, verify_sha256 + +PAPER_RESULTS_URL = ( + "https://zenodo.org/records/15045515/files/matching_results.zip?download=1" +) +PAPER_RESULTS_SHA256 = "dbfd11f19ffbfd78acfeaeee634c3e7e7ca8cb4d72b526ff8d191f9e880312c8" +PAPER_RESULTS_FILENAME = "matching_results.zip" +TRACKS = { + "21": {"directory": "TREC21", "prefix": "trec-2021", "topics": 75}, + "22": {"directory": "TREC22", "prefix": "trec-2022", "topics": 50}, +} +QRELS_SHA256 = { + "21": "ba7a2cddc90285e75cd76adcd483394a6c9bacf7017113222058ba6537e6d8ac", + "22": "e569a531489e03f7b1fab03fe169c8ea66f4a59e8180fa9858b1a6e4bdcb0c5c", +} +RANKING_CUTOFFS = (5, 10, 20) +RECALL_CUTOFFS = (10, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000) +_REPRODUCTION_FILENAMES = { + "average_metrics.json", + "evaluation_metrics.json", + "ranked_trials.json", + "nct_ids.txt", + *(f"nct_ids_{cutoff}.txt" for cutoff in RECALL_CUTOFFS), +} + + +@retry( + retry=retry_if_exception_type(requests.RequestException), + stop=stop_after_attempt(4), + wait=wait_exponential(multiplier=1, min=1, max=20), + reraise=True, +) +def _download_once(url: str, destination: Path) -> None: + partial = destination.with_name(f".{destination.name}.part-{uuid.uuid4().hex}") + try: + with requests.get(url, stream=True, timeout=(15, 180)) as response: + response.raise_for_status() + with partial.open("wb") as output: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if chunk: + output.write(chunk) + partial.replace(destination) + finally: + partial.unlink(missing_ok=True) + + +def obtain_results_archive( + workdir: Path, + *, + archive: Path | None = None, + allow_download: bool = True, +) -> Path: + """Return a checksum-verified copy of the official matching-results archive.""" + workdir = Path(workdir).resolve() + workdir.mkdir(parents=True, exist_ok=True) + source = Path(archive).resolve() if archive is not None else workdir / PAPER_RESULTS_FILENAME + if source.is_symlink() or (source.exists() and not source.is_file()): + raise ValueError(f"Paper result archive must be a regular file: {source}") + if not source.exists(): + if archive is not None or not allow_download: + raise FileNotFoundError(f"Paper result archive not found: {source}") + _download_once(PAPER_RESULTS_URL, source) + try: + verify_sha256(source, PAPER_RESULTS_SHA256) + except ValueError: + if archive is not None or not allow_download: + raise + quarantine = source.with_name(f".{source.name}.corrupt-{uuid.uuid4().hex}") + source.replace(quarantine) + _download_once(PAPER_RESULTS_URL, source) + verify_sha256(source, PAPER_RESULTS_SHA256) + return source + + +def _validated_member(name: str) -> PurePosixPath: + path = PurePosixPath(name) + if ( + not name + or path.is_absolute() + or "\\" in name + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError(f"Archive contains an unsafe path: {name!r}") + return path + + +def extract_reproduction_files(archive: Path, workdir: Path) -> Path: + """Extract only files needed for metric reproduction, atomically and safely.""" + workdir = Path(workdir).resolve() + target = workdir / "matching_results" + marker = target / ".paper-results.json" + try: + state = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + state = {} + cache_complete = state.get("archive_sha256") == PAPER_RESULTS_SHA256 and all( + (target / str(spec["directory"]) / "average_metrics.json").is_file() + and len(list((target / str(spec["directory"])).glob("*/evaluation_metrics.json"))) + == int(spec["topics"]) + and len(list((target / str(spec["directory"])).glob("*/ranked_trials.json"))) + == int(spec["topics"]) + for spec in TRACKS.values() + ) + if cache_complete: + return target + + workdir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".paper-results-", dir=workdir) as temporary: + staging = Path(temporary) / "matching_results" + staging.mkdir() + extracted = 0 + with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + member_path = _validated_member(member.filename) + mode = member.external_attr >> 16 + if stat.S_ISLNK(mode): + raise ValueError(f"Archive contains a symbolic link: {member.filename}") + parts = member_path.parts + if member.is_dir() or len(parts) < 3 or parts[0] != "matching_results": + continue + if parts[1] not in {spec["directory"] for spec in TRACKS.values()}: + continue + if parts[-1] not in _REPRODUCTION_FILENAMES: + continue + relative = Path(*parts[1:]) + destination = staging / relative + destination.parent.mkdir(parents=True, exist_ok=True) + with bundle.open(member) as source, destination.open("wb") as output: + shutil.copyfileobj(source, output) + extracted += 1 + if extracted < sum(spec["topics"] for spec in TRACKS.values()) * 3: + raise ValueError("Paper archive is missing the expected TREC reproduction files") + write_json_file( + {"schema": 1, "archive_sha256": PAPER_RESULTS_SHA256, "files": extracted}, + str(staging / ".paper-results.json"), + ) + previous = None + if target.exists() or target.is_symlink(): + if target.is_symlink() or not target.is_dir(): + raise ValueError(f"Extraction target must be a directory: {target}") + previous = workdir / f".matching-results-previous-{uuid.uuid4().hex}" + target.replace(previous) + try: + staging.replace(target) + except BaseException: + if previous is not None and not target.exists(): + previous.replace(target) + raise + if previous is not None: + shutil.rmtree(previous) + return target + + +def _ranked_ids(path: Path) -> list[str]: + value = json.loads(path.read_text(encoding="utf-8")) + items = value.get("RankedTrials", []) if isinstance(value, dict) else value + if not isinstance(items, list): + raise ValueError(f"Invalid ranked trial payload: {path}") + return [str(item["TrialID"]) for item in items if isinstance(item, dict) and item.get("TrialID")] + + +def _dcg(grades: Sequence[int], cutoff: int) -> float: + return sum(grade / math.log2(rank + 2) for rank, grade in enumerate(grades[:cutoff])) + + +def paper_ranking_metrics( + ranked_ids: Sequence[str], judgments: Mapping[str, int] +) -> dict[str, float]: + """Recalculate the metric convention used by the published result files. + + Unjudged trials are removed before applying each cutoff. nDCG uses linear + grades and preserves the archived order within score ties. ``p@k`` is the + normalized graded precision ``sum(grade) / (2*k)``. + """ + grades = [int(judgments[trial]) for trial in ranked_ids if trial in judgments] + ideal = sorted((int(grade) for grade in judgments.values()), reverse=True) + metrics: dict[str, float] = {} + for cutoff in RANKING_CUTOFFS: + ideal_dcg = _dcg(ideal, cutoff) + metrics[f"ndcg@{cutoff}"] = _dcg(grades, cutoff) / ideal_dcg if ideal_dcg else 0.0 + metrics[f"p@{cutoff}"] = sum(grades[:cutoff]) / (2.0 * cutoff) + return metrics + + +def _summaries(rows: Sequence[Mapping[str, float]], keys: Sequence[str]) -> dict[str, dict]: + return { + key: {"mean": mean(float(row[key]) for row in rows), "median": median(float(row[key]) for row in rows)} + for key in keys + } + + +def _close(left: float, right: float, tolerance: float = 1e-12) -> bool: + return math.isclose(float(left), float(right), rel_tol=0.0, abs_tol=tolerance) + + +def audit_track(track: str, results_root: Path, qrels_path: Path) -> dict: + """Audit one archived TREC track against official qrels.""" + spec = TRACKS[track] + track_dir = Path(results_root) / str(spec["directory"]) + qrels = parse_qrels(Path(qrels_path), str(spec["prefix"])) + expected_topics = int(spec["topics"]) + if len(qrels) != expected_topics: + raise ValueError( + f"TREC {track} qrels contain {len(qrels)} topics; expected {expected_topics}" + ) + + metric_keys = tuple( + key for cutoff in RANKING_CUTOFFS for key in (f"ndcg@{cutoff}", f"p@{cutoff}") + ) + stored_rows: list[dict[str, float]] = [] + recalculated_rows: list[dict[str, float]] = [] + mismatch_topics: list[dict] = [] + recall_rows: list[dict[str, float]] = [] + + for query_id, judgments in qrels.items(): + patient_dir = track_dir / query_id + if not patient_dir.is_dir(): + raise ValueError(f"Paper archive is missing topic directory: {patient_dir}") + stored = json.loads((patient_dir / "evaluation_metrics.json").read_text(encoding="utf-8")) + ranked = _ranked_ids(patient_dir / "ranked_trials.json") + recalculated = paper_ranking_metrics(ranked, judgments) + stored_rows.append(stored) + recalculated_rows.append(recalculated) + differences = { + key: {"stored": float(stored[key]), "recalculated": recalculated[key]} + for key in metric_keys + if not _close(stored[key], recalculated[key]) + } + if differences: + mismatch_topics.append({"topic": query_id, "metrics": differences}) + + relevant = {trial for trial, grade in judgments.items() if grade >= 1} + recall_row = {} + for cutoff in RECALL_CUTOFFS: + ids_path = patient_dir / f"nct_ids_{cutoff}.txt" + retrieved = {line.strip() for line in ids_path.read_text().splitlines() if line.strip()} + recall_row[f"recall@{cutoff}"] = len(retrieved & relevant) / len(relevant) + recall_rows.append(recall_row) + + archived_summary = json.loads((track_dir / "average_metrics.json").read_text(encoding="utf-8")) + stored_summary = _summaries(stored_rows, metric_keys) + recalculated_summary = _summaries(recalculated_rows, metric_keys) + recall_keys = tuple(f"recall@{cutoff}" for cutoff in RECALL_CUTOFFS) + recall_summary = _summaries(recall_rows, recall_keys) + summary_mismatches = { + key: {"archived": float(archived_summary[key]), "aggregated": stored_summary[key]["mean"]} + for key in metric_keys + if not _close(archived_summary[key], stored_summary[key]["mean"]) + } + + current = evaluate(qrels, track_dir, cutoffs=RECALL_CUTOFFS) + current_keys = tuple(current["mean"]) + current_summary = { + key: { + "mean": current["mean"][key], + "median": median( + float(row[key]) for row in current["per_query"].values() if key in row + ), + } + for key in current_keys + if current["mean"][key] is not None + } + + return { + "track": f"TREC20{track}", + "topics": len(qrels), + "archived_summary": archived_summary, + "stored_per_topic": stored_summary, + "recalculated_from_rankings": recalculated_summary, + "paper_retrieval_recall": recall_summary, + "current_evaluator": current_summary, + "checks": { + "archived_summary_matches_stored_topics": not summary_mismatches, + "rankings_match_stored_topic_metrics": not mismatch_topics, + "summary_mismatches": summary_mismatches, + "ranking_mismatch_count": len(mismatch_topics), + "ranking_mismatches": mismatch_topics, + }, + } + + +def reproduce_paper_results(results_root: Path, qrels_dir: Path) -> dict: + for track, expected in QRELS_SHA256.items(): + verify_sha256(Path(qrels_dir) / f"qrels_{track}.txt", expected) + tracks = { + track: audit_track(track, results_root, Path(qrels_dir) / f"qrels_{track}.txt") + for track in TRACKS + } + total_topics = sum(item["topics"] for item in tracks.values()) + + def pooled(section: str, key: str) -> float: + return sum( + item[section][key]["mean"] * item["topics"] for item in tracks.values() + ) / total_topics + + pooled_means = { + "stored_per_topic": { + key: pooled("stored_per_topic", key) + for key in tracks["21"]["stored_per_topic"] + }, + "recalculated_from_rankings": { + key: pooled("recalculated_from_rankings", key) + for key in tracks["21"]["recalculated_from_rankings"] + }, + "current_evaluator": { + key: pooled("current_evaluator", key) + for key in tracks["21"]["current_evaluator"] + if key in tracks["22"]["current_evaluator"] + }, + } + return { + "schema": 1, + "scope": "published-result artifact audit; model inference was not rerun", + "paper": "https://doi.org/10.1038/s41467-026-70509-w", + "artifact": { + "url": PAPER_RESULTS_URL, + "sha256": PAPER_RESULTS_SHA256, + }, + "qrels_sha256": { + track: sha256_file(Path(qrels_dir) / f"qrels_{track}.txt") for track in TRACKS + }, + "status": ( + "verified_with_ranking_discrepancies" + if any(not item["checks"]["rankings_match_stored_topic_metrics"] for item in tracks.values()) + else "verified" + ), + "all_125_topics_weighted_mean": pooled_means, + "tracks": tracks, + } diff --git a/tests/test_paper_reproduction.py b/tests/test_paper_reproduction.py new file mode 100644 index 00000000..20511d7e --- /dev/null +++ b/tests/test_paper_reproduction.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import math +import subprocess +import sys +import zipfile + +import pytest + +from trialmatchai.trec import paper_reproduction as reproduction + + +def test_trec_package_keeps_runner_import_lazy(): + probe = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import trialmatchai.trec; " + "assert 'trialmatchai.trec.runner' not in sys.modules", + ], + check=False, + text=True, + capture_output=True, + ) + assert probe.returncode == 0, probe.stderr + + +def test_paper_archive_extraction_rejects_traversal(tmp_path): + archive = tmp_path / "unsafe.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("../outside.json", "{}") + + with pytest.raises(ValueError, match="unsafe path"): + reproduction.extract_reproduction_files(archive, tmp_path / "work") + assert not (tmp_path / "outside.json").exists() + + +def test_paper_ranking_metric_drops_unjudged_and_preserves_archived_order(): + metrics = reproduction.paper_ranking_metrics( + ["UNJUDGED", "NCT_GRADE_1", "NCT_GRADE_2"], + {"NCT_GRADE_1": 1, "NCT_GRADE_2": 2}, + ) + + dcg = 1 / math.log2(2) + 2 / math.log2(3) + ideal = 2 / math.log2(2) + 1 / math.log2(3) + assert metrics["ndcg@5"] == pytest.approx(dcg / ideal) + assert metrics["p@5"] == pytest.approx(3 / (2 * 5)) + + +def test_audit_track_separates_stored_metrics_from_ranking_recalculation( + tmp_path, monkeypatch +): + monkeypatch.setitem( + reproduction.TRACKS, + "21", + {"directory": "TREC21", "prefix": "trec-2021", "topics": 1}, + ) + qrels = tmp_path / "qrels_21.txt" + qrels.write_text("1 0 NCT1 2\n1 0 NCT2 1\n", encoding="utf-8") + topic = tmp_path / "results" / "TREC21" / "trec-20211" + topic.mkdir(parents=True) + + stored = reproduction.paper_ranking_metrics(["NCT1", "NCT2"], {"NCT1": 2, "NCT2": 1}) + (topic / "evaluation_metrics.json").write_text(json.dumps(stored), encoding="utf-8") + (topic.parent / "average_metrics.json").write_text(json.dumps(stored), encoding="utf-8") + # The archived ranking has the reverse order, so recomputation must expose a mismatch. + (topic / "ranked_trials.json").write_text( + json.dumps( + [ + {"TrialID": "NCT2", "Score": 1.0}, + {"TrialID": "NCT1", "Score": 1.0}, + ] + ), + encoding="utf-8", + ) + (topic / "nct_ids.txt").write_text("NCT2\nNCT1\n", encoding="utf-8") + for cutoff in reproduction.RECALL_CUTOFFS: + (topic / f"nct_ids_{cutoff}.txt").write_text("NCT2\nNCT1\n", encoding="utf-8") + + result = reproduction.audit_track("21", tmp_path / "results", qrels) + + assert result["checks"]["archived_summary_matches_stored_topics"] is True + assert result["checks"]["rankings_match_stored_topic_metrics"] is False + assert result["checks"]["ranking_mismatch_count"] == 1 + assert result["paper_retrieval_recall"]["recall@10"]["mean"] == 1.0 + # Current evaluation tie-averages the equal scores; the paper method preserves file order. + assert result["current_evaluator"]["ndcg@5"]["mean"] != pytest.approx( + result["recalculated_from_rankings"]["ndcg@5"]["mean"] + ) From e8b6cff72d2108a9c7f5f34db90fda5e4538babb Mon Sep 17 00:00:00 2001 From: Majd Abdallah Date: Sat, 19 Sep 2026 21:18:04 +0200 Subject: [PATCH 2/3] feat: compare TREC unjudged evaluation policies --- README.md | 8 + benchmarks/trec/unjudged-policy-trec21.json | 20408 ++++++++++++++++++ benchmarks/trec/unjudged-policy-trec22.json | 13808 ++++++++++++ docs/paper-reproduction.md | 10 + docs/trec-evaluation.md | 86 + mkdocs.yml | 1 + pyproject.toml | 2 +- src/trialmatchai/cli/main.py | 7 + src/trialmatchai/cli/reproduce_paper.py | 6 +- src/trialmatchai/cli/trec_evaluate.py | 160 + src/trialmatchai/trec/metrics.py | 42 +- src/trialmatchai/trec/paper_reproduction.py | 35 +- src/trialmatchai/trec/qrels.py | 71 +- tests/test_paper_reproduction.py | 9 +- tests/test_qrels_eval.py | 27 + tests/test_trec_evaluate_cli.py | 31 + 16 files changed, 34677 insertions(+), 34 deletions(-) create mode 100644 benchmarks/trec/unjudged-policy-trec21.json create mode 100644 benchmarks/trec/unjudged-policy-trec22.json create mode 100644 docs/trec-evaluation.md create mode 100644 src/trialmatchai/cli/trec_evaluate.py create mode 100644 tests/test_trec_evaluate_cli.py diff --git a/README.md b/README.md index d562f680..b5205c49 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ Run `trialmatchai --help` for the current options. | `report` | Render a patient report or the multi-patient front page | | `update-registry` | Fetch and upsert ClinicalTrials.gov studies; optional watch mode | | `trec` | Run TREC Clinical Trials evaluation presets | +| `trec-evaluate` | Re-score completed TREC rankings with explicit unjudged policies; no GPU inference | | `reproduce-paper` | Verify and recalculate the published TREC 2021/2022 result artifact | | `finetune` | Train supported reasoning, reranker, or NER components | | `healthcheck` | Inspect configured dependencies, paths, and services | @@ -273,6 +274,13 @@ The current evaluator and retrieval pipeline have open audit findings; this release does not claim a newly measured ranking improvement or a validated comparison against another system. +Completed rankings can be evaluated without repeating retrieval or model +inference. `trialmatchai trec-evaluate` reports both condensed metrics that +exclude unjudged trials and metrics that retain unjudged trials with gain zero. +See the [TREC evaluation guide](https://github.com/cbib/TrialMatchAI/blob/main/docs/trec-evaluation.md) +for commands, input fingerprints, and measured policy sensitivity across the +existing complete runs. + ## Reproduce the paper The published TREC result artifact can be audited on CPU with one command: diff --git a/benchmarks/trec/unjudged-policy-trec21.json b/benchmarks/trec/unjudged-policy-trec21.json new file mode 100644 index 00000000..266fd30f --- /dev/null +++ b/benchmarks/trec/unjudged-policy-trec21.json @@ -0,0 +1,20408 @@ +{ + "schema": 1, + "track": "TREC2021", + "qrels": { + "path": "data/trec/qrels/qrels_21.txt", + "sha256": "ba7a2cddc90285e75cd76adcd483394a6c9bacf7017113222058ba6537e6d8ac" + }, + "policies": { + "exclude": "remove unjudged trials before applying rank cutoffs", + "include_as_zero": "retain unjudged trials at their ranks with relevance grade zero" + }, + "runs": [ + { + "name": "h100_rrf", + "results_dir": "h100_rrf/results_trec21", + "evaluation_input_sha256": "7e77496e335c92920faa2e6afa97b8b3297d5f415bbe766d9130ab287b595551", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.07913867480634595, + "recall@50": 0.262806004824187, + "recall@100": 0.40253058229454475, + "recall@200": 0.5525817008151617, + "recall@300": 0.6354865277172483, + "recall@500": 0.7161949471889308, + "recall@1000": 0.7833828707617181, + "ndcg@5": 0.8007191901563339, + "ndcg@10": 0.790936519510314, + "ndcg@20": 0.7911113689434736, + "ndcg_full@5": 0.7994768421382488, + "ndcg_full@10": 0.7864313773750026, + "ndcg_full@20": 0.7635023626971176, + "P@10(rel>=1)": 0.8399999999999995, + "P@10(eligible)": 0.7106666666666664, + "graded_P@10": 0.7753333333333334, + "unjudged_fraction@10": 0.2626666666666666 + }, + "median": { + "recall@10": 0.045454545454545456, + "recall@50": 0.20625, + "recall@100": 0.34782608695652173, + "recall@200": 0.5108225108225108, + "recall@300": 0.6187683284457478, + "recall@500": 0.7293233082706767, + "recall@1000": 0.8181818181818182, + "ndcg@5": 0.8421757737602428, + "ndcg@10": 0.8007055007764234, + "ndcg@20": 0.8229801987513429, + "ndcg_full@5": 0.8421757737602428, + "ndcg_full@10": 0.8007055007764234, + "ndcg_full@20": 0.7774989681009015, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8, + "unjudged_fraction@10": 0.2 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 1000, + "num_ranked": 190, + "recall@10": 0.047337278106508875, + "recall@50": 0.24260355029585798, + "recall@100": 0.46153846153846156, + "recall@200": 0.7100591715976331, + "recall@300": 0.7692307692307693, + "recall@500": 0.7988165680473372, + "recall@1000": 0.8284023668639053, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7632636606403641, + "ndcg_full@5": 0.7632636606403641, + "ndcg@10": 0.7753647857722433, + "ndcg_full@10": 0.7753647857722433, + "ndcg@20": 0.7825008864559888, + "ndcg_full@20": 0.7825008864559888, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 1000, + "num_ranked": 247, + "recall@10": 0.03333333333333333, + "recall@50": 0.16296296296296298, + "recall@100": 0.2777777777777778, + "recall@200": 0.45555555555555555, + "recall@300": 0.6074074074074074, + "recall@500": 0.7074074074074074, + "recall@1000": 0.7962962962962963, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8368975253577313, + "ndcg_full@10": 0.8368975253577313, + "ndcg@20": 0.8414572735589413, + "ndcg_full@20": 0.8414572735589413, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 1000, + "num_ranked": 132, + "recall@10": 0.047619047619047616, + "recall@50": 0.27380952380952384, + "recall@100": 0.4642857142857143, + "recall@200": 0.6190476190476191, + "recall@300": 0.6904761904761905, + "recall@500": 0.7857142857142857, + "recall@1000": 0.8928571428571429, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.45608677546257415, + "ndcg_full@10": 0.45608677546257415, + "ndcg@20": 0.5297620999041701, + "ndcg_full@20": 0.5036340336548162, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.10526315789473684, + "recall@100": 0.24210526315789474, + "recall@200": 0.3263157894736842, + "recall@300": 0.42105263157894735, + "recall@500": 0.5578947368421052, + "recall@1000": 0.6526315789473685, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.7012236839470684, + "ndcg_full@5": 0.7012236839470684, + "ndcg@10": 0.6640955573742984, + "ndcg_full@10": 0.6640955573742984, + "ndcg@20": 0.6061961899180915, + "ndcg_full@20": 0.6061961899180915, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 1000, + "num_ranked": 233, + "recall@10": 0.014925373134328358, + "recall@50": 0.0845771144278607, + "recall@100": 0.18407960199004975, + "recall@200": 0.263681592039801, + "recall@300": 0.3283582089552239, + "recall@500": 0.42786069651741293, + "recall@1000": 0.5870646766169154, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.7014116859302243, + "ndcg_full@10": 0.7014116859302243, + "ndcg@20": 0.7530226719179621, + "ndcg_full@20": 0.7530226719179621, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.04285714285714286, + "recall@50": 0.1619047619047619, + "recall@100": 0.26666666666666666, + "recall@200": 0.4142857142857143, + "recall@300": 0.5523809523809524, + "recall@500": 0.7095238095238096, + "recall@1000": 0.8142857142857143, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.7887125008147068, + "ndcg_full@10": 0.7887125008147068, + "ndcg@20": 0.8295481148462642, + "ndcg_full@20": 0.8295481148462642, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.043478260869565216, + "recall@50": 0.12422360248447205, + "recall@100": 0.22981366459627328, + "recall@200": 0.33540372670807456, + "recall@300": 0.4409937888198758, + "recall@500": 0.6273291925465838, + "recall@1000": 0.8136645962732919, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6442227883107535, + "ndcg_full@10": 0.6442227883107535, + "ndcg@20": 0.7039917408453371, + "ndcg_full@20": 0.7039917408453371, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.21568627450980393, + "recall@100": 0.37254901960784315, + "recall@200": 0.6601307189542484, + "recall@300": 0.8431372549019608, + "recall@500": 0.9019607843137255, + "recall@1000": 0.9411764705882353, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.4777969189511402, + "ndcg_full@5": 0.4777969189511402, + "ndcg@10": 0.6611259319718193, + "ndcg_full@10": 0.6611259319718193, + "ndcg@20": 0.6452657761145029, + "ndcg_full@20": 0.6452657761145029, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.20093457943925233, + "recall@100": 0.3925233644859813, + "recall@200": 0.6542056074766355, + "recall@300": 0.8130841121495327, + "recall@500": 0.8878504672897196, + "recall@1000": 0.9439252336448598, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.565602538756171, + "ndcg_full@5": 0.565602538756171, + "ndcg@10": 0.5014060742501815, + "ndcg_full@10": 0.5014060742501815, + "ndcg@20": 0.5448828146590452, + "ndcg_full@20": 0.5091967261258555, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 1000, + "num_ranked": 66, + "recall@10": 0.19148936170212766, + "recall@50": 0.723404255319149, + "recall@100": 0.8297872340425532, + "recall@200": 0.8297872340425532, + "recall@300": 0.8297872340425532, + "recall@500": 0.851063829787234, + "recall@1000": 0.8723404255319149, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6887676151990412, + "ndcg_full@5": 0.6887676151990412, + "ndcg@10": 0.7221503826978086, + "ndcg_full@10": 0.7221503826978086, + "ndcg@20": 0.7638092890002919, + "ndcg_full@20": 0.7458489643009246, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 1000, + "num_ranked": 136, + "recall@10": 0.07936507936507936, + "recall@50": 0.35714285714285715, + "recall@100": 0.5396825396825397, + "recall@200": 0.7698412698412699, + "recall@300": 0.8015873015873016, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8412698412698413, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.8669798428057267, + "ndcg_full@20": 0.8669798428057267, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 1000, + "num_ranked": 149, + "recall@10": 0.053691275167785234, + "recall@50": 0.22818791946308725, + "recall@100": 0.4563758389261745, + "recall@200": 0.6577181208053692, + "recall@300": 0.7516778523489933, + "recall@500": 0.7919463087248322, + "recall@1000": 0.8590604026845637, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6975961695061491, + "ndcg_full@10": 0.6975961695061491, + "ndcg@20": 0.8048384417500536, + "ndcg_full@20": 0.8048384417500536, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 1000, + "num_ranked": 146, + "recall@10": 0.0, + "recall@50": 0.043478260869565216, + "recall@100": 0.10144927536231885, + "recall@200": 0.18115942028985507, + "recall@300": 0.2826086956521739, + "recall@500": 0.4492753623188406, + "recall@1000": 0.6376811594202898, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9032521000738538, + "ndcg_full@10": 0.9032521000738538, + "ndcg@20": 0.8524640152663249, + "ndcg_full@20": 0.7659543012978098, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 1000, + "num_ranked": 195, + "recall@10": 0.013605442176870748, + "recall@50": 0.05782312925170068, + "recall@100": 0.12244897959183673, + "recall@200": 0.20748299319727892, + "recall@300": 0.272108843537415, + "recall@500": 0.40476190476190477, + "recall@1000": 0.5306122448979592, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.861363286322476, + "ndcg_full@5": 0.861363286322476, + "ndcg@10": 0.8753186480260255, + "ndcg_full@10": 0.8753186480260255, + "ndcg@20": 0.8313009154165549, + "ndcg_full@20": 0.7761425512324938, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 1000, + "num_ranked": 168, + "recall@10": 0.019230769230769232, + "recall@50": 0.1423076923076923, + "recall@100": 0.2423076923076923, + "recall@200": 0.38846153846153847, + "recall@300": 0.43846153846153846, + "recall@500": 0.5615384615384615, + "recall@1000": 0.6884615384615385, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9608008655106622, + "ndcg_full@10": 0.9608008655106622, + "ndcg@20": 0.9206766624538901, + "ndcg_full@20": 0.9206766624538901, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.08695652173913043, + "recall@100": 0.18840579710144928, + "recall@200": 0.2971014492753623, + "recall@300": 0.391304347826087, + "recall@500": 0.5, + "recall@1000": 0.6159420289855072, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.033127111727194514, + "ndcg_full@10": 0.033127111727194514, + "ndcg@20": 0.14505171470181188, + "ndcg_full@20": 0.14505171470181188, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.035856573705179286, + "recall@50": 0.16733067729083664, + "recall@100": 0.3187250996015936, + "recall@200": 0.6055776892430279, + "recall@300": 0.8247011952191236, + "recall@500": 0.9282868525896414, + "recall@1000": 0.9641434262948207, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.4984338108996547, + "ndcg_full@10": 0.4984338108996547, + "ndcg@20": 0.5697131929114893, + "ndcg_full@20": 0.5697131929114893, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 1000, + "num_ranked": 236, + "recall@10": 0.03398058252427184, + "recall@50": 0.16990291262135923, + "recall@100": 0.2621359223300971, + "recall@200": 0.470873786407767, + "recall@300": 0.6407766990291263, + "recall@500": 0.8203883495145631, + "recall@1000": 0.8980582524271845, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5413905936110929, + "ndcg_full@5": 0.5413905936110929, + "ndcg@10": 0.6938761640430795, + "ndcg_full@10": 0.5992765032036573, + "ndcg@20": 0.7673995074232148, + "ndcg_full@20": 0.6274848640223715, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.013071895424836602, + "recall@50": 0.0915032679738562, + "recall@100": 0.16339869281045752, + "recall@200": 0.33986928104575165, + "recall@300": 0.40522875816993464, + "recall@500": 0.6274509803921569, + "recall@1000": 0.8627450980392157, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.8343572359526547, + "ndcg_full@20": 0.8343572359526547, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 1000, + "num_ranked": 172, + "recall@10": 0.14285714285714285, + "recall@50": 0.4714285714285714, + "recall@100": 0.7285714285714285, + "recall@200": 0.8571428571428571, + "recall@300": 0.9, + "recall@500": 0.9285714285714286, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.6766261041613127, + "ndcg_full@10": 0.6766261041613127, + "ndcg@20": 0.6831417147630375, + "ndcg_full@20": 0.6831417147630375, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 1000, + "num_ranked": 178, + "recall@10": 0.07547169811320754, + "recall@50": 0.5094339622641509, + "recall@100": 0.6792452830188679, + "recall@200": 0.8490566037735849, + "recall@300": 0.8679245283018868, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8389574121760602, + "ndcg_full@10": 0.8389574121760602, + "ndcg@20": 0.8129712837092171, + "ndcg_full@20": 0.8129712837092171, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 1000, + "num_ranked": 169, + "recall@10": 0.072992700729927, + "recall@50": 0.29927007299270075, + "recall@100": 0.5109489051094891, + "recall@200": 0.708029197080292, + "recall@300": 0.7737226277372263, + "recall@500": 0.8175182481751825, + "recall@1000": 0.8321167883211679, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0000000000000002, + "ndcg_full@5": 1.0000000000000002, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9832812543076433, + "ndcg_full@20": 0.9832812543076433, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 1000, + "num_ranked": 203, + "recall@10": 0.0299625468164794, + "recall@50": 0.13857677902621723, + "recall@100": 0.250936329588015, + "recall@200": 0.45692883895131087, + "recall@300": 0.6179775280898876, + "recall@500": 0.7715355805243446, + "recall@1000": 0.850187265917603, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9126316630185216, + "ndcg_full@20": 0.9126316630185216, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 1000, + "num_ranked": 181, + "recall@10": 0.04430379746835443, + "recall@50": 0.16455696202531644, + "recall@100": 0.31645569620253167, + "recall@200": 0.5253164556962026, + "recall@300": 0.6392405063291139, + "recall@500": 0.7468354430379747, + "recall@1000": 0.8227848101265823, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9648931575331846, + "ndcg_full@10": 0.9648931575331846, + "ndcg@20": 0.9209919775247694, + "ndcg_full@20": 0.9209919775247694, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.06862745098039216, + "recall@100": 0.17647058823529413, + "recall@200": 0.3431372549019608, + "recall@300": 0.46078431372549017, + "recall@500": 0.6176470588235294, + "recall@1000": 0.7254901960784313, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4759908431563517, + "ndcg_full@5": 0.4759908431563517, + "ndcg@10": 0.542750893882018, + "ndcg_full@10": 0.542750893882018, + "ndcg@20": 0.5672627510877496, + "ndcg_full@20": 0.5672627510877496, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 1000, + "num_ranked": 115, + "recall@10": 0.05, + "recall@50": 0.20625, + "recall@100": 0.38125, + "recall@200": 0.54375, + "recall@300": 0.625, + "recall@500": 0.70625, + "recall@1000": 0.7875, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.77603392959832, + "ndcg_full@10": 0.77603392959832, + "ndcg@20": 0.7317806757211407, + "ndcg_full@20": 0.7317806757211407, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 1000, + "num_ranked": 244, + "recall@10": 0.04291845493562232, + "recall@50": 0.14163090128755365, + "recall@100": 0.26180257510729615, + "recall@200": 0.45493562231759654, + "recall@300": 0.5836909871244635, + "recall@500": 0.8240343347639485, + "recall@1000": 0.9399141630901288, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5949016878400769, + "ndcg_full@5": 0.5949016878400769, + "ndcg@10": 0.6200055120319127, + "ndcg_full@10": 0.6200055120319127, + "ndcg@20": 0.6083480840531117, + "ndcg_full@20": 0.6083480840531117, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 1000, + "num_ranked": 234, + "recall@10": 0.018518518518518517, + "recall@50": 0.1111111111111111, + "recall@100": 0.22839506172839505, + "recall@200": 0.42592592592592593, + "recall@300": 0.5802469135802469, + "recall@500": 0.7098765432098766, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.860099657059182, + "ndcg_full@20": 0.860099657059182, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.031007751937984496, + "recall@50": 0.13565891472868216, + "recall@100": 0.22093023255813954, + "recall@200": 0.38372093023255816, + "recall@300": 0.5193798449612403, + "recall@500": 0.6434108527131783, + "recall@1000": 0.7248062015503876, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6386367136775242, + "ndcg_full@5": 0.6386367136775242, + "ndcg@10": 0.7654999533396626, + "ndcg_full@10": 0.7654999533396626, + "ndcg@20": 0.7598651656401886, + "ndcg_full@20": 0.7598651656401886, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 1000, + "num_ranked": 75, + "recall@10": 0.06, + "recall@50": 0.29333333333333333, + "recall@100": 0.5, + "recall@200": 0.66, + "recall@300": 0.7066666666666667, + "recall@500": 0.7533333333333333, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.8899701444399659, + "ndcg_full@20": 0.8899701444399659, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 1000, + "num_ranked": 174, + "recall@10": 0.03488372093023256, + "recall@50": 0.13372093023255813, + "recall@100": 0.21511627906976744, + "recall@200": 0.37209302325581395, + "recall@300": 0.46511627906976744, + "recall@500": 0.5755813953488372, + "recall@1000": 0.7209302325581395, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.3274584397097874, + "ndcg_full@10": 0.3274584397097874, + "ndcg@20": 0.3931518132796809, + "ndcg_full@20": 0.3931518132796809, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 1000, + "num_ranked": 67, + "recall@10": 0.09433962264150944, + "recall@50": 0.24528301886792453, + "recall@100": 0.2830188679245283, + "recall@200": 0.33962264150943394, + "recall@300": 0.41509433962264153, + "recall@500": 0.4528301886792453, + "recall@1000": 0.49056603773584906, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.728748749097909, + "ndcg_full@10": 0.728748749097909, + "ndcg@20": 0.8112477674670605, + "ndcg_full@20": 0.6155482446719344, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 1000, + "num_ranked": 27, + "recall@10": 0.0430622009569378, + "recall@50": 0.15789473684210525, + "recall@100": 0.2727272727272727, + "recall@200": 0.3875598086124402, + "recall@300": 0.4784688995215311, + "recall@500": 0.5789473684210527, + "recall@1000": 0.6698564593301436, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.5617131059245698, + "ndcg_full@10": 0.5617131059245698, + "ndcg@20": 0.796724598120624, + "ndcg_full@20": 0.5141800413419203, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 1000, + "num_ranked": 218, + "recall@10": 0.04, + "recall@50": 0.14, + "recall@100": 0.22, + "recall@200": 0.37333333333333335, + "recall@300": 0.4666666666666667, + "recall@500": 0.5666666666666667, + "recall@1000": 0.6733333333333333, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.968189605900524, + "ndcg_full@10": 0.968189605900524, + "ndcg@20": 0.9255971045226652, + "ndcg_full@20": 0.9255971045226652, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 1000, + "num_ranked": 84, + "recall@10": 0.05172413793103448, + "recall@50": 0.25287356321839083, + "recall@100": 0.4367816091954023, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.9241910479142424, + "ndcg_full@20": 0.9241910479142424, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.022653721682847898, + "recall@50": 0.0744336569579288, + "recall@100": 0.12944983818770225, + "recall@200": 0.24271844660194175, + "recall@300": 0.31715210355987056, + "recall@500": 0.4368932038834951, + "recall@1000": 0.5728155339805825, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.911900180951869, + "ndcg_full@20": 0.911900180951869, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.04032258064516129, + "recall@50": 0.1935483870967742, + "recall@100": 0.375, + "recall@200": 0.6895161290322581, + "recall@300": 0.8669354838709677, + "recall@500": 0.9112903225806451, + "recall@1000": 0.9314516129032258, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 1000, + "num_ranked": 104, + "recall@10": 0.12345679012345678, + "recall@50": 0.5555555555555556, + "recall@100": 0.9382716049382716, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9631344311641977, + "ndcg_full@10": 0.9631344311641977, + "ndcg@20": 0.9030733547267193, + "ndcg_full@20": 0.9030733547267193, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 1000, + "num_ranked": 81, + "recall@10": 0.1016949152542373, + "recall@50": 0.5254237288135594, + "recall@100": 0.7796610169491526, + "recall@200": 0.8813559322033898, + "recall@300": 0.9152542372881356, + "recall@500": 0.9322033898305084, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7477870352439605, + "ndcg_full@10": 0.7477870352439605, + "ndcg@20": 0.7829463377098402, + "ndcg_full@20": 0.7702867643986545, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 1000, + "num_ranked": 51, + "recall@10": 0.6923076923076923, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.800693766409882, + "ndcg_full@10": 0.800693766409882, + "ndcg@20": 0.9095918738743377, + "ndcg_full@20": 0.837039077084839, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.02922077922077922, + "recall@50": 0.12012987012987013, + "recall@100": 0.22402597402597402, + "recall@200": 0.41883116883116883, + "recall@300": 0.577922077922078, + "recall@500": 0.724025974025974, + "recall@1000": 0.827922077922078, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9722726572644951, + "ndcg_full@5": 0.9722726572644951, + "ndcg@10": 0.9598875104974018, + "ndcg_full@10": 0.9598875104974018, + "ndcg@20": 0.9196199076274632, + "ndcg_full@20": 0.9196199076274632, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 1000, + "num_ranked": 95, + "recall@10": 0.08450704225352113, + "recall@50": 0.23943661971830985, + "recall@100": 0.36619718309859156, + "recall@200": 0.4788732394366197, + "recall@300": 0.5070422535211268, + "recall@500": 0.5774647887323944, + "recall@1000": 0.6197183098591549, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.7984599822057986, + "ndcg_full@20": 0.7984599822057986, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 1000, + "num_ranked": 150, + "recall@10": 0.06451612903225806, + "recall@50": 0.25161290322580643, + "recall@100": 0.44516129032258067, + "recall@200": 0.6645161290322581, + "recall@300": 0.7354838709677419, + "recall@500": 0.8580645161290322, + "recall@1000": 0.9161290322580645, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4374130297304557, + "ndcg_full@5": 0.4374130297304557, + "ndcg@10": 0.5590384555827961, + "ndcg_full@10": 0.5590384555827961, + "ndcg@20": 0.608556007420607, + "ndcg_full@20": 0.608556007420607, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 1000, + "num_ranked": 128, + "recall@10": 0.05128205128205128, + "recall@50": 0.24615384615384617, + "recall@100": 0.46153846153846156, + "recall@200": 0.676923076923077, + "recall@300": 0.7230769230769231, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7384615384615385, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6720882144762628, + "ndcg_full@5": 0.5789121131198787, + "ndcg@10": 0.7504540869478651, + "ndcg_full@10": 0.6382203881849413, + "ndcg@20": 0.8731539043632495, + "ndcg_full@20": 0.774085840199104, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.1, + "graded_P@10": 0.55 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 1000, + "num_ranked": 210, + "recall@10": 0.053763440860215055, + "recall@50": 0.24731182795698925, + "recall@100": 0.43548387096774194, + "recall@200": 0.7526881720430108, + "recall@300": 0.8602150537634409, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.9551913563983283, + "ndcg_full@20": 0.9551913563983283, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 1000, + "num_ranked": 164, + "recall@10": 0.06306306306306306, + "recall@50": 0.3153153153153153, + "recall@100": 0.5315315315315315, + "recall@200": 0.8738738738738738, + "recall@300": 0.954954954954955, + "recall@500": 0.990990990990991, + "recall@1000": 0.990990990990991, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.9120975666589385, + "ndcg_full@20": 0.9120975666589385, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 1000, + "num_ranked": 243, + "recall@10": 0.03759398496240601, + "recall@50": 0.16917293233082706, + "recall@100": 0.2894736842105263, + "recall@200": 0.4548872180451128, + "recall@300": 0.5902255639097744, + "recall@500": 0.7293233082706767, + "recall@1000": 0.8270676691729323, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9328156361182762, + "ndcg_full@10": 0.9328156361182762, + "ndcg@20": 0.9175014582119495, + "ndcg_full@20": 0.9175014582119495, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 1000, + "num_ranked": 57, + "recall@10": 0.15151515151515152, + "recall@50": 0.6818181818181818, + "recall@100": 0.803030303030303, + "recall@200": 0.8181818181818182, + "recall@300": 0.8333333333333334, + "recall@500": 0.8333333333333334, + "recall@1000": 0.8484848484848485, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.9356218339941231, + "ndcg_full@20": 0.9356218339941231, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 1000, + "num_ranked": 105, + "recall@10": 0.12987012987012986, + "recall@50": 0.4805194805194805, + "recall@100": 0.6623376623376623, + "recall@200": 0.7792207792207793, + "recall@300": 0.7792207792207793, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8690959628864622, + "ndcg_full@10": 0.8690959628864622, + "ndcg@20": 0.8823181769631833, + "ndcg_full@20": 0.8823181769631833, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 1000, + "num_ranked": 45, + "recall@10": 0.391304347826087, + "recall@50": 0.7391304347826086, + "recall@100": 0.782608695652174, + "recall@200": 0.782608695652174, + "recall@300": 0.782608695652174, + "recall@500": 0.782608695652174, + "recall@1000": 0.782608695652174, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6187490087643146, + "ndcg_full@5": 0.6187490087643146, + "ndcg@10": 0.6149183319922086, + "ndcg_full@10": 0.5069791269361719, + "ndcg@20": 0.7765795572735829, + "ndcg_full@20": 0.4769959493243629, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 1000, + "num_ranked": 87, + "recall@10": 0.11267605633802817, + "recall@50": 0.4225352112676056, + "recall@100": 0.6619718309859155, + "recall@200": 0.8309859154929577, + "recall@300": 0.8732394366197183, + "recall@500": 0.8732394366197183, + "recall@1000": 0.8873239436619719, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8790301722005143, + "ndcg_full@10": 0.8790301722005143, + "ndcg@20": 0.8523086985766511, + "ndcg_full@20": 0.7658147464492903, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 1000, + "num_ranked": 67, + "recall@10": 0.10294117647058823, + "recall@50": 0.38235294117647056, + "recall@100": 0.5, + "recall@200": 0.6470588235294118, + "recall@300": 0.7205882352941176, + "recall@500": 0.7794117647058824, + "recall@1000": 0.8088235294117647, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7559632399276095, + "ndcg_full@10": 0.7559632399276095, + "ndcg@20": 0.703195971056409, + "ndcg_full@20": 0.6443195178312635, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 1000, + "num_ranked": 143, + "recall@10": 0.06521739130434782, + "recall@50": 0.1956521739130435, + "recall@100": 0.34782608695652173, + "recall@200": 0.5869565217391305, + "recall@300": 0.6304347826086957, + "recall@500": 0.717391304347826, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.872443023559078, + "ndcg_full@10": 0.872443023559078, + "ndcg@20": 0.7603712995742892, + "ndcg_full@20": 0.7603712995742892, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 1000, + "num_ranked": 106, + "recall@10": 0.10666666666666667, + "recall@50": 0.5066666666666667, + "recall@100": 0.84, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.5397882006722999, + "ndcg_full@10": 0.5397882006722999, + "ndcg@20": 0.5468946563947155, + "ndcg_full@20": 0.5468946563947155, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 1000, + "num_ranked": 80, + "recall@10": 0.20512820512820512, + "recall@50": 0.5641025641025641, + "recall@100": 0.6923076923076923, + "recall@200": 0.717948717948718, + "recall@300": 0.7435897435897436, + "recall@500": 0.8205128205128205, + "recall@1000": 0.8461538461538461, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8211303807792082, + "ndcg_full@5": 0.8211303807792082, + "ndcg@10": 0.8007055007764234, + "ndcg_full@10": 0.8007055007764234, + "ndcg@20": 0.7644517632828929, + "ndcg_full@20": 0.7644517632828929, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 1000, + "num_ranked": 178, + "recall@10": 0.10465116279069768, + "recall@50": 0.37209302325581395, + "recall@100": 0.6395348837209303, + "recall@200": 0.9302325581395349, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.94497705842548, + "ndcg_full@10": 0.94497705842548, + "ndcg@20": 0.8229801987513429, + "ndcg_full@20": 0.8229801987513429, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 1000, + "num_ranked": 112, + "recall@10": 0.04938271604938271, + "recall@50": 0.21604938271604937, + "recall@100": 0.36419753086419754, + "recall@200": 0.5246913580246914, + "recall@300": 0.5802469135802469, + "recall@500": 0.6481481481481481, + "recall@1000": 0.7037037037037037, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.9526058178152246, + "ndcg_full@10": 0.9526058178152246, + "ndcg@20": 0.9176965360163978, + "ndcg_full@20": 0.9176965360163978, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 1000, + "num_ranked": 93, + "recall@10": 0.07352941176470588, + "recall@50": 0.33088235294117646, + "recall@100": 0.5661764705882353, + "recall@200": 0.6544117647058824, + "recall@300": 0.6764705882352942, + "recall@500": 0.6764705882352942, + "recall@1000": 0.6838235294117647, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.9173950256592021, + "ndcg_full@5": 0.9173950256592021, + "ndcg@10": 0.946395023870779, + "ndcg_full@10": 0.946395023870779, + "ndcg@20": 0.9467517235080039, + "ndcg_full@20": 0.9467517235080039, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1000, + "num_ranked": 165, + "recall@10": 0.02564102564102564, + "recall@50": 0.1339031339031339, + "recall@100": 0.23931623931623933, + "recall@200": 0.4586894586894587, + "recall@300": 0.6153846153846154, + "recall@500": 0.6296296296296297, + "recall@1000": 0.6324786324786325, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.9112105684172009, + "ndcg_full@20": 0.9112105684172009, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 1000, + "num_ranked": 166, + "recall@10": 0.035175879396984924, + "recall@50": 0.1507537688442211, + "recall@100": 0.32663316582914576, + "recall@200": 0.48743718592964824, + "recall@300": 0.6030150753768844, + "recall@500": 0.6633165829145728, + "recall@1000": 0.8090452261306532, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 1.0000000000000002, + "ndcg_full@20": 1.0000000000000002, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.015789473684210527, + "recall@50": 0.06315789473684211, + "recall@100": 0.12105263157894737, + "recall@200": 0.24210526315789474, + "recall@300": 0.4, + "recall@500": 0.5526315789473685, + "recall@1000": 0.6789473684210526, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8633297260334926, + "ndcg_full@20": 0.8633297260334926, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 1000, + "num_ranked": 126, + "recall@10": 0.03636363636363636, + "recall@50": 0.15636363636363637, + "recall@100": 0.27636363636363637, + "recall@200": 0.44727272727272727, + "recall@300": 0.5490909090909091, + "recall@500": 0.6436363636363637, + "recall@1000": 0.7418181818181818, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8554853481239506, + "ndcg_full@5": 0.8554853481239506, + "ndcg@10": 0.7638127786926596, + "ndcg_full@10": 0.7638127786926596, + "ndcg@20": 0.7774989681009015, + "ndcg_full@20": 0.7774989681009015, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 1000, + "num_ranked": 133, + "recall@10": 0.045454545454545456, + "recall@50": 0.29545454545454547, + "recall@100": 0.45454545454545453, + "recall@200": 0.5, + "recall@300": 0.5681818181818182, + "recall@500": 0.5909090909090909, + "recall@1000": 0.7272727272727273, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.7166953784267104, + "ndcg_full@5": 0.7166953784267104, + "ndcg@10": 0.7427907635242925, + "ndcg_full@10": 0.7427907635242925, + "ndcg@20": 0.6388141504177526, + "ndcg_full@20": 0.6284850714248146, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.024630541871921183, + "recall@50": 0.08866995073891626, + "recall@100": 0.20689655172413793, + "recall@200": 0.3891625615763547, + "recall@300": 0.5123152709359606, + "recall@500": 0.7438423645320197, + "recall@1000": 0.8817733990147784, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7854294331056967, + "ndcg_full@10": 0.7854294331056967, + "ndcg@20": 0.7430570042046561, + "ndcg_full@20": 0.7430570042046561, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 1000, + "num_ranked": 230, + "recall@10": 0.017699115044247787, + "recall@50": 0.09734513274336283, + "recall@100": 0.20353982300884957, + "recall@200": 0.34513274336283184, + "recall@300": 0.4778761061946903, + "recall@500": 0.6460176991150443, + "recall@1000": 0.8053097345132744, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6802551169068608, + "ndcg_full@10": 0.6802551169068608, + "ndcg@20": 0.6543726226509218, + "ndcg_full@20": 0.6543726226509218, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 1000, + "num_ranked": 44, + "recall@10": 0.10416666666666667, + "recall@50": 0.20833333333333334, + "recall@100": 0.22916666666666666, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.7967610662472993, + "ndcg_full@10": 0.7967610662472993, + "ndcg@20": 0.8520839135688071, + "ndcg_full@20": 0.7121960847579404, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 1000, + "num_ranked": 151, + "recall@10": 0.06369426751592357, + "recall@50": 0.25477707006369427, + "recall@100": 0.43312101910828027, + "recall@200": 0.6815286624203821, + "recall@300": 0.7579617834394905, + "recall@500": 0.802547770700637, + "recall@1000": 0.821656050955414, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.9285714285714286, + "ndcg_full@5": 0.9285714285714286, + "ndcg@10": 0.9428075881135799, + "ndcg_full@10": 0.9428075881135799, + "ndcg@20": 0.8955320813352615, + "ndcg_full@20": 0.8955320813352615, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 1000, + "num_ranked": 141, + "recall@10": 0.06097560975609756, + "recall@50": 0.23170731707317074, + "recall@100": 0.3597560975609756, + "recall@200": 0.43902439024390244, + "recall@300": 0.47560975609756095, + "recall@500": 0.5304878048780488, + "recall@1000": 0.6097560975609756, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.7221864957468423, + "ndcg_full@10": 0.7221864957468423, + "ndcg@20": 0.6885893416988016, + "ndcg_full@20": 0.6429001562566643, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.0, + "recall@50": 0.02531645569620253, + "recall@100": 0.06329113924050633, + "recall@200": 0.1518987341772152, + "recall@300": 0.22784810126582278, + "recall@500": 0.34177215189873417, + "recall@1000": 0.4050632911392405, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.4690000933206748, + "ndcg_full@10": 0.4690000933206748, + "ndcg@20": 0.6017947512515609, + "ndcg_full@20": 0.4118503949172273, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 1000, + "num_ranked": 234, + "recall@10": 0.03896103896103896, + "recall@50": 0.1341991341991342, + "recall@100": 0.2554112554112554, + "recall@200": 0.5108225108225108, + "recall@300": 0.670995670995671, + "recall@500": 0.7922077922077922, + "recall@1000": 0.8398268398268398, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8587926983189565, + "ndcg_full@20": 0.8587926983189565, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.1358695652173913, + "recall@100": 0.20652173913043478, + "recall@200": 0.32065217391304346, + "recall@300": 0.44565217391304346, + "recall@500": 0.5978260869565217, + "recall@1000": 0.7228260869565217, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.8920218064796811, + "ndcg_full@20": 0.8920218064796811, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 1000, + "num_ranked": 51, + "recall@10": 0.08791208791208792, + "recall@50": 0.21978021978021978, + "recall@100": 0.24175824175824176, + "recall@200": 0.25274725274725274, + "recall@300": 0.26373626373626374, + "recall@500": 0.27472527472527475, + "recall@1000": 0.31868131868131866, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7265894102945099, + "ndcg_full@10": 0.7034763148045359, + "ndcg@20": 0.7871324361969577, + "ndcg_full@20": 0.675207457634565, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 1000, + "num_ranked": 86, + "recall@10": 0.12121212121212122, + "recall@50": 0.4696969696969697, + "recall@100": 0.7121212121212122, + "recall@200": 0.8484848484848485, + "recall@300": 0.9090909090909091, + "recall@500": 0.9545454545454546, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8611375561264454, + "ndcg_full@10": 0.8611375561264454, + "ndcg@20": 0.8154384621103938, + "ndcg_full@20": 0.7326861736234048, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 1000, + "num_ranked": 51, + "recall@10": 0.6923076923076923, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7677078645873638, + "ndcg_full@10": 0.7677078645873638, + "ndcg@20": 0.8258589937101908, + "ndcg_full@20": 0.8045469700388569, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.10557184750733138, + "recall@100": 0.22580645161290322, + "recall@200": 0.4222873900293255, + "recall@300": 0.6187683284457478, + "recall@500": 0.7712609970674487, + "recall@1000": 0.8651026392961877, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9840948029502621, + "ndcg_full@10": 0.9840948029502621, + "ndcg@20": 0.9520877121931672, + "ndcg_full@20": 0.9520877121931672, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.07913867480634595, + "recall@50": 0.262806004824187, + "recall@100": 0.40253058229454475, + "recall@200": 0.5525817008151617, + "recall@300": 0.6354865277172483, + "recall@500": 0.7161949471889308, + "recall@1000": 0.7833828707617181, + "ndcg@5": 0.6246179095971478, + "ndcg@10": 0.6083436732336542, + "ndcg@20": 0.5962571940330675, + "ndcg_full@5": 0.6233755615790627, + "ndcg_full@10": 0.6049301034488778, + "ndcg_full@20": 0.5768530577307897, + "P@10(rel>=1)": 0.6373333333333333, + "P@10(eligible)": 0.536, + "graded_P@10": 0.5866666666666668, + "unjudged_fraction@10": 0.2626666666666666 + }, + "median": { + "recall@10": 0.045454545454545456, + "recall@50": 0.20625, + "recall@100": 0.34782608695652173, + "recall@200": 0.5108225108225108, + "recall@300": 0.6187683284457478, + "recall@500": 0.7293233082706767, + "recall@1000": 0.8181818181818182, + "ndcg@5": 0.6548086577531307, + "ndcg@10": 0.6538571657758918, + "ndcg@20": 0.5998376854531831, + "ndcg_full@5": 0.6548086577531307, + "ndcg_full@10": 0.6442227883107535, + "ndcg_full@20": 0.5810169801605163, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6, + "unjudged_fraction@10": 0.2 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 1000, + "num_ranked": 190, + "recall@10": 0.047337278106508875, + "recall@50": 0.24260355029585798, + "recall@100": 0.46153846153846156, + "recall@200": 0.7100591715976331, + "recall@300": 0.7692307692307693, + "recall@500": 0.7988165680473372, + "recall@1000": 0.8284023668639053, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7632636606403641, + "ndcg_full@5": 0.7632636606403641, + "ndcg@10": 0.7753647857722433, + "ndcg_full@10": 0.7753647857722433, + "ndcg@20": 0.7490633950712754, + "ndcg_full@20": 0.7490633950712754, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 1000, + "num_ranked": 247, + "recall@10": 0.03333333333333333, + "recall@50": 0.16296296296296298, + "recall@100": 0.2777777777777778, + "recall@200": 0.45555555555555555, + "recall@300": 0.6074074074074074, + "recall@500": 0.7074074074074074, + "recall@1000": 0.7962962962962963, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.697116599874871, + "ndcg_full@10": 0.697116599874871, + "ndcg@20": 0.7534684645572371, + "ndcg_full@20": 0.7534684645572371, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 1000, + "num_ranked": 132, + "recall@10": 0.047619047619047616, + "recall@50": 0.27380952380952384, + "recall@100": 0.4642857142857143, + "recall@200": 0.6190476190476191, + "recall@300": 0.6904761904761905, + "recall@500": 0.7857142857142857, + "recall@1000": 0.8928571428571429, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.404762744029787, + "ndcg_full@5": 0.404762744029787, + "ndcg@10": 0.36548324780718, + "ndcg_full@10": 0.36548324780718, + "ndcg@20": 0.3421106194105923, + "ndcg_full@20": 0.3252375948394038, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.10526315789473684, + "recall@100": 0.24210526315789474, + "recall@200": 0.3263157894736842, + "recall@300": 0.42105263157894735, + "recall@500": 0.5578947368421052, + "recall@1000": 0.6526315789473685, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.23561034379970078, + "ndcg_full@10": 0.23561034379970078, + "ndcg@20": 0.19043986101313123, + "ndcg_full@20": 0.19043986101313123, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 1000, + "num_ranked": 233, + "recall@10": 0.014925373134328358, + "recall@50": 0.0845771144278607, + "recall@100": 0.18407960199004975, + "recall@200": 0.263681592039801, + "recall@300": 0.3283582089552239, + "recall@500": 0.42786069651741293, + "recall@1000": 0.5870646766169154, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.33013764944712026, + "ndcg_full@10": 0.33013764944712026, + "ndcg@20": 0.35321434816425606, + "ndcg_full@20": 0.35321434816425606, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.04285714285714286, + "recall@50": 0.1619047619047619, + "recall@100": 0.26666666666666666, + "recall@200": 0.4142857142857143, + "recall@300": 0.5523809523809524, + "recall@500": 0.7095238095238096, + "recall@1000": 0.8142857142857143, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.19178318368566782, + "ndcg_full@5": 0.19178318368566782, + "ndcg@10": 0.24110975616177097, + "ndcg_full@10": 0.24110975616177097, + "ndcg@20": 0.3600868923102385, + "ndcg_full@20": 0.3600868923102385, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.043478260869565216, + "recall@50": 0.12422360248447205, + "recall@100": 0.22981366459627328, + "recall@200": 0.33540372670807456, + "recall@300": 0.4409937888198758, + "recall@500": 0.6273291925465838, + "recall@1000": 0.8136645962732919, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6442227883107535, + "ndcg_full@10": 0.6442227883107535, + "ndcg@20": 0.5924467421836522, + "ndcg_full@20": 0.5924467421836522, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.21568627450980393, + "recall@100": 0.37254901960784315, + "recall@200": 0.6601307189542484, + "recall@300": 0.8431372549019608, + "recall@500": 0.9019607843137255, + "recall@1000": 0.9411764705882353, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.404762744029787, + "ndcg_full@5": 0.404762744029787, + "ndcg@10": 0.2973789360872793, + "ndcg_full@10": 0.2973789360872793, + "ndcg@20": 0.5124883347545613, + "ndcg_full@20": 0.5124883347545613, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.1, + "graded_P@10": 0.2 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.20093457943925233, + "recall@100": 0.3925233644859813, + "recall@200": 0.6542056074766355, + "recall@300": 0.8130841121495327, + "recall@500": 0.8878504672897196, + "recall@1000": 0.9439252336448598, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4343974612438291, + "ndcg_full@5": 0.4343974612438291, + "ndcg@10": 0.458357115377653, + "ndcg_full@10": 0.458357115377653, + "ndcg@20": 0.5318201067815951, + "ndcg_full@20": 0.49698953605380647, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 1000, + "num_ranked": 66, + "recall@10": 0.19148936170212766, + "recall@50": 0.723404255319149, + "recall@100": 0.8297872340425532, + "recall@200": 0.8297872340425532, + "recall@300": 0.8297872340425532, + "recall@500": 0.851063829787234, + "recall@1000": 0.8723404255319149, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6813359790338589, + "ndcg_full@5": 0.6813359790338589, + "ndcg@10": 0.5755702911805343, + "ndcg_full@10": 0.5755702911805343, + "ndcg@20": 0.7158524319443089, + "ndcg_full@20": 0.6990197718815081, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 1000, + "num_ranked": 136, + "recall@10": 0.07936507936507936, + "recall@50": 0.35714285714285715, + "recall@100": 0.5396825396825397, + "recall@200": 0.7698412698412699, + "recall@300": 0.8015873015873016, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8412698412698413, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8701249883466592, + "ndcg_full@10": 0.8701249883466592, + "ndcg@20": 0.7957198923066257, + "ndcg_full@20": 0.7957198923066257, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 1000, + "num_ranked": 149, + "recall@10": 0.053691275167785234, + "recall@50": 0.22818791946308725, + "recall@100": 0.4563758389261745, + "recall@200": 0.6577181208053692, + "recall@300": 0.7516778523489933, + "recall@500": 0.7919463087248322, + "recall@1000": 0.8590604026845637, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6975961695061491, + "ndcg_full@10": 0.6975961695061491, + "ndcg@20": 0.8048384417500536, + "ndcg_full@20": 0.8048384417500536, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 1000, + "num_ranked": 146, + "recall@10": 0.0, + "recall@50": 0.043478260869565216, + "recall@100": 0.10144927536231885, + "recall@200": 0.18115942028985507, + "recall@300": 0.2826086956521739, + "recall@500": 0.4492753623188406, + "recall@1000": 0.6376811594202898, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.727329844310522, + "ndcg_full@10": 0.727329844310522, + "ndcg@20": 0.62299997157158, + "ndcg_full@20": 0.5597767171258045, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 1000, + "num_ranked": 195, + "recall@10": 0.013605442176870748, + "recall@50": 0.05782312925170068, + "recall@100": 0.12244897959183673, + "recall@200": 0.20748299319727892, + "recall@300": 0.272108843537415, + "recall@500": 0.40476190476190477, + "recall@1000": 0.5306122448979592, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.4777969189511402, + "ndcg_full@5": 0.4777969189511402, + "ndcg@10": 0.31005750730366605, + "ndcg_full@10": 0.31005750730366605, + "ndcg@20": 0.32714955537944124, + "ndcg_full@20": 0.3054425730056389, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 1000, + "num_ranked": 168, + "recall@10": 0.019230769230769232, + "recall@50": 0.1423076923076923, + "recall@100": 0.2423076923076923, + "recall@200": 0.38846153846153847, + "recall@300": 0.43846153846153846, + "recall@500": 0.5615384615384615, + "recall@1000": 0.6884615384615385, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.666794353754856, + "ndcg_full@10": 0.666794353754856, + "ndcg@20": 0.7146997729836069, + "ndcg_full@20": 0.7146997729836069, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.08695652173913043, + "recall@100": 0.18840579710144928, + "recall@200": 0.2971014492753623, + "recall@300": 0.391304347826087, + "recall@500": 0.5, + "recall@1000": 0.6159420289855072, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.01616914557415991, + "ndcg_full@20": 0.01616914557415991, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.035856573705179286, + "recall@50": 0.16733067729083664, + "recall@100": 0.3187250996015936, + "recall@200": 0.6055776892430279, + "recall@300": 0.8247011952191236, + "recall@500": 0.9282868525896414, + "recall@1000": 0.9641434262948207, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.44202197604759985, + "ndcg_full@10": 0.44202197604759985, + "ndcg@20": 0.44378552800733956, + "ndcg_full@20": 0.44378552800733956, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 1000, + "num_ranked": 236, + "recall@10": 0.03398058252427184, + "recall@50": 0.16990291262135923, + "recall@100": 0.2621359223300971, + "recall@200": 0.470873786407767, + "recall@300": 0.6407766990291263, + "recall@500": 0.8203883495145631, + "recall@1000": 0.8980582524271845, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.2620045784218241, + "ndcg_full@5": 0.2620045784218241, + "ndcg@10": 0.383452137359286, + "ndcg_full@10": 0.33117416036267616, + "ndcg@20": 0.4901157655890233, + "ndcg_full@20": 0.4007563486175692, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.013071895424836602, + "recall@50": 0.0915032679738562, + "recall@100": 0.16339869281045752, + "recall@200": 0.33986928104575165, + "recall@300": 0.40522875816993464, + "recall@500": 0.6274509803921569, + "recall@1000": 0.8627450980392157, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.1780615783628922, + "ndcg_full@10": 0.1780615783628922, + "ndcg@20": 0.3294514496252314, + "ndcg_full@20": 0.3294514496252314, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 1000, + "num_ranked": 172, + "recall@10": 0.14285714285714285, + "recall@50": 0.4714285714285714, + "recall@100": 0.7285714285714285, + "recall@200": 0.8571428571428571, + "recall@300": 0.9, + "recall@500": 0.9285714285714286, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.35262297841205176, + "ndcg_full@5": 0.35262297841205176, + "ndcg@10": 0.3587031965324812, + "ndcg_full@10": 0.3587031965324812, + "ndcg@20": 0.3195007948678155, + "ndcg_full@20": 0.3195007948678155, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 1000, + "num_ranked": 178, + "recall@10": 0.07547169811320754, + "recall@50": 0.5094339622641509, + "recall@100": 0.6792452830188679, + "recall@200": 0.8490566037735849, + "recall@300": 0.8679245283018868, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.7503853954919861, + "ndcg_full@10": 0.7503853954919861, + "ndcg@20": 0.5846396672200014, + "ndcg_full@20": 0.5846396672200014, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 1000, + "num_ranked": 169, + "recall@10": 0.072992700729927, + "recall@50": 0.29927007299270075, + "recall@100": 0.5109489051094891, + "recall@200": 0.708029197080292, + "recall@300": 0.7737226277372263, + "recall@500": 0.8175182481751825, + "recall@1000": 0.8321167883211679, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.7057807535511937, + "ndcg_full@10": 0.7057807535511937, + "ndcg@20": 0.6459910318884723, + "ndcg_full@20": 0.6459910318884723, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 1000, + "num_ranked": 203, + "recall@10": 0.0299625468164794, + "recall@50": 0.13857677902621723, + "recall@100": 0.250936329588015, + "recall@200": 0.45692883895131087, + "recall@300": 0.6179775280898876, + "recall@500": 0.7715355805243446, + "recall@1000": 0.850187265917603, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5538466623591201, + "ndcg_full@5": 0.5538466623591201, + "ndcg@10": 0.6345959162261248, + "ndcg_full@10": 0.6345959162261248, + "ndcg@20": 0.5764755747649879, + "ndcg_full@20": 0.5764755747649879, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 1000, + "num_ranked": 181, + "recall@10": 0.04430379746835443, + "recall@50": 0.16455696202531644, + "recall@100": 0.31645569620253167, + "recall@200": 0.5253164556962026, + "recall@300": 0.6392405063291139, + "recall@500": 0.7468354430379747, + "recall@1000": 0.8227848101265823, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.7165226333593651, + "ndcg_full@10": 0.7165226333593651, + "ndcg@20": 0.6415333525306326, + "ndcg_full@20": 0.6415333525306326, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.06862745098039216, + "recall@100": 0.17647058823529413, + "recall@200": 0.3431372549019608, + "recall@300": 0.46078431372549017, + "recall@500": 0.6176470588235294, + "recall@1000": 0.7254901960784313, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4528847242100977, + "ndcg_full@5": 0.4528847242100977, + "ndcg@10": 0.3318317452948964, + "ndcg_full@10": 0.3318317452948964, + "ndcg@20": 0.3353157661369267, + "ndcg_full@20": 0.3353157661369267, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 1000, + "num_ranked": 115, + "recall@10": 0.05, + "recall@50": 0.20625, + "recall@100": 0.38125, + "recall@200": 0.54375, + "recall@300": 0.625, + "recall@500": 0.70625, + "recall@1000": 0.7875, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.6538571657758918, + "ndcg_full@10": 0.6538571657758918, + "ndcg@20": 0.5240081425028282, + "ndcg_full@20": 0.5240081425028282, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 1000, + "num_ranked": 244, + "recall@10": 0.04291845493562232, + "recall@50": 0.14163090128755365, + "recall@100": 0.26180257510729615, + "recall@200": 0.45493562231759654, + "recall@300": 0.5836909871244635, + "recall@500": 0.8240343347639485, + "recall@1000": 0.9399141630901288, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5949016878400769, + "ndcg_full@5": 0.5949016878400769, + "ndcg@10": 0.6200055120319127, + "ndcg_full@10": 0.6200055120319127, + "ndcg@20": 0.5564221353513422, + "ndcg_full@20": 0.5564221353513422, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 1000, + "num_ranked": 234, + "recall@10": 0.018518518518518517, + "recall@50": 0.1111111111111111, + "recall@100": 0.22839506172839505, + "recall@200": 0.42592592592592593, + "recall@300": 0.5802469135802469, + "recall@500": 0.7098765432098766, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.618884020521281, + "ndcg_full@10": 0.618884020521281, + "ndcg@20": 0.503016751646783, + "ndcg_full@20": 0.503016751646783, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.031007751937984496, + "recall@50": 0.13565891472868216, + "recall@100": 0.22093023255813954, + "recall@200": 0.38372093023255816, + "recall@300": 0.5193798449612403, + "recall@500": 0.6434108527131783, + "recall@1000": 0.7248062015503876, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4925683638348177, + "ndcg_full@5": 0.4925683638348177, + "ndcg@10": 0.45512586161821045, + "ndcg_full@10": 0.45512586161821045, + "ndcg@20": 0.527390318705576, + "ndcg_full@20": 0.527390318705576, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 1000, + "num_ranked": 75, + "recall@10": 0.06, + "recall@50": 0.29333333333333333, + "recall@100": 0.5, + "recall@200": 0.66, + "recall@300": 0.7066666666666667, + "recall@500": 0.7533333333333333, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.8899701444399659, + "ndcg_full@20": 0.8899701444399659, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 1000, + "num_ranked": 174, + "recall@10": 0.03488372093023256, + "recall@50": 0.13372093023255813, + "recall@100": 0.21511627906976744, + "recall@200": 0.37209302325581395, + "recall@300": 0.46511627906976744, + "recall@500": 0.5755813953488372, + "recall@1000": 0.7209302325581395, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.3274584397097874, + "ndcg_full@10": 0.3274584397097874, + "ndcg@20": 0.3711086913250021, + "ndcg_full@20": 0.3711086913250021, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 1000, + "num_ranked": 67, + "recall@10": 0.09433962264150944, + "recall@50": 0.24528301886792453, + "recall@100": 0.2830188679245283, + "recall@200": 0.33962264150943394, + "recall@300": 0.41509433962264153, + "recall@500": 0.4528301886792453, + "recall@1000": 0.49056603773584906, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6390097281865127, + "ndcg_full@10": 0.6390097281865127, + "ndcg@20": 0.613263586770639, + "ndcg_full@20": 0.4653243306129762, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 1000, + "num_ranked": 27, + "recall@10": 0.0430622009569378, + "recall@50": 0.15789473684210525, + "recall@100": 0.2727272727272727, + "recall@200": 0.3875598086124402, + "recall@300": 0.4784688995215311, + "recall@500": 0.5789473684210527, + "recall@1000": 0.6698564593301436, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.31017134364653787, + "ndcg_full@10": 0.31017134364653787, + "ndcg@20": 0.4278986430298445, + "ndcg_full@20": 0.2761518126617782, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 1000, + "num_ranked": 218, + "recall@10": 0.04, + "recall@50": 0.14, + "recall@100": 0.22, + "recall@200": 0.37333333333333335, + "recall@300": 0.4666666666666667, + "recall@500": 0.5666666666666667, + "recall@1000": 0.6733333333333333, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.542526795349146, + "ndcg_full@10": 0.542526795349146, + "ndcg@20": 0.5810169801605163, + "ndcg_full@20": 0.5810169801605163, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 1000, + "num_ranked": 84, + "recall@10": 0.05172413793103448, + "recall@50": 0.25287356321839083, + "recall@100": 0.4367816091954023, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.8904408160938945, + "ndcg_full@20": 0.8904408160938945, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.022653721682847898, + "recall@50": 0.0744336569579288, + "recall@100": 0.12944983818770225, + "recall@200": 0.24271844660194175, + "recall@300": 0.31715210355987056, + "recall@500": 0.4368932038834951, + "recall@1000": 0.5728155339805825, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.31516255047698366, + "ndcg_full@10": 0.31516255047698366, + "ndcg@20": 0.34018057312746086, + "ndcg_full@20": 0.34018057312746086, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.04032258064516129, + "recall@50": 0.1935483870967742, + "recall@100": 0.375, + "recall@200": 0.6895161290322581, + "recall@300": 0.8669354838709677, + "recall@500": 0.9112903225806451, + "recall@1000": 0.9314516129032258, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263838, + "ndcg_full@5": 0.6608397947263838, + "ndcg@10": 0.7799082337019199, + "ndcg_full@10": 0.7799082337019199, + "ndcg@20": 0.7890124590732149, + "ndcg_full@20": 0.7890124590732149, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 1000, + "num_ranked": 104, + "recall@10": 0.12345679012345678, + "recall@50": 0.5555555555555556, + "recall@100": 0.9382716049382716, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9631344311641977, + "ndcg_full@10": 0.9631344311641977, + "ndcg@20": 0.7957855085946132, + "ndcg_full@20": 0.7957855085946132, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 1000, + "num_ranked": 81, + "recall@10": 0.1016949152542373, + "recall@50": 0.5254237288135594, + "recall@100": 0.7796610169491526, + "recall@200": 0.8813559322033898, + "recall@300": 0.9152542372881356, + "recall@500": 0.9322033898305084, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7477870352439605, + "ndcg_full@10": 0.7477870352439605, + "ndcg@20": 0.7318414585432462, + "ndcg_full@20": 0.720008207462855, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 1000, + "num_ranked": 51, + "recall@10": 0.6923076923076923, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.7917267193679839, + "ndcg_full@10": 0.7917267193679839, + "ndcg@20": 0.8167632740705711, + "ndcg_full@20": 0.7516148689992276, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.02922077922077922, + "recall@50": 0.12012987012987013, + "recall@100": 0.22402597402597402, + "recall@200": 0.41883116883116883, + "recall@300": 0.577922077922078, + "recall@500": 0.724025974025974, + "recall@1000": 0.827922077922078, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6164336326286642, + "ndcg_full@5": 0.6164336326286642, + "ndcg@10": 0.71598483051059, + "ndcg_full@10": 0.71598483051059, + "ndcg@20": 0.6797356233238855, + "ndcg_full@20": 0.6797356233238855, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 1000, + "num_ranked": 95, + "recall@10": 0.08450704225352113, + "recall@50": 0.23943661971830985, + "recall@100": 0.36619718309859156, + "recall@200": 0.4788732394366197, + "recall@300": 0.5070422535211268, + "recall@500": 0.5774647887323944, + "recall@1000": 0.6197183098591549, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.6942343603822474, + "ndcg_full@20": 0.6942343603822474, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 1000, + "num_ranked": 150, + "recall@10": 0.06451612903225806, + "recall@50": 0.25161290322580643, + "recall@100": 0.44516129032258067, + "recall@200": 0.6645161290322581, + "recall@300": 0.7354838709677419, + "recall@500": 0.8580645161290322, + "recall@1000": 0.9161290322580645, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4374130297304557, + "ndcg_full@5": 0.4374130297304557, + "ndcg@10": 0.5590384555827961, + "ndcg_full@10": 0.5590384555827961, + "ndcg@20": 0.5343198809535961, + "ndcg_full@20": 0.5343198809535961, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 1000, + "num_ranked": 128, + "recall@10": 0.05128205128205128, + "recall@50": 0.24615384615384617, + "recall@100": 0.46153846153846156, + "recall@200": 0.676923076923077, + "recall@300": 0.7230769230769231, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7384615384615385, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6720882144762628, + "ndcg_full@5": 0.5789121131198787, + "ndcg@10": 0.7504540869478651, + "ndcg_full@10": 0.6382203881849413, + "ndcg@20": 0.8731539043632495, + "ndcg_full@20": 0.774085840199104, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.1, + "graded_P@10": 0.55 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 1000, + "num_ranked": 210, + "recall@10": 0.053763440860215055, + "recall@50": 0.24731182795698925, + "recall@100": 0.43548387096774194, + "recall@200": 0.7526881720430108, + "recall@300": 0.8602150537634409, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.8838502126394092, + "ndcg_full@20": 0.8838502126394092, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 1000, + "num_ranked": 164, + "recall@10": 0.06306306306306306, + "recall@50": 0.3153153153153153, + "recall@100": 0.5315315315315315, + "recall@200": 0.8738738738738738, + "recall@300": 0.954954954954955, + "recall@500": 0.990990990990991, + "recall@1000": 0.990990990990991, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8757412786703995, + "ndcg_full@20": 0.8757412786703995, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 1000, + "num_ranked": 243, + "recall@10": 0.03759398496240601, + "recall@50": 0.16917293233082706, + "recall@100": 0.2894736842105263, + "recall@200": 0.4548872180451128, + "recall@300": 0.5902255639097744, + "recall@500": 0.7293233082706767, + "recall@1000": 0.8270676691729323, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8292298191606344, + "ndcg_full@10": 0.8292298191606344, + "ndcg@20": 0.7659570920721864, + "ndcg_full@20": 0.7659570920721864, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 1000, + "num_ranked": 57, + "recall@10": 0.15151515151515152, + "recall@50": 0.6818181818181818, + "recall@100": 0.803030303030303, + "recall@200": 0.8181818181818182, + "recall@300": 0.8333333333333334, + "recall@500": 0.8333333333333334, + "recall@1000": 0.8484848484848485, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.9356218339941231, + "ndcg_full@20": 0.9356218339941231, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 1000, + "num_ranked": 105, + "recall@10": 0.12987012987012986, + "recall@50": 0.4805194805194805, + "recall@100": 0.6623376623376623, + "recall@200": 0.7792207792207793, + "recall@300": 0.7792207792207793, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7840048711692542, + "ndcg_full@5": 0.7840048711692542, + "ndcg@10": 0.7884363687657903, + "ndcg_full@10": 0.7884363687657903, + "ndcg@20": 0.8467449945874643, + "ndcg_full@20": 0.8467449945874643, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 1000, + "num_ranked": 45, + "recall@10": 0.391304347826087, + "recall@50": 0.7391304347826086, + "recall@100": 0.782608695652174, + "recall@200": 0.782608695652174, + "recall@300": 0.782608695652174, + "recall@500": 0.782608695652174, + "recall@1000": 0.782608695652174, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.3896275358478457, + "ndcg_full@10": 0.3212345732391268, + "ndcg@20": 0.44323933443145414, + "ndcg_full@20": 0.27224946256285165, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 1000, + "num_ranked": 87, + "recall@10": 0.11267605633802817, + "recall@50": 0.4225352112676056, + "recall@100": 0.6619718309859155, + "recall@200": 0.8309859154929577, + "recall@300": 0.8732394366197183, + "recall@500": 0.8732394366197183, + "recall@1000": 0.8873239436619719, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.476663423592926, + "ndcg_full@10": 0.476663423592926, + "ndcg@20": 0.5998376854531831, + "ndcg_full@20": 0.5389649850613911, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 1000, + "num_ranked": 67, + "recall@10": 0.10294117647058823, + "recall@50": 0.38235294117647056, + "recall@100": 0.5, + "recall@200": 0.6470588235294118, + "recall@300": 0.7205882352941176, + "recall@500": 0.7794117647058824, + "recall@1000": 0.8088235294117647, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.5239294981581574, + "ndcg_full@10": 0.5239294981581574, + "ndcg@20": 0.5079932171422576, + "ndcg_full@20": 0.465460494944155, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 1000, + "num_ranked": 143, + "recall@10": 0.06521739130434782, + "recall@50": 0.1956521739130435, + "recall@100": 0.34782608695652173, + "recall@200": 0.5869565217391305, + "recall@300": 0.6304347826086957, + "recall@500": 0.717391304347826, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8454256604216017, + "ndcg_full@10": 0.8454256604216017, + "ndcg@20": 0.6195576118905802, + "ndcg_full@20": 0.6195576118905802, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 1000, + "num_ranked": 106, + "recall@10": 0.10666666666666667, + "recall@50": 0.5066666666666667, + "recall@100": 0.84, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.5185812712726494, + "ndcg_full@10": 0.5185812712726494, + "ndcg@20": 0.5263634356642185, + "ndcg_full@20": 0.5263634356642185, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 1000, + "num_ranked": 80, + "recall@10": 0.20512820512820512, + "recall@50": 0.5641025641025641, + "recall@100": 0.6923076923076923, + "recall@200": 0.717948717948718, + "recall@300": 0.7435897435897436, + "recall@500": 0.8205128205128205, + "recall@1000": 0.8461538461538461, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.7059714630011186, + "ndcg_full@10": 0.7059714630011186, + "ndcg@20": 0.6342840422792451, + "ndcg_full@20": 0.6342840422792451, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 1000, + "num_ranked": 178, + "recall@10": 0.10465116279069768, + "recall@50": 0.37209302325581395, + "recall@100": 0.6395348837209303, + "recall@200": 0.9302325581395349, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.94497705842548, + "ndcg_full@10": 0.94497705842548, + "ndcg@20": 0.8229801987513429, + "ndcg_full@20": 0.8229801987513429, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 1000, + "num_ranked": 112, + "recall@10": 0.04938271604938271, + "recall@50": 0.21604938271604937, + "recall@100": 0.36419753086419754, + "recall@200": 0.5246913580246914, + "recall@300": 0.5802469135802469, + "recall@500": 0.6481481481481481, + "recall@1000": 0.7037037037037037, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8227308061618839, + "ndcg_full@10": 0.8227308061618839, + "ndcg@20": 0.7084973524904892, + "ndcg_full@20": 0.7084973524904892, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 1000, + "num_ranked": 93, + "recall@10": 0.07352941176470588, + "recall@50": 0.33088235294117646, + "recall@100": 0.5661764705882353, + "recall@200": 0.6544117647058824, + "recall@300": 0.6764705882352942, + "recall@500": 0.6764705882352942, + "recall@1000": 0.6838235294117647, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8017480615820849, + "ndcg_full@5": 0.8017480615820849, + "ndcg@10": 0.8478285765962675, + "ndcg_full@10": 0.8478285765962675, + "ndcg@20": 0.775247671691344, + "ndcg_full@20": 0.775247671691344, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1000, + "num_ranked": 165, + "recall@10": 0.02564102564102564, + "recall@50": 0.1339031339031339, + "recall@100": 0.23931623931623933, + "recall@200": 0.4586894586894587, + "recall@300": 0.6153846153846154, + "recall@500": 0.6296296296296297, + "recall@1000": 0.6324786324786325, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7471589728148227, + "ndcg_full@10": 0.7471589728148227, + "ndcg@20": 0.7827903339691237, + "ndcg_full@20": 0.7827903339691237, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 1000, + "num_ranked": 166, + "recall@10": 0.035175879396984924, + "recall@50": 0.1507537688442211, + "recall@100": 0.32663316582914576, + "recall@200": 0.48743718592964824, + "recall@300": 0.6030150753768844, + "recall@500": 0.6633165829145728, + "recall@1000": 0.8090452261306532, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.710970696247901, + "ndcg_full@5": 0.710970696247901, + "ndcg@10": 0.7037716763453934, + "ndcg_full@10": 0.7037716763453934, + "ndcg@20": 0.6502920998209335, + "ndcg_full@20": 0.6502920998209335, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.015789473684210527, + "recall@50": 0.06315789473684211, + "recall@100": 0.12105263157894737, + "recall@200": 0.24210526315789474, + "recall@300": 0.4, + "recall@500": 0.5526315789473685, + "recall@1000": 0.6789473684210526, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.7482066088715277, + "ndcg_full@10": 0.7482066088715277, + "ndcg@20": 0.6835893710557087, + "ndcg_full@20": 0.6835893710557087, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 1000, + "num_ranked": 126, + "recall@10": 0.03636363636363636, + "recall@50": 0.15636363636363637, + "recall@100": 0.27636363636363637, + "recall@200": 0.44727272727272727, + "recall@300": 0.5490909090909091, + "recall@500": 0.6436363636363637, + "recall@1000": 0.7418181818181818, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5915214378876218, + "ndcg_full@5": 0.5915214378876218, + "ndcg@10": 0.5245495896166291, + "ndcg_full@10": 0.5245495896166291, + "ndcg@20": 0.568884806879336, + "ndcg_full@20": 0.568884806879336, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 1000, + "num_ranked": 133, + "recall@10": 0.045454545454545456, + "recall@50": 0.29545454545454547, + "recall@100": 0.45454545454545453, + "recall@200": 0.5, + "recall@300": 0.5681818181818182, + "recall@500": 0.5909090909090909, + "recall@1000": 0.7272727272727273, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.3564268441428192, + "ndcg_full@10": 0.3564268441428192, + "ndcg@20": 0.3598151043527821, + "ndcg_full@20": 0.3539972015507204, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.024630541871921183, + "recall@50": 0.08866995073891626, + "recall@100": 0.20689655172413793, + "recall@200": 0.3891625615763547, + "recall@300": 0.5123152709359606, + "recall@500": 0.7438423645320197, + "recall@1000": 0.8817733990147784, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.5672274908036508, + "ndcg_full@10": 0.5672274908036508, + "ndcg@20": 0.4851753038569542, + "ndcg_full@20": 0.4851753038569542, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 1000, + "num_ranked": 230, + "recall@10": 0.017699115044247787, + "recall@50": 0.09734513274336283, + "recall@100": 0.20353982300884957, + "recall@200": 0.34513274336283184, + "recall@300": 0.4778761061946903, + "recall@500": 0.6460176991150443, + "recall@1000": 0.8053097345132744, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.48328431553011136, + "ndcg_full@10": 0.48328431553011136, + "ndcg@20": 0.46163947064034444, + "ndcg_full@20": 0.46163947064034444, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 1000, + "num_ranked": 44, + "recall@10": 0.10416666666666667, + "recall@50": 0.20833333333333334, + "recall@100": 0.22916666666666666, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.710970696247901, + "ndcg_full@5": 0.710970696247901, + "ndcg@10": 0.6825647469457429, + "ndcg_full@10": 0.6825647469457429, + "ndcg@20": 0.6818222146026295, + "ndcg_full@20": 0.5698864912343738, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 1000, + "num_ranked": 151, + "recall@10": 0.06369426751592357, + "recall@50": 0.25477707006369427, + "recall@100": 0.43312101910828027, + "recall@200": 0.6815286624203821, + "recall@300": 0.7579617834394905, + "recall@500": 0.802547770700637, + "recall@1000": 0.821656050955414, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.39049624233831776, + "ndcg_full@5": 0.39049624233831776, + "ndcg@10": 0.46085486536931625, + "ndcg_full@10": 0.46085486536931625, + "ndcg@20": 0.5837856211765606, + "ndcg_full@20": 0.5837856211765606, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 1000, + "num_ranked": 141, + "recall@10": 0.06097560975609756, + "recall@50": 0.23170731707317074, + "recall@100": 0.3597560975609756, + "recall@200": 0.43902439024390244, + "recall@300": 0.47560975609756095, + "recall@500": 0.5304878048780488, + "recall@1000": 0.6097560975609756, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.7140318610876282, + "ndcg_full@10": 0.7140318610876282, + "ndcg@20": 0.5860630083650272, + "ndcg_full@20": 0.5471766361131617, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.0, + "recall@50": 0.02531645569620253, + "recall@100": 0.06329113924050633, + "recall@200": 0.1518987341772152, + "recall@300": 0.22784810126582278, + "recall@500": 0.34177215189873417, + "recall@1000": 0.4050632911392405, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.4713281642188929, + "ndcg_full@5": 0.4713281642188929, + "ndcg@10": 0.3215393819005685, + "ndcg_full@10": 0.3215393819005685, + "ndcg@20": 0.35772710194877216, + "ndcg_full@20": 0.24481776868906335, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 1000, + "num_ranked": 234, + "recall@10": 0.03896103896103896, + "recall@50": 0.1341991341991342, + "recall@100": 0.2554112554112554, + "recall@200": 0.5108225108225108, + "recall@300": 0.670995670995671, + "recall@500": 0.7922077922077922, + "recall@1000": 0.8398268398268398, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.06560253875617089, + "ndcg_full@5": 0.06560253875617089, + "ndcg@10": 0.25795453809779756, + "ndcg_full@10": 0.25795453809779756, + "ndcg@20": 0.30751306977937265, + "ndcg_full@20": 0.30751306977937265, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.1358695652173913, + "recall@100": 0.20652173913043478, + "recall@200": 0.32065217391304346, + "recall@300": 0.44565217391304346, + "recall@500": 0.5978260869565217, + "recall@1000": 0.7228260869565217, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.55206785919664, + "ndcg_full@10": 0.55206785919664, + "ndcg@20": 0.4652210441095134, + "ndcg_full@20": 0.4652210441095134, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 1000, + "num_ranked": 51, + "recall@10": 0.08791208791208792, + "recall@50": 0.21978021978021978, + "recall@100": 0.24175824175824176, + "recall@200": 0.25274725274725274, + "recall@300": 0.26373626373626374, + "recall@500": 0.27472527472527475, + "recall@1000": 0.31868131868131866, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7265894102945099, + "ndcg_full@10": 0.7034763148045359, + "ndcg@20": 0.7154304359894833, + "ndcg_full@20": 0.6137010032679866, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 1000, + "num_ranked": 86, + "recall@10": 0.12121212121212122, + "recall@50": 0.4696969696969697, + "recall@100": 0.7121212121212122, + "recall@200": 0.8484848484848485, + "recall@300": 0.9090909090909091, + "recall@500": 0.9545454545454546, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7191184989488184, + "ndcg_full@10": 0.7191184989488184, + "ndcg@20": 0.7112302576791576, + "ndcg_full@20": 0.6390532213987996, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 1000, + "num_ranked": 51, + "recall@10": 0.6923076923076923, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7677078645873638, + "ndcg_full@10": 0.7677078645873638, + "ndcg@20": 0.7322907253666109, + "ndcg_full@20": 0.7133933138324716, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.10557184750733138, + "recall@100": 0.22580645161290322, + "recall@200": 0.4222873900293255, + "recall@300": 0.6187683284457478, + "recall@500": 0.7712609970674487, + "recall@1000": 0.8651026392961877, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8613313565017253, + "ndcg_full@10": 0.8613313565017253, + "ndcg@20": 0.7511634622848141, + "ndcg_full@20": 0.7511634622848141, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + } + } + } + } + }, + { + "name": "h100_medcpt", + "results_dir": "h100_medcpt/results_trec21", + "evaluation_input_sha256": "9d89f695629b5d4c826725923637cbfa9048f37ef76b9c4116c3292ca1fea192", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.08033430480469368, + "recall@50": 0.27627747582078804, + "recall@100": 0.4219591213693766, + "recall@200": 0.5871692603169641, + "recall@300": 0.6782443020293144, + "recall@500": 0.7718410466727308, + "recall@1000": 0.837050997496093, + "ndcg@5": 0.8128546828087836, + "ndcg@10": 0.7987271514510064, + "ndcg@20": 0.7864347490329242, + "ndcg_full@5": 0.8113827270438009, + "ndcg_full@10": 0.7951760762411342, + "ndcg_full@20": 0.768931171940641, + "P@10(rel>=1)": 0.8346666666666663, + "P@10(eligible)": 0.7306666666666665, + "graded_P@10": 0.7826666666666665, + "unjudged_fraction@10": 0.2826666666666667 + }, + "median": { + "recall@10": 0.05128205128205128, + "recall@50": 0.1935483870967742, + "recall@100": 0.34274193548387094, + "recall@200": 0.5352112676056338, + "recall@300": 0.6466165413533834, + "recall@500": 0.8089887640449438, + "recall@1000": 0.8717948717948718, + "ndcg@5": 0.8687949224876582, + "ndcg@10": 0.8630152897016882, + "ndcg@20": 0.8247257280380528, + "ndcg_full@5": 0.8687949224876582, + "ndcg_full@10": 0.84860265890399, + "ndcg_full@20": 0.8102363410554485, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8, + "unjudged_fraction@10": 0.3 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 209, + "recall@10": 0.05917159763313609, + "recall@50": 0.25443786982248523, + "recall@100": 0.4437869822485207, + "recall@200": 0.7100591715976331, + "recall@300": 0.8106508875739645, + "recall@500": 0.8698224852071006, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7135842986869696, + "ndcg_full@10": 0.7135842986869696, + "ndcg@20": 0.690843093804615, + "ndcg_full@20": 0.690843093804615, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.037037037037037035, + "recall@50": 0.17407407407407408, + "recall@100": 0.3, + "recall@200": 0.45555555555555555, + "recall@300": 0.5814814814814815, + "recall@500": 0.7296296296296296, + "recall@1000": 0.8481481481481481, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.84860265890399, + "ndcg_full@10": 0.84860265890399, + "ndcg@20": 0.9022930994129825, + "ndcg_full@20": 0.9022930994129825, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 164, + "recall@10": 0.05952380952380952, + "recall@50": 0.27380952380952384, + "recall@100": 0.4523809523809524, + "recall@200": 0.7142857142857143, + "recall@300": 0.7857142857142857, + "recall@500": 0.9047619047619048, + "recall@1000": 0.9523809523809523, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.47820400441738703, + "ndcg_full@10": 0.47820400441738703, + "ndcg@20": 0.5351885668579532, + "ndcg_full@20": 0.5087928652151008, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.042105263157894736, + "recall@50": 0.15789473684210525, + "recall@100": 0.23157894736842105, + "recall@200": 0.4421052631578947, + "recall@300": 0.5368421052631579, + "recall@500": 0.7789473684210526, + "recall@1000": 0.9052631578947369, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.4458567890377042, + "ndcg_full@10": 0.4458567890377042, + "ndcg@20": 0.5002944179061221, + "ndcg_full@20": 0.5002944179061221, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01990049751243781, + "recall@50": 0.1044776119402985, + "recall@100": 0.18407960199004975, + "recall@200": 0.29850746268656714, + "recall@300": 0.4427860696517413, + "recall@500": 0.5671641791044776, + "recall@1000": 0.736318407960199, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7877736340270854, + "ndcg_full@10": 0.7877736340270854, + "ndcg@20": 0.8131395762232025, + "ndcg_full@20": 0.8131395762232025, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0380952380952381, + "recall@50": 0.15714285714285714, + "recall@100": 0.28095238095238095, + "recall@200": 0.47619047619047616, + "recall@300": 0.6285714285714286, + "recall@500": 0.8238095238095238, + "recall@1000": 0.8857142857142857, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9283354595867189, + "ndcg_full@20": 0.9283354595867189, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018633540372670808, + "recall@50": 0.08074534161490683, + "recall@100": 0.14285714285714285, + "recall@200": 0.3105590062111801, + "recall@300": 0.4161490683229814, + "recall@500": 0.639751552795031, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5508310938724935, + "ndcg_full@5": 0.5508310938724935, + "ndcg@10": 0.518276764441841, + "ndcg_full@10": 0.518276764441841, + "ndcg@20": 0.5687327688194965, + "ndcg_full@20": 0.5687327688194965, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.2222222222222222, + "recall@100": 0.41830065359477125, + "recall@200": 0.7058823529411765, + "recall@300": 0.8431372549019608, + "recall@500": 0.9281045751633987, + "recall@1000": 0.9411764705882353, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8189117618793237, + "ndcg_full@20": 0.8189117618793237, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.21962616822429906, + "recall@100": 0.40654205607476634, + "recall@200": 0.7383177570093458, + "recall@300": 0.9112149532710281, + "recall@500": 0.9299065420560748, + "recall@1000": 0.9392523364485982, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.31494820288354997, + "ndcg_full@5": 0.31494820288354997, + "ndcg@10": 0.37952600234997763, + "ndcg_full@10": 0.37952600234997763, + "ndcg@20": 0.5119092557431947, + "ndcg_full@20": 0.47838270924559234, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 85, + "recall@10": 0.19148936170212766, + "recall@50": 0.8085106382978723, + "recall@100": 0.8085106382978723, + "recall@200": 0.8297872340425532, + "recall@300": 0.8297872340425532, + "recall@500": 0.851063829787234, + "recall@1000": 0.8723404255319149, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7618017901203944, + "ndcg_full@5": 0.7618017901203944, + "ndcg@10": 0.6698914480875249, + "ndcg_full@10": 0.6698914480875249, + "ndcg@20": 0.8241953233627917, + "ndcg_full@20": 0.8048150725116003, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 156, + "recall@10": 0.07936507936507936, + "recall@50": 0.35714285714285715, + "recall@100": 0.5396825396825397, + "recall@200": 0.7936507936507936, + "recall@300": 0.8412698412698413, + "recall@500": 0.8492063492063492, + "recall@1000": 0.8650793650793651, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.7612554570768364, + "ndcg_full@20": 0.7612554570768364, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.06040268456375839, + "recall@50": 0.26174496644295303, + "recall@100": 0.46308724832214765, + "recall@200": 0.7919463087248322, + "recall@300": 0.8389261744966443, + "recall@500": 0.8590604026845637, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.702728403557943, + "ndcg_full@10": 0.702728403557943, + "ndcg@20": 0.7758123268834158, + "ndcg_full@20": 0.7758123268834158, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 198, + "recall@10": 0.036231884057971016, + "recall@50": 0.10869565217391304, + "recall@100": 0.18840579710144928, + "recall@200": 0.34782608695652173, + "recall@300": 0.4927536231884058, + "recall@500": 0.6304347826086957, + "recall@1000": 0.6956521739130435, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010481, + "ndcg_full@10": 0.9363792118010481, + "ndcg@20": 0.8683589753409113, + "ndcg_full@20": 0.8683589753409113, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01020408163265306, + "recall@50": 0.061224489795918366, + "recall@100": 0.13945578231292516, + "recall@200": 0.2755102040816326, + "recall@300": 0.3707482993197279, + "recall@500": 0.5272108843537415, + "recall@1000": 0.6870748299319728, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503199, + "ndcg_full@10": 0.9633180389503199, + "ndcg@20": 0.8736980482128412, + "ndcg_full@20": 0.8736980482128412, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.026923076923076925, + "recall@50": 0.1423076923076923, + "recall@100": 0.26153846153846155, + "recall@200": 0.3923076923076923, + "recall@300": 0.4807692307692308, + "recall@500": 0.5807692307692308, + "recall@1000": 0.6884615384615385, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8440072240027022, + "ndcg_full@10": 0.8440072240027022, + "ndcg@20": 0.8801350366930267, + "ndcg_full@20": 0.8801350366930267, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.10144927536231885, + "recall@100": 0.17391304347826086, + "recall@200": 0.2753623188405797, + "recall@300": 0.427536231884058, + "recall@500": 0.5869565217391305, + "recall@1000": 0.7246376811594203, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.1467278441987201, + "ndcg_full@10": 0.1467278441987201, + "ndcg@20": 0.23808444886209718, + "ndcg_full@20": 0.23808444886209718, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035856573705179286, + "recall@50": 0.15139442231075698, + "recall@100": 0.2868525896414343, + "recall@200": 0.6095617529880478, + "recall@300": 0.8286852589641435, + "recall@500": 0.9561752988047809, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.36222708529148695, + "ndcg_full@10": 0.36222708529148695, + "ndcg@20": 0.4995628036468117, + "ndcg_full@20": 0.4995628036468117, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.043689320388349516, + "recall@50": 0.14563106796116504, + "recall@100": 0.2766990291262136, + "recall@200": 0.5, + "recall@300": 0.6262135922330098, + "recall@500": 0.8009708737864077, + "recall@1000": 0.9077669902912622, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.849607409925425, + "ndcg_full@5": 0.849607409925425, + "ndcg@10": 0.8847947874311091, + "ndcg_full@10": 0.7966222291395775, + "ndcg@20": 0.9129157739579024, + "ndcg_full@20": 0.7705228345844314, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.1895424836601307, + "recall@100": 0.3137254901960784, + "recall@200": 0.5620915032679739, + "recall@300": 0.6993464052287581, + "recall@500": 0.8300653594771242, + "recall@1000": 0.9084967320261438, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6468562235661909, + "ndcg_full@10": 0.6468562235661909, + "ndcg@20": 0.7039048732941999, + "ndcg_full@20": 0.7039048732941999, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 193, + "recall@10": 0.12857142857142856, + "recall@50": 0.38571428571428573, + "recall@100": 0.7142857142857143, + "recall@200": 0.9142857142857143, + "recall@300": 0.9285714285714286, + "recall@500": 0.9428571428571428, + "recall@1000": 0.9571428571428572, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.8195989752263492, + "ndcg_full@20": 0.8195989752263492, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 246, + "recall@10": 0.1320754716981132, + "recall@50": 0.5660377358490566, + "recall@100": 0.7547169811320755, + "recall@200": 0.9056603773584906, + "recall@300": 0.9245283018867925, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8263333286520083, + "ndcg_full@10": 0.8263333286520083, + "ndcg@20": 0.8879212007962823, + "ndcg_full@20": 0.8879212007962823, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 203, + "recall@10": 0.06569343065693431, + "recall@50": 0.291970802919708, + "recall@100": 0.5182481751824818, + "recall@200": 0.7445255474452555, + "recall@300": 0.7591240875912408, + "recall@500": 0.8102189781021898, + "recall@1000": 0.8613138686131386, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9652462405176582, + "ndcg_full@20": 0.9652462405176582, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 237, + "recall@10": 0.033707865168539325, + "recall@50": 0.12359550561797752, + "recall@100": 0.25842696629213485, + "recall@200": 0.45692883895131087, + "recall@300": 0.6104868913857678, + "recall@500": 0.8089887640449438, + "recall@1000": 0.8726591760299626, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9118620531846015, + "ndcg_full@20": 0.9118620531846015, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0189873417721519, + "recall@50": 0.11392405063291139, + "recall@100": 0.23417721518987342, + "recall@200": 0.4430379746835443, + "recall@300": 0.6139240506329114, + "recall@500": 0.7974683544303798, + "recall@1000": 0.9493670886075949, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0000000000000002, + "ndcg_full@5": 1.0000000000000002, + "ndcg@10": 0.8630152897016882, + "ndcg_full@10": 0.8630152897016882, + "ndcg@20": 0.8393422739918858, + "ndcg_full@20": 0.8393422739918858, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.09803921568627451, + "recall@100": 0.22549019607843138, + "recall@200": 0.38235294117647056, + "recall@300": 0.5, + "recall@500": 0.6568627450980392, + "recall@1000": 0.8137254901960784, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6922500167496441, + "ndcg_full@5": 0.6922500167496441, + "ndcg@10": 0.5186541158296211, + "ndcg_full@10": 0.5186541158296211, + "ndcg@20": 0.6054984461417369, + "ndcg_full@20": 0.6054984461417369, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.0625, + "recall@50": 0.1875, + "recall@100": 0.34375, + "recall@200": 0.5625, + "recall@300": 0.65, + "recall@500": 0.775, + "recall@1000": 0.89375, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9227128302108009, + "ndcg_full@10": 0.9227128302108009, + "ndcg@20": 0.8102363410554485, + "ndcg_full@20": 0.8102363410554485, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.14163090128755365, + "recall@100": 0.24892703862660945, + "recall@200": 0.463519313304721, + "recall@300": 0.6223175965665236, + "recall@500": 0.8540772532188842, + "recall@1000": 0.9570815450643777, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6204544287741587, + "ndcg_full@5": 0.6204544287741587, + "ndcg@10": 0.6446926872879769, + "ndcg_full@10": 0.6446926872879769, + "ndcg@20": 0.6226256652136167, + "ndcg_full@20": 0.6226256652136167, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.3, + "graded_P@10": 0.65 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.024691358024691357, + "recall@50": 0.10493827160493827, + "recall@100": 0.22839506172839505, + "recall@200": 0.4351851851851852, + "recall@300": 0.6203703703703703, + "recall@500": 0.8364197530864198, + "recall@1000": 0.9351851851851852, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8250166110242896, + "ndcg_full@10": 0.8250166110242896, + "ndcg@20": 0.8331670686625122, + "ndcg_full@20": 0.8331670686625122, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03488372093023256, + "recall@50": 0.16279069767441862, + "recall@100": 0.2596899224806202, + "recall@200": 0.437984496124031, + "recall@300": 0.6162790697674418, + "recall@500": 0.7635658914728682, + "recall@1000": 0.8023255813953488, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632229, + "ndcg_full@10": 0.9305687780632229, + "ndcg@20": 0.8824594286645165, + "ndcg_full@20": 0.8824594286645165, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 76, + "recall@10": 0.06666666666666667, + "recall@50": 0.28, + "recall@100": 0.52, + "recall@200": 0.6733333333333333, + "recall@300": 0.76, + "recall@500": 0.8, + "recall@1000": 0.84, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8734012415309734, + "ndcg_full@10": 0.8734012415309734, + "ndcg@20": 0.8305454061292459, + "ndcg_full@20": 0.8305454061292459, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 240, + "recall@10": 0.040697674418604654, + "recall@50": 0.14534883720930233, + "recall@100": 0.26744186046511625, + "recall@200": 0.436046511627907, + "recall@300": 0.5988372093023255, + "recall@500": 0.6918604651162791, + "recall@1000": 0.7558139534883721, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.38541299932114065, + "ndcg_full@10": 0.38541299932114065, + "ndcg@20": 0.4765544518323515, + "ndcg_full@20": 0.4765544518323515, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 99, + "recall@10": 0.11320754716981132, + "recall@50": 0.2830188679245283, + "recall@100": 0.39622641509433965, + "recall@200": 0.4339622641509434, + "recall@300": 0.49056603773584906, + "recall@500": 0.5283018867924528, + "recall@1000": 0.5660377358490566, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.835780413693672, + "ndcg_full@10": 0.835780413693672, + "ndcg@20": 0.7221876126184087, + "ndcg_full@20": 0.613280037554036, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 22, + "recall@10": 0.03349282296650718, + "recall@50": 0.14354066985645933, + "recall@100": 0.27751196172248804, + "recall@200": 0.40669856459330145, + "recall@300": 0.48325358851674644, + "recall@500": 0.569377990430622, + "recall@1000": 0.7320574162679426, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7799082337019199, + "ndcg_full@10": 0.7799082337019199, + "ndcg@20": 0.8413012793066663, + "ndcg_full@20": 0.5429483758821007, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 247, + "recall@10": 0.04, + "recall@50": 0.12666666666666668, + "recall@100": 0.22, + "recall@200": 0.4066666666666667, + "recall@300": 0.5333333333333333, + "recall@500": 0.66, + "recall@1000": 0.7733333333333333, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7852564605874278, + "ndcg_full@10": 0.7852564605874278, + "ndcg@20": 0.7723388959426615, + "ndcg_full@20": 0.7723388959426615, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 2000, + "num_ranked": 98, + "recall@10": 0.05747126436781609, + "recall@50": 0.2471264367816092, + "recall@100": 0.4540229885057471, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8713986118041845, + "ndcg_full@20": 0.8713986118041845, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.006472491909385114, + "recall@50": 0.061488673139158574, + "recall@100": 0.13268608414239483, + "recall@200": 0.2621359223300971, + "recall@300": 0.3333333333333333, + "recall@500": 0.4692556634304207, + "recall@1000": 0.6893203883495146, + "unjudged_fraction@10": 0.7, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9490083336684577, + "ndcg_full@20": 0.9490083336684577, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03225806451612903, + "recall@50": 0.18548387096774194, + "recall@100": 0.34274193548387094, + "recall@200": 0.5887096774193549, + "recall@300": 0.7338709677419355, + "recall@500": 0.9153225806451613, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.948494763225445, + "ndcg_full@20": 0.948494763225445, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 116, + "recall@10": 0.12345679012345678, + "recall@50": 0.5679012345679012, + "recall@100": 0.9506172839506173, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.9255244116680239, + "ndcg_full@5": 0.9255244116680239, + "ndcg@10": 0.8783065170406047, + "ndcg_full@10": 0.8783065170406047, + "ndcg@20": 0.8686742263053125, + "ndcg_full@20": 0.8686742263053125, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 140, + "recall@10": 0.13559322033898305, + "recall@50": 0.6779661016949152, + "recall@100": 0.8813559322033898, + "recall@200": 0.9491525423728814, + "recall@300": 0.9491525423728814, + "recall@500": 0.9661016949152542, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.7731287127294072, + "ndcg_full@10": 0.7731287127294072, + "ndcg@20": 0.7814545416787357, + "ndcg_full@20": 0.7814545416787357, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 68, + "recall@10": 0.5384615384615384, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.9059366708707217, + "ndcg_full@20": 0.8336754281378971, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032467532467532464, + "recall@50": 0.12012987012987013, + "recall@100": 0.22077922077922077, + "recall@200": 0.4383116883116883, + "recall@300": 0.6006493506493507, + "recall@500": 0.8344155844155844, + "recall@1000": 0.9285714285714286, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9574284411791895, + "ndcg_full@10": 0.9574284411791895, + "ndcg@20": 0.9173049253486589, + "ndcg_full@20": 0.9173049253486589, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.09859154929577464, + "recall@50": 0.323943661971831, + "recall@100": 0.4647887323943662, + "recall@200": 0.5352112676056338, + "recall@300": 0.5774647887323944, + "recall@500": 0.676056338028169, + "recall@1000": 0.7183098591549296, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9673983691420516, + "ndcg_full@20": 0.9673983691420516, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 153, + "recall@10": 0.05806451612903226, + "recall@50": 0.1935483870967742, + "recall@100": 0.3741935483870968, + "recall@200": 0.5741935483870968, + "recall@300": 0.7161290322580646, + "recall@500": 0.8580645161290322, + "recall@1000": 0.9225806451612903, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.8791360339581893, + "ndcg_full@10": 0.8791360339581893, + "ndcg@20": 0.7681720646347323, + "ndcg_full@20": 0.7681720646347323, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.05128205128205128, + "recall@50": 0.24615384615384617, + "recall@100": 0.4717948717948718, + "recall@200": 0.7128205128205128, + "recall@300": 0.7230769230769231, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7435897435897436, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7963019278608471, + "ndcg_full@5": 0.6859052454871424, + "ndcg@10": 0.892246906946592, + "ndcg_full@10": 0.7588074703200698, + "ndcg@20": 0.9215809282983327, + "ndcg_full@20": 0.8170183327686337, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.3, + "graded_P@10": 0.65 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.053763440860215055, + "recall@50": 0.25268817204301075, + "recall@100": 0.46774193548387094, + "recall@200": 0.8279569892473119, + "recall@300": 0.8817204301075269, + "recall@500": 0.8870967741935484, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9999999999999999, + "ndcg_full@20": 0.9999999999999999, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 167, + "recall@10": 0.08108108108108109, + "recall@50": 0.2972972972972973, + "recall@100": 0.5855855855855856, + "recall@200": 0.9279279279279279, + "recall@300": 0.9819819819819819, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.7510916729774053, + "ndcg_full@10": 0.7510916729774053, + "ndcg@20": 0.8030064049912891, + "ndcg_full@20": 0.8030064049912891, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03383458646616541, + "recall@50": 0.13533834586466165, + "recall@100": 0.2593984962406015, + "recall@200": 0.4699248120300752, + "recall@300": 0.6466165413533834, + "recall@500": 0.8007518796992481, + "recall@1000": 0.8984962406015038, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9321572773044168, + "ndcg_full@10": 0.9321572773044168, + "ndcg@20": 0.921571994265504, + "ndcg_full@20": 0.921571994265504, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 63, + "recall@10": 0.15151515151515152, + "recall@50": 0.6666666666666666, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8484848484848485, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.9289799801831713, + "ndcg_full@20": 0.9289799801831713, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 115, + "recall@10": 0.12987012987012986, + "recall@50": 0.4935064935064935, + "recall@100": 0.7402597402597403, + "recall@200": 0.7792207792207793, + "recall@300": 0.8051948051948052, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.888044966673629, + "ndcg_full@20": 0.888044966673629, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.7391304347826086, + "recall@300": 0.8260869565217391, + "recall@500": 0.8260869565217391, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6161372885453322, + "ndcg_full@10": 0.5965377185773229, + "ndcg@20": 0.6194232933630814, + "ndcg_full@20": 0.530516711679157, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 104, + "recall@10": 0.09859154929577464, + "recall@50": 0.4084507042253521, + "recall@100": 0.676056338028169, + "recall@200": 0.8732394366197183, + "recall@300": 0.8873239436619719, + "recall@500": 0.9154929577464789, + "recall@1000": 0.9577464788732394, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7590911424516827, + "ndcg_full@5": 0.7590911424516827, + "ndcg@10": 0.6677443797964433, + "ndcg_full@10": 0.6677443797964433, + "ndcg@20": 0.7846295899771154, + "ndcg_full@20": 0.7189349482914524, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 123, + "recall@10": 0.07352941176470588, + "recall@50": 0.39705882352941174, + "recall@100": 0.6323529411764706, + "recall@200": 0.75, + "recall@300": 0.7794117647058824, + "recall@500": 0.8088235294117647, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.5309999066793252, + "ndcg_full@10": 0.5309999066793252, + "ndcg@20": 0.4959172790384222, + "ndcg_full@20": 0.4959172790384222, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 184, + "recall@10": 0.08695652173913043, + "recall@50": 0.391304347826087, + "recall@100": 0.5869565217391305, + "recall@200": 0.6304347826086957, + "recall@300": 0.6304347826086957, + "recall@500": 0.6956521739130435, + "recall@1000": 0.7608695652173914, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9125583249939405, + "ndcg_full@20": 0.9125583249939405, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.12, + "recall@50": 0.5066666666666667, + "recall@100": 0.8666666666666667, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.5638177562145632, + "ndcg_full@10": 0.5638177562145632, + "ndcg@20": 0.5733033143863729, + "ndcg_full@20": 0.5733033143863729, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 117, + "recall@10": 0.20512820512820512, + "recall@50": 0.6666666666666666, + "recall@100": 0.6923076923076923, + "recall@200": 0.6923076923076923, + "recall@300": 0.7692307692307693, + "recall@500": 0.7692307692307693, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8165901947515999, + "ndcg_full@10": 0.8165901947515999, + "ndcg@20": 0.710227879078707, + "ndcg_full@20": 0.710227879078707, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 187, + "recall@10": 0.10465116279069768, + "recall@50": 0.38372093023255816, + "recall@100": 0.6511627906976745, + "recall@200": 0.9883720930232558, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.8108831892248393, + "ndcg_full@20": 0.8108831892248393, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 163, + "recall@10": 0.05555555555555555, + "recall@50": 0.19135802469135801, + "recall@100": 0.3271604938271605, + "recall@200": 0.5123456790123457, + "recall@300": 0.6296296296296297, + "recall@500": 0.7037037037037037, + "recall@1000": 0.8024691358024691, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9335645888032941, + "ndcg_full@10": 0.9335645888032941, + "ndcg@20": 0.9232974368381567, + "ndcg_full@20": 0.9232974368381567, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 96, + "recall@10": 0.07352941176470588, + "recall@50": 0.3088235294117647, + "recall@100": 0.5661764705882353, + "recall@200": 0.6691176470588235, + "recall@300": 0.6764705882352942, + "recall@500": 0.6838235294117647, + "recall@1000": 0.6911764705882353, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9651774792426178, + "ndcg_full@20": 0.9651774792426178, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1923, + "num_ranked": 186, + "recall@10": 0.02564102564102564, + "recall@50": 0.13105413105413105, + "recall@100": 0.24786324786324787, + "recall@200": 0.46438746438746437, + "recall@300": 0.6182336182336182, + "recall@500": 0.6324786324786325, + "recall@1000": 0.6353276353276354, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9775956781991643, + "ndcg_full@20": 0.9775956781991643, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 243, + "recall@10": 0.04020100502512563, + "recall@50": 0.1507537688442211, + "recall@100": 0.32160804020100503, + "recall@200": 0.5276381909547738, + "recall@300": 0.6683417085427136, + "recall@500": 0.8241206030150754, + "recall@1000": 0.949748743718593, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8513642017789715, + "ndcg_full@10": 0.8513642017789715, + "ndcg@20": 0.867719021027329, + "ndcg_full@20": 0.867719021027329, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02631578947368421, + "recall@50": 0.06315789473684211, + "recall@100": 0.11578947368421053, + "recall@200": 0.2789473684210526, + "recall@300": 0.41578947368421054, + "recall@500": 0.6578947368421053, + "recall@1000": 0.8894736842105263, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8970638154959308, + "ndcg_full@10": 0.8970638154959308, + "ndcg@20": 0.8247257280380528, + "ndcg_full@20": 0.8247257280380528, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.03272727272727273, + "recall@50": 0.15636363636363637, + "recall@100": 0.27636363636363637, + "recall@200": 0.4618181818181818, + "recall@300": 0.5527272727272727, + "recall@500": 0.6763636363636364, + "recall@1000": 0.7781818181818182, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6472260379105316, + "ndcg_full@5": 0.6472260379105316, + "ndcg@10": 0.7122751353112177, + "ndcg_full@10": 0.7122751353112177, + "ndcg@20": 0.7078103938346311, + "ndcg_full@20": 0.7078103938346311, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 157, + "recall@10": 0.09090909090909091, + "recall@50": 0.29545454545454547, + "recall@100": 0.4318181818181818, + "recall@200": 0.6136363636363636, + "recall@300": 0.7727272727272727, + "recall@500": 0.8863636363636364, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.6549028116336734, + "ndcg_full@10": 0.6549028116336734, + "ndcg@20": 0.6072750668600051, + "ndcg_full@20": 0.6072750668600051, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.024630541871921183, + "recall@50": 0.10344827586206896, + "recall@100": 0.21182266009852216, + "recall@200": 0.3694581280788177, + "recall@300": 0.541871921182266, + "recall@500": 0.8325123152709359, + "recall@1000": 0.9704433497536946, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.7189230730997338, + "ndcg_full@10": 0.7189230730997338, + "ndcg@20": 0.6550415857721922, + "ndcg_full@20": 0.6550415857721922, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02654867256637168, + "recall@50": 0.12389380530973451, + "recall@100": 0.26548672566371684, + "recall@200": 0.49557522123893805, + "recall@300": 0.6371681415929203, + "recall@500": 0.7168141592920354, + "recall@1000": 0.8584070796460177, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9350624941733298, + "ndcg_full@10": 0.9350624941733298, + "ndcg@20": 0.7450232445389794, + "ndcg_full@20": 0.7450232445389794, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 1971, + "num_ranked": 40, + "recall@10": 0.16666666666666666, + "recall@50": 0.25, + "recall@100": 0.2916666666666667, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8656312722365523, + "ndcg_full@10": 0.8656312722365523, + "ndcg@20": 0.8672595238008426, + "ndcg_full@20": 0.7248802934596406, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 167, + "recall@10": 0.06369426751592357, + "recall@50": 0.2611464968152866, + "recall@100": 0.4585987261146497, + "recall@200": 0.7261146496815286, + "recall@300": 0.7961783439490446, + "recall@500": 0.8280254777070064, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8017480615820849, + "ndcg_full@5": 0.8017480615820849, + "ndcg@10": 0.7815743531418785, + "ndcg_full@10": 0.7815743531418785, + "ndcg@20": 0.7915347558108304, + "ndcg_full@20": 0.7915347558108304, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06097560975609756, + "recall@50": 0.2621951219512195, + "recall@100": 0.36585365853658536, + "recall@200": 0.4451219512195122, + "recall@300": 0.5487804878048781, + "recall@500": 0.6646341463414634, + "recall@1000": 0.7378048780487805, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.7825823471446738, + "ndcg_full@10": 0.7825823471446738, + "ndcg@20": 0.6599773854570147, + "ndcg_full@20": 0.6599773854570147, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.012658227848101266, + "recall@50": 0.11392405063291139, + "recall@100": 0.27848101265822783, + "recall@200": 0.43037974683544306, + "recall@300": 0.5063291139240507, + "recall@500": 0.5443037974683544, + "recall@1000": 0.569620253164557, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.30078518014914984, + "ndcg_full@5": 0.30078518014914984, + "ndcg@10": 0.4469185010287862, + "ndcg_full@10": 0.4469185010287862, + "ndcg@20": 0.4628772000405961, + "ndcg_full@20": 0.4628772000405961, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03463203463203463, + "recall@50": 0.14285714285714285, + "recall@100": 0.2683982683982684, + "recall@200": 0.5411255411255411, + "recall@300": 0.7662337662337663, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.7, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9260852545422736, + "ndcg_full@10": 0.9260852545422736, + "ndcg@20": 0.8864955741517561, + "ndcg_full@20": 0.8864955741517561, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02717391304347826, + "recall@50": 0.14130434782608695, + "recall@100": 0.18478260869565216, + "recall@200": 0.375, + "recall@300": 0.46195652173913043, + "recall@500": 0.6739130434782609, + "recall@1000": 0.8532608695652174, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.9035757210193455, + "ndcg_full@20": 0.9035757210193455, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 73, + "recall@10": 0.0989010989010989, + "recall@50": 0.23076923076923078, + "recall@100": 0.24175824175824176, + "recall@200": 0.26373626373626374, + "recall@300": 0.27472527472527475, + "recall@500": 0.27472527472527475, + "recall@1000": 0.3076923076923077, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7204111965093014, + "ndcg_full@5": 0.7204111965093014, + "ndcg@10": 0.7131126393619082, + "ndcg_full@10": 0.7131126393619082, + "ndcg@20": 0.760827488323229, + "ndcg_full@20": 0.6995652129407604, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.13636363636363635, + "recall@50": 0.5909090909090909, + "recall@100": 0.8181818181818182, + "recall@200": 0.8939393939393939, + "recall@300": 0.9545454545454546, + "recall@500": 0.9848484848484849, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8630152897016883, + "ndcg_full@10": 0.8630152897016883, + "ndcg@20": 0.8671599472865638, + "ndcg_full@20": 0.809622269544087, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 67, + "recall@10": 0.5384615384615384, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.735182641392979, + "ndcg_full@5": 0.735182641392979, + "ndcg@10": 0.78964994195933, + "ndcg_full@10": 0.7645308661049757, + "ndcg@20": 0.84322849037214, + "ndcg_full@20": 0.7520203704032108, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.017595307917888565, + "recall@50": 0.11436950146627566, + "recall@100": 0.21700879765395895, + "recall@200": 0.4222873900293255, + "recall@300": 0.6041055718475073, + "recall@500": 0.8592375366568915, + "recall@1000": 0.9530791788856305, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9321572773044168, + "ndcg_full@10": 0.9321572773044168, + "ndcg@20": 0.8895783988547835, + "ndcg_full@20": 0.8895783988547835, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.08033430480469368, + "recall@50": 0.27627747582078804, + "recall@100": 0.4219591213693766, + "recall@200": 0.5871692603169641, + "recall@300": 0.6782443020293144, + "recall@500": 0.7718410466727308, + "recall@1000": 0.837050997496093, + "ndcg@5": 0.5993409139840837, + "ndcg@10": 0.5853233346289564, + "ndcg@20": 0.5751464650098916, + "ndcg_full@5": 0.597868958219101, + "ndcg_full@10": 0.5825852201115219, + "ndcg_full@20": 0.5609098561846095, + "P@10(rel>=1)": 0.6066666666666668, + "P@10(eligible)": 0.5373333333333334, + "graded_P@10": 0.5720000000000001, + "unjudged_fraction@10": 0.2826666666666667 + }, + "median": { + "recall@10": 0.05128205128205128, + "recall@50": 0.1935483870967742, + "recall@100": 0.34274193548387094, + "recall@200": 0.5352112676056338, + "recall@300": 0.6466165413533834, + "recall@500": 0.8089887640449438, + "recall@1000": 0.8717948717948718, + "ndcg@5": 0.6282776154664089, + "ndcg@10": 0.5970883119045803, + "ndcg@20": 0.5733033143863729, + "ndcg_full@5": 0.6282776154664089, + "ndcg_full@10": 0.5894280199323518, + "ndcg_full@20": 0.5443705630380683, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6, + "unjudged_fraction@10": 0.3 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 209, + "recall@10": 0.05917159763313609, + "recall@50": 0.25443786982248523, + "recall@100": 0.4437869822485207, + "recall@200": 0.7100591715976331, + "recall@300": 0.8106508875739645, + "recall@500": 0.8698224852071006, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.6512757762532352, + "ndcg_full@10": 0.6512757762532352, + "ndcg@20": 0.6123596597570286, + "ndcg_full@20": 0.6123596597570286, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.037037037037037035, + "recall@50": 0.17407407407407408, + "recall@100": 0.3, + "recall@200": 0.45555555555555555, + "recall@300": 0.5814814814814815, + "recall@500": 0.7296296296296296, + "recall@1000": 0.8481481481481481, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.77523873680463, + "ndcg_full@10": 0.77523873680463, + "ndcg@20": 0.6727060211238942, + "ndcg_full@20": 0.6727060211238942, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 164, + "recall@10": 0.05952380952380952, + "recall@50": 0.27380952380952384, + "recall@100": 0.4523809523809524, + "recall@200": 0.7142857142857143, + "recall@300": 0.7857142857142857, + "recall@500": 0.9047619047619048, + "recall@1000": 0.9523809523809523, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.42914113378860647, + "ndcg_full@10": 0.42914113378860647, + "ndcg@20": 0.3611895151145599, + "ndcg_full@20": 0.3433755122230877, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.042105263157894736, + "recall@50": 0.15789473684210525, + "recall@100": 0.23157894736842105, + "recall@200": 0.4421052631578947, + "recall@300": 0.5368421052631579, + "recall@500": 0.7789473684210526, + "recall@1000": 0.9052631578947369, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.07839826897867533, + "ndcg_full@10": 0.07839826897867533, + "ndcg@20": 0.19199079468911026, + "ndcg_full@20": 0.19199079468911026, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01990049751243781, + "recall@50": 0.1044776119402985, + "recall@100": 0.18407960199004975, + "recall@200": 0.29850746268656714, + "recall@300": 0.4427860696517413, + "recall@500": 0.5671641791044776, + "recall@1000": 0.736318407960199, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.39375843764607193, + "ndcg_full@10": 0.39375843764607193, + "ndcg@20": 0.42762063198137434, + "ndcg_full@20": 0.42762063198137434, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0380952380952381, + "recall@50": 0.15714285714285714, + "recall@100": 0.28095238095238095, + "recall@200": 0.47619047619047616, + "recall@300": 0.6285714285714286, + "recall@500": 0.8238095238095238, + "recall@1000": 0.8857142857142857, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5289635809537698, + "ndcg_full@5": 0.5289635809537698, + "ndcg@10": 0.5295340122986827, + "ndcg_full@10": 0.5295340122986827, + "ndcg@20": 0.6273738180328927, + "ndcg_full@20": 0.6273738180328927, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018633540372670808, + "recall@50": 0.08074534161490683, + "recall@100": 0.14285714285714285, + "recall@200": 0.3105590062111801, + "recall@300": 0.4161490683229814, + "recall@500": 0.639751552795031, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.15782422623975725, + "ndcg_full@5": 0.15782422623975725, + "ndcg@10": 0.20497545742326723, + "ndcg_full@10": 0.20497545742326723, + "ndcg@20": 0.27121842060658197, + "ndcg_full@20": 0.27121842060658197, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.2222222222222222, + "recall@100": 0.41830065359477125, + "recall@200": 0.7058823529411765, + "recall@300": 0.8431372549019608, + "recall@500": 0.9281045751633987, + "recall@1000": 0.9411764705882353, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7136540102475308, + "ndcg_full@10": 0.7136540102475308, + "ndcg@20": 0.5715597859894963, + "ndcg_full@20": 0.5715597859894963, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.21962616822429906, + "recall@100": 0.40654205607476634, + "recall@200": 0.7383177570093458, + "recall@300": 0.9112149532710281, + "recall@500": 0.9299065420560748, + "recall@1000": 0.9392523364485982, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.10699313236726378, + "ndcg_full@5": 0.10699313236726378, + "ndcg@10": 0.24950708869969282, + "ndcg_full@10": 0.24950708869969282, + "ndcg@20": 0.3899527367074409, + "ndcg_full@20": 0.364413506048076, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 85, + "recall@10": 0.19148936170212766, + "recall@50": 0.8085106382978723, + "recall@100": 0.8085106382978723, + "recall@200": 0.8297872340425532, + "recall@300": 0.8297872340425532, + "recall@500": 0.851063829787234, + "recall@1000": 0.8723404255319149, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7618017901203944, + "ndcg_full@5": 0.7618017901203944, + "ndcg@10": 0.5988819194987114, + "ndcg_full@10": 0.5988819194987114, + "ndcg@20": 0.7242905647855642, + "ndcg_full@20": 0.7072594892179145, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 156, + "recall@10": 0.07936507936507936, + "recall@50": 0.35714285714285715, + "recall@100": 0.5396825396825397, + "recall@200": 0.7936507936507936, + "recall@300": 0.8412698412698413, + "recall@500": 0.8492063492063492, + "recall@1000": 0.8650793650793651, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.7603851774361361, + "ndcg_full@20": 0.7603851774361361, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.06040268456375839, + "recall@50": 0.26174496644295303, + "recall@100": 0.46308724832214765, + "recall@200": 0.7919463087248322, + "recall@300": 0.8389261744966443, + "recall@500": 0.8590604026845637, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.702728403557943, + "ndcg_full@10": 0.702728403557943, + "ndcg@20": 0.7717943300431969, + "ndcg_full@20": 0.7717943300431969, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 198, + "recall@10": 0.036231884057971016, + "recall@50": 0.10869565217391304, + "recall@100": 0.18840579710144928, + "recall@200": 0.34782608695652173, + "recall@300": 0.4927536231884058, + "recall@500": 0.6304347826086957, + "recall@1000": 0.6956521739130435, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8482378089219645, + "ndcg_full@10": 0.8482378089219645, + "ndcg@20": 0.6306406353587494, + "ndcg_full@20": 0.6306406353587494, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01020408163265306, + "recall@50": 0.061224489795918366, + "recall@100": 0.13945578231292516, + "recall@200": 0.2755102040816326, + "recall@300": 0.3707482993197279, + "recall@500": 0.5272108843537415, + "recall@1000": 0.6870748299319728, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.5481540639795447, + "ndcg_full@10": 0.5481540639795447, + "ndcg@20": 0.5122732174924165, + "ndcg_full@20": 0.5122732174924165, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.026923076923076925, + "recall@50": 0.1423076923076923, + "recall@100": 0.26153846153846155, + "recall@200": 0.3923076923076923, + "recall@300": 0.4807692307692308, + "recall@500": 0.5807692307692308, + "recall@1000": 0.6884615384615385, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.7748652363999737, + "ndcg_full@10": 0.7748652363999737, + "ndcg@20": 0.6942803465774617, + "ndcg_full@20": 0.6942803465774617, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.10144927536231885, + "recall@100": 0.17391304347826086, + "recall@200": 0.2753623188405797, + "recall@300": 0.427536231884058, + "recall@500": 0.5869565217391305, + "recall@1000": 0.7246376811594203, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.08514311764162098, + "ndcg_full@10": 0.08514311764162098, + "ndcg@20": 0.07166733376434263, + "ndcg_full@20": 0.07166733376434263, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035856573705179286, + "recall@50": 0.15139442231075698, + "recall@100": 0.2868525896414343, + "recall@200": 0.6095617529880478, + "recall@300": 0.8286852589641435, + "recall@500": 0.9561752988047809, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.35762729995475817, + "ndcg_full@10": 0.35762729995475817, + "ndcg@20": 0.369246164210239, + "ndcg_full@20": 0.369246164210239, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.043689320388349516, + "recall@50": 0.14563106796116504, + "recall@100": 0.2766990291262136, + "recall@200": 0.5, + "recall@300": 0.6262135922330098, + "recall@500": 0.8009708737864077, + "recall@1000": 0.9077669902912622, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.13216795894527678, + "ndcg_full@5": 0.13216795894527678, + "ndcg@10": 0.3416369534475153, + "ndcg_full@10": 0.307591766224102, + "ndcg@20": 0.5439073277056091, + "ndcg_full@20": 0.4590708451427141, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.1895424836601307, + "recall@100": 0.3137254901960784, + "recall@200": 0.5620915032679739, + "recall@300": 0.6993464052287581, + "recall@500": 0.8300653594771242, + "recall@1000": 0.9084967320261438, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.28902930375209906, + "ndcg_full@5": 0.28902930375209906, + "ndcg@10": 0.46526474396970974, + "ndcg_full@10": 0.46526474396970974, + "ndcg@20": 0.4731148157576676, + "ndcg_full@20": 0.4731148157576676, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 193, + "recall@10": 0.12857142857142856, + "recall@50": 0.38571428571428573, + "recall@100": 0.7142857142857143, + "recall@200": 0.9142857142857143, + "recall@300": 0.9285714285714286, + "recall@500": 0.9428571428571428, + "recall@1000": 0.9571428571428572, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.669580102636808, + "ndcg_full@5": 0.669580102636808, + "ndcg@10": 0.5078755929143235, + "ndcg_full@10": 0.5078755929143235, + "ndcg@20": 0.5200807547341597, + "ndcg_full@20": 0.5200807547341597, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 246, + "recall@10": 0.1320754716981132, + "recall@50": 0.5660377358490566, + "recall@100": 0.7547169811320755, + "recall@200": 0.9056603773584906, + "recall@300": 0.9245283018867925, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.5157051110116302, + "ndcg_full@5": 0.5157051110116302, + "ndcg@10": 0.33465733009546195, + "ndcg_full@10": 0.33465733009546195, + "ndcg@20": 0.429098225894732, + "ndcg_full@20": 0.429098225894732, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 203, + "recall@10": 0.06569343065693431, + "recall@50": 0.291970802919708, + "recall@100": 0.5182481751824818, + "recall@200": 0.7445255474452555, + "recall@300": 0.7591240875912408, + "recall@500": 0.8102189781021898, + "recall@1000": 0.8613138686131386, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5491264122284197, + "ndcg_full@5": 0.5491264122284197, + "ndcg@10": 0.5481961796134488, + "ndcg_full@10": 0.5481961796134488, + "ndcg@20": 0.49482375774161697, + "ndcg_full@20": 0.49482375774161697, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 237, + "recall@10": 0.033707865168539325, + "recall@50": 0.12359550561797752, + "recall@100": 0.25842696629213485, + "recall@200": 0.45692883895131087, + "recall@300": 0.6104868913857678, + "recall@500": 0.8089887640449438, + "recall@1000": 0.8726591760299626, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5773584151532217, + "ndcg_full@5": 0.5773584151532217, + "ndcg@10": 0.6563033086078546, + "ndcg_full@10": 0.6563033086078546, + "ndcg@20": 0.6369111959803669, + "ndcg_full@20": 0.6369111959803669, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0189873417721519, + "recall@50": 0.11392405063291139, + "recall@100": 0.23417721518987342, + "recall@200": 0.4430379746835443, + "recall@300": 0.6139240506329114, + "recall@500": 0.7974683544303798, + "recall@1000": 0.9493670886075949, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.40578566530816096, + "ndcg_full@10": 0.40578566530816096, + "ndcg@20": 0.4310460877190185, + "ndcg_full@20": 0.4310460877190185, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.09803921568627451, + "recall@100": 0.22549019607843138, + "recall@200": 0.38235294117647056, + "recall@300": 0.5, + "recall@500": 0.6568627450980392, + "recall@1000": 0.8137254901960784, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.2765732350040718, + "ndcg_full@5": 0.2765732350040718, + "ndcg@10": 0.32693938042902576, + "ndcg_full@10": 0.32693938042902576, + "ndcg@20": 0.3136547509954461, + "ndcg_full@20": 0.3136547509954461, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.0625, + "recall@50": 0.1875, + "recall@100": 0.34375, + "recall@200": 0.5625, + "recall@300": 0.65, + "recall@500": 0.775, + "recall@1000": 0.89375, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.7329604910053826, + "ndcg_full@10": 0.7329604910053826, + "ndcg@20": 0.5848710978032684, + "ndcg_full@20": 0.5848710978032684, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.14163090128755365, + "recall@100": 0.24892703862660945, + "recall@200": 0.463519313304721, + "recall@300": 0.6223175965665236, + "recall@500": 0.8540772532188842, + "recall@1000": 0.9570815450643777, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6204544287741587, + "ndcg_full@5": 0.6204544287741587, + "ndcg@10": 0.5421343536240051, + "ndcg_full@10": 0.5421343536240051, + "ndcg@20": 0.4422822108000591, + "ndcg_full@20": 0.4422822108000591, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.024691358024691357, + "recall@50": 0.10493827160493827, + "recall@100": 0.22839506172839505, + "recall@200": 0.4351851851851852, + "recall@300": 0.6203703703703703, + "recall@500": 0.8364197530864198, + "recall@1000": 0.9351851851851852, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.2477742047067549, + "ndcg_full@10": 0.2477742047067549, + "ndcg@20": 0.3531597477722202, + "ndcg_full@20": 0.3531597477722202, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03488372093023256, + "recall@50": 0.16279069767441862, + "recall@100": 0.2596899224806202, + "recall@200": 0.437984496124031, + "recall@300": 0.6162790697674418, + "recall@500": 0.7635658914728682, + "recall@1000": 0.8023255813953488, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5385625784044538, + "ndcg_full@5": 0.5385625784044538, + "ndcg@10": 0.6002534228960357, + "ndcg_full@10": 0.6002534228960357, + "ndcg@20": 0.5253967142840115, + "ndcg_full@20": 0.5253967142840115, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 76, + "recall@10": 0.06666666666666667, + "recall@50": 0.28, + "recall@100": 0.52, + "recall@200": 0.6733333333333333, + "recall@300": 0.76, + "recall@500": 0.8, + "recall@1000": 0.84, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.7026305163854645, + "ndcg_full@10": 0.7026305163854645, + "ndcg@20": 0.6840258201832651, + "ndcg_full@20": 0.6840258201832651, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 240, + "recall@10": 0.040697674418604654, + "recall@50": 0.14534883720930233, + "recall@100": 0.26744186046511625, + "recall@200": 0.436046511627907, + "recall@300": 0.5988372093023255, + "recall@500": 0.6918604651162791, + "recall@1000": 0.7558139534883721, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.38541299932114065, + "ndcg_full@10": 0.38541299932114065, + "ndcg@20": 0.35149845591358814, + "ndcg_full@20": 0.35149845591358814, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 99, + "recall@10": 0.11320754716981132, + "recall@50": 0.2830188679245283, + "recall@100": 0.39622641509433965, + "recall@200": 0.4339622641509434, + "recall@300": 0.49056603773584906, + "recall@500": 0.5283018867924528, + "recall@1000": 0.5660377358490566, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.4073174795910971, + "ndcg_full@10": 0.4073174795910971, + "ndcg@20": 0.5228467718940314, + "ndcg_full@20": 0.4440002601811514, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 22, + "recall@10": 0.03349282296650718, + "recall@50": 0.14354066985645933, + "recall@100": 0.27751196172248804, + "recall@200": 0.40669856459330145, + "recall@300": 0.48325358851674644, + "recall@500": 0.569377990430622, + "recall@1000": 0.7320574162679426, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.5747915663739762, + "ndcg_full@10": 0.5747915663739762, + "ndcg@20": 0.7484425540550442, + "ndcg_full@20": 0.483020386585091, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 247, + "recall@10": 0.04, + "recall@50": 0.12666666666666668, + "recall@100": 0.22, + "recall@200": 0.4066666666666667, + "recall@300": 0.5333333333333333, + "recall@500": 0.66, + "recall@1000": 0.7733333333333333, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.5661326586116142, + "ndcg_full@10": 0.5661326586116142, + "ndcg@20": 0.4043663853753545, + "ndcg_full@20": 0.4043663853753545, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 2000, + "num_ranked": 98, + "recall@10": 0.05747126436781609, + "recall@50": 0.2471264367816092, + "recall@100": 0.4540229885057471, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8713986118041845, + "ndcg_full@20": 0.8713986118041845, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.006472491909385114, + "recall@50": 0.061488673139158574, + "recall@100": 0.13268608414239483, + "recall@200": 0.2621359223300971, + "recall@300": 0.3333333333333333, + "recall@500": 0.4692556634304207, + "recall@1000": 0.6893203883495146, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.21128749918529321, + "ndcg_full@10": 0.21128749918529321, + "ndcg@20": 0.2927239176267991, + "ndcg_full@20": 0.2927239176267991, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03225806451612903, + "recall@50": 0.18548387096774194, + "recall@100": 0.34274193548387094, + "recall@200": 0.5887096774193549, + "recall@300": 0.7338709677419355, + "recall@500": 0.9153225806451613, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.84860265890399, + "ndcg_full@10": 0.84860265890399, + "ndcg@20": 0.8297841410906505, + "ndcg_full@20": 0.8297841410906505, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 116, + "recall@10": 0.12345679012345678, + "recall@50": 0.5679012345679012, + "recall@100": 0.9506172839506173, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.9255244116680239, + "ndcg_full@5": 0.9255244116680239, + "ndcg@10": 0.8088752951038273, + "ndcg_full@10": 0.8088752951038273, + "ndcg@20": 0.7753685635345205, + "ndcg_full@20": 0.7753685635345205, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 140, + "recall@10": 0.13559322033898305, + "recall@50": 0.6779661016949152, + "recall@100": 0.8813559322033898, + "recall@200": 0.9491525423728814, + "recall@300": 0.9491525423728814, + "recall@500": 0.9661016949152542, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.7029417891124354, + "ndcg_full@10": 0.7029417891124354, + "ndcg@20": 0.6806040661857976, + "ndcg_full@20": 0.6806040661857976, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 68, + "recall@10": 0.5384615384615384, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8512360941594274, + "ndcg_full@10": 0.8512360941594274, + "ndcg@20": 0.8854489363006828, + "ndcg_full@20": 0.8148218797184069, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032467532467532464, + "recall@50": 0.12012987012987013, + "recall@100": 0.22077922077922077, + "recall@200": 0.4383116883116883, + "recall@300": 0.6006493506493507, + "recall@500": 0.8344155844155844, + "recall@1000": 0.9285714285714286, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.4398841196769108, + "ndcg_full@10": 0.4398841196769108, + "ndcg@20": 0.42588069842964194, + "ndcg_full@20": 0.42588069842964194, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.09859154929577464, + "recall@50": 0.323943661971831, + "recall@100": 0.4647887323943662, + "recall@200": 0.5352112676056338, + "recall@300": 0.5774647887323944, + "recall@500": 0.676056338028169, + "recall@1000": 0.7183098591549296, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8957954109547271, + "ndcg_full@20": 0.8957954109547271, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 153, + "recall@10": 0.05806451612903226, + "recall@50": 0.1935483870967742, + "recall@100": 0.3741935483870968, + "recall@200": 0.5741935483870968, + "recall@300": 0.7161290322580646, + "recall@500": 0.8580645161290322, + "recall@1000": 0.9225806451612903, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.8064798752202919, + "ndcg_full@10": 0.8064798752202919, + "ndcg@20": 0.7389664462112809, + "ndcg_full@20": 0.7389664462112809, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.05128205128205128, + "recall@50": 0.24615384615384617, + "recall@100": 0.4717948717948718, + "recall@200": 0.7128205128205128, + "recall@300": 0.7230769230769231, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7435897435897436, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7963019278608471, + "ndcg_full@5": 0.6859052454871424, + "ndcg@10": 0.8480416641273905, + "ndcg_full@10": 0.7212133153643377, + "ndcg@20": 0.8894098415295254, + "ndcg_full@20": 0.7884973783216497, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.053763440860215055, + "recall@50": 0.25268817204301075, + "recall@100": 0.46774193548387094, + "recall@200": 0.8279569892473119, + "recall@300": 0.8817204301075269, + "recall@500": 0.8870967741935484, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8643145546088337, + "ndcg_full@10": 0.8643145546088337, + "ndcg@20": 0.8728119768274811, + "ndcg_full@20": 0.8728119768274811, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 167, + "recall@10": 0.08108108108108109, + "recall@50": 0.2972972972972973, + "recall@100": 0.5855855855855856, + "recall@200": 0.9279279279279279, + "recall@300": 0.9819819819819819, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.7510916729774053, + "ndcg_full@10": 0.7510916729774053, + "ndcg@20": 0.8030064049912891, + "ndcg_full@20": 0.8030064049912891, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03383458646616541, + "recall@50": 0.13533834586466165, + "recall@100": 0.2593984962406015, + "recall@200": 0.4699248120300752, + "recall@300": 0.6466165413533834, + "recall@500": 0.8007518796992481, + "recall@1000": 0.8984962406015038, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.6453699114846652, + "ndcg_full@10": 0.6453699114846652, + "ndcg@20": 0.5920404001584784, + "ndcg_full@20": 0.5920404001584784, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 63, + "recall@10": 0.15151515151515152, + "recall@50": 0.6666666666666666, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8484848484848485, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.9289799801831713, + "ndcg_full@20": 0.9289799801831713, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 115, + "recall@10": 0.12987012987012986, + "recall@50": 0.4935064935064935, + "recall@100": 0.7402597402597403, + "recall@200": 0.7792207792207793, + "recall@300": 0.8051948051948052, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.7534494445524138, + "ndcg_full@10": 0.7534494445524138, + "ndcg@20": 0.773795857922277, + "ndcg_full@20": 0.773795857922277, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.7391304347826086, + "recall@300": 0.8260869565217391, + "recall@500": 0.8260869565217391, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6087939968991074, + "ndcg_full@10": 0.5894280199323518, + "ndcg@20": 0.5394978255244429, + "ndcg_full@20": 0.4620630438376436, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 104, + "recall@10": 0.09859154929577464, + "recall@50": 0.4084507042253521, + "recall@100": 0.676056338028169, + "recall@200": 0.8732394366197183, + "recall@300": 0.8873239436619719, + "recall@500": 0.9154929577464789, + "recall@1000": 0.9577464788732394, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.627886064939341, + "ndcg_full@5": 0.627886064939341, + "ndcg@10": 0.5470732388037497, + "ndcg_full@10": 0.5470732388037497, + "ndcg@20": 0.5573353291637821, + "ndcg_full@20": 0.5106713424675328, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 123, + "recall@10": 0.07352941176470588, + "recall@50": 0.39705882352941174, + "recall@100": 0.6323529411764706, + "recall@200": 0.75, + "recall@300": 0.7794117647058824, + "recall@500": 0.8088235294117647, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.4362115423097744, + "ndcg_full@10": 0.4362115423097744, + "ndcg@20": 0.319901324593464, + "ndcg_full@20": 0.319901324593464, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 184, + "recall@10": 0.08695652173913043, + "recall@50": 0.391304347826087, + "recall@100": 0.5869565217391305, + "recall@200": 0.6304347826086957, + "recall@300": 0.6304347826086957, + "recall@500": 0.6956521739130435, + "recall@1000": 0.7608695652173914, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.8191007513227553, + "ndcg_full@20": 0.8191007513227553, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.12, + "recall@50": 0.5066666666666667, + "recall@100": 0.8666666666666667, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.5638177562145632, + "ndcg_full@10": 0.5638177562145632, + "ndcg@20": 0.5733033143863729, + "ndcg_full@20": 0.5733033143863729, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 117, + "recall@10": 0.20512820512820512, + "recall@50": 0.6666666666666666, + "recall@100": 0.6923076923076923, + "recall@200": 0.6923076923076923, + "recall@300": 0.7692307692307693, + "recall@500": 0.7692307692307693, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.568003577662161, + "ndcg_full@10": 0.568003577662161, + "ndcg@20": 0.5066249248121251, + "ndcg_full@20": 0.5066249248121251, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 187, + "recall@10": 0.10465116279069768, + "recall@50": 0.38372093023255816, + "recall@100": 0.6511627906976745, + "recall@200": 0.9883720930232558, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.7785448980765195, + "ndcg_full@20": 0.7785448980765195, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 163, + "recall@10": 0.05555555555555555, + "recall@50": 0.19135802469135801, + "recall@100": 0.3271604938271605, + "recall@200": 0.5123456790123457, + "recall@300": 0.6296296296296297, + "recall@500": 0.7037037037037037, + "recall@1000": 0.8024691358024691, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6627919420317118, + "ndcg_full@10": 0.6627919420317118, + "ndcg@20": 0.5984638563253503, + "ndcg_full@20": 0.5984638563253503, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 96, + "recall@10": 0.07352941176470588, + "recall@50": 0.3088235294117647, + "recall@100": 0.5661764705882353, + "recall@200": 0.6691176470588235, + "recall@300": 0.6764705882352942, + "recall@500": 0.6838235294117647, + "recall@1000": 0.6911764705882353, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9422007889499151, + "ndcg_full@20": 0.9422007889499151, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1923, + "num_ranked": 186, + "recall@10": 0.02564102564102564, + "recall@50": 0.13105413105413105, + "recall@100": 0.24786324786324787, + "recall@200": 0.46438746438746437, + "recall@300": 0.6182336182336182, + "recall@500": 0.6324786324786325, + "recall@1000": 0.6353276353276354, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8071470180765843, + "ndcg_full@10": 0.8071470180765843, + "ndcg@20": 0.8371543442481091, + "ndcg_full@20": 0.8371543442481091, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 243, + "recall@10": 0.04020100502512563, + "recall@50": 0.1507537688442211, + "recall@100": 0.32160804020100503, + "recall@200": 0.5276381909547738, + "recall@300": 0.6683417085427136, + "recall@500": 0.8241206030150754, + "recall@1000": 0.949748743718593, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.565602538756171, + "ndcg_full@5": 0.565602538756171, + "ndcg@10": 0.5490316250122088, + "ndcg_full@10": 0.5490316250122088, + "ndcg@20": 0.5443705630380683, + "ndcg_full@20": 0.5443705630380683, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02631578947368421, + "recall@50": 0.06315789473684211, + "recall@100": 0.11578947368421053, + "recall@200": 0.2789473684210526, + "recall@300": 0.41578947368421054, + "recall@500": 0.6578947368421053, + "recall@1000": 0.8894736842105263, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.6653882795046744, + "ndcg_full@10": 0.6653882795046744, + "ndcg@20": 0.48310802354293847, + "ndcg_full@20": 0.48310802354293847, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.03272727272727273, + "recall@50": 0.15636363636363637, + "recall@100": 0.27636363636363637, + "recall@200": 0.4618181818181818, + "recall@300": 0.5527272727272727, + "recall@500": 0.6763636363636364, + "recall@1000": 0.7781818181818182, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5433994577073111, + "ndcg_full@5": 0.5433994577073111, + "ndcg@10": 0.4600008358307626, + "ndcg_full@10": 0.4600008358307626, + "ndcg@20": 0.5985375361730464, + "ndcg_full@20": 0.5985375361730464, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 157, + "recall@10": 0.09090909090909091, + "recall@50": 0.29545454545454547, + "recall@100": 0.4318181818181818, + "recall@200": 0.6136363636363636, + "recall@300": 0.7727272727272727, + "recall@500": 0.8863636363636364, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.3296507969226586, + "ndcg_full@10": 0.3296507969226586, + "ndcg@20": 0.2776712439643846, + "ndcg_full@20": 0.2776712439643846, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.024630541871921183, + "recall@50": 0.10344827586206896, + "recall@100": 0.21182266009852216, + "recall@200": 0.3694581280788177, + "recall@300": 0.541871921182266, + "recall@500": 0.8325123152709359, + "recall@1000": 0.9704433497536946, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.3499667779514209, + "ndcg_full@10": 0.3499667779514209, + "ndcg@20": 0.336203380182094, + "ndcg_full@20": 0.336203380182094, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02654867256637168, + "recall@50": 0.12389380530973451, + "recall@100": 0.26548672566371684, + "recall@200": 0.49557522123893805, + "recall@300": 0.6371681415929203, + "recall@500": 0.7168141592920354, + "recall@1000": 0.8584070796460177, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3544290542068402, + "ndcg_full@5": 0.3544290542068402, + "ndcg@10": 0.4534423562354316, + "ndcg_full@10": 0.4534423562354316, + "ndcg@20": 0.42222235862934604, + "ndcg_full@20": 0.42222235862934604, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 1971, + "num_ranked": 40, + "recall@10": 0.16666666666666666, + "recall@50": 0.25, + "recall@100": 0.2916666666666667, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.48068140169045714, + "ndcg_full@10": 0.48068140169045714, + "ndcg@20": 0.6235059027790082, + "ndcg_full@20": 0.5211440513209692, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 167, + "recall@10": 0.06369426751592357, + "recall@50": 0.2611464968152866, + "recall@100": 0.4585987261146497, + "recall@200": 0.7261146496815286, + "recall@300": 0.7961783439490446, + "recall@500": 0.8280254777070064, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6282776154664089, + "ndcg_full@5": 0.6282776154664089, + "ndcg@10": 0.5970883119045803, + "ndcg_full@10": 0.5970883119045803, + "ndcg@20": 0.6660793041165426, + "ndcg_full@20": 0.6660793041165426, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06097560975609756, + "recall@50": 0.2621951219512195, + "recall@100": 0.36585365853658536, + "recall@200": 0.4451219512195122, + "recall@300": 0.5487804878048781, + "recall@500": 0.6646341463414634, + "recall@1000": 0.7378048780487805, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3378515335283742, + "ndcg_full@5": 0.3378515335283742, + "ndcg@10": 0.36707201879630064, + "ndcg_full@10": 0.36707201879630064, + "ndcg@20": 0.3914966618470787, + "ndcg_full@20": 0.3914966618470787, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.012658227848101266, + "recall@50": 0.11392405063291139, + "recall@100": 0.27848101265822783, + "recall@200": 0.43037974683544306, + "recall@300": 0.5063291139240507, + "recall@500": 0.5443037974683544, + "recall@1000": 0.569620253164557, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.13863671367752411, + "ndcg_full@5": 0.13863671367752411, + "ndcg@10": 0.08996574100558588, + "ndcg_full@10": 0.08996574100558588, + "ndcg@20": 0.13087771199986784, + "ndcg_full@20": 0.13087771199986784, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03463203463203463, + "recall@50": 0.14285714285714285, + "recall@100": 0.2683982683982684, + "recall@200": 0.5411255411255411, + "recall@300": 0.7662337662337663, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.24966402870278917, + "ndcg_full@10": 0.24966402870278917, + "ndcg@20": 0.4793492836123742, + "ndcg_full@20": 0.4793492836123742, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02717391304347826, + "recall@50": 0.14130434782608695, + "recall@100": 0.18478260869565216, + "recall@200": 0.375, + "recall@300": 0.46195652173913043, + "recall@500": 0.6739130434782609, + "recall@1000": 0.8532608695652174, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.6334783693634827, + "ndcg_full@10": 0.6334783693634827, + "ndcg@20": 0.44169122611041584, + "ndcg_full@20": 0.44169122611041584, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 73, + "recall@10": 0.0989010989010989, + "recall@50": 0.23076923076923078, + "recall@100": 0.24175824175824176, + "recall@200": 0.26373626373626374, + "recall@300": 0.27472527472527475, + "recall@500": 0.27472527472527475, + "recall@1000": 0.3076923076923077, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6364925547468551, + "ndcg_full@10": 0.6364925547468551, + "ndcg@20": 0.6108389342762156, + "ndcg_full@20": 0.5616538251939512, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.13636363636363635, + "recall@50": 0.5909090909090909, + "recall@100": 0.8181818181818182, + "recall@200": 0.8939393939393939, + "recall@300": 0.9545454545454546, + "recall@500": 0.9848484848484849, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7791714369672127, + "ndcg_full@10": 0.7791714369672127, + "ndcg@20": 0.7276806217353219, + "ndcg_full@20": 0.6793976570482826, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 67, + "recall@10": 0.5384615384615384, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.735182641392979, + "ndcg_full@5": 0.735182641392979, + "ndcg@10": 0.78964994195933, + "ndcg_full@10": 0.7645308661049757, + "ndcg@20": 0.7653834047107615, + "ndcg_full@20": 0.682595427079363, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.017595307917888565, + "recall@50": 0.11436950146627566, + "recall@100": 0.21700879765395895, + "recall@200": 0.4222873900293255, + "recall@300": 0.6041055718475073, + "recall@500": 0.8592375366568915, + "recall@1000": 0.9530791788856305, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.7832109464728314, + "ndcg_full@10": 0.7832109464728314, + "ndcg@20": 0.7887502795911117, + "ndcg_full@20": 0.7887502795911117, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + } + } + } + } + }, + { + "name": "h100_baichuan", + "results_dir": "h100_baichuan/results_trec21", + "evaluation_input_sha256": "c1bf01e08c7d79779d38e2e46bedad87d4881cd617ff2f5479ab78ed45cf5a4a", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.08201653038371211, + "recall@50": 0.2820748984045244, + "recall@100": 0.42521830967576446, + "recall@200": 0.5873393737363496, + "recall@300": 0.678937984786285, + "recall@500": 0.7726304624552182, + "recall@1000": 0.8362067497153736, + "ndcg@5": 0.8301100845175586, + "ndcg@10": 0.8224955649390504, + "ndcg@20": 0.7997381294026057, + "ndcg_full@5": 0.828491202748756, + "ndcg_full@10": 0.8178471054898836, + "ndcg_full@20": 0.7866779994796045, + "P@10(rel>=1)": 0.8653333333333331, + "P@10(eligible)": 0.7399999999999999, + "graded_P@10": 0.8026666666666668, + "unjudged_fraction@10": 0.2613333333333334 + }, + "median": { + "recall@10": 0.05228758169934641, + "recall@50": 0.2222222222222222, + "recall@100": 0.3629032258064516, + "recall@200": 0.5541125541125541, + "recall@300": 0.6764705882352942, + "recall@500": 0.7933333333333333, + "recall@1000": 0.8698224852071006, + "ndcg@5": 0.8757862866154155, + "ndcg@10": 0.8521705090845474, + "ndcg@20": 0.8244208909022631, + "ndcg_full@5": 0.8687949224876582, + "ndcg_full@10": 0.8512360941594274, + "ndcg_full@20": 0.8215843983276304, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85, + "unjudged_fraction@10": 0.2 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 246, + "recall@10": 0.05325443786982249, + "recall@50": 0.26627218934911245, + "recall@100": 0.4556213017751479, + "recall@200": 0.6923076923076923, + "recall@300": 0.7988165680473372, + "recall@500": 0.863905325443787, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7883291114011227, + "ndcg_full@5": 0.7883291114011227, + "ndcg@10": 0.7269546314184724, + "ndcg_full@10": 0.7269546314184724, + "ndcg@20": 0.7220843982479043, + "ndcg_full@20": 0.7220843982479043, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02962962962962963, + "recall@50": 0.12222222222222222, + "recall@100": 0.21851851851851853, + "recall@200": 0.3962962962962963, + "recall@300": 0.5074074074074074, + "recall@500": 0.6888888888888889, + "recall@1000": 0.8296296296296296, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6578242262397573, + "ndcg_full@5": 0.6578242262397573, + "ndcg@10": 0.7779513360933721, + "ndcg_full@10": 0.7779513360933721, + "ndcg@20": 0.8170759754896911, + "ndcg_full@20": 0.8170759754896911, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.08333333333333333, + "recall@50": 0.39285714285714285, + "recall@100": 0.5952380952380952, + "recall@200": 0.7857142857142857, + "recall@300": 0.8333333333333334, + "recall@500": 0.9523809523809523, + "recall@1000": 0.9761904761904762, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.7965812792043342, + "ndcg_full@10": 0.7965812792043342, + "ndcg@20": 0.7174474844247547, + "ndcg_full@20": 0.7058469716072759, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.18947368421052632, + "recall@100": 0.35789473684210527, + "recall@200": 0.5368421052631579, + "recall@300": 0.7368421052631579, + "recall@500": 0.8736842105263158, + "recall@1000": 0.9368421052631579, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8042418012076715, + "ndcg_full@10": 0.8042418012076715, + "ndcg@20": 0.7366236960910839, + "ndcg_full@20": 0.7366236960910839, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01990049751243781, + "recall@50": 0.0945273631840796, + "recall@100": 0.16417910447761194, + "recall@200": 0.2935323383084577, + "recall@300": 0.4427860696517413, + "recall@500": 0.6417910447761194, + "recall@1000": 0.7711442786069652, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8611375561264454, + "ndcg_full@10": 0.8611375561264454, + "ndcg@20": 0.8286351016545931, + "ndcg_full@20": 0.8286351016545931, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.04285714285714286, + "recall@50": 0.18095238095238095, + "recall@100": 0.29523809523809524, + "recall@200": 0.44761904761904764, + "recall@300": 0.5714285714285714, + "recall@500": 0.719047619047619, + "recall@1000": 0.8904761904761904, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9249596882658544, + "ndcg_full@10": 0.9249596882658544, + "ndcg@20": 0.9317608994606191, + "ndcg_full@20": 0.9317608994606191, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.012422360248447204, + "recall@50": 0.13043478260869565, + "recall@100": 0.18633540372670807, + "recall@200": 0.3105590062111801, + "recall@300": 0.4658385093167702, + "recall@500": 0.6956521739130435, + "recall@1000": 0.9316770186335404, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.5743371894486219, + "ndcg_full@10": 0.5743371894486219, + "ndcg@20": 0.6237419884653742, + "ndcg_full@20": 0.6237419884653742, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.2222222222222222, + "recall@100": 0.42483660130718953, + "recall@200": 0.7124183006535948, + "recall@300": 0.8235294117647058, + "recall@500": 0.9150326797385621, + "recall@1000": 0.9477124183006536, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.825051169203569, + "ndcg_full@20": 0.825051169203569, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.2102803738317757, + "recall@100": 0.40654205607476634, + "recall@200": 0.7336448598130841, + "recall@300": 0.9158878504672897, + "recall@500": 0.9439252336448598, + "recall@1000": 0.9485981308411215, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3421757737602428, + "ndcg_full@5": 0.3421757737602428, + "ndcg@10": 0.39599437699951046, + "ndcg_full@10": 0.39599437699951046, + "ndcg@20": 0.45844104588909845, + "ndcg_full@20": 0.44872721511907493, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 95, + "recall@10": 0.19148936170212766, + "recall@50": 0.7872340425531915, + "recall@100": 0.8085106382978723, + "recall@200": 0.8085106382978723, + "recall@300": 0.8085106382978723, + "recall@500": 0.8297872340425532, + "recall@1000": 0.8297872340425532, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.7694428460914031, + "ndcg_full@10": 0.7694428460914031, + "ndcg@20": 0.8244208909022631, + "ndcg_full@20": 0.8050353360225546, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.4, + "graded_P@10": 0.7 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.07936507936507936, + "recall@50": 0.36507936507936506, + "recall@100": 0.5714285714285714, + "recall@200": 0.8015873015873016, + "recall@300": 0.8174603174603174, + "recall@500": 0.8412698412698413, + "recall@1000": 0.8492063492063492, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.7907694447671567, + "ndcg_full@10": 0.7907694447671567, + "ndcg@20": 0.7964073349935761, + "ndcg_full@20": 0.7964073349935761, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 229, + "recall@10": 0.06040268456375839, + "recall@50": 0.2751677852348993, + "recall@100": 0.47651006711409394, + "recall@200": 0.785234899328859, + "recall@300": 0.8322147651006712, + "recall@500": 0.8657718120805369, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7788359968786707, + "ndcg_full@10": 0.7788359968786707, + "ndcg@20": 0.734366727824257, + "ndcg_full@20": 0.734366727824257, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.036231884057971016, + "recall@50": 0.13043478260869565, + "recall@100": 0.2463768115942029, + "recall@200": 0.36231884057971014, + "recall@300": 0.45652173913043476, + "recall@500": 0.6304347826086957, + "recall@1000": 0.6956521739130435, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9041084081571661, + "ndcg_full@5": 0.9041084081571661, + "ndcg@10": 0.9059625241448755, + "ndcg_full@10": 0.9059625241448755, + "ndcg@20": 0.8168647175546953, + "ndcg_full@20": 0.8168647175546953, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027210884353741496, + "recall@50": 0.08843537414965986, + "recall@100": 0.16666666666666666, + "recall@200": 0.3129251700680272, + "recall@300": 0.4387755102040816, + "recall@500": 0.5884353741496599, + "recall@1000": 0.7346938775510204, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8385058694009951, + "ndcg_full@5": 0.8385058694009951, + "ndcg@10": 0.8253922866466662, + "ndcg_full@10": 0.8253922866466662, + "ndcg@20": 0.7998652855192124, + "ndcg_full@20": 0.7998652855192124, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.023076923076923078, + "recall@50": 0.13076923076923078, + "recall@100": 0.2423076923076923, + "recall@200": 0.3923076923076923, + "recall@300": 0.47307692307692306, + "recall@500": 0.5884615384615385, + "recall@1000": 0.7076923076923077, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9321131406841483, + "ndcg_full@20": 0.9321131406841483, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.036231884057971016, + "recall@50": 0.10144927536231885, + "recall@100": 0.18840579710144928, + "recall@200": 0.35507246376811596, + "recall@300": 0.47101449275362317, + "recall@500": 0.6086956521739131, + "recall@1000": 0.7536231884057971, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.23518264139297895, + "ndcg_full@5": 0.23518264139297895, + "ndcg@10": 0.2860473029456768, + "ndcg_full@10": 0.2860473029456768, + "ndcg@20": 0.226748120662614, + "ndcg_full@20": 0.2193557621350379, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01593625498007968, + "recall@50": 0.10756972111553785, + "recall@100": 0.23904382470119523, + "recall@200": 0.5338645418326693, + "recall@300": 0.7928286852589641, + "recall@500": 0.9442231075697212, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.37381935507050307, + "ndcg_full@5": 0.37381935507050307, + "ndcg@10": 0.4907154231395208, + "ndcg_full@10": 0.4907154231395208, + "ndcg@20": 0.5331480904513666, + "ndcg_full@20": 0.5331480904513666, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.043689320388349516, + "recall@50": 0.1407766990291262, + "recall@100": 0.2524271844660194, + "recall@200": 0.46116504854368934, + "recall@300": 0.5970873786407767, + "recall@500": 0.7766990291262136, + "recall@1000": 0.8883495145631068, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8554853481239504, + "ndcg_full@5": 0.8554853481239504, + "ndcg@10": 0.7592101141786671, + "ndcg_full@10": 0.7099089029655229, + "ndcg@20": 0.8051393624804617, + "ndcg_full@20": 0.6996329030997857, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 209, + "recall@10": 0.05228758169934641, + "recall@50": 0.19607843137254902, + "recall@100": 0.38562091503267976, + "recall@200": 0.6797385620915033, + "recall@300": 0.8366013071895425, + "recall@500": 0.9019607843137255, + "recall@1000": 0.9150326797385621, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.865283257607045, + "ndcg_full@20": 0.865283257607045, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 204, + "recall@10": 0.12857142857142856, + "recall@50": 0.44285714285714284, + "recall@100": 0.7285714285714285, + "recall@200": 0.9, + "recall@300": 0.9285714285714286, + "recall@500": 0.9571428571428572, + "recall@1000": 0.9714285714285714, + "unjudged_fraction@10": 0.9, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8165901947515999, + "ndcg_full@10": 0.8165901947515999, + "ndcg@20": 0.7330643252712293, + "ndcg_full@20": 0.7330643252712293, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1320754716981132, + "recall@50": 0.41509433962264153, + "recall@100": 0.660377358490566, + "recall@200": 0.8490566037735849, + "recall@300": 0.8867924528301887, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8448653845904918, + "ndcg_full@10": 0.8448653845904918, + "ndcg@20": 0.7973711725164713, + "ndcg_full@20": 0.7973711725164713, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 198, + "recall@10": 0.058394160583941604, + "recall@50": 0.29927007299270075, + "recall@100": 0.48905109489051096, + "recall@200": 0.7299270072992701, + "recall@300": 0.7883211678832117, + "recall@500": 0.8248175182481752, + "recall@1000": 0.8613138686131386, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.861713382497964, + "ndcg_full@5": 0.861713382497964, + "ndcg@10": 0.875154604990276, + "ndcg_full@10": 0.875154604990276, + "ndcg@20": 0.91134428566806, + "ndcg_full@20": 0.91134428566806, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.026217228464419477, + "recall@50": 0.12734082397003746, + "recall@100": 0.23595505617977527, + "recall@200": 0.44569288389513106, + "recall@300": 0.6142322097378277, + "recall@500": 0.8202247191011236, + "recall@1000": 0.8764044943820225, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.945661763505564, + "ndcg_full@20": 0.945661763505564, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.056962025316455694, + "recall@50": 0.24050632911392406, + "recall@100": 0.4430379746835443, + "recall@200": 0.6835443037974683, + "recall@300": 0.8354430379746836, + "recall@500": 0.9113924050632911, + "recall@1000": 0.9746835443037974, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9838308544258403, + "ndcg_full@20": 0.9838308544258403, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.1568627450980392, + "recall@100": 0.21568627450980393, + "recall@200": 0.3627450980392157, + "recall@300": 0.47058823529411764, + "recall@500": 0.6274509803921569, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.7584837914317291, + "ndcg_full@10": 0.7584837914317291, + "ndcg@20": 0.7416233127838551, + "ndcg_full@20": 0.7416233127838551, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05625, + "recall@50": 0.19375, + "recall@100": 0.325, + "recall@200": 0.5625, + "recall@300": 0.66875, + "recall@500": 0.74375, + "recall@1000": 0.81875, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8601645736749576, + "ndcg_full@10": 0.8601645736749576, + "ndcg@20": 0.8174436937833316, + "ndcg_full@20": 0.8174436937833316, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.13304721030042918, + "recall@100": 0.2532188841201717, + "recall@200": 0.45064377682403434, + "recall@300": 0.6266094420600858, + "recall@500": 0.8497854077253219, + "recall@1000": 0.9527896995708155, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.588272452869007, + "ndcg_full@5": 0.588272452869007, + "ndcg@10": 0.5212504533025837, + "ndcg_full@10": 0.5212504533025837, + "ndcg@20": 0.5527753151606718, + "ndcg_full@20": 0.5527753151606718, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027777777777777776, + "recall@50": 0.11419753086419752, + "recall@100": 0.23148148148148148, + "recall@200": 0.44135802469135804, + "recall@300": 0.5648148148148148, + "recall@500": 0.7901234567901234, + "recall@1000": 0.9197530864197531, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9487449939588293, + "ndcg_full@20": 0.9487449939588293, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027131782945736434, + "recall@50": 0.13953488372093023, + "recall@100": 0.2364341085271318, + "recall@200": 0.42248062015503873, + "recall@300": 0.6162790697674418, + "recall@500": 0.7751937984496124, + "recall@1000": 0.810077519379845, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.826333328652008, + "ndcg_full@10": 0.826333328652008, + "ndcg@20": 0.7805158954167937, + "ndcg_full@20": 0.7805158954167937, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 90, + "recall@10": 0.06, + "recall@50": 0.2733333333333333, + "recall@100": 0.5, + "recall@200": 0.66, + "recall@300": 0.74, + "recall@500": 0.7933333333333333, + "recall@1000": 0.8466666666666667, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.898758383963747, + "ndcg_full@10": 0.898758383963747, + "ndcg@20": 0.8618507684253244, + "ndcg_full@20": 0.8618507684253244, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 217, + "recall@10": 0.01744186046511628, + "recall@50": 0.14534883720930233, + "recall@100": 0.29651162790697677, + "recall@200": 0.47674418604651164, + "recall@300": 0.6162790697674418, + "recall@500": 0.7093023255813954, + "recall@1000": 0.7441860465116279, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5117558763970509, + "ndcg_full@5": 0.5117558763970509, + "ndcg@10": 0.49569664046935485, + "ndcg_full@10": 0.49569664046935485, + "ndcg@20": 0.4981174546306571, + "ndcg_full@20": 0.4981174546306571, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 103, + "recall@10": 0.07547169811320754, + "recall@50": 0.3584905660377358, + "recall@100": 0.41509433962264153, + "recall@200": 0.4716981132075472, + "recall@300": 0.49056603773584906, + "recall@500": 0.5471698113207547, + "recall@1000": 0.5471698113207547, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7162874455029682, + "ndcg_full@10": 0.7162874455029682, + "ndcg@20": 0.6310181772747843, + "ndcg_full@20": 0.6104459555946162, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 248, + "recall@10": 0.019138755980861243, + "recall@50": 0.0861244019138756, + "recall@100": 0.16267942583732056, + "recall@200": 0.2631578947368421, + "recall@300": 0.3444976076555024, + "recall@500": 0.46411483253588515, + "recall@1000": 0.631578947368421, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7799082337019199, + "ndcg_full@10": 0.7799082337019199, + "ndcg@20": 0.8195224751704618, + "ndcg_full@20": 0.8195224751704618, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04, + "recall@50": 0.12, + "recall@100": 0.22, + "recall@200": 0.36, + "recall@300": 0.52, + "recall@500": 0.6933333333333334, + "recall@1000": 0.7733333333333333, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.9100342589944143, + "ndcg_full@10": 0.9100342589944143, + "ndcg@20": 0.8415736140667184, + "ndcg_full@20": 0.8415736140667184, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 2000, + "num_ranked": 111, + "recall@10": 0.04597701149425287, + "recall@50": 0.25862068965517243, + "recall@100": 0.45977011494252873, + "recall@200": 0.5057471264367817, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.9097832499695674, + "ndcg_full@20": 0.9097832499695674, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.025889967637540454, + "recall@50": 0.08090614886731391, + "recall@100": 0.1650485436893204, + "recall@200": 0.2621359223300971, + "recall@300": 0.35275080906148865, + "recall@500": 0.4919093851132686, + "recall@1000": 0.6245954692556634, + "unjudged_fraction@10": 0.9, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9835675147162116, + "ndcg_full@20": 0.9835675147162116, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03225806451612903, + "recall@50": 0.18548387096774194, + "recall@100": 0.3629032258064516, + "recall@200": 0.5967741935483871, + "recall@300": 0.7862903225806451, + "recall@500": 0.9395161290322581, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9651774792426175, + "ndcg_full@20": 0.9651774792426175, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.12345679012345678, + "recall@50": 0.5679012345679012, + "recall@100": 0.9506172839506173, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9255244116680239, + "ndcg_full@5": 0.9255244116680239, + "ndcg@10": 0.8822392172031873, + "ndcg_full@10": 0.8822392172031873, + "ndcg@20": 0.8884910259454395, + "ndcg_full@20": 0.8884910259454395, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 151, + "recall@10": 0.13559322033898305, + "recall@50": 0.6610169491525424, + "recall@100": 0.8813559322033898, + "recall@200": 0.9491525423728814, + "recall@300": 0.9491525423728814, + "recall@500": 0.9491525423728814, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7691415988388896, + "ndcg_full@5": 0.7691415988388896, + "ndcg@10": 0.850188694055929, + "ndcg_full@10": 0.850188694055929, + "ndcg@20": 0.8525195358117057, + "ndcg_full@20": 0.8525195358117057, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 71, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.8464134707954625, + "ndcg_full@10": 0.8464134707954625, + "ndcg@20": 0.8535890582699065, + "ndcg_full@20": 0.8535890582699065, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032467532467532464, + "recall@50": 0.1266233766233766, + "recall@100": 0.237012987012987, + "recall@200": 0.3961038961038961, + "recall@300": 0.5714285714285714, + "recall@500": 0.7922077922077922, + "recall@1000": 0.9253246753246753, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.918335168447527, + "ndcg_full@10": 0.918335168447527, + "ndcg@20": 0.9135459523782047, + "ndcg_full@20": 0.9135459523782047, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 166, + "recall@10": 0.11267605633802817, + "recall@50": 0.36619718309859156, + "recall@100": 0.4647887323943662, + "recall@200": 0.5211267605633803, + "recall@300": 0.6056338028169014, + "recall@500": 0.6338028169014085, + "recall@1000": 0.7183098591549296, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006398, + "ndcg_full@10": 0.9266360779006398, + "ndcg@20": 0.8854181392871684, + "ndcg_full@20": 0.8854181392871684, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.06451612903225806, + "recall@50": 0.2645161290322581, + "recall@100": 0.45161290322580644, + "recall@200": 0.7032258064516129, + "recall@300": 0.832258064516129, + "recall@500": 0.896774193548387, + "recall@1000": 0.9161290322580645, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8777926945437562, + "ndcg_full@10": 0.8777926945437562, + "ndcg@20": 0.8510328403022259, + "ndcg_full@20": 0.8510328403022259, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.05128205128205128, + "recall@50": 0.24615384615384617, + "recall@100": 0.4666666666666667, + "recall@200": 0.7025641025641025, + "recall@300": 0.7230769230769231, + "recall@500": 0.7333333333333333, + "recall@1000": 0.7487179487179487, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8757862866154155, + "ndcg_full@5": 0.7543701539552121, + "ndcg@10": 0.9054714533341821, + "ndcg_full@10": 0.7700542278177658, + "ndcg@20": 0.9591142047939843, + "ndcg_full@20": 0.8502930827598733, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.2, + "graded_P@10": 0.6 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 216, + "recall@10": 0.053763440860215055, + "recall@50": 0.25268817204301075, + "recall@100": 0.4731182795698925, + "recall@200": 0.8333333333333334, + "recall@300": 0.8817204301075269, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 225, + "recall@10": 0.08108108108108109, + "recall@50": 0.35135135135135137, + "recall@100": 0.5855855855855856, + "recall@200": 0.9099099099099099, + "recall@300": 0.9819819819819819, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.9179031474232457, + "ndcg_full@20": 0.9179031474232457, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.015037593984962405, + "recall@50": 0.10526315789473684, + "recall@100": 0.24060150375939848, + "recall@200": 0.42105263157894735, + "recall@300": 0.6090225563909775, + "recall@500": 0.7894736842105263, + "recall@1000": 0.924812030075188, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9257772306718706, + "ndcg_full@20": 0.9257772306718706, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 76, + "recall@10": 0.15151515151515152, + "recall@50": 0.6212121212121212, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.8787878787878788, + "recall@1000": 0.9545454545454546, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.8975828762407047, + "ndcg_full@10": 0.8975828762407047, + "ndcg@20": 0.93390333240257, + "ndcg_full@20": 0.93390333240257, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.12987012987012986, + "recall@50": 0.42857142857142855, + "recall@100": 0.7012987012987013, + "recall@200": 0.7792207792207793, + "recall@300": 0.8181818181818182, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7840048711692542, + "ndcg_full@5": 0.7840048711692542, + "ndcg@10": 0.79357971732947, + "ndcg_full@10": 0.79357971732947, + "ndcg@20": 0.7955334807716082, + "ndcg_full@20": 0.7955334807716082, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 83, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.7391304347826086, + "recall@300": 0.782608695652174, + "recall@500": 0.8260869565217391, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7840048711692542, + "ndcg_full@5": 0.7840048711692542, + "ndcg@10": 0.7028229111688272, + "ndcg_full@10": 0.680465837382406, + "ndcg@20": 0.6327322773639398, + "ndcg_full@20": 0.5419154409545626, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 179, + "recall@10": 0.09859154929577464, + "recall@50": 0.4225352112676056, + "recall@100": 0.676056338028169, + "recall@200": 0.8732394366197183, + "recall@300": 0.8873239436619719, + "recall@500": 0.9154929577464789, + "recall@1000": 0.9295774647887324, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.811925048289751, + "ndcg_full@10": 0.811925048289751, + "ndcg@20": 0.7877124953749433, + "ndcg_full@20": 0.7877124953749433, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 163, + "recall@10": 0.08823529411764706, + "recall@50": 0.4117647058823529, + "recall@100": 0.5882352941176471, + "recall@200": 0.75, + "recall@300": 0.7647058823529411, + "recall@500": 0.7941176470588235, + "recall@1000": 0.8088235294117647, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.6090528004170105, + "ndcg_full@10": 0.6090528004170105, + "ndcg@20": 0.6083112722190241, + "ndcg_full@20": 0.6083112722190241, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 190, + "recall@10": 0.10869565217391304, + "recall@50": 0.43478260869565216, + "recall@100": 0.5652173913043478, + "recall@200": 0.6304347826086957, + "recall@300": 0.6956521739130435, + "recall@500": 0.717391304347826, + "recall@1000": 0.782608695652174, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8729617039182878, + "ndcg_full@20": 0.8729617039182878, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.12, + "recall@50": 0.52, + "recall@100": 0.88, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.38962811929307617, + "ndcg_full@10": 0.38962811929307617, + "ndcg@20": 0.4080437739626595, + "ndcg_full@20": 0.4080437739626595, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 90, + "recall@10": 0.20512820512820512, + "recall@50": 0.6666666666666666, + "recall@100": 0.6923076923076923, + "recall@200": 0.717948717948718, + "recall@300": 0.7435897435897436, + "recall@500": 0.7692307692307693, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8568270051237653, + "ndcg_full@10": 0.8568270051237653, + "ndcg@20": 0.8724707326220141, + "ndcg_full@20": 0.8724707326220141, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.10465116279069768, + "recall@50": 0.37209302325581395, + "recall@100": 0.6627906976744186, + "recall@200": 0.9767441860465116, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.907575524214984, + "ndcg_full@5": 0.907575524214984, + "ndcg@10": 0.9138900830033845, + "ndcg_full@10": 0.9138900830033845, + "ndcg@20": 0.7644780409738634, + "ndcg_full@20": 0.7644780409738634, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 139, + "recall@10": 0.06172839506172839, + "recall@50": 0.19753086419753085, + "recall@100": 0.3271604938271605, + "recall@200": 0.5679012345679012, + "recall@300": 0.6666666666666666, + "recall@500": 0.7654320987654321, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9318123359164181, + "ndcg_full@20": 0.9318123359164181, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.07352941176470588, + "recall@50": 0.3235294117647059, + "recall@100": 0.5735294117647058, + "recall@200": 0.6764705882352942, + "recall@300": 0.6764705882352942, + "recall@500": 0.6911764705882353, + "recall@1000": 0.6911764705882353, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9173950256592021, + "ndcg_full@5": 0.9173950256592021, + "ndcg@10": 0.9463950238707795, + "ndcg_full@10": 0.9463950238707795, + "ndcg@20": 0.9467304859378322, + "ndcg_full@20": 0.9467304859378322, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1317, + "num_ranked": 200, + "recall@10": 0.019943019943019943, + "recall@50": 0.1225071225071225, + "recall@100": 0.23931623931623933, + "recall@200": 0.45014245014245013, + "recall@300": 0.6125356125356125, + "recall@500": 0.6296296296296297, + "recall@1000": 0.6324786324786325, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9631684808225083, + "ndcg_full@20": 0.9631684808225083, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 166, + "recall@10": 0.04522613065326633, + "recall@50": 0.16080402010050251, + "recall@100": 0.271356783919598, + "recall@200": 0.507537688442211, + "recall@300": 0.6030150753768844, + "recall@500": 0.7386934673366834, + "recall@1000": 0.8944723618090452, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8512360941594274, + "ndcg_full@10": 0.8512360941594274, + "ndcg@20": 0.903992632541125, + "ndcg_full@20": 0.903992632541125, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03684210526315789, + "recall@50": 0.07368421052631578, + "recall@100": 0.14210526315789473, + "recall@200": 0.28421052631578947, + "recall@300": 0.4421052631578947, + "recall@500": 0.7105263157894737, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9260852545422736, + "ndcg_full@10": 0.9260852545422736, + "ndcg@20": 0.8327400834208911, + "ndcg_full@20": 0.8327400834208911, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 152, + "recall@10": 0.03636363636363636, + "recall@50": 0.15272727272727274, + "recall@100": 0.2727272727272727, + "recall@200": 0.4509090909090909, + "recall@300": 0.5490909090909091, + "recall@500": 0.6581818181818182, + "recall@1000": 0.7563636363636363, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6069931323672637, + "ndcg_full@5": 0.6069931323672637, + "ndcg@10": 0.676473079121698, + "ndcg_full@10": 0.676473079121698, + "ndcg@20": 0.6879071022021657, + "ndcg_full@20": 0.6879071022021657, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.4, + "graded_P@10": 0.7 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.09090909090909091, + "recall@50": 0.3409090909090909, + "recall@100": 0.4318181818181818, + "recall@200": 0.6590909090909091, + "recall@300": 0.7954545454545454, + "recall@500": 0.8636363636363636, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.4845283055203581, + "ndcg_full@5": 0.4845283055203581, + "ndcg@10": 0.601873353063398, + "ndcg_full@10": 0.601873353063398, + "ndcg@20": 0.4602515395253163, + "ndcg_full@20": 0.4602515395253163, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.019704433497536946, + "recall@50": 0.11330049261083744, + "recall@100": 0.23645320197044334, + "recall@200": 0.3842364532019704, + "recall@300": 0.5517241379310345, + "recall@500": 0.8275862068965517, + "recall@1000": 0.9655172413793104, + "unjudged_fraction@10": 0.7, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.7987274163285906, + "ndcg_full@10": 0.7987274163285906, + "ndcg@20": 0.7598477495069642, + "ndcg_full@20": 0.7350754736692953, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035398230088495575, + "recall@50": 0.17699115044247787, + "recall@100": 0.30973451327433627, + "recall@200": 0.5132743362831859, + "recall@300": 0.6283185840707964, + "recall@500": 0.7433628318584071, + "recall@1000": 0.8584070796460177, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.8215843983276304, + "ndcg_full@20": 0.8215843983276304, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 59, + "recall@10": 0.125, + "recall@50": 0.25, + "recall@100": 0.2916666666666667, + "recall@200": 0.3125, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8603818544462509, + "ndcg_full@10": 0.8603818544462509, + "ndcg@20": 0.79054307746183, + "ndcg_full@20": 0.6782616383344992, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.06369426751592357, + "recall@50": 0.25477707006369427, + "recall@100": 0.4713375796178344, + "recall@200": 0.7388535031847133, + "recall@300": 0.7898089171974523, + "recall@500": 0.8280254777070064, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.814958199839295, + "ndcg_full@10": 0.814958199839295, + "ndcg@20": 0.7858263149708833, + "ndcg_full@20": 0.7858263149708833, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.06097560975609756, + "recall@50": 0.27439024390243905, + "recall@100": 0.3475609756097561, + "recall@200": 0.42073170731707316, + "recall@300": 0.47560975609756095, + "recall@500": 0.5975609756097561, + "recall@1000": 0.7317073170731707, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7245844530637532, + "ndcg_full@5": 0.7245844530637532, + "ndcg@10": 0.7049266880849643, + "ndcg_full@10": 0.7049266880849643, + "ndcg@20": 0.7220543869199896, + "ndcg_full@20": 0.674144733392813, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 213, + "recall@10": 0.0, + "recall@50": 0.0379746835443038, + "recall@100": 0.08860759493670886, + "recall@200": 0.16455696202531644, + "recall@300": 0.26582278481012656, + "recall@500": 0.4430379746835443, + "recall@1000": 0.5316455696202531, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.4315846810586322, + "ndcg_full@5": 0.4315846810586322, + "ndcg@10": 0.5191582034842978, + "ndcg_full@10": 0.3775992553127645, + "ndcg@20": 0.6735007524040898, + "ndcg_full@20": 0.3161378406030131, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03896103896103896, + "recall@50": 0.1774891774891775, + "recall@100": 0.3116883116883117, + "recall@200": 0.5541125541125541, + "recall@300": 0.7835497835497836, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8860308691611207, + "ndcg_full@10": 0.8860308691611207, + "ndcg@20": 0.8709631498635143, + "ndcg_full@20": 0.8709631498635143, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.016304347826086956, + "recall@50": 0.11956521739130435, + "recall@100": 0.18478260869565216, + "recall@200": 0.3532608695652174, + "recall@300": 0.45652173913043476, + "recall@500": 0.6304347826086957, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8521705090845474, + "ndcg_full@10": 0.8521705090845474, + "ndcg@20": 0.8722573822753913, + "ndcg_full@20": 0.8722573822753913, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 98, + "recall@10": 0.10989010989010989, + "recall@50": 0.23076923076923078, + "recall@100": 0.24175824175824176, + "recall@200": 0.27472527472527475, + "recall@300": 0.27472527472527475, + "recall@500": 0.2857142857142857, + "recall@1000": 0.32967032967032966, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.7412693750436921, + "ndcg_full@10": 0.7412693750436921, + "ndcg@20": 0.8227408964253164, + "ndcg_full@20": 0.7737265051920472, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 202, + "recall@10": 0.13636363636363635, + "recall@50": 0.5757575757575758, + "recall@100": 0.803030303030303, + "recall@200": 0.8939393939393939, + "recall@300": 0.9545454545454546, + "recall@500": 0.9696969696969697, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8881221418648991, + "ndcg_full@20": 0.873761965665451, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 84, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7035390600827187, + "ndcg_full@5": 0.7035390600827187, + "ndcg@10": 0.768418000745781, + "ndcg_full@10": 0.768418000745781, + "ndcg@20": 0.8394704064093075, + "ndcg_full@20": 0.8394704064093075, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.11143695014662756, + "recall@100": 0.21994134897360704, + "recall@200": 0.4222873900293255, + "recall@300": 0.5806451612903226, + "recall@500": 0.841642228739003, + "recall@1000": 0.9354838709677419, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9651031103133739, + "ndcg_full@20": 0.9651031103133739, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.08201653038371211, + "recall@50": 0.2820748984045244, + "recall@100": 0.42521830967576446, + "recall@200": 0.5873393737363496, + "recall@300": 0.678937984786285, + "recall@500": 0.7726304624552182, + "recall@1000": 0.8362067497153736, + "ndcg@5": 0.6399903951866057, + "ndcg@10": 0.6210606733330393, + "ndcg@20": 0.6044124310806349, + "ndcg_full@5": 0.6383715134178031, + "ndcg_full@10": 0.6180902908499666, + "ndcg_full@20": 0.5964576791345534, + "P@10(rel>=1)": 0.648, + "P@10(eligible)": 0.5546666666666666, + "graded_P@10": 0.6013333333333334, + "unjudged_fraction@10": 0.2613333333333334 + }, + "median": { + "recall@10": 0.05228758169934641, + "recall@50": 0.2222222222222222, + "recall@100": 0.3629032258064516, + "recall@200": 0.5541125541125541, + "recall@300": 0.6764705882352942, + "recall@500": 0.7933333333333333, + "recall@1000": 0.8698224852071006, + "ndcg@5": 0.6992148198508501, + "ndcg@10": 0.6429424283490954, + "ndcg@20": 0.6167198376863421, + "ndcg_full@5": 0.6992148198508501, + "ndcg_full@10": 0.6429424283490954, + "ndcg_full@20": 0.6013293567127181, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6, + "unjudged_fraction@10": 0.2 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 246, + "recall@10": 0.05325443786982249, + "recall@50": 0.26627218934911245, + "recall@100": 0.4556213017751479, + "recall@200": 0.6923076923076923, + "recall@300": 0.7988165680473372, + "recall@500": 0.863905325443787, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6357368530666605, + "ndcg_full@10": 0.6357368530666605, + "ndcg@20": 0.6240095992246932, + "ndcg_full@20": 0.6240095992246932, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02962962962962963, + "recall@50": 0.12222222222222222, + "recall@100": 0.21851851851851853, + "recall@200": 0.3962962962962963, + "recall@300": 0.5074074074074074, + "recall@500": 0.6888888888888889, + "recall@1000": 0.8296296296296296, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.565602538756171, + "ndcg_full@5": 0.565602538756171, + "ndcg@10": 0.6789066366655496, + "ndcg_full@10": 0.6789066366655496, + "ndcg@20": 0.7203292170753954, + "ndcg_full@20": 0.7203292170753954, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.08333333333333333, + "recall@50": 0.39285714285714285, + "recall@100": 0.5952380952380952, + "recall@200": 0.7857142857142857, + "recall@300": 0.8333333333333334, + "recall@500": 0.9523809523809523, + "recall@1000": 0.9761904761904762, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.6595965689060225, + "ndcg_full@10": 0.6595965689060225, + "ndcg@20": 0.6310404710920504, + "ndcg_full@20": 0.6208370858517767, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.18947368421052632, + "recall@100": 0.35789473684210527, + "recall@200": 0.5368421052631579, + "recall@300": 0.7368421052631579, + "recall@500": 0.8736842105263158, + "recall@1000": 0.9368421052631579, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6792465385586782, + "ndcg_full@10": 0.6792465385586782, + "ndcg@20": 0.6191576539400564, + "ndcg_full@20": 0.6191576539400564, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01990049751243781, + "recall@50": 0.0945273631840796, + "recall@100": 0.16417910447761194, + "recall@200": 0.2935323383084577, + "recall@300": 0.4427860696517413, + "recall@500": 0.6417910447761194, + "recall@1000": 0.7711442786069652, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.23365080824310533, + "ndcg_full@10": 0.23365080824310533, + "ndcg@20": 0.32116329724066206, + "ndcg_full@20": 0.32116329724066206, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.04285714285714286, + "recall@50": 0.18095238095238095, + "recall@100": 0.29523809523809524, + "recall@200": 0.44761904761904764, + "recall@300": 0.5714285714285714, + "recall@500": 0.719047619047619, + "recall@1000": 0.8904761904761904, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9249596882658544, + "ndcg_full@10": 0.9249596882658544, + "ndcg@20": 0.8277894682056347, + "ndcg_full@20": 0.8277894682056347, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.012422360248447204, + "recall@50": 0.13043478260869565, + "recall@100": 0.18633540372670807, + "recall@200": 0.3105590062111801, + "recall@300": 0.4658385093167702, + "recall@500": 0.6956521739130435, + "recall@1000": 0.9316770186335404, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.1635413866202963, + "ndcg_full@10": 0.1635413866202963, + "ndcg@20": 0.31571475772492513, + "ndcg_full@20": 0.31571475772492513, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0392156862745098, + "recall@50": 0.2222222222222222, + "recall@100": 0.42483660130718953, + "recall@200": 0.7124183006535948, + "recall@300": 0.8235294117647058, + "recall@500": 0.9150326797385621, + "recall@1000": 0.9477124183006536, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.7873611989440605, + "ndcg_full@20": 0.7873611989440605, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.2102803738317757, + "recall@100": 0.40654205607476634, + "recall@200": 0.7336448598130841, + "recall@300": 0.9158878504672897, + "recall@500": 0.9439252336448598, + "recall@1000": 0.9485981308411215, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.26481735860702105, + "ndcg_full@5": 0.26481735860702105, + "ndcg@10": 0.3460658404024307, + "ndcg_full@10": 0.3460658404024307, + "ndcg@20": 0.4222236150536102, + "ndcg_full@20": 0.4132771893779073, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 95, + "recall@10": 0.19148936170212766, + "recall@50": 0.7872340425531915, + "recall@100": 0.8085106382978723, + "recall@200": 0.8085106382978723, + "recall@300": 0.8085106382978723, + "recall@500": 0.8297872340425532, + "recall@1000": 0.8297872340425532, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.7363157343642086, + "ndcg_full@10": 0.7363157343642086, + "ndcg@20": 0.7378927796670297, + "ndcg_full@20": 0.7205418596049231, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.07936507936507936, + "recall@50": 0.36507936507936506, + "recall@100": 0.5714285714285714, + "recall@200": 0.8015873015873016, + "recall@300": 0.8174603174603174, + "recall@500": 0.8412698412698413, + "recall@1000": 0.8492063492063492, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.7907694447671567, + "ndcg_full@10": 0.7907694447671567, + "ndcg@20": 0.7586613053387534, + "ndcg_full@20": 0.7586613053387534, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 229, + "recall@10": 0.06040268456375839, + "recall@50": 0.2751677852348993, + "recall@100": 0.47651006711409394, + "recall@200": 0.785234899328859, + "recall@300": 0.8322147651006712, + "recall@500": 0.8657718120805369, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7788359968786707, + "ndcg_full@10": 0.7788359968786707, + "ndcg@20": 0.734366727824257, + "ndcg_full@20": 0.734366727824257, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.036231884057971016, + "recall@50": 0.13043478260869565, + "recall@100": 0.2463768115942029, + "recall@200": 0.36231884057971014, + "recall@300": 0.45652173913043476, + "recall@500": 0.6304347826086957, + "recall@1000": 0.6956521739130435, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8385058694009951, + "ndcg_full@5": 0.8385058694009951, + "ndcg@10": 0.7910647191075327, + "ndcg_full@10": 0.7910647191075327, + "ndcg@20": 0.6363712321080411, + "ndcg_full@20": 0.6363712321080411, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027210884353741496, + "recall@50": 0.08843537414965986, + "recall@100": 0.16666666666666666, + "recall@200": 0.3129251700680272, + "recall@300": 0.4387755102040816, + "recall@500": 0.5884353741496599, + "recall@1000": 0.7346938775510204, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7580400583144596, + "ndcg_full@5": 0.7580400583144596, + "ndcg@10": 0.6336735973599565, + "ndcg_full@10": 0.6336735973599565, + "ndcg@20": 0.5517403294912991, + "ndcg_full@20": 0.5517403294912991, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.023076923076923078, + "recall@50": 0.13076923076923078, + "recall@100": 0.2423076923076923, + "recall@200": 0.3923076923076923, + "recall@300": 0.47307692307692306, + "recall@500": 0.5884615384615385, + "recall@1000": 0.7076923076923077, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.8443973426956417, + "ndcg_full@20": 0.8443973426956417, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.036231884057971016, + "recall@50": 0.10144927536231885, + "recall@100": 0.18840579710144928, + "recall@200": 0.35507246376811596, + "recall@300": 0.47101449275362317, + "recall@500": 0.6086956521739131, + "recall@1000": 0.7536231884057971, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.11152538070586984, + "ndcg_full@10": 0.11152538070586984, + "ndcg@20": 0.11216074154500127, + "ndcg_full@20": 0.10850411845239738, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.01593625498007968, + "recall@50": 0.10756972111553785, + "recall@100": 0.23904382470119523, + "recall@200": 0.5338645418326693, + "recall@300": 0.7928286852589641, + "recall@500": 0.9442231075697212, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.37381935507050307, + "ndcg_full@5": 0.37381935507050307, + "ndcg@10": 0.41428350424213684, + "ndcg_full@10": 0.41428350424213684, + "ndcg@20": 0.4640195612355714, + "ndcg_full@20": 0.4640195612355714, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.043689320388349516, + "recall@50": 0.1407766990291262, + "recall@100": 0.2524271844660194, + "recall@200": 0.46116504854368934, + "recall@300": 0.5970873786407767, + "recall@500": 0.7766990291262136, + "recall@1000": 0.8883495145631068, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.3077499832503558, + "ndcg_full@5": 0.3077499832503558, + "ndcg@10": 0.40785835152149574, + "ndcg_full@10": 0.3813730474431125, + "ndcg@20": 0.5243892521445487, + "ndcg_full@20": 0.45567263498574645, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 209, + "recall@10": 0.05228758169934641, + "recall@50": 0.19607843137254902, + "recall@100": 0.38562091503267976, + "recall@200": 0.6797385620915033, + "recall@300": 0.8366013071895425, + "recall@500": 0.9019607843137255, + "recall@1000": 0.9150326797385621, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.7770226618949087, + "ndcg_full@10": 0.7770226618949087, + "ndcg@20": 0.7316556576565596, + "ndcg_full@20": 0.7316556576565596, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 204, + "recall@10": 0.12857142857142856, + "recall@50": 0.44285714285714284, + "recall@100": 0.7285714285714285, + "recall@200": 0.9, + "recall@300": 0.9285714285714286, + "recall@500": 0.9571428571428572, + "recall@1000": 0.9714285714285714, + "unjudged_fraction@10": 0.9, + "ndcg@5": 0.05545468547100965, + "ndcg_full@5": 0.05545468547100965, + "ndcg@10": 0.11203537310467275, + "ndcg_full@10": 0.11203537310467275, + "ndcg@20": 0.12686467790105935, + "ndcg_full@20": 0.12686467790105935, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1320754716981132, + "recall@50": 0.41509433962264153, + "recall@100": 0.660377358490566, + "recall@200": 0.8490566037735849, + "recall@300": 0.8867924528301887, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.49685832173576006, + "ndcg_full@10": 0.49685832173576006, + "ndcg@20": 0.5037111767585656, + "ndcg_full@20": 0.5037111767585656, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 198, + "recall@10": 0.058394160583941604, + "recall@50": 0.29927007299270075, + "recall@100": 0.48905109489051096, + "recall@200": 0.7299270072992701, + "recall@300": 0.7883211678832117, + "recall@500": 0.8248175182481752, + "recall@1000": 0.8613138686131386, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.861713382497964, + "ndcg_full@5": 0.861713382497964, + "ndcg@10": 0.822494341290053, + "ndcg_full@10": 0.822494341290053, + "ndcg@20": 0.6923474437884973, + "ndcg_full@20": 0.6923474437884973, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.026217228464419477, + "recall@50": 0.12734082397003746, + "recall@100": 0.23595505617977527, + "recall@200": 0.44569288389513106, + "recall@300": 0.6142322097378277, + "recall@500": 0.8202247191011236, + "recall@1000": 0.8764044943820225, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7020976466720958, + "ndcg_full@5": 0.7020976466720958, + "ndcg@10": 0.6668824232637928, + "ndcg_full@10": 0.6668824232637928, + "ndcg@20": 0.6013293567127181, + "ndcg_full@20": 0.6013293567127181, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.056962025316455694, + "recall@50": 0.24050632911392406, + "recall@100": 0.4430379746835443, + "recall@200": 0.6835443037974683, + "recall@300": 0.8354430379746836, + "recall@500": 0.9113924050632911, + "recall@1000": 0.9746835443037974, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8909319377242088, + "ndcg_full@10": 0.8909319377242088, + "ndcg@20": 0.7410007085850071, + "ndcg_full@20": 0.7410007085850071, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.1568627450980392, + "recall@100": 0.21568627450980393, + "recall@200": 0.3627450980392157, + "recall@300": 0.47058823529411764, + "recall@500": 0.6274509803921569, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.42257499837058643, + "ndcg_full@10": 0.42257499837058643, + "ndcg@20": 0.5055579649256534, + "ndcg_full@20": 0.5055579649256534, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05625, + "recall@50": 0.19375, + "recall@100": 0.325, + "recall@200": 0.5625, + "recall@300": 0.66875, + "recall@500": 0.74375, + "recall@1000": 0.81875, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6425171760065149, + "ndcg_full@5": 0.6425171760065149, + "ndcg@10": 0.580354131555013, + "ndcg_full@10": 0.580354131555013, + "ndcg@20": 0.5037746426042288, + "ndcg_full@20": 0.5037746426042288, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.13304721030042918, + "recall@100": 0.2532188841201717, + "recall@200": 0.45064377682403434, + "recall@300": 0.6266094420600858, + "recall@500": 0.8497854077253219, + "recall@1000": 0.9527896995708155, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.588272452869007, + "ndcg_full@5": 0.588272452869007, + "ndcg@10": 0.4566464900629862, + "ndcg_full@10": 0.4566464900629862, + "ndcg@20": 0.4905386429325821, + "ndcg_full@20": 0.4905386429325821, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.1, + "graded_P@10": 0.4 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027777777777777776, + "recall@50": 0.11419753086419752, + "recall@100": 0.23148148148148148, + "recall@200": 0.44135802469135804, + "recall@300": 0.5648148148148148, + "recall@500": 0.7901234567901234, + "recall@1000": 0.9197530864197531, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.28902930375209906, + "ndcg_full@5": 0.28902930375209906, + "ndcg@10": 0.5386286660690698, + "ndcg_full@10": 0.5386286660690698, + "ndcg@20": 0.5234982150965332, + "ndcg_full@20": 0.5234982150965332, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027131782945736434, + "recall@50": 0.13953488372093023, + "recall@100": 0.2364341085271318, + "recall@200": 0.42248062015503873, + "recall@300": 0.6162790697674418, + "recall@500": 0.7751937984496124, + "recall@1000": 0.810077519379845, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.36899771078908794, + "ndcg_full@5": 0.36899771078908794, + "ndcg@10": 0.4983959310538874, + "ndcg_full@10": 0.4983959310538874, + "ndcg@20": 0.45485055466196744, + "ndcg_full@20": 0.45485055466196744, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 90, + "recall@10": 0.06, + "recall@50": 0.2733333333333333, + "recall@100": 0.5, + "recall@200": 0.66, + "recall@300": 0.74, + "recall@500": 0.7933333333333333, + "recall@1000": 0.8466666666666667, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.800693766409882, + "ndcg_full@10": 0.800693766409882, + "ndcg@20": 0.7983984789268523, + "ndcg_full@20": 0.7983984789268523, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 217, + "recall@10": 0.01744186046511628, + "recall@50": 0.14534883720930233, + "recall@100": 0.29651162790697677, + "recall@200": 0.47674418604651164, + "recall@300": 0.6162790697674418, + "recall@500": 0.7093023255813954, + "recall@1000": 0.7441860465116279, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.44615333764087994, + "ndcg_full@5": 0.44615333764087994, + "ndcg@10": 0.4066471297161996, + "ndcg_full@10": 0.4066471297161996, + "ndcg@20": 0.3821999712679613, + "ndcg_full@20": 0.3821999712679613, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 103, + "recall@10": 0.07547169811320754, + "recall@50": 0.3584905660377358, + "recall@100": 0.41509433962264153, + "recall@200": 0.4716981132075472, + "recall@300": 0.49056603773584906, + "recall@500": 0.5471698113207547, + "recall@1000": 0.5471698113207547, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.5708588662113935, + "ndcg_full@10": 0.5708588662113935, + "ndcg@20": 0.5652116332861817, + "ndcg_full@20": 0.5467848122611675, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 248, + "recall@10": 0.019138755980861243, + "recall@50": 0.0861244019138756, + "recall@100": 0.16267942583732056, + "recall@200": 0.2631578947368421, + "recall@300": 0.3444976076555024, + "recall@500": 0.46411483253588515, + "recall@1000": 0.631578947368421, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.07336392209936005, + "ndcg_full@10": 0.07336392209936005, + "ndcg@20": 0.09726605727239368, + "ndcg_full@20": 0.09726605727239368, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04, + "recall@50": 0.12, + "recall@100": 0.22, + "recall@200": 0.36, + "recall@300": 0.52, + "recall@500": 0.6933333333333334, + "recall@1000": 0.7733333333333333, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.6983320793199449, + "ndcg_full@10": 0.6983320793199449, + "ndcg@20": 0.591822911933455, + "ndcg_full@20": 0.591822911933455, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 2000, + "num_ranked": 111, + "recall@10": 0.04597701149425287, + "recall@50": 0.25862068965517243, + "recall@100": 0.45977011494252873, + "recall@200": 0.5057471264367817, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.9097832499695674, + "ndcg_full@20": 0.9097832499695674, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.025889967637540454, + "recall@50": 0.08090614886731391, + "recall@100": 0.1650485436893204, + "recall@200": 0.2621359223300971, + "recall@300": 0.35275080906148865, + "recall@500": 0.4919093851132686, + "recall@1000": 0.6245954692556634, + "unjudged_fraction@10": 0.9, + "ndcg@5": 0.026241015502468355, + "ndcg_full@5": 0.026241015502468355, + "ndcg@10": 0.07451815082216454, + "ndcg_full@10": 0.07451815082216454, + "ndcg@20": 0.13711996239760144, + "ndcg_full@20": 0.13711996239760144, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03225806451612903, + "recall@50": 0.18548387096774194, + "recall@100": 0.3629032258064516, + "recall@200": 0.5967741935483871, + "recall@300": 0.7862903225806451, + "recall@500": 0.9395161290322581, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.9370132124040589, + "ndcg_full@20": 0.9370132124040589, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 143, + "recall@10": 0.12345679012345678, + "recall@50": 0.5679012345679012, + "recall@100": 0.9506172839506173, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9255244116680239, + "ndcg_full@5": 0.9255244116680239, + "ndcg@10": 0.7523642055498466, + "ndcg_full@10": 0.7523642055498466, + "ndcg@20": 0.7726834803112372, + "ndcg_full@20": 0.7726834803112372, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 151, + "recall@10": 0.13559322033898305, + "recall@50": 0.6610169491525424, + "recall@100": 0.8813559322033898, + "recall@200": 0.9491525423728814, + "recall@300": 0.9491525423728814, + "recall@500": 0.9491525423728814, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7691415988388896, + "ndcg_full@5": 0.7691415988388896, + "ndcg@10": 0.850188694055929, + "ndcg_full@10": 0.850188694055929, + "ndcg@20": 0.8331763212785439, + "ndcg_full@20": 0.8331763212785439, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 71, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.7348195014414517, + "ndcg_full@10": 0.7348195014414517, + "ndcg@20": 0.7559842621715005, + "ndcg_full@20": 0.7559842621715005, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032467532467532464, + "recall@50": 0.1266233766233766, + "recall@100": 0.237012987012987, + "recall@200": 0.3961038961038961, + "recall@300": 0.5714285714285714, + "recall@500": 0.7922077922077922, + "recall@1000": 0.9253246753246753, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.641626450837791, + "ndcg_full@10": 0.641626450837791, + "ndcg@20": 0.6438118704988053, + "ndcg_full@20": 0.6438118704988053, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 166, + "recall@10": 0.11267605633802817, + "recall@50": 0.36619718309859156, + "recall@100": 0.4647887323943662, + "recall@200": 0.5211267605633803, + "recall@300": 0.6056338028169014, + "recall@500": 0.6338028169014085, + "recall@1000": 0.7183098591549296, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7765732350040719, + "ndcg_full@5": 0.7765732350040719, + "ndcg@10": 0.6859371712772155, + "ndcg_full@10": 0.6859371712772155, + "ndcg@20": 0.6596720245473612, + "ndcg_full@20": 0.6596720245473612, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.06451612903225806, + "recall@50": 0.2645161290322581, + "recall@100": 0.45161290322580644, + "recall@200": 0.7032258064516129, + "recall@300": 0.832258064516129, + "recall@500": 0.896774193548387, + "recall@1000": 0.9161290322580645, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7840048711692542, + "ndcg_full@5": 0.7840048711692542, + "ndcg@10": 0.7496252777057077, + "ndcg_full@10": 0.7496252777057077, + "ndcg@20": 0.679952279989837, + "ndcg_full@20": 0.679952279989837, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.05128205128205128, + "recall@50": 0.24615384615384617, + "recall@100": 0.4666666666666667, + "recall@200": 0.7025641025641025, + "recall@300": 0.7230769230769231, + "recall@500": 0.7333333333333333, + "recall@1000": 0.7487179487179487, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8757862866154155, + "ndcg_full@5": 0.7543701539552121, + "ndcg@10": 0.9054714533341821, + "ndcg_full@10": 0.7700542278177658, + "ndcg@20": 0.9591142047939843, + "ndcg_full@20": 0.8502930827598733, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.2, + "graded_P@10": 0.6 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 216, + "recall@10": 0.053763440860215055, + "recall@50": 0.25268817204301075, + "recall@100": 0.4731182795698925, + "recall@200": 0.8333333333333334, + "recall@300": 0.8817204301075269, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7799082337019199, + "ndcg_full@10": 0.7799082337019199, + "ndcg@20": 0.821128441188851, + "ndcg_full@20": 0.821128441188851, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 225, + "recall@10": 0.08108108108108109, + "recall@50": 0.35135135135135137, + "recall@100": 0.5855855855855856, + "recall@200": 0.9099099099099099, + "recall@300": 0.9819819819819819, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.8830803379577175, + "ndcg_full@20": 0.8830803379577175, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.015037593984962405, + "recall@50": 0.10526315789473684, + "recall@100": 0.24060150375939848, + "recall@200": 0.42105263157894735, + "recall@300": 0.6090225563909775, + "recall@500": 0.7894736842105263, + "recall@1000": 0.924812030075188, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.6777277508780454, + "ndcg_full@10": 0.6777277508780454, + "ndcg@20": 0.6164813757175781, + "ndcg_full@20": 0.6164813757175781, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 76, + "recall@10": 0.15151515151515152, + "recall@50": 0.6212121212121212, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.8787878787878788, + "recall@1000": 0.9545454545454546, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.8975828762407047, + "ndcg_full@10": 0.8975828762407047, + "ndcg@20": 0.93390333240257, + "ndcg_full@20": 0.93390333240257, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.12987012987012986, + "recall@50": 0.42857142857142855, + "recall@100": 0.7012987012987013, + "recall@200": 0.7792207792207793, + "recall@300": 0.8181818181818182, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7632636606403641, + "ndcg_full@5": 0.7632636606403641, + "ndcg@10": 0.6946121232830215, + "ndcg_full@10": 0.6946121232830215, + "ndcg@20": 0.6553339483657566, + "ndcg_full@20": 0.6553339483657566, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 83, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.7391304347826086, + "recall@300": 0.782608695652174, + "recall@500": 0.8260869565217391, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7691415988388896, + "ndcg_full@5": 0.7691415988388896, + "ndcg@10": 0.6168057199769172, + "ndcg_full@10": 0.5971848869416406, + "ndcg@20": 0.5679699503589901, + "ndcg_full@20": 0.486448529826929, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 179, + "recall@10": 0.09859154929577464, + "recall@50": 0.4225352112676056, + "recall@100": 0.676056338028169, + "recall@200": 0.8732394366197183, + "recall@300": 0.8873239436619719, + "recall@500": 0.9154929577464789, + "recall@1000": 0.9295774647887324, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.745670824835362, + "ndcg_full@10": 0.745670824835362, + "ndcg@20": 0.6905998838917691, + "ndcg_full@20": 0.6905998838917691, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 163, + "recall@10": 0.08823529411764706, + "recall@50": 0.4117647058823529, + "recall@100": 0.5882352941176471, + "recall@200": 0.75, + "recall@300": 0.7647058823529411, + "recall@500": 0.7941176470588235, + "recall@1000": 0.8088235294117647, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.5354170967041749, + "ndcg_full@10": 0.5354170967041749, + "ndcg@20": 0.5566955448751011, + "ndcg_full@20": 0.5566955448751011, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 190, + "recall@10": 0.10869565217391304, + "recall@50": 0.43478260869565216, + "recall@100": 0.5652173913043478, + "recall@200": 0.6304347826086957, + "recall@300": 0.6956521739130435, + "recall@500": 0.717391304347826, + "recall@1000": 0.782608695652174, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9262688623283957, + "ndcg_full@10": 0.9262688623283957, + "ndcg@20": 0.8102474820527624, + "ndcg_full@20": 0.8102474820527624, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.12, + "recall@50": 0.52, + "recall@100": 0.88, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.38962811929307617, + "ndcg_full@10": 0.38962811929307617, + "ndcg@20": 0.4080437739626595, + "ndcg_full@20": 0.4080437739626595, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 90, + "recall@10": 0.20512820512820512, + "recall@50": 0.6666666666666666, + "recall@100": 0.6923076923076923, + "recall@200": 0.717948717948718, + "recall@300": 0.7435897435897436, + "recall@500": 0.7692307692307693, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.5582604437617528, + "ndcg_full@10": 0.5582604437617528, + "ndcg@20": 0.5903124953539537, + "ndcg_full@20": 0.5903124953539537, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.10465116279069768, + "recall@50": 0.37209302325581395, + "recall@100": 0.6627906976744186, + "recall@200": 0.9767441860465116, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.907575524214984, + "ndcg_full@5": 0.907575524214984, + "ndcg@10": 0.9138900830033845, + "ndcg_full@10": 0.9138900830033845, + "ndcg@20": 0.7644780409738634, + "ndcg_full@20": 0.7644780409738634, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 139, + "recall@10": 0.06172839506172839, + "recall@50": 0.19753086419753085, + "recall@100": 0.3271604938271605, + "recall@200": 0.5679012345679012, + "recall@300": 0.6666666666666666, + "recall@500": 0.7654320987654321, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.866947989864271, + "ndcg_full@10": 0.866947989864271, + "ndcg@20": 0.8104918300996614, + "ndcg_full@20": 0.8104918300996614, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.07352941176470588, + "recall@50": 0.3235294117647059, + "recall@100": 0.5735294117647058, + "recall@200": 0.6764705882352942, + "recall@300": 0.6764705882352942, + "recall@500": 0.6911764705882353, + "recall@1000": 0.6911764705882353, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7246500855306732, + "ndcg_full@5": 0.7246500855306732, + "ndcg@10": 0.758082499953416, + "ndcg_full@10": 0.758082499953416, + "ndcg@20": 0.787484299361068, + "ndcg_full@20": 0.787484299361068, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1317, + "num_ranked": 200, + "recall@10": 0.019943019943019943, + "recall@50": 0.1225071225071225, + "recall@100": 0.23931623931623933, + "recall@200": 0.45014245014245013, + "recall@300": 0.6125356125356125, + "recall@500": 0.6296296296296297, + "recall@1000": 0.6324786324786325, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8516545733528995, + "ndcg_full@20": 0.8516545733528995, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 166, + "recall@10": 0.04522613065326633, + "recall@50": 0.16080402010050251, + "recall@100": 0.271356783919598, + "recall@200": 0.507537688442211, + "recall@300": 0.6030150753768844, + "recall@500": 0.7386934673366834, + "recall@1000": 0.8944723618090452, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.16520994868159597, + "ndcg_full@5": 0.16520994868159597, + "ndcg@10": 0.3920241534722059, + "ndcg_full@10": 0.3920241534722059, + "ndcg@20": 0.44871687419383327, + "ndcg_full@20": 0.44871687419383327, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03684210526315789, + "recall@50": 0.07368421052631578, + "recall@100": 0.14210526315789473, + "recall@200": 0.28421052631578947, + "recall@300": 0.4421052631578947, + "recall@500": 0.7105263157894737, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.5054799843174406, + "ndcg_full@10": 0.5054799843174406, + "ndcg@20": 0.5373996799325468, + "ndcg_full@20": 0.5373996799325468, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 152, + "recall@10": 0.03636363636363636, + "recall@50": 0.15272727272727274, + "recall@100": 0.2727272727272727, + "recall@200": 0.4509090909090909, + "recall@300": 0.5490909090909091, + "recall@500": 0.6581818181818182, + "recall@1000": 0.7563636363636363, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5222030810488597, + "ndcg_full@5": 0.5222030810488597, + "ndcg@10": 0.47165429655043406, + "ndcg_full@10": 0.47165429655043406, + "ndcg@20": 0.5735575173164446, + "ndcg_full@20": 0.5735575173164446, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.2, + "graded_P@10": 0.45 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.09090909090909091, + "recall@50": 0.3409090909090909, + "recall@100": 0.4318181818181818, + "recall@200": 0.6590909090909091, + "recall@300": 0.7954545454545454, + "recall@500": 0.8636363636363636, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.16520994868159597, + "ndcg_full@5": 0.16520994868159597, + "ndcg@10": 0.20461621099844715, + "ndcg_full@10": 0.20461621099844715, + "ndcg@20": 0.23809189569732941, + "ndcg_full@20": 0.23809189569732941, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.019704433497536946, + "recall@50": 0.11330049261083744, + "recall@100": 0.23645320197044334, + "recall@200": 0.3842364532019704, + "recall@300": 0.5517241379310345, + "recall@500": 0.8275862068965517, + "recall@1000": 0.9655172413793104, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.36885567213865283, + "ndcg_full@10": 0.36885567213865283, + "ndcg@20": 0.38881960565438467, + "ndcg_full@20": 0.37614345240050734, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035398230088495575, + "recall@50": 0.17699115044247787, + "recall@100": 0.30973451327433627, + "recall@200": 0.5132743362831859, + "recall@300": 0.6283185840707964, + "recall@500": 0.7433628318584071, + "recall@1000": 0.8584070796460177, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.4322938718959355, + "ndcg_full@10": 0.4322938718959355, + "ndcg@20": 0.46577593205270507, + "ndcg_full@20": 0.46577593205270507, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 59, + "recall@10": 0.125, + "recall@50": 0.25, + "recall@100": 0.2916666666666667, + "recall@200": 0.3125, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6429424283490954, + "ndcg_full@10": 0.6429424283490954, + "ndcg@20": 0.6769261208474632, + "ndcg_full@20": 0.5807817851388194, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.06369426751592357, + "recall@50": 0.25477707006369427, + "recall@100": 0.4713375796178344, + "recall@200": 0.7388535031847133, + "recall@300": 0.7898089171974523, + "recall@500": 0.8280254777070064, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6145101197429427, + "ndcg_full@5": 0.6145101197429427, + "ndcg@10": 0.6295051058353064, + "ndcg_full@10": 0.6295051058353064, + "ndcg@20": 0.5480693112728487, + "ndcg_full@20": 0.5480693112728487, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.06097560975609756, + "recall@50": 0.27439024390243905, + "recall@100": 0.3475609756097561, + "recall@200": 0.42073170731707316, + "recall@300": 0.47560975609756095, + "recall@500": 0.5975609756097561, + "recall@1000": 0.7317073170731707, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.48824412360294916, + "ndcg_full@5": 0.48824412360294916, + "ndcg@10": 0.5981545110753473, + "ndcg_full@10": 0.5981545110753473, + "ndcg@20": 0.6125610089688943, + "ndcg_full@20": 0.5719164450197116, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 213, + "recall@10": 0.0, + "recall@50": 0.0379746835443038, + "recall@100": 0.08860759493670886, + "recall@200": 0.16455696202531644, + "recall@300": 0.26582278481012656, + "recall@500": 0.4430379746835443, + "recall@1000": 0.5316455696202531, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.15130120674940672, + "ndcg_full@10": 0.11004588314904008, + "ndcg@20": 0.15130120674940672, + "ndcg_full@20": 0.07102001981682865, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03896103896103896, + "recall@50": 0.1774891774891775, + "recall@100": 0.3116883116883117, + "recall@200": 0.5541125541125541, + "recall@300": 0.7835497835497836, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.4226415848467783, + "ndcg_full@5": 0.4226415848467783, + "ndcg@10": 0.4071362918487801, + "ndcg_full@10": 0.4071362918487801, + "ndcg@20": 0.48812670872690933, + "ndcg_full@20": 0.48812670872690933, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.016304347826086956, + "recall@50": 0.11956521739130435, + "recall@100": 0.18478260869565216, + "recall@200": 0.3532608695652174, + "recall@300": 0.45652173913043476, + "recall@500": 0.6304347826086957, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.6069108398694022, + "ndcg_full@10": 0.6069108398694022, + "ndcg@20": 0.4960289822645006, + "ndcg_full@20": 0.4960289822645006, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 98, + "recall@10": 0.10989010989010989, + "recall@50": 0.23076923076923078, + "recall@100": 0.24175824175824176, + "recall@200": 0.27472527472527475, + "recall@300": 0.27472527472527475, + "recall@500": 0.2857142857142857, + "recall@1000": 0.32967032967032966, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6010212420982718, + "ndcg_full@10": 0.6010212420982718, + "ndcg@20": 0.6167198376863421, + "ndcg_full@20": 0.5799790514473045, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 202, + "recall@10": 0.13636363636363635, + "recall@50": 0.5757575757575758, + "recall@100": 0.803030303030303, + "recall@200": 0.8939393939393939, + "recall@300": 0.9545454545454546, + "recall@500": 0.9696969696969697, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.7715951962267015, + "ndcg_full@20": 0.7591191611745894, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 84, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6261806449294969, + "ndcg_full@5": 0.6261806449294969, + "ndcg@10": 0.6815357214855459, + "ndcg_full@10": 0.6815357214855459, + "ndcg@20": 0.7667082767206322, + "ndcg_full@20": 0.7667082767206322, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.11143695014662756, + "recall@100": 0.21994134897360704, + "recall@200": 0.4222873900293255, + "recall@300": 0.5806451612903226, + "recall@500": 0.841642228739003, + "recall@1000": 0.9354838709677419, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.6265706800017662, + "ndcg_full@10": 0.6265706800017662, + "ndcg@20": 0.5715013535975216, + "ndcg_full@20": 0.5715013535975216, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + } + } + } + } + }, + { + "name": "qwen36_medcpt", + "results_dir": "qwen36_medcpt/results_trec21", + "evaluation_input_sha256": "217228f467f3a2ae14a2a88f9c78a345e833e800bcae0f9dcc99ca83cb37e876", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.0818034764464546, + "recall@50": 0.2778819305429651, + "recall@100": 0.4269902560041185, + "recall@200": 0.5945583921870587, + "recall@300": 0.6778828178518908, + "recall@500": 0.7669052888565161, + "recall@1000": 0.8361387639997722, + "ndcg@5": 0.8342761073727945, + "ndcg@10": 0.8214584944628943, + "ndcg@20": 0.8013157236312944, + "ndcg_full@5": 0.8338080514581386, + "ndcg_full@10": 0.8197380535510176, + "ndcg_full@20": 0.7860709157876381, + "P@10(rel>=1)": 0.857333333333333, + "P@10(eligible)": 0.7679999999999998, + "graded_P@10": 0.8126666666666668, + "unjudged_fraction@10": 0.2786666666666667 + }, + "median": { + "recall@10": 0.04938271604938271, + "recall@50": 0.22875816993464052, + "recall@100": 0.3588709677419355, + "recall@200": 0.549800796812749, + "recall@300": 0.6739130434782609, + "recall@500": 0.7954545454545454, + "recall@1000": 0.8636363636363636, + "ndcg@5": 0.8930068676327362, + "ndcg@10": 0.8643145546088337, + "ndcg@20": 0.8473333391091306, + "ndcg_full@5": 0.8930068676327362, + "ndcg_full@10": 0.8643145546088337, + "ndcg_full@20": 0.842108482619296, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85, + "unjudged_fraction@10": 0.3 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 247, + "recall@10": 0.05917159763313609, + "recall@50": 0.25443786982248523, + "recall@100": 0.4911242603550296, + "recall@200": 0.757396449704142, + "recall@300": 0.8284023668639053, + "recall@500": 0.863905325443787, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.826813366651774, + "ndcg_full@10": 0.826813366651774, + "ndcg@20": 0.7461685368362136, + "ndcg_full@20": 0.7461685368362136, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03333333333333333, + "recall@50": 0.15925925925925927, + "recall@100": 0.24814814814814815, + "recall@200": 0.40370370370370373, + "recall@300": 0.5259259259259259, + "recall@500": 0.6703703703703704, + "recall@1000": 0.837037037037037, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213246, + "ndcg_full@10": 0.9216017310213246, + "ndcg@20": 0.9494043170253829, + "ndcg_full@20": 0.9494043170253829, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 133, + "recall@10": 0.11904761904761904, + "recall@50": 0.36904761904761907, + "recall@100": 0.5238095238095238, + "recall@200": 0.7023809523809523, + "recall@300": 0.8095238095238095, + "recall@500": 0.8928571428571429, + "recall@1000": 0.9761904761904762, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.8473825580301494, + "ndcg_full@10": 0.8473825580301494, + "ndcg@20": 0.7610012419404353, + "ndcg_full@20": 0.7610012419404353, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021052631578947368, + "recall@50": 0.14736842105263157, + "recall@100": 0.29473684210526313, + "recall@200": 0.47368421052631576, + "recall@300": 0.6105263157894737, + "recall@500": 0.7894736842105263, + "recall@1000": 0.9368421052631579, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.5567484706058157, + "ndcg_full@10": 0.5567484706058157, + "ndcg@20": 0.5237308271921702, + "ndcg_full@20": 0.5237308271921702, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04975124378109453, + "recall@50": 0.1791044776119403, + "recall@100": 0.2885572139303483, + "recall@200": 0.46766169154228854, + "recall@300": 0.5621890547263682, + "recall@500": 0.681592039800995, + "recall@1000": 0.7711442786069652, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8389574121760602, + "ndcg_full@10": 0.8389574121760602, + "ndcg@20": 0.8596599919695072, + "ndcg_full@20": 0.8596599919695072, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.047619047619047616, + "recall@50": 0.17142857142857143, + "recall@100": 0.2904761904761905, + "recall@200": 0.46190476190476193, + "recall@300": 0.5761904761904761, + "recall@500": 0.7380952380952381, + "recall@1000": 0.8666666666666667, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.948458733550261, + "ndcg_full@20": 0.948458733550261, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031055900621118012, + "recall@50": 0.11180124223602485, + "recall@100": 0.2360248447204969, + "recall@200": 0.42857142857142855, + "recall@300": 0.5527950310559007, + "recall@500": 0.6832298136645962, + "recall@1000": 0.8322981366459627, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.5747915663739762, + "ndcg_full@10": 0.5747915663739762, + "ndcg@20": 0.5114684177948652, + "ndcg_full@20": 0.5114684177948652, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032679738562091505, + "recall@50": 0.22875816993464052, + "recall@100": 0.43137254901960786, + "recall@200": 0.6993464052287581, + "recall@300": 0.8235294117647058, + "recall@500": 0.9084967320261438, + "recall@1000": 0.9477124183006536, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8200617090286295, + "ndcg_full@20": 0.8200617090286295, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04672897196261682, + "recall@50": 0.205607476635514, + "recall@100": 0.40186915887850466, + "recall@200": 0.7289719626168224, + "recall@300": 0.9112149532710281, + "recall@500": 0.9345794392523364, + "recall@1000": 0.9439252336448598, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.44484466589563804, + "ndcg_full@5": 0.44484466589563804, + "ndcg@10": 0.39478693280408267, + "ndcg_full@10": 0.39478693280408267, + "ndcg@20": 0.46012600992293606, + "ndcg_full@20": 0.4299909109901354, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 89, + "recall@10": 0.1702127659574468, + "recall@50": 0.7872340425531915, + "recall@100": 0.8085106382978723, + "recall@200": 0.8085106382978723, + "recall@300": 0.8085106382978723, + "recall@500": 0.8297872340425532, + "recall@1000": 0.851063829787234, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7883291114011227, + "ndcg_full@5": 0.7883291114011227, + "ndcg@10": 0.7961140717417743, + "ndcg_full@10": 0.7961140717417743, + "ndcg@20": 0.8868126169097346, + "ndcg_full@20": 0.8659599737479304, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 231, + "recall@10": 0.07936507936507936, + "recall@50": 0.35714285714285715, + "recall@100": 0.5634920634920635, + "recall@200": 0.8095238095238095, + "recall@300": 0.8333333333333334, + "recall@500": 0.8412698412698413, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.7922673501371923, + "ndcg_full@10": 0.7922673501371923, + "ndcg@20": 0.7737303470419717, + "ndcg_full@20": 0.7737303470419717, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 174, + "recall@10": 0.06040268456375839, + "recall@50": 0.2483221476510067, + "recall@100": 0.46308724832214765, + "recall@200": 0.7651006711409396, + "recall@300": 0.825503355704698, + "recall@500": 0.8590604026845637, + "recall@1000": 0.912751677852349, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8945734077057459, + "ndcg_full@20": 0.8945734077057459, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 184, + "recall@10": 0.028985507246376812, + "recall@50": 0.18840579710144928, + "recall@100": 0.2971014492753623, + "recall@200": 0.4927536231884058, + "recall@300": 0.6231884057971014, + "recall@500": 0.6811594202898551, + "recall@1000": 0.7463768115942029, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8643145546088337, + "ndcg_full@10": 0.8643145546088337, + "ndcg@20": 0.7909830297264099, + "ndcg_full@20": 0.7909830297264099, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.030612244897959183, + "recall@50": 0.11904761904761904, + "recall@100": 0.2108843537414966, + "recall@200": 0.3707482993197279, + "recall@300": 0.45918367346938777, + "recall@500": 0.5748299319727891, + "recall@1000": 0.717687074829932, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7972801269920728, + "ndcg_full@10": 0.7972801269920728, + "ndcg@20": 0.8011840902923884, + "ndcg_full@20": 0.8011840902923884, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 198, + "recall@10": 0.019230769230769232, + "recall@50": 0.11923076923076924, + "recall@100": 0.23461538461538461, + "recall@200": 0.39615384615384613, + "recall@300": 0.4653846153846154, + "recall@500": 0.5769230769230769, + "recall@1000": 0.7, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9675312470866648, + "ndcg_full@10": 0.9675312470866648, + "ndcg@20": 0.9232860025097027, + "ndcg_full@20": 0.9232860025097027, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.014492753623188406, + "recall@50": 0.10869565217391304, + "recall@100": 0.2028985507246377, + "recall@200": 0.34057971014492755, + "recall@300": 0.37681159420289856, + "recall@500": 0.4927536231884058, + "recall@1000": 0.6884057971014492, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.10699313236726378, + "ndcg_full@5": 0.10699313236726378, + "ndcg@10": 0.14279514403613733, + "ndcg_full@10": 0.14279514403613733, + "ndcg@20": 0.14961782505341292, + "ndcg_full@20": 0.14474003995125245, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02390438247011952, + "recall@50": 0.13545816733067728, + "recall@100": 0.28286852589641437, + "recall@200": 0.549800796812749, + "recall@300": 0.7768924302788844, + "recall@500": 0.9362549800796812, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4121943801949693, + "ndcg_full@5": 0.4121943801949693, + "ndcg@10": 0.5491231512142316, + "ndcg_full@10": 0.5491231512142316, + "ndcg@20": 0.5507072623073134, + "ndcg_full@20": 0.5507072623073134, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03398058252427184, + "recall@50": 0.18932038834951456, + "recall@100": 0.32038834951456313, + "recall@200": 0.5048543689320388, + "recall@300": 0.616504854368932, + "recall@500": 0.7572815533980582, + "recall@1000": 0.8883495145631068, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6578242262397573, + "ndcg_full@5": 0.6578242262397573, + "ndcg@10": 0.6744711422671819, + "ndcg_full@10": 0.6306726685362858, + "ndcg@20": 0.7230267968779449, + "ndcg_full@20": 0.6282804698061588, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032679738562091505, + "recall@50": 0.16339869281045752, + "recall@100": 0.38562091503267976, + "recall@200": 0.6666666666666666, + "recall@300": 0.7843137254901961, + "recall@500": 0.869281045751634, + "recall@1000": 0.9281045751633987, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8701249883466593, + "ndcg_full@10": 0.8701249883466593, + "ndcg@20": 0.8120224873996634, + "ndcg_full@20": 0.8120224873996634, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.11428571428571428, + "recall@50": 0.37142857142857144, + "recall@100": 0.7285714285714285, + "recall@200": 0.8857142857142857, + "recall@300": 0.9142857142857143, + "recall@500": 0.9428571428571428, + "recall@1000": 0.9571428571428572, + "unjudged_fraction@10": 0.8, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8365833032144286, + "ndcg_full@20": 0.8365833032144286, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 210, + "recall@10": 0.11320754716981132, + "recall@50": 0.41509433962264153, + "recall@100": 0.7169811320754716, + "recall@200": 0.8679245283018868, + "recall@300": 0.9056603773584906, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8165901947515999, + "ndcg_full@10": 0.8165901947515999, + "ndcg@20": 0.8492950091569659, + "ndcg_full@20": 0.8492950091569659, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 244, + "recall@10": 0.06569343065693431, + "recall@50": 0.291970802919708, + "recall@100": 0.48175182481751827, + "recall@200": 0.7445255474452555, + "recall@300": 0.7737226277372263, + "recall@500": 0.8321167883211679, + "recall@1000": 0.8613138686131386, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8937600634656412, + "ndcg_full@20": 0.8937600634656412, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0299625468164794, + "recall@50": 0.13857677902621723, + "recall@100": 0.2397003745318352, + "recall@200": 0.4232209737827715, + "recall@300": 0.550561797752809, + "recall@500": 0.7790262172284644, + "recall@1000": 0.8726591760299626, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9059769837392324, + "ndcg_full@20": 0.9059769837392324, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04430379746835443, + "recall@50": 0.24050632911392406, + "recall@100": 0.41139240506329117, + "recall@200": 0.689873417721519, + "recall@300": 0.8354430379746836, + "recall@500": 0.9177215189873418, + "recall@1000": 0.9746835443037974, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0000000000000002, + "ndcg_full@5": 1.0000000000000002, + "ndcg@10": 0.9337457765456112, + "ndcg_full@10": 0.9337457765456112, + "ndcg@20": 0.9410725419108087, + "ndcg_full@20": 0.9410725419108087, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.13725490196078433, + "recall@100": 0.20588235294117646, + "recall@200": 0.3431372549019608, + "recall@300": 0.4117647058823529, + "recall@500": 0.6078431372549019, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7426142775581612, + "ndcg_full@5": 0.7426142775581612, + "ndcg@10": 0.6541569976248054, + "ndcg_full@10": 0.6541569976248054, + "ndcg@20": 0.6553155695233592, + "ndcg_full@20": 0.6553155695233592, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.05625, + "recall@50": 0.175, + "recall@100": 0.30625, + "recall@200": 0.50625, + "recall@300": 0.6125, + "recall@500": 0.725, + "recall@1000": 0.83125, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8919204669322514, + "ndcg_full@10": 0.8919204669322514, + "ndcg@20": 0.7710312777268998, + "ndcg_full@20": 0.7710312777268998, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.13733905579399142, + "recall@100": 0.26180257510729615, + "recall@200": 0.4592274678111588, + "recall@300": 0.630901287553648, + "recall@500": 0.8454935622317596, + "recall@1000": 0.9527896995708155, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3766321352557, + "ndcg_full@5": 0.3766321352557, + "ndcg@10": 0.4648194694487437, + "ndcg_full@10": 0.4648194694487437, + "ndcg@20": 0.5485565175278263, + "ndcg_full@20": 0.5485565175278263, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.2, + "graded_P@10": 0.55 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018518518518518517, + "recall@50": 0.09567901234567901, + "recall@100": 0.19753086419753085, + "recall@200": 0.3950617283950617, + "recall@300": 0.5771604938271605, + "recall@500": 0.8024691358024691, + "recall@1000": 0.9074074074074074, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9145767929877944, + "ndcg_full@20": 0.9145767929877944, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027131782945736434, + "recall@50": 0.1511627906976744, + "recall@100": 0.24031007751937986, + "recall@200": 0.43410852713178294, + "recall@300": 0.6162790697674418, + "recall@500": 0.7751937984496124, + "recall@1000": 0.8062015503875969, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.904108408157166, + "ndcg_full@5": 0.904108408157166, + "ndcg@10": 0.9046458065171568, + "ndcg_full@10": 0.9046458065171568, + "ndcg@20": 0.8458863153335652, + "ndcg_full@20": 0.8458863153335652, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.06666666666666667, + "recall@50": 0.28, + "recall@100": 0.5133333333333333, + "recall@200": 0.68, + "recall@300": 0.7466666666666667, + "recall@500": 0.82, + "recall@1000": 0.8466666666666667, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9478384225683674, + "ndcg_full@20": 0.9478384225683674, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 234, + "recall@10": 0.023255813953488372, + "recall@50": 0.11627906976744186, + "recall@100": 0.23255813953488372, + "recall@200": 0.45348837209302323, + "recall@300": 0.5988372093023255, + "recall@500": 0.7093023255813954, + "recall@1000": 0.7790697674418605, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5030155684866267, + "ndcg_full@5": 0.5030155684866267, + "ndcg@10": 0.4864522230035559, + "ndcg_full@10": 0.4864522230035559, + "ndcg@20": 0.44528875095347514, + "ndcg_full@20": 0.44528875095347514, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 94, + "recall@10": 0.11320754716981132, + "recall@50": 0.2641509433962264, + "recall@100": 0.37735849056603776, + "recall@200": 0.4339622641509434, + "recall@300": 0.4716981132075472, + "recall@500": 0.4716981132075472, + "recall@1000": 0.5660377358490566, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8603818544462509, + "ndcg_full@10": 0.8603818544462509, + "ndcg@20": 0.8492302150114562, + "ndcg_full@20": 0.6292899956002662, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03349282296650718, + "recall@50": 0.1291866028708134, + "recall@100": 0.23923444976076555, + "recall@200": 0.3444976076555024, + "recall@300": 0.4019138755980861, + "recall@500": 0.5167464114832536, + "recall@1000": 0.6028708133971292, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9999999999999997, + "ndcg_full@20": 0.9999999999999997, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04, + "recall@50": 0.13333333333333333, + "recall@100": 0.24, + "recall@200": 0.42, + "recall@300": 0.5666666666666667, + "recall@500": 0.6866666666666666, + "recall@1000": 0.76, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8917480892315613, + "ndcg_full@20": 0.8917480892315613, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 2000, + "num_ranked": 94, + "recall@10": 0.05172413793103448, + "recall@50": 0.25862068965517243, + "recall@100": 0.4540229885057471, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.8681235945057538, + "ndcg_full@20": 0.8681235945057538, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.006472491909385114, + "recall@50": 0.032362459546925564, + "recall@100": 0.09061488673139159, + "recall@200": 0.20064724919093851, + "recall@300": 0.3074433656957929, + "recall@500": 0.4627831715210356, + "recall@1000": 0.6375404530744336, + "unjudged_fraction@10": 0.8, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9652498273011314, + "ndcg_full@20": 0.9652498273011314, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03225806451612903, + "recall@50": 0.1935483870967742, + "recall@100": 0.3588709677419355, + "recall@200": 0.6451612903225806, + "recall@300": 0.7862903225806451, + "recall@500": 0.9274193548387096, + "recall@1000": 0.9516129032258065, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9832812543076433, + "ndcg_full@20": 0.9832812543076433, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 108, + "recall@10": 0.12345679012345678, + "recall@50": 0.5802469135802469, + "recall@100": 0.9506172839506173, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9753086419753086, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.7980647256682447, + "ndcg_full@10": 0.7980647256682447, + "ndcg@20": 0.8532450821601955, + "ndcg_full@20": 0.8532450821601955, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 137, + "recall@10": 0.13559322033898305, + "recall@50": 0.6949152542372882, + "recall@100": 0.864406779661017, + "recall@200": 0.9152542372881356, + "recall@300": 0.9491525423728814, + "recall@500": 0.9491525423728814, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.8589002902359547, + "ndcg_full@20": 0.8589002902359547, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 57, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8285714603467751, + "ndcg_full@10": 0.8285714603467751, + "ndcg@20": 0.9117519654203567, + "ndcg_full@20": 0.863235239534642, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02922077922077922, + "recall@50": 0.12337662337662338, + "recall@100": 0.22727272727272727, + "recall@200": 0.4318181818181818, + "recall@300": 0.5876623376623377, + "recall@500": 0.8344155844155844, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8938076529802379, + "ndcg_full@10": 0.8938076529802379, + "ndcg@20": 0.8788966169171863, + "ndcg_full@20": 0.8788966169171863, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 118, + "recall@10": 0.08450704225352113, + "recall@50": 0.36619718309859156, + "recall@100": 0.5070422535211268, + "recall@200": 0.6197183098591549, + "recall@300": 0.704225352112676, + "recall@500": 0.7605633802816901, + "recall@1000": 0.8309859154929577, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9478384225683675, + "ndcg_full@20": 0.9478384225683675, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 136, + "recall@10": 0.05806451612903226, + "recall@50": 0.24516129032258063, + "recall@100": 0.44516129032258067, + "recall@200": 0.6903225806451613, + "recall@300": 0.7741935483870968, + "recall@500": 0.864516129032258, + "recall@1000": 0.896774193548387, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7867461507334154, + "ndcg_full@10": 0.7867461507334154, + "ndcg@20": 0.7779224608960016, + "ndcg_full@20": 0.7779224608960016, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.05128205128205128, + "recall@50": 0.2512820512820513, + "recall@100": 0.441025641025641, + "recall@200": 0.6820512820512821, + "recall@300": 0.7076923076923077, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7384615384615385, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5351041935991797, + "ndcg_full@5": 0.5, + "ndcg@10": 0.7323959400285283, + "ndcg_full@10": 0.6630536265254862, + "ndcg@20": 0.8019791722639285, + "ndcg_full@20": 0.7443743002001465, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.2, + "graded_P@10": 0.6 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.053763440860215055, + "recall@50": 0.24193548387096775, + "recall@100": 0.478494623655914, + "recall@200": 0.8279569892473119, + "recall@300": 0.8817204301075269, + "recall@500": 0.8870967741935484, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9347967382841031, + "ndcg_full@20": 0.9347967382841031, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.08108108108108109, + "recall@50": 0.34234234234234234, + "recall@100": 0.5765765765765766, + "recall@200": 0.9279279279279279, + "recall@300": 0.9819819819819819, + "recall@500": 0.990990990990991, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9307787414438842, + "ndcg_full@20": 0.9307787414438842, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03007518796992481, + "recall@50": 0.14285714285714285, + "recall@100": 0.2593984962406015, + "recall@200": 0.4473684210526316, + "recall@300": 0.6240601503759399, + "recall@500": 0.7969924812030075, + "recall@1000": 0.9360902255639098, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9681896059005243, + "ndcg_full@10": 0.9681896059005243, + "ndcg@20": 0.9436797643683754, + "ndcg_full@20": 0.9436797643683754, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.15151515151515152, + "recall@50": 0.5909090909090909, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.803030303030303, + "recall@1000": 0.8636363636363636, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8611375561264454, + "ndcg_full@10": 0.8611375561264454, + "ndcg@20": 0.9103827127966565, + "ndcg_full@20": 0.9103827127966565, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.11688311688311688, + "recall@50": 0.4805194805194805, + "recall@100": 0.7402597402597403, + "recall@200": 0.7922077922077922, + "recall@300": 0.8181818181818182, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.7951657524814092, + "ndcg_full@10": 0.7951657524814092, + "ndcg@20": 0.7993964029807279, + "ndcg_full@20": 0.7993964029807279, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 91, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.782608695652174, + "recall@300": 0.782608695652174, + "recall@500": 0.8260869565217391, + "recall@1000": 0.9565217391304348, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5878056198050307, + "ndcg_full@5": 0.5878056198050307, + "ndcg@10": 0.4995939725591009, + "ndcg_full@10": 0.4837016914022732, + "ndcg@20": 0.5531364610904368, + "ndcg_full@20": 0.4737441093864981, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 115, + "recall@10": 0.1267605633802817, + "recall@50": 0.4084507042253521, + "recall@100": 0.6901408450704225, + "recall@200": 0.8732394366197183, + "recall@300": 0.9014084507042254, + "recall@500": 0.9154929577464789, + "recall@1000": 0.9577464788732394, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.7510916729774053, + "ndcg_full@10": 0.7510916729774053, + "ndcg@20": 0.7568637389638984, + "ndcg_full@20": 0.7195349343609271, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.08823529411764706, + "recall@50": 0.4264705882352941, + "recall@100": 0.6176470588235294, + "recall@200": 0.7352941176470589, + "recall@300": 0.7794117647058824, + "recall@500": 0.7794117647058824, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.26187778578763665, + "ndcg_full@10": 0.26187778578763665, + "ndcg@20": 0.4399933352584693, + "ndcg_full@20": 0.4399933352584693, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.13043478260869565, + "recall@50": 0.3695652173913043, + "recall@100": 0.5652173913043478, + "recall@200": 0.6739130434782609, + "recall@300": 0.6739130434782609, + "recall@500": 0.7608695652173914, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9133234723572442, + "ndcg_full@20": 0.9133234723572442, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.12, + "recall@50": 0.49333333333333335, + "recall@100": 0.8933333333333333, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.55206785919664, + "ndcg_full@10": 0.55206785919664, + "ndcg@20": 0.49308855316831873, + "ndcg_full@20": 0.49308855316831873, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 109, + "recall@10": 0.1794871794871795, + "recall@50": 0.6410256410256411, + "recall@100": 0.6923076923076923, + "recall@200": 0.7435897435897436, + "recall@300": 0.7435897435897436, + "recall@500": 0.7692307692307693, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7828577588238569, + "ndcg_full@10": 0.7828577588238569, + "ndcg@20": 0.842108482619296, + "ndcg_full@20": 0.842108482619296, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 203, + "recall@10": 0.10465116279069768, + "recall@50": 0.37209302325581395, + "recall@100": 0.627906976744186, + "recall@200": 0.9883720930232558, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8712459207538756, + "ndcg_full@10": 0.8712459207538756, + "ndcg@20": 0.8473333391091306, + "ndcg_full@20": 0.8473333391091306, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04938271604938271, + "recall@50": 0.17901234567901234, + "recall@100": 0.3271604938271605, + "recall@200": 0.5185185185185185, + "recall@300": 0.6419753086419753, + "recall@500": 0.7777777777777778, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9278651300142691, + "ndcg_full@20": 0.9278651300142691, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 126, + "recall@10": 0.07352941176470588, + "recall@50": 0.3235294117647059, + "recall@100": 0.5588235294117647, + "recall@200": 0.6691176470588235, + "recall@300": 0.6764705882352942, + "recall@500": 0.6911764705882353, + "recall@1000": 0.6911764705882353, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8795455712258413, + "ndcg_full@5": 0.8795455712258413, + "ndcg@10": 0.9218333177798875, + "ndcg_full@10": 0.9218333177798875, + "ndcg@20": 0.8807319727020904, + "ndcg_full@20": 0.8807319727020904, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02564102564102564, + "recall@50": 0.1225071225071225, + "recall@100": 0.23076923076923078, + "recall@200": 0.46438746438746437, + "recall@300": 0.6153846153846154, + "recall@500": 0.6296296296296297, + "recall@1000": 0.6353276353276354, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.9407630766188416, + "ndcg_full@20": 0.9407630766188416, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03015075376884422, + "recall@50": 0.11557788944723618, + "recall@100": 0.24623115577889448, + "recall@200": 0.45226130653266333, + "recall@300": 0.5829145728643216, + "recall@500": 0.7236180904522613, + "recall@1000": 0.8793969849246231, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9603789329441844, + "ndcg_full@20": 0.9603789329441844, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.07368421052631578, + "recall@100": 0.13157894736842105, + "recall@200": 0.30526315789473685, + "recall@300": 0.4263157894736842, + "recall@500": 0.7052631578947368, + "recall@1000": 0.9526315789473684, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.847582794887513, + "ndcg_full@20": 0.847582794887513, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03272727272727273, + "recall@50": 0.16, + "recall@100": 0.2690909090909091, + "recall@200": 0.4254545454545455, + "recall@300": 0.5309090909090909, + "recall@500": 0.6690909090909091, + "recall@1000": 0.7672727272727272, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.72307666882044, + "ndcg_full@5": 0.72307666882044, + "ndcg@10": 0.6431729948762345, + "ndcg_full@10": 0.6431729948762345, + "ndcg@20": 0.6651743446787199, + "ndcg_full@20": 0.6651743446787199, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.4, + "graded_P@10": 0.6 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 234, + "recall@10": 0.045454545454545456, + "recall@50": 0.3181818181818182, + "recall@100": 0.5, + "recall@200": 0.5909090909090909, + "recall@300": 0.6818181818181818, + "recall@500": 0.7954545454545454, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7600791051976192, + "ndcg_full@10": 0.7600791051976192, + "ndcg@20": 0.7027842698325941, + "ndcg_full@20": 0.7027842698325941, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029556650246305417, + "recall@50": 0.09359605911330049, + "recall@100": 0.2019704433497537, + "recall@200": 0.3793103448275862, + "recall@300": 0.5270935960591133, + "recall@500": 0.8472906403940886, + "recall@1000": 0.9605911330049262, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.800693766409882, + "ndcg_full@10": 0.800693766409882, + "ndcg@20": 0.7157600739167878, + "ndcg_full@20": 0.7157600739167878, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.008849557522123894, + "recall@50": 0.11504424778761062, + "recall@100": 0.20353982300884957, + "recall@200": 0.34513274336283184, + "recall@300": 0.46017699115044247, + "recall@500": 0.6460176991150443, + "recall@1000": 0.8141592920353983, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.857980942822373, + "ndcg_full@10": 0.857980942822373, + "ndcg@20": 0.7619648519413039, + "ndcg_full@20": 0.7619648519413039, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 53, + "recall@10": 0.14583333333333334, + "recall@50": 0.25, + "recall@100": 0.2916666666666667, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8701249883466593, + "ndcg_full@10": 0.8701249883466593, + "ndcg@20": 0.781262846811849, + "ndcg_full@20": 0.6871564665686474, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.06369426751592357, + "recall@50": 0.2611464968152866, + "recall@100": 0.445859872611465, + "recall@200": 0.7261146496815286, + "recall@300": 0.7961783439490446, + "recall@500": 0.8089171974522293, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7456298460447879, + "ndcg_full@5": 0.7456298460447879, + "ndcg@10": 0.7982492142267598, + "ndcg_full@10": 0.7982492142267598, + "ndcg@20": 0.834424509460235, + "ndcg_full@20": 0.834424509460235, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.06097560975609756, + "recall@50": 0.2682926829268293, + "recall@100": 0.3353658536585366, + "recall@200": 0.4329268292682927, + "recall@300": 0.5, + "recall@500": 0.5548780487804879, + "recall@1000": 0.6707317073170732, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.8310568711728401, + "ndcg_full@10": 0.8310568711728401, + "ndcg@20": 0.7976723472351668, + "ndcg_full@20": 0.7167229988003833, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0, + "recall@50": 0.05063291139240506, + "recall@100": 0.16455696202531644, + "recall@200": 0.379746835443038, + "recall@300": 0.45569620253164556, + "recall@500": 0.5189873417721519, + "recall@1000": 0.5443037974683544, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.6548086577531306, + "ndcg_full@5": 0.6548086577531306, + "ndcg@10": 0.6084985989941823, + "ndcg_full@10": 0.6084985989941823, + "ndcg@20": 0.7303910418580247, + "ndcg_full@20": 0.5003094024692321, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03896103896103896, + "recall@50": 0.18614718614718614, + "recall@100": 0.30303030303030304, + "recall@200": 0.5627705627705628, + "recall@300": 0.7705627705627706, + "recall@500": 0.8961038961038961, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.9232067353977313, + "ndcg_full@10": 0.9232067353977313, + "ndcg@20": 0.8926409233939663, + "ndcg_full@20": 0.8926409233939663, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02717391304347826, + "recall@50": 0.11956521739130435, + "recall@100": 0.19021739130434784, + "recall@200": 0.3695652173913043, + "recall@300": 0.47282608695652173, + "recall@500": 0.6304347826086957, + "recall@1000": 0.8097826086956522, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.887701366452186, + "ndcg_full@20": 0.887701366452186, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 72, + "recall@10": 0.0989010989010989, + "recall@50": 0.24175824175824176, + "recall@100": 0.25274725274725274, + "recall@200": 0.26373626373626374, + "recall@300": 0.27472527472527475, + "recall@500": 0.2967032967032967, + "recall@1000": 0.34065934065934067, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7129795603441191, + "ndcg_full@5": 0.7129795603441191, + "ndcg@10": 0.7501225857427184, + "ndcg_full@10": 0.7501225857427184, + "ndcg@20": 0.7263375895523618, + "ndcg_full@20": 0.667852461038063, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 171, + "recall@10": 0.10606060606060606, + "recall@50": 0.5454545454545454, + "recall@100": 0.7878787878787878, + "recall@200": 0.9393939393939394, + "recall@300": 0.9545454545454546, + "recall@500": 0.9848484848484849, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9675312470866648, + "ndcg_full@10": 0.9675312470866648, + "ndcg@20": 0.8536964598274087, + "ndcg_full@20": 0.8115918289690586, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 58, + "recall@10": 0.6923076923076923, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7012236839470684, + "ndcg_full@5": 0.7012236839470684, + "ndcg@10": 0.7628229440008117, + "ndcg_full@10": 0.7628229440008117, + "ndcg@20": 0.8313557800224182, + "ndcg_full@20": 0.7871171471237698, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.10263929618768329, + "recall@100": 0.20821114369501467, + "recall@200": 0.4046920821114369, + "recall@300": 0.5894428152492669, + "recall@500": 0.8357771260997068, + "recall@1000": 0.9354838709677419, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9659370277440176, + "ndcg_full@20": 0.9659370277440176, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.0818034764464546, + "recall@50": 0.2778819305429651, + "recall@100": 0.4269902560041185, + "recall@200": 0.5945583921870587, + "recall@300": 0.6778828178518908, + "recall@500": 0.7669052888565161, + "recall@1000": 0.8361387639997722, + "ndcg@5": 0.6248463085621521, + "ndcg@10": 0.6137183452317222, + "ndcg@20": 0.5979962585782981, + "ndcg_full@5": 0.6243782526474962, + "ndcg_full@10": 0.6123461737634085, + "ndcg_full@20": 0.5872458125338067, + "P@10(rel>=1)": 0.6373333333333332, + "P@10(eligible)": 0.5653333333333332, + "graded_P@10": 0.6013333333333332, + "unjudged_fraction@10": 0.2786666666666667 + }, + "median": { + "recall@10": 0.04938271604938271, + "recall@50": 0.22875816993464052, + "recall@100": 0.3588709677419355, + "recall@200": 0.549800796812749, + "recall@300": 0.6739130434782609, + "recall@500": 0.7954545454545454, + "recall@1000": 0.8636363636363636, + "ndcg@5": 0.6548086577531307, + "ndcg@10": 0.6539659222111618, + "ndcg@20": 0.6058302386787767, + "ndcg_full@5": 0.6548086577531307, + "ndcg_full@10": 0.6539659222111618, + "ndcg_full@20": 0.5893476913276927, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6, + "unjudged_fraction@10": 0.3 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 247, + "recall@10": 0.05917159763313609, + "recall@50": 0.25443786982248523, + "recall@100": 0.4911242603550296, + "recall@200": 0.757396449704142, + "recall@300": 0.8284023668639053, + "recall@500": 0.863905325443787, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6548086577531306, + "ndcg_full@5": 0.6548086577531306, + "ndcg@10": 0.7026305163854645, + "ndcg_full@10": 0.7026305163854645, + "ndcg@20": 0.68365334476219, + "ndcg_full@20": 0.68365334476219, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03333333333333333, + "recall@50": 0.15925925925925927, + "recall@100": 0.24814814814814815, + "recall@200": 0.40370370370370373, + "recall@300": 0.5259259259259259, + "recall@500": 0.6703703703703704, + "recall@1000": 0.837037037037037, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7877736340270854, + "ndcg_full@10": 0.7877736340270854, + "ndcg@20": 0.7241487318910759, + "ndcg_full@20": 0.7241487318910759, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 133, + "recall@10": 0.11904761904761904, + "recall@50": 0.36904761904761907, + "recall@100": 0.5238095238095238, + "recall@200": 0.7023809523809523, + "recall@300": 0.8095238095238095, + "recall@500": 0.8928571428571429, + "recall@1000": 0.9761904761904762, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.764817358607021, + "ndcg_full@5": 0.764817358607021, + "ndcg@10": 0.8473825580301494, + "ndcg_full@10": 0.8473825580301494, + "ndcg@20": 0.7610012419404353, + "ndcg_full@20": 0.7610012419404353, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021052631578947368, + "recall@50": 0.14736842105263157, + "recall@100": 0.29473684210526313, + "recall@200": 0.47368421052631576, + "recall@300": 0.6105263157894737, + "recall@500": 0.7894736842105263, + "recall@1000": 0.9368421052631579, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.5567484706058157, + "ndcg_full@10": 0.5567484706058157, + "ndcg@20": 0.5044855641502418, + "ndcg_full@20": 0.5044855641502418, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04975124378109453, + "recall@50": 0.1791044776119403, + "recall@100": 0.2885572139303483, + "recall@200": 0.46766169154228854, + "recall@300": 0.5621890547263682, + "recall@500": 0.681592039800995, + "recall@1000": 0.7711442786069652, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6015720654566381, + "ndcg_full@10": 0.6015720654566381, + "ndcg@20": 0.5795633217581742, + "ndcg_full@20": 0.5795633217581742, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.047619047619047616, + "recall@50": 0.17142857142857143, + "recall@100": 0.2904761904761905, + "recall@200": 0.46190476190476193, + "recall@300": 0.5761904761904761, + "recall@500": 0.7380952380952381, + "recall@1000": 0.8666666666666667, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.5688421557538149, + "ndcg_full@10": 0.5688421557538149, + "ndcg@20": 0.5945914674475643, + "ndcg_full@20": 0.5945914674475643, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031055900621118012, + "recall@50": 0.11180124223602485, + "recall@100": 0.2360248447204969, + "recall@200": 0.42857142857142855, + "recall@300": 0.5527950310559007, + "recall@500": 0.6832298136645962, + "recall@1000": 0.8322981366459627, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.2816372027313761, + "ndcg_full@10": 0.2816372027313761, + "ndcg@20": 0.3643091120477429, + "ndcg_full@20": 0.3643091120477429, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032679738562091505, + "recall@50": 0.22875816993464052, + "recall@100": 0.43137254901960786, + "recall@200": 0.6993464052287581, + "recall@300": 0.8235294117647058, + "recall@500": 0.9084967320261438, + "recall@1000": 0.9477124183006536, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7702043899253147, + "ndcg_full@10": 0.7702043899253147, + "ndcg@20": 0.6779640135755526, + "ndcg_full@20": 0.6779640135755526, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04672897196261682, + "recall@50": 0.205607476635514, + "recall@100": 0.40186915887850466, + "recall@200": 0.7289719626168224, + "recall@300": 0.9112149532710281, + "recall@500": 0.9345794392523364, + "recall@1000": 0.9439252336448598, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4299813935652734, + "ndcg_full@5": 0.4299813935652734, + "ndcg@10": 0.37999833751247325, + "ndcg_full@10": 0.37999833751247325, + "ndcg@20": 0.42592404913209103, + "ndcg_full@20": 0.39802894413551776, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 89, + "recall@10": 0.1702127659574468, + "recall@50": 0.7872340425531915, + "recall@100": 0.8085106382978723, + "recall@200": 0.8085106382978723, + "recall@300": 0.8085106382978723, + "recall@500": 0.8297872340425532, + "recall@1000": 0.851063829787234, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7324847529928067, + "ndcg_full@10": 0.7324847529928067, + "ndcg@20": 0.8355684611302212, + "ndcg_full@20": 0.8159207806338353, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 231, + "recall@10": 0.07936507936507936, + "recall@50": 0.35714285714285715, + "recall@100": 0.5634920634920635, + "recall@200": 0.8095238095238095, + "recall@300": 0.8333333333333334, + "recall@500": 0.8412698412698413, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.7509288930482942, + "ndcg_full@10": 0.7509288930482942, + "ndcg@20": 0.642939943100263, + "ndcg_full@20": 0.642939943100263, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 174, + "recall@10": 0.06040268456375839, + "recall@50": 0.2483221476510067, + "recall@100": 0.46308724832214765, + "recall@200": 0.7651006711409396, + "recall@300": 0.825503355704698, + "recall@500": 0.8590604026845637, + "recall@1000": 0.912751677852349, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8945734077057459, + "ndcg_full@20": 0.8945734077057459, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 184, + "recall@10": 0.028985507246376812, + "recall@50": 0.18840579710144928, + "recall@100": 0.2971014492753623, + "recall@200": 0.4927536231884058, + "recall@300": 0.6231884057971014, + "recall@500": 0.6811594202898551, + "recall@1000": 0.7463768115942029, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.800693766409882, + "ndcg_full@10": 0.800693766409882, + "ndcg@20": 0.7007486476808517, + "ndcg_full@20": 0.7007486476808517, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.030612244897959183, + "recall@50": 0.11904761904761904, + "recall@100": 0.2108843537414966, + "recall@200": 0.3707482993197279, + "recall@300": 0.45918367346938777, + "recall@500": 0.5748299319727891, + "recall@1000": 0.717687074829932, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6187490087643146, + "ndcg_full@5": 0.6187490087643146, + "ndcg@10": 0.5737769136175119, + "ndcg_full@10": 0.5737769136175119, + "ndcg@20": 0.46059566511896194, + "ndcg_full@20": 0.46059566511896194, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 198, + "recall@10": 0.019230769230769232, + "recall@50": 0.11923076923076924, + "recall@100": 0.23461538461538461, + "recall@200": 0.39615384615384613, + "recall@300": 0.4653846153846154, + "recall@500": 0.5769230769230769, + "recall@1000": 0.7, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9090874918426615, + "ndcg_full@10": 0.9090874918426615, + "ndcg@20": 0.757598843208195, + "ndcg_full@20": 0.757598843208195, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.014492753623188406, + "recall@50": 0.10869565217391304, + "recall@100": 0.2028985507246377, + "recall@200": 0.34057971014492755, + "recall@300": 0.37681159420289856, + "recall@500": 0.4927536231884058, + "recall@1000": 0.6884057971014492, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.039199134489337664, + "ndcg_full@10": 0.039199134489337664, + "ndcg@20": 0.06285709521058114, + "ndcg_full@20": 0.06080785139572285, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02390438247011952, + "recall@50": 0.13545816733067728, + "recall@100": 0.28286852589641437, + "recall@200": 0.549800796812749, + "recall@300": 0.7768924302788844, + "recall@500": 0.9362549800796812, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4121943801949693, + "ndcg_full@5": 0.4121943801949693, + "ndcg@10": 0.3337401719372446, + "ndcg_full@10": 0.3337401719372446, + "ndcg@20": 0.3776540927388128, + "ndcg_full@20": 0.3776540927388128, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03398058252427184, + "recall@50": 0.18932038834951456, + "recall@100": 0.32038834951456313, + "recall@200": 0.5048543689320388, + "recall@300": 0.616504854368932, + "recall@500": 0.7572815533980582, + "recall@1000": 0.8883495145631068, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.06560253875617089, + "ndcg_full@5": 0.06560253875617089, + "ndcg@10": 0.3096085502059411, + "ndcg_full@10": 0.28950334317295584, + "ndcg@20": 0.40034478857824246, + "ndcg_full@20": 0.3478831115782924, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.2, + "graded_P@10": 0.4 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032679738562091505, + "recall@50": 0.16339869281045752, + "recall@100": 0.38562091503267976, + "recall@200": 0.6666666666666666, + "recall@300": 0.7843137254901961, + "recall@500": 0.869281045751634, + "recall@1000": 0.9281045751633987, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.2765732350040718, + "ndcg_full@5": 0.2765732350040718, + "ndcg@10": 0.4521472607752954, + "ndcg_full@10": 0.4521472607752954, + "ndcg@20": 0.4038487553203449, + "ndcg_full@20": 0.4038487553203449, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.11428571428571428, + "recall@50": 0.37142857142857144, + "recall@100": 0.7285714285714285, + "recall@200": 0.8857142857142857, + "recall@300": 0.9142857142857143, + "recall@500": 0.9428571428571428, + "recall@1000": 0.9571428571428572, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.21719873062191455, + "ndcg_full@5": 0.21719873062191455, + "ndcg@10": 0.21934538340123172, + "ndcg_full@10": 0.21934538340123172, + "ndcg@20": 0.3562254997153661, + "ndcg_full@20": 0.3562254997153661, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 210, + "recall@10": 0.11320754716981132, + "recall@50": 0.41509433962264153, + "recall@100": 0.7169811320754716, + "recall@200": 0.8679245283018868, + "recall@300": 0.9056603773584906, + "recall@500": 0.9245283018867925, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.530943388959284, + "ndcg_full@5": 0.530943388959284, + "ndcg@10": 0.40948343563604783, + "ndcg_full@10": 0.40948343563604783, + "ndcg@20": 0.46488443973260785, + "ndcg_full@20": 0.46488443973260785, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 244, + "recall@10": 0.06569343065693431, + "recall@50": 0.291970802919708, + "recall@100": 0.48175182481751827, + "recall@200": 0.7445255474452555, + "recall@300": 0.7737226277372263, + "recall@500": 0.8321167883211679, + "recall@1000": 0.8613138686131386, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.49839197688811054, + "ndcg_full@5": 0.49839197688811054, + "ndcg@10": 0.5639978093055565, + "ndcg_full@10": 0.5639978093055565, + "ndcg@20": 0.4756217027657174, + "ndcg_full@20": 0.4756217027657174, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0299625468164794, + "recall@50": 0.13857677902621723, + "recall@100": 0.2397003745318352, + "recall@200": 0.4232209737827715, + "recall@300": 0.550561797752809, + "recall@500": 0.7790262172284644, + "recall@1000": 0.8726591760299626, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.15782422623975725, + "ndcg_full@5": 0.15782422623975725, + "ndcg@10": 0.3165008381291371, + "ndcg_full@10": 0.3165008381291371, + "ndcg@20": 0.30894640680642405, + "ndcg_full@20": 0.30894640680642405, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04430379746835443, + "recall@50": 0.24050632911392406, + "recall@100": 0.41139240506329117, + "recall@200": 0.689873417721519, + "recall@300": 0.8354430379746836, + "recall@500": 0.9177215189873418, + "recall@1000": 0.9746835443037974, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4374130297304556, + "ndcg_full@5": 0.4374130297304556, + "ndcg@10": 0.45938533878773713, + "ndcg_full@10": 0.45938533878773713, + "ndcg@20": 0.40743676110654664, + "ndcg_full@20": 0.40743676110654664, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.13725490196078433, + "recall@100": 0.20588235294117646, + "recall@200": 0.3431372549019608, + "recall@300": 0.4117647058823529, + "recall@500": 0.6078431372549019, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6770117388019904, + "ndcg_full@5": 0.6770117388019904, + "ndcg@10": 0.5130654318505325, + "ndcg_full@10": 0.5130654318505325, + "ndcg@20": 0.5609141066234424, + "ndcg_full@20": 0.5609141066234424, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.05625, + "recall@50": 0.175, + "recall@100": 0.30625, + "recall@200": 0.50625, + "recall@300": 0.6125, + "recall@500": 0.725, + "recall@1000": 0.83125, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7116179486056782, + "ndcg_full@10": 0.7116179486056782, + "ndcg@20": 0.5639791983380493, + "ndcg_full@20": 0.5639791983380493, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.13733905579399142, + "recall@100": 0.26180257510729615, + "recall@200": 0.4592274678111588, + "recall@300": 0.630901287553648, + "recall@500": 0.8454935622317596, + "recall@1000": 0.9527896995708155, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3766321352557, + "ndcg_full@5": 0.3766321352557, + "ndcg@10": 0.4648194694487437, + "ndcg_full@10": 0.4648194694487437, + "ndcg@20": 0.4792494773726009, + "ndcg_full@20": 0.4792494773726009, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.2, + "graded_P@10": 0.55 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018518518518518517, + "recall@50": 0.09567901234567901, + "recall@100": 0.19753086419753085, + "recall@200": 0.3950617283950617, + "recall@300": 0.5771604938271605, + "recall@500": 0.8024691358024691, + "recall@1000": 0.9074074074074074, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.21167088859887737, + "ndcg_full@5": 0.21167088859887737, + "ndcg@10": 0.31935420171583623, + "ndcg_full@10": 0.31935420171583623, + "ndcg@20": 0.42020423275146035, + "ndcg_full@20": 0.42020423275146035, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027131782945736434, + "recall@50": 0.1511627906976744, + "recall@100": 0.24031007751937986, + "recall@200": 0.43410852713178294, + "recall@300": 0.6162790697674418, + "recall@500": 0.7751937984496124, + "recall@1000": 0.8062015503875969, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5163251428503343, + "ndcg_full@5": 0.5163251428503343, + "ndcg@10": 0.5787614395897542, + "ndcg_full@10": 0.5787614395897542, + "ndcg@20": 0.48174051161493053, + "ndcg_full@20": 0.48174051161493053, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.06666666666666667, + "recall@50": 0.28, + "recall@100": 0.5133333333333333, + "recall@200": 0.68, + "recall@300": 0.7466666666666667, + "recall@500": 0.82, + "recall@1000": 0.8466666666666667, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8603818544462509, + "ndcg_full@10": 0.8603818544462509, + "ndcg@20": 0.8273739993546265, + "ndcg_full@20": 0.8273739993546265, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 234, + "recall@10": 0.023255813953488372, + "recall@50": 0.11627906976744186, + "recall@100": 0.23255813953488372, + "recall@200": 0.45348837209302323, + "recall@300": 0.5988372093023255, + "recall@500": 0.7093023255813954, + "recall@1000": 0.7790697674418605, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5030155684866267, + "ndcg_full@5": 0.5030155684866267, + "ndcg@10": 0.4276643013107241, + "ndcg_full@10": 0.4276643013107241, + "ndcg@20": 0.3812620109672555, + "ndcg_full@20": 0.3812620109672555, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 94, + "recall@10": 0.11320754716981132, + "recall@50": 0.2641509433962264, + "recall@100": 0.37735849056603776, + "recall@200": 0.4339622641509434, + "recall@300": 0.4716981132075472, + "recall@500": 0.4716981132075472, + "recall@1000": 0.5660377358490566, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.6906478832608419, + "ndcg_full@10": 0.6906478832608419, + "ndcg@20": 0.7512863383580382, + "ndcg_full@20": 0.5567123828177641, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03349282296650718, + "recall@50": 0.1291866028708134, + "recall@100": 0.23923444976076555, + "recall@200": 0.3444976076555024, + "recall@300": 0.4019138755980861, + "recall@500": 0.5167464114832536, + "recall@1000": 0.6028708133971292, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.40348266660099286, + "ndcg_full@10": 0.40348266660099286, + "ndcg@20": 0.4425832694056781, + "ndcg_full@20": 0.4425832694056781, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04, + "recall@50": 0.13333333333333333, + "recall@100": 0.24, + "recall@200": 0.42, + "recall@300": 0.5666666666666667, + "recall@500": 0.6866666666666666, + "recall@1000": 0.76, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.7151857987862356, + "ndcg_full@10": 0.7151857987862356, + "ndcg@20": 0.6291275874308192, + "ndcg_full@20": 0.6291275874308192, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 2000, + "num_ranked": 94, + "recall@10": 0.05172413793103448, + "recall@50": 0.25862068965517243, + "recall@100": 0.4540229885057471, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.8681235945057538, + "ndcg_full@20": 0.8681235945057538, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.006472491909385114, + "recall@50": 0.032362459546925564, + "recall@100": 0.09061488673139159, + "recall@200": 0.20064724919093851, + "recall@300": 0.3074433656957929, + "recall@500": 0.4627831715210356, + "recall@1000": 0.6375404530744336, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.24090885754831728, + "ndcg_full@5": 0.24090885754831728, + "ndcg@10": 0.21995415263917664, + "ndcg_full@10": 0.21995415263917664, + "ndcg@20": 0.29037619373721874, + "ndcg_full@20": 0.29037619373721874, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03225806451612903, + "recall@50": 0.1935483870967742, + "recall@100": 0.3588709677419355, + "recall@200": 0.6451612903225806, + "recall@300": 0.7862903225806451, + "recall@500": 0.9274193548387096, + "recall@1000": 0.9516129032258065, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8572048559638628, + "ndcg_full@10": 0.8572048559638628, + "ndcg@20": 0.8916755309462826, + "ndcg_full@20": 0.8916755309462826, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 108, + "recall@10": 0.12345679012345678, + "recall@50": 0.5802469135802469, + "recall@100": 0.9506172839506173, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9753086419753086, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.6552695816321074, + "ndcg_full@10": 0.6552695816321074, + "ndcg@20": 0.7062726352314666, + "ndcg_full@20": 0.7062726352314666, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 137, + "recall@10": 0.13559322033898305, + "recall@50": 0.6949152542372882, + "recall@100": 0.864406779661017, + "recall@200": 0.9152542372881356, + "recall@300": 0.9491525423728814, + "recall@500": 0.9491525423728814, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8273957295043396, + "ndcg_full@10": 0.8273957295043396, + "ndcg@20": 0.8115389026161676, + "ndcg_full@20": 0.8115389026161676, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 57, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.78180487222265, + "ndcg_full@10": 0.78180487222265, + "ndcg@20": 0.8191110232191385, + "ndcg_full@20": 0.7755239661129132, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02922077922077922, + "recall@50": 0.12337662337662338, + "recall@100": 0.22727272727272727, + "recall@200": 0.4318181818181818, + "recall@300": 0.5876623376623377, + "recall@500": 0.8344155844155844, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.46453859504280076, + "ndcg_full@10": 0.46453859504280076, + "ndcg@20": 0.4801706105980668, + "ndcg_full@20": 0.4801706105980668, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 118, + "recall@10": 0.08450704225352113, + "recall@50": 0.36619718309859156, + "recall@100": 0.5070422535211268, + "recall@200": 0.6197183098591549, + "recall@300": 0.704225352112676, + "recall@500": 0.7605633802816901, + "recall@1000": 0.8309859154929577, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.7333857000528786, + "ndcg_full@20": 0.7333857000528786, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 136, + "recall@10": 0.05806451612903226, + "recall@50": 0.24516129032258063, + "recall@100": 0.44516129032258067, + "recall@200": 0.6903225806451613, + "recall@300": 0.7741935483870968, + "recall@500": 0.864516129032258, + "recall@1000": 0.896774193548387, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.6539659222111618, + "ndcg_full@10": 0.6539659222111618, + "ndcg@20": 0.6690214615885423, + "ndcg_full@20": 0.6690214615885423, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.05128205128205128, + "recall@50": 0.2512820512820513, + "recall@100": 0.441025641025641, + "recall@200": 0.6820512820512821, + "recall@300": 0.7076923076923077, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7384615384615385, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5351041935991797, + "ndcg_full@5": 0.5, + "ndcg@10": 0.7323959400285283, + "ndcg_full@10": 0.6630536265254862, + "ndcg@20": 0.8019791722639285, + "ndcg_full@20": 0.7443743002001465, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.2, + "graded_P@10": 0.6 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.053763440860215055, + "recall@50": 0.24193548387096775, + "recall@100": 0.478494623655914, + "recall@200": 0.8279569892473119, + "recall@300": 0.8817204301075269, + "recall@500": 0.8870967741935484, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9347967382841031, + "ndcg_full@20": 0.9347967382841031, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.08108108108108109, + "recall@50": 0.34234234234234234, + "recall@100": 0.5765765765765766, + "recall@200": 0.9279279279279279, + "recall@300": 0.9819819819819819, + "recall@500": 0.990990990990991, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9307787414438842, + "ndcg_full@20": 0.9307787414438842, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03007518796992481, + "recall@50": 0.14285714285714285, + "recall@100": 0.2593984962406015, + "recall@200": 0.4473684210526316, + "recall@300": 0.6240601503759399, + "recall@500": 0.7969924812030075, + "recall@1000": 0.9360902255639098, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7555853890875123, + "ndcg_full@10": 0.7555853890875123, + "ndcg@20": 0.6739470295794178, + "ndcg_full@20": 0.6739470295794178, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.15151515151515152, + "recall@50": 0.5909090909090909, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.803030303030303, + "recall@1000": 0.8636363636363636, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8611375561264454, + "ndcg_full@10": 0.8611375561264454, + "ndcg@20": 0.9103827127966565, + "ndcg_full@20": 0.9103827127966565, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.11688311688311688, + "recall@50": 0.4805194805194805, + "recall@100": 0.7402597402597403, + "recall@200": 0.7922077922077922, + "recall@300": 0.8181818181818182, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.7218018303820491, + "ndcg_full@10": 0.7218018303820491, + "ndcg@20": 0.7692674885883332, + "ndcg_full@20": 0.7692674885883332, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 91, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.782608695652174, + "recall@300": 0.782608695652174, + "recall@500": 0.8260869565217391, + "recall@1000": 0.9565217391304348, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.42329999261824636, + "ndcg_full@10": 0.4098346530307547, + "ndcg@20": 0.35630491216846444, + "ndcg_full@20": 0.30516403303539513, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 115, + "recall@10": 0.1267605633802817, + "recall@50": 0.4084507042253521, + "recall@100": 0.6901408450704225, + "recall@200": 0.8732394366197183, + "recall@300": 0.9014084507042254, + "recall@500": 0.9154929577464789, + "recall@1000": 0.9577464788732394, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5773584151532217, + "ndcg_full@5": 0.5773584151532217, + "ndcg@10": 0.6594803070902429, + "ndcg_full@10": 0.6594803070902429, + "ndcg@20": 0.6905624291791583, + "ndcg_full@20": 0.6565036301405489, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.08823529411764706, + "recall@50": 0.4264705882352941, + "recall@100": 0.6176470588235294, + "recall@200": 0.7352941176470589, + "recall@300": 0.7794117647058824, + "recall@500": 0.7794117647058824, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.26187778578763665, + "ndcg_full@10": 0.26187778578763665, + "ndcg@20": 0.3876925203073306, + "ndcg_full@20": 0.3876925203073306, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.13043478260869565, + "recall@50": 0.3695652173913043, + "recall@100": 0.5652173913043478, + "recall@200": 0.6739130434782609, + "recall@300": 0.6739130434782609, + "recall@500": 0.7608695652173914, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9096182159526598, + "ndcg_full@20": 0.9096182159526598, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.12, + "recall@50": 0.49333333333333335, + "recall@100": 0.8933333333333333, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.5337225282363589, + "ndcg_full@10": 0.5337225282363589, + "ndcg@20": 0.46274145073173484, + "ndcg_full@20": 0.46274145073173484, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 109, + "recall@10": 0.1794871794871795, + "recall@50": 0.6410256410256411, + "recall@100": 0.6923076923076923, + "recall@200": 0.7435897435897436, + "recall@300": 0.7435897435897436, + "recall@500": 0.7692307692307693, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.6126163760997826, + "ndcg_full@10": 0.6126163760997826, + "ndcg@20": 0.6058302386787767, + "ndcg_full@20": 0.6058302386787767, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 203, + "recall@10": 0.10465116279069768, + "recall@50": 0.37209302325581395, + "recall@100": 0.627906976744186, + "recall@200": 0.9883720930232558, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8712459207538756, + "ndcg_full@10": 0.8712459207538756, + "ndcg@20": 0.8473333391091306, + "ndcg_full@20": 0.8473333391091306, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04938271604938271, + "recall@50": 0.17901234567901234, + "recall@100": 0.3271604938271605, + "recall@200": 0.5185185185185185, + "recall@300": 0.6419753086419753, + "recall@500": 0.7777777777777778, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8304198973631918, + "ndcg_full@5": 0.8304198973631918, + "ndcg@10": 0.8205228949141826, + "ndcg_full@10": 0.8205228949141826, + "ndcg@20": 0.6344605786743323, + "ndcg_full@20": 0.6344605786743323, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 126, + "recall@10": 0.07352941176470588, + "recall@50": 0.3235294117647059, + "recall@100": 0.5588235294117647, + "recall@200": 0.6691176470588235, + "recall@300": 0.6764705882352942, + "recall@500": 0.6911764705882353, + "recall@1000": 0.6911764705882353, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6742019040671282, + "ndcg_full@5": 0.6742019040671282, + "ndcg@10": 0.7461654695668642, + "ndcg_full@10": 0.7461654695668642, + "ndcg@20": 0.7662131041917555, + "ndcg_full@20": 0.7662131041917555, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02564102564102564, + "recall@50": 0.1225071225071225, + "recall@100": 0.23076923076923078, + "recall@200": 0.46438746438746437, + "recall@300": 0.6153846153846154, + "recall@500": 0.6296296296296297, + "recall@1000": 0.6353276353276354, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9350624941733298, + "ndcg_full@10": 0.9350624941733298, + "ndcg@20": 0.828622140590374, + "ndcg_full@20": 0.828622140590374, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03015075376884422, + "recall@50": 0.11557788944723618, + "recall@100": 0.24623115577889448, + "recall@200": 0.45226130653266333, + "recall@300": 0.5829145728643216, + "recall@500": 0.7236180904522613, + "recall@1000": 0.8793969849246231, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.42857142857142855, + "ndcg_full@5": 0.42857142857142855, + "ndcg@10": 0.40940869477291, + "ndcg_full@10": 0.40940869477291, + "ndcg@20": 0.4838828092294392, + "ndcg_full@20": 0.4838828092294392, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.07368421052631578, + "recall@100": 0.13157894736842105, + "recall@200": 0.30526315789473685, + "recall@300": 0.4263157894736842, + "recall@500": 0.7052631578947368, + "recall@1000": 0.9526315789473684, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.6920178444619898, + "ndcg_full@10": 0.6920178444619898, + "ndcg@20": 0.5502149336861343, + "ndcg_full@20": 0.5502149336861343, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03272727272727273, + "recall@50": 0.16, + "recall@100": 0.2690909090909091, + "recall@200": 0.4254545454545455, + "recall@300": 0.5309090909090909, + "recall@500": 0.6690909090909091, + "recall@1000": 0.7672727272727272, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.23673633935963592, + "ndcg_full@5": 0.23673633935963592, + "ndcg@10": 0.2890088619194199, + "ndcg_full@10": 0.2890088619194199, + "ndcg@20": 0.22975905374372782, + "ndcg_full@20": 0.22975905374372782, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 234, + "recall@10": 0.045454545454545456, + "recall@50": 0.3181818181818182, + "recall@100": 0.5, + "recall@200": 0.5909090909090909, + "recall@300": 0.6818181818181818, + "recall@500": 0.7954545454545454, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.2979023533279042, + "ndcg_full@5": 0.2979023533279042, + "ndcg@10": 0.323193255093482, + "ndcg_full@10": 0.323193255093482, + "ndcg@20": 0.3229403672895556, + "ndcg_full@20": 0.3229403672895556, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029556650246305417, + "recall@50": 0.09359605911330049, + "recall@100": 0.2019704433497537, + "recall@200": 0.3793103448275862, + "recall@300": 0.5270935960591133, + "recall@500": 0.8472906403940886, + "recall@1000": 0.9605911330049262, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.530263109944618, + "ndcg_full@10": 0.530263109944618, + "ndcg@20": 0.41332095793848844, + "ndcg_full@20": 0.41332095793848844, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.008849557522123894, + "recall@50": 0.11504424778761062, + "recall@100": 0.20353982300884957, + "recall@200": 0.34513274336283184, + "recall@300": 0.46017699115044247, + "recall@500": 0.6460176991150443, + "recall@1000": 0.8141592920353983, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5222030810488598, + "ndcg_full@5": 0.5222030810488598, + "ndcg@10": 0.5522906972238762, + "ndcg_full@10": 0.5522906972238762, + "ndcg@20": 0.4292124150220645, + "ndcg_full@20": 0.4292124150220645, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 53, + "recall@10": 0.14583333333333334, + "recall@50": 0.25, + "recall@100": 0.2916666666666667, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.7026305163854645, + "ndcg_full@10": 0.7026305163854645, + "ndcg@20": 0.665408510925426, + "ndcg_full@20": 0.5852572704027965, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.06369426751592357, + "recall@50": 0.2611464968152866, + "recall@100": 0.445859872611465, + "recall@200": 0.7261146496815286, + "recall@300": 0.7961783439490446, + "recall@500": 0.8089171974522293, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4394230586203177, + "ndcg_full@5": 0.4394230586203177, + "ndcg@10": 0.5358150428799577, + "ndcg_full@10": 0.5358150428799577, + "ndcg@20": 0.608020050804262, + "ndcg_full@20": 0.608020050804262, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.06097560975609756, + "recall@50": 0.2682926829268293, + "recall@100": 0.3353658536585366, + "recall@200": 0.4329268292682927, + "recall@300": 0.5, + "recall@500": 0.5548780487804879, + "recall@1000": 0.6707317073170732, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.8310568711728401, + "ndcg_full@10": 0.8310568711728401, + "ndcg@20": 0.7793839162385808, + "ndcg_full@20": 0.7002905135165948, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0, + "recall@50": 0.05063291139240506, + "recall@100": 0.16455696202531644, + "recall@200": 0.379746835443038, + "recall@300": 0.45569620253164556, + "recall@500": 0.5189873417721519, + "recall@1000": 0.5443037974683544, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.19178318368566782, + "ndcg_full@5": 0.19178318368566782, + "ndcg@10": 0.1244541635112973, + "ndcg_full@10": 0.1244541635112973, + "ndcg@20": 0.1172554917583688, + "ndcg_full@20": 0.08031865351008606, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03896103896103896, + "recall@50": 0.18614718614718614, + "recall@100": 0.30303030303030304, + "recall@200": 0.5627705627705628, + "recall@300": 0.7705627705627706, + "recall@500": 0.8961038961038961, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5030155684866267, + "ndcg_full@5": 0.5030155684866267, + "ndcg@10": 0.43094736901973413, + "ndcg_full@10": 0.43094736901973413, + "ndcg@20": 0.4771657975864908, + "ndcg_full@20": 0.4771657975864908, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02717391304347826, + "recall@50": 0.11956521739130435, + "recall@100": 0.19021739130434784, + "recall@200": 0.3695652173913043, + "recall@300": 0.47282608695652173, + "recall@500": 0.6304347826086957, + "recall@1000": 0.8097826086956522, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.4696395331850896, + "ndcg_full@10": 0.4696395331850896, + "ndcg@20": 0.5893476913276927, + "ndcg_full@20": 0.5893476913276927, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 72, + "recall@10": 0.0989010989010989, + "recall@50": 0.24175824175824176, + "recall@100": 0.25274725274725274, + "recall@200": 0.26373626373626374, + "recall@300": 0.27472527472527475, + "recall@500": 0.2967032967032967, + "recall@1000": 0.34065934065934067, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.49987162063252927, + "ndcg_full@10": 0.49987162063252927, + "ndcg@20": 0.5821461672802244, + "ndcg_full@20": 0.5352714166171431, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 171, + "recall@10": 0.10606060606060606, + "recall@50": 0.5454545454545454, + "recall@100": 0.7878787878787878, + "recall@200": 0.9393939393939394, + "recall@300": 0.9545454545454546, + "recall@500": 0.9848484848484849, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8630152897016883, + "ndcg_full@10": 0.8630152897016883, + "ndcg@20": 0.8261831471993974, + "ndcg_full@20": 0.7854354832800071, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 58, + "recall@10": 0.6923076923076923, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7012236839470684, + "ndcg_full@5": 0.7012236839470684, + "ndcg@10": 0.6762396028985845, + "ndcg_full@10": 0.6762396028985845, + "ndcg@20": 0.7415585975489226, + "ndcg_full@20": 0.7020983094771676, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.10263929618768329, + "recall@100": 0.20821114369501467, + "recall@200": 0.4046920821114369, + "recall@300": 0.5894428152492669, + "recall@500": 0.8357771260997068, + "recall@1000": 0.9354838709677419, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9321572773044168, + "ndcg_full@10": 0.9321572773044168, + "ndcg@20": 0.8505069319171649, + "ndcg_full@20": 0.8505069319171649, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + } + } + } + } + }, + { + "name": "huatuo_medcpt", + "results_dir": "huatuo_medcpt/results_trec21", + "evaluation_input_sha256": "8297d951cbce51d0fa93d992ba866e0c6733dc13e5519ac4d59250175d30947b", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.07987159484786008, + "recall@50": 0.2697296003572918, + "recall@100": 0.407365139825072, + "recall@200": 0.5657282405172425, + "recall@300": 0.6525827512818093, + "recall@500": 0.7501550294140854, + "recall@1000": 0.8178965483941935, + "ndcg@5": 0.6797536659737168, + "ndcg@10": 0.6817217836184921, + "ndcg@20": 0.6952041733412063, + "ndcg_full@5": 0.6773235446376489, + "ndcg_full@10": 0.6765537795309075, + "ndcg_full@20": 0.6823792626861027, + "P@10(rel>=1)": 0.7679999999999999, + "P@10(eligible)": 0.584, + "graded_P@10": 0.6760000000000002, + "unjudged_fraction@10": 0.3906666666666666 + }, + "median": { + "recall@10": 0.05063291139240506, + "recall@50": 0.20253164556962025, + "recall@100": 0.33544303797468356, + "recall@200": 0.5238095238095238, + "recall@300": 0.654320987654321, + "recall@500": 0.7883211678832117, + "recall@1000": 0.8592592592592593, + "ndcg@5": 0.7227265726449519, + "ndcg@10": 0.674385164197632, + "ndcg@20": 0.6995192997906257, + "ndcg_full@5": 0.7227265726449519, + "ndcg_full@10": 0.670086666467663, + "ndcg_full@20": 0.6978591264295058, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65, + "unjudged_fraction@10": 0.4 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05917159763313609, + "recall@50": 0.26627218934911245, + "recall@100": 0.47928994082840237, + "recall@200": 0.7869822485207101, + "recall@300": 0.834319526627219, + "recall@500": 0.8520710059171598, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7532124658873871, + "ndcg_full@5": 0.7532124658873871, + "ndcg@10": 0.6803622823041467, + "ndcg_full@10": 0.6803622823041467, + "ndcg@20": 0.6327750733637245, + "ndcg_full@20": 0.6327750733637245, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.022222222222222223, + "recall@50": 0.1259259259259259, + "recall@100": 0.24074074074074073, + "recall@200": 0.4185185185185185, + "recall@300": 0.5518518518518518, + "recall@500": 0.7037037037037037, + "recall@1000": 0.8592592592592593, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8847630951196416, + "ndcg_full@10": 0.8847630951196416, + "ndcg@20": 0.9089111185602757, + "ndcg_full@20": 0.9089111185602757, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 136, + "recall@10": 0.09523809523809523, + "recall@50": 0.3333333333333333, + "recall@100": 0.47619047619047616, + "recall@200": 0.6190476190476191, + "recall@300": 0.7619047619047619, + "recall@500": 0.8690476190476191, + "recall@1000": 0.9523809523809523, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.4569970750177365, + "ndcg_full@10": 0.4569970750177365, + "ndcg@20": 0.4943353578751961, + "ndcg_full@20": 0.4943353578751961, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.12631578947368421, + "recall@100": 0.25263157894736843, + "recall@200": 0.3263157894736842, + "recall@300": 0.4842105263157895, + "recall@500": 0.6842105263157895, + "recall@1000": 0.8421052631578947, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.41770005335592036, + "ndcg_full@10": 0.41770005335592036, + "ndcg@20": 0.32425181260176567, + "ndcg_full@20": 0.32425181260176567, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.004975124378109453, + "recall@50": 0.03482587064676617, + "recall@100": 0.08955223880597014, + "recall@200": 0.24875621890547264, + "recall@300": 0.3383084577114428, + "recall@500": 0.4925373134328358, + "recall@1000": 0.6666666666666666, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8189445882621463, + "ndcg_full@10": 0.8189445882621463, + "ndcg@20": 0.7235111676747824, + "ndcg_full@20": 0.7235111676747824, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.047619047619047616, + "recall@50": 0.14285714285714285, + "recall@100": 0.2714285714285714, + "recall@200": 0.4238095238095238, + "recall@300": 0.5666666666666667, + "recall@500": 0.7571428571428571, + "recall@1000": 0.861904761904762, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8690959628864622, + "ndcg_full@10": 0.8690959628864622, + "ndcg@20": 0.8973406646710649, + "ndcg_full@20": 0.8973406646710649, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 196, + "recall@10": 0.024844720496894408, + "recall@50": 0.14285714285714285, + "recall@100": 0.2484472049689441, + "recall@200": 0.37888198757763975, + "recall@300": 0.4472049689440994, + "recall@500": 0.639751552795031, + "recall@1000": 0.8757763975155279, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.19178318368566782, + "ndcg_full@5": 0.19178318368566782, + "ndcg@10": 0.23253369657904602, + "ndcg_full@10": 0.23253369657904602, + "ndcg@20": 0.2944467059827191, + "ndcg_full@20": 0.2944467059827191, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05228758169934641, + "recall@50": 0.22875816993464052, + "recall@100": 0.47058823529411764, + "recall@200": 0.8300653594771242, + "recall@300": 0.9150326797385621, + "recall@500": 0.934640522875817, + "recall@1000": 0.954248366013072, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8660125011411115, + "ndcg_full@10": 0.8660125011411115, + "ndcg@20": 0.828140082691736, + "ndcg_full@20": 0.828140082691736, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.1822429906542056, + "recall@100": 0.3598130841121495, + "recall@200": 0.602803738317757, + "recall@300": 0.780373831775701, + "recall@500": 0.9345794392523364, + "recall@1000": 0.9485981308411215, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.2573857224418387, + "ndcg_full@5": 0.2573857224418387, + "ndcg@10": 0.4100691147746947, + "ndcg_full@10": 0.4100691147746947, + "ndcg@20": 0.5058219994885607, + "ndcg_full@20": 0.47269412654017473, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 166, + "recall@10": 0.2127659574468085, + "recall@50": 0.8085106382978723, + "recall@100": 0.8085106382978723, + "recall@200": 0.8085106382978723, + "recall@300": 0.8085106382978723, + "recall@500": 0.8085106382978723, + "recall@1000": 0.8085106382978723, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.28902930375209906, + "ndcg_full@5": 0.28902930375209906, + "ndcg@10": 0.4383259168204381, + "ndcg_full@10": 0.4383259168204381, + "ndcg@20": 0.49122794426517297, + "ndcg_full@20": 0.47967713766009407, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 161, + "recall@10": 0.07936507936507936, + "recall@50": 0.3492063492063492, + "recall@100": 0.5555555555555556, + "recall@200": 0.8095238095238095, + "recall@300": 0.8333333333333334, + "recall@500": 0.8492063492063492, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.849607409925425, + "ndcg_full@5": 0.849607409925425, + "ndcg@10": 0.9024054996046694, + "ndcg_full@10": 0.9024054996046694, + "ndcg@20": 0.7553336894698629, + "ndcg_full@20": 0.7553336894698629, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.06040268456375839, + "recall@50": 0.28187919463087246, + "recall@100": 0.4966442953020134, + "recall@200": 0.7986577181208053, + "recall@300": 0.8322147651006712, + "recall@500": 0.8859060402684564, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.6785038377365556, + "ndcg_full@10": 0.6785038377365556, + "ndcg@20": 0.6853190910997596, + "ndcg_full@20": 0.6853190910997596, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.11594202898550725, + "recall@100": 0.2028985507246377, + "recall@200": 0.391304347826087, + "recall@300": 0.5362318840579711, + "recall@500": 0.644927536231884, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.62713337324863, + "ndcg_full@10": 0.62713337324863, + "ndcg@20": 0.5906061576325216, + "ndcg_full@20": 0.5614772395451568, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.013605442176870748, + "recall@50": 0.08503401360544217, + "recall@100": 0.17006802721088435, + "recall@200": 0.30952380952380953, + "recall@300": 0.3877551020408163, + "recall@500": 0.5544217687074829, + "recall@1000": 0.6768707482993197, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8510488233360478, + "ndcg_full@5": 0.8510488233360478, + "ndcg@10": 0.8384033724532591, + "ndcg_full@10": 0.8384033724532591, + "ndcg@20": 0.7734642986338568, + "ndcg_full@20": 0.7734642986338568, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 190, + "recall@10": 0.026923076923076925, + "recall@50": 0.1423076923076923, + "recall@100": 0.26153846153846155, + "recall@200": 0.3923076923076923, + "recall@300": 0.4807692307692308, + "recall@500": 0.5807692307692308, + "recall@1000": 0.6884615384615385, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9333480446521031, + "ndcg_full@10": 0.9333480446521031, + "ndcg@20": 0.956985004307457, + "ndcg_full@20": 0.956985004307457, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.028985507246376812, + "recall@50": 0.10144927536231885, + "recall@100": 0.14492753623188406, + "recall@200": 0.2391304347826087, + "recall@300": 0.4057971014492754, + "recall@500": 0.5507246376811594, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.404762744029787, + "ndcg_full@5": 0.404762744029787, + "ndcg@10": 0.341061594097566, + "ndcg_full@10": 0.341061594097566, + "ndcg@20": 0.31128342808197756, + "ndcg_full@20": 0.31128342808197756, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035856573705179286, + "recall@50": 0.14342629482071714, + "recall@100": 0.29880478087649404, + "recall@200": 0.601593625498008, + "recall@300": 0.852589641434263, + "recall@500": 0.9641434262948207, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5859477393862291, + "ndcg_full@5": 0.5859477393862291, + "ndcg@10": 0.5925360809153932, + "ndcg_full@10": 0.5925360809153932, + "ndcg@20": 0.5844590177746554, + "ndcg_full@20": 0.5844590177746554, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03398058252427184, + "recall@50": 0.1796116504854369, + "recall@100": 0.30097087378640774, + "recall@200": 0.49029126213592233, + "recall@300": 0.6407766990291263, + "recall@500": 0.7961165048543689, + "recall@1000": 0.8737864077669902, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3072118018694012, + "ndcg_full@5": 0.3072118018694012, + "ndcg@10": 0.4713256956120693, + "ndcg_full@10": 0.4407189805070011, + "ndcg@20": 0.629737161022784, + "ndcg_full@20": 0.5472156233907632, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.3, + "graded_P@10": 0.55 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0457516339869281, + "recall@50": 0.16339869281045752, + "recall@100": 0.2875816993464052, + "recall@200": 0.5228758169934641, + "recall@300": 0.6601307189542484, + "recall@500": 0.8431372549019608, + "recall@1000": 0.9215686274509803, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5433994577073111, + "ndcg_full@5": 0.5433994577073111, + "ndcg@10": 0.5616784336150028, + "ndcg_full@10": 0.5616784336150028, + "ndcg@20": 0.6656835143238892, + "ndcg_full@20": 0.6656835143238892, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.07142857142857142, + "recall@50": 0.35714285714285715, + "recall@100": 0.6428571428571429, + "recall@200": 0.8714285714285714, + "recall@300": 0.9, + "recall@500": 0.9571428571428572, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.2870204396558808, + "ndcg_full@5": 0.2870204396558808, + "ndcg@10": 0.3587596712434614, + "ndcg_full@10": 0.3587596712434614, + "ndcg@20": 0.42689047709351396, + "ndcg_full@20": 0.42689047709351396, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.07547169811320754, + "recall@50": 0.37735849056603776, + "recall@100": 0.7169811320754716, + "recall@200": 0.8867924528301887, + "recall@300": 0.9056603773584906, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.8551490041618764, + "ndcg_full@20": 0.8551490041618764, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 1494, + "num_ranked": 204, + "recall@10": 0.058394160583941604, + "recall@50": 0.31386861313868614, + "recall@100": 0.5474452554744526, + "recall@200": 0.7153284671532847, + "recall@300": 0.7445255474452555, + "recall@500": 0.7883211678832117, + "recall@1000": 0.8248175182481752, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.913116627268135, + "ndcg_full@10": 0.913116627268135, + "ndcg@20": 0.9287614337571612, + "ndcg_full@20": 0.9287614337571612, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 1496, + "num_ranked": 250, + "recall@10": 0.00749063670411985, + "recall@50": 0.07116104868913857, + "recall@100": 0.1348314606741573, + "recall@200": 0.2696629213483146, + "recall@300": 0.41198501872659177, + "recall@500": 0.6404494382022472, + "recall@1000": 0.8689138576779026, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.7611015405244298, + "ndcg_full@5": 0.7611015405244298, + "ndcg@10": 0.670086666467663, + "ndcg_full@10": 0.670086666467663, + "ndcg@20": 0.6783173887495398, + "ndcg_full@20": 0.6783173887495398, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05063291139240506, + "recall@50": 0.20253164556962025, + "recall@100": 0.33544303797468356, + "recall@200": 0.5822784810126582, + "recall@300": 0.7531645569620253, + "recall@500": 0.8987341772151899, + "recall@1000": 0.9683544303797469, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7897399841083766, + "ndcg_full@10": 0.7897399841083766, + "ndcg@20": 0.8136725175836266, + "ndcg_full@20": 0.8136725175836266, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0196078431372549, + "recall@50": 0.11764705882352941, + "recall@100": 0.24509803921568626, + "recall@200": 0.3627450980392157, + "recall@300": 0.4411764705882353, + "recall@500": 0.6568627450980392, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.30078518014914984, + "ndcg_full@5": 0.30078518014914984, + "ndcg@10": 0.2974403091742153, + "ndcg_full@10": 0.2974403091742153, + "ndcg@20": 0.28434405240635147, + "ndcg_full@20": 0.28434405240635147, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04375, + "recall@50": 0.2125, + "recall@100": 0.35625, + "recall@200": 0.575, + "recall@300": 0.66875, + "recall@500": 0.725, + "recall@1000": 0.76875, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7456298460447879, + "ndcg_full@5": 0.7456298460447879, + "ndcg@10": 0.6941125739107518, + "ndcg_full@10": 0.6941125739107518, + "ndcg@20": 0.6978591264295058, + "ndcg_full@20": 0.6978591264295058, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 242, + "recall@10": 0.034334763948497854, + "recall@50": 0.1459227467811159, + "recall@100": 0.26609442060085836, + "recall@200": 0.4291845493562232, + "recall@300": 0.5836909871244635, + "recall@500": 0.7939914163090128, + "recall@1000": 0.9613733905579399, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6699156707669441, + "ndcg_full@5": 0.6699156707669441, + "ndcg@10": 0.5531595512918787, + "ndcg_full@10": 0.5531595512918787, + "ndcg@20": 0.46418891807338547, + "ndcg_full@20": 0.46418891807338547, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018518518518518517, + "recall@50": 0.10802469135802469, + "recall@100": 0.2191358024691358, + "recall@200": 0.4074074074074074, + "recall@300": 0.5925925925925926, + "recall@500": 0.8827160493827161, + "recall@1000": 0.941358024691358, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.849607409925425, + "ndcg_full@5": 0.849607409925425, + "ndcg@10": 0.8664745450460192, + "ndcg_full@10": 0.8664745450460192, + "ndcg@20": 0.8297408087851117, + "ndcg_full@20": 0.8297408087851117, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03875968992248062, + "recall@50": 0.16279069767441862, + "recall@100": 0.27906976744186046, + "recall@200": 0.4418604651162791, + "recall@300": 0.6356589147286822, + "recall@500": 0.7790697674418605, + "recall@1000": 0.8023255813953488, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6814687206992398, + "ndcg_full@5": 0.6814687206992398, + "ndcg@10": 0.7932949951308893, + "ndcg_full@10": 0.7932949951308893, + "ndcg@20": 0.7925984258813938, + "ndcg_full@20": 0.7925984258813938, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 87, + "recall@10": 0.06, + "recall@50": 0.28, + "recall@100": 0.52, + "recall@200": 0.6533333333333333, + "recall@300": 0.7133333333333334, + "recall@500": 0.78, + "recall@1000": 0.8266666666666667, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.765471694479642, + "ndcg_full@5": 0.765471694479642, + "ndcg@10": 0.7719260816997477, + "ndcg_full@10": 0.7719260816997477, + "ndcg@20": 0.783284629068035, + "ndcg_full@20": 0.783284629068035, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03488372093023256, + "recall@50": 0.1511627906976744, + "recall@100": 0.2558139534883721, + "recall@200": 0.4883720930232558, + "recall@300": 0.5813953488372093, + "recall@500": 0.6744186046511628, + "recall@1000": 0.7558139534883721, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.09242447578501607, + "ndcg_full@5": 0.09242447578501607, + "ndcg@10": 0.21941382769240952, + "ndcg_full@10": 0.21941382769240952, + "ndcg@20": 0.3176985297044959, + "ndcg_full@20": 0.3176985297044959, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.1509433962264151, + "recall@50": 0.39622641509433965, + "recall@100": 0.4528301886792453, + "recall@200": 0.5094339622641509, + "recall@300": 0.5283018867924528, + "recall@500": 0.5471698113207547, + "recall@1000": 0.5660377358490566, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.46905661104071605, + "ndcg_full@5": 0.46905661104071605, + "ndcg@10": 0.4917193822254317, + "ndcg_full@10": 0.4917193822254317, + "ndcg@20": 0.5421985282903503, + "ndcg_full@20": 0.5421985282903503, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 240, + "recall@10": 0.009569377990430622, + "recall@50": 0.06220095693779904, + "recall@100": 0.12440191387559808, + "recall@200": 0.1674641148325359, + "recall@300": 0.21052631578947367, + "recall@500": 0.2822966507177033, + "recall@1000": 0.42105263157894735, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.8354722964753316, + "ndcg_full@20": 0.8354722964753316, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 242, + "recall@10": 0.02, + "recall@50": 0.14, + "recall@100": 0.22, + "recall@200": 0.3933333333333333, + "recall@300": 0.5133333333333333, + "recall@500": 0.6733333333333333, + "recall@1000": 0.7666666666666667, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.7457783952156486, + "ndcg_full@20": 0.7457783952156486, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 1206, + "num_ranked": 150, + "recall@10": 0.05172413793103448, + "recall@50": 0.27011494252873564, + "recall@100": 0.47126436781609193, + "recall@200": 0.5057471264367817, + "recall@300": 0.5057471264367817, + "recall@500": 0.5057471264367817, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8934699702747572, + "ndcg_full@20": 0.8934699702747572, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.006472491909385114, + "recall@50": 0.038834951456310676, + "recall@100": 0.06796116504854369, + "recall@200": 0.16181229773462782, + "recall@300": 0.2750809061488673, + "recall@500": 0.41423948220064727, + "recall@1000": 0.6116504854368932, + "unjudged_fraction@10": 0.8, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9350624941733298, + "ndcg_full@10": 0.9350624941733298, + "ndcg@20": 0.8897703623599434, + "ndcg_full@20": 0.8897703623599434, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.036290322580645164, + "recall@50": 0.1935483870967742, + "recall@100": 0.3870967741935484, + "recall@200": 0.7540322580645161, + "recall@300": 0.9354838709677419, + "recall@500": 0.9556451612903226, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.9289799801831713, + "ndcg_full@20": 0.9289799801831713, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 239, + "recall@10": 0.12345679012345678, + "recall@50": 0.6172839506172839, + "recall@100": 0.9629629629629629, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9286196799704268, + "ndcg_full@20": 0.9286196799704268, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 1373, + "num_ranked": 126, + "recall@10": 0.13559322033898305, + "recall@50": 0.5932203389830508, + "recall@100": 0.8305084745762712, + "recall@200": 0.9152542372881356, + "recall@300": 0.9661016949152542, + "recall@500": 0.9661016949152542, + "recall@1000": 0.9661016949152542, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3229882611980096, + "ndcg_full@5": 0.3229882611980096, + "ndcg@10": 0.4292021949265368, + "ndcg_full@10": 0.4292021949265368, + "ndcg@20": 0.5977984160390049, + "ndcg_full@20": 0.5977984160390049, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.9, + "ndcg@5": 0.512936578048227, + "ndcg_full@5": 0.512936578048227, + "ndcg@10": 0.4693873512313333, + "ndcg_full@10": 0.4693873512313333, + "ndcg@20": 0.6612194038506906, + "ndcg_full@20": 0.6612194038506906, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02922077922077922, + "recall@50": 0.13636363636363635, + "recall@100": 0.24025974025974026, + "recall@200": 0.42857142857142855, + "recall@300": 0.577922077922078, + "recall@500": 0.8214285714285714, + "recall@1000": 0.9253246753246753, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.8500474841512529, + "ndcg_full@10": 0.8500474841512529, + "ndcg@20": 0.8126601132537674, + "ndcg_full@20": 0.8126601132537674, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.09859154929577464, + "recall@50": 0.30985915492957744, + "recall@100": 0.4647887323943662, + "recall@200": 0.5633802816901409, + "recall@300": 0.6056338028169014, + "recall@500": 0.704225352112676, + "recall@1000": 0.7746478873239436, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0000000000000002, + "ndcg_full@5": 1.0000000000000002, + "ndcg@10": 0.9652843890316112, + "ndcg_full@10": 0.9652843890316112, + "ndcg@20": 0.9435327059431814, + "ndcg_full@20": 0.9435327059431814, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05806451612903226, + "recall@50": 0.25161290322580643, + "recall@100": 0.4064516129032258, + "recall@200": 0.6451612903225806, + "recall@300": 0.7741935483870968, + "recall@500": 0.896774193548387, + "recall@1000": 0.9225806451612903, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.4566005422926888, + "ndcg_full@5": 0.4566005422926888, + "ndcg@10": 0.5470681846268917, + "ndcg_full@10": 0.5470681846268917, + "ndcg@20": 0.6349076729266059, + "ndcg_full@20": 0.6349076729266059, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 1069, + "num_ranked": 152, + "recall@10": 0.05128205128205128, + "recall@50": 0.2564102564102564, + "recall@100": 0.4666666666666667, + "recall@200": 0.7076923076923077, + "recall@300": 0.717948717948718, + "recall@500": 0.7230769230769231, + "recall@1000": 0.7333333333333333, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5351041935991797, + "ndcg_full@5": 0.5, + "ndcg@10": 0.6809984436127783, + "ndcg_full@10": 0.616522379517937, + "ndcg@20": 0.763946211236007, + "ndcg_full@20": 0.7090731854969065, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.1, + "graded_P@10": 0.55 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.053763440860215055, + "recall@50": 0.25806451612903225, + "recall@100": 0.489247311827957, + "recall@200": 0.8387096774193549, + "recall@300": 0.8817204301075269, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7469385177900298, + "ndcg_full@5": 0.7469385177900298, + "ndcg@10": 0.7990984526439919, + "ndcg_full@10": 0.7990984526439919, + "ndcg@20": 0.7839153728999108, + "ndcg_full@20": 0.7839153728999108, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06306306306306306, + "recall@50": 0.2972972972972973, + "recall@100": 0.5855855855855856, + "recall@200": 0.9369369369369369, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6601585067762746, + "ndcg_full@10": 0.6601585067762746, + "ndcg@20": 0.6907253835201204, + "ndcg_full@20": 0.6907253835201204, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018796992481203006, + "recall@50": 0.09398496240601503, + "recall@100": 0.19548872180451127, + "recall@200": 0.40977443609022557, + "recall@300": 0.5789473684210527, + "recall@500": 0.8458646616541353, + "recall@1000": 0.9360902255639098, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.6505098819838827, + "ndcg_full@10": 0.6505098819838827, + "ndcg@20": 0.6850652121317031, + "ndcg_full@20": 0.6850652121317031, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.15151515151515152, + "recall@50": 0.6666666666666666, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.803030303030303, + "recall@1000": 0.803030303030303, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.9526533201221146, + "ndcg_full@20": 0.9526533201221146, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 1563, + "num_ranked": 184, + "recall@10": 0.12987012987012986, + "recall@50": 0.5064935064935064, + "recall@100": 0.7402597402597403, + "recall@200": 0.7922077922077922, + "recall@300": 0.8051948051948052, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.7, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8521705090845474, + "ndcg_full@10": 0.8521705090845474, + "ndcg@20": 0.8689708690572314, + "ndcg_full@20": 0.8689708690572314, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 138, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.782608695652174, + "recall@300": 0.8260869565217391, + "recall@500": 0.8260869565217391, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.2795888034906985, + "ndcg_full@5": 0.2795888034906985, + "ndcg@10": 0.336512033796832, + "ndcg_full@10": 0.32580745338253864, + "ndcg@20": 0.3648094155615161, + "ndcg_full@20": 0.3124478746714593, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1267605633802817, + "recall@50": 0.43661971830985913, + "recall@100": 0.676056338028169, + "recall@200": 0.8591549295774648, + "recall@300": 0.8732394366197183, + "recall@500": 0.8873239436619719, + "recall@1000": 0.9295774647887324, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.9562649741625526, + "ndcg_full@5": 0.9562649741625526, + "ndcg@10": 0.9210315637601145, + "ndcg_full@10": 0.9210315637601145, + "ndcg@20": 0.8665590275038055, + "ndcg_full@20": 0.8665590275038055, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.10294117647058823, + "recall@50": 0.4411764705882353, + "recall@100": 0.6470588235294118, + "recall@200": 0.7352941176470589, + "recall@300": 0.7794117647058824, + "recall@500": 0.7794117647058824, + "recall@1000": 0.8088235294117647, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3082168163143322, + "ndcg_full@5": 0.3082168163143322, + "ndcg@10": 0.2636324123535777, + "ndcg_full@10": 0.2636324123535777, + "ndcg@20": 0.1701397509287266, + "ndcg_full@20": 0.1701397509287266, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 217, + "recall@10": 0.0, + "recall@50": 0.021739130434782608, + "recall@100": 0.043478260869565216, + "recall@200": 0.2391304347826087, + "recall@300": 0.34782608695652173, + "recall@500": 0.6304347826086957, + "recall@1000": 0.7608695652173914, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4777969189511402, + "ndcg_full@5": 0.4777969189511402, + "ndcg@10": 0.5929583930494499, + "ndcg_full@10": 0.5929583930494499, + "ndcg@20": 0.6544855117675548, + "ndcg_full@20": 0.6331482167110335, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 197, + "recall@10": 0.12, + "recall@50": 0.5066666666666667, + "recall@100": 0.8266666666666667, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.92, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.5786288355058145, + "ndcg_full@10": 0.5786288355058145, + "ndcg@20": 0.480167283678999, + "ndcg_full@20": 0.480167283678999, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 183, + "recall@10": 0.23076923076923078, + "recall@50": 0.6666666666666666, + "recall@100": 0.6923076923076923, + "recall@200": 0.6923076923076923, + "recall@300": 0.6923076923076923, + "recall@500": 0.7435897435897436, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6468562235661909, + "ndcg_full@10": 0.6468562235661909, + "ndcg@20": 0.6995192997906257, + "ndcg_full@20": 0.6995192997906257, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 1836, + "num_ranked": 225, + "recall@10": 0.11627906976744186, + "recall@50": 0.37209302325581395, + "recall@100": 0.6511627906976745, + "recall@200": 0.9883720930232558, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.7933660361310324, + "ndcg_full@20": 0.7933660361310324, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05555555555555555, + "recall@50": 0.19753086419753085, + "recall@100": 0.35802469135802467, + "recall@200": 0.5802469135802469, + "recall@300": 0.654320987654321, + "recall@500": 0.7098765432098766, + "recall@1000": 0.7654320987654321, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503199, + "ndcg_full@10": 0.9633180389503199, + "ndcg@20": 0.976326660061057, + "ndcg_full@20": 0.976326660061057, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 1562, + "num_ranked": 250, + "recall@10": 0.07352941176470588, + "recall@50": 0.34558823529411764, + "recall@100": 0.5808823529411765, + "recall@200": 0.6691176470588235, + "recall@300": 0.6764705882352942, + "recall@500": 0.6764705882352942, + "recall@1000": 0.6764705882352942, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6144247685324461, + "ndcg_full@5": 0.6144247685324461, + "ndcg@10": 0.7129224887990167, + "ndcg_full@10": 0.7129224887990167, + "ndcg@20": 0.798560402219582, + "ndcg_full@20": 0.798560402219582, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1114, + "num_ranked": 183, + "recall@10": 0.02564102564102564, + "recall@50": 0.13105413105413105, + "recall@100": 0.24786324786324787, + "recall@200": 0.4472934472934473, + "recall@300": 0.5868945868945868, + "recall@500": 0.6210826210826211, + "recall@1000": 0.6296296296296297, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9775956781991642, + "ndcg_full@20": 0.9775956781991642, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 1031, + "num_ranked": 246, + "recall@10": 0.035175879396984924, + "recall@50": 0.17587939698492464, + "recall@100": 0.31155778894472363, + "recall@200": 0.5728643216080402, + "recall@300": 0.6984924623115578, + "recall@500": 0.8844221105527639, + "recall@1000": 0.9597989949748744, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8874369434113021, + "ndcg_full@10": 0.8874369434113021, + "ndcg@20": 0.9273554786348056, + "ndcg_full@20": 0.9273554786348056, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02631578947368421, + "recall@50": 0.07368421052631578, + "recall@100": 0.12631578947368421, + "recall@200": 0.2578947368421053, + "recall@300": 0.42105263157894735, + "recall@500": 0.6947368421052632, + "recall@1000": 0.9315789473684211, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.930635970923374, + "ndcg_full@20": 0.930635970923374, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03272727272727273, + "recall@50": 0.14181818181818182, + "recall@100": 0.26545454545454544, + "recall@200": 0.44363636363636366, + "recall@300": 0.5381818181818182, + "recall@500": 0.6327272727272727, + "recall@1000": 0.7527272727272727, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.565602538756171, + "ndcg_full@5": 0.565602538756171, + "ndcg@10": 0.6376932336781874, + "ndcg_full@10": 0.6376932336781874, + "ndcg@20": 0.6667929946747841, + "ndcg_full@20": 0.6667929946747841, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.3, + "graded_P@10": 0.65 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06818181818181818, + "recall@50": 0.20454545454545456, + "recall@100": 0.29545454545454547, + "recall@200": 0.4772727272727273, + "recall@300": 0.5909090909090909, + "recall@500": 0.6818181818181818, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.6887676151990412, + "ndcg_full@5": 0.6887676151990412, + "ndcg@10": 0.5454254030160456, + "ndcg_full@10": 0.5454254030160456, + "ndcg@20": 0.6005974221562572, + "ndcg_full@20": 0.4864193939095867, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 1407, + "num_ranked": 222, + "recall@10": 0.029556650246305417, + "recall@50": 0.13793103448275862, + "recall@100": 0.20689655172413793, + "recall@200": 0.37438423645320196, + "recall@300": 0.5320197044334976, + "recall@500": 0.8423645320197044, + "recall@1000": 0.9605911330049262, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.674385164197632, + "ndcg_full@10": 0.674385164197632, + "ndcg@20": 0.613786404575557, + "ndcg_full@20": 0.613786404575557, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035398230088495575, + "recall@50": 0.1415929203539823, + "recall@100": 0.23893805309734514, + "recall@200": 0.4247787610619469, + "recall@300": 0.584070796460177, + "recall@500": 0.7699115044247787, + "recall@1000": 0.8495575221238938, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.5662568746287918, + "ndcg_full@5": 0.5662568746287918, + "ndcg@10": 0.6426492948244856, + "ndcg_full@10": 0.6426492948244856, + "ndcg@20": 0.6484754545039223, + "ndcg_full@20": 0.6484754545039223, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 1796, + "num_ranked": 40, + "recall@10": 0.14583333333333334, + "recall@50": 0.25, + "recall@100": 0.25, + "recall@200": 0.2916666666666667, + "recall@300": 0.2916666666666667, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.44916890612750654, + "ndcg_full@5": 0.44916890612750654, + "ndcg@10": 0.5688171728399543, + "ndcg_full@10": 0.5688171728399543, + "ndcg@20": 0.7753040543541553, + "ndcg_full@20": 0.6297906699931223, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06369426751592357, + "recall@50": 0.2611464968152866, + "recall@100": 0.46496815286624205, + "recall@200": 0.6815286624203821, + "recall@300": 0.7834394904458599, + "recall@500": 0.8152866242038217, + "recall@1000": 0.821656050955414, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.669290626270562, + "ndcg_full@10": 0.669290626270562, + "ndcg@20": 0.7083327855703434, + "ndcg_full@20": 0.7083327855703434, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06097560975609756, + "recall@50": 0.23780487804878048, + "recall@100": 0.3597560975609756, + "recall@200": 0.45121951219512196, + "recall@300": 0.4878048780487805, + "recall@500": 0.5792682926829268, + "recall@1000": 0.6707317073170732, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.680027307288617, + "ndcg_full@5": 0.680027307288617, + "ndcg@10": 0.6461186247813709, + "ndcg_full@10": 0.6461186247813709, + "ndcg@20": 0.6649576759850306, + "ndcg_full@20": 0.6649576759850306, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.0, + "recall@200": 0.0379746835443038, + "recall@300": 0.06329113924050633, + "recall@500": 0.06329113924050633, + "recall@1000": 0.10126582278481013, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.5307212739772434, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.5307212739772434, + "ndcg_full@10": 0.2489083270225946, + "ndcg@20": 0.5307212739772434, + "ndcg_full@20": 0.16063730702017212, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 248, + "recall@10": 0.04329004329004329, + "recall@50": 0.16883116883116883, + "recall@100": 0.2813852813852814, + "recall@200": 0.5238095238095238, + "recall@300": 0.7489177489177489, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.46604104255408946, + "ndcg_full@5": 0.46604104255408946, + "ndcg@10": 0.5750989036033994, + "ndcg_full@10": 0.5750989036033994, + "ndcg@20": 0.6638968079034273, + "ndcg_full@20": 0.6638968079034273, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03260869565217391, + "recall@50": 0.07065217391304347, + "recall@100": 0.10869565217391304, + "recall@200": 0.20652173913043478, + "recall@300": 0.30978260869565216, + "recall@500": 0.5597826086956522, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.8200685179888283, + "ndcg_full@10": 0.8200685179888283, + "ndcg@20": 0.7756150022106207, + "ndcg_full@20": 0.7756150022106207, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 1324, + "num_ranked": 159, + "recall@10": 0.0989010989010989, + "recall@50": 0.24175824175824176, + "recall@100": 0.25274725274725274, + "recall@200": 0.25274725274725274, + "recall@300": 0.25274725274725274, + "recall@500": 0.25274725274725274, + "recall@1000": 0.2857142857142857, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6386367136775242, + "ndcg_full@5": 0.6386367136775242, + "ndcg@10": 0.5940714136864377, + "ndcg_full@10": 0.5940714136864377, + "ndcg@20": 0.6706752031705321, + "ndcg_full@20": 0.6307200521120642, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.4, + "graded_P@10": 0.6 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.13636363636363635, + "recall@50": 0.6060606060606061, + "recall@100": 0.8787878787878788, + "recall@200": 0.9242424242424242, + "recall@300": 0.9696969696969697, + "recall@500": 0.9848484848484849, + "recall@1000": 0.9848484848484849, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.17685092996179536, + "ndcg_full@10": 0.17685092996179536, + "ndcg@20": 0.4475667262567596, + "ndcg_full@20": 0.44032995470576386, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.1, + "graded_P@10": 0.25 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 90, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6445343056547514, + "ndcg_full@10": 0.6445343056547514, + "ndcg@20": 0.8009851267125935, + "ndcg_full@20": 0.8009851267125935, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 1784, + "num_ranked": 250, + "recall@10": 0.02346041055718475, + "recall@50": 0.11730205278592376, + "recall@100": 0.21994134897360704, + "recall@200": 0.4310850439882698, + "recall@300": 0.6363636363636364, + "recall@500": 0.8768328445747801, + "recall@1000": 0.9618768328445748, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8085523844444683, + "ndcg_full@5": 0.8085523844444683, + "ndcg@10": 0.839866434031192, + "ndcg_full@10": 0.839866434031192, + "ndcg@20": 0.8472630581732761, + "ndcg_full@20": 0.8472630581732761, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.07987159484786008, + "recall@50": 0.2697296003572918, + "recall@100": 0.407365139825072, + "recall@200": 0.5657282405172425, + "recall@300": 0.6525827512818093, + "recall@500": 0.7501550294140854, + "recall@1000": 0.8178965483941935, + "ndcg@5": 0.42956820971859067, + "ndcg@10": 0.4304568730021463, + "ndcg@20": 0.43860460440569804, + "ndcg_full@5": 0.4288637633335286, + "ndcg_full@10": 0.4280606057622515, + "ndcg_full@20": 0.4309262106463552, + "P@10(rel>=1)": 0.4866666666666666, + "P@10(eligible)": 0.3679999999999999, + "graded_P@10": 0.4273333333333333, + "unjudged_fraction@10": 0.3906666666666666 + }, + "median": { + "recall@10": 0.05063291139240506, + "recall@50": 0.20253164556962025, + "recall@100": 0.33544303797468356, + "recall@200": 0.5238095238095238, + "recall@300": 0.654320987654321, + "recall@500": 0.7883211678832117, + "recall@1000": 0.8592592592592593, + "ndcg@5": 0.44916890612750654, + "ndcg@10": 0.4163824138054737, + "ndcg@20": 0.41316227106581255, + "ndcg_full@5": 0.44916890612750654, + "ndcg_full@10": 0.4163824138054737, + "ndcg_full@20": 0.41316227106581255, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4, + "unjudged_fraction@10": 0.4 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05917159763313609, + "recall@50": 0.26627218934911245, + "recall@100": 0.47928994082840237, + "recall@200": 0.7869822485207101, + "recall@300": 0.834319526627219, + "recall@500": 0.8520710059171598, + "recall@1000": 0.8698224852071006, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.5622295221244623, + "ndcg_full@10": 0.5622295221244623, + "ndcg@20": 0.4187664584041546, + "ndcg_full@20": 0.4187664584041546, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.022222222222222223, + "recall@50": 0.1259259259259259, + "recall@100": 0.24074074074074073, + "recall@200": 0.4185185185185185, + "recall@300": 0.5518518518518518, + "recall@500": 0.7037037037037037, + "recall@1000": 0.8592592592592593, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.35970991185182927, + "ndcg_full@10": 0.35970991185182927, + "ndcg@20": 0.39121603563319574, + "ndcg_full@20": 0.39121603563319574, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 136, + "recall@10": 0.09523809523809523, + "recall@50": 0.3333333333333333, + "recall@100": 0.47619047619047616, + "recall@200": 0.6190476190476191, + "recall@300": 0.7619047619047619, + "recall@500": 0.8690476190476191, + "recall@1000": 0.9523809523809523, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.45503072493644514, + "ndcg_full@10": 0.45503072493644514, + "ndcg@20": 0.41316227106581255, + "ndcg_full@20": 0.41316227106581255, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.12631578947368421, + "recall@100": 0.25263157894736843, + "recall@200": 0.3263157894736842, + "recall@300": 0.4842105263157895, + "recall@500": 0.6842105263157895, + "recall@1000": 0.8421052631578947, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.08479005131840403, + "ndcg_full@5": 0.08479005131840403, + "ndcg@10": 0.05502294157452004, + "ndcg_full@10": 0.05502294157452004, + "ndcg@20": 0.12946931345866583, + "ndcg_full@20": 0.12946931345866583, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.004975124378109453, + "recall@50": 0.03482587064676617, + "recall@100": 0.08955223880597014, + "recall@200": 0.24875621890547264, + "recall@300": 0.3383084577114428, + "recall@500": 0.4925373134328358, + "recall@1000": 0.6666666666666666, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.5639512376193366, + "ndcg_full@10": 0.5639512376193366, + "ndcg@20": 0.40357678193890983, + "ndcg_full@20": 0.40357678193890983, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.047619047619047616, + "recall@50": 0.14285714285714285, + "recall@100": 0.2714285714285714, + "recall@200": 0.4238095238095238, + "recall@300": 0.5666666666666667, + "recall@500": 0.7571428571428571, + "recall@1000": 0.861904761904762, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.6404931013741779, + "ndcg_full@10": 0.6404931013741779, + "ndcg@20": 0.6040701989340967, + "ndcg_full@20": 0.6040701989340967, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 196, + "recall@10": 0.024844720496894408, + "recall@50": 0.14285714285714285, + "recall@100": 0.2484472049689441, + "recall@200": 0.37888198757763975, + "recall@300": 0.4472049689440994, + "recall@500": 0.639751552795031, + "recall@1000": 0.8757763975155279, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.16520994868159597, + "ndcg_full@5": 0.16520994868159597, + "ndcg@10": 0.2052745698123065, + "ndcg_full@10": 0.2052745698123065, + "ndcg@20": 0.20571740414094478, + "ndcg_full@20": 0.20571740414094478, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05228758169934641, + "recall@50": 0.22875816993464052, + "recall@100": 0.47058823529411764, + "recall@200": 0.8300653594771242, + "recall@300": 0.9150326797385621, + "recall@500": 0.934640522875817, + "recall@1000": 0.954248366013072, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.56023360836774, + "ndcg_full@10": 0.56023360836774, + "ndcg@20": 0.6838508387421722, + "ndcg_full@20": 0.6838508387421722, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04205607476635514, + "recall@50": 0.1822429906542056, + "recall@100": 0.3598130841121495, + "recall@200": 0.602803738317757, + "recall@300": 0.780373831775701, + "recall@500": 0.9345794392523364, + "recall@1000": 0.9485981308411215, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.06560253875617089, + "ndcg_full@5": 0.06560253875617089, + "ndcg@10": 0.14670819913681854, + "ndcg_full@10": 0.14670819913681854, + "ndcg@20": 0.26543176964986726, + "ndcg_full@20": 0.24804780859179365, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.1, + "graded_P@10": 0.2 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 166, + "recall@10": 0.2127659574468085, + "recall@50": 0.8085106382978723, + "recall@100": 0.8085106382978723, + "recall@200": 0.8085106382978723, + "recall@300": 0.8085106382978723, + "recall@500": 0.8085106382978723, + "recall@1000": 0.8085106382978723, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.15039259007457492, + "ndcg_full@5": 0.15039259007457492, + "ndcg@10": 0.4139473140950953, + "ndcg_full@10": 0.4139473140950953, + "ndcg@20": 0.40747457581830754, + "ndcg_full@20": 0.3978931583181193, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 161, + "recall@10": 0.07936507936507936, + "recall@50": 0.3492063492063492, + "recall@100": 0.5555555555555556, + "recall@200": 0.8095238095238095, + "recall@300": 0.8333333333333334, + "recall@500": 0.8492063492063492, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7129795603441191, + "ndcg_full@5": 0.7129795603441191, + "ndcg@10": 0.7745442394523322, + "ndcg_full@10": 0.7745442394523322, + "ndcg@20": 0.6060985013023914, + "ndcg_full@20": 0.6060985013023914, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.06040268456375839, + "recall@50": 0.28187919463087246, + "recall@100": 0.4966442953020134, + "recall@200": 0.7986577181208053, + "recall@300": 0.8322147651006712, + "recall@500": 0.8859060402684564, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5551553341043619, + "ndcg_full@5": 0.5551553341043619, + "ndcg@10": 0.6379623280830145, + "ndcg_full@10": 0.6379623280830145, + "ndcg@20": 0.6219244265693683, + "ndcg_full@20": 0.6219244265693683, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.11594202898550725, + "recall@100": 0.2028985507246377, + "recall@200": 0.391304347826087, + "recall@300": 0.5362318840579711, + "recall@500": 0.644927536231884, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.24982680863190024, + "ndcg_full@10": 0.24982680863190024, + "ndcg@20": 0.20072070878499107, + "ndcg_full@20": 0.19082108784627116, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.013605442176870748, + "recall@50": 0.08503401360544217, + "recall@100": 0.17006802721088435, + "recall@200": 0.30952380952380953, + "recall@300": 0.3877551020408163, + "recall@500": 0.5544217687074829, + "recall@1000": 0.6768707482993197, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.3218570557413087, + "ndcg_full@10": 0.3218570557413087, + "ndcg@20": 0.3628712678450955, + "ndcg_full@20": 0.3628712678450955, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 190, + "recall@10": 0.026923076923076925, + "recall@50": 0.1423076923076923, + "recall@100": 0.26153846153846155, + "recall@200": 0.3923076923076923, + "recall@300": 0.4807692307692308, + "recall@500": 0.5807692307692308, + "recall@1000": 0.6884615384615385, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8355590206883058, + "ndcg_full@10": 0.8355590206883058, + "ndcg@20": 0.7638323732745789, + "ndcg_full@20": 0.7638323732745789, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.028985507246376812, + "recall@50": 0.10144927536231885, + "recall@100": 0.14492753623188406, + "recall@200": 0.2391304347826087, + "recall@300": 0.4057971014492754, + "recall@500": 0.5507246376811594, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.10699313236726378, + "ndcg_full@5": 0.10699313236726378, + "ndcg@10": 0.14279514403613733, + "ndcg_full@10": 0.14279514403613733, + "ndcg@20": 0.1486543929286936, + "ndcg_full@20": 0.1486543929286936, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035856573705179286, + "recall@50": 0.14342629482071714, + "recall@100": 0.29880478087649404, + "recall@200": 0.601593625498008, + "recall@300": 0.852589641434263, + "recall@500": 0.9641434262948207, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.29938256369037275, + "ndcg_full@10": 0.29938256369037275, + "ndcg@20": 0.4245161699161455, + "ndcg_full@20": 0.4245161699161455, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03398058252427184, + "recall@50": 0.1796116504854369, + "recall@100": 0.30097087378640774, + "recall@200": 0.49029126213592233, + "recall@300": 0.6407766990291263, + "recall@500": 0.7961165048543689, + "recall@1000": 0.8737864077669902, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.13863671367752411, + "ndcg_full@5": 0.13863671367752411, + "ndcg@10": 0.28393819129103864, + "ndcg_full@10": 0.2654999533396626, + "ndcg@20": 0.3828055846779671, + "ndcg_full@20": 0.33264226668281444, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0457516339869281, + "recall@50": 0.16339869281045752, + "recall@100": 0.2875816993464052, + "recall@200": 0.5228758169934641, + "recall@300": 0.6601307189542484, + "recall@500": 0.8431372549019608, + "recall@1000": 0.9215686274509803, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.21101571757181778, + "ndcg_full@10": 0.21101571757181778, + "ndcg@20": 0.20666406548434613, + "ndcg_full@20": 0.20666406548434613, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.07142857142857142, + "recall@50": 0.35714285714285715, + "recall@100": 0.6428571428571429, + "recall@200": 0.8714285714285714, + "recall@300": 0.9, + "recall@500": 0.9571428571428572, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.1780615783628922, + "ndcg_full@10": 0.1780615783628922, + "ndcg@20": 0.23827968693968588, + "ndcg_full@20": 0.23827968693968588, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.07547169811320754, + "recall@50": 0.37735849056603776, + "recall@100": 0.7169811320754716, + "recall@200": 0.8867924528301887, + "recall@300": 0.9056603773584906, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.727329844310522, + "ndcg_full@10": 0.727329844310522, + "ndcg@20": 0.6761399613494413, + "ndcg_full@20": 0.6761399613494413, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 1494, + "num_ranked": 204, + "recall@10": 0.058394160583941604, + "recall@50": 0.31386861313868614, + "recall@100": 0.5474452554744526, + "recall@200": 0.7153284671532847, + "recall@300": 0.7445255474452555, + "recall@500": 0.7883211678832117, + "recall@1000": 0.8248175182481752, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8156211075169129, + "ndcg_full@10": 0.8156211075169129, + "ndcg@20": 0.7340721386505599, + "ndcg_full@20": 0.7340721386505599, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 1496, + "num_ranked": 250, + "recall@10": 0.00749063670411985, + "recall@50": 0.07116104868913857, + "recall@100": 0.1348314606741573, + "recall@200": 0.2696629213483146, + "recall@300": 0.41198501872659177, + "recall@500": 0.6404494382022472, + "recall@1000": 0.8689138576779026, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.204239252433695, + "ndcg_full@5": 0.204239252433695, + "ndcg@10": 0.1961580880253481, + "ndcg_full@10": 0.1961580880253481, + "ndcg@20": 0.2358677441998373, + "ndcg_full@20": 0.2358677441998373, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05063291139240506, + "recall@50": 0.20253164556962025, + "recall@100": 0.33544303797468356, + "recall@200": 0.5822784810126582, + "recall@300": 0.7531645569620253, + "recall@500": 0.8987341772151899, + "recall@1000": 0.9683544303797469, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.45449827622138, + "ndcg_full@10": 0.45449827622138, + "ndcg@20": 0.4358005062324582, + "ndcg_full@20": 0.4358005062324582, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0196078431372549, + "recall@50": 0.11764705882352941, + "recall@100": 0.24509803921568626, + "recall@200": 0.3627450980392157, + "recall@300": 0.4411764705882353, + "recall@500": 0.6568627450980392, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.09478836436955078, + "ndcg_full@10": 0.09478836436955078, + "ndcg@20": 0.15153535942481638, + "ndcg_full@20": 0.15153535942481638, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04375, + "recall@50": 0.2125, + "recall@100": 0.35625, + "recall@200": 0.575, + "recall@300": 0.66875, + "recall@500": 0.725, + "recall@1000": 0.76875, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.4808124874377669, + "ndcg_full@5": 0.4808124874377669, + "ndcg@10": 0.40340017505622316, + "ndcg_full@10": 0.40340017505622316, + "ndcg@20": 0.3218858711846758, + "ndcg_full@20": 0.3218858711846758, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 242, + "recall@10": 0.034334763948497854, + "recall@50": 0.1459227467811159, + "recall@100": 0.26609442060085836, + "recall@200": 0.4291845493562232, + "recall@300": 0.5836909871244635, + "recall@500": 0.7939914163090128, + "recall@1000": 0.9613733905579399, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6524216604319653, + "ndcg_full@5": 0.6524216604319653, + "ndcg@10": 0.5126769372780504, + "ndcg_full@10": 0.5126769372780504, + "ndcg@20": 0.38473130953878876, + "ndcg_full@20": 0.38473130953878876, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.2, + "graded_P@10": 0.45 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018518518518518517, + "recall@50": 0.10802469135802469, + "recall@100": 0.2191358024691358, + "recall@200": 0.4074074074074074, + "recall@300": 0.5925925925925926, + "recall@500": 0.8827160493827161, + "recall@1000": 0.941358024691358, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.44916890612750654, + "ndcg_full@5": 0.44916890612750654, + "ndcg@10": 0.48232053933620916, + "ndcg_full@10": 0.48232053933620916, + "ndcg@20": 0.5363594066175308, + "ndcg_full@20": 0.5363594066175308, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03875968992248062, + "recall@50": 0.16279069767441862, + "recall@100": 0.27906976744186046, + "recall@200": 0.4418604651162791, + "recall@300": 0.6356589147286822, + "recall@500": 0.7790697674418605, + "recall@1000": 0.8023255813953488, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.08260497434079798, + "ndcg_full@5": 0.08260497434079798, + "ndcg@10": 0.3137833489008815, + "ndcg_full@10": 0.3137833489008815, + "ndcg@20": 0.42679056940725635, + "ndcg_full@20": 0.42679056940725635, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 87, + "recall@10": 0.06, + "recall@50": 0.28, + "recall@100": 0.52, + "recall@200": 0.6533333333333333, + "recall@300": 0.7133333333333334, + "recall@500": 0.78, + "recall@1000": 0.8266666666666667, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5163251428503343, + "ndcg_full@5": 0.5163251428503343, + "ndcg@10": 0.5119106182951034, + "ndcg_full@10": 0.5119106182951034, + "ndcg@20": 0.5813824096442224, + "ndcg_full@20": 0.5813824096442224, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03488372093023256, + "recall@50": 0.1511627906976744, + "recall@100": 0.2558139534883721, + "recall@200": 0.4883720930232558, + "recall@300": 0.5813953488372093, + "recall@500": 0.6744186046511628, + "recall@1000": 0.7558139534883721, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.11702280822271781, + "ndcg_full@10": 0.11702280822271781, + "ndcg@20": 0.21519567641133594, + "ndcg_full@20": 0.21519567641133594, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.1509433962264151, + "recall@50": 0.39622641509433965, + "recall@100": 0.4528301886792453, + "recall@200": 0.5094339622641509, + "recall@300": 0.5283018867924528, + "recall@500": 0.5471698113207547, + "recall@1000": 0.5660377358490566, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.16946218957193354, + "ndcg_full@10": 0.16946218957193354, + "ndcg@20": 0.1615141886244825, + "ndcg_full@20": 0.1615141886244825, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 240, + "recall@10": 0.009569377990430622, + "recall@50": 0.06220095693779904, + "recall@100": 0.12440191387559808, + "recall@200": 0.1674641148325359, + "recall@300": 0.21052631578947367, + "recall@500": 0.2822966507177033, + "recall@1000": 0.42105263157894735, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.504370153955212, + "ndcg_full@5": 0.504370153955212, + "ndcg@10": 0.430974021773298, + "ndcg_full@10": 0.430974021773298, + "ndcg@20": 0.2979470952148586, + "ndcg_full@20": 0.2979470952148586, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 242, + "recall@10": 0.02, + "recall@50": 0.14, + "recall@100": 0.22, + "recall@200": 0.3933333333333333, + "recall@300": 0.5133333333333333, + "recall@500": 0.6733333333333333, + "recall@1000": 0.7666666666666667, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.3885264725763437, + "ndcg_full@10": 0.3885264725763437, + "ndcg@20": 0.46790056511373657, + "ndcg_full@20": 0.46790056511373657, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 1206, + "num_ranked": 150, + "recall@10": 0.05172413793103448, + "recall@50": 0.27011494252873564, + "recall@100": 0.47126436781609193, + "recall@200": 0.5057471264367817, + "recall@300": 0.5057471264367817, + "recall@500": 0.5057471264367817, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8263333286520083, + "ndcg_full@10": 0.8263333286520083, + "ndcg@20": 0.8314366075254482, + "ndcg_full@20": 0.8314366075254482, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.006472491909385114, + "recall@50": 0.038834951456310676, + "recall@100": 0.06796116504854369, + "recall@200": 0.16181229773462782, + "recall@300": 0.2750809061488673, + "recall@500": 0.41423948220064727, + "recall@1000": 0.6116504854368932, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.1635413866202963, + "ndcg_full@10": 0.1635413866202963, + "ndcg@20": 0.21346500076185326, + "ndcg_full@20": 0.21346500076185326, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.036290322580645164, + "recall@50": 0.1935483870967742, + "recall@100": 0.3870967741935484, + "recall@200": 0.7540322580645161, + "recall@300": 0.9354838709677419, + "recall@500": 0.9556451612903226, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.9289799801831713, + "ndcg_full@20": 0.9289799801831713, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 239, + "recall@10": 0.12345679012345678, + "recall@50": 0.6172839506172839, + "recall@100": 0.9629629629629629, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.7885560429799757, + "ndcg_full@10": 0.7885560429799757, + "ndcg@20": 0.7751595906548149, + "ndcg_full@20": 0.7751595906548149, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 1373, + "num_ranked": 126, + "recall@10": 0.13559322033898305, + "recall@50": 0.5932203389830508, + "recall@100": 0.8305084745762712, + "recall@200": 0.9152542372881356, + "recall@300": 0.9661016949152542, + "recall@500": 0.9661016949152542, + "recall@1000": 0.9661016949152542, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.137535533656678, + "ndcg_full@10": 0.137535533656678, + "ndcg@20": 0.362967492342105, + "ndcg_full@20": 0.362967492342105, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.1, + "graded_P@10": 0.2 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.9, + "ndcg@5": 0.2765732350040718, + "ndcg_full@5": 0.2765732350040718, + "ndcg@10": 0.17947710508581735, + "ndcg_full@10": 0.17947710508581735, + "ndcg@20": 0.24132442752404118, + "ndcg_full@20": 0.24132442752404118, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02922077922077922, + "recall@50": 0.13636363636363635, + "recall@100": 0.24025974025974026, + "recall@200": 0.42857142857142855, + "recall@300": 0.577922077922078, + "recall@500": 0.8214285714285714, + "recall@1000": 0.9253246753246753, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.6749013152464313, + "ndcg_full@10": 0.6749013152464313, + "ndcg@20": 0.5413776680853959, + "ndcg_full@20": 0.5413776680853959, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.09859154929577464, + "recall@50": 0.30985915492957744, + "recall@100": 0.4647887323943662, + "recall@200": 0.5633802816901409, + "recall@300": 0.6056338028169014, + "recall@500": 0.704225352112676, + "recall@1000": 0.7746478873239436, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7379954215781759, + "ndcg_full@5": 0.7379954215781759, + "ndcg@10": 0.7341610773575021, + "ndcg_full@10": 0.7341610773575021, + "ndcg@20": 0.6695736617908444, + "ndcg_full@20": 0.6695736617908444, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05806451612903226, + "recall@50": 0.25161290322580643, + "recall@100": 0.4064516129032258, + "recall@200": 0.6451612903225806, + "recall@300": 0.7741935483870968, + "recall@500": 0.896774193548387, + "recall@1000": 0.9225806451612903, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.19163415146817556, + "ndcg_full@10": 0.19163415146817556, + "ndcg@20": 0.2420490425897011, + "ndcg_full@20": 0.2420490425897011, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 1069, + "num_ranked": 152, + "recall@10": 0.05128205128205128, + "recall@50": 0.2564102564102564, + "recall@100": 0.4666666666666667, + "recall@200": 0.7076923076923077, + "recall@300": 0.717948717948718, + "recall@500": 0.7230769230769231, + "recall@1000": 0.7333333333333333, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5351041935991797, + "ndcg_full@5": 0.5, + "ndcg@10": 0.6809984436127783, + "ndcg_full@10": 0.616522379517937, + "ndcg@20": 0.763946211236007, + "ndcg_full@20": 0.7090731854969065, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.1, + "graded_P@10": 0.55 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.053763440860215055, + "recall@50": 0.25806451612903225, + "recall@100": 0.489247311827957, + "recall@200": 0.8387096774193549, + "recall@300": 0.8817204301075269, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7469385177900298, + "ndcg_full@5": 0.7469385177900298, + "ndcg@10": 0.7990984526439919, + "ndcg_full@10": 0.7990984526439919, + "ndcg@20": 0.7619104890706867, + "ndcg_full@20": 0.7619104890706867, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06306306306306306, + "recall@50": 0.2972972972972973, + "recall@100": 0.5855855855855856, + "recall@200": 0.9369369369369369, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.48153499054313026, + "ndcg_full@10": 0.48153499054313026, + "ndcg@20": 0.5093249723654392, + "ndcg_full@20": 0.5093249723654392, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018796992481203006, + "recall@50": 0.09398496240601503, + "recall@100": 0.19548872180451127, + "recall@200": 0.40977443609022557, + "recall@300": 0.5789473684210527, + "recall@500": 0.8458646616541353, + "recall@1000": 0.9360902255639098, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.578724266536559, + "ndcg_full@10": 0.578724266536559, + "ndcg@20": 0.48859662485365013, + "ndcg_full@20": 0.48859662485365013, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.15151515151515152, + "recall@50": 0.6666666666666666, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.803030303030303, + "recall@1000": 0.803030303030303, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7542686714597936, + "ndcg_full@10": 0.7542686714597936, + "ndcg@20": 0.8058742005345274, + "ndcg_full@20": 0.8058742005345274, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 1563, + "num_ranked": 184, + "recall@10": 0.12987012987012986, + "recall@50": 0.5064935064935064, + "recall@100": 0.7402597402597403, + "recall@200": 0.7922077922077922, + "recall@300": 0.8051948051948052, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.4690000933206748, + "ndcg_full@10": 0.4690000933206748, + "ndcg@20": 0.40688193569910247, + "ndcg_full@20": 0.40688193569910247, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 138, + "recall@10": 0.43478260869565216, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.782608695652174, + "recall@300": 0.8260869565217391, + "recall@500": 0.8260869565217391, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.19178318368566782, + "ndcg_full@5": 0.19178318368566782, + "ndcg@10": 0.27071759228396924, + "ndcg_full@10": 0.262105958983755, + "ndcg@20": 0.2676389641722811, + "ndcg_full@20": 0.22922441682648695, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1267605633802817, + "recall@50": 0.43661971830985913, + "recall@100": 0.676056338028169, + "recall@200": 0.8591549295774648, + "recall@300": 0.8732394366197183, + "recall@500": 0.8873239436619719, + "recall@1000": 0.9295774647887324, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.4427195269179057, + "ndcg_full@10": 0.4427195269179057, + "ndcg@20": 0.5113370034489968, + "ndcg_full@20": 0.5113370034489968, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.10294117647058823, + "recall@50": 0.4411764705882353, + "recall@100": 0.6470588235294118, + "recall@200": 0.7352941176470589, + "recall@300": 0.7794117647058824, + "recall@500": 0.7794117647058824, + "recall@1000": 0.8088235294117647, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.15700502675892142, + "ndcg_full@10": 0.15700502675892142, + "ndcg@20": 0.13768220580282625, + "ndcg_full@20": 0.13768220580282625, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 217, + "recall@10": 0.0, + "recall@50": 0.021739130434782608, + "recall@100": 0.043478260869565216, + "recall@200": 0.2391304347826087, + "recall@300": 0.34782608695652173, + "recall@500": 0.6304347826086957, + "recall@1000": 0.7608695652173914, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.404762744029787, + "ndcg_full@5": 0.404762744029787, + "ndcg@10": 0.47263264044759384, + "ndcg_full@10": 0.47263264044759384, + "ndcg@20": 0.4816230603340715, + "ndcg_full@20": 0.46592136310838467, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 197, + "recall@10": 0.12, + "recall@50": 0.5066666666666667, + "recall@100": 0.8266666666666667, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.92, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.5105143311967558, + "ndcg_full@10": 0.5105143311967558, + "ndcg@20": 0.43949724692610725, + "ndcg_full@20": 0.43949724692610725, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 183, + "recall@10": 0.23076923076923078, + "recall@50": 0.6666666666666666, + "recall@100": 0.6923076923076923, + "recall@200": 0.6923076923076923, + "recall@300": 0.6923076923076923, + "recall@500": 0.7435897435897436, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.30078518014914984, + "ndcg_full@5": 0.30078518014914984, + "ndcg@10": 0.4163824138054737, + "ndcg_full@10": 0.4163824138054737, + "ndcg@20": 0.3021571057277026, + "ndcg_full@20": 0.3021571057277026, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 1836, + "num_ranked": 225, + "recall@10": 0.11627906976744186, + "recall@50": 0.37209302325581395, + "recall@100": 0.6511627906976745, + "recall@200": 0.9883720930232558, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.7933660361310324, + "ndcg_full@20": 0.7933660361310324, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05555555555555555, + "recall@50": 0.19753086419753085, + "recall@100": 0.35802469135802467, + "recall@200": 0.5802469135802469, + "recall@300": 0.654320987654321, + "recall@500": 0.7098765432098766, + "recall@1000": 0.7654320987654321, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.7220039004349634, + "ndcg_full@10": 0.7220039004349634, + "ndcg@20": 0.6318459164406772, + "ndcg_full@20": 0.6318459164406772, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 1562, + "num_ranked": 250, + "recall@10": 0.07352941176470588, + "recall@50": 0.34558823529411764, + "recall@100": 0.5808823529411765, + "recall@200": 0.6691176470588235, + "recall@300": 0.6764705882352942, + "recall@500": 0.6764705882352942, + "recall@1000": 0.6764705882352942, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.3210616007208579, + "ndcg_full@10": 0.3210616007208579, + "ndcg@20": 0.47209457608093713, + "ndcg_full@20": 0.47209457608093713, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1114, + "num_ranked": 183, + "recall@10": 0.02564102564102564, + "recall@50": 0.13105413105413105, + "recall@100": 0.24786324786324787, + "recall@200": 0.4472934472934473, + "recall@300": 0.5868945868945868, + "recall@500": 0.6210826210826211, + "recall@1000": 0.6296296296296297, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6786666176656667, + "ndcg_full@10": 0.6786666176656667, + "ndcg@20": 0.6884737708530224, + "ndcg_full@20": 0.6884737708530224, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 1031, + "num_ranked": 246, + "recall@10": 0.035175879396984924, + "recall@50": 0.17587939698492464, + "recall@100": 0.31155778894472363, + "recall@200": 0.5728643216080402, + "recall@300": 0.6984924623115578, + "recall@500": 0.8844221105527639, + "recall@1000": 0.9597989949748744, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.49698443151337335, + "ndcg_full@5": 0.49698443151337335, + "ndcg@10": 0.5768294147993828, + "ndcg_full@10": 0.5768294147993828, + "ndcg@20": 0.5495831957128068, + "ndcg_full@20": 0.5495831957128068, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02631578947368421, + "recall@50": 0.07368421052631578, + "recall@100": 0.12631578947368421, + "recall@200": 0.2578947368421053, + "recall@300": 0.42105263157894735, + "recall@500": 0.6947368421052632, + "recall@1000": 0.9315789473684211, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7808860545751688, + "ndcg_full@10": 0.7808860545751688, + "ndcg@20": 0.6362873618234165, + "ndcg_full@20": 0.6362873618234165, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03272727272727273, + "recall@50": 0.14181818181818182, + "recall@100": 0.26545454545454544, + "recall@200": 0.44363636363636366, + "recall@300": 0.5381818181818182, + "recall@500": 0.6327272727272727, + "recall@1000": 0.7527272727272727, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.08479005131840403, + "ndcg_full@5": 0.08479005131840403, + "ndcg@10": 0.22765193703968395, + "ndcg_full@10": 0.22765193703968395, + "ndcg@20": 0.23566654395864134, + "ndcg_full@20": 0.23566654395864134, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.1, + "graded_P@10": 0.3 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06818181818181818, + "recall@50": 0.20454545454545456, + "recall@100": 0.29545454545454547, + "recall@200": 0.4772727272727273, + "recall@300": 0.5909090909090909, + "recall@500": 0.6818181818181818, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.1780615783628922, + "ndcg_full@10": 0.1780615783628922, + "ndcg@20": 0.18631598589137718, + "ndcg_full@20": 0.15089593393121858, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 1407, + "num_ranked": 222, + "recall@10": 0.029556650246305417, + "recall@50": 0.13793103448275862, + "recall@100": 0.20689655172413793, + "recall@200": 0.37438423645320196, + "recall@300": 0.5320197044334976, + "recall@500": 0.8423645320197044, + "recall@1000": 0.9605911330049262, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.456899187845258, + "ndcg_full@10": 0.456899187845258, + "ndcg@20": 0.3227768003824389, + "ndcg_full@20": 0.3227768003824389, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.035398230088495575, + "recall@50": 0.1415929203539823, + "recall@100": 0.23893805309734514, + "recall@200": 0.4247787610619469, + "recall@300": 0.584070796460177, + "recall@500": 0.7699115044247787, + "recall@1000": 0.8495575221238938, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.12997650650805187, + "ndcg_full@10": 0.12997650650805187, + "ndcg@20": 0.22952140585978442, + "ndcg_full@20": 0.22952140585978442, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 1796, + "num_ranked": 40, + "recall@10": 0.14583333333333334, + "recall@50": 0.25, + "recall@100": 0.25, + "recall@200": 0.2916666666666667, + "recall@300": 0.2916666666666667, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.37438703096168285, + "ndcg_full@10": 0.37438703096168285, + "ndcg@20": 0.6422869342174823, + "ndcg_full@20": 0.5217389440399857, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06369426751592357, + "recall@50": 0.2611464968152866, + "recall@100": 0.46496815286624205, + "recall@200": 0.6815286624203821, + "recall@300": 0.7834394904458599, + "recall@500": 0.8152866242038217, + "recall@1000": 0.821656050955414, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.39168308588041617, + "ndcg_full@10": 0.39168308588041617, + "ndcg@20": 0.47941617689158617, + "ndcg_full@20": 0.47941617689158617, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06097560975609756, + "recall@50": 0.23780487804878048, + "recall@100": 0.3597560975609756, + "recall@200": 0.45121951219512196, + "recall@300": 0.4878048780487805, + "recall@500": 0.5792682926829268, + "recall@1000": 0.6707317073170732, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4808124874377669, + "ndcg_full@5": 0.4808124874377669, + "ndcg@10": 0.34673001588060237, + "ndcg_full@10": 0.34673001588060237, + "ndcg@20": 0.31819083215173055, + "ndcg_full@20": 0.31819083215173055, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.0, + "recall@200": 0.0379746835443038, + "recall@300": 0.06329113924050633, + "recall@500": 0.06329113924050633, + "recall@1000": 0.10126582278481013, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.06394152317298338, + "ndcg_full@5": 0.046212237892508036, + "ndcg@10": 0.16609062739240826, + "ndcg_full@10": 0.07789651974672891, + "ndcg@20": 0.28777637488779567, + "ndcg_full@20": 0.08710338958069572, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 248, + "recall@10": 0.04329004329004329, + "recall@50": 0.16883116883116883, + "recall@100": 0.2813852813852814, + "recall@200": 0.5238095238095238, + "recall@300": 0.7489177489177489, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.10699313236726378, + "ndcg_full@5": 0.10699313236726378, + "ndcg@10": 0.28033067769281533, + "ndcg_full@10": 0.28033067769281533, + "ndcg@20": 0.3575897780586847, + "ndcg_full@20": 0.3575897780586847, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03260869565217391, + "recall@50": 0.07065217391304347, + "recall@100": 0.10869565217391304, + "recall@200": 0.20652173913043478, + "recall@300": 0.30978260869565216, + "recall@500": 0.5597826086956522, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.27454788926472085, + "ndcg_full@10": 0.27454788926472085, + "ndcg@20": 0.2489879778205072, + "ndcg_full@20": 0.2489879778205072, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 1324, + "num_ranked": 159, + "recall@10": 0.0989010989010989, + "recall@50": 0.24175824175824176, + "recall@100": 0.25274725274725274, + "recall@200": 0.25274725274725274, + "recall@300": 0.25274725274725274, + "recall@500": 0.25274725274725274, + "recall@1000": 0.2857142857142857, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.34960740992542505, + "ndcg_full@5": 0.34960740992542505, + "ndcg@10": 0.3688903444482198, + "ndcg_full@10": 0.3688903444482198, + "ndcg@20": 0.36555938973102176, + "ndcg_full@20": 0.34378136578068574, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.13636363636363635, + "recall@50": 0.6060606060606061, + "recall@100": 0.8787878787878788, + "recall@200": 0.9242424242424242, + "recall@300": 0.9696969696969697, + "recall@500": 0.9848484848484849, + "recall@1000": 0.9848484848484849, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.08923741494505459, + "ndcg_full@20": 0.0877945221921463, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 90, + "recall@10": 0.6153846153846154, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.4690000933206748, + "ndcg_full@10": 0.4690000933206748, + "ndcg@20": 0.5944714132411807, + "ndcg_full@20": 0.5944714132411807, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 1784, + "num_ranked": 250, + "recall@10": 0.02346041055718475, + "recall@50": 0.11730205278592376, + "recall@100": 0.21994134897360704, + "recall@200": 0.4310850439882698, + "recall@300": 0.6363636363636364, + "recall@500": 0.8768328445747801, + "recall@1000": 0.9618768328445748, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.47822575712515014, + "ndcg_full@10": 0.47822575712515014, + "ndcg@20": 0.5768683265960067, + "ndcg_full@20": 0.5768683265960067, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + } + } + } + } + }, + { + "name": "iimedical_medcpt", + "results_dir": "iimedical_medcpt/results_trec21", + "evaluation_input_sha256": "10e37480beb5050ede0cc142d17acf20a57d4a3cf2f7a2a88f4181a24519e373", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.07726188662520815, + "recall@50": 0.2746256904930421, + "recall@100": 0.4211739931883072, + "recall@200": 0.5847221382572916, + "recall@300": 0.6719627150675345, + "recall@500": 0.7591384771954316, + "recall@1000": 0.8285043615113159, + "ndcg@5": 0.7494473920472198, + "ndcg@10": 0.7539097396513507, + "ndcg@20": 0.753630952064058, + "ndcg_full@5": 0.7481148308461779, + "ndcg_full@10": 0.7457878532835107, + "ndcg_full@20": 0.7291379637392056, + "P@10(rel>=1)": 0.808, + "P@10(eligible)": 0.6666666666666666, + "graded_P@10": 0.7373333333333335, + "unjudged_fraction@10": 0.2960000000000001 + }, + "median": { + "recall@10": 0.05228758169934641, + "recall@50": 0.2222222222222222, + "recall@100": 0.3584905660377358, + "recall@200": 0.5555555555555556, + "recall@300": 0.678391959798995, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8695652173913043, + "ndcg@5": 0.830419897363192, + "ndcg@10": 0.8135221979535759, + "ndcg@20": 0.8029812255711771, + "ndcg_full@5": 0.830419897363192, + "ndcg_full@10": 0.8116978598270739, + "ndcg_full@20": 0.7897479123727567, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8, + "unjudged_fraction@10": 0.3 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05917159763313609, + "recall@50": 0.27218934911242604, + "recall@100": 0.48520710059171596, + "recall@200": 0.7810650887573964, + "recall@300": 0.8284023668639053, + "recall@500": 0.8579881656804734, + "recall@1000": 0.8757396449704142, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7012236839470684, + "ndcg_full@5": 0.7012236839470684, + "ndcg@10": 0.7694326535022453, + "ndcg_full@10": 0.7694326535022453, + "ndcg@20": 0.6407928797474942, + "ndcg_full@20": 0.6407928797474942, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02962962962962963, + "recall@50": 0.14074074074074075, + "recall@100": 0.2518518518518518, + "recall@200": 0.44074074074074077, + "recall@300": 0.5370370370370371, + "recall@500": 0.7148148148148148, + "recall@1000": 0.8555555555555555, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.9354500338834016, + "ndcg_full@20": 0.9354500338834016, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.10714285714285714, + "recall@50": 0.40476190476190477, + "recall@100": 0.5476190476190477, + "recall@200": 0.6428571428571429, + "recall@300": 0.7619047619047619, + "recall@500": 0.8452380952380952, + "recall@1000": 0.9404761904761905, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.6920237987577761, + "ndcg_full@10": 0.6920237987577761, + "ndcg@20": 0.600736647422733, + "ndcg_full@20": 0.600736647422733, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.18947368421052632, + "recall@100": 0.2631578947368421, + "recall@200": 0.49473684210526314, + "recall@300": 0.6526315789473685, + "recall@500": 0.8210526315789474, + "recall@1000": 0.9263157894736842, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.43079934459036184, + "ndcg_full@10": 0.43079934459036184, + "ndcg@20": 0.4029615933722065, + "ndcg_full@20": 0.4029615933722065, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 229, + "recall@10": 0.004975124378109453, + "recall@50": 0.06467661691542288, + "recall@100": 0.12437810945273632, + "recall@200": 0.23880597014925373, + "recall@300": 0.3333333333333333, + "recall@500": 0.4925373134328358, + "recall@1000": 0.7313432835820896, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.5309999066793252, + "ndcg_full@10": 0.5309999066793252, + "ndcg@20": 0.6973226533461705, + "ndcg_full@20": 0.6973226533461705, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.3380952380952381, + "recall@200": 0.5285714285714286, + "recall@300": 0.6952380952380952, + "recall@500": 0.819047619047619, + "recall@1000": 0.9095238095238095, + "unjudged_fraction@10": 0.7, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9681896059005243, + "ndcg_full@10": 0.9681896059005243, + "ndcg@20": 0.9127751474068162, + "ndcg_full@20": 0.9127751474068162, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031055900621118012, + "recall@50": 0.09316770186335403, + "recall@100": 0.14285714285714285, + "recall@200": 0.2236024844720497, + "recall@300": 0.32298136645962733, + "recall@500": 0.5838509316770186, + "recall@1000": 0.9130434782608695, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8521705090845474, + "ndcg_full@10": 0.8521705090845474, + "ndcg@20": 0.8000517213515904, + "ndcg_full@20": 0.8000517213515904, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0457516339869281, + "recall@50": 0.2222222222222222, + "recall@100": 0.4117647058823529, + "recall@200": 0.7189542483660131, + "recall@300": 0.8169934640522876, + "recall@500": 0.9281045751633987, + "recall@1000": 0.9477124183006536, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8611375561264454, + "ndcg_full@10": 0.8611375561264454, + "ndcg@20": 0.8059874985830235, + "ndcg_full@20": 0.8059874985830235, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 238, + "recall@10": 0.037383177570093455, + "recall@50": 0.2102803738317757, + "recall@100": 0.3925233644859813, + "recall@200": 0.7289719626168224, + "recall@300": 0.9252336448598131, + "recall@500": 0.9392523364485982, + "recall@1000": 0.9485981308411215, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4925683638348177, + "ndcg_full@5": 0.4925683638348177, + "ndcg@10": 0.504154616267091, + "ndcg_full@10": 0.504154616267091, + "ndcg@20": 0.6009492664780393, + "ndcg_full@20": 0.5751145381462109, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.19148936170212766, + "recall@50": 0.8085106382978723, + "recall@100": 0.8297872340425532, + "recall@200": 0.8297872340425532, + "recall@300": 0.8297872340425532, + "recall@500": 0.8297872340425532, + "recall@1000": 0.851063829787234, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6763446997542273, + "ndcg_full@10": 0.6763446997542273, + "ndcg@20": 0.7837538156156325, + "ndcg_full@20": 0.7653245123647533, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.4, + "graded_P@10": 0.6 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 138, + "recall@10": 0.07936507936507936, + "recall@50": 0.36507936507936506, + "recall@100": 0.5634920634920635, + "recall@200": 0.8015873015873016, + "recall@300": 0.8253968253968254, + "recall@500": 0.8412698412698413, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8185565448328912, + "ndcg_full@10": 0.8185565448328912, + "ndcg@20": 0.8111634091008674, + "ndcg_full@20": 0.8111634091008674, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.053691275167785234, + "recall@50": 0.2751677852348993, + "recall@100": 0.5033557046979866, + "recall@200": 0.7516778523489933, + "recall@300": 0.825503355704698, + "recall@500": 0.87248322147651, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8165901947515999, + "ndcg_full@10": 0.8165901947515999, + "ndcg@20": 0.7539767897783569, + "ndcg_full@20": 0.7539767897783569, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 224, + "recall@10": 0.028985507246376812, + "recall@50": 0.12318840579710146, + "recall@100": 0.2608695652173913, + "recall@200": 0.4855072463768116, + "recall@300": 0.5869565217391305, + "recall@500": 0.6521739130434783, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5030155684866267, + "ndcg_full@5": 0.5030155684866267, + "ndcg@10": 0.541805664551458, + "ndcg_full@10": 0.541805664551458, + "ndcg@20": 0.5217201993318236, + "ndcg_full@20": 0.5217201993318236, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.023809523809523808, + "recall@50": 0.08503401360544217, + "recall@100": 0.1598639455782313, + "recall@200": 0.30612244897959184, + "recall@300": 0.41156462585034015, + "recall@500": 0.564625850340136, + "recall@1000": 0.6972789115646258, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.8274043288765652, + "ndcg_full@5": 0.8274043288765652, + "ndcg@10": 0.8181881464655376, + "ndcg_full@10": 0.8181881464655376, + "ndcg@20": 0.7613720786560311, + "ndcg_full@20": 0.7613720786560311, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.026923076923076925, + "recall@50": 0.15, + "recall@100": 0.27307692307692305, + "recall@200": 0.4269230769230769, + "recall@300": 0.5076923076923077, + "recall@500": 0.6038461538461538, + "recall@1000": 0.7269230769230769, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.9446532192903555, + "ndcg_full@10": 0.9446532192903555, + "ndcg@20": 0.8996142469184861, + "ndcg_full@20": 0.8996142469184861, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.014492753623188406, + "recall@50": 0.07246376811594203, + "recall@100": 0.16666666666666666, + "recall@200": 0.2971014492753623, + "recall@300": 0.39855072463768115, + "recall@500": 0.5144927536231884, + "recall@1000": 0.6811594202898551, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.2795888034906985, + "ndcg_full@5": 0.2795888034906985, + "ndcg@10": 0.21324439679384088, + "ndcg_full@10": 0.21324439679384088, + "ndcg@20": 0.2843610829595168, + "ndcg_full@20": 0.2843610829595168, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.1, + "graded_P@10": 0.2 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0398406374501992, + "recall@50": 0.1553784860557769, + "recall@100": 0.3107569721115538, + "recall@200": 0.6215139442231076, + "recall@300": 0.8565737051792829, + "recall@500": 0.9641434262948207, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.6936634693435663, + "ndcg_full@10": 0.6936634693435663, + "ndcg@20": 0.7855816597814829, + "ndcg_full@20": 0.7855816597814829, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 1740, + "num_ranked": 234, + "recall@10": 0.038834951456310676, + "recall@50": 0.17475728155339806, + "recall@100": 0.3155339805825243, + "recall@200": 0.49514563106796117, + "recall@300": 0.6310679611650486, + "recall@500": 0.7378640776699029, + "recall@1000": 0.8883495145631068, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.495629846044788, + "ndcg_full@5": 0.495629846044788, + "ndcg@10": 0.569820227847299, + "ndcg_full@10": 0.569820227847299, + "ndcg@20": 0.7067942316910101, + "ndcg_full@20": 0.6471412321978238, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05228758169934641, + "recall@50": 0.20915032679738563, + "recall@100": 0.3464052287581699, + "recall@200": 0.6013071895424836, + "recall@300": 0.7320261437908496, + "recall@500": 0.8300653594771242, + "recall@1000": 0.9215686274509803, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7849818707050383, + "ndcg_full@10": 0.7849818707050383, + "ndcg@20": 0.7921995899977302, + "ndcg_full@20": 0.7921995899977302, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1, + "recall@50": 0.4, + "recall@100": 0.6714285714285714, + "recall@200": 0.8714285714285714, + "recall@300": 0.9285714285714286, + "recall@500": 0.9428571428571428, + "recall@1000": 0.9428571428571428, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.646910188523972, + "ndcg_full@5": 0.646910188523972, + "ndcg@10": 0.5644529401701474, + "ndcg_full@10": 0.5644529401701474, + "ndcg@20": 0.536574460828698, + "ndcg_full@20": 0.536574460828698, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 104, + "recall@10": 0.05660377358490566, + "recall@50": 0.39622641509433965, + "recall@100": 0.6415094339622641, + "recall@200": 0.8490566037735849, + "recall@300": 0.8867924528301887, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.8274823029254629, + "ndcg_full@20": 0.8274823029254629, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 130, + "recall@10": 0.06569343065693431, + "recall@50": 0.2846715328467153, + "recall@100": 0.5109489051094891, + "recall@200": 0.7372262773722628, + "recall@300": 0.7883211678832117, + "recall@500": 0.8248175182481752, + "recall@1000": 0.8540145985401459, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0000000000000002, + "ndcg_full@5": 1.0000000000000002, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.965906167958209, + "ndcg_full@20": 0.965906167958209, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02247191011235955, + "recall@50": 0.12734082397003746, + "recall@100": 0.2397003745318352, + "recall@200": 0.39325842696629215, + "recall@300": 0.5430711610486891, + "recall@500": 0.7340823970037453, + "recall@1000": 0.8689138576779026, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6725956711234347, + "ndcg_full@5": 0.6725956711234347, + "ndcg@10": 0.7875369930916644, + "ndcg_full@10": 0.7875369930916644, + "ndcg@20": 0.8115840757079397, + "ndcg_full@20": 0.8115840757079397, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05063291139240506, + "recall@50": 0.25949367088607594, + "recall@100": 0.5063291139240507, + "recall@200": 0.7848101265822784, + "recall@300": 0.8987341772151899, + "recall@500": 0.9556962025316456, + "recall@1000": 0.9746835443037974, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.842997893697932, + "ndcg_full@5": 0.842997893697932, + "ndcg@10": 0.8688606737713013, + "ndcg_full@10": 0.8688606737713013, + "ndcg@20": 0.8029812255711771, + "ndcg_full@20": 0.8029812255711771, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.049019607843137254, + "recall@50": 0.16666666666666666, + "recall@100": 0.24509803921568626, + "recall@200": 0.35294117647058826, + "recall@300": 0.45098039215686275, + "recall@500": 0.6764705882352942, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.38697720739102376, + "ndcg_full@5": 0.38697720739102376, + "ndcg@10": 0.38877352428223344, + "ndcg_full@10": 0.38877352428223344, + "ndcg@20": 0.4222248955546446, + "ndcg_full@20": 0.4222248955546446, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 125, + "recall@10": 0.0625, + "recall@50": 0.19375, + "recall@100": 0.33125, + "recall@200": 0.49375, + "recall@300": 0.6, + "recall@500": 0.66875, + "recall@1000": 0.78125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8379352012405591, + "ndcg_full@10": 0.8379352012405591, + "ndcg@20": 0.737664582463361, + "ndcg_full@20": 0.737664582463361, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.15450643776824036, + "recall@100": 0.26180257510729615, + "recall@200": 0.45493562231759654, + "recall@300": 0.592274678111588, + "recall@500": 0.8283261802575107, + "recall@1000": 0.9399141630901288, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.40489548569516803, + "ndcg_full@5": 0.40489548569516803, + "ndcg@10": 0.5067760328601512, + "ndcg_full@10": 0.5067760328601512, + "ndcg@20": 0.4976695686501552, + "ndcg_full@20": 0.4976695686501552, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018518518518518517, + "recall@50": 0.10802469135802469, + "recall@100": 0.22839506172839505, + "recall@200": 0.4351851851851852, + "recall@300": 0.5987654320987654, + "recall@500": 0.8395061728395061, + "recall@1000": 0.9382716049382716, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8824086793035105, + "ndcg_full@5": 0.8824086793035105, + "ndcg@10": 0.9040917117704057, + "ndcg_full@10": 0.9040917117704057, + "ndcg@20": 0.863227613755116, + "ndcg_full@20": 0.863227613755116, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03488372093023256, + "recall@50": 0.13953488372093023, + "recall@100": 0.26744186046511625, + "recall@200": 0.4496124031007752, + "recall@300": 0.6124031007751938, + "recall@500": 0.7751937984496124, + "recall@1000": 0.8062015503875969, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0000000000000002, + "ndcg_full@5": 1.0000000000000002, + "ndcg@10": 0.9681896059005243, + "ndcg_full@10": 0.9681896059005243, + "ndcg@20": 0.8346985755816003, + "ndcg_full@20": 0.8346985755816003, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 43, + "recall@10": 0.06, + "recall@50": 0.2866666666666667, + "recall@100": 0.44666666666666666, + "recall@200": 0.5866666666666667, + "recall@300": 0.6333333333333333, + "recall@500": 0.7266666666666667, + "recall@1000": 0.8133333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.758317633438156, + "ndcg_full@10": 0.758317633438156, + "ndcg@20": 0.8163626666780033, + "ndcg_full@20": 0.7897479123727567, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 156, + "recall@10": 0.023255813953488372, + "recall@50": 0.13953488372093023, + "recall@100": 0.27325581395348836, + "recall@200": 0.5, + "recall@300": 0.6104651162790697, + "recall@500": 0.6918604651162791, + "recall@1000": 0.7441860465116279, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3421757737602428, + "ndcg_full@5": 0.3421757737602428, + "ndcg@10": 0.38011185155442145, + "ndcg_full@10": 0.38011185155442145, + "ndcg@20": 0.32618693268980203, + "ndcg_full@20": 0.32618693268980203, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 128, + "recall@10": 0.09433962264150944, + "recall@50": 0.24528301886792453, + "recall@100": 0.3584905660377358, + "recall@200": 0.41509433962264153, + "recall@300": 0.4528301886792453, + "recall@500": 0.4716981132075472, + "recall@1000": 0.5094339622641509, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.6848374495230163, + "ndcg_full@10": 0.6848374495230163, + "ndcg@20": 0.6584299829203382, + "ndcg_full@20": 0.5122319115320215, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 246, + "recall@10": 0.028708133971291867, + "recall@50": 0.11004784688995216, + "recall@100": 0.22488038277511962, + "recall@200": 0.3588516746411483, + "recall@300": 0.45933014354066987, + "recall@500": 0.5645933014354066, + "recall@1000": 0.6889952153110048, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.804810999209339, + "ndcg_full@10": 0.804810999209339, + "ndcg@20": 0.8740313921111853, + "ndcg_full@20": 0.8740313921111853, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02, + "recall@50": 0.14666666666666667, + "recall@100": 0.24, + "recall@200": 0.41333333333333333, + "recall@300": 0.52, + "recall@500": 0.72, + "recall@1000": 0.76, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8135221979535759, + "ndcg_full@10": 0.8135221979535759, + "ndcg@20": 0.8016476101254628, + "ndcg_full@20": 0.8016476101254628, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 1764, + "num_ranked": 91, + "recall@10": 0.05747126436781609, + "recall@50": 0.2471264367816092, + "recall@100": 0.45977011494252873, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.9173456621324145, + "ndcg_full@20": 0.9173456621324145, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.022653721682847898, + "recall@50": 0.09061488673139159, + "recall@100": 0.15210355987055016, + "recall@200": 0.2815533980582524, + "recall@300": 0.34627831715210355, + "recall@500": 0.459546925566343, + "recall@1000": 0.6407766990291263, + "unjudged_fraction@10": 0.9, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9668728882728057, + "ndcg_full@10": 0.9668728882728057, + "ndcg@20": 0.9599674685592627, + "ndcg_full@20": 0.9599674685592627, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 249, + "recall@10": 0.036290322580645164, + "recall@50": 0.1935483870967742, + "recall@100": 0.36693548387096775, + "recall@200": 0.6370967741935484, + "recall@300": 0.8225806451612904, + "recall@500": 0.9475806451612904, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.982244995045793, + "ndcg_full@20": 0.982244995045793, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 110, + "recall@10": 0.12345679012345678, + "recall@50": 0.5802469135802469, + "recall@100": 0.9629629629629629, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.913166664326004, + "ndcg_full@10": 0.913166664326004, + "ndcg@20": 0.8914191043617355, + "ndcg_full@20": 0.8914191043617355, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 160, + "recall@10": 0.11864406779661017, + "recall@50": 0.559322033898305, + "recall@100": 0.8305084745762712, + "recall@200": 0.8983050847457628, + "recall@300": 0.9491525423728814, + "recall@500": 0.9491525423728814, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.42696582507864683, + "ndcg_full@5": 0.42696582507864683, + "ndcg@10": 0.6217779513294063, + "ndcg_full@10": 0.6217779513294063, + "ndcg@20": 0.7236784032811977, + "ndcg_full@20": 0.7236784032811977, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 81, + "recall@10": 0.46153846153846156, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8347900513184041, + "ndcg_full@5": 0.8347900513184041, + "ndcg@10": 0.8927900477415585, + "ndcg_full@10": 0.8927900477415585, + "ndcg@20": 0.8609641350016063, + "ndcg_full@20": 0.8151499634730085, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032467532467532464, + "recall@50": 0.12012987012987013, + "recall@100": 0.2305194805194805, + "recall@200": 0.42207792207792205, + "recall@300": 0.5941558441558441, + "recall@500": 0.8474025974025974, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.9178902068468361, + "ndcg_full@10": 0.9178902068468361, + "ndcg@20": 0.9225577973666869, + "ndcg_full@20": 0.9225577973666869, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.08450704225352113, + "recall@50": 0.30985915492957744, + "recall@100": 0.4507042253521127, + "recall@200": 0.5352112676056338, + "recall@300": 0.6619718309859155, + "recall@500": 0.7183098591549296, + "recall@1000": 0.7605633802816901, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8934267498394676, + "ndcg_full@20": 0.8934267498394676, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 174, + "recall@10": 0.06451612903225806, + "recall@50": 0.25161290322580643, + "recall@100": 0.41935483870967744, + "recall@200": 0.6967741935483871, + "recall@300": 0.8, + "recall@500": 0.864516129032258, + "recall@1000": 0.9096774193548387, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7456298460447879, + "ndcg_full@5": 0.7456298460447879, + "ndcg@10": 0.782998012810738, + "ndcg_full@10": 0.782998012810738, + "ndcg@20": 0.7796215387666382, + "ndcg_full@20": 0.7796215387666382, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 126, + "recall@10": 0.05128205128205128, + "recall@50": 0.24102564102564103, + "recall@100": 0.4256410256410256, + "recall@200": 0.6974358974358974, + "recall@300": 0.7230769230769231, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7384615384615385, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5804751699305776, + "ndcg_full@5": 0.49999999999999994, + "ndcg@10": 0.6807351507646947, + "ndcg_full@10": 0.5789282245622048, + "ndcg@20": 0.8253241523896192, + "ndcg_full@20": 0.7316828530991122, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.5 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 211, + "recall@10": 0.053763440860215055, + "recall@50": 0.24193548387096775, + "recall@100": 0.46236559139784944, + "recall@200": 0.8225806451612904, + "recall@300": 0.8817204301075269, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503199, + "ndcg_full@10": 0.9633180389503199, + "ndcg@20": 0.9589515737116228, + "ndcg_full@20": 0.9589515737116228, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.08108108108108109, + "recall@50": 0.3333333333333333, + "recall@100": 0.5945945945945946, + "recall@200": 0.9369369369369369, + "recall@300": 0.990990990990991, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.5550616925716014, + "ndcg_full@10": 0.5550616925716014, + "ndcg@20": 0.6446636802230978, + "ndcg_full@20": 0.6446636802230978, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03759398496240601, + "recall@50": 0.14661654135338345, + "recall@100": 0.23308270676691728, + "recall@200": 0.43609022556390975, + "recall@300": 0.631578947368421, + "recall@500": 0.8007518796992481, + "recall@1000": 0.9210526315789473, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.9537877621074919, + "ndcg_full@5": 0.9537877621074919, + "ndcg@10": 0.9569450415016922, + "ndcg_full@10": 0.9569450415016922, + "ndcg@20": 0.9159849626432935, + "ndcg_full@20": 0.9159849626432935, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 68, + "recall@10": 0.09090909090909091, + "recall@50": 0.5151515151515151, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.8333333333333334, + "recall@1000": 0.9090909090909091, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8581437227514841, + "ndcg_full@10": 0.8581437227514841, + "ndcg@20": 0.9084505904897269, + "ndcg_full@20": 0.9084505904897269, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 1567, + "num_ranked": 114, + "recall@10": 0.12987012987012986, + "recall@50": 0.5194805194805194, + "recall@100": 0.7142857142857143, + "recall@200": 0.7792207792207793, + "recall@300": 0.7922077922077922, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8568270051237654, + "ndcg_full@10": 0.8568270051237654, + "ndcg@20": 0.8512610111774833, + "ndcg_full@20": 0.8512610111774833, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 60, + "recall@10": 0.391304347826087, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.782608695652174, + "recall@300": 0.782608695652174, + "recall@500": 0.782608695652174, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6278201736892663, + "ndcg_full@10": 0.6078489665406095, + "ndcg@20": 0.6529515740302835, + "ndcg_full@20": 0.5347063997843936, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.11267605633802817, + "recall@50": 0.4225352112676056, + "recall@100": 0.676056338028169, + "recall@200": 0.8732394366197183, + "recall@300": 0.8732394366197183, + "recall@500": 0.9014084507042254, + "recall@1000": 0.9577464788732394, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8234550942619859, + "ndcg_full@5": 0.8234550942619859, + "ndcg@10": 0.8529656832892831, + "ndcg_full@10": 0.8529656832892831, + "ndcg@20": 0.8570295452581681, + "ndcg_full@20": 0.8001640385599554, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 208, + "recall@10": 0.08823529411764706, + "recall@50": 0.39705882352941174, + "recall@100": 0.6323529411764706, + "recall@200": 0.7352941176470589, + "recall@300": 0.7794117647058824, + "recall@500": 0.7941176470588235, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.4317939459725493, + "ndcg_full@10": 0.4317939459725493, + "ndcg@20": 0.5316998326563354, + "ndcg_full@20": 0.5316998326563354, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 148, + "recall@10": 0.15217391304347827, + "recall@50": 0.4782608695652174, + "recall@100": 0.6086956521739131, + "recall@200": 0.6521739130434783, + "recall@300": 0.717391304347826, + "recall@500": 0.7608695652173914, + "recall@1000": 0.7608695652173914, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.7858634295666242, + "ndcg_full@20": 0.7858634295666242, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 122, + "recall@10": 0.12, + "recall@50": 0.5066666666666667, + "recall@100": 0.8666666666666667, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5222030810488597, + "ndcg_full@5": 0.5222030810488597, + "ndcg@10": 0.47585877832649226, + "ndcg_full@10": 0.47585877832649226, + "ndcg@20": 0.44875023454113666, + "ndcg_full@20": 0.44875023454113666, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 172, + "recall@10": 0.20512820512820512, + "recall@50": 0.5128205128205128, + "recall@100": 0.717948717948718, + "recall@200": 0.7435897435897436, + "recall@300": 0.7435897435897436, + "recall@500": 0.7435897435897436, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.06362078819895171, + "ndcg_full@10": 0.06362078819895171, + "ndcg@20": 0.10855924302758507, + "ndcg_full@20": 0.10855924302758507, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 187, + "recall@10": 0.09302325581395349, + "recall@50": 0.36046511627906974, + "recall@100": 0.6511627906976745, + "recall@200": 0.9767441860465116, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6621484664716257, + "ndcg_full@5": 0.6621484664716257, + "ndcg@10": 0.6103386248132282, + "ndcg_full@10": 0.6103386248132282, + "ndcg@20": 0.6038129293688367, + "ndcg_full@20": 0.6038129293688367, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 113, + "recall@10": 0.06172839506172839, + "recall@50": 0.17901234567901234, + "recall@100": 0.3271604938271605, + "recall@200": 0.5555555555555556, + "recall@300": 0.691358024691358, + "recall@500": 0.7839506172839507, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.948793096849316, + "ndcg_full@20": 0.948793096849316, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 103, + "recall@10": 0.07352941176470588, + "recall@50": 0.3235294117647059, + "recall@100": 0.5661764705882353, + "recall@200": 0.6764705882352942, + "recall@300": 0.6764705882352942, + "recall@500": 0.6764705882352942, + "recall@1000": 0.6838235294117647, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.9445453145289904, + "ndcg_full@5": 0.9445453145289904, + "ndcg@10": 0.9197750209948032, + "ndcg_full@10": 0.9197750209948032, + "ndcg@20": 0.9308503316697273, + "ndcg_full@20": 0.9308503316697273, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1578, + "num_ranked": 187, + "recall@10": 0.02564102564102564, + "recall@50": 0.1225071225071225, + "recall@100": 0.23931623931623933, + "recall@200": 0.46438746438746437, + "recall@300": 0.6068376068376068, + "recall@500": 0.6239316239316239, + "recall@1000": 0.6239316239316239, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 1.0000000000000002, + "ndcg_full@20": 1.0000000000000002, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04020100502512563, + "recall@50": 0.20100502512562815, + "recall@100": 0.37185929648241206, + "recall@200": 0.542713567839196, + "recall@300": 0.678391959798995, + "recall@500": 0.8442211055276382, + "recall@1000": 0.9547738693467337, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.76110154052443, + "ndcg_full@5": 0.76110154052443, + "ndcg@10": 0.7739035868514635, + "ndcg_full@10": 0.7739035868514635, + "ndcg@20": 0.8193345846511568, + "ndcg_full@20": 0.8193345846511568, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.05263157894736842, + "recall@100": 0.1368421052631579, + "recall@200": 0.28421052631578947, + "recall@300": 0.4473684210526316, + "recall@500": 0.6736842105263158, + "recall@1000": 0.9052631578947369, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8974416663360282, + "ndcg_full@10": 0.8974416663360282, + "ndcg@20": 0.8290049861336718, + "ndcg_full@20": 0.8290049861336718, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.03272727272727273, + "recall@50": 0.16, + "recall@100": 0.28363636363636363, + "recall@200": 0.4763636363636364, + "recall@300": 0.5745454545454546, + "recall@500": 0.6872727272727273, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7515077842433132, + "ndcg_full@5": 0.7515077842433132, + "ndcg@10": 0.8030467689622777, + "ndcg_full@10": 0.8030467689622777, + "ndcg@20": 0.7626518712490503, + "ndcg_full@20": 0.7626518712490503, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 141, + "recall@10": 0.06818181818181818, + "recall@50": 0.29545454545454547, + "recall@100": 0.38636363636363635, + "recall@200": 0.5681818181818182, + "recall@300": 0.7727272727272727, + "recall@500": 0.8636363636363636, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.6578242262397573, + "ndcg_full@5": 0.6578242262397573, + "ndcg@10": 0.5002468335245788, + "ndcg_full@10": 0.5002468335245788, + "ndcg@20": 0.5677834039443346, + "ndcg_full@20": 0.4503509378026054, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.024630541871921183, + "recall@50": 0.10344827586206896, + "recall@100": 0.21674876847290642, + "recall@200": 0.4088669950738916, + "recall@300": 0.541871921182266, + "recall@500": 0.8275862068965517, + "recall@1000": 0.9753694581280788, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.8116978598270739, + "ndcg_full@10": 0.8116978598270739, + "ndcg@20": 0.7468756648686027, + "ndcg_full@20": 0.7225263001457717, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 109, + "recall@10": 0.017699115044247787, + "recall@50": 0.02654867256637168, + "recall@100": 0.05309734513274336, + "recall@200": 0.18584070796460178, + "recall@300": 0.30973451327433627, + "recall@500": 0.49557522123893805, + "recall@1000": 0.7610619469026548, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.2967403475026077, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.2967403475026077, + "ndcg_full@10": 0.17993148201117176, + "ndcg@20": 0.5199511419221059, + "ndcg_full@20": 0.20346985777009874, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 29, + "recall@10": 0.16666666666666666, + "recall@50": 0.25, + "recall@100": 0.2708333333333333, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9202054614375518, + "ndcg_full@10": 0.800693766409882, + "ndcg@20": 0.9808642748886733, + "ndcg_full@20": 0.6537784831520479, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.06369426751592357, + "recall@50": 0.25477707006369427, + "recall@100": 0.4585987261146497, + "recall@200": 0.7261146496815286, + "recall@300": 0.802547770700637, + "recall@500": 0.821656050955414, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.8558289404669284, + "ndcg_full@10": 0.8558289404669284, + "ndcg@20": 0.873920661444281, + "ndcg_full@20": 0.873920661444281, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.06097560975609756, + "recall@50": 0.25609756097560976, + "recall@100": 0.3475609756097561, + "recall@200": 0.4634146341463415, + "recall@300": 0.5121951219512195, + "recall@500": 0.6524390243902439, + "recall@1000": 0.7439024390243902, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6445146518760496, + "ndcg_full@5": 0.6445146518760496, + "ndcg@10": 0.7534091359847971, + "ndcg_full@10": 0.7534091359847971, + "ndcg@20": 0.7448098585669178, + "ndcg_full@20": 0.7448098585669178, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 201, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.0, + "recall@200": 0.0759493670886076, + "recall@300": 0.10126582278481013, + "recall@500": 0.12658227848101267, + "recall@1000": 0.189873417721519, + "unjudged_fraction@10": 1.0, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.2890648263178879, + "ndcg_full@10": 0.06362078819895171, + "ndcg@20": 0.2890648263178879, + "ndcg_full@20": 0.04105877938688906, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03896103896103896, + "recall@50": 0.1645021645021645, + "recall@100": 0.3116883116883117, + "recall@200": 0.5670995670995671, + "recall@300": 0.8095238095238095, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8943654543139561, + "ndcg_full@10": 0.8943654543139561, + "ndcg@20": 0.8700657339424483, + "ndcg_full@20": 0.8700657339424483, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.016304347826086956, + "recall@50": 0.1358695652173913, + "recall@100": 0.24456521739130435, + "recall@200": 0.3858695652173913, + "recall@300": 0.4782608695652174, + "recall@500": 0.625, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8905583796478245, + "ndcg_full@20": 0.8905583796478245, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 69, + "recall@10": 0.0989010989010989, + "recall@50": 0.23076923076923078, + "recall@100": 0.24175824175824176, + "recall@200": 0.25274725274725274, + "recall@300": 0.25274725274725274, + "recall@500": 0.26373626373626374, + "recall@1000": 0.2967032967032967, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.6782122407403123, + "ndcg_full@10": 0.6782122407403123, + "ndcg@20": 0.7731252716519106, + "ndcg_full@20": 0.7108727715464419, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 115, + "recall@10": 0.13636363636363635, + "recall@50": 0.6060606060606061, + "recall@100": 0.8181818181818182, + "recall@200": 0.9393939393939394, + "recall@300": 0.9545454545454546, + "recall@500": 0.9696969696969697, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.94497705842548, + "ndcg_full@10": 0.94497705842548, + "ndcg@20": 0.898265191797176, + "ndcg_full@20": 0.8386636231423563, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 79, + "recall@10": 0.5384615384615384, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7765732350040719, + "ndcg_full@5": 0.7765732350040719, + "ndcg@10": 0.8047289674796688, + "ndcg_full@10": 0.7791302218808763, + "ndcg@20": 0.8363961132830305, + "ndcg_full@20": 0.7459270198962575, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.10263929618768329, + "recall@100": 0.19648093841642228, + "recall@200": 0.3870967741935484, + "recall@300": 0.5454545454545454, + "recall@500": 0.8357771260997068, + "recall@1000": 0.9413489736070382, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503199, + "ndcg_full@10": 0.9633180389503199, + "ndcg@20": 0.9412407995940463, + "ndcg_full@20": 0.9412407995940463, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 75, + "num_queries_ranked": 75, + "mean": { + "recall@10": 0.07726188662520815, + "recall@50": 0.2746256904930421, + "recall@100": 0.4211739931883072, + "recall@200": 0.5847221382572916, + "recall@300": 0.6719627150675345, + "recall@500": 0.7591384771954316, + "recall@1000": 0.8285043615113159, + "ndcg@5": 0.5443532657862635, + "ndcg@10": 0.5457336226585257, + "ndcg@20": 0.5385372310307441, + "ndcg_full@5": 0.5430207045852218, + "ndcg_full@10": 0.5408349455612907, + "ndcg_full@20": 0.5229422157749569, + "P@10(rel>=1)": 0.5839999999999999, + "P@10(eligible)": 0.48533333333333323, + "graded_P@10": 0.5346666666666666, + "unjudged_fraction@10": 0.2960000000000001 + }, + "median": { + "recall@10": 0.05228758169934641, + "recall@50": 0.2222222222222222, + "recall@100": 0.3584905660377358, + "recall@200": 0.5555555555555556, + "recall@300": 0.678391959798995, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8695652173913043, + "ndcg@5": 0.5863642063944079, + "ndcg@10": 0.5797016289380371, + "ndcg@20": 0.5795710185298536, + "ndcg_full@5": 0.5863642063944079, + "ndcg_full@10": 0.5730379245414764, + "ndcg_full@20": 0.5589267633918229, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55, + "unjudged_fraction@10": 0.3 + }, + "per_query": { + "trec-20211": { + "num_relevant": 169, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05917159763313609, + "recall@50": 0.27218934911242604, + "recall@100": 0.48520710059171596, + "recall@200": 0.7810650887573964, + "recall@300": 0.8284023668639053, + "recall@500": 0.8579881656804734, + "recall@1000": 0.8757396449704142, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5551553341043619, + "ndcg_full@5": 0.5551553341043619, + "ndcg@10": 0.43865609449289644, + "ndcg_full@10": 0.43865609449289644, + "ndcg@20": 0.4496337173381476, + "ndcg_full@20": 0.4496337173381476, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-20212": { + "num_relevant": 270, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02962962962962963, + "recall@50": 0.14074074074074075, + "recall@100": 0.2518518518518518, + "recall@200": 0.44074074074074077, + "recall@300": 0.5370370370370371, + "recall@500": 0.7148148148148148, + "recall@1000": 0.8555555555555555, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.5647069392995312, + "ndcg_full@10": 0.5647069392995312, + "ndcg@20": 0.6285426087711155, + "ndcg_full@20": 0.6285426087711155, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-20213": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.10714285714285714, + "recall@50": 0.40476190476190477, + "recall@100": 0.5476190476190477, + "recall@200": 0.6428571428571429, + "recall@300": 0.7619047619047619, + "recall@500": 0.8452380952380952, + "recall@1000": 0.9404761904761905, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.6659870905314846, + "ndcg_full@10": 0.6659870905314846, + "ndcg@20": 0.5795710185298536, + "ndcg_full@20": 0.5795710185298536, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20214": { + "num_relevant": 95, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.18947368421052632, + "recall@100": 0.2631578947368421, + "recall@200": 0.49473684210526314, + "recall@300": 0.6526315789473685, + "recall@500": 0.8210526315789474, + "recall@1000": 0.9263157894736842, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.21599512883074581, + "ndcg_full@5": 0.21599512883074581, + "ndcg@10": 0.20378684741509273, + "ndcg_full@10": 0.20378684741509273, + "ndcg@20": 0.21746004640507852, + "ndcg_full@20": 0.21746004640507852, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-20215": { + "num_relevant": 201, + "num_retrieved": 2000, + "num_ranked": 229, + "recall@10": 0.004975124378109453, + "recall@50": 0.06467661691542288, + "recall@100": 0.12437810945273632, + "recall@200": 0.23880597014925373, + "recall@300": 0.3333333333333333, + "recall@500": 0.4925373134328358, + "recall@1000": 0.7313432835820896, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.06625422345438903, + "ndcg_full@10": 0.06625422345438903, + "ndcg@20": 0.0782683224234459, + "ndcg_full@20": 0.0782683224234459, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-20216": { + "num_relevant": 210, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.3380952380952381, + "recall@200": 0.5285714285714286, + "recall@300": 0.6952380952380952, + "recall@500": 0.819047619047619, + "recall@1000": 0.9095238095238095, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.4440973278132557, + "ndcg_full@10": 0.4440973278132557, + "ndcg@20": 0.5041539813973969, + "ndcg_full@20": 0.5041539813973969, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-20217": { + "num_relevant": 161, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031055900621118012, + "recall@50": 0.09316770186335403, + "recall@100": 0.14285714285714285, + "recall@200": 0.2236024844720497, + "recall@300": 0.32298136645962733, + "recall@500": 0.5838509316770186, + "recall@1000": 0.9130434782608695, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.646910188523972, + "ndcg_full@5": 0.646910188523972, + "ndcg@10": 0.5715626388151184, + "ndcg_full@10": 0.5715626388151184, + "ndcg@20": 0.4565728794065688, + "ndcg_full@20": 0.4565728794065688, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20218": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0457516339869281, + "recall@50": 0.2222222222222222, + "recall@100": 0.4117647058823529, + "recall@200": 0.7189542483660131, + "recall@300": 0.8169934640522876, + "recall@500": 0.9281045751633987, + "recall@1000": 0.9477124183006536, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.7510916729774053, + "ndcg_full@10": 0.7510916729774053, + "ndcg@20": 0.6628593827035717, + "ndcg_full@20": 0.6628593827035717, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20219": { + "num_relevant": 214, + "num_retrieved": 2000, + "num_ranked": 238, + "recall@10": 0.037383177570093455, + "recall@50": 0.2102803738317757, + "recall@100": 0.3925233644859813, + "recall@200": 0.7289719626168224, + "recall@300": 0.9252336448598131, + "recall@500": 0.9392523364485982, + "recall@1000": 0.9485981308411215, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4925683638348177, + "ndcg_full@5": 0.4925683638348177, + "ndcg@10": 0.5009776177847028, + "ndcg_full@10": 0.5009776177847028, + "ndcg@20": 0.5840343204641236, + "ndcg_full@20": 0.5589267633918229, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202110": { + "num_relevant": 47, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.19148936170212766, + "recall@50": 0.8085106382978723, + "recall@100": 0.8297872340425532, + "recall@200": 0.8297872340425532, + "recall@300": 0.8297872340425532, + "recall@500": 0.8297872340425532, + "recall@1000": 0.851063829787234, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6763446997542273, + "ndcg_full@10": 0.6763446997542273, + "ndcg@20": 0.7837538156156325, + "ndcg_full@20": 0.7653245123647533, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.4, + "graded_P@10": 0.6 + }, + "trec-202111": { + "num_relevant": 126, + "num_retrieved": 2000, + "num_ranked": 138, + "recall@10": 0.07936507936507936, + "recall@50": 0.36507936507936506, + "recall@100": 0.5634920634920635, + "recall@200": 0.8015873015873016, + "recall@300": 0.8253968253968254, + "recall@500": 0.8412698412698413, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.6822214669881522, + "ndcg_full@10": 0.6822214669881522, + "ndcg@20": 0.7262860183620484, + "ndcg_full@20": 0.7262860183620484, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202112": { + "num_relevant": 149, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.053691275167785234, + "recall@50": 0.2751677852348993, + "recall@100": 0.5033557046979866, + "recall@200": 0.7516778523489933, + "recall@300": 0.825503355704698, + "recall@500": 0.87248322147651, + "recall@1000": 0.8993288590604027, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8165901947515999, + "ndcg_full@10": 0.8165901947515999, + "ndcg@20": 0.7539767897783569, + "ndcg_full@20": 0.7539767897783569, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202113": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 224, + "recall@10": 0.028985507246376812, + "recall@50": 0.12318840579710146, + "recall@100": 0.2608695652173913, + "recall@200": 0.4855072463768116, + "recall@300": 0.5869565217391305, + "recall@500": 0.6521739130434783, + "recall@1000": 0.717391304347826, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3256996472324916, + "ndcg_full@5": 0.3256996472324916, + "ndcg@10": 0.37057449796202546, + "ndcg_full@10": 0.37057449796202546, + "ndcg@20": 0.3458662120596307, + "ndcg_full@20": 0.3458662120596307, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202114": { + "num_relevant": 294, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.023809523809523808, + "recall@50": 0.08503401360544217, + "recall@100": 0.1598639455782313, + "recall@200": 0.30612244897959184, + "recall@300": 0.41156462585034015, + "recall@500": 0.564625850340136, + "recall@1000": 0.6972789115646258, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.2388984594755701, + "ndcg_full@5": 0.2388984594755701, + "ndcg@10": 0.304824594648577, + "ndcg_full@10": 0.304824594648577, + "ndcg@20": 0.3163212263483235, + "ndcg_full@20": 0.3163212263483235, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202115": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.026923076923076925, + "recall@50": 0.15, + "recall@100": 0.27307692307692305, + "recall@200": 0.4269230769230769, + "recall@300": 0.5076923076923077, + "recall@500": 0.6038461538461538, + "recall@1000": 0.7269230769230769, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.13863671367752411, + "ndcg_full@5": 0.13863671367752411, + "ndcg@10": 0.32950878496786945, + "ndcg_full@10": 0.32950878496786945, + "ndcg@20": 0.5500667816080315, + "ndcg_full@20": 0.5500667816080315, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202116": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.014492753623188406, + "recall@50": 0.07246376811594203, + "recall@100": 0.16666666666666666, + "recall@200": 0.2971014492753623, + "recall@300": 0.39855072463768115, + "recall@500": 0.5144927536231884, + "recall@1000": 0.6811594202898551, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.08514311764162098, + "ndcg_full@10": 0.08514311764162098, + "ndcg@20": 0.07414090715467733, + "ndcg_full@20": 0.07414090715467733, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202117": { + "num_relevant": 251, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0398406374501992, + "recall@50": 0.1553784860557769, + "recall@100": 0.3107569721115538, + "recall@200": 0.6215139442231076, + "recall@300": 0.8565737051792829, + "recall@500": 0.9641434262948207, + "recall@1000": 0.9721115537848606, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.6936634693435663, + "ndcg_full@10": 0.6936634693435663, + "ndcg@20": 0.7527166892139058, + "ndcg_full@20": 0.7527166892139058, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202118": { + "num_relevant": 206, + "num_retrieved": 1740, + "num_ranked": 234, + "recall@10": 0.038834951456310676, + "recall@50": 0.17475728155339806, + "recall@100": 0.3155339805825243, + "recall@200": 0.49514563106796117, + "recall@300": 0.6310679611650486, + "recall@500": 0.7378640776699029, + "recall@1000": 0.8883495145631068, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.1828037168148503, + "ndcg_full@5": 0.1828037168148503, + "ndcg@10": 0.26224600492978584, + "ndcg_full@10": 0.26224600492978584, + "ndcg@20": 0.3685125049730349, + "ndcg_full@20": 0.3374102756582949, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202119": { + "num_relevant": 153, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05228758169934641, + "recall@50": 0.20915032679738563, + "recall@100": 0.3464052287581699, + "recall@200": 0.6013071895424836, + "recall@300": 0.7320261437908496, + "recall@500": 0.8300653594771242, + "recall@1000": 0.9215686274509803, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.4881113819375682, + "ndcg_full@5": 0.4881113819375682, + "ndcg@10": 0.38318629921485675, + "ndcg_full@10": 0.38318629921485675, + "ndcg@20": 0.28246578660342125, + "ndcg_full@20": 0.28246578660342125, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202120": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1, + "recall@50": 0.4, + "recall@100": 0.6714285714285714, + "recall@200": 0.8714285714285714, + "recall@300": 0.9285714285714286, + "recall@500": 0.9428571428571428, + "recall@1000": 0.9428571428571428, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4713281642188929, + "ndcg_full@5": 0.4713281642188929, + "ndcg@10": 0.45722482729173486, + "ndcg_full@10": 0.45722482729173486, + "ndcg@20": 0.32851546582403557, + "ndcg_full@20": 0.32851546582403557, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202121": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 104, + "recall@10": 0.05660377358490566, + "recall@50": 0.39622641509433965, + "recall@100": 0.6415094339622641, + "recall@200": 0.8490566037735849, + "recall@300": 0.8867924528301887, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.5730379245414764, + "ndcg_full@10": 0.5730379245414764, + "ndcg@20": 0.5872497725000436, + "ndcg_full@20": 0.5872497725000436, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202122": { + "num_relevant": 137, + "num_retrieved": 2000, + "num_ranked": 130, + "recall@10": 0.06569343065693431, + "recall@50": 0.2846715328467153, + "recall@100": 0.5109489051094891, + "recall@200": 0.7372262773722628, + "recall@300": 0.7883211678832117, + "recall@500": 0.8248175182481752, + "recall@1000": 0.8540145985401459, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8304198973631921, + "ndcg_full@5": 0.8304198973631921, + "ndcg@10": 0.7542686714597936, + "ndcg_full@10": 0.7542686714597936, + "ndcg@20": 0.7325029016583638, + "ndcg_full@20": 0.7325029016583638, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202123": { + "num_relevant": 267, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02247191011235955, + "recall@50": 0.12734082397003746, + "recall@100": 0.2397003745318352, + "recall@200": 0.39325842696629215, + "recall@300": 0.5430711610486891, + "recall@500": 0.7340823970037453, + "recall@1000": 0.8689138576779026, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.415209948681596, + "ndcg_full@5": 0.415209948681596, + "ndcg@10": 0.4090609916451523, + "ndcg_full@10": 0.4090609916451523, + "ndcg@20": 0.4741227255898057, + "ndcg_full@20": 0.4741227255898057, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202124": { + "num_relevant": 158, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05063291139240506, + "recall@50": 0.25949367088607594, + "recall@100": 0.5063291139240507, + "recall@200": 0.7848101265822784, + "recall@300": 0.8987341772151899, + "recall@500": 0.9556962025316456, + "recall@1000": 0.9746835443037974, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6620877268881592, + "ndcg_full@10": 0.6620877268881592, + "ndcg@20": 0.5625764063145297, + "ndcg_full@20": 0.5625764063145297, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202125": { + "num_relevant": 102, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.049019607843137254, + "recall@50": 0.16666666666666666, + "recall@100": 0.24509803921568626, + "recall@200": 0.35294117647058826, + "recall@300": 0.45098039215686275, + "recall@500": 0.6764705882352942, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.21719873062191455, + "ndcg_full@5": 0.21719873062191455, + "ndcg@10": 0.21037833635933365, + "ndcg_full@10": 0.21037833635933365, + "ndcg@20": 0.20581508660618617, + "ndcg_full@20": 0.20581508660618617, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202126": { + "num_relevant": 160, + "num_retrieved": 2000, + "num_ranked": 125, + "recall@10": 0.0625, + "recall@50": 0.19375, + "recall@100": 0.33125, + "recall@200": 0.49375, + "recall@300": 0.6, + "recall@500": 0.66875, + "recall@1000": 0.78125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7691415988388896, + "ndcg_full@5": 0.7691415988388896, + "ndcg@10": 0.6781548302021558, + "ndcg_full@10": 0.6781548302021558, + "ndcg@20": 0.5762518675836106, + "ndcg_full@20": 0.5762518675836106, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202127": { + "num_relevant": 233, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03862660944206009, + "recall@50": 0.15450643776824036, + "recall@100": 0.26180257510729615, + "recall@200": 0.45493562231759654, + "recall@300": 0.592274678111588, + "recall@500": 0.8283261802575107, + "recall@1000": 0.9399141630901288, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.2696390028884361, + "ndcg_full@5": 0.2696390028884361, + "ndcg@10": 0.36198935417129635, + "ndcg_full@10": 0.36198935417129635, + "ndcg@20": 0.3271834022411263, + "ndcg_full@20": 0.3271834022411263, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.2, + "graded_P@10": 0.45 + }, + "trec-202128": { + "num_relevant": 324, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.018518518518518517, + "recall@50": 0.10802469135802469, + "recall@100": 0.22839506172839505, + "recall@200": 0.4351851851851852, + "recall@300": 0.5987654320987654, + "recall@500": 0.8395061728395061, + "recall@1000": 0.9382716049382716, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.5019534903461996, + "ndcg_full@10": 0.5019534903461996, + "ndcg@20": 0.4914919289735387, + "ndcg_full@20": 0.4914919289735387, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202129": { + "num_relevant": 258, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03488372093023256, + "recall@50": 0.13953488372093023, + "recall@100": 0.26744186046511625, + "recall@200": 0.4496124031007752, + "recall@300": 0.6124031007751938, + "recall@500": 0.7751937984496124, + "recall@1000": 0.8062015503875969, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5863642063944079, + "ndcg_full@5": 0.5863642063944079, + "ndcg@10": 0.6297755981794121, + "ndcg_full@10": 0.6297755981794121, + "ndcg@20": 0.5494893444706881, + "ndcg_full@20": 0.5494893444706881, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202130": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 43, + "recall@10": 0.06, + "recall@50": 0.2866666666666667, + "recall@100": 0.44666666666666666, + "recall@200": 0.5866666666666667, + "recall@300": 0.6333333333333333, + "recall@500": 0.7266666666666667, + "recall@1000": 0.8133333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7556841981827186, + "ndcg_full@10": 0.7556841981827186, + "ndcg@20": 0.6820865228552002, + "ndcg_full@20": 0.6598493898238934, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202131": { + "num_relevant": 172, + "num_retrieved": 2000, + "num_ranked": 156, + "recall@10": 0.023255813953488372, + "recall@50": 0.13953488372093023, + "recall@100": 0.27325581395348836, + "recall@200": 0.5, + "recall@300": 0.6104651162790697, + "recall@500": 0.6918604651162791, + "recall@1000": 0.7441860465116279, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3421757737602428, + "ndcg_full@5": 0.3421757737602428, + "ndcg@10": 0.38011185155442145, + "ndcg_full@10": 0.38011185155442145, + "ndcg@20": 0.3237750511392533, + "ndcg_full@20": 0.3237750511392533, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.2, + "graded_P@10": 0.35 + }, + "trec-202132": { + "num_relevant": 53, + "num_retrieved": 2000, + "num_ranked": 128, + "recall@10": 0.09433962264150944, + "recall@50": 0.24528301886792453, + "recall@100": 0.3584905660377358, + "recall@200": 0.41509433962264153, + "recall@300": 0.4528301886792453, + "recall@500": 0.4716981132075472, + "recall@1000": 0.5094339622641509, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.42333070005078094, + "ndcg_full@10": 0.42333070005078094, + "ndcg@20": 0.4907354966124646, + "ndcg_full@20": 0.3817723797624072, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202133": { + "num_relevant": 209, + "num_retrieved": 2000, + "num_ranked": 246, + "recall@10": 0.028708133971291867, + "recall@50": 0.11004784688995216, + "recall@100": 0.22488038277511962, + "recall@200": 0.3588516746411483, + "recall@300": 0.45933014354066987, + "recall@500": 0.5645933014354066, + "recall@1000": 0.6889952153110048, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.2, + "ndcg_full@5": 0.2, + "ndcg@10": 0.19946943756321145, + "ndcg_full@10": 0.19946943756321145, + "ndcg@20": 0.1621685534036793, + "ndcg_full@20": 0.1621685534036793, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202134": { + "num_relevant": 150, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02, + "recall@50": 0.14666666666666667, + "recall@100": 0.24, + "recall@200": 0.41333333333333333, + "recall@300": 0.52, + "recall@500": 0.72, + "recall@1000": 0.76, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.6455709457931598, + "ndcg_full@5": 0.6455709457931598, + "ndcg@10": 0.5100016329949207, + "ndcg_full@10": 0.5100016329949207, + "ndcg@20": 0.3845545258895468, + "ndcg_full@20": 0.3845545258895468, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202135": { + "num_relevant": 174, + "num_retrieved": 1764, + "num_ranked": 91, + "recall@10": 0.05747126436781609, + "recall@50": 0.2471264367816092, + "recall@100": 0.45977011494252873, + "recall@200": 0.5114942528735632, + "recall@300": 0.5114942528735632, + "recall@500": 0.5114942528735632, + "recall@1000": 0.5114942528735632, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8389574121760602, + "ndcg_full@10": 0.8389574121760602, + "ndcg@20": 0.8251738806734623, + "ndcg_full@20": 0.8251738806734623, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202136": { + "num_relevant": 309, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.022653721682847898, + "recall@50": 0.09061488673139159, + "recall@100": 0.15210355987055016, + "recall@200": 0.2815533980582524, + "recall@300": 0.34627831715210355, + "recall@500": 0.459546925566343, + "recall@1000": 0.6407766990291263, + "unjudged_fraction@10": 0.9, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.09478836436955078, + "ndcg_full@10": 0.09478836436955078, + "ndcg@20": 0.1793841992247178, + "ndcg_full@20": 0.1793841992247178, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202137": { + "num_relevant": 248, + "num_retrieved": 2000, + "num_ranked": 249, + "recall@10": 0.036290322580645164, + "recall@50": 0.1935483870967742, + "recall@100": 0.36693548387096775, + "recall@200": 0.6370967741935484, + "recall@300": 0.8225806451612904, + "recall@500": 0.9475806451612904, + "recall@1000": 0.9556451612903226, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9147766125938318, + "ndcg_full@20": 0.9147766125938318, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202138": { + "num_relevant": 81, + "num_retrieved": 2000, + "num_ranked": 110, + "recall@10": 0.12345679012345678, + "recall@50": 0.5802469135802469, + "recall@100": 0.9629629629629629, + "recall@200": 0.9629629629629629, + "recall@300": 0.9629629629629629, + "recall@500": 0.9629629629629629, + "recall@1000": 0.9629629629629629, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5743428466665951, + "ndcg_full@5": 0.5743428466665951, + "ndcg@10": 0.7237776329360842, + "ndcg_full@10": 0.7237776329360842, + "ndcg@20": 0.6698390589433475, + "ndcg_full@20": 0.6698390589433475, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202139": { + "num_relevant": 59, + "num_retrieved": 2000, + "num_ranked": 160, + "recall@10": 0.11864406779661017, + "recall@50": 0.559322033898305, + "recall@100": 0.8305084745762712, + "recall@200": 0.8983050847457628, + "recall@300": 0.9491525423728814, + "recall@500": 0.9491525423728814, + "recall@1000": 0.9830508474576272, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.36136328632247594, + "ndcg_full@5": 0.36136328632247594, + "ndcg@10": 0.5463693368391531, + "ndcg_full@10": 0.5463693368391531, + "ndcg@20": 0.642762917819385, + "ndcg_full@20": 0.642762917819385, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202140": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 81, + "recall@10": 0.46153846153846156, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4375640134078725, + "ndcg_full@5": 0.4375640134078725, + "ndcg@10": 0.5797191759437461, + "ndcg_full@10": 0.5797191759437461, + "ndcg@20": 0.7151747435119429, + "ndcg_full@20": 0.6771184098737061, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202141": { + "num_relevant": 308, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.032467532467532464, + "recall@50": 0.12012987012987013, + "recall@100": 0.2305194805194805, + "recall@200": 0.42207792207792205, + "recall@300": 0.5941558441558441, + "recall@500": 0.8474025974025974, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7555278420230372, + "ndcg_full@5": 0.7555278420230372, + "ndcg@10": 0.7189348074662754, + "ndcg_full@10": 0.7189348074662754, + "ndcg@20": 0.5824179501132626, + "ndcg_full@20": 0.5824179501132626, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202142": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.08450704225352113, + "recall@50": 0.30985915492957744, + "recall@100": 0.4507042253521127, + "recall@200": 0.5352112676056338, + "recall@300": 0.6619718309859155, + "recall@500": 0.7183098591549296, + "recall@1000": 0.7605633802816901, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6386367136775242, + "ndcg_full@5": 0.6386367136775242, + "ndcg@10": 0.6960687314028853, + "ndcg_full@10": 0.6960687314028853, + "ndcg@20": 0.6819788859558782, + "ndcg_full@20": 0.6819788859558782, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202143": { + "num_relevant": 155, + "num_retrieved": 2000, + "num_ranked": 174, + "recall@10": 0.06451612903225806, + "recall@50": 0.25161290322580643, + "recall@100": 0.41935483870967744, + "recall@200": 0.6967741935483871, + "recall@300": 0.8, + "recall@500": 0.864516129032258, + "recall@1000": 0.9096774193548387, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7456298460447879, + "ndcg_full@5": 0.7456298460447879, + "ndcg@10": 0.6693573776140648, + "ndcg_full@10": 0.6693573776140648, + "ndcg@20": 0.6245294166927976, + "ndcg_full@20": 0.6245294166927976, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202144": { + "num_relevant": 195, + "num_retrieved": 2000, + "num_ranked": 126, + "recall@10": 0.05128205128205128, + "recall@50": 0.24102564102564103, + "recall@100": 0.4256410256410256, + "recall@200": 0.6974358974358974, + "recall@300": 0.7230769230769231, + "recall@500": 0.7282051282051282, + "recall@1000": 0.7384615384615385, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5804751699305776, + "ndcg_full@5": 0.49999999999999994, + "ndcg@10": 0.6807351507646947, + "ndcg_full@10": 0.5789282245622048, + "ndcg@20": 0.7989119540307507, + "ndcg_full@20": 0.7082673834368195, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.5 + }, + "trec-202145": { + "num_relevant": 186, + "num_retrieved": 2000, + "num_ranked": 211, + "recall@10": 0.053763440860215055, + "recall@50": 0.24193548387096775, + "recall@100": 0.46236559139784944, + "recall@200": 0.8225806451612904, + "recall@300": 0.8817204301075269, + "recall@500": 0.8817204301075269, + "recall@1000": 0.8870967741935484, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503199, + "ndcg_full@10": 0.9633180389503199, + "ndcg@20": 0.8527954486968281, + "ndcg_full@20": 0.8527954486968281, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202146": { + "num_relevant": 111, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.08108108108108109, + "recall@50": 0.3333333333333333, + "recall@100": 0.5945945945945946, + "recall@200": 0.9369369369369369, + "recall@300": 0.990990990990991, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.5550616925716014, + "ndcg_full@10": 0.5550616925716014, + "ndcg@20": 0.6446636802230978, + "ndcg_full@20": 0.6446636802230978, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202147": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03759398496240601, + "recall@50": 0.14661654135338345, + "recall@100": 0.23308270676691728, + "recall@200": 0.43609022556390975, + "recall@300": 0.631578947368421, + "recall@500": 0.8007518796992481, + "recall@1000": 0.9210526315789473, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6281895090257151, + "ndcg_full@5": 0.6281895090257151, + "ndcg@10": 0.6509275686911992, + "ndcg_full@10": 0.6509275686911992, + "ndcg@20": 0.6476667753702007, + "ndcg_full@20": 0.6476667753702007, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202148": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 68, + "recall@10": 0.09090909090909091, + "recall@50": 0.5151515151515151, + "recall@100": 0.7878787878787878, + "recall@200": 0.803030303030303, + "recall@300": 0.803030303030303, + "recall@500": 0.8333333333333334, + "recall@1000": 0.9090909090909091, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8581437227514841, + "ndcg_full@10": 0.8581437227514841, + "ndcg@20": 0.9084505904897269, + "ndcg_full@20": 0.9084505904897269, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202149": { + "num_relevant": 77, + "num_retrieved": 1567, + "num_ranked": 114, + "recall@10": 0.12987012987012986, + "recall@50": 0.5194805194805194, + "recall@100": 0.7142857142857143, + "recall@200": 0.7792207792207793, + "recall@300": 0.7922077922077922, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7797454537728086, + "ndcg_full@10": 0.7797454537728086, + "ndcg@20": 0.7706994929176189, + "ndcg_full@20": 0.7706994929176189, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202150": { + "num_relevant": 23, + "num_retrieved": 2000, + "num_ranked": 60, + "recall@10": 0.391304347826087, + "recall@50": 0.7391304347826086, + "recall@100": 0.7391304347826086, + "recall@200": 0.782608695652174, + "recall@300": 0.782608695652174, + "recall@500": 0.782608695652174, + "recall@1000": 0.8695652173913043, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.5797016289380371, + "ndcg_full@10": 0.56126109166141, + "ndcg@20": 0.5885727170009951, + "ndcg_full@20": 0.4819861242946091, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202151": { + "num_relevant": 71, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.11267605633802817, + "recall@50": 0.4225352112676056, + "recall@100": 0.676056338028169, + "recall@200": 0.8732394366197183, + "recall@300": 0.8732394366197183, + "recall@500": 0.9014084507042254, + "recall@1000": 0.9577464788732394, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8234550942619859, + "ndcg_full@5": 0.8234550942619859, + "ndcg@10": 0.6127642805131401, + "ndcg_full@10": 0.6127642805131401, + "ndcg@20": 0.6333774033426927, + "ndcg_full@20": 0.5913516328525625, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202152": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 208, + "recall@10": 0.08823529411764706, + "recall@50": 0.39705882352941174, + "recall@100": 0.6323529411764706, + "recall@200": 0.7352941176470589, + "recall@300": 0.7794117647058824, + "recall@500": 0.7941176470588235, + "recall@1000": 0.8235294117647058, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.20248323207250624, + "ndcg_full@10": 0.20248323207250624, + "ndcg@20": 0.33962060278367706, + "ndcg_full@20": 0.33962060278367706, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202153": { + "num_relevant": 46, + "num_retrieved": 2000, + "num_ranked": 148, + "recall@10": 0.15217391304347827, + "recall@50": 0.4782608695652174, + "recall@100": 0.6086956521739131, + "recall@200": 0.6521739130434783, + "recall@300": 0.717391304347826, + "recall@500": 0.7608695652173914, + "recall@1000": 0.7608695652173914, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.8205228949141826, + "ndcg_full@10": 0.8205228949141826, + "ndcg@20": 0.6201731525796832, + "ndcg_full@20": 0.6201731525796832, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202154": { + "num_relevant": 75, + "num_retrieved": 2000, + "num_ranked": 122, + "recall@10": 0.12, + "recall@50": 0.5066666666666667, + "recall@100": 0.8666666666666667, + "recall@200": 0.9066666666666666, + "recall@300": 0.9066666666666666, + "recall@500": 0.9066666666666666, + "recall@1000": 0.9066666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4566005422926888, + "ndcg_full@5": 0.4566005422926888, + "ndcg@10": 0.40493286563348496, + "ndcg_full@10": 0.40493286563348496, + "ndcg@20": 0.43949636336662334, + "ndcg_full@20": 0.43949636336662334, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202155": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 172, + "recall@10": 0.20512820512820512, + "recall@50": 0.5128205128205128, + "recall@100": 0.717948717948718, + "recall@200": 0.7435897435897436, + "recall@300": 0.7435897435897436, + "recall@500": 0.7435897435897436, + "recall@1000": 0.8205128205128205, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.039621067055815605, + "ndcg_full@20": 0.039621067055815605, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202156": { + "num_relevant": 86, + "num_retrieved": 2000, + "num_ranked": 187, + "recall@10": 0.09302325581395349, + "recall@50": 0.36046511627906974, + "recall@100": 0.6511627906976745, + "recall@200": 0.9767441860465116, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6621484664716257, + "ndcg_full@5": 0.6621484664716257, + "ndcg@10": 0.6103386248132282, + "ndcg_full@10": 0.6103386248132282, + "ndcg@20": 0.6038129293688367, + "ndcg_full@20": 0.6038129293688367, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202157": { + "num_relevant": 162, + "num_retrieved": 2000, + "num_ranked": 113, + "recall@10": 0.06172839506172839, + "recall@50": 0.17901234567901234, + "recall@100": 0.3271604938271605, + "recall@200": 0.5555555555555556, + "recall@300": 0.691358024691358, + "recall@500": 0.7839506172839507, + "recall@1000": 0.8209876543209876, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.7863847170475807, + "ndcg_full@20": 0.7863847170475807, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202158": { + "num_relevant": 136, + "num_retrieved": 2000, + "num_ranked": 103, + "recall@10": 0.07352941176470588, + "recall@50": 0.3235294117647059, + "recall@100": 0.5661764705882353, + "recall@200": 0.6764705882352942, + "recall@300": 0.6764705882352942, + "recall@500": 0.6764705882352942, + "recall@1000": 0.6838235294117647, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.9075755242149841, + "ndcg_full@5": 0.9075755242149841, + "ndcg@10": 0.8442069605065422, + "ndcg_full@10": 0.8442069605065422, + "ndcg@20": 0.8468043645527589, + "ndcg_full@20": 0.8468043645527589, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202159": { + "num_relevant": 351, + "num_retrieved": 1578, + "num_ranked": 187, + "recall@10": 0.02564102564102564, + "recall@50": 0.1225071225071225, + "recall@100": 0.23931623931623933, + "recall@200": 0.46438746438746437, + "recall@300": 0.6068376068376068, + "recall@500": 0.6239316239316239, + "recall@1000": 0.6239316239316239, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.9266029294647912, + "ndcg_full@20": 0.9266029294647912, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202160": { + "num_relevant": 199, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.04020100502512563, + "recall@50": 0.20100502512562815, + "recall@100": 0.37185929648241206, + "recall@200": 0.542713567839196, + "recall@300": 0.678391959798995, + "recall@500": 0.8442211055276382, + "recall@1000": 0.9547738693467337, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6363284982146581, + "ndcg_full@5": 0.6363284982146581, + "ndcg@10": 0.5182541821754323, + "ndcg_full@10": 0.5182541821754323, + "ndcg@20": 0.5214301914937111, + "ndcg_full@20": 0.5214301914937111, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202161": { + "num_relevant": 190, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.031578947368421054, + "recall@50": 0.05263157894736842, + "recall@100": 0.1368421052631579, + "recall@200": 0.28421052631578947, + "recall@300": 0.4473684210526316, + "recall@500": 0.6736842105263158, + "recall@1000": 0.9052631578947369, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6164336326286642, + "ndcg_full@5": 0.6164336326286642, + "ndcg@10": 0.6147565951326662, + "ndcg_full@10": 0.6147565951326662, + "ndcg@20": 0.486261357500415, + "ndcg_full@20": 0.486261357500415, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202162": { + "num_relevant": 275, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.03272727272727273, + "recall@50": 0.16, + "recall@100": 0.28363636363636363, + "recall@200": 0.4763636363636364, + "recall@300": 0.5745454545454546, + "recall@500": 0.6872727272727273, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.48197017550559196, + "ndcg_full@5": 0.48197017550559196, + "ndcg@10": 0.6117657697639597, + "ndcg_full@10": 0.6117657697639597, + "ndcg@20": 0.5583093297201626, + "ndcg_full@20": 0.5583093297201626, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202163": { + "num_relevant": 44, + "num_retrieved": 2000, + "num_ranked": 141, + "recall@10": 0.06818181818181818, + "recall@50": 0.29545454545454547, + "recall@100": 0.38636363636363635, + "recall@200": 0.5681818181818182, + "recall@300": 0.7727272727272727, + "recall@500": 0.8636363636363636, + "recall@1000": 0.8863636363636364, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.17654490573801404, + "ndcg_full@5": 0.17654490573801404, + "ndcg@10": 0.15376469828671946, + "ndcg_full@10": 0.15376469828671946, + "ndcg@20": 0.192863084044742, + "ndcg_full@20": 0.15297395127027633, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202164": { + "num_relevant": 203, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.024630541871921183, + "recall@50": 0.10344827586206896, + "recall@100": 0.21674876847290642, + "recall@200": 0.4088669950738916, + "recall@300": 0.541871921182266, + "recall@500": 0.8275862068965517, + "recall@1000": 0.9753694581280788, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3077499832503558, + "ndcg_full@5": 0.3077499832503558, + "ndcg@10": 0.3395080147350688, + "ndcg_full@10": 0.3395080147350688, + "ndcg@20": 0.3564260516695249, + "ndcg_full@20": 0.344805981104839, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202165": { + "num_relevant": 113, + "num_retrieved": 2000, + "num_ranked": 109, + "recall@10": 0.017699115044247787, + "recall@50": 0.02654867256637168, + "recall@100": 0.05309734513274336, + "recall@200": 0.18584070796460178, + "recall@300": 0.30973451327433627, + "recall@500": 0.49557522123893805, + "recall@1000": 0.7610619469026548, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.2967403475026077, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.2967403475026077, + "ndcg_full@10": 0.17993148201117176, + "ndcg@20": 0.2967403475026077, + "ndcg_full@20": 0.11612190345001776, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202166": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 29, + "recall@10": 0.16666666666666666, + "recall@50": 0.25, + "recall@100": 0.2708333333333333, + "recall@200": 0.2916666666666667, + "recall@300": 0.3125, + "recall@500": 0.3125, + "recall@1000": 0.3125, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8292135270631279, + "ndcg_full@10": 0.7215194105726964, + "ndcg@20": 0.8292135270631279, + "ndcg_full@20": 0.5526982435913689, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202167": { + "num_relevant": 157, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.06369426751592357, + "recall@50": 0.25477707006369427, + "recall@100": 0.4585987261146497, + "recall@200": 0.7261146496815286, + "recall@300": 0.802547770700637, + "recall@500": 0.821656050955414, + "recall@1000": 0.8343949044585988, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7123252244714983, + "ndcg_full@5": 0.7123252244714983, + "ndcg@10": 0.7375583312686051, + "ndcg_full@10": 0.7375583312686051, + "ndcg@20": 0.6913735644243053, + "ndcg_full@20": 0.6913735644243053, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202168": { + "num_relevant": 164, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.06097560975609756, + "recall@50": 0.25609756097560976, + "recall@100": 0.3475609756097561, + "recall@200": 0.4634146341463415, + "recall@300": 0.5121951219512195, + "recall@500": 0.6524390243902439, + "recall@1000": 0.7439024390243902, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6445146518760496, + "ndcg_full@5": 0.6445146518760496, + "ndcg@10": 0.6336288876433688, + "ndcg_full@10": 0.6336288876433688, + "ndcg@20": 0.6181545510450168, + "ndcg_full@20": 0.6181545510450168, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202169": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 201, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.0, + "recall@200": 0.0759493670886076, + "recall@300": 0.10126582278481013, + "recall@500": 0.12658227848101267, + "recall@1000": 0.189873417721519, + "unjudged_fraction@10": 1.0, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.0, + "ndcg_full@20": 0.0, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202170": { + "num_relevant": 231, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03896103896103896, + "recall@50": 0.1645021645021645, + "recall@100": 0.3116883116883117, + "recall@200": 0.5670995670995671, + "recall@300": 0.8095238095238095, + "recall@500": 0.9004329004329005, + "recall@1000": 0.9004329004329005, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.30145631640942205, + "ndcg_full@5": 0.30145631640942205, + "ndcg@10": 0.4642951558439675, + "ndcg_full@10": 0.4642951558439675, + "ndcg@20": 0.4578448867362132, + "ndcg_full@20": 0.4578448867362132, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202171": { + "num_relevant": 184, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.016304347826086956, + "recall@50": 0.1358695652173913, + "recall@100": 0.24456521739130435, + "recall@200": 0.3858695652173913, + "recall@300": 0.4782608695652174, + "recall@500": 0.625, + "recall@1000": 0.8043478260869565, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.3312392961638527, + "ndcg_full@10": 0.3312392961638527, + "ndcg@20": 0.343309574224581, + "ndcg_full@20": 0.343309574224581, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202172": { + "num_relevant": 91, + "num_retrieved": 2000, + "num_ranked": 69, + "recall@10": 0.0989010989010989, + "recall@50": 0.23076923076923078, + "recall@100": 0.24175824175824176, + "recall@200": 0.25274725274725274, + "recall@300": 0.25274725274725274, + "recall@500": 0.26373626373626374, + "recall@1000": 0.2967032967032967, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.3787833386759354, + "ndcg_full@10": 0.3787833386759354, + "ndcg@20": 0.4841692529473041, + "ndcg_full@20": 0.4451836608636745, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202173": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 115, + "recall@10": 0.13636363636363635, + "recall@50": 0.6060606060606061, + "recall@100": 0.8181818181818182, + "recall@200": 0.9393939393939394, + "recall@300": 0.9545454545454546, + "recall@500": 0.9696969696969697, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.94497705842548, + "ndcg_full@10": 0.94497705842548, + "ndcg@20": 0.7858490162432362, + "ndcg_full@20": 0.7337064702315909, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202174": { + "num_relevant": 13, + "num_retrieved": 2000, + "num_ranked": 79, + "recall@10": 0.5384615384615384, + "recall@50": 1.0, + "recall@100": 1.0, + "recall@200": 1.0, + "recall@300": 1.0, + "recall@500": 1.0, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6917831836856677, + "ndcg_full@5": 0.6917831836856677, + "ndcg@10": 0.7120420061696041, + "ndcg_full@10": 0.6893916693379677, + "ndcg@20": 0.8015732750758164, + "ndcg_full@20": 0.7148708067985206, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202175": { + "num_relevant": 341, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020527859237536656, + "recall@50": 0.10263929618768329, + "recall@100": 0.19648093841642228, + "recall@200": 0.3870967741935484, + "recall@300": 0.5454545454545454, + "recall@500": 0.8357771260997068, + "recall@1000": 0.9413489736070382, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7644817904768849, + "ndcg_full@5": 0.7644817904768849, + "ndcg@10": 0.7634502885216227, + "ndcg_full@10": 0.7634502885216227, + "ndcg@20": 0.6893604029763079, + "ndcg_full@20": 0.6893604029763079, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/trec/unjudged-policy-trec22.json b/benchmarks/trec/unjudged-policy-trec22.json new file mode 100644 index 00000000..a707f432 --- /dev/null +++ b/benchmarks/trec/unjudged-policy-trec22.json @@ -0,0 +1,13808 @@ +{ + "schema": 1, + "track": "TREC2022", + "qrels": { + "path": "data/trec/qrels/qrels_22.txt", + "sha256": "e569a531489e03f7b1fab03fe169c8ea66f4a59e8180fa9858b1a6e4bdcb0c5c" + }, + "policies": { + "exclude": "remove unjudged trials before applying rank cutoffs", + "include_as_zero": "retain unjudged trials at their ranks with relevance grade zero" + }, + "runs": [ + { + "name": "h100_rrf", + "results_dir": "h100_rrf/results_trec22", + "evaluation_input_sha256": "e92490bb73cab9b5c6d8da39513cb8d3e5f35bd8f66b6753036635ca2e12838d", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.07596359209010367, + "recall@50": 0.27199682239397766, + "recall@100": 0.4089733562644178, + "recall@200": 0.5572015900971085, + "recall@300": 0.6312335663196081, + "recall@500": 0.6966403365562905, + "recall@1000": 0.7499263497134883, + "ndcg@5": 0.7446605265789237, + "ndcg@10": 0.752820505770411, + "ndcg@20": 0.7574534782631782, + "ndcg_full@5": 0.7336229774913444, + "ndcg_full@10": 0.7131621504300781, + "ndcg_full@20": 0.6751936220831244, + "P@10(rel>=1)": 0.742, + "P@10(eligible)": 0.6580000000000001, + "graded_P@10": 0.6999999999999998, + "unjudged_fraction@10": 0.15200000000000002 + }, + "median": { + "recall@10": 0.058141500185917586, + "recall@50": 0.2376207729468599, + "recall@100": 0.3548387096774194, + "recall@200": 0.52898419194086, + "recall@300": 0.6719261223936654, + "recall@500": 0.7483716891011724, + "recall@1000": 0.7799668416689693, + "ndcg@5": 0.831242017300881, + "ndcg@10": 0.8241853605421747, + "ndcg@20": 0.795319701749207, + "ndcg_full@5": 0.830419897363192, + "ndcg_full@10": 0.7482480764578422, + "ndcg_full@20": 0.736195689165244, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 1000, + "num_ranked": 110, + "recall@10": 0.046632124352331605, + "recall@50": 0.20725388601036268, + "recall@100": 0.38341968911917096, + "recall@200": 0.538860103626943, + "recall@300": 0.5854922279792746, + "recall@500": 0.5958549222797928, + "recall@1000": 0.6113989637305699, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632229, + "ndcg_full@10": 0.9305687780632229, + "ndcg@20": 0.8872042336139788, + "ndcg_full@20": 0.8872042336139788, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 1000, + "num_ranked": 47, + "recall@10": 0.022613065326633167, + "recall@50": 0.09045226130653267, + "recall@100": 0.18341708542713567, + "recall@200": 0.3065326633165829, + "recall@300": 0.4020100502512563, + "recall@500": 0.5376884422110553, + "recall@1000": 0.7336683417085427, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.849607409925425, + "ndcg_full@5": 0.849607409925425, + "ndcg@10": 0.8310079275866007, + "ndcg_full@10": 0.8310079275866007, + "ndcg@20": 0.9336682177878284, + "ndcg_full@20": 0.8219453873700663, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 1000, + "num_ranked": 115, + "recall@10": 0.12658227848101267, + "recall@50": 0.5443037974683544, + "recall@100": 0.8227848101265823, + "recall@200": 0.8987341772151899, + "recall@300": 0.9240506329113924, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9240506329113924, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8510488233360478, + "ndcg_full@5": 0.8510488233360478, + "ndcg@10": 0.8249426093012541, + "ndcg_full@10": 0.8249426093012541, + "ndcg@20": 0.8190859538780906, + "ndcg_full@20": 0.8190859538780906, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.01749271137026239, + "recall@50": 0.09329446064139942, + "recall@100": 0.1836734693877551, + "recall@200": 0.3498542274052478, + "recall@300": 0.478134110787172, + "recall@500": 0.6559766763848397, + "recall@1000": 0.7959183673469388, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 1000, + "num_ranked": 121, + "recall@10": 0.09090909090909091, + "recall@50": 0.36363636363636365, + "recall@100": 0.5454545454545454, + "recall@200": 0.6636363636363637, + "recall@300": 0.7090909090909091, + "recall@500": 0.7636363636363637, + "recall@1000": 0.7818181818181819, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.7510916729774053, + "ndcg_full@10": 0.7510916729774053, + "ndcg@20": 0.6968865096699933, + "ndcg_full@20": 0.6968865096699933, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 1000, + "num_ranked": 168, + "recall@10": 0.06521739130434782, + "recall@50": 0.2391304347826087, + "recall@100": 0.4420289855072464, + "recall@200": 0.7971014492753623, + "recall@300": 0.8623188405797102, + "recall@500": 0.9130434782608695, + "recall@1000": 0.9202898550724637, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 1.0000000000000002, + "ndcg_full@20": 1.0000000000000002, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 1000, + "num_ranked": 113, + "recall@10": 0.16129032258064516, + "recall@50": 0.3548387096774194, + "recall@100": 0.3548387096774194, + "recall@200": 0.45161290322580644, + "recall@300": 0.46774193548387094, + "recall@500": 0.5161290322580645, + "recall@1000": 0.5483870967741935, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.6619811828636347, + "ndcg_full@10": 0.6619811828636347, + "ndcg@20": 0.6900988461684888, + "ndcg_full@20": 0.6630354892333141, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 1000, + "num_ranked": 27, + "recall@10": 0.25, + "recall@50": 0.45, + "recall@100": 0.55, + "recall@200": 0.65, + "recall@300": 0.7, + "recall@500": 0.8, + "recall@1000": 0.85, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8282996787332996, + "ndcg_full@10": 0.8282996787332996, + "ndcg@20": 0.9221134932034663, + "ndcg_full@20": 0.75836202758774, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 1000, + "num_ranked": 21, + "recall@10": 0.05384615384615385, + "recall@50": 0.13076923076923078, + "recall@100": 0.2, + "recall@200": 0.3, + "recall@300": 0.38461538461538464, + "recall@500": 0.5538461538461539, + "recall@1000": 0.7230769230769231, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7309662895581445, + "ndcg_full@10": 0.6834991618708629, + "ndcg@20": 0.9101714165263287, + "ndcg_full@20": 0.5847502347040391, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 1000, + "num_ranked": 20, + "recall@10": 0.03211009174311927, + "recall@50": 0.0779816513761468, + "recall@100": 0.11467889908256881, + "recall@200": 0.1559633027522936, + "recall@300": 0.1926605504587156, + "recall@500": 0.28440366972477066, + "recall@1000": 0.43119266055045874, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9673577739920146, + "ndcg_full@10": 0.6656685916681657, + "ndcg@20": 0.9673577739920146, + "ndcg_full@20": 0.4296007739579482, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 1000, + "num_ranked": 166, + "recall@10": 0.03289473684210526, + "recall@50": 0.1513157894736842, + "recall@100": 0.2894736842105263, + "recall@200": 0.45394736842105265, + "recall@300": 0.5723684210526315, + "recall@500": 0.6480263157894737, + "recall@1000": 0.7401315789473685, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.83206413723857, + "ndcg_full@5": 0.83206413723857, + "ndcg@10": 0.8657274175105066, + "ndcg_full@10": 0.8657274175105066, + "ndcg@20": 0.9133448594678868, + "ndcg_full@20": 0.9133448594678868, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 1000, + "num_ranked": 175, + "recall@10": 0.09090909090909091, + "recall@50": 0.24242424242424243, + "recall@100": 0.3939393939393939, + "recall@200": 0.6060606060606061, + "recall@300": 0.7424242424242424, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8787878787878788, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.43024278050533904, + "ndcg_full@10": 0.43024278050533904, + "ndcg@20": 0.3160492806141465, + "ndcg_full@20": 0.3160492806141465, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 1000, + "num_ranked": 157, + "recall@10": 0.23809523809523808, + "recall@50": 0.5238095238095238, + "recall@100": 0.5238095238095238, + "recall@200": 0.5714285714285714, + "recall@300": 0.7142857142857143, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8095238095238095, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.2194775303813488, + "ndcg_full@5": 0.2194775303813488, + "ndcg@10": 0.14710537964084, + "ndcg_full@10": 0.1424258995403119, + "ndcg@20": 0.37102191329673817, + "ndcg_full@20": 0.2661924466346444, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 1000, + "num_ranked": 116, + "recall@10": 0.08196721311475409, + "recall@50": 0.3442622950819672, + "recall@100": 0.5983606557377049, + "recall@200": 0.860655737704918, + "recall@300": 0.9344262295081968, + "recall@500": 0.9590163934426229, + "recall@1000": 0.9590163934426229, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5538466623591201, + "ndcg_full@5": 0.5538466623591201, + "ndcg@10": 0.7104770117651424, + "ndcg_full@10": 0.7104770117651424, + "ndcg@20": 0.6696538371420097, + "ndcg_full@20": 0.6696538371420097, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1000, + "num_ranked": 134, + "recall@10": 0.05847953216374269, + "recall@50": 0.2807017543859649, + "recall@100": 0.52046783625731, + "recall@200": 0.695906432748538, + "recall@300": 0.7017543859649122, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9808076809173084, + "ndcg_full@20": 0.9808076809173084, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 1000, + "num_ranked": 113, + "recall@10": 0.14583333333333334, + "recall@50": 0.6458333333333334, + "recall@100": 0.9583333333333334, + "recall@200": 0.9583333333333334, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.5747915663739762, + "ndcg_full@10": 0.5747915663739762, + "ndcg@20": 0.5843290810624311, + "ndcg_full@20": 0.5843290810624311, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 1000, + "num_ranked": 132, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.1, + "recall@200": 0.23333333333333334, + "recall@300": 0.36666666666666664, + "recall@500": 0.6, + "recall@1000": 0.7333333333333333, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.1309019901681972, + "ndcg_full@20": 0.039621067055815605, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1000, + "num_ranked": 55, + "recall@10": 0.14285714285714285, + "recall@50": 0.5102040816326531, + "recall@100": 0.6938775510204082, + "recall@200": 0.7551020408163265, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6379365213265478, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.6132833853362728, + "ndcg_full@10": 0.6132833853362728, + "ndcg@20": 0.579480508312642, + "ndcg_full@20": 0.579480508312642, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 1000, + "num_ranked": 102, + "recall@10": 0.07142857142857142, + "recall@50": 0.14285714285714285, + "recall@100": 0.21428571428571427, + "recall@200": 0.5, + "recall@300": 0.6428571428571429, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.42870185423738655, + "ndcg_full@10": 0.2781981696179509, + "ndcg@20": 0.5085431896180544, + "ndcg_full@20": 0.29530920473153227, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 1000, + "num_ranked": 171, + "recall@10": 0.042735042735042736, + "recall@50": 0.2094017094017094, + "recall@100": 0.3974358974358974, + "recall@200": 0.6452991452991453, + "recall@300": 0.8034188034188035, + "recall@500": 0.8547008547008547, + "recall@1000": 0.8632478632478633, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9603895103778499, + "ndcg_full@5": 0.9603895103778499, + "ndcg@10": 0.9241428704743823, + "ndcg_full@10": 0.9241428704743823, + "ndcg@20": 0.9144740142506342, + "ndcg_full@20": 0.9144740142506342, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 1000, + "num_ranked": 180, + "recall@10": 0.057803468208092484, + "recall@50": 0.24855491329479767, + "recall@100": 0.5202312138728323, + "recall@200": 0.8670520231213873, + "recall@300": 0.953757225433526, + "recall@500": 0.9595375722543352, + "recall@1000": 0.9595375722543352, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 1000, + "num_ranked": 121, + "recall@10": 0.072, + "recall@50": 0.288, + "recall@100": 0.4, + "recall@200": 0.472, + "recall@300": 0.544, + "recall@500": 0.6, + "recall@1000": 0.608, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6975961695061491, + "ndcg_full@10": 0.6975961695061491, + "ndcg@20": 0.7366507776664716, + "ndcg_full@20": 0.7366507776664716, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 1000, + "num_ranked": 129, + "recall@10": 0.05319148936170213, + "recall@50": 0.2553191489361702, + "recall@100": 0.48936170212765956, + "recall@200": 0.6914893617021277, + "recall@300": 0.7819148936170213, + "recall@500": 0.8351063829787234, + "recall@1000": 0.8617021276595744, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9025937412599945, + "ndcg_full@10": 0.9025937412599945, + "ndcg@20": 0.8840772334129118, + "ndcg_full@20": 0.8840772334129118, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1000, + "num_ranked": 30, + "recall@10": 0.22580645161290322, + "recall@50": 0.3548387096774194, + "recall@100": 0.3548387096774194, + "recall@200": 0.41935483870967744, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.7881868575021682, + "ndcg_full@10": 0.6845690459740027, + "ndcg@20": 0.8881887186822485, + "ndcg_full@20": 0.5655225455907407, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 1000, + "num_ranked": 61, + "recall@10": 0.016574585635359115, + "recall@50": 0.12154696132596685, + "recall@100": 0.2154696132596685, + "recall@200": 0.32044198895027626, + "recall@300": 0.36464088397790057, + "recall@500": 0.48066298342541436, + "recall@1000": 0.6795580110497238, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.5634440045622183, + "ndcg_full@10": 0.5455206287241073, + "ndcg@20": 0.48151421662111105, + "ndcg_full@20": 0.38624869192787353, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 1000, + "num_ranked": 61, + "recall@10": 0.16071428571428573, + "recall@50": 0.6607142857142857, + "recall@100": 0.75, + "recall@200": 0.9285714285714286, + "recall@300": 0.9464285714285714, + "recall@500": 0.9642857142857143, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7274710542151983, + "ndcg_full@10": 0.7274710542151983, + "ndcg@20": 0.7357406006640165, + "ndcg_full@20": 0.7357406006640165, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 1000, + "num_ranked": 76, + "recall@10": 0.13793103448275862, + "recall@50": 0.6551724137931034, + "recall@100": 0.7931034482758621, + "recall@200": 0.8275862068965517, + "recall@300": 0.8275862068965517, + "recall@500": 0.8275862068965517, + "recall@1000": 0.8275862068965517, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010481, + "ndcg_full@10": 0.9363792118010481, + "ndcg@20": 0.7861389447347518, + "ndcg_full@20": 0.7861389447347518, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 1000, + "num_ranked": 158, + "recall@10": 0.014705882352941176, + "recall@50": 0.058823529411764705, + "recall@100": 0.1323529411764706, + "recall@200": 0.29411764705882354, + "recall@300": 0.35294117647058826, + "recall@500": 0.4264705882352941, + "recall@1000": 0.5882352941176471, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.571098066588972, + "ndcg_full@10": 0.5141863642577489, + "ndcg@20": 0.6276589772414871, + "ndcg_full@20": 0.36470406108285847, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 1000, + "num_ranked": 182, + "recall@10": 0.0, + "recall@50": 0.18181818181818182, + "recall@100": 0.2727272727272727, + "recall@200": 0.5151515151515151, + "recall@300": 0.6363636363636364, + "recall@500": 0.696969696969697, + "recall@1000": 0.7575757575757576, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.42600000043464953, + "ndcg_full@10": 0.3679212572135327, + "ndcg@20": 0.4152405208930943, + "ndcg_full@20": 0.3008390738232198, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 1000, + "num_ranked": 245, + "recall@10": 0.02356020942408377, + "recall@50": 0.1099476439790576, + "recall@100": 0.20680628272251309, + "recall@200": 0.3743455497382199, + "recall@300": 0.518324607329843, + "recall@500": 0.6518324607329843, + "recall@1000": 0.7120418848167539, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 1.0000000000000002, + "ndcg_full@20": 1.0000000000000002, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 1000, + "num_ranked": 97, + "recall@10": 0.1282051282051282, + "recall@50": 0.46153846153846156, + "recall@100": 0.6410256410256411, + "recall@200": 0.7435897435897436, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.6726934039987301, + "ndcg_full@10": 0.6726934039987301, + "ndcg@20": 0.6486495997225513, + "ndcg_full@20": 0.6056104905554288, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 1000, + "num_ranked": 148, + "recall@10": 0.0228310502283105, + "recall@50": 0.0776255707762557, + "recall@100": 0.1506849315068493, + "recall@200": 0.2785388127853881, + "recall@300": 0.3607305936073059, + "recall@500": 0.4337899543378995, + "recall@1000": 0.5251141552511416, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.9166666666666667, + "ndcg_full@5": 0.9166666666666667, + "ndcg@10": 0.9046735686724011, + "ndcg_full@10": 0.9046735686724011, + "ndcg@20": 0.7946692526049672, + "ndcg_full@20": 0.7946692526049672, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 1000, + "num_ranked": 61, + "recall@10": 0.08333333333333333, + "recall@50": 0.2361111111111111, + "recall@100": 0.2638888888888889, + "recall@200": 0.3055555555555556, + "recall@300": 0.3472222222222222, + "recall@500": 0.4027777777777778, + "recall@1000": 0.5416666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.7959701508934467, + "ndcg_full@20": 0.7440703008266993, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 1000, + "num_ranked": 11, + "recall@10": 0.075, + "recall@50": 0.1, + "recall@100": 0.125, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8972754076830647, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.9746507957568686, + "ndcg_full@10": 0.44967593775518905, + "ndcg@20": 0.9746507957568686, + "ndcg_full@20": 0.3367885635619303, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 1000, + "num_ranked": 171, + "recall@10": 0.023076923076923078, + "recall@50": 0.1076923076923077, + "recall@100": 0.17307692307692307, + "recall@200": 0.2653846153846154, + "recall@300": 0.36538461538461536, + "recall@500": 0.46153846153846156, + "recall@1000": 0.5807692307692308, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8251154201194959, + "ndcg_full@10": 0.8251154201194959, + "ndcg@20": 0.7787544430261462, + "ndcg_full@20": 0.7787544430261462, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 1000, + "num_ranked": 174, + "recall@10": 0.03896103896103896, + "recall@50": 0.24675324675324675, + "recall@100": 0.4025974025974026, + "recall@200": 0.6493506493506493, + "recall@300": 0.7142857142857143, + "recall@500": 0.8051948051948052, + "recall@1000": 0.8961038961038961, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.6268473080189253, + "ndcg_full@10": 0.6268473080189253, + "ndcg@20": 0.6124791522474271, + "ndcg_full@20": 0.6124791522474271, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 1000, + "num_ranked": 85, + "recall@10": 0.11538461538461539, + "recall@50": 0.47435897435897434, + "recall@100": 0.7692307692307693, + "recall@200": 0.8717948717948718, + "recall@300": 0.9102564102564102, + "recall@500": 0.9102564102564102, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.735182641392979, + "ndcg_full@5": 0.735182641392979, + "ndcg@10": 0.692466208912761, + "ndcg_full@10": 0.692466208912761, + "ndcg@20": 0.6963513930026646, + "ndcg_full@20": 0.6963513930026646, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 1000, + "num_ranked": 118, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.5786163522012578, + "recall@200": 0.8930817610062893, + "recall@300": 0.9056603773584906, + "recall@500": 0.9182389937106918, + "recall@1000": 0.9182389937106918, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.9122612344908148, + "ndcg_full@20": 0.9122612344908148, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 1000, + "num_ranked": 179, + "recall@10": 0.026515151515151516, + "recall@50": 0.1590909090909091, + "recall@100": 0.29924242424242425, + "recall@200": 0.5606060606060606, + "recall@300": 0.696969696969697, + "recall@500": 0.7803030303030303, + "recall@1000": 0.7992424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9270848374537759, + "ndcg_full@20": 0.9270848374537759, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 1000, + "num_ranked": 143, + "recall@10": 0.06134969325153374, + "recall@50": 0.25153374233128833, + "recall@100": 0.3312883435582822, + "recall@200": 0.3619631901840491, + "recall@300": 0.4171779141104294, + "recall@500": 0.5214723926380368, + "recall@1000": 0.6134969325153374, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9320985963249863, + "ndcg_full@20": 0.9320985963249863, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 1000, + "num_ranked": 101, + "recall@10": 0.12857142857142856, + "recall@50": 0.6142857142857143, + "recall@100": 0.9571428571428572, + "recall@200": 0.9571428571428572, + "recall@300": 0.9571428571428572, + "recall@500": 0.9571428571428572, + "recall@1000": 0.9571428571428572, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8552385058825712, + "ndcg_full@10": 0.8552385058825712, + "ndcg@20": 0.8904065128081756, + "ndcg_full@20": 0.8904065128081756, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1000, + "num_ranked": 153, + "recall@10": 0.055865921787709494, + "recall@50": 0.20670391061452514, + "recall@100": 0.3407821229050279, + "recall@200": 0.4692737430167598, + "recall@300": 0.553072625698324, + "recall@500": 0.6368715083798883, + "recall@1000": 0.6536312849162011, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.8900796236705368, + "ndcg_full@10": 0.8900796236705368, + "ndcg@20": 0.8765610579098814, + "ndcg_full@20": 0.8765610579098814, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 1000, + "num_ranked": 53, + "recall@10": 0.0, + "recall@50": 0.03571428571428571, + "recall@100": 0.13095238095238096, + "recall@200": 0.16666666666666666, + "recall@300": 0.20238095238095238, + "recall@500": 0.35714285714285715, + "recall@1000": 0.5357142857142857, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.3010299956639812, + "ndcg_full@10": 0.033127111727194514, + "ndcg@20": 0.3010299956639812, + "ndcg_full@20": 0.02428511556543983, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 1000, + "num_ranked": 201, + "recall@10": 0.03007518796992481, + "recall@50": 0.16165413533834586, + "recall@100": 0.31203007518796994, + "recall@200": 0.556390977443609, + "recall@300": 0.7330827067669173, + "recall@500": 0.8759398496240601, + "recall@1000": 0.924812030075188, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7632636606403641, + "ndcg_full@5": 0.7632636606403641, + "ndcg@10": 0.7454044799382791, + "ndcg_full@10": 0.7454044799382791, + "ndcg@20": 0.7099968437863164, + "ndcg_full@20": 0.7099968437863164, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 1000, + "num_ranked": 77, + "recall@10": 0.0, + "recall@50": 0.09523809523809523, + "recall@100": 0.19047619047619047, + "recall@200": 0.23809523809523808, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7386919056633241, + "ndcg_full@5": 0.44615333764087994, + "ndcg@10": 0.7386919056633241, + "ndcg_full@10": 0.28952298823485745, + "ndcg@20": 0.8073829738811472, + "ndcg_full@20": 0.2424037669389584, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 1000, + "num_ranked": 212, + "recall@10": 0.028662420382165606, + "recall@50": 0.14331210191082802, + "recall@100": 0.28343949044585987, + "recall@200": 0.5191082802547771, + "recall@300": 0.6751592356687898, + "recall@500": 0.8121019108280255, + "recall@1000": 0.8885350318471338, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9318324610776306, + "ndcg_full@10": 0.9318324610776306, + "ndcg@20": 0.900178643540874, + "ndcg_full@20": 0.900178643540874, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 1000, + "num_ranked": 168, + "recall@10": 0.10416666666666667, + "recall@50": 0.4270833333333333, + "recall@100": 0.65625, + "recall@200": 0.9270833333333334, + "recall@300": 0.9375, + "recall@500": 0.9479166666666666, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8521705090845474, + "ndcg_full@10": 0.8521705090845474, + "ndcg@20": 0.79912257141114, + "ndcg_full@20": 0.79912257141114, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 1000, + "num_ranked": 199, + "recall@10": 0.030395136778115502, + "recall@50": 0.13677811550151975, + "recall@100": 0.270516717325228, + "recall@200": 0.5106382978723404, + "recall@300": 0.668693009118541, + "recall@500": 0.7416413373860182, + "recall@1000": 0.7781155015197568, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8234281117830955, + "ndcg_full@10": 0.8234281117830955, + "ndcg@20": 0.7798397133245715, + "ndcg_full@20": 0.7798397133245715, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 1000, + "num_ranked": 75, + "recall@10": 0.2222222222222222, + "recall@50": 0.6666666666666666, + "recall@100": 0.7222222222222222, + "recall@200": 0.8888888888888888, + "recall@300": 0.8888888888888888, + "recall@500": 0.8888888888888888, + "recall@1000": 0.9444444444444444, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.6163668470816234, + "ndcg_full@10": 0.6163668470816234, + "ndcg@20": 0.7694749091601946, + "ndcg_full@20": 0.601355649126305, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 1000, + "num_ranked": 223, + "recall@10": 0.026490066225165563, + "recall@50": 0.152317880794702, + "recall@100": 0.24503311258278146, + "recall@200": 0.45695364238410596, + "recall@300": 0.6357615894039735, + "recall@500": 0.8344370860927153, + "recall@1000": 0.8741721854304636, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.6214990811334172, + "ndcg_full@10": 0.6214990811334172, + "ndcg@20": 0.6153084465426503, + "ndcg_full@20": 0.6153084465426503, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.07596359209010367, + "recall@50": 0.27199682239397766, + "recall@100": 0.4089733562644178, + "recall@200": 0.5572015900971085, + "recall@300": 0.6312335663196081, + "recall@500": 0.6966403365562905, + "recall@1000": 0.7499263497134883, + "ndcg@5": 0.6551123523304655, + "ndcg@10": 0.6537015390494615, + "ndcg@20": 0.665201916855575, + "ndcg_full@5": 0.6440748032428862, + "ndcg_full@10": 0.6210422677430445, + "ndcg_full@20": 0.5945579489071843, + "P@10(rel>=1)": 0.6399999999999999, + "P@10(eligible)": 0.5660000000000002, + "graded_P@10": 0.6029999999999999, + "unjudged_fraction@10": 0.15200000000000002 + }, + "median": { + "recall@10": 0.058141500185917586, + "recall@50": 0.2376207729468599, + "recall@100": 0.3548387096774194, + "recall@200": 0.52898419194086, + "recall@300": 0.6719261223936654, + "recall@500": 0.7483716891011724, + "recall@1000": 0.7799668416689693, + "ndcg@5": 0.710970696247901, + "ndcg@10": 0.6979878442880916, + "ndcg@20": 0.6809403349376675, + "ndcg_full@5": 0.6917831836856678, + "ndcg_full@10": 0.6511316667873446, + "ndcg_full@20": 0.6556236156935058, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.625, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 1000, + "num_ranked": 110, + "recall@10": 0.046632124352331605, + "recall@50": 0.20725388601036268, + "recall@100": 0.38341968911917096, + "recall@200": 0.538860103626943, + "recall@300": 0.5854922279792746, + "recall@500": 0.5958549222797928, + "recall@1000": 0.6113989637305699, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8930068676327362, + "ndcg_full@5": 0.8930068676327362, + "ndcg@10": 0.9305687780632229, + "ndcg_full@10": 0.9305687780632229, + "ndcg@20": 0.8353783173169202, + "ndcg_full@20": 0.8353783173169202, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 1000, + "num_ranked": 47, + "recall@10": 0.022613065326633167, + "recall@50": 0.09045226130653267, + "recall@100": 0.18341708542713567, + "recall@200": 0.3065326633165829, + "recall@300": 0.4020100502512563, + "recall@500": 0.5376884422110553, + "recall@1000": 0.7336683417085427, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.849607409925425, + "ndcg_full@5": 0.849607409925425, + "ndcg@10": 0.6975780666107746, + "ndcg_full@10": 0.6975780666107746, + "ndcg@20": 0.7037650470325512, + "ndcg_full@20": 0.6195524525524061, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.4, + "graded_P@10": 0.6 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 1000, + "num_ranked": 115, + "recall@10": 0.12658227848101267, + "recall@50": 0.5443037974683544, + "recall@100": 0.8227848101265823, + "recall@200": 0.8987341772151899, + "recall@300": 0.9240506329113924, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9240506329113924, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5684153189413678, + "ndcg_full@5": 0.5684153189413678, + "ndcg@10": 0.620434394605944, + "ndcg_full@10": 0.620434394605944, + "ndcg@20": 0.6690513467057668, + "ndcg_full@20": 0.6690513467057668, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 1000, + "num_ranked": 250, + "recall@10": 0.01749271137026239, + "recall@50": 0.09329446064139942, + "recall@100": 0.1836734693877551, + "recall@200": 0.3498542274052478, + "recall@300": 0.478134110787172, + "recall@500": 0.6559766763848397, + "recall@1000": 0.7959183673469388, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.857980942822373, + "ndcg_full@10": 0.857980942822373, + "ndcg@20": 0.6567954189979142, + "ndcg_full@20": 0.6567954189979142, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 1000, + "num_ranked": 121, + "recall@10": 0.09090909090909091, + "recall@50": 0.36363636363636365, + "recall@100": 0.5454545454545454, + "recall@200": 0.6636363636363637, + "recall@300": 0.7090909090909091, + "recall@500": 0.7636363636363637, + "recall@1000": 0.7818181818181819, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.508473817692402, + "ndcg_full@10": 0.508473817692402, + "ndcg@20": 0.5476694441183657, + "ndcg_full@20": 0.5476694441183657, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 1000, + "num_ranked": 168, + "recall@10": 0.06521739130434782, + "recall@50": 0.2391304347826087, + "recall@100": 0.4420289855072464, + "recall@200": 0.7971014492753623, + "recall@300": 0.8623188405797102, + "recall@500": 0.9130434782608695, + "recall@1000": 0.9202898550724637, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6370625586015203, + "ndcg_full@5": 0.6370625586015203, + "ndcg@10": 0.764478434406375, + "ndcg_full@10": 0.764478434406375, + "ndcg@20": 0.8116457837191688, + "ndcg_full@20": 0.8116457837191688, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 1000, + "num_ranked": 113, + "recall@10": 0.16129032258064516, + "recall@50": 0.3548387096774194, + "recall@100": 0.3548387096774194, + "recall@200": 0.45161290322580644, + "recall@300": 0.46774193548387094, + "recall@500": 0.5161290322580645, + "recall@1000": 0.5483870967741935, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.5957269594092457, + "ndcg_full@10": 0.5957269594092457, + "ndcg@20": 0.681164806313508, + "ndcg_full@20": 0.6544518123890973, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 1000, + "num_ranked": 27, + "recall@10": 0.25, + "recall@50": 0.45, + "recall@100": 0.55, + "recall@200": 0.65, + "recall@300": 0.7, + "recall@500": 0.8, + "recall@1000": 0.85, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8240777442366682, + "ndcg_full@10": 0.8240777442366682, + "ndcg@20": 0.9106645857301331, + "ndcg_full@20": 0.7489462487827055, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 1000, + "num_ranked": 21, + "recall@10": 0.05384615384615385, + "recall@50": 0.13076923076923078, + "recall@100": 0.2, + "recall@200": 0.3, + "recall@300": 0.38461538461538464, + "recall@500": 0.5538461538461539, + "recall@1000": 0.7230769230769231, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7309662895581445, + "ndcg_full@10": 0.6834991618708629, + "ndcg@20": 0.8819681009102182, + "ndcg_full@20": 0.5666306858734528, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 1000, + "num_ranked": 20, + "recall@10": 0.03211009174311927, + "recall@50": 0.0779816513761468, + "recall@100": 0.11467889908256881, + "recall@200": 0.1559633027522936, + "recall@300": 0.1926605504587156, + "recall@500": 0.28440366972477066, + "recall@1000": 0.43119266055045874, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.8622790868454683, + "ndcg_full@10": 0.5933607200949346, + "ndcg@20": 0.9032119016250898, + "ndcg_full@20": 0.4011137786022194, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 1000, + "num_ranked": 166, + "recall@10": 0.03289473684210526, + "recall@50": 0.1513157894736842, + "recall@100": 0.2894736842105263, + "recall@200": 0.45394736842105265, + "recall@300": 0.5723684210526315, + "recall@500": 0.6480263157894737, + "recall@1000": 0.7401315789473685, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.83206413723857, + "ndcg_full@5": 0.83206413723857, + "ndcg@10": 0.8657274175105066, + "ndcg_full@10": 0.8657274175105066, + "ndcg@20": 0.9133448594678868, + "ndcg_full@20": 0.9133448594678868, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 1000, + "num_ranked": 175, + "recall@10": 0.09090909090909091, + "recall@50": 0.24242424242424243, + "recall@100": 0.3939393939393939, + "recall@200": 0.6060606060606061, + "recall@300": 0.7424242424242424, + "recall@500": 0.8181818181818182, + "recall@1000": 0.8787878787878788, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.43024278050533904, + "ndcg_full@10": 0.43024278050533904, + "ndcg@20": 0.3149713928152083, + "ndcg_full@20": 0.3149713928152083, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 1000, + "num_ranked": 157, + "recall@10": 0.23809523809523808, + "recall@50": 0.5238095238095238, + "recall@100": 0.5238095238095238, + "recall@200": 0.5714285714285714, + "recall@300": 0.7142857142857143, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8095238095238095, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.03723779416598803, + "ndcg_full@5": 0.03723779416598803, + "ndcg@10": 0.11242817278653126, + "ndcg_full@10": 0.10885178830230774, + "ndcg@20": 0.21011862018878344, + "ndcg_full@20": 0.15075117556955248, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 1000, + "num_ranked": 116, + "recall@10": 0.08196721311475409, + "recall@50": 0.3442622950819672, + "recall@100": 0.5983606557377049, + "recall@200": 0.860655737704918, + "recall@300": 0.9344262295081968, + "recall@500": 0.9590163934426229, + "recall@1000": 0.9590163934426229, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5538466623591201, + "ndcg_full@5": 0.5538466623591201, + "ndcg@10": 0.7104770117651424, + "ndcg_full@10": 0.7104770117651424, + "ndcg@20": 0.6696538371420097, + "ndcg_full@20": 0.6696538371420097, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1000, + "num_ranked": 134, + "recall@10": 0.05847953216374269, + "recall@50": 0.2807017543859649, + "recall@100": 0.52046783625731, + "recall@200": 0.695906432748538, + "recall@300": 0.7017543859649122, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9263980367447917, + "ndcg_full@20": 0.9263980367447917, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 1000, + "num_ranked": 113, + "recall@10": 0.14583333333333334, + "recall@50": 0.6458333333333334, + "recall@100": 0.9583333333333334, + "recall@200": 0.9583333333333334, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.4673791184803734, + "ndcg_full@10": 0.4673791184803734, + "ndcg@20": 0.5126842532184583, + "ndcg_full@20": 0.5126842532184583, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 1000, + "num_ranked": 132, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.1, + "recall@200": 0.23333333333333334, + "recall@300": 0.36666666666666664, + "recall@500": 0.6, + "recall@1000": 0.7333333333333333, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.12325583919272352, + "ndcg_full@20": 0.03730675036644464, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1000, + "num_ranked": 55, + "recall@10": 0.14285714285714285, + "recall@50": 0.5102040816326531, + "recall@100": 0.6938775510204082, + "recall@200": 0.7551020408163265, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6379365213265478, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.5438521633994955, + "ndcg_full@10": 0.5438521633994955, + "ndcg@20": 0.5374033449223863, + "ndcg_full@20": 0.5374033449223863, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 1000, + "num_ranked": 102, + "recall@10": 0.07142857142857142, + "recall@50": 0.14285714285714285, + "recall@100": 0.21428571428571427, + "recall@200": 0.5, + "recall@300": 0.6428571428571429, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.41774584760242617, + "ndcg_full@10": 0.2710884709729799, + "ndcg@20": 0.41774584760242617, + "ndcg_full@20": 0.2425835141515238, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 1000, + "num_ranked": 171, + "recall@10": 0.042735042735042736, + "recall@50": 0.2094017094017094, + "recall@100": 0.3974358974358974, + "recall@200": 0.6452991452991453, + "recall@300": 0.8034188034188035, + "recall@500": 0.8547008547008547, + "recall@1000": 0.8632478632478633, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9603895103778499, + "ndcg_full@5": 0.9603895103778499, + "ndcg@10": 0.9241428704743823, + "ndcg_full@10": 0.9241428704743823, + "ndcg@20": 0.9144740142506342, + "ndcg_full@20": 0.9144740142506342, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 1000, + "num_ranked": 180, + "recall@10": 0.057803468208092484, + "recall@50": 0.24855491329479767, + "recall@100": 0.5202312138728323, + "recall@200": 0.8670520231213873, + "recall@300": 0.953757225433526, + "recall@500": 0.9595375722543352, + "recall@1000": 0.9595375722543352, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.5181822849033655, + "ndcg_full@5": 0.5181822849033655, + "ndcg@10": 0.6210790476651611, + "ndcg_full@10": 0.6210790476651611, + "ndcg@20": 0.6807158635618271, + "ndcg_full@20": 0.6807158635618271, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 1000, + "num_ranked": 121, + "recall@10": 0.072, + "recall@50": 0.288, + "recall@100": 0.4, + "recall@200": 0.472, + "recall@300": 0.544, + "recall@500": 0.6, + "recall@1000": 0.608, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.5135081645717171, + "ndcg_full@10": 0.5135081645717171, + "ndcg@20": 0.6168127955661226, + "ndcg_full@20": 0.6168127955661226, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 1000, + "num_ranked": 129, + "recall@10": 0.05319148936170213, + "recall@50": 0.2553191489361702, + "recall@100": 0.48936170212765956, + "recall@200": 0.6914893617021277, + "recall@300": 0.7819148936170213, + "recall@500": 0.8351063829787234, + "recall@1000": 0.8617021276595744, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9025937412599945, + "ndcg_full@10": 0.9025937412599945, + "ndcg@20": 0.8840772334129118, + "ndcg_full@20": 0.8840772334129118, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1000, + "num_ranked": 30, + "recall@10": 0.22580645161290322, + "recall@50": 0.3548387096774194, + "recall@100": 0.3548387096774194, + "recall@200": 0.41935483870967744, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.6345607719553766, + "ndcg_full@10": 0.5511391849981765, + "ndcg@20": 0.8334588529510848, + "ndcg_full@20": 0.5306752520627985, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 1000, + "num_ranked": 61, + "recall@10": 0.016574585635359115, + "recall@50": 0.12154696132596685, + "recall@100": 0.2154696132596685, + "recall@200": 0.32044198895027626, + "recall@300": 0.36464088397790057, + "recall@500": 0.48066298342541436, + "recall@1000": 0.6795580110497238, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.5634440045622183, + "ndcg_full@10": 0.5455206287241073, + "ndcg@20": 0.46055515149777837, + "ndcg_full@20": 0.3694362880393121, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 1000, + "num_ranked": 61, + "recall@10": 0.16071428571428573, + "recall@50": 0.6607142857142857, + "recall@100": 0.75, + "recall@200": 0.9285714285714286, + "recall@300": 0.9464285714285714, + "recall@500": 0.9642857142857143, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.7274710542151983, + "ndcg_full@10": 0.7274710542151983, + "ndcg@20": 0.7357406006640165, + "ndcg_full@20": 0.7357406006640165, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 1000, + "num_ranked": 76, + "recall@10": 0.13793103448275862, + "recall@50": 0.6551724137931034, + "recall@100": 0.7931034482758621, + "recall@200": 0.8275862068965517, + "recall@300": 0.8275862068965517, + "recall@500": 0.8275862068965517, + "recall@1000": 0.8275862068965517, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010481, + "ndcg_full@10": 0.9363792118010481, + "ndcg@20": 0.7861389447347518, + "ndcg_full@20": 0.7861389447347518, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 1000, + "num_ranked": 158, + "recall@10": 0.014705882352941176, + "recall@50": 0.058823529411764705, + "recall@100": 0.1323529411764706, + "recall@200": 0.29411764705882354, + "recall@300": 0.35294117647058826, + "recall@500": 0.4264705882352941, + "recall@1000": 0.5882352941176471, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.34973201611667765, + "ndcg_full@10": 0.31488013066763093, + "ndcg@20": 0.534855677572025, + "ndcg_full@20": 0.3107802879854167, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 1000, + "num_ranked": 182, + "recall@10": 0.0, + "recall@50": 0.18181818181818182, + "recall@100": 0.2727272727272727, + "recall@200": 0.5151515151515151, + "recall@300": 0.6363636363636364, + "recall@500": 0.696969696969697, + "recall@1000": 0.7575757575757576, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.42600000043464953, + "ndcg_full@10": 0.3679212572135327, + "ndcg@20": 0.36475440872136095, + "ndcg_full@20": 0.2642622118300481, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 1000, + "num_ranked": 245, + "recall@10": 0.02356020942408377, + "recall@50": 0.1099476439790576, + "recall@100": 0.20680628272251309, + "recall@200": 0.3743455497382199, + "recall@300": 0.518324607329843, + "recall@500": 0.6518324607329843, + "recall@1000": 0.7120418848167539, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6754160255557637, + "ndcg_full@10": 0.6754160255557637, + "ndcg@20": 0.7521394628949508, + "ndcg_full@20": 0.7521394628949508, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 1000, + "num_ranked": 97, + "recall@10": 0.1282051282051282, + "recall@50": 0.46153846153846156, + "recall@100": 0.6410256410256411, + "recall@200": 0.7435897435897436, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.42049964660493067, + "ndcg_full@10": 0.42049964660493067, + "ndcg@20": 0.48342282822382243, + "ndcg_full@20": 0.45134682310996177, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 1000, + "num_ranked": 148, + "recall@10": 0.0228310502283105, + "recall@50": 0.0776255707762557, + "recall@100": 0.1506849315068493, + "recall@200": 0.2785388127853881, + "recall@300": 0.3607305936073059, + "recall@500": 0.4337899543378995, + "recall@1000": 0.5251141552511416, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6057698118325185, + "ndcg_full@5": 0.6057698118325185, + "ndcg@10": 0.6983976219654084, + "ndcg_full@10": 0.6983976219654084, + "ndcg@20": 0.5267003765028994, + "ndcg_full@20": 0.5267003765028994, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 1000, + "num_ranked": 61, + "recall@10": 0.08333333333333333, + "recall@50": 0.2361111111111111, + "recall@100": 0.2638888888888889, + "recall@200": 0.3055555555555556, + "recall@300": 0.3472222222222222, + "recall@500": 0.4027777777777778, + "recall@1000": 0.5416666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9305687780632227, + "ndcg_full@10": 0.9305687780632227, + "ndcg@20": 0.7959701508934467, + "ndcg_full@20": 0.7440703008266993, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 1000, + "num_ranked": 11, + "recall@10": 0.075, + "recall@50": 0.1, + "recall@100": 0.125, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8972754076830647, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.9707983139693702, + "ndcg_full@10": 0.4478985130939463, + "ndcg@20": 0.9707983139693702, + "ndcg_full@20": 0.33545734646037084, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 1000, + "num_ranked": 171, + "recall@10": 0.023076923076923078, + "recall@50": 0.1076923076923077, + "recall@100": 0.17307692307692307, + "recall@200": 0.2653846153846154, + "recall@300": 0.36538461538461536, + "recall@500": 0.46153846153846156, + "recall@1000": 0.5807692307692308, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8251154201194959, + "ndcg_full@10": 0.8251154201194959, + "ndcg@20": 0.7428762311758056, + "ndcg_full@20": 0.7428762311758056, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 1000, + "num_ranked": 174, + "recall@10": 0.03896103896103896, + "recall@50": 0.24675324675324675, + "recall@100": 0.4025974025974026, + "recall@200": 0.6493506493506493, + "recall@300": 0.7142857142857143, + "recall@500": 0.8051948051948052, + "recall@1000": 0.8961038961038961, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.6268473080189253, + "ndcg_full@10": 0.6268473080189253, + "ndcg@20": 0.5760042984408958, + "ndcg_full@20": 0.5760042984408958, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 1000, + "num_ranked": 85, + "recall@10": 0.11538461538461539, + "recall@50": 0.47435897435897434, + "recall@100": 0.7692307692307693, + "recall@200": 0.8717948717948718, + "recall@300": 0.9102564102564102, + "recall@500": 0.9102564102564102, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.735182641392979, + "ndcg_full@5": 0.735182641392979, + "ndcg@10": 0.692466208912761, + "ndcg_full@10": 0.692466208912761, + "ndcg@20": 0.6771603966021937, + "ndcg_full@20": 0.6771603966021937, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 1000, + "num_ranked": 118, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.5786163522012578, + "recall@200": 0.8930817610062893, + "recall@300": 0.9056603773584906, + "recall@500": 0.9182389937106918, + "recall@1000": 0.9182389937106918, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8761912069108442, + "ndcg_full@20": 0.8761912069108442, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 1000, + "num_ranked": 179, + "recall@10": 0.026515151515151516, + "recall@50": 0.1590909090909091, + "recall@100": 0.29924242424242425, + "recall@200": 0.5606060606060606, + "recall@300": 0.696969696969697, + "recall@500": 0.7803030303030303, + "recall@1000": 0.7992424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9270848374537759, + "ndcg_full@20": 0.9270848374537759, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 1000, + "num_ranked": 143, + "recall@10": 0.06134969325153374, + "recall@50": 0.25153374233128833, + "recall@100": 0.3312883435582822, + "recall@200": 0.3619631901840491, + "recall@300": 0.4171779141104294, + "recall@500": 0.5214723926380368, + "recall@1000": 0.6134969325153374, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.5988751049740155, + "ndcg_full@10": 0.5988751049740155, + "ndcg@20": 0.6740386262486193, + "ndcg_full@20": 0.6740386262486193, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 1000, + "num_ranked": 101, + "recall@10": 0.12857142857142856, + "recall@50": 0.6142857142857143, + "recall@100": 0.9571428571428572, + "recall@200": 0.9571428571428572, + "recall@300": 0.9571428571428572, + "recall@500": 0.9571428571428572, + "recall@1000": 0.9571428571428572, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7080630992831928, + "ndcg_full@10": 0.7080630992831928, + "ndcg@20": 0.8115934564667806, + "ndcg_full@20": 0.8115934564667806, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1000, + "num_ranked": 153, + "recall@10": 0.055865921787709494, + "recall@50": 0.20670391061452514, + "recall@100": 0.3407821229050279, + "recall@200": 0.4692737430167598, + "recall@300": 0.553072625698324, + "recall@500": 0.6368715083798883, + "recall@1000": 0.6536312849162011, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7754155469362467, + "ndcg_full@5": 0.7754155469362467, + "ndcg@10": 0.7710397016346769, + "ndcg_full@10": 0.7710397016346769, + "ndcg@20": 0.6866317912042463, + "ndcg_full@20": 0.6866317912042463, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 1000, + "num_ranked": 53, + "recall@10": 0.0, + "recall@50": 0.03571428571428571, + "recall@100": 0.13095238095238096, + "recall@200": 0.16666666666666666, + "recall@300": 0.20238095238095238, + "recall@500": 0.35714285714285715, + "recall@1000": 0.5357142857142857, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.0, + "ndcg_full@20": 0.0, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 1000, + "num_ranked": 201, + "recall@10": 0.03007518796992481, + "recall@50": 0.16165413533834586, + "recall@100": 0.31203007518796994, + "recall@200": 0.556390977443609, + "recall@300": 0.7330827067669173, + "recall@500": 0.8759398496240601, + "recall@1000": 0.924812030075188, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7632636606403641, + "ndcg_full@5": 0.7632636606403641, + "ndcg@10": 0.7454044799382791, + "ndcg_full@10": 0.7454044799382791, + "ndcg@20": 0.6612260673542082, + "ndcg_full@20": 0.6612260673542082, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 1000, + "num_ranked": 77, + "recall@10": 0.0, + "recall@50": 0.09523809523809523, + "recall@100": 0.19047619047619047, + "recall@200": 0.23809523809523808, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7386919056633241, + "ndcg_full@5": 0.44615333764087994, + "ndcg@10": 0.7386919056633241, + "ndcg_full@10": 0.28952298823485745, + "ndcg@20": 0.8047881815478757, + "ndcg_full@20": 0.2416247222273939, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 1000, + "num_ranked": 212, + "recall@10": 0.028662420382165606, + "recall@50": 0.14331210191082802, + "recall@100": 0.28343949044585987, + "recall@200": 0.5191082802547771, + "recall@300": 0.6751592356687898, + "recall@500": 0.8121019108280255, + "recall@1000": 0.8885350318471338, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8401275584534302, + "ndcg_full@10": 0.8401275584534302, + "ndcg@20": 0.81564575011142, + "ndcg_full@20": 0.81564575011142, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 1000, + "num_ranked": 168, + "recall@10": 0.10416666666666667, + "recall@50": 0.4270833333333333, + "recall@100": 0.65625, + "recall@200": 0.9270833333333334, + "recall@300": 0.9375, + "recall@500": 0.9479166666666666, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.7885497208855957, + "ndcg_full@10": 0.7885497208855957, + "ndcg@20": 0.7921671714143951, + "ndcg_full@20": 0.7921671714143951, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 1000, + "num_ranked": 199, + "recall@10": 0.030395136778115502, + "recall@50": 0.13677811550151975, + "recall@100": 0.270516717325228, + "recall@200": 0.5106382978723404, + "recall@300": 0.668693009118541, + "recall@500": 0.7416413373860182, + "recall@1000": 0.7781155015197568, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.40777831251641367, + "ndcg_full@5": 0.40777831251641367, + "ndcg@10": 0.5462574254588145, + "ndcg_full@10": 0.5462574254588145, + "ndcg@20": 0.5484103205782435, + "ndcg_full@20": 0.5484103205782435, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 1000, + "num_ranked": 75, + "recall@10": 0.2222222222222222, + "recall@50": 0.6666666666666666, + "recall@100": 0.7222222222222222, + "recall@200": 0.8888888888888888, + "recall@300": 0.8888888888888888, + "recall@500": 0.8888888888888888, + "recall@1000": 0.9444444444444444, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.44916890612750654, + "ndcg_full@5": 0.44916890612750654, + "ndcg@10": 0.33067902033274277, + "ndcg_full@10": 0.33067902033274277, + "ndcg@20": 0.5523919288036004, + "ndcg_full@20": 0.4317021945268711, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 1000, + "num_ranked": 223, + "recall@10": 0.026490066225165563, + "recall@50": 0.152317880794702, + "recall@100": 0.24503311258278146, + "recall@200": 0.45695364238410596, + "recall@300": 0.6357615894039735, + "recall@500": 0.8344370860927153, + "recall@1000": 0.8741721854304636, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.5182838217145528, + "ndcg_full@10": 0.5182838217145528, + "ndcg@20": 0.546380119284524, + "ndcg_full@20": 0.546380119284524, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + } + } + } + } + }, + { + "name": "h100_medcpt", + "results_dir": "h100_medcpt/results_trec22", + "evaluation_input_sha256": "667b1a3c9716f25e609203587528a755fa1e4df5ef4314ed17720b03963e1e11", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08118516647690757, + "recall@50": 0.2880848623399002, + "recall@100": 0.43222345110104016, + "recall@200": 0.5799385656724092, + "recall@300": 0.6546541934415785, + "recall@500": 0.7331629170029889, + "recall@1000": 0.7802276425435689, + "ndcg@5": 0.7408053618433074, + "ndcg@10": 0.7144332567540345, + "ndcg@20": 0.7298898350582133, + "ndcg_full@5": 0.7332641778086277, + "ndcg_full@10": 0.6900931299177415, + "ndcg_full@20": 0.6639315492247831, + "P@10(rel>=1)": 0.7119999999999997, + "P@10(eligible)": 0.6299999999999999, + "graded_P@10": 0.6709999999999999, + "unjudged_fraction@10": 0.17800000000000002 + }, + "median": { + "recall@10": 0.05691846814023156, + "recall@50": 0.22503532736693357, + "recall@100": 0.37528211322673954, + "recall@200": 0.5508658008658008, + "recall@300": 0.6754061072082771, + "recall@500": 0.7571321722406239, + "recall@1000": 0.8233823821226322, + "ndcg@5": 0.8613632863224759, + "ndcg@10": 0.8515805547309923, + "ndcg@20": 0.7778724559592018, + "ndcg_full@5": 0.8480537119587682, + "ndcg_full@10": 0.818346058117099, + "ndcg_full@20": 0.7552010964998854, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 167, + "recall@10": 0.046632124352331605, + "recall@50": 0.22279792746113988, + "recall@100": 0.38860103626943004, + "recall@200": 0.5803108808290155, + "recall@300": 0.5906735751295337, + "recall@500": 0.6113989637305699, + "recall@1000": 0.616580310880829, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8551663198246187, + "ndcg_full@10": 0.8551663198246187, + "ndcg@20": 0.890359926283359, + "ndcg_full@20": 0.890359926283359, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 81, + "recall@10": 0.02512562814070352, + "recall@50": 0.10301507537688442, + "recall@100": 0.18341708542713567, + "recall@200": 0.32160804020100503, + "recall@300": 0.43467336683417085, + "recall@500": 0.6155778894472361, + "recall@1000": 0.8266331658291457, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.669580102636808, + "ndcg_full@5": 0.669580102636808, + "ndcg@10": 0.753769701383641, + "ndcg_full@10": 0.753769701383641, + "ndcg@20": 0.8137059539641261, + "ndcg_full@20": 0.787177812826073, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 106, + "recall@10": 0.12658227848101267, + "recall@50": 0.46835443037974683, + "recall@100": 0.759493670886076, + "recall@200": 0.8734177215189873, + "recall@300": 0.9113924050632911, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8267954144698881, + "ndcg_full@20": 0.8267954144698881, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02040816326530612, + "recall@50": 0.09329446064139942, + "recall@100": 0.17784256559766765, + "recall@200": 0.35276967930029157, + "recall@300": 0.5043731778425656, + "recall@500": 0.7055393586005831, + "recall@1000": 0.8221574344023324, + "unjudged_fraction@10": 0.8, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9826249136505656, + "ndcg_full@20": 0.9826249136505656, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 147, + "recall@10": 0.09090909090909091, + "recall@50": 0.38181818181818183, + "recall@100": 0.6, + "recall@200": 0.7181818181818181, + "recall@300": 0.7636363636363637, + "recall@500": 0.7727272727272727, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8643145546088337, + "ndcg_full@10": 0.8643145546088337, + "ndcg@20": 0.7182504375706894, + "ndcg_full@20": 0.7182504375706894, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 187, + "recall@10": 0.057971014492753624, + "recall@50": 0.2608695652173913, + "recall@100": 0.4927536231884058, + "recall@200": 0.7753623188405797, + "recall@300": 0.8623188405797102, + "recall@500": 0.9057971014492754, + "recall@1000": 0.9202898550724637, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9603789329441844, + "ndcg_full@20": 0.9603789329441844, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 144, + "recall@10": 0.12903225806451613, + "recall@50": 0.3225806451612903, + "recall@100": 0.4032258064516129, + "recall@200": 0.5, + "recall@300": 0.5806451612903226, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.45577863618303804, + "ndcg_full@10": 0.45577863618303804, + "ndcg@20": 0.6033755841431186, + "ndcg_full@20": 0.5797132220187688, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 31, + "recall@10": 0.2, + "recall@50": 0.5, + "recall@100": 0.6, + "recall@200": 0.75, + "recall@300": 0.8, + "recall@500": 0.925, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7015099647232446, + "ndcg_full@10": 0.7015099647232446, + "ndcg@20": 0.7654399485601343, + "ndcg_full@20": 0.6611891206035, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 21, + "recall@10": 0.05384615384615385, + "recall@50": 0.13846153846153847, + "recall@100": 0.23076923076923078, + "recall@200": 0.33076923076923076, + "recall@300": 0.46923076923076923, + "recall@500": 0.6307692307692307, + "recall@1000": 0.7769230769230769, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.9270734479801118, + "ndcg_full@10": 0.8975828762407047, + "ndcg@20": 0.9508641731861271, + "ndcg_full@20": 0.6489592704080166, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 22, + "recall@10": 0.03211009174311927, + "recall@50": 0.06880733944954129, + "recall@100": 0.11009174311926606, + "recall@200": 0.1559633027522936, + "recall@300": 0.1834862385321101, + "recall@500": 0.2798165137614679, + "recall@1000": 0.43577981651376146, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8899187097780359, + "ndcg_full@10": 0.7125523635307982, + "ndcg@20": 0.960275513017743, + "ndcg_full@20": 0.4962143174792751, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 197, + "recall@10": 0.029605263157894735, + "recall@50": 0.15460526315789475, + "recall@100": 0.27631578947368424, + "recall@200": 0.4407894736842105, + "recall@300": 0.5723684210526315, + "recall@500": 0.7335526315789473, + "recall@1000": 0.819078947368421, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.6698623505528798, + "ndcg_full@10": 0.6698623505528798, + "ndcg@20": 0.7682865653662918, + "ndcg_full@20": 0.7682865653662918, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.06060606060606061, + "recall@50": 0.22727272727272727, + "recall@100": 0.3484848484848485, + "recall@200": 0.5303030303030303, + "recall@300": 0.696969696969697, + "recall@500": 0.8636363636363636, + "recall@1000": 0.9090909090909091, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.21726071285222986, + "ndcg_full@10": 0.21726071285222986, + "ndcg@20": 0.1402129701779605, + "ndcg_full@20": 0.1402129701779605, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 213, + "recall@10": 0.23809523809523808, + "recall@50": 0.5238095238095238, + "recall@100": 0.6190476190476191, + "recall@200": 0.7619047619047619, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.22466337602289155, + "ndcg_full@10": 0.22466337602289155, + "ndcg@20": 0.2960297159957152, + "ndcg_full@20": 0.24413716341789202, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.08196721311475409, + "recall@50": 0.38524590163934425, + "recall@100": 0.639344262295082, + "recall@200": 0.8934426229508197, + "recall@300": 0.9426229508196722, + "recall@500": 0.9590163934426229, + "recall@1000": 0.9590163934426229, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8601100728327754, + "ndcg_full@10": 0.8601100728327754, + "ndcg@20": 0.8356308955502236, + "ndcg_full@20": 0.8356308955502236, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1624, + "num_ranked": 124, + "recall@10": 0.05847953216374269, + "recall@50": 0.2807017543859649, + "recall@100": 0.5380116959064327, + "recall@200": 0.7076023391812866, + "recall@300": 0.7134502923976608, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9775956781991642, + "ndcg_full@20": 0.9775956781991642, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 134, + "recall@10": 0.1875, + "recall@50": 0.6458333333333334, + "recall@100": 0.9166666666666666, + "recall@200": 0.9375, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.8129980723195801, + "ndcg_full@10": 0.8129980723195801, + "ndcg@20": 0.7721172786557556, + "ndcg_full@20": 0.7721172786557556, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 126, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.1, + "recall@200": 0.3333333333333333, + "recall@300": 0.5, + "recall@500": 0.7333333333333333, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.25, + "ndcg_full@20": 0.035510009908414325, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1733, + "num_ranked": 90, + "recall@10": 0.16326530612244897, + "recall@50": 0.5714285714285714, + "recall@100": 0.6938775510204082, + "recall@200": 0.7551020408163265, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.795760747566305, + "ndcg_full@5": 0.795760747566305, + "ndcg@10": 0.7317772547824374, + "ndcg_full@10": 0.7317772547824374, + "ndcg@20": 0.6819851510343515, + "ndcg_full@20": 0.6819851510343515, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 1833, + "num_ranked": 119, + "recall@10": 0.21428571428571427, + "recall@50": 0.35714285714285715, + "recall@100": 0.5, + "recall@200": 0.5714285714285714, + "recall@300": 0.5714285714285714, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.41720250447561474, + "ndcg_full@10": 0.3340514446642156, + "ndcg@20": 0.5296870329176367, + "ndcg_full@20": 0.37952118535960916, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 194, + "recall@10": 0.042735042735042736, + "recall@50": 0.21367521367521367, + "recall@100": 0.3974358974358974, + "recall@200": 0.6581196581196581, + "recall@300": 0.7863247863247863, + "recall@500": 0.8675213675213675, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9055943150390882, + "ndcg_full@5": 0.9055943150390882, + "ndcg@10": 0.9071381111359174, + "ndcg_full@10": 0.9071381111359174, + "ndcg@20": 0.9203418183906833, + "ndcg_full@20": 0.9203418183906833, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 206, + "recall@10": 0.04046242774566474, + "recall@50": 0.24277456647398843, + "recall@100": 0.47398843930635837, + "recall@200": 0.8497109826589595, + "recall@300": 0.9653179190751445, + "recall@500": 0.976878612716763, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.9572416874849684, + "ndcg_full@20": 0.9572416874849684, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.08, + "recall@50": 0.344, + "recall@100": 0.464, + "recall@200": 0.528, + "recall@300": 0.568, + "recall@500": 0.608, + "recall@1000": 0.624, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.8491484326232737, + "ndcg_full@20": 0.8491484326232737, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 158, + "recall@10": 0.05319148936170213, + "recall@50": 0.2553191489361702, + "recall@100": 0.48936170212765956, + "recall@200": 0.7872340425531915, + "recall@300": 0.8191489361702128, + "recall@500": 0.8404255319148937, + "recall@1000": 0.8723404255319149, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8463645272092231, + "ndcg_full@10": 0.8463645272092231, + "ndcg@20": 0.7927074974404416, + "ndcg_full@20": 0.7927074974404416, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1424, + "num_ranked": 59, + "recall@10": 0.22580645161290322, + "recall@50": 0.2903225806451613, + "recall@100": 0.3548387096774194, + "recall@200": 0.45161290322580644, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8543630247089099, + "ndcg_full@10": 0.7420454619022024, + "ndcg@20": 0.9517835178240142, + "ndcg_full@20": 0.606014269861165, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 67, + "recall@10": 0.011049723756906077, + "recall@50": 0.11602209944751381, + "recall@100": 0.22099447513812154, + "recall@200": 0.30939226519337015, + "recall@300": 0.40331491712707185, + "recall@500": 0.569060773480663, + "recall@1000": 0.7182320441988951, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5817744828317774, + "ndcg_full@5": 0.5817744828317774, + "ndcg@10": 0.5195508888095226, + "ndcg_full@10": 0.5195508888095226, + "ndcg@20": 0.5609585160389517, + "ndcg_full@20": 0.4614914130655171, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 63, + "recall@10": 0.16071428571428573, + "recall@50": 0.6964285714285714, + "recall@100": 0.7857142857142857, + "recall@200": 0.9464285714285714, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8556265493118265, + "ndcg_full@10": 0.8556265493118265, + "ndcg@20": 0.7974332121324702, + "ndcg_full@20": 0.7974332121324702, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.13793103448275862, + "recall@50": 0.6896551724137931, + "recall@100": 0.8275862068965517, + "recall@200": 0.8275862068965517, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9999999999999999, + "ndcg_full@5": 0.9999999999999999, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.8691288518422549, + "ndcg_full@20": 0.8691288518422549, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.014705882352941176, + "recall@50": 0.058823529411764705, + "recall@100": 0.19117647058823528, + "recall@200": 0.3382352941176471, + "recall@300": 0.38235294117647056, + "recall@500": 0.47058823529411764, + "recall@1000": 0.5588235294117647, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.20483424751859086, + "ndcg_full@10": 0.20483424751859086, + "ndcg@20": 0.3348195907112801, + "ndcg_full@20": 0.24147246479371767, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 248, + "recall@10": 0.030303030303030304, + "recall@50": 0.15151515151515152, + "recall@100": 0.2727272727272727, + "recall@200": 0.3333333333333333, + "recall@300": 0.48484848484848486, + "recall@500": 0.696969696969697, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.20052526032438814, + "ndcg_full@10": 0.1731866333482261, + "ndcg@20": 0.15919336459842842, + "ndcg_full@20": 0.12439260273955823, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1256544502617801, + "recall@100": 0.21204188481675393, + "recall@200": 0.387434554973822, + "recall@300": 0.5549738219895288, + "recall@500": 0.7591623036649214, + "recall@1000": 0.824607329842932, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9808076809173086, + "ndcg_full@20": 0.9808076809173086, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 100, + "recall@10": 0.15384615384615385, + "recall@50": 0.5384615384615384, + "recall@100": 0.6923076923076923, + "recall@200": 0.8205128205128205, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.35266373843404347, + "ndcg_full@10": 0.35266373843404347, + "ndcg@20": 0.4646757760748367, + "ndcg_full@20": 0.44952658795461414, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 179, + "recall@10": 0.0273972602739726, + "recall@50": 0.091324200913242, + "recall@100": 0.1780821917808219, + "recall@200": 0.2557077625570776, + "recall@300": 0.2831050228310502, + "recall@500": 0.3515981735159817, + "recall@1000": 0.4885844748858447, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8, + "ndcg_full@5": 0.8, + "ndcg@10": 0.8702136849336308, + "ndcg_full@10": 0.8702136849336308, + "ndcg@20": 0.7449141540181888, + "ndcg_full@20": 0.7449141540181888, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.08333333333333333, + "recall@50": 0.2222222222222222, + "recall@100": 0.2361111111111111, + "recall@200": 0.3194444444444444, + "recall@300": 0.3472222222222222, + "recall@500": 0.5138888888888888, + "recall@1000": 0.5972222222222222, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.7197044482006457, + "ndcg_full@20": 0.6727773707065239, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 1445, + "num_ranked": 14, + "recall@10": 0.0625, + "recall@50": 0.1125, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9434113026660673, + "ndcg_full@5": 0.6707377907046331, + "ndcg@10": 0.985892414523795, + "ndcg_full@10": 0.4548624984012288, + "ndcg@20": 0.985892414523795, + "ndcg_full@20": 0.34067308164071963, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.03461538461538462, + "recall@50": 0.1, + "recall@100": 0.14615384615384616, + "recall@200": 0.23846153846153847, + "recall@300": 0.3153846153846154, + "recall@500": 0.45384615384615384, + "recall@1000": 0.6230769230769231, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9527971575195441, + "ndcg_full@5": 0.9527971575195441, + "ndcg@10": 0.823694043914618, + "ndcg_full@10": 0.823694043914618, + "ndcg@20": 0.8699170771745077, + "ndcg_full@20": 0.8699170771745077, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.012987012987012988, + "recall@50": 0.22077922077922077, + "recall@100": 0.45454545454545453, + "recall@200": 0.7792207792207793, + "recall@300": 0.8961038961038961, + "recall@500": 0.935064935064935, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.246787534112613, + "ndcg_full@5": 0.246787534112613, + "ndcg@10": 0.3225429345647659, + "ndcg_full@10": 0.3225429345647659, + "ndcg@20": 0.3466168649141672, + "ndcg_full@20": 0.3466168649141672, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 93, + "recall@10": 0.11538461538461539, + "recall@50": 0.5256410256410257, + "recall@100": 0.8076923076923077, + "recall@200": 0.8974358974358975, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7080630992831928, + "ndcg_full@10": 0.7080630992831928, + "ndcg@20": 0.7780492245431865, + "ndcg_full@20": 0.7780492245431865, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 129, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.5974842767295597, + "recall@200": 0.8867924528301887, + "recall@300": 0.9056603773584906, + "recall@500": 0.9119496855345912, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.03787878787878788, + "recall@50": 0.17045454545454544, + "recall@100": 0.3333333333333333, + "recall@200": 0.6136363636363636, + "recall@300": 0.7348484848484849, + "recall@500": 0.7916666666666666, + "recall@1000": 0.7954545454545454, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.921071679667761, + "ndcg_full@20": 0.921071679667761, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 190, + "recall@10": 0.06134969325153374, + "recall@50": 0.24539877300613497, + "recall@100": 0.3619631901840491, + "recall@200": 0.43558282208588955, + "recall@300": 0.49693251533742333, + "recall@500": 0.588957055214724, + "recall@1000": 0.6687116564417178, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8603818544462509, + "ndcg_full@10": 0.8603818544462509, + "ndcg@20": 0.765488038981582, + "ndcg_full@20": 0.765488038981582, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.12857142857142856, + "recall@50": 0.6571428571428571, + "recall@100": 0.9571428571428572, + "recall@200": 0.9571428571428572, + "recall@300": 0.9571428571428572, + "recall@500": 0.9857142857142858, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9256180470797137, + "ndcg_full@10": 0.9256180470797137, + "ndcg@20": 0.9519963162705626, + "ndcg_full@20": 0.9519963162705626, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1351, + "num_ranked": 191, + "recall@10": 0.055865921787709494, + "recall@50": 0.2011173184357542, + "recall@100": 0.35195530726256985, + "recall@200": 0.45251396648044695, + "recall@300": 0.5418994413407822, + "recall@500": 0.6480446927374302, + "recall@1000": 0.6759776536312849, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8874369434113023, + "ndcg_full@10": 0.8874369434113023, + "ndcg@20": 0.8738052137649573, + "ndcg_full@20": 0.8738052137649573, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 54, + "recall@10": 0.023809523809523808, + "recall@50": 0.09523809523809523, + "recall@100": 0.16666666666666666, + "recall@200": 0.2976190476190476, + "recall@300": 0.44047619047619047, + "recall@500": 0.6190476190476191, + "recall@1000": 0.7380952380952381, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.1403860944330722, + "ndcg_full@5": 0.08479005131840403, + "ndcg@10": 0.1403860944330722, + "ndcg_full@10": 0.05502294157452004, + "ndcg@20": 0.43329012620788643, + "ndcg_full@20": 0.12449591655912726, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.03759398496240601, + "recall@50": 0.18045112781954886, + "recall@100": 0.34210526315789475, + "recall@200": 0.6353383458646616, + "recall@300": 0.868421052631579, + "recall@500": 0.9360902255639098, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8274043288765652, + "ndcg_full@5": 0.8274043288765652, + "ndcg@10": 0.8487980847530745, + "ndcg_full@10": 0.8487980847530745, + "ndcg@20": 0.7776956873752173, + "ndcg_full@20": 0.7776956873752173, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 84, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.23809523809523808, + "recall@200": 0.3333333333333333, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.21836974929469322, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.34461577660059567, + "ndcg_full@10": 0.1736666713479918, + "ndcg@20": 0.34461577660059567, + "ndcg_full@20": 0.1330321302765889, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 232, + "recall@10": 0.028662420382165606, + "recall@50": 0.14331210191082802, + "recall@100": 0.267515923566879, + "recall@200": 0.4394904458598726, + "recall@300": 0.5923566878980892, + "recall@500": 0.8280254777070064, + "recall@1000": 0.9522292993630573, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6934886036955118, + "ndcg_full@5": 0.6934886036955118, + "ndcg@10": 0.7173471569936435, + "ndcg_full@10": 0.7173471569936435, + "ndcg@20": 0.715618284514998, + "ndcg_full@20": 0.715618284514998, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.10416666666666667, + "recall@50": 0.5, + "recall@100": 0.7916666666666666, + "recall@200": 0.9375, + "recall@300": 0.9375, + "recall@500": 0.96875, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5222030810488598, + "ndcg_full@5": 0.5222030810488598, + "ndcg@10": 0.5421130017808813, + "ndcg_full@10": 0.5421130017808813, + "ndcg@20": 0.6689846721566308, + "ndcg_full@20": 0.6689846721566308, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 235, + "recall@10": 0.030395136778115502, + "recall@50": 0.13677811550151975, + "recall@100": 0.2826747720364742, + "recall@200": 0.5227963525835866, + "recall@300": 0.668693009118541, + "recall@500": 0.7446808510638298, + "recall@1000": 0.78419452887538, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9276737537834677, + "ndcg_full@10": 0.9276737537834677, + "ndcg@20": 0.8453780727302511, + "ndcg_full@20": 0.8453780727302511, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 75, + "recall@10": 0.2777777777777778, + "recall@50": 0.7222222222222222, + "recall@100": 0.7222222222222222, + "recall@200": 0.8333333333333334, + "recall@300": 0.9444444444444444, + "recall@500": 0.9444444444444444, + "recall@1000": 0.9444444444444444, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.5173633627401372, + "ndcg_full@10": 0.5173633627401372, + "ndcg@20": 0.7318592596688709, + "ndcg_full@20": 0.5719584809433288, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 233, + "recall@10": 0.033112582781456956, + "recall@50": 0.152317880794702, + "recall@100": 0.2781456953642384, + "recall@200": 0.4966887417218543, + "recall@300": 0.6821192052980133, + "recall@500": 0.9602649006622517, + "recall@1000": 0.9867549668874173, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.6594803070902429, + "ndcg_full@10": 0.6594803070902429, + "ndcg@20": 0.6577384098309015, + "ndcg_full@20": 0.6577384098309015, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08118516647690757, + "recall@50": 0.2880848623399002, + "recall@100": 0.43222345110104016, + "recall@200": 0.5799385656724092, + "recall@300": 0.6546541934415785, + "recall@500": 0.7331629170029889, + "recall@1000": 0.7802276425435689, + "ndcg@5": 0.6437170705078057, + "ndcg@10": 0.6254178149701387, + "ndcg@20": 0.6225373823030882, + "ndcg_full@5": 0.6375544998475177, + "ndcg_full@10": 0.6042336184679615, + "ndcg_full@20": 0.5731118113507265, + "P@10(rel>=1)": 0.6219999999999999, + "P@10(eligible)": 0.546, + "graded_P@10": 0.584, + "unjudged_fraction@10": 0.17800000000000002 + }, + "median": { + "recall@10": 0.05691846814023156, + "recall@50": 0.22503532736693357, + "recall@100": 0.37528211322673954, + "recall@200": 0.5508658008658008, + "recall@300": 0.6754061072082771, + "recall@500": 0.7571321722406239, + "recall@1000": 0.8233823821226322, + "ndcg@5": 0.6917831836856678, + "ndcg@10": 0.6929341708736322, + "ndcg@20": 0.6712732078051686, + "ndcg_full@5": 0.6725956711234347, + "ndcg_full@10": 0.6687648839507221, + "ndcg_full@20": 0.6389544762588648, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 167, + "recall@10": 0.046632124352331605, + "recall@50": 0.22279792746113988, + "recall@100": 0.38860103626943004, + "recall@200": 0.5803108808290155, + "recall@300": 0.5906735751295337, + "recall@500": 0.6113989637305699, + "recall@1000": 0.616580310880829, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8551663198246187, + "ndcg_full@10": 0.8551663198246187, + "ndcg@20": 0.7792374667503018, + "ndcg_full@20": 0.7792374667503018, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 81, + "recall@10": 0.02512562814070352, + "recall@50": 0.10301507537688442, + "recall@100": 0.18341708542713567, + "recall@200": 0.32160804020100503, + "recall@300": 0.43467336683417085, + "recall@500": 0.6155778894472361, + "recall@1000": 0.8266331658291457, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.500654335872621, + "ndcg_full@5": 0.500654335872621, + "ndcg@10": 0.5338235125492861, + "ndcg_full@10": 0.5338235125492861, + "ndcg@20": 0.5723297106624574, + "ndcg_full@20": 0.5536708287064035, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 106, + "recall@10": 0.12658227848101267, + "recall@50": 0.46835443037974683, + "recall@100": 0.759493670886076, + "recall@200": 0.8734177215189873, + "recall@300": 0.9113924050632911, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7065443116025597, + "ndcg_full@10": 0.7065443116025597, + "ndcg@20": 0.7064872975670231, + "ndcg_full@20": 0.7064872975670231, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02040816326530612, + "recall@50": 0.09329446064139942, + "recall@100": 0.17784256559766765, + "recall@200": 0.35276967930029157, + "recall@300": 0.5043731778425656, + "recall@500": 0.7055393586005831, + "recall@1000": 0.8221574344023324, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.3589542101716347, + "ndcg_full@10": 0.3589542101716347, + "ndcg@20": 0.5107049419492824, + "ndcg_full@20": 0.5107049419492824, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 147, + "recall@10": 0.09090909090909091, + "recall@50": 0.38181818181818183, + "recall@100": 0.6, + "recall@200": 0.7181818181818181, + "recall@300": 0.7636363636363637, + "recall@500": 0.7727272727272727, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8643145546088337, + "ndcg_full@10": 0.8643145546088337, + "ndcg@20": 0.7182504375706894, + "ndcg_full@20": 0.7182504375706894, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 187, + "recall@10": 0.057971014492753624, + "recall@50": 0.2608695652173913, + "recall@100": 0.4927536231884058, + "recall@200": 0.7753623188405797, + "recall@300": 0.8623188405797102, + "recall@500": 0.9057971014492754, + "recall@1000": 0.9202898550724637, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.888556803394088, + "ndcg_full@20": 0.888556803394088, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 144, + "recall@10": 0.12903225806451613, + "recall@50": 0.3225806451612903, + "recall@100": 0.4032258064516129, + "recall@200": 0.5, + "recall@300": 0.5806451612903226, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.37183505243062814, + "ndcg_full@10": 0.37183505243062814, + "ndcg@20": 0.555346022875127, + "ndcg_full@20": 0.5335672186892557, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 31, + "recall@10": 0.2, + "recall@50": 0.5, + "recall@100": 0.6, + "recall@200": 0.75, + "recall@300": 0.8, + "recall@500": 0.925, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.611755947233009, + "ndcg_full@10": 0.611755947233009, + "ndcg@20": 0.7379910247823316, + "ndcg_full@20": 0.6374786651872419, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 21, + "recall@10": 0.05384615384615385, + "recall@50": 0.13846153846153847, + "recall@100": 0.23076923076923078, + "recall@200": 0.33076923076923076, + "recall@300": 0.46923076923076923, + "recall@500": 0.6307692307692307, + "recall@1000": 0.7769230769230769, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8421757737602428, + "ndcg_full@5": 0.8421757737602428, + "ndcg@10": 0.9270734479801118, + "ndcg_full@10": 0.8975828762407047, + "ndcg@20": 0.9383673882383845, + "ndcg_full@20": 0.6404302873304878, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 22, + "recall@10": 0.03211009174311927, + "recall@50": 0.06880733944954129, + "recall@100": 0.11009174311926606, + "recall@200": 0.1559633027522936, + "recall@300": 0.1834862385321101, + "recall@500": 0.2798165137614679, + "recall@1000": 0.43577981651376146, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.7837042816946693, + "ndcg_full@10": 0.627507133061656, + "ndcg@20": 0.8496230530920482, + "ndcg_full@20": 0.43903558685968364, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 197, + "recall@10": 0.029605263157894735, + "recall@50": 0.15460526315789475, + "recall@100": 0.27631578947368424, + "recall@200": 0.4407894736842105, + "recall@300": 0.5723684210526315, + "recall@500": 0.7335526315789473, + "recall@1000": 0.819078947368421, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.5750739861833288, + "ndcg_full@10": 0.5750739861833288, + "ndcg@20": 0.6355747785471889, + "ndcg_full@20": 0.6355747785471889, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.06060606060606061, + "recall@50": 0.22727272727272727, + "recall@100": 0.3484848484848485, + "recall@200": 0.5303030303030303, + "recall@300": 0.696969696969697, + "recall@500": 0.8636363636363636, + "recall@1000": 0.9090909090909091, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.21726071285222986, + "ndcg_full@10": 0.21726071285222986, + "ndcg@20": 0.1402129701779605, + "ndcg_full@20": 0.1402129701779605, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 213, + "recall@10": 0.23809523809523808, + "recall@50": 0.5238095238095238, + "recall@100": 0.6190476190476191, + "recall@200": 0.7619047619047619, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.08514311764162098, + "ndcg_full@10": 0.08514311764162098, + "ndcg@20": 0.2171055301786763, + "ndcg_full@20": 0.17904799902225615, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 119, + "recall@10": 0.08196721311475409, + "recall@50": 0.38524590163934425, + "recall@100": 0.639344262295082, + "recall@200": 0.8934426229508197, + "recall@300": 0.9426229508196722, + "recall@500": 0.9590163934426229, + "recall@1000": 0.9590163934426229, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8601100728327754, + "ndcg_full@10": 0.8601100728327754, + "ndcg@20": 0.8356308955502236, + "ndcg_full@20": 0.8356308955502236, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1624, + "num_ranked": 124, + "recall@10": 0.05847953216374269, + "recall@50": 0.2807017543859649, + "recall@100": 0.5380116959064327, + "recall@200": 0.7076023391812866, + "recall@300": 0.7134502923976608, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9428455055002956, + "ndcg_full@20": 0.9428455055002956, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 134, + "recall@10": 0.1875, + "recall@50": 0.6458333333333334, + "recall@100": 0.9166666666666666, + "recall@200": 0.9375, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.930681643161238, + "ndcg_full@5": 0.930681643161238, + "ndcg@10": 0.8087761378229487, + "ndcg_full@10": 0.8087761378229487, + "ndcg@20": 0.6970037957414373, + "ndcg_full@20": 0.6970037957414373, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 126, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.1, + "recall@200": 0.3333333333333333, + "recall@300": 0.5, + "recall@500": 0.7333333333333333, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.0, + "ndcg_full@20": 0.0, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1733, + "num_ranked": 90, + "recall@10": 0.16326530612244897, + "recall@50": 0.5714285714285714, + "recall@100": 0.6938775510204082, + "recall@200": 0.7551020408163265, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.5940001599593494, + "ndcg_full@10": 0.5940001599593494, + "ndcg@20": 0.45471803084017415, + "ndcg_full@20": 0.45471803084017415, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 1833, + "num_ranked": 119, + "recall@10": 0.21428571428571427, + "recall@50": 0.35714285714285715, + "recall@100": 0.5, + "recall@200": 0.5714285714285714, + "recall@300": 0.5714285714285714, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.40877874879535264, + "ndcg_full@10": 0.32730659600126993, + "ndcg@20": 0.40877874879535264, + "ndcg_full@20": 0.2928903024831147, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 194, + "recall@10": 0.042735042735042736, + "recall@50": 0.21367521367521367, + "recall@100": 0.3974358974358974, + "recall@200": 0.6581196581196581, + "recall@300": 0.7863247863247863, + "recall@500": 0.8675213675213675, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9055943150390882, + "ndcg_full@5": 0.9055943150390882, + "ndcg@10": 0.9071381111359174, + "ndcg_full@10": 0.9071381111359174, + "ndcg@20": 0.9203418183906833, + "ndcg_full@20": 0.9203418183906833, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 206, + "recall@10": 0.04046242774566474, + "recall@50": 0.24277456647398843, + "recall@100": 0.47398843930635837, + "recall@200": 0.8497109826589595, + "recall@300": 0.9653179190751445, + "recall@500": 0.976878612716763, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.716485063693381, + "ndcg_full@10": 0.716485063693381, + "ndcg@20": 0.7082248188971036, + "ndcg_full@20": 0.7082248188971036, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.08, + "recall@50": 0.344, + "recall@100": 0.464, + "recall@200": 0.528, + "recall@300": 0.568, + "recall@500": 0.608, + "recall@1000": 0.624, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.6846562617806994, + "ndcg_full@10": 0.6846562617806994, + "ndcg@20": 0.6552092661905347, + "ndcg_full@20": 0.6552092661905347, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 158, + "recall@10": 0.05319148936170213, + "recall@50": 0.2553191489361702, + "recall@100": 0.48936170212765956, + "recall@200": 0.7872340425531915, + "recall@300": 0.8191489361702128, + "recall@500": 0.8404255319148937, + "recall@1000": 0.8723404255319149, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8463645272092231, + "ndcg_full@10": 0.8463645272092231, + "ndcg@20": 0.756811484737759, + "ndcg_full@20": 0.756811484737759, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1424, + "num_ranked": 59, + "recall@10": 0.22580645161290322, + "recall@50": 0.2903225806451613, + "recall@100": 0.3548387096774194, + "recall@200": 0.45161290322580644, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.517113190207065, + "ndcg_full@10": 0.4491316746925709, + "ndcg@20": 0.6334385122329913, + "ndcg_full@20": 0.40331942117513897, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 67, + "recall@10": 0.011049723756906077, + "recall@50": 0.11602209944751381, + "recall@100": 0.22099447513812154, + "recall@200": 0.30939226519337015, + "recall@300": 0.40331491712707185, + "recall@500": 0.569060773480663, + "recall@1000": 0.7182320441988951, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5508310938724935, + "ndcg_full@5": 0.5508310938724935, + "ndcg@10": 0.43081561158780146, + "ndcg_full@10": 0.43081561158780146, + "ndcg@20": 0.5359281372259453, + "ndcg_full@20": 0.440899329056265, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 63, + "recall@10": 0.16071428571428573, + "recall@50": 0.6964285714285714, + "recall@100": 0.7857142857142857, + "recall@200": 0.9464285714285714, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8556265493118265, + "ndcg_full@10": 0.8556265493118265, + "ndcg@20": 0.7974332121324702, + "ndcg_full@20": 0.7974332121324702, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.13793103448275862, + "recall@50": 0.6896551724137931, + "recall@100": 0.8275862068965517, + "recall@200": 0.8275862068965517, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9999999999999999, + "ndcg_full@5": 0.9999999999999999, + "ndcg@10": 0.933745776545611, + "ndcg_full@10": 0.933745776545611, + "ndcg@20": 0.8691288518422549, + "ndcg_full@20": 0.8691288518422549, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.014705882352941176, + "recall@50": 0.058823529411764705, + "recall@100": 0.19117647058823528, + "recall@200": 0.3382352941176471, + "recall@300": 0.38235294117647056, + "recall@500": 0.47058823529411764, + "recall@1000": 0.5588235294117647, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.15850703974098102, + "ndcg_full@10": 0.15850703974098102, + "ndcg@20": 0.14184002211571345, + "ndcg_full@20": 0.10229526794987165, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 248, + "recall@10": 0.030303030303030304, + "recall@50": 0.15151515151515152, + "recall@100": 0.2727272727272727, + "recall@200": 0.3333333333333333, + "recall@300": 0.48484848484848486, + "recall@500": 0.696969696969697, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.20052526032438814, + "ndcg_full@10": 0.1731866333482261, + "ndcg@20": 0.15919336459842842, + "ndcg_full@20": 0.12439260273955823, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1256544502617801, + "recall@100": 0.21204188481675393, + "recall@200": 0.387434554973822, + "recall@300": 0.5549738219895288, + "recall@500": 0.7591623036649214, + "recall@1000": 0.824607329842932, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8364586133797037, + "ndcg_full@10": 0.8364586133797037, + "ndcg@20": 0.7684269938763747, + "ndcg_full@20": 0.7684269938763747, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 100, + "recall@10": 0.15384615384615385, + "recall@50": 0.5384615384615384, + "recall@100": 0.6923076923076923, + "recall@200": 0.8205128205128205, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.31954962756492084, + "ndcg_full@10": 0.31954962756492084, + "ndcg@20": 0.4351865776631515, + "ndcg_full@20": 0.42099878550384356, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 179, + "recall@10": 0.0273972602739726, + "recall@50": 0.091324200913242, + "recall@100": 0.1780821917808219, + "recall@200": 0.2557077625570776, + "recall@300": 0.2831050228310502, + "recall@500": 0.3515981735159817, + "recall@1000": 0.4885844748858447, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8, + "ndcg_full@5": 0.8, + "ndcg@10": 0.8023709622380476, + "ndcg_full@10": 0.8023709622380476, + "ndcg@20": 0.6800835588217071, + "ndcg_full@20": 0.6800835588217071, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 102, + "recall@10": 0.08333333333333333, + "recall@50": 0.2222222222222222, + "recall@100": 0.2361111111111111, + "recall@200": 0.3194444444444444, + "recall@300": 0.3472222222222222, + "recall@500": 0.5138888888888888, + "recall@1000": 0.5972222222222222, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7421246259355073, + "ndcg_full@10": 0.7421246259355073, + "ndcg@20": 0.6745977171470955, + "ndcg_full@20": 0.6306117456430069, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 1445, + "num_ranked": 14, + "recall@10": 0.0625, + "recall@50": 0.1125, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8972754076830647, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.9795097056968828, + "ndcg_full@10": 0.4519176995156636, + "ndcg@20": 0.9795097056968828, + "ndcg_full@20": 0.33846754982685545, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.03461538461538462, + "recall@50": 0.1, + "recall@100": 0.14615384615384616, + "recall@200": 0.23846153846153847, + "recall@300": 0.3153846153846154, + "recall@500": 0.45384615384615384, + "recall@1000": 0.6230769230769231, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9527971575195441, + "ndcg_full@5": 0.9527971575195441, + "ndcg@10": 0.823694043914618, + "ndcg_full@10": 0.823694043914618, + "ndcg@20": 0.7796898836809215, + "ndcg_full@20": 0.7796898836809215, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.012987012987012988, + "recall@50": 0.22077922077922077, + "recall@100": 0.45454545454545453, + "recall@200": 0.7792207792207793, + "recall@300": 0.8961038961038961, + "recall@500": 0.935064935064935, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.3124884370342297, + "ndcg_full@10": 0.3124884370342297, + "ndcg@20": 0.3029556264746201, + "ndcg_full@20": 0.3029556264746201, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 93, + "recall@10": 0.11538461538461539, + "recall@50": 0.5256410256410257, + "recall@100": 0.8076923076923077, + "recall@200": 0.8974358974358975, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7067463816554741, + "ndcg_full@10": 0.7067463816554741, + "ndcg@20": 0.7196537044556686, + "ndcg_full@20": 0.7196537044556686, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 129, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.5974842767295597, + "recall@200": 0.8867924528301887, + "recall@300": 0.9056603773584906, + "recall@500": 0.9119496855345912, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.03787878787878788, + "recall@50": 0.17045454545454544, + "recall@100": 0.3333333333333333, + "recall@200": 0.6136363636363636, + "recall@300": 0.7348484848484849, + "recall@500": 0.7916666666666666, + "recall@1000": 0.7954545454545454, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.921071679667761, + "ndcg_full@20": 0.921071679667761, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 190, + "recall@10": 0.06134969325153374, + "recall@50": 0.24539877300613497, + "recall@100": 0.3619631901840491, + "recall@200": 0.43558282208588955, + "recall@300": 0.49693251533742333, + "recall@500": 0.588957055214724, + "recall@1000": 0.6687116564417178, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6780494608112012, + "ndcg_full@10": 0.6780494608112012, + "ndcg@20": 0.5127220596126485, + "ndcg_full@20": 0.5127220596126485, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.12857142857142856, + "recall@50": 0.6571428571428571, + "recall@100": 0.9571428571428572, + "recall@200": 0.9571428571428572, + "recall@300": 0.9571428571428572, + "recall@500": 0.9857142857142858, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.9015961918657437, + "ndcg_full@5": 0.9015961918657437, + "ndcg@10": 0.7377258144810287, + "ndcg_full@10": 0.7377258144810287, + "ndcg@20": 0.7741740221624782, + "ndcg_full@20": 0.7741740221624782, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1351, + "num_ranked": 191, + "recall@10": 0.055865921787709494, + "recall@50": 0.2011173184357542, + "recall@100": 0.35195530726256985, + "recall@200": 0.45251396648044695, + "recall@300": 0.5418994413407822, + "recall@500": 0.6480446927374302, + "recall@1000": 0.6759776536312849, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.701212079966565, + "ndcg_full@10": 0.701212079966565, + "ndcg@20": 0.6679486984632417, + "ndcg_full@20": 0.6679486984632417, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 54, + "recall@10": 0.023809523809523808, + "recall@50": 0.09523809523809523, + "recall@100": 0.16666666666666666, + "recall@200": 0.2976190476190476, + "recall@300": 0.44047619047619047, + "recall@500": 0.6190476190476191, + "recall@1000": 0.7380952380952381, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.10001307161417164, + "ndcg_full@10": 0.039199134489337664, + "ndcg@20": 0.10001307161417164, + "ndcg_full@20": 0.028736447625686293, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.03759398496240601, + "recall@50": 0.18045112781954886, + "recall@100": 0.34210526315789475, + "recall@200": 0.6353383458646616, + "recall@300": 0.868421052631579, + "recall@500": 0.9360902255639098, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8274043288765652, + "ndcg_full@5": 0.8274043288765652, + "ndcg@10": 0.8487980847530745, + "ndcg_full@10": 0.8487980847530745, + "ndcg@20": 0.7748857280282307, + "ndcg_full@20": 0.7748857280282307, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 84, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.23809523809523808, + "recall@200": 0.3333333333333333, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.21836974929469322, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.21836974929469322, + "ndcg_full@10": 0.11004588314904008, + "ndcg@20": 0.3275546239420398, + "ndcg_full@20": 0.1264460084642602, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 232, + "recall@10": 0.028662420382165606, + "recall@50": 0.14331210191082802, + "recall@100": 0.267515923566879, + "recall@200": 0.4394904458598726, + "recall@300": 0.5923566878980892, + "recall@500": 0.8280254777070064, + "recall@1000": 0.9522292993630573, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6130227926089763, + "ndcg_full@5": 0.6130227926089763, + "ndcg@10": 0.6447416308742162, + "ndcg_full@10": 0.6447416308742162, + "ndcg@20": 0.6480731782306153, + "ndcg_full@20": 0.6480731782306153, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 195, + "recall@10": 0.10416666666666667, + "recall@50": 0.5, + "recall@100": 0.7916666666666666, + "recall@200": 0.9375, + "recall@300": 0.9375, + "recall@500": 0.96875, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5222030810488598, + "ndcg_full@5": 0.5222030810488598, + "ndcg@10": 0.47192607816390947, + "ndcg_full@10": 0.47192607816390947, + "ndcg@20": 0.5910106691902174, + "ndcg_full@20": 0.5910106691902174, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 235, + "recall@10": 0.030395136778115502, + "recall@50": 0.13677811550151975, + "recall@100": 0.2826747720364742, + "recall@200": 0.5227963525835866, + "recall@300": 0.668693009118541, + "recall@500": 0.7446808510638298, + "recall@1000": 0.78419452887538, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.5715259513855394, + "ndcg_full@10": 0.5715259513855394, + "ndcg@20": 0.6023773086598001, + "ndcg_full@20": 0.6023773086598001, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 75, + "recall@10": 0.2777777777777778, + "recall@50": 0.7222222222222222, + "recall@100": 0.7222222222222222, + "recall@200": 0.8333333333333334, + "recall@300": 0.9444444444444444, + "recall@500": 0.9444444444444444, + "recall@1000": 0.9444444444444444, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.43231813227099475, + "ndcg_full@10": 0.43231813227099475, + "ndcg@20": 0.44288200735895317, + "ndcg_full@20": 0.346118624338741, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 233, + "recall@10": 0.033112582781456956, + "recall@50": 0.152317880794702, + "recall@100": 0.2781456953642384, + "recall@200": 0.4966887417218543, + "recall@300": 0.6821192052980133, + "recall@500": 0.9602649006622517, + "recall@1000": 0.9867549668874173, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.6594803070902429, + "ndcg_full@10": 0.6594803070902429, + "ndcg@20": 0.6577384098309015, + "ndcg_full@20": 0.6577384098309015, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + } + } + } + } + }, + { + "name": "h100_baichuan", + "results_dir": "h100_baichuan/results_trec22", + "evaluation_input_sha256": "33765f5507d837a342e57a3c84210136c4c8343eaa221e02d9fdc0af75922a44", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08621931490019455, + "recall@50": 0.28497839609949105, + "recall@100": 0.4392981988236679, + "recall@200": 0.578969461533681, + "recall@300": 0.6512134072613851, + "recall@500": 0.7233007904524495, + "recall@1000": 0.785858357332761, + "ndcg@5": 0.7301254516330676, + "ndcg@10": 0.7277868524527126, + "ndcg@20": 0.7247762806476042, + "ndcg_full@5": 0.7245067573784278, + "ndcg_full@10": 0.7053886427733006, + "ndcg_full@20": 0.6765682696969997, + "P@10(rel>=1)": 0.7519999999999998, + "P@10(eligible)": 0.6379999999999999, + "graded_P@10": 0.6949999999999998, + "unjudged_fraction@10": 0.17599999999999993 + }, + "median": { + "recall@10": 0.059001670843776104, + "recall@50": 0.2251007484168106, + "recall@100": 0.35648553900087643, + "recall@200": 0.546516717325228, + "recall@300": 0.6600731962356222, + "recall@500": 0.7558232717170638, + "recall@1000": 0.8255524014116925, + "ndcg@5": 0.695499001768259, + "ndcg@10": 0.7598733197984213, + "ndcg@20": 0.7298117714589301, + "ndcg_full@5": 0.6880673656030767, + "ndcg_full@10": 0.7446459794897398, + "ndcg_full@20": 0.7127266921608797, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 232, + "recall@10": 0.046632124352331605, + "recall@50": 0.22797927461139897, + "recall@100": 0.40414507772020725, + "recall@200": 0.5854922279792746, + "recall@300": 0.5958549222797928, + "recall@500": 0.616580310880829, + "recall@1000": 0.6321243523316062, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8869084343117906, + "ndcg_full@20": 0.8869084343117906, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 184, + "recall@10": 0.02512562814070352, + "recall@50": 0.09798994974874371, + "recall@100": 0.16834170854271358, + "recall@200": 0.27889447236180903, + "recall@300": 0.4045226130653266, + "recall@500": 0.5628140703517588, + "recall@1000": 0.8592964824120602, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8886025434429607, + "ndcg_full@10": 0.8886025434429607, + "ndcg@20": 0.8266257648546642, + "ndcg_full@20": 0.8266257648546642, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 99, + "recall@10": 0.12658227848101267, + "recall@50": 0.4936708860759494, + "recall@100": 0.7721518987341772, + "recall@200": 0.8734177215189873, + "recall@300": 0.9240506329113924, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.8823356828696407, + "ndcg_full@20": 0.8823356828696407, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.026239067055393587, + "recall@50": 0.10204081632653061, + "recall@100": 0.19533527696793002, + "recall@200": 0.37317784256559766, + "recall@300": 0.5014577259475219, + "recall@500": 0.6793002915451894, + "recall@1000": 0.8221574344023324, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9835675147162116, + "ndcg_full@20": 0.9835675147162116, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 180, + "recall@10": 0.09090909090909091, + "recall@50": 0.21818181818181817, + "recall@100": 0.3090909090909091, + "recall@200": 0.45454545454545453, + "recall@300": 0.5363636363636364, + "recall@500": 0.6545454545454545, + "recall@1000": 0.7909090909090909, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6578242262397573, + "ndcg_full@5": 0.6578242262397573, + "ndcg@10": 0.6384494523554027, + "ndcg_full@10": 0.6384494523554027, + "ndcg@20": 0.559613117577904, + "ndcg_full@20": 0.559613117577904, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 180, + "recall@10": 0.057971014492753624, + "recall@50": 0.2826086956521739, + "recall@100": 0.47101449275362317, + "recall@200": 0.7391304347826086, + "recall@300": 0.8405797101449275, + "recall@500": 0.8768115942028986, + "recall@1000": 0.9637681159420289, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.9427172181013092, + "ndcg_full@10": 0.9427172181013092, + "ndcg@20": 0.8950807585326132, + "ndcg_full@20": 0.8950807585326132, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.11290322580645161, + "recall@50": 0.3387096774193548, + "recall@100": 0.43548387096774194, + "recall@200": 0.5, + "recall@300": 0.5645161290322581, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6379365213265478, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.6203930839812437, + "ndcg_full@10": 0.6203930839812437, + "ndcg@20": 0.7272894955643874, + "ndcg_full@20": 0.7131965037611401, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 53, + "recall@10": 0.125, + "recall@50": 0.3, + "recall@100": 0.525, + "recall@200": 0.725, + "recall@300": 0.75, + "recall@500": 0.9, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7104770117651426, + "ndcg_full@10": 0.7104770117651426, + "ndcg@20": 0.7237455705330033, + "ndcg_full@20": 0.6543512459533934, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 211, + "recall@10": 0.05384615384615385, + "recall@50": 0.16153846153846155, + "recall@100": 0.27692307692307694, + "recall@200": 0.33076923076923076, + "recall@300": 0.4230769230769231, + "recall@500": 0.5538461538461539, + "recall@1000": 0.7, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6770117388019904, + "ndcg_full@5": 0.6770117388019904, + "ndcg@10": 0.7904027188470817, + "ndcg_full@10": 0.7904027188470817, + "ndcg@20": 0.7105062886338618, + "ndcg_full@20": 0.7105062886338618, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 149, + "recall@10": 0.01834862385321101, + "recall@50": 0.07339449541284404, + "recall@100": 0.09174311926605505, + "recall@200": 0.15137614678899083, + "recall@300": 0.18807339449541285, + "recall@500": 0.25229357798165136, + "recall@1000": 0.39908256880733944, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.7373236366182006, + "ndcg_full@10": 0.7138690811585169, + "ndcg@20": 0.7323340473534729, + "ndcg_full@20": 0.5635696095200388, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03289473684210526, + "recall@50": 0.15460526315789475, + "recall@100": 0.3026315789473684, + "recall@200": 0.5, + "recall@300": 0.6578947368421053, + "recall@500": 0.75, + "recall@1000": 0.8289473684210527, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7799082337019199, + "ndcg_full@10": 0.7799082337019199, + "ndcg@20": 0.7661875480844298, + "ndcg_full@20": 0.7661875480844298, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.21212121212121213, + "recall@100": 0.25757575757575757, + "recall@200": 0.4393939393939394, + "recall@300": 0.5606060606060606, + "recall@500": 0.8181818181818182, + "recall@1000": 0.9242424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.46506739315809204, + "ndcg_full@10": 0.46506739315809204, + "ndcg@20": 0.37039949298489827, + "ndcg_full@20": 0.37039949298489827, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 233, + "recall@10": 0.3333333333333333, + "recall@50": 0.47619047619047616, + "recall@100": 0.7142857142857143, + "recall@200": 0.8095238095238095, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.15139734109601002, + "ndcg_full@10": 0.15139734109601002, + "ndcg@20": 0.35357068279481824, + "ndcg_full@20": 0.31296758201402863, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 122, + "recall@10": 0.08196721311475409, + "recall@50": 0.3770491803278688, + "recall@100": 0.639344262295082, + "recall@200": 0.9016393442622951, + "recall@300": 0.9180327868852459, + "recall@500": 0.9426229508196722, + "recall@1000": 0.9508196721311475, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8601100728327754, + "ndcg_full@10": 0.8601100728327754, + "ndcg@20": 0.8041803184121914, + "ndcg_full@20": 0.8041803184121914, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.05847953216374269, + "recall@50": 0.27485380116959063, + "recall@100": 0.5497076023391813, + "recall@200": 0.7076023391812866, + "recall@300": 0.7134502923976608, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9801894664720919, + "ndcg_full@20": 0.9801894664720919, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1875, + "recall@50": 0.6666666666666666, + "recall@100": 0.9166666666666666, + "recall@200": 0.9583333333333334, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.5571394580649888, + "ndcg_full@10": 0.5571394580649888, + "ndcg@20": 0.5612405408635118, + "ndcg_full@20": 0.5612405408635118, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 211, + "recall@10": 0.0, + "recall@50": 0.13333333333333333, + "recall@100": 0.5, + "recall@200": 0.6666666666666666, + "recall@300": 0.8, + "recall@500": 0.8, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.43810570633311313, + "ndcg_full@10": 0.43810570633311313, + "ndcg@20": 0.40453770804013073, + "ndcg_full@20": 0.31507740918969124, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1882, + "num_ranked": 86, + "recall@10": 0.14285714285714285, + "recall@50": 0.6122448979591837, + "recall@100": 0.7346938775510204, + "recall@200": 0.7346938775510204, + "recall@300": 0.7346938775510204, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6379365213265478, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.7650455764143083, + "ndcg_full@10": 0.7650455764143083, + "ndcg@20": 0.708402586334294, + "ndcg_full@20": 0.708402586334294, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 2000, + "num_ranked": 43, + "recall@10": 0.2857142857142857, + "recall@50": 0.2857142857142857, + "recall@100": 0.35714285714285715, + "recall@200": 0.35714285714285715, + "recall@300": 0.42857142857142855, + "recall@500": 0.42857142857142855, + "recall@1000": 0.7857142857142857, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5413996682199069, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.664550862590272, + "ndcg_full@10": 0.3746661058764784, + "ndcg@20": 0.664550862590272, + "ndcg_full@20": 0.335269959178906, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 216, + "recall@10": 0.042735042735042736, + "recall@50": 0.21367521367521367, + "recall@100": 0.38461538461538464, + "recall@200": 0.6367521367521367, + "recall@300": 0.7735042735042735, + "recall@500": 0.8589743589743589, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9627622058340121, + "ndcg_full@5": 0.9627622058340121, + "ndcg@10": 0.9465795175143028, + "ndcg_full@10": 0.9465795175143028, + "ndcg@20": 0.9528851029000728, + "ndcg_full@20": 0.9528851029000728, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 218, + "recall@10": 0.046242774566473986, + "recall@50": 0.2543352601156069, + "recall@100": 0.4797687861271676, + "recall@200": 0.8208092485549133, + "recall@300": 0.9653179190751445, + "recall@500": 0.976878612716763, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0000000000000002, + "ndcg_full@20": 1.0000000000000002, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.08, + "recall@50": 0.36, + "recall@100": 0.48, + "recall@200": 0.552, + "recall@300": 0.568, + "recall@500": 0.608, + "recall@1000": 0.632, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.750637296052051, + "ndcg_full@10": 0.750637296052051, + "ndcg@20": 0.7701115341018233, + "ndcg_full@20": 0.7701115341018233, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 181, + "recall@10": 0.05319148936170213, + "recall@50": 0.25, + "recall@100": 0.48404255319148937, + "recall@200": 0.7127659574468085, + "recall@300": 0.8085106382978723, + "recall@500": 0.851063829787234, + "recall@1000": 0.8829787234042553, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6673451771132222, + "ndcg_full@10": 0.6673451771132222, + "ndcg@20": 0.7122568805606194, + "ndcg_full@20": 0.7122568805606194, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1977, + "num_ranked": 66, + "recall@10": 0.25806451612903225, + "recall@50": 0.3225806451612903, + "recall@100": 0.3548387096774194, + "recall@200": 0.3870967741935484, + "recall@300": 0.3870967741935484, + "recall@500": 0.41935483870967744, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8177724279489998, + "ndcg_full@10": 0.7386546629274284, + "ndcg@20": 0.9459885112530222, + "ndcg_full@20": 0.6263995601327007, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 96, + "recall@10": 0.03314917127071823, + "recall@50": 0.13259668508287292, + "recall@100": 0.2265193370165746, + "recall@200": 0.30939226519337015, + "recall@300": 0.39226519337016574, + "recall@500": 0.5580110497237569, + "recall@1000": 0.7182320441988951, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.47038652250895463, + "ndcg_full@10": 0.47038652250895463, + "ndcg@20": 0.5004906346819915, + "ndcg_full@20": 0.4216604607566425, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 69, + "recall@10": 0.16071428571428573, + "recall@50": 0.6964285714285714, + "recall@100": 0.8214285714285714, + "recall@200": 0.9285714285714286, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9041084081571661, + "ndcg_full@5": 0.9041084081571661, + "ndcg@10": 0.8348367337402822, + "ndcg_full@10": 0.8348367337402822, + "ndcg@20": 0.7875970815083083, + "ndcg_full@20": 0.7875970815083083, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.20689655172413793, + "recall@50": 0.7586206896551724, + "recall@100": 0.8620689655172413, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7162874455029681, + "ndcg_full@10": 0.7162874455029681, + "ndcg@20": 0.6372173602358271, + "ndcg_full@20": 0.6372173602358271, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.014705882352941176, + "recall@50": 0.07352941176470588, + "recall@100": 0.16176470588235295, + "recall@200": 0.29411764705882354, + "recall@300": 0.39705882352941174, + "recall@500": 0.47058823529411764, + "recall@1000": 0.5147058823529411, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3679414168719777, + "ndcg_full@5": 0.3679414168719777, + "ndcg@10": 0.3812902487137119, + "ndcg_full@10": 0.3432934870258273, + "ndcg@20": 0.4843981918304296, + "ndcg_full@20": 0.2910579264301099, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.2, + "graded_P@10": 0.4 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 177, + "recall@10": 0.06060606060606061, + "recall@50": 0.21212121212121213, + "recall@100": 0.3333333333333333, + "recall@200": 0.48484848484848486, + "recall@300": 0.6060606060606061, + "recall@500": 0.7272727272727273, + "recall@1000": 0.7878787878787878, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.18148604814387062, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.22211018439732447, + "ndcg_full@10": 0.1736666713479918, + "ndcg@20": 0.22211018439732447, + "ndcg_full@20": 0.12473739364547462, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1099476439790576, + "recall@100": 0.19895287958115182, + "recall@200": 0.36649214659685864, + "recall@300": 0.5235602094240838, + "recall@500": 0.756544502617801, + "recall@1000": 0.8219895287958116, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9681896059005243, + "ndcg_full@10": 0.9681896059005243, + "ndcg@20": 0.9794706103065556, + "ndcg_full@20": 0.9794706103065556, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 117, + "recall@10": 0.1282051282051282, + "recall@50": 0.38461538461538464, + "recall@100": 0.6666666666666666, + "recall@200": 0.8205128205128205, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.5357086937004182, + "ndcg_full@10": 0.5357086937004182, + "ndcg@20": 0.586994790734602, + "ndcg_full@20": 0.5775035865119408, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 218, + "recall@10": 0.0319634703196347, + "recall@50": 0.0867579908675799, + "recall@100": 0.1689497716894977, + "recall@200": 0.2420091324200913, + "recall@300": 0.3059360730593607, + "recall@500": 0.4200913242009132, + "recall@1000": 0.589041095890411, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7727272727272727, + "ndcg_full@5": 0.7727272727272727, + "ndcg@10": 0.7727272727272728, + "ndcg_full@10": 0.7727272727272728, + "ndcg@20": 0.78508310042985, + "ndcg_full@20": 0.78508310042985, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 111, + "recall@10": 0.06944444444444445, + "recall@50": 0.2222222222222222, + "recall@100": 0.2916666666666667, + "recall@200": 0.4027777777777778, + "recall@300": 0.4722222222222222, + "recall@500": 0.5277777777777778, + "recall@1000": 0.6111111111111112, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8443670565516415, + "ndcg_full@20": 0.7893115703790233, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 2000, + "num_ranked": 26, + "recall@10": 0.0625, + "recall@50": 0.1125, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7406126212438987, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.8808685441719454, + "ndcg_full@10": 0.49237542072483004, + "ndcg@20": 0.9319187367118184, + "ndcg_full@20": 0.3901404615683145, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.03076923076923077, + "recall@50": 0.12307692307692308, + "recall@100": 0.16923076923076924, + "recall@200": 0.28076923076923077, + "recall@300": 0.3923076923076923, + "recall@500": 0.49615384615384617, + "recall@1000": 0.6269230769230769, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9000000000000001, + "ndcg_full@5": 0.9000000000000001, + "ndcg@10": 0.7547010631825345, + "ndcg_full@10": 0.7547010631825345, + "ndcg@20": 0.7027213985904829, + "ndcg_full@20": 0.7027213985904829, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.05194805194805195, + "recall@50": 0.24675324675324675, + "recall@100": 0.4805194805194805, + "recall@200": 0.7532467532467533, + "recall@300": 0.8441558441558441, + "recall@500": 0.8961038961038961, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.3479118113640809, + "ndcg_full@10": 0.3479118113640809, + "ndcg@20": 0.38333954846388724, + "ndcg_full@20": 0.38333954846388724, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 116, + "recall@10": 0.11538461538461539, + "recall@50": 0.5384615384615384, + "recall@100": 0.8333333333333334, + "recall@200": 0.8974358974358975, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9230769230769231, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.6897123945376824, + "ndcg_full@10": 0.6897123945376824, + "ndcg@20": 0.7835813652286394, + "ndcg_full@20": 0.7835813652286394, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6163522012578616, + "recall@200": 0.89937106918239, + "recall@300": 0.9056603773584906, + "recall@500": 0.9182389937106918, + "recall@1000": 0.9308176100628931, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.9526533201221142, + "ndcg_full@20": 0.9526533201221142, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 225, + "recall@10": 0.03409090909090909, + "recall@50": 0.18181818181818182, + "recall@100": 0.3371212121212121, + "recall@200": 0.5643939393939394, + "recall@300": 0.7121212121212122, + "recall@500": 0.7840909090909091, + "recall@1000": 0.7992424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.8048109992093391, + "ndcg_full@10": 0.8048109992093391, + "ndcg@20": 0.8392088713538033, + "ndcg_full@20": 0.8392088713538033, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 147, + "recall@10": 0.06134969325153374, + "recall@50": 0.25766871165644173, + "recall@100": 0.3558282208588957, + "recall@200": 0.4171779141104294, + "recall@300": 0.49693251533742333, + "recall@500": 0.5828220858895705, + "recall@1000": 0.6748466257668712, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8876683119885542, + "ndcg_full@10": 0.8876683119885542, + "ndcg@20": 0.8215050149281953, + "ndcg_full@20": 0.8215050149281953, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 124, + "recall@10": 0.14285714285714285, + "recall@50": 0.6857142857142857, + "recall@100": 0.9571428571428572, + "recall@200": 0.9714285714285714, + "recall@300": 0.9714285714285714, + "recall@500": 0.9857142857142858, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.8708351245050765, + "ndcg_full@10": 0.8708351245050765, + "ndcg@20": 0.8988862018334755, + "ndcg_full@20": 0.8988862018334755, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0446927374301676, + "recall@50": 0.1564245810055866, + "recall@100": 0.2905027932960894, + "recall@200": 0.44692737430167595, + "recall@300": 0.49162011173184356, + "recall@500": 0.6312849162011173, + "recall@1000": 0.6759776536312849, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9441317067873845, + "ndcg_full@20": 0.9441317067873845, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05952380952380952, + "recall@50": 0.17857142857142858, + "recall@100": 0.30952380952380953, + "recall@200": 0.42857142857142855, + "recall@300": 0.5476190476190477, + "recall@500": 0.6666666666666666, + "recall@1000": 0.75, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.4674824559647123, + "ndcg_full@5": 0.4674824559647123, + "ndcg@10": 0.4846306748583207, + "ndcg_full@10": 0.4531599675859253, + "ndcg@20": 0.49327347590400805, + "ndcg_full@20": 0.43748518105359774, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.3, + "graded_P@10": 0.45 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03759398496240601, + "recall@50": 0.18045112781954886, + "recall@100": 0.33458646616541354, + "recall@200": 0.5977443609022557, + "recall@300": 0.7969924812030075, + "recall@500": 0.9172932330827067, + "recall@1000": 0.9511278195488722, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6614941305990049, + "ndcg_full@5": 0.6614941305990049, + "ndcg@10": 0.7124901302149528, + "ndcg_full@10": 0.7124901302149528, + "ndcg@20": 0.6906341376736428, + "ndcg_full@20": 0.6906341376736428, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 183, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.19047619047619047, + "recall@200": 0.3333333333333333, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6875399693572165, + "ndcg_full@5": 0.5922216874835864, + "ndcg@10": 0.6053606539232432, + "ndcg_full@10": 0.38431135260440824, + "ndcg@20": 0.6477689632396401, + "ndcg_full@20": 0.31501354254896735, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.028662420382165606, + "recall@50": 0.12738853503184713, + "recall@100": 0.24203821656050956, + "recall@200": 0.42038216560509556, + "recall@300": 0.5668789808917197, + "recall@500": 0.8280254777070064, + "recall@1000": 0.9490445859872612, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6917831836856677, + "ndcg_full@5": 0.6917831836856677, + "ndcg@10": 0.693130126446575, + "ndcg_full@10": 0.693130126446575, + "ndcg@20": 0.7149721051935736, + "ndcg_full@20": 0.7149721051935736, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.4, + "graded_P@10": 0.7 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 220, + "recall@10": 0.10416666666666667, + "recall@50": 0.4791666666666667, + "recall@100": 0.78125, + "recall@200": 0.9375, + "recall@300": 0.9479166666666666, + "recall@500": 0.96875, + "recall@1000": 0.9895833333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263838, + "ndcg_full@5": 0.6608397947263838, + "ndcg@10": 0.5750808007080248, + "ndcg_full@10": 0.5750808007080248, + "ndcg@20": 0.7257710230432163, + "ndcg_full@20": 0.7257710230432163, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 207, + "recall@10": 0.030395136778115502, + "recall@50": 0.1458966565349544, + "recall@100": 0.2887537993920973, + "recall@200": 0.541033434650456, + "recall@300": 0.6869300911854104, + "recall@500": 0.7446808510638298, + "recall@1000": 0.7811550151975684, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.924301329451995, + "ndcg_full@10": 0.924301329451995, + "ndcg@20": 0.8801038569514527, + "ndcg_full@20": 0.8801038569514527, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 206, + "recall@10": 0.2777777777777778, + "recall@50": 0.6111111111111112, + "recall@100": 0.8333333333333334, + "recall@200": 0.9444444444444444, + "recall@300": 0.9444444444444444, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.46692474155501895, + "ndcg_full@10": 0.46692474155501895, + "ndcg@20": 0.3912366691524068, + "ndcg_full@20": 0.3764330411587517, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.039735099337748346, + "recall@50": 0.15894039735099338, + "recall@100": 0.271523178807947, + "recall@200": 0.47019867549668876, + "recall@300": 0.6622516556291391, + "recall@500": 0.9337748344370861, + "recall@1000": 0.9867549668874173, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.7787979365625564, + "ndcg_full@10": 0.7787979365625564, + "ndcg@20": 0.6868707161862224, + "ndcg_full@20": 0.6868707161862224, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08621931490019455, + "recall@50": 0.28497839609949105, + "recall@100": 0.4392981988236679, + "recall@200": 0.578969461533681, + "recall@300": 0.6512134072613851, + "recall@500": 0.7233007904524495, + "recall@1000": 0.785858357332761, + "ndcg@5": 0.6439044480268573, + "ndcg@10": 0.6226981360587238, + "ndcg@20": 0.6260207182644039, + "ndcg_full@5": 0.6396975039226828, + "ndcg_full@10": 0.6047377994397054, + "ndcg_full@20": 0.5864712278406966, + "P@10(rel>=1)": 0.64, + "P@10(eligible)": 0.5379999999999999, + "graded_P@10": 0.5890000000000001, + "unjudged_fraction@10": 0.17599999999999993 + }, + "median": { + "recall@10": 0.059001670843776104, + "recall@50": 0.2251007484168106, + "recall@100": 0.35648553900087643, + "recall@200": 0.546516717325228, + "recall@300": 0.6600731962356222, + "recall@500": 0.7558232717170638, + "recall@1000": 0.8255524014116925, + "ndcg@5": 0.6608397947263839, + "ndcg@10": 0.6736704424329529, + "ndcg@20": 0.6684594478973622, + "ndcg_full@5": 0.6608397947263839, + "ndcg_full@10": 0.6436905138436246, + "ndcg_full@20": 0.6587672368162076, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.55, + "graded_P@10": 0.65, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 232, + "recall@10": 0.046632124352331605, + "recall@50": 0.22797927461139897, + "recall@100": 0.40414507772020725, + "recall@200": 0.5854922279792746, + "recall@300": 0.5958549222797928, + "recall@500": 0.616580310880829, + "recall@1000": 0.6321243523316062, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.833071909033374, + "ndcg_full@20": 0.833071909033374, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 184, + "recall@10": 0.02512562814070352, + "recall@50": 0.09798994974874371, + "recall@100": 0.16834170854271358, + "recall@200": 0.27889447236180903, + "recall@300": 0.4045226130653266, + "recall@500": 0.5628140703517588, + "recall@1000": 0.8592964824120602, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.8132984167532468, + "ndcg_full@10": 0.8132984167532468, + "ndcg@20": 0.7083763623979372, + "ndcg_full@20": 0.7083763623979372, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 99, + "recall@10": 0.12658227848101267, + "recall@50": 0.4936708860759494, + "recall@100": 0.7721518987341772, + "recall@200": 0.8734177215189873, + "recall@300": 0.9240506329113924, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7104770117651426, + "ndcg_full@10": 0.7104770117651426, + "ndcg@20": 0.6521873448733686, + "ndcg_full@20": 0.6521873448733686, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.026239067055393587, + "recall@50": 0.10204081632653061, + "recall@100": 0.19533527696793002, + "recall@200": 0.37317784256559766, + "recall@300": 0.5014577259475219, + "recall@500": 0.6793002915451894, + "recall@1000": 0.8221574344023324, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.5294438830372382, + "ndcg_full@10": 0.5294438830372382, + "ndcg@20": 0.5895357014426971, + "ndcg_full@20": 0.5895357014426971, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 180, + "recall@10": 0.09090909090909091, + "recall@50": 0.21818181818181817, + "recall@100": 0.3090909090909091, + "recall@200": 0.45454545454545453, + "recall@300": 0.5363636363636364, + "recall@500": 0.6545454545454545, + "recall@1000": 0.7909090909090909, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6578242262397573, + "ndcg_full@5": 0.6578242262397573, + "ndcg@10": 0.6384494523554027, + "ndcg_full@10": 0.6384494523554027, + "ndcg@20": 0.559613117577904, + "ndcg_full@20": 0.559613117577904, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 180, + "recall@10": 0.057971014492753624, + "recall@50": 0.2826086956521739, + "recall@100": 0.47101449275362317, + "recall@200": 0.7391304347826086, + "recall@300": 0.8405797101449275, + "recall@500": 0.8768115942028986, + "recall@1000": 0.9637681159420289, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.9427172181013092, + "ndcg_full@10": 0.9427172181013092, + "ndcg@20": 0.8869961857455332, + "ndcg_full@20": 0.8869961857455332, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.11290322580645161, + "recall@50": 0.3387096774193548, + "recall@100": 0.43548387096774194, + "recall@200": 0.5, + "recall@300": 0.5645161290322581, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5817744828317774, + "ndcg_full@5": 0.5817744828317774, + "ndcg@10": 0.513217277023062, + "ndcg_full@10": 0.513217277023062, + "ndcg@20": 0.6090826629799275, + "ndcg_full@20": 0.5972802142587137, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 53, + "recall@10": 0.125, + "recall@50": 0.3, + "recall@100": 0.525, + "recall@200": 0.725, + "recall@300": 0.75, + "recall@500": 0.9, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7104770117651426, + "ndcg_full@10": 0.7104770117651426, + "ndcg@20": 0.7230527300798051, + "ndcg_full@20": 0.6537248365738331, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 211, + "recall@10": 0.05384615384615385, + "recall@50": 0.16153846153846155, + "recall@100": 0.27692307692307694, + "recall@200": 0.33076923076923076, + "recall@300": 0.4230769230769231, + "recall@500": 0.5538461538461539, + "recall@1000": 0.7, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6281895090257151, + "ndcg_full@5": 0.6281895090257151, + "ndcg@10": 0.5488586524939398, + "ndcg_full@10": 0.5488586524939398, + "ndcg@20": 0.6064101520473948, + "ndcg_full@20": 0.6064101520473948, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 149, + "recall@10": 0.01834862385321101, + "recall@50": 0.07339449541284404, + "recall@100": 0.09174311926605505, + "recall@200": 0.15137614678899083, + "recall@300": 0.18807339449541285, + "recall@500": 0.25229357798165136, + "recall@1000": 0.39908256880733944, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.6702525738522754, + "ndcg_full@10": 0.6489315753318465, + "ndcg@20": 0.5948938003476213, + "ndcg_full@20": 0.45780210271444643, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03289473684210526, + "recall@50": 0.15460526315789475, + "recall@100": 0.3026315789473684, + "recall@200": 0.5, + "recall@300": 0.6578947368421053, + "recall@500": 0.75, + "recall@1000": 0.8289473684210527, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7015099647232446, + "ndcg_full@10": 0.7015099647232446, + "ndcg@20": 0.6840520595770958, + "ndcg_full@20": 0.6840520595770958, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.21212121212121213, + "recall@100": 0.25757575757575757, + "recall@200": 0.4393939393939394, + "recall@300": 0.5606060606060606, + "recall@500": 0.8181818181818182, + "recall@1000": 0.9242424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.46506739315809204, + "ndcg_full@10": 0.46506739315809204, + "ndcg@20": 0.36534257209351223, + "ndcg_full@20": 0.36534257209351223, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 233, + "recall@10": 0.3333333333333333, + "recall@50": 0.47619047619047616, + "recall@100": 0.7142857142857143, + "recall@200": 0.8095238095238095, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.09350936543551687, + "ndcg_full@20": 0.0827710028577313, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 122, + "recall@10": 0.08196721311475409, + "recall@50": 0.3770491803278688, + "recall@100": 0.639344262295082, + "recall@200": 0.9016393442622951, + "recall@300": 0.9180327868852459, + "recall@500": 0.9426229508196722, + "recall@1000": 0.9508196721311475, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8601100728327754, + "ndcg_full@10": 0.8601100728327754, + "ndcg@20": 0.7857689728009066, + "ndcg_full@20": 0.7857689728009066, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.05847953216374269, + "recall@50": 0.27485380116959063, + "recall@100": 0.5497076023391813, + "recall@200": 0.7076023391812866, + "recall@300": 0.7134502923976608, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.8750200335287022, + "ndcg_full@20": 0.8750200335287022, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1875, + "recall@50": 0.6666666666666666, + "recall@100": 0.9166666666666666, + "recall@200": 0.9583333333333334, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.5571394580649888, + "ndcg_full@10": 0.5571394580649888, + "ndcg@20": 0.5276425284551693, + "ndcg_full@20": 0.5276425284551693, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 211, + "recall@10": 0.0, + "recall@50": 0.13333333333333333, + "recall@100": 0.5, + "recall@200": 0.6666666666666666, + "recall@300": 0.8, + "recall@500": 0.8, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.28952298823485745, + "ndcg_full@10": 0.28952298823485745, + "ndcg@20": 0.3372669633581292, + "ndcg_full@20": 0.26268305502342004, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1882, + "num_ranked": 86, + "recall@10": 0.14285714285714285, + "recall@50": 0.6122448979591837, + "recall@100": 0.7346938775510204, + "recall@200": 0.7346938775510204, + "recall@300": 0.7346938775510204, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6379365213265478, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.6230265192366811, + "ndcg_full@10": 0.6230265192366811, + "ndcg@20": 0.6541151505244592, + "ndcg_full@20": 0.6541151505244592, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 2000, + "num_ranked": 43, + "recall@10": 0.2857142857142857, + "recall@50": 0.2857142857142857, + "recall@100": 0.35714285714285715, + "recall@200": 0.35714285714285715, + "recall@300": 0.42857142857142855, + "recall@500": 0.42857142857142855, + "recall@1000": 0.7857142857142857, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3903800499921017, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.5078961547485289, + "ndcg_full@10": 0.2863459897524692, + "ndcg@20": 0.6133917389836392, + "ndcg_full@20": 0.3094598696150023, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 216, + "recall@10": 0.042735042735042736, + "recall@50": 0.21367521367521367, + "recall@100": 0.38461538461538464, + "recall@200": 0.6367521367521367, + "recall@300": 0.7735042735042735, + "recall@500": 0.8589743589743589, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9627622058340121, + "ndcg_full@5": 0.9627622058340121, + "ndcg@10": 0.9465795175143028, + "ndcg_full@10": 0.9465795175143028, + "ndcg@20": 0.9528851029000728, + "ndcg_full@20": 0.9528851029000728, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 218, + "recall@10": 0.046242774566473986, + "recall@50": 0.2543352601156069, + "recall@100": 0.4797687861271676, + "recall@200": 0.8208092485549133, + "recall@300": 0.9653179190751445, + "recall@500": 0.976878612716763, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.716485063693381, + "ndcg_full@10": 0.716485063693381, + "ndcg@20": 0.8170287173070079, + "ndcg_full@20": 0.8170287173070079, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.08, + "recall@50": 0.36, + "recall@100": 0.48, + "recall@200": 0.552, + "recall@300": 0.568, + "recall@500": 0.608, + "recall@1000": 0.632, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.5357086937004182, + "ndcg_full@10": 0.5357086937004182, + "ndcg@20": 0.6656113780415676, + "ndcg_full@20": 0.6656113780415676, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 181, + "recall@10": 0.05319148936170213, + "recall@50": 0.25, + "recall@100": 0.48404255319148937, + "recall@200": 0.7127659574468085, + "recall@300": 0.8085106382978723, + "recall@500": 0.851063829787234, + "recall@1000": 0.8829787234042553, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.601090953658833, + "ndcg_full@10": 0.601090953658833, + "ndcg@20": 0.6713075177531569, + "ndcg_full@20": 0.6713075177531569, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1977, + "num_ranked": 66, + "recall@10": 0.25806451612903225, + "recall@50": 0.3225806451612903, + "recall@100": 0.3548387096774194, + "recall@200": 0.3870967741935484, + "recall@300": 0.3870967741935484, + "recall@500": 0.41935483870967744, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5531464700081437, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.47427006488375206, + "ndcg_full@10": 0.42838543210841196, + "ndcg@20": 0.6947829605710375, + "ndcg_full@20": 0.4600602816126464, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 96, + "recall@10": 0.03314917127071823, + "recall@50": 0.13259668508287292, + "recall@100": 0.2265193370165746, + "recall@200": 0.30939226519337015, + "recall@300": 0.39226519337016574, + "recall@500": 0.5580110497237569, + "recall@1000": 0.7182320441988951, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.43370456145927466, + "ndcg_full@10": 0.43370456145927466, + "ndcg@20": 0.41601593351360744, + "ndcg_full@20": 0.35049101432020124, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 69, + "recall@10": 0.16071428571428573, + "recall@50": 0.6964285714285714, + "recall@100": 0.8214285714285714, + "recall@200": 0.9285714285714286, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9041084081571661, + "ndcg_full@5": 0.9041084081571661, + "ndcg@10": 0.8348367337402822, + "ndcg_full@10": 0.8348367337402822, + "ndcg@20": 0.7875970815083083, + "ndcg_full@20": 0.7875970815083083, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.20689655172413793, + "recall@50": 0.7586206896551724, + "recall@100": 0.8620689655172413, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7162874455029681, + "ndcg_full@10": 0.7162874455029681, + "ndcg@20": 0.6012232911835376, + "ndcg_full@20": 0.6012232911835376, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.014705882352941176, + "recall@50": 0.07352941176470588, + "recall@100": 0.16176470588235295, + "recall@200": 0.29411764705882354, + "recall@300": 0.39705882352941174, + "recall@500": 0.47058823529411764, + "recall@1000": 0.5147058823529411, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.3679414168719777, + "ndcg_full@5": 0.3679414168719777, + "ndcg@10": 0.3812902487137119, + "ndcg_full@10": 0.3432934870258273, + "ndcg@20": 0.42655267671085423, + "ndcg_full@20": 0.2563005801642944, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.2, + "graded_P@10": 0.4 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 177, + "recall@10": 0.06060606060606061, + "recall@50": 0.21212121212121213, + "recall@100": 0.3333333333333333, + "recall@200": 0.48484848484848486, + "recall@300": 0.6060606060606061, + "recall@500": 0.7272727272727273, + "recall@1000": 0.7878787878787878, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.15632357310588862, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.12122913927828419, + "ndcg_full@10": 0.09478836436955078, + "ndcg@20": 0.19516113410716093, + "ndcg_full@20": 0.10960276889363314, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1099476439790576, + "recall@100": 0.19895287958115182, + "recall@200": 0.36649214659685864, + "recall@300": 0.5235602094240838, + "recall@500": 0.756544502617801, + "recall@1000": 0.8219895287958116, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7065836017263629, + "ndcg_full@10": 0.7065836017263629, + "ndcg@20": 0.6880593033255568, + "ndcg_full@20": 0.6880593033255568, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 117, + "recall@10": 0.1282051282051282, + "recall@50": 0.38461538461538464, + "recall@100": 0.6666666666666666, + "recall@200": 0.8205128205128205, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.4595326231006953, + "ndcg_full@10": 0.4595326231006953, + "ndcg@20": 0.46329929904589134, + "ndcg_full@20": 0.45580814523521207, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 218, + "recall@10": 0.0319634703196347, + "recall@50": 0.0867579908675799, + "recall@100": 0.1689497716894977, + "recall@200": 0.2420091324200913, + "recall@300": 0.3059360730593607, + "recall@500": 0.4200913242009132, + "recall@1000": 0.589041095890411, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7727272727272727, + "ndcg_full@5": 0.7727272727272727, + "ndcg@10": 0.7727272727272728, + "ndcg_full@10": 0.7727272727272728, + "ndcg@20": 0.7077661091828052, + "ndcg_full@20": 0.7077661091828052, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 111, + "recall@10": 0.06944444444444445, + "recall@50": 0.2222222222222222, + "recall@100": 0.2916666666666667, + "recall@200": 0.4027777777777778, + "recall@300": 0.4722222222222222, + "recall@500": 0.5277777777777778, + "recall@1000": 0.6111111111111112, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.741670249010153, + "ndcg_full@10": 0.741670249010153, + "ndcg@20": 0.7440352954205577, + "ndcg_full@20": 0.6955217673273865, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 2000, + "num_ranked": 26, + "recall@10": 0.0625, + "recall@50": 0.1125, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7406126212438987, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.8718620064371599, + "ndcg_full@10": 0.4873410738455148, + "ndcg@20": 0.8718620064371599, + "ndcg_full@20": 0.3649981830126634, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 154, + "recall@10": 0.03076923076923077, + "recall@50": 0.12307692307692308, + "recall@100": 0.16923076923076924, + "recall@200": 0.28076923076923077, + "recall@300": 0.3923076923076923, + "recall@500": 0.49615384615384617, + "recall@1000": 0.6269230769230769, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9000000000000001, + "ndcg_full@5": 0.9000000000000001, + "ndcg@10": 0.7547010631825345, + "ndcg_full@10": 0.7547010631825345, + "ndcg@20": 0.6738744423253507, + "ndcg_full@20": 0.6738744423253507, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 127, + "recall@10": 0.05194805194805195, + "recall@50": 0.24675324675324675, + "recall@100": 0.4805194805194805, + "recall@200": 0.7532467532467533, + "recall@300": 0.8441558441558441, + "recall@500": 0.8961038961038961, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.24573132854020638, + "ndcg_full@10": 0.24573132854020638, + "ndcg@20": 0.26785308450411205, + "ndcg_full@20": 0.26785308450411205, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 116, + "recall@10": 0.11538461538461539, + "recall@50": 0.5384615384615384, + "recall@100": 0.8333333333333334, + "recall@200": 0.8974358974358975, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9230769230769231, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5087403079104241, + "ndcg_full@5": 0.5087403079104241, + "ndcg@10": 0.580903324866642, + "ndcg_full@10": 0.580903324866642, + "ndcg@20": 0.6899076229649914, + "ndcg_full@20": 0.6899076229649914, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 145, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6163522012578616, + "recall@200": 0.89937106918239, + "recall@300": 0.9056603773584906, + "recall@500": 0.9182389937106918, + "recall@1000": 0.9308176100628931, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8454256604216017, + "ndcg_full@10": 0.8454256604216017, + "ndcg@20": 0.9002427683263423, + "ndcg_full@20": 0.9002427683263423, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 225, + "recall@10": 0.03409090909090909, + "recall@50": 0.18181818181818182, + "recall@100": 0.3371212121212121, + "recall@200": 0.5643939393939394, + "recall@300": 0.7121212121212122, + "recall@500": 0.7840909090909091, + "recall@1000": 0.7992424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.8048109992093391, + "ndcg_full@10": 0.8048109992093391, + "ndcg@20": 0.8392088713538033, + "ndcg_full@20": 0.8392088713538033, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 147, + "recall@10": 0.06134969325153374, + "recall@50": 0.25766871165644173, + "recall@100": 0.3558282208588957, + "recall@200": 0.4171779141104294, + "recall@300": 0.49693251533742333, + "recall@500": 0.5828220858895705, + "recall@1000": 0.6748466257668712, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7204111965093014, + "ndcg_full@5": 0.7204111965093014, + "ndcg@10": 0.7491347753688576, + "ndcg_full@10": 0.7491347753688576, + "ndcg@20": 0.5908858000492448, + "ndcg_full@20": 0.5908858000492448, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 124, + "recall@10": 0.14285714285714285, + "recall@50": 0.6857142857142857, + "recall@100": 0.9571428571428572, + "recall@200": 0.9714285714285714, + "recall@300": 0.9714285714285714, + "recall@500": 0.9857142857142858, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8613632863224759, + "ndcg_full@5": 0.8613632863224759, + "ndcg@10": 0.7974712024057163, + "ndcg_full@10": 0.7974712024057163, + "ndcg@20": 0.8519194405603625, + "ndcg_full@20": 0.8519194405603625, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0446927374301676, + "recall@50": 0.1564245810055866, + "recall@100": 0.2905027932960894, + "recall@200": 0.44692737430167595, + "recall@300": 0.49162011173184356, + "recall@500": 0.6312849162011173, + "recall@1000": 0.6759776536312849, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.7360439527507333, + "ndcg_full@10": 0.7360439527507333, + "ndcg@20": 0.6634193231079559, + "ndcg_full@20": 0.6634193231079559, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05952380952380952, + "recall@50": 0.17857142857142858, + "recall@100": 0.30952380952380953, + "recall@200": 0.42857142857142855, + "recall@300": 0.5476190476190477, + "recall@500": 0.6666666666666666, + "recall@1000": 0.75, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.10699313236726378, + "ndcg_full@5": 0.10699313236726378, + "ndcg@10": 0.23195615120007243, + "ndcg_full@10": 0.21689349727998572, + "ndcg@20": 0.317706212920753, + "ndcg_full@20": 0.2817742426283155, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03759398496240601, + "recall@50": 0.18045112781954886, + "recall@100": 0.33458646616541354, + "recall@200": 0.5977443609022557, + "recall@300": 0.7969924812030075, + "recall@500": 0.9172932330827067, + "recall@1000": 0.9511278195488722, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6614941305990049, + "ndcg_full@5": 0.6614941305990049, + "ndcg@10": 0.7124901302149528, + "ndcg_full@10": 0.7124901302149528, + "ndcg@20": 0.6906341376736428, + "ndcg_full@20": 0.6906341376736428, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 183, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.19047619047619047, + "recall@200": 0.3333333333333333, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.33321647696559303, + "ndcg_full@5": 0.2870204396558808, + "ndcg@10": 0.3977507610634655, + "ndcg_full@10": 0.25251084951271896, + "ndcg@20": 0.3977507610634655, + "ndcg_full@20": 0.19342834159190328, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.028662420382165606, + "recall@50": 0.12738853503184713, + "recall@100": 0.24203821656050956, + "recall@200": 0.42038216560509556, + "recall@300": 0.5668789808917197, + "recall@500": 0.8280254777070064, + "recall@1000": 0.9490445859872612, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6917831836856677, + "ndcg_full@5": 0.6917831836856677, + "ndcg@10": 0.6281926206199047, + "ndcg_full@10": 0.6281926206199047, + "ndcg@20": 0.6907441952675765, + "ndcg_full@20": 0.6907441952675765, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 220, + "recall@10": 0.10416666666666667, + "recall@50": 0.4791666666666667, + "recall@100": 0.78125, + "recall@200": 0.9375, + "recall@300": 0.9479166666666666, + "recall@500": 0.96875, + "recall@1000": 0.9895833333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263838, + "ndcg_full@5": 0.6608397947263838, + "ndcg@10": 0.5750808007080248, + "ndcg_full@10": 0.5750808007080248, + "ndcg@20": 0.6867681704326171, + "ndcg_full@20": 0.6867681704326171, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 207, + "recall@10": 0.030395136778115502, + "recall@50": 0.1458966565349544, + "recall@100": 0.2887537993920973, + "recall@200": 0.541033434650456, + "recall@300": 0.6869300911854104, + "recall@500": 0.7446808510638298, + "recall@1000": 0.7811550151975684, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6770883110136303, + "ndcg_full@10": 0.6770883110136303, + "ndcg@20": 0.664846661672017, + "ndcg_full@20": 0.664846661672017, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 206, + "recall@10": 0.2777777777777778, + "recall@50": 0.6111111111111112, + "recall@100": 0.8333333333333334, + "recall@200": 0.9444444444444444, + "recall@300": 0.9444444444444444, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.22400556151517553, + "ndcg_full@10": 0.22400556151517553, + "ndcg@20": 0.2858865465207548, + "ndcg_full@20": 0.275069160481116, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.039735099337748346, + "recall@50": 0.15894039735099338, + "recall@100": 0.271523178807947, + "recall@200": 0.47019867549668876, + "recall@300": 0.6622516556291391, + "recall@500": 0.9337748344370861, + "recall@1000": 0.9867549668874173, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.7787979365625564, + "ndcg_full@10": 0.7787979365625564, + "ndcg@20": 0.6868707161862224, + "ndcg_full@20": 0.6868707161862224, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + } + } + } + } + }, + { + "name": "qwen36_medcpt", + "results_dir": "qwen36_medcpt/results_trec22", + "evaluation_input_sha256": "2187bae91f457b49765006bd9be07e4a2711b916afe18b1f89e46f8609cce6fc", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08495420999498725, + "recall@50": 0.2835665389548808, + "recall@100": 0.4327706931049727, + "recall@200": 0.5792057708977656, + "recall@300": 0.6572540528790871, + "recall@500": 0.7293634903186007, + "recall@1000": 0.7806891099020966, + "ndcg@5": 0.7950047726531426, + "ndcg@10": 0.7832053952230806, + "ndcg@20": 0.7736417029219084, + "ndcg_full@5": 0.7842578572137406, + "ndcg_full@10": 0.7481034096160062, + "ndcg_full@20": 0.7099926806891068, + "P@10(rel>=1)": 0.7679999999999997, + "P@10(eligible)": 0.6939999999999997, + "graded_P@10": 0.7309999999999998, + "unjudged_fraction@10": 0.14200000000000002 + }, + "median": { + "recall@10": 0.05954279638490165, + "recall@50": 0.2494824016563147, + "recall@100": 0.37057757716392764, + "recall@200": 0.5498095238095239, + "recall@300": 0.6772401163957456, + "recall@500": 0.7608843537414967, + "recall@1000": 0.8216273522395972, + "ndcg@5": 0.8687949224876582, + "ndcg@10": 0.8258564692677821, + "ndcg@20": 0.8375634893697077, + "ndcg_full@5": 0.8496074099254252, + "ndcg_full@10": 0.796323464574994, + "ndcg_full@20": 0.7614666500955124, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.75, + "graded_P@10": 0.8, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 183, + "recall@10": 0.046632124352331605, + "recall@50": 0.22797927461139897, + "recall@100": 0.37305699481865284, + "recall@200": 0.5647668393782384, + "recall@300": 0.5906735751295337, + "recall@500": 0.6010362694300518, + "recall@1000": 0.6217616580310881, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509597, + "ndcg_full@10": 0.8899541168509597, + "ndcg@20": 0.8738765963254316, + "ndcg_full@20": 0.8738765963254316, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 165, + "recall@10": 0.02512562814070352, + "recall@50": 0.10552763819095477, + "recall@100": 0.17587939698492464, + "recall@200": 0.31155778894472363, + "recall@300": 0.39949748743718594, + "recall@500": 0.5552763819095478, + "recall@1000": 0.7889447236180904, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9608008655106622, + "ndcg_full@10": 0.9608008655106622, + "ndcg@20": 0.9027207441983794, + "ndcg_full@20": 0.9027207441983794, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 148, + "recall@10": 0.12658227848101267, + "recall@50": 0.4810126582278481, + "recall@100": 0.7341772151898734, + "recall@200": 0.8987341772151899, + "recall@300": 0.9240506329113924, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8625437478340825, + "ndcg_full@20": 0.8625437478340825, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029154518950437316, + "recall@50": 0.12536443148688048, + "recall@100": 0.21865889212827988, + "recall@200": 0.4198250728862974, + "recall@300": 0.5510204081632653, + "recall@500": 0.7201166180758017, + "recall@1000": 0.8250728862973761, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9775956781991642, + "ndcg_full@20": 0.9775956781991642, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 144, + "recall@10": 0.09090909090909091, + "recall@50": 0.36363636363636365, + "recall@100": 0.5909090909090909, + "recall@200": 0.7272727272727273, + "recall@300": 0.7545454545454545, + "recall@500": 0.7818181818181819, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8935089661734454, + "ndcg_full@10": 0.8935089661734454, + "ndcg@20": 0.7321825369831829, + "ndcg_full@20": 0.7321825369831829, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 202, + "recall@10": 0.057971014492753624, + "recall@50": 0.2608695652173913, + "recall@100": 0.5289855072463768, + "recall@200": 0.7608695652173914, + "recall@300": 0.8333333333333334, + "recall@500": 0.9057971014492754, + "recall@1000": 0.927536231884058, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9272087893571915, + "ndcg_full@20": 0.9272087893571915, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 128, + "recall@10": 0.08064516129032258, + "recall@50": 0.2903225806451613, + "recall@100": 0.43548387096774194, + "recall@200": 0.5, + "recall@300": 0.5645161290322581, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.5015472841548577, + "ndcg_full@10": 0.5015472841548577, + "ndcg@20": 0.6632400741976093, + "ndcg_full@20": 0.637230028011647, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 45, + "recall@10": 0.2, + "recall@50": 0.5, + "recall@100": 0.65, + "recall@200": 0.75, + "recall@300": 0.9, + "recall@500": 0.925, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7162874455029682, + "ndcg_full@10": 0.7162874455029682, + "ndcg@20": 0.7479444965395781, + "ndcg_full@20": 0.7193531217557249, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.038461538461538464, + "recall@50": 0.13846153846153847, + "recall@100": 0.26153846153846155, + "recall@200": 0.36153846153846153, + "recall@300": 0.4230769230769231, + "recall@500": 0.5384615384615384, + "recall@1000": 0.676923076923077, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4299813935652734, + "ndcg_full@5": 0.4299813935652734, + "ndcg@10": 0.6300969277578489, + "ndcg_full@10": 0.6300969277578489, + "ndcg@20": 0.6536707383583976, + "ndcg_full@20": 0.6536707383583976, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 41, + "recall@10": 0.022935779816513763, + "recall@50": 0.06422018348623854, + "recall@100": 0.09174311926605505, + "recall@200": 0.12844036697247707, + "recall@300": 0.1651376146788991, + "recall@500": 0.24311926605504589, + "recall@1000": 0.3669724770642202, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7795044443395203, + "ndcg_full@10": 0.6512053225092734, + "ndcg@20": 0.8394850341271144, + "ndcg_full@20": 0.4526049434391312, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 237, + "recall@10": 0.03289473684210526, + "recall@50": 0.1513157894736842, + "recall@100": 0.27631578947368424, + "recall@200": 0.4407894736842105, + "recall@300": 0.5855263157894737, + "recall@500": 0.7105263157894737, + "recall@1000": 0.8618421052631579, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7799082337019199, + "ndcg_full@10": 0.7799082337019199, + "ndcg@20": 0.8579599603663427, + "ndcg_full@20": 0.8579599603663427, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.09090909090909091, + "recall@50": 0.2878787878787879, + "recall@100": 0.4696969696969697, + "recall@200": 0.5757575757575758, + "recall@300": 0.7272727272727273, + "recall@500": 0.8181818181818182, + "recall@1000": 0.9242424242424242, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.30523488393970116, + "ndcg_full@10": 0.30523488393970116, + "ndcg@20": 0.2324986376140575, + "ndcg_full@20": 0.2324986376140575, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.2857142857142857, + "recall@50": 0.47619047619047616, + "recall@100": 0.6666666666666666, + "recall@200": 0.8095238095238095, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.3077499832503558, + "ndcg_full@5": 0.3077499832503558, + "ndcg@10": 0.32958369309234353, + "ndcg_full@10": 0.32958369309234353, + "ndcg@20": 0.36147843196785717, + "ndcg_full@20": 0.31996722666300437, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 152, + "recall@10": 0.08196721311475409, + "recall@50": 0.36885245901639346, + "recall@100": 0.6639344262295082, + "recall@200": 0.8852459016393442, + "recall@300": 0.9098360655737705, + "recall@500": 0.9508196721311475, + "recall@1000": 0.9508196721311475, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.8439539777664462, + "ndcg_full@20": 0.8439539777664462, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.05847953216374269, + "recall@50": 0.2807017543859649, + "recall@100": 0.543859649122807, + "recall@200": 0.695906432748538, + "recall@300": 0.7076023391812866, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9668728882728055, + "ndcg_full@10": 0.9668728882728055, + "ndcg@20": 0.9786208437424843, + "ndcg_full@20": 0.9786208437424843, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 229, + "recall@10": 0.16666666666666666, + "recall@50": 0.5833333333333334, + "recall@100": 0.8541666666666666, + "recall@200": 0.9375, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7386730375707296, + "ndcg_full@10": 0.7386730375707296, + "ndcg@20": 0.6928899247063153, + "ndcg_full@20": 0.6928899247063153, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 165, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.03333333333333333, + "recall@200": 0.43333333333333335, + "recall@300": 0.6666666666666666, + "recall@500": 0.7666666666666667, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.31834121936641585, + "ndcg_full@5": 0.2765732350040718, + "ndcg@10": 0.43585732412284295, + "ndcg_full@10": 0.24573132854020638, + "ndcg@20": 0.43585732412284295, + "ndcg_full@20": 0.15858697593353197, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 2000, + "num_ranked": 80, + "recall@10": 0.10204081632653061, + "recall@50": 0.5714285714285714, + "recall@100": 0.7551020408163265, + "recall@200": 0.7551020408163265, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.6530488081313035, + "ndcg_full@10": 0.6530488081313035, + "ndcg@20": 0.6656474680279866, + "ndcg_full@20": 0.6656474680279866, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 2000, + "num_ranked": 36, + "recall@10": 0.21428571428571427, + "recall@50": 0.2857142857142857, + "recall@100": 0.2857142857142857, + "recall@200": 0.42857142857142855, + "recall@300": 0.42857142857142855, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8529278650606567, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.8529278650606567, + "ndcg_full@10": 0.40002324830925196, + "ndcg@20": 0.8529278650606567, + "ndcg_full@20": 0.357960797701493, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 174, + "recall@10": 0.042735042735042736, + "recall@50": 0.20512820512820512, + "recall@100": 0.38461538461538464, + "recall@200": 0.5854700854700855, + "recall@300": 0.7350427350427351, + "recall@500": 0.8333333333333334, + "recall@1000": 0.8675213675213675, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9553146470008144, + "ndcg_full@5": 0.9553146470008144, + "ndcg@10": 0.9358954210171636, + "ndcg_full@10": 0.9358954210171636, + "ndcg@20": 0.950828427315586, + "ndcg_full@20": 0.950828427315586, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 196, + "recall@10": 0.046242774566473986, + "recall@50": 0.2138728323699422, + "recall@100": 0.4277456647398844, + "recall@200": 0.6936416184971098, + "recall@300": 0.884393063583815, + "recall@500": 0.9653179190751445, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0, + "ndcg_full@20": 1.0, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 2000, + "num_ranked": 177, + "recall@10": 0.08, + "recall@50": 0.344, + "recall@100": 0.488, + "recall@200": 0.552, + "recall@300": 0.584, + "recall@500": 0.608, + "recall@1000": 0.616, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7104770117651426, + "ndcg_full@10": 0.7104770117651426, + "ndcg@20": 0.7380202398004411, + "ndcg_full@20": 0.7380202398004411, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 147, + "recall@10": 0.05319148936170213, + "recall@50": 0.26595744680851063, + "recall@100": 0.5106382978723404, + "recall@200": 0.7712765957446809, + "recall@300": 0.8138297872340425, + "recall@500": 0.8297872340425532, + "recall@1000": 0.8563829787234043, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9102649208458158, + "ndcg_full@20": 0.9102649208458158, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 2000, + "num_ranked": 55, + "recall@10": 0.1935483870967742, + "recall@50": 0.3225806451612903, + "recall@100": 0.3548387096774194, + "recall@200": 0.41935483870967744, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.795760747566305, + "ndcg_full@5": 0.795760747566305, + "ndcg@10": 0.7890615516075404, + "ndcg_full@10": 0.6591894195415876, + "ndcg@20": 0.9247109299102612, + "ndcg_full@20": 0.5663200515397971, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.027624309392265192, + "recall@50": 0.11602209944751381, + "recall@100": 0.22099447513812154, + "recall@200": 0.30939226519337015, + "recall@300": 0.39226519337016574, + "recall@500": 0.5580110497237569, + "recall@1000": 0.7458563535911602, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.5965377185773229, + "ndcg_full@10": 0.5965377185773229, + "ndcg@20": 0.5389364883368243, + "ndcg_full@20": 0.4744472953394869, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 84, + "recall@10": 0.16071428571428573, + "recall@50": 0.625, + "recall@100": 0.8035714285714286, + "recall@200": 0.9107142857142857, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.7774107053241637, + "ndcg_full@10": 0.7774107053241637, + "ndcg@20": 0.8005507442715386, + "ndcg_full@20": 0.8005507442715386, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 106, + "recall@10": 0.27586206896551724, + "recall@50": 0.7241379310344828, + "recall@100": 0.8275862068965517, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.890225066021651, + "ndcg_full@20": 0.890225066021651, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 164, + "recall@10": 0.014705882352941176, + "recall@50": 0.07352941176470588, + "recall@100": 0.17647058823529413, + "recall@200": 0.3235294117647059, + "recall@300": 0.35294117647058826, + "recall@500": 0.4411764705882353, + "recall@1000": 0.5735294117647058, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.34299006184340053, + "ndcg_full@10": 0.3098064936645125, + "ndcg@20": 0.4039065452592942, + "ndcg_full@20": 0.23544900526035273, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.09090909090909091, + "recall@100": 0.21212121212121213, + "recall@200": 0.42424242424242425, + "recall@300": 0.5454545454545454, + "recall@500": 0.7272727272727273, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.20833482684714794, + "ndcg_full@10": 0.17993148201117176, + "ndcg@20": 0.1653932377156717, + "ndcg_full@20": 0.12923714105090264, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1099476439790576, + "recall@100": 0.20680628272251309, + "recall@200": 0.35602094240837695, + "recall@300": 0.5314136125654451, + "recall@500": 0.743455497382199, + "recall@1000": 0.8167539267015707, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9652843890316114, + "ndcg_full@10": 0.9652843890316114, + "ndcg@20": 0.9452573870508446, + "ndcg_full@20": 0.9452573870508446, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 100, + "recall@10": 0.15384615384615385, + "recall@50": 0.5384615384615384, + "recall@100": 0.6666666666666666, + "recall@200": 0.7692307692307693, + "recall@300": 0.8205128205128205, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.6975961695061491, + "ndcg_full@10": 0.6975961695061491, + "ndcg@20": 0.6915072621288827, + "ndcg_full@20": 0.6689629976333663, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0228310502283105, + "recall@50": 0.0867579908675799, + "recall@100": 0.1598173515981735, + "recall@200": 0.2876712328767123, + "recall@300": 0.3561643835616438, + "recall@500": 0.4246575342465753, + "recall@1000": 0.5525114155251142, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8914006346890427, + "ndcg_full@5": 0.8914006346890427, + "ndcg@10": 0.8311900436213815, + "ndcg_full@10": 0.8311900436213815, + "ndcg@20": 0.8218278533027776, + "ndcg_full@20": 0.8218278533027776, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.013888888888888888, + "recall@50": 0.19444444444444445, + "recall@100": 0.2916666666666667, + "recall@200": 0.4166666666666667, + "recall@300": 0.5138888888888888, + "recall@500": 0.5416666666666666, + "recall@1000": 0.5833333333333334, + "unjudged_fraction@10": 0.5, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8066054772684166, + "ndcg_full@20": 0.7540121692326082, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 2000, + "num_ranked": 23, + "recall@10": 0.0625, + "recall@50": 0.1, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8741527823288849, + "ndcg_full@5": 0.8168061405473396, + "ndcg@10": 0.9499852182706902, + "ndcg_full@10": 0.6132716509697314, + "ndcg@20": 0.9499852182706902, + "ndcg_full@20": 0.45931494452299254, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 171, + "recall@10": 0.03076923076923077, + "recall@50": 0.11153846153846154, + "recall@100": 0.15384615384615385, + "recall@200": 0.2692307692307692, + "recall@300": 0.3346153846153846, + "recall@500": 0.4576923076923077, + "recall@1000": 0.6115384615384616, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.927951759766568, + "ndcg_full@20": 0.927951759766568, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 219, + "recall@10": 0.06493506493506493, + "recall@50": 0.19480519480519481, + "recall@100": 0.36363636363636365, + "recall@200": 0.6623376623376623, + "recall@300": 0.8701298701298701, + "recall@500": 0.935064935064935, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.45503072493644514, + "ndcg_full@10": 0.45503072493644514, + "ndcg@20": 0.46890556143627277, + "ndcg_full@20": 0.46890556143627277, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.11538461538461539, + "recall@50": 0.5256410256410257, + "recall@100": 0.7948717948717948, + "recall@200": 0.8974358974358975, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.772837825180752, + "ndcg_full@10": 0.772837825180752, + "ndcg@20": 0.8356419446123009, + "ndcg_full@20": 0.8356419446123009, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6037735849056604, + "recall@200": 0.8930817610062893, + "recall@300": 0.89937106918239, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9119496855345912, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9832812543076435, + "ndcg_full@20": 0.9832812543076435, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 208, + "recall@10": 0.03787878787878788, + "recall@50": 0.17424242424242425, + "recall@100": 0.3181818181818182, + "recall@200": 0.5340909090909091, + "recall@300": 0.6856060606060606, + "recall@500": 0.7727272727272727, + "recall@1000": 0.7954545454545454, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503198, + "ndcg_full@10": 0.9633180389503198, + "ndcg@20": 0.976326660061057, + "ndcg_full@20": 0.976326660061057, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 141, + "recall@10": 0.06134969325153374, + "recall@50": 0.26380368098159507, + "recall@100": 0.36809815950920244, + "recall@200": 0.4539877300613497, + "recall@300": 0.48466257668711654, + "recall@500": 0.6073619631901841, + "recall@1000": 0.6871165644171779, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010483, + "ndcg_full@10": 0.9363792118010483, + "ndcg@20": 0.820191148239455, + "ndcg_full@20": 0.820191148239455, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 171, + "recall@10": 0.12857142857142856, + "recall@50": 0.5714285714285714, + "recall@100": 0.8571428571428571, + "recall@200": 0.9571428571428572, + "recall@300": 0.9857142857142858, + "recall@500": 0.9857142857142858, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9182293066898518, + "ndcg_full@10": 0.9182293066898518, + "ndcg@20": 0.9310587189025386, + "ndcg_full@20": 0.9310587189025386, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.05027932960893855, + "recall@50": 0.15083798882681565, + "recall@100": 0.29608938547486036, + "recall@200": 0.45251396648044695, + "recall@300": 0.5027932960893855, + "recall@500": 0.6145251396648045, + "recall@1000": 0.6703910614525139, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9609971473894008, + "ndcg_full@20": 0.9609971473894008, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 53, + "recall@10": 0.09523809523809523, + "recall@50": 0.36904761904761907, + "recall@100": 0.47619047619047616, + "recall@200": 0.5476190476190477, + "recall@300": 0.6190476190476191, + "recall@500": 0.6666666666666666, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5430575929537645, + "ndcg_full@5": 0.5074316361651824, + "ndcg@10": 0.643956544454796, + "ndcg_full@10": 0.5035059057362463, + "ndcg@20": 0.6057075227759188, + "ndcg_full@20": 0.46919070920023387, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.03383458646616541, + "recall@50": 0.16917293233082706, + "recall@100": 0.3308270676691729, + "recall@200": 0.6127819548872181, + "recall@300": 0.8157894736842105, + "recall@500": 0.9398496240601504, + "recall@1000": 0.9511278195488722, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8841134626660687, + "ndcg_full@10": 0.8841134626660687, + "ndcg@20": 0.8579161489346545, + "ndcg_full@20": 0.8579161489346545, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 97, + "recall@10": 0.047619047619047616, + "recall@50": 0.23809523809523808, + "recall@100": 0.23809523809523808, + "recall@200": 0.23809523809523808, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7434693717588585, + "ndcg_full@5": 0.5773584151532217, + "ndcg@10": 0.689812400965576, + "ndcg_full@10": 0.3746661058764784, + "ndcg@20": 0.689812400965576, + "ndcg_full@20": 0.2870017017099826, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.025477707006369428, + "recall@50": 0.14968152866242038, + "recall@100": 0.27070063694267515, + "recall@200": 0.4554140127388535, + "recall@300": 0.5923566878980892, + "recall@500": 0.8312101910828026, + "recall@1000": 0.9426751592356688, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7906538248839006, + "ndcg_full@5": 0.7906538248839006, + "ndcg@10": 0.8127386954480682, + "ndcg_full@10": 0.8127386954480682, + "ndcg@20": 0.7689211309584166, + "ndcg_full@20": 0.7689211309584166, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.10416666666666667, + "recall@50": 0.40625, + "recall@100": 0.6875, + "recall@200": 0.8854166666666666, + "recall@300": 0.9479166666666666, + "recall@500": 0.9791666666666666, + "recall@1000": 0.9895833333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4777969189511402, + "ndcg_full@5": 0.4777969189511402, + "ndcg@10": 0.5827276629931442, + "ndcg_full@10": 0.5827276629931442, + "ndcg@20": 0.7307060582802021, + "ndcg_full@20": 0.7307060582802021, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 223, + "recall@10": 0.030395136778115502, + "recall@50": 0.1398176291793313, + "recall@100": 0.2765957446808511, + "recall@200": 0.5319148936170213, + "recall@300": 0.6930091185410334, + "recall@500": 0.7203647416413373, + "recall@1000": 0.7720364741641338, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503198, + "ndcg_full@10": 0.9633180389503198, + "ndcg@20": 0.8676251901686959, + "ndcg_full@20": 0.8676251901686959, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 75, + "recall@10": 0.3333333333333333, + "recall@50": 0.6666666666666666, + "recall@100": 0.8333333333333334, + "recall@200": 0.8888888888888888, + "recall@300": 0.8888888888888888, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7503359712972109, + "ndcg_full@10": 0.7503359712972109, + "ndcg@20": 0.8906391437211598, + "ndcg_full@20": 0.696047231734013, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.039735099337748346, + "recall@50": 0.16556291390728478, + "recall@100": 0.2913907284768212, + "recall@200": 0.4768211920529801, + "recall@300": 0.6688741721854304, + "recall@500": 0.9668874172185431, + "recall@1000": 0.9933774834437086, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8205228949141826, + "ndcg_full@10": 0.8205228949141826, + "ndcg@20": 0.7260806885817352, + "ndcg_full@20": 0.7260806885817352, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08495420999498725, + "recall@50": 0.2835665389548808, + "recall@100": 0.4327706931049727, + "recall@200": 0.5792057708977656, + "recall@300": 0.6572540528790871, + "recall@500": 0.7293634903186007, + "recall@1000": 0.7806891099020966, + "ndcg@5": 0.6973101175835921, + "ndcg@10": 0.6913562712839112, + "ndcg@20": 0.6835007043019152, + "ndcg_full@5": 0.6875742778387257, + "ndcg_full@10": 0.660279992137347, + "ndcg_full@20": 0.6263238353396199, + "P@10(rel>=1)": 0.67, + "P@10(eligible)": 0.6080000000000001, + "graded_P@10": 0.639, + "unjudged_fraction@10": 0.14200000000000002 + }, + "median": { + "recall@10": 0.05954279638490165, + "recall@50": 0.2494824016563147, + "recall@100": 0.37057757716392764, + "recall@200": 0.5498095238095239, + "recall@300": 0.6772401163957456, + "recall@500": 0.7608843537414967, + "recall@1000": 0.8216273522395972, + "ndcg@5": 0.7971152757899023, + "ndcg@10": 0.7176385351469365, + "ndcg@20": 0.716961747564057, + "ndcg_full@5": 0.7426142775581612, + "ndcg_full@10": 0.6990822295864291, + "ndcg_full@20": 0.6819911237383554, + "P@10(rel>=1)": 0.75, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 183, + "recall@10": 0.046632124352331605, + "recall@50": 0.22797927461139897, + "recall@100": 0.37305699481865284, + "recall@200": 0.5647668393782384, + "recall@300": 0.5906735751295337, + "recall@500": 0.6010362694300518, + "recall@1000": 0.6217616580310881, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.7663491917568945, + "ndcg_full@10": 0.7663491917568945, + "ndcg@20": 0.7033340787032365, + "ndcg_full@20": 0.7033340787032365, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 165, + "recall@10": 0.02512562814070352, + "recall@50": 0.10552763819095477, + "recall@100": 0.17587939698492464, + "recall@200": 0.31155778894472363, + "recall@300": 0.39949748743718594, + "recall@500": 0.5552763819095478, + "recall@1000": 0.7889447236180904, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8913696435738849, + "ndcg_full@10": 0.8913696435738849, + "ndcg@20": 0.7685650782990054, + "ndcg_full@20": 0.7685650782990054, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 148, + "recall@10": 0.12658227848101267, + "recall@50": 0.4810126582278481, + "recall@100": 0.7341772151898734, + "recall@200": 0.8987341772151899, + "recall@300": 0.9240506329113924, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.8286559156890452, + "ndcg_full@20": 0.8286559156890452, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029154518950437316, + "recall@50": 0.12536443148688048, + "recall@100": 0.21865889212827988, + "recall@200": 0.4198250728862974, + "recall@300": 0.5510204081632653, + "recall@500": 0.7201166180758017, + "recall@1000": 0.8250728862973761, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.6968404678259545, + "ndcg_full@10": 0.6968404678259545, + "ndcg@20": 0.643593975447791, + "ndcg_full@20": 0.643593975447791, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 144, + "recall@10": 0.09090909090909091, + "recall@50": 0.36363636363636365, + "recall@100": 0.5909090909090909, + "recall@200": 0.7272727272727273, + "recall@300": 0.7545454545454545, + "recall@500": 0.7818181818181819, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8935089661734454, + "ndcg_full@10": 0.8935089661734454, + "ndcg@20": 0.6972592634387595, + "ndcg_full@20": 0.6972592634387595, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 202, + "recall@10": 0.057971014492753624, + "recall@50": 0.2608695652173913, + "recall@100": 0.5289855072463768, + "recall@200": 0.7608695652173914, + "recall@300": 0.8333333333333334, + "recall@500": 0.9057971014492754, + "recall@1000": 0.927536231884058, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.8265528575457551, + "ndcg_full@20": 0.8265528575457551, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 128, + "recall@10": 0.08064516129032258, + "recall@50": 0.2903225806451613, + "recall@100": 0.43548387096774194, + "recall@200": 0.5, + "recall@300": 0.5645161290322581, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.45718160765461063, + "ndcg_full@10": 0.45718160765461063, + "ndcg@20": 0.5875075158518697, + "ndcg_full@20": 0.5644674460243728, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 45, + "recall@10": 0.2, + "recall@50": 0.5, + "recall@100": 0.65, + "recall@200": 0.75, + "recall@300": 0.9, + "recall@500": 0.925, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7104770117651426, + "ndcg_full@10": 0.7104770117651426, + "ndcg@20": 0.7383471063660573, + "ndcg_full@20": 0.7101226071734645, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.038461538461538464, + "recall@50": 0.13846153846153847, + "recall@100": 0.26153846153846155, + "recall@200": 0.36153846153846153, + "recall@300": 0.4230769230769231, + "recall@500": 0.5384615384615384, + "recall@1000": 0.676923076923077, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.2870204396558808, + "ndcg_full@5": 0.2870204396558808, + "ndcg@10": 0.5373250507264833, + "ndcg_full@10": 0.5373250507264833, + "ndcg@20": 0.5962496433653335, + "ndcg_full@20": 0.5962496433653335, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 41, + "recall@10": 0.022935779816513763, + "recall@50": 0.06422018348623854, + "recall@100": 0.09174311926605505, + "recall@200": 0.12844036697247707, + "recall@300": 0.1651376146788991, + "recall@500": 0.24311926605504589, + "recall@1000": 0.3669724770642202, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7248000585287305, + "ndcg_full@10": 0.605504765619221, + "ndcg@20": 0.7609710190203133, + "ndcg_full@20": 0.41027443137283515, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 237, + "recall@10": 0.03289473684210526, + "recall@50": 0.1513157894736842, + "recall@100": 0.27631578947368424, + "recall@200": 0.4407894736842105, + "recall@300": 0.5855263157894737, + "recall@500": 0.7105263157894737, + "recall@1000": 0.8618421052631579, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4912596920895758, + "ndcg_full@5": 0.4912596920895758, + "ndcg@10": 0.6698623505528798, + "ndcg_full@10": 0.6698623505528798, + "ndcg@20": 0.7514299306410998, + "ndcg_full@20": 0.7514299306410998, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.09090909090909091, + "recall@50": 0.2878787878787879, + "recall@100": 0.4696969696969697, + "recall@200": 0.5757575757575758, + "recall@300": 0.7272727272727273, + "recall@500": 0.8181818181818182, + "recall@1000": 0.9242424242424242, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.30523488393970116, + "ndcg_full@10": 0.30523488393970116, + "ndcg@20": 0.22985359827322016, + "ndcg_full@20": 0.22985359827322016, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.2857142857142857, + "recall@50": 0.47619047619047616, + "recall@100": 0.6666666666666666, + "recall@200": 0.8095238095238095, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.1489511766639521, + "ndcg_full@5": 0.1489511766639521, + "ndcg@10": 0.17505739069874593, + "ndcg_full@10": 0.17505739069874593, + "ndcg@20": 0.2752629519016821, + "ndcg_full@20": 0.24365249910922726, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 152, + "recall@10": 0.08196721311475409, + "recall@50": 0.36885245901639346, + "recall@100": 0.6639344262295082, + "recall@200": 0.8852459016393442, + "recall@300": 0.9098360655737705, + "recall@500": 0.9508196721311475, + "recall@1000": 0.9508196721311475, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9266360779006401, + "ndcg_full@10": 0.9266360779006401, + "ndcg@20": 0.8089706472772892, + "ndcg_full@20": 0.8089706472772892, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.05847953216374269, + "recall@50": 0.2807017543859649, + "recall@100": 0.543859649122807, + "recall@200": 0.695906432748538, + "recall@300": 0.7076023391812866, + "recall@500": 0.7192982456140351, + "recall@1000": 0.7192982456140351, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9668728882728055, + "ndcg_full@10": 0.9668728882728055, + "ndcg@20": 0.9786208437424843, + "ndcg_full@20": 0.9786208437424843, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 229, + "recall@10": 0.16666666666666666, + "recall@50": 0.5833333333333334, + "recall@100": 0.8541666666666666, + "recall@200": 0.9375, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.732862603832904, + "ndcg_full@10": 0.732862603832904, + "ndcg@20": 0.6512554577085207, + "ndcg_full@20": 0.6512554577085207, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 165, + "recall@10": 0.0, + "recall@50": 0.0, + "recall@100": 0.03333333333333333, + "recall@200": 0.43333333333333335, + "recall@300": 0.6666666666666666, + "recall@500": 0.7666666666666667, + "recall@1000": 0.8, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.22074620686839042, + "ndcg_full@5": 0.19178318368566782, + "ndcg@10": 0.22074620686839042, + "ndcg_full@10": 0.1244541635112973, + "ndcg@20": 0.32624179110350077, + "ndcg_full@20": 0.11870329167546895, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 2000, + "num_ranked": 80, + "recall@10": 0.10204081632653061, + "recall@50": 0.5714285714285714, + "recall@100": 0.7551020408163265, + "recall@200": 0.7551020408163265, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.4760705018418426, + "ndcg_full@10": 0.4760705018418426, + "ndcg@20": 0.4913659579961583, + "ndcg_full@20": 0.4913659579961583, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 2000, + "num_ranked": 36, + "recall@10": 0.21428571428571427, + "recall@50": 0.2857142857142857, + "recall@100": 0.2857142857142857, + "recall@200": 0.42857142857142855, + "recall@300": 0.42857142857142855, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8529278650606567, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.8529278650606567, + "ndcg_full@10": 0.40002324830925196, + "ndcg@20": 0.8529278650606567, + "ndcg_full@20": 0.357960797701493, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 174, + "recall@10": 0.042735042735042736, + "recall@50": 0.20512820512820512, + "recall@100": 0.38461538461538464, + "recall@200": 0.5854700854700855, + "recall@300": 0.7350427350427351, + "recall@500": 0.8333333333333334, + "recall@1000": 0.8675213675213675, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9553146470008144, + "ndcg_full@5": 0.9553146470008144, + "ndcg@10": 0.9358954210171636, + "ndcg_full@10": 0.9358954210171636, + "ndcg@20": 0.950828427315586, + "ndcg_full@20": 0.950828427315586, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 196, + "recall@10": 0.046242774566473986, + "recall@50": 0.2138728323699422, + "recall@100": 0.4277456647398844, + "recall@200": 0.6936416184971098, + "recall@300": 0.884393063583815, + "recall@500": 0.9653179190751445, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.7917063341896681, + "ndcg_full@10": 0.7917063341896681, + "ndcg@20": 0.790833143041063, + "ndcg_full@20": 0.790833143041063, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 2000, + "num_ranked": 177, + "recall@10": 0.08, + "recall@50": 0.344, + "recall@100": 0.488, + "recall@200": 0.552, + "recall@300": 0.584, + "recall@500": 0.608, + "recall@1000": 0.616, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.34519134224686937, + "ndcg_full@5": 0.34519134224686937, + "ndcg@10": 0.43808927588501717, + "ndcg_full@10": 0.43808927588501717, + "ndcg@20": 0.49769073529987684, + "ndcg_full@20": 0.49769073529987684, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 147, + "recall@10": 0.05319148936170213, + "recall@50": 0.26595744680851063, + "recall@100": 0.5106382978723404, + "recall@200": 0.7712765957446809, + "recall@300": 0.8138297872340425, + "recall@500": 0.8297872340425532, + "recall@1000": 0.8563829787234043, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9102649208458158, + "ndcg_full@20": 0.9102649208458158, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 2000, + "num_ranked": 55, + "recall@10": 0.1935483870967742, + "recall@50": 0.3225806451612903, + "recall@100": 0.3548387096774194, + "recall@200": 0.41935483870967744, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.4947447660984327, + "ndcg_full@10": 0.4133144170074498, + "ndcg@20": 0.6266319901381197, + "ndcg_full@20": 0.38376778025749564, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.027624309392265192, + "recall@50": 0.11602209944751381, + "recall@100": 0.22099447513812154, + "recall@200": 0.30939226519337015, + "recall@300": 0.39226519337016574, + "recall@500": 0.5580110497237569, + "recall@1000": 0.7458563535911602, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.5933607200949346, + "ndcg_full@10": 0.5933607200949346, + "ndcg@20": 0.5343244447126205, + "ndcg_full@20": 0.47038712930722587, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 84, + "recall@10": 0.16071428571428573, + "recall@50": 0.625, + "recall@100": 0.8035714285714286, + "recall@200": 0.9107142857142857, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7357678207018167, + "ndcg_full@10": 0.7357678207018167, + "ndcg@20": 0.7412876650646402, + "ndcg_full@20": 0.7412876650646402, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 106, + "recall@10": 0.27586206896551724, + "recall@50": 0.7241379310344828, + "recall@100": 0.8275862068965517, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.890225066021651, + "ndcg_full@20": 0.890225066021651, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 164, + "recall@10": 0.014705882352941176, + "recall@50": 0.07352941176470588, + "recall@100": 0.17647058823529413, + "recall@200": 0.3235294117647059, + "recall@300": 0.35294117647058826, + "recall@500": 0.4411764705882353, + "recall@1000": 0.5735294117647058, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.27255481104935336, + "ndcg_full@10": 0.24618570546556082, + "ndcg@20": 0.3989578329107922, + "ndcg_full@20": 0.23256425527684743, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.09090909090909091, + "recall@100": 0.21212121212121213, + "recall@200": 0.42424242424242425, + "recall@300": 0.5454545454545454, + "recall@500": 0.7272727272727273, + "recall@1000": 0.8181818181818182, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.18935744919705852, + "ndcg_full@10": 0.1635413866202963, + "ndcg@20": 0.1503274420424204, + "ndcg_full@20": 0.11746483169073851, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1099476439790576, + "recall@100": 0.20680628272251309, + "recall@200": 0.35602094240837695, + "recall@300": 0.5314136125654451, + "recall@500": 0.743455497382199, + "recall@1000": 0.8167539267015707, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7058075148678526, + "ndcg_full@10": 0.7058075148678526, + "ndcg@20": 0.5757701414299161, + "ndcg_full@20": 0.5757701414299161, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 100, + "recall@10": 0.15384615384615385, + "recall@50": 0.5384615384615384, + "recall@100": 0.6666666666666666, + "recall@200": 0.7692307692307693, + "recall@300": 0.8205128205128205, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.4419240888751214, + "ndcg_full@10": 0.4419240888751214, + "ndcg@20": 0.5576509243737237, + "ndcg_full@20": 0.5394705947896978, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0228310502283105, + "recall@50": 0.0867579908675799, + "recall@100": 0.1598173515981735, + "recall@200": 0.2876712328767123, + "recall@300": 0.3561643835616438, + "recall@500": 0.4246575342465753, + "recall@1000": 0.5525114155251142, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5782348203855859, + "ndcg_full@5": 0.5782348203855859, + "ndcg@10": 0.6282386400188342, + "ndcg_full@10": 0.6282386400188342, + "ndcg@20": 0.5825610015522906, + "ndcg_full@20": 0.5825610015522906, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 78, + "recall@10": 0.013888888888888888, + "recall@50": 0.19444444444444445, + "recall@100": 0.2916666666666667, + "recall@200": 0.4166666666666667, + "recall@300": 0.5138888888888888, + "recall@500": 0.5416666666666666, + "recall@1000": 0.5833333333333334, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.642186726668901, + "ndcg_full@10": 0.642186726668901, + "ndcg@20": 0.7146220504808295, + "ndcg_full@20": 0.6680263618953771, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 2000, + "num_ranked": 23, + "recall@10": 0.0625, + "recall@50": 0.1, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8741527823288849, + "ndcg_full@5": 0.8168061405473396, + "ndcg@10": 0.8514337716675965, + "ndcg_full@10": 0.5496508627707797, + "ndcg@20": 0.9386980505354017, + "ndcg_full@20": 0.45385763348020275, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 171, + "recall@10": 0.03076923076923077, + "recall@50": 0.11153846153846154, + "recall@100": 0.15384615384615385, + "recall@200": 0.2692307692307692, + "recall@300": 0.3346153846153846, + "recall@500": 0.4576923076923077, + "recall@1000": 0.6115384615384616, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.927951759766568, + "ndcg_full@20": 0.927951759766568, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 219, + "recall@10": 0.06493506493506493, + "recall@50": 0.19480519480519481, + "recall@100": 0.36363636363636365, + "recall@200": 0.6623376623376623, + "recall@300": 0.8701298701298701, + "recall@500": 0.935064935064935, + "recall@1000": 0.935064935064935, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.4348251131214945, + "ndcg_full@10": 0.4348251131214945, + "ndcg@20": 0.3504157096722658, + "ndcg_full@20": 0.3504157096722658, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.11538461538461539, + "recall@50": 0.5256410256410257, + "recall@100": 0.7948717948717948, + "recall@200": 0.8974358974358975, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.772837825180752, + "ndcg_full@10": 0.772837825180752, + "ndcg@20": 0.7976372250516909, + "ndcg_full@20": 0.7976372250516909, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 132, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6037735849056604, + "recall@200": 0.8930817610062893, + "recall@300": 0.89937106918239, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9119496855345912, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213246, + "ndcg_full@10": 0.9216017310213246, + "ndcg@20": 0.9329718317415944, + "ndcg_full@20": 0.9329718317415944, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 208, + "recall@10": 0.03787878787878788, + "recall@50": 0.17424242424242425, + "recall@100": 0.3181818181818182, + "recall@200": 0.5340909090909091, + "recall@300": 0.6856060606060606, + "recall@500": 0.7727272727272727, + "recall@1000": 0.7954545454545454, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9633180389503198, + "ndcg_full@10": 0.9633180389503198, + "ndcg@20": 0.9367055930052414, + "ndcg_full@20": 0.9367055930052414, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 141, + "recall@10": 0.06134969325153374, + "recall@50": 0.26380368098159507, + "recall@100": 0.36809815950920244, + "recall@200": 0.4539877300613497, + "recall@300": 0.48466257668711654, + "recall@500": 0.6073619631901841, + "recall@1000": 0.6871165644171779, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8205228949141826, + "ndcg_full@10": 0.8205228949141826, + "ndcg@20": 0.7426855468018326, + "ndcg_full@20": 0.7426855468018326, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 171, + "recall@10": 0.12857142857142856, + "recall@50": 0.5714285714285714, + "recall@100": 0.8571428571428571, + "recall@200": 0.9571428571428572, + "recall@300": 0.9857142857142858, + "recall@500": 0.9857142857142858, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7013239913469036, + "ndcg_full@10": 0.7013239913469036, + "ndcg@20": 0.8072442562469524, + "ndcg_full@20": 0.8072442562469524, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 2000, + "num_ranked": 173, + "recall@10": 0.05027932960893855, + "recall@50": 0.15083798882681565, + "recall@100": 0.29608938547486036, + "recall@200": 0.45251396648044695, + "recall@300": 0.5027932960893855, + "recall@500": 0.6145251396648045, + "recall@1000": 0.6703910614525139, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.8061146145519253, + "ndcg_full@10": 0.8061146145519253, + "ndcg@20": 0.801632803573321, + "ndcg_full@20": 0.801632803573321, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 53, + "recall@10": 0.09523809523809523, + "recall@50": 0.36904761904761907, + "recall@100": 0.47619047619047616, + "recall@200": 0.5476190476190477, + "recall@300": 0.6190476190476191, + "recall@500": 0.6666666666666666, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5430575929537645, + "ndcg_full@5": 0.5074316361651824, + "ndcg@10": 0.643956544454796, + "ndcg_full@10": 0.5035059057362463, + "ndcg@20": 0.556289967933473, + "ndcg_full@20": 0.4309110829258111, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.03383458646616541, + "recall@50": 0.16917293233082706, + "recall@100": 0.3308270676691729, + "recall@200": 0.6127819548872181, + "recall@300": 0.8157894736842105, + "recall@500": 0.9398496240601504, + "recall@1000": 0.9511278195488722, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.8841134626660687, + "ndcg_full@10": 0.8841134626660687, + "ndcg@20": 0.797884530907093, + "ndcg_full@20": 0.797884530907093, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 97, + "recall@10": 0.047619047619047616, + "recall@50": 0.23809523809523808, + "recall@100": 0.23809523809523808, + "recall@200": 0.23809523809523808, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5745154706993483, + "ndcg_full@5": 0.44615333764087994, + "ndcg@10": 0.6773941966568309, + "ndcg_full@10": 0.36792125721353275, + "ndcg@20": 0.6773941966568309, + "ndcg_full@20": 0.2818350132541021, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.025477707006369428, + "recall@50": 0.14968152866242038, + "recall@100": 0.27070063694267515, + "recall@200": 0.4554140127388535, + "recall@300": 0.5923566878980892, + "recall@500": 0.8312101910828026, + "recall@1000": 0.9426751592356688, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6922500167496441, + "ndcg_full@5": 0.6922500167496441, + "ndcg@10": 0.7623507707914884, + "ndcg_full@10": 0.7623507707914884, + "ndcg@20": 0.7193014446472844, + "ndcg_full@20": 0.7193014446472844, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.10416666666666667, + "recall@50": 0.40625, + "recall@100": 0.6875, + "recall@200": 0.8854166666666666, + "recall@300": 0.9479166666666666, + "recall@500": 0.9791666666666666, + "recall@1000": 0.9895833333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.4777969189511402, + "ndcg_full@5": 0.4777969189511402, + "ndcg@10": 0.5827276629931442, + "ndcg_full@10": 0.5827276629931442, + "ndcg@20": 0.6959558855813335, + "ndcg_full@20": 0.6959558855813335, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 223, + "recall@10": 0.030395136778115502, + "recall@50": 0.1398176291793313, + "recall@100": 0.2765957446808511, + "recall@200": 0.5319148936170213, + "recall@300": 0.6930091185410334, + "recall@500": 0.7203647416413373, + "recall@1000": 0.7720364741641338, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.6519927576051744, + "ndcg_full@10": 0.6519927576051744, + "ndcg@20": 0.6583033689381178, + "ndcg_full@20": 0.6583033689381178, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 75, + "recall@10": 0.3333333333333333, + "recall@50": 0.6666666666666666, + "recall@100": 0.8333333333333334, + "recall@200": 0.8888888888888888, + "recall@300": 0.8888888888888888, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.530943388959284, + "ndcg_full@5": 0.530943388959284, + "ndcg@10": 0.5625623443418019, + "ndcg_full@10": 0.5625623443418019, + "ndcg@20": 0.6689410727939019, + "ndcg_full@20": 0.5227870178330611, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.039735099337748346, + "recall@50": 0.16556291390728478, + "recall@100": 0.2913907284768212, + "recall@200": 0.4768211920529801, + "recall@300": 0.6688741721854304, + "recall@500": 0.9668874172185431, + "recall@1000": 0.9933774834437086, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8205228949141826, + "ndcg_full@10": 0.8205228949141826, + "ndcg@20": 0.7260806885817352, + "ndcg_full@20": 0.7260806885817352, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + } + } + } + } + }, + { + "name": "huatuo_medcpt", + "results_dir": "huatuo_medcpt/results_trec22", + "evaluation_input_sha256": "651b27235a6bb67773a3950f8a0a16f57c1152b9d7fc42f389fae6b6903a5f65", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.0781627074484694, + "recall@50": 0.28600996393064365, + "recall@100": 0.42541886913233784, + "recall@200": 0.5560649060417724, + "recall@300": 0.6276470014652813, + "recall@500": 0.6986965480131504, + "recall@1000": 0.7612156405464531, + "ndcg@5": 0.6013560883638938, + "ndcg@10": 0.6103701188641527, + "ndcg@20": 0.6324017781611648, + "ndcg_full@5": 0.6001873787264176, + "ndcg_full@10": 0.6012514579020997, + "ndcg_full@20": 0.6006096809026558, + "P@10(rel>=1)": 0.6539999999999999, + "P@10(eligible)": 0.5419999999999999, + "graded_P@10": 0.5979999999999999, + "unjudged_fraction@10": 0.2280000000000001 + }, + "median": { + "recall@10": 0.0547008547008547, + "recall@50": 0.21473805411629243, + "recall@100": 0.36262050832602977, + "recall@200": 0.5159574468085106, + "recall@300": 0.6037549407114624, + "recall@500": 0.7320885801128962, + "recall@1000": 0.8038537549407114, + "ndcg@5": 0.7156909463786012, + "ndcg@10": 0.6769174245240719, + "ndcg@20": 0.653153749055559, + "ndcg_full@5": 0.7156909463786012, + "ndcg_full@10": 0.6564079405371519, + "ndcg_full@20": 0.6294229493137738, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.55, + "graded_P@10": 0.65, + "unjudged_fraction@10": 0.2 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 125, + "recall@10": 0.04145077720207254, + "recall@50": 0.20725388601036268, + "recall@100": 0.38860103626943004, + "recall@200": 0.5595854922279793, + "recall@300": 0.5906735751295337, + "recall@500": 0.6062176165803109, + "recall@1000": 0.616580310880829, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.7124433618464341, + "ndcg_full@10": 0.7124433618464341, + "ndcg@20": 0.7820820437544583, + "ndcg_full@20": 0.7820820437544583, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 175, + "recall@10": 0.02512562814070352, + "recall@50": 0.09547738693467336, + "recall@100": 0.16331658291457288, + "recall@200": 0.24874371859296482, + "recall@300": 0.31155778894472363, + "recall@500": 0.4296482412060301, + "recall@1000": 0.5954773869346733, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7012236839470684, + "ndcg_full@5": 0.7012236839470684, + "ndcg@10": 0.6595488066508806, + "ndcg_full@10": 0.6595488066508806, + "ndcg@20": 0.6548809366782311, + "ndcg_full@20": 0.6225819622856443, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.12658227848101267, + "recall@50": 0.5569620253164557, + "recall@100": 0.7974683544303798, + "recall@200": 0.8860759493670886, + "recall@300": 0.8987341772151899, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9240506329113924, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6281895090257151, + "ndcg_full@5": 0.6281895090257151, + "ndcg@10": 0.6532670744234234, + "ndcg_full@10": 0.6532670744234234, + "ndcg@20": 0.5313824602304471, + "ndcg_full@20": 0.5313824602304471, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 1778, + "num_ranked": 250, + "recall@10": 0.014577259475218658, + "recall@50": 0.0641399416909621, + "recall@100": 0.14285714285714285, + "recall@200": 0.2886297376093295, + "recall@300": 0.4110787172011662, + "recall@500": 0.6064139941690962, + "recall@1000": 0.793002915451895, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.9449770584254795, + "ndcg_full@10": 0.9449770584254795, + "ndcg@20": 0.9644899900915856, + "ndcg_full@20": 0.9644899900915856, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 1748, + "num_ranked": 173, + "recall@10": 0.09090909090909091, + "recall@50": 0.36363636363636365, + "recall@100": 0.6090909090909091, + "recall@200": 0.7363636363636363, + "recall@300": 0.7727272727272727, + "recall@500": 0.7818181818181819, + "recall@1000": 0.7909090909090909, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.4818809355796682, + "ndcg_full@10": 0.4818809355796682, + "ndcg@20": 0.4544811155580991, + "ndcg_full@20": 0.4544811155580991, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.15942028985507245, + "recall@100": 0.26811594202898553, + "recall@200": 0.463768115942029, + "recall@300": 0.6014492753623188, + "recall@500": 0.6884057971014492, + "recall@1000": 0.7971014492753623, + "unjudged_fraction@10": 0.3, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8911186912154221, + "ndcg_full@20": 0.8911186912154221, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 1564, + "num_ranked": 139, + "recall@10": 0.16129032258064516, + "recall@50": 0.3548387096774194, + "recall@100": 0.4032258064516129, + "recall@200": 0.5, + "recall@300": 0.5806451612903226, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.5075394027672818, + "ndcg_full@10": 0.5075394027672818, + "ndcg@20": 0.6127382820030616, + "ndcg_full@20": 0.5887087463419544, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 129, + "recall@10": 0.2, + "recall@50": 0.65, + "recall@100": 0.775, + "recall@200": 0.875, + "recall@300": 0.9, + "recall@500": 0.925, + "recall@1000": 0.975, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.1860868145872509, + "ndcg_full@10": 0.1860868145872509, + "ndcg@20": 0.35455340031572785, + "ndcg_full@20": 0.34738034152947445, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05384615384615385, + "recall@50": 0.16153846153846155, + "recall@100": 0.27692307692307694, + "recall@200": 0.33076923076923076, + "recall@300": 0.38461538461538464, + "recall@500": 0.5, + "recall@1000": 0.7153846153846154, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6942860423972633, + "ndcg_full@10": 0.6942860423972633, + "ndcg@20": 0.707593569953991, + "ndcg_full@20": 0.707593569953991, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 87, + "recall@10": 0.027522935779816515, + "recall@50": 0.05963302752293578, + "recall@100": 0.0963302752293578, + "recall@200": 0.12385321100917432, + "recall@300": 0.14678899082568808, + "recall@500": 0.23853211009174313, + "recall@1000": 0.3256880733944954, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.710970696247901, + "ndcg_full@5": 0.710970696247901, + "ndcg@10": 0.5666918613313762, + "ndcg_full@10": 0.5666918613313762, + "ndcg@20": 0.5073061231564634, + "ndcg_full@20": 0.45662822268550707, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029605263157894735, + "recall@50": 0.1513157894736842, + "recall@100": 0.2993421052631579, + "recall@200": 0.5526315789473685, + "recall@300": 0.6973684210526315, + "recall@500": 0.7796052631578947, + "recall@1000": 0.875, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8140730213119423, + "ndcg_full@10": 0.8140730213119423, + "ndcg@20": 0.8416241605915367, + "ndcg_full@20": 0.8416241605915367, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.19696969696969696, + "recall@100": 0.3484848484848485, + "recall@200": 0.48484848484848486, + "recall@300": 0.6060606060606061, + "recall@500": 0.7575757575757576, + "recall@1000": 0.9090909090909091, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.06625422345438903, + "ndcg_full@10": 0.06625422345438903, + "ndcg@20": 0.15006207166565283, + "ndcg_full@20": 0.15006207166565283, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 222, + "recall@10": 0.2857142857142857, + "recall@50": 0.6190476190476191, + "recall@100": 0.7619047619047619, + "recall@200": 0.8095238095238095, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.09668388883949189, + "ndcg_full@20": 0.08371134856579455, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 170, + "recall@10": 0.08196721311475409, + "recall@50": 0.4016393442622951, + "recall@100": 0.7622950819672131, + "recall@200": 0.9344262295081968, + "recall@300": 0.9426229508196722, + "recall@500": 0.9590163934426229, + "recall@1000": 0.9672131147540983, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6578242262397573, + "ndcg_full@5": 0.6578242262397573, + "ndcg@10": 0.600450773678004, + "ndcg_full@10": 0.600450773678004, + "ndcg@20": 0.5877566690799427, + "ndcg_full@20": 0.5877566690799427, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.3, + "graded_P@10": 0.6 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1219, + "num_ranked": 175, + "recall@10": 0.05263157894736842, + "recall@50": 0.28654970760233917, + "recall@100": 0.5555555555555556, + "recall@200": 0.7017543859649122, + "recall@300": 0.7076023391812866, + "recall@500": 0.7134502923976608, + "recall@1000": 0.7134502923976608, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 1.0000000000000002, + "ndcg_full@20": 1.0000000000000002, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 243, + "recall@10": 0.16666666666666666, + "recall@50": 0.6458333333333334, + "recall@100": 0.8958333333333334, + "recall@200": 0.9375, + "recall@300": 0.9375, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8546876825242049, + "ndcg_full@10": 0.8546876825242049, + "ndcg@20": 0.8228368261657892, + "ndcg_full@20": 0.8228368261657892, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 144, + "recall@10": 0.0, + "recall@50": 0.03333333333333333, + "recall@100": 0.1, + "recall@200": 0.23333333333333334, + "recall@300": 0.5, + "recall@500": 0.6666666666666666, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.14887095502556022, + "ndcg_full@20": 0.07692781742226024, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 690, + "num_ranked": 116, + "recall@10": 0.12244897959183673, + "recall@50": 0.6530612244897959, + "recall@100": 0.7142857142857143, + "recall@200": 0.7346938775510204, + "recall@300": 0.7346938775510204, + "recall@500": 0.7346938775510204, + "recall@1000": 0.7346938775510204, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8981000251498876, + "ndcg_full@10": 0.8981000251498876, + "ndcg@20": 0.6514265614328869, + "ndcg_full@20": 0.6514265614328869, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 780, + "num_ranked": 135, + "recall@10": 0.21428571428571427, + "recall@50": 0.35714285714285715, + "recall@100": 0.35714285714285715, + "recall@200": 0.5, + "recall@300": 0.5714285714285714, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.16781537886316344, + "ndcg_full@10": 0.13436872776344766, + "ndcg@20": 0.44727220023144587, + "ndcg_full@20": 0.320470891415297, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 1411, + "num_ranked": 218, + "recall@10": 0.042735042735042736, + "recall@50": 0.20512820512820512, + "recall@100": 0.4017094017094017, + "recall@200": 0.6495726495726496, + "recall@300": 0.8162393162393162, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7204111965093014, + "ndcg_full@5": 0.7204111965093014, + "ndcg@10": 0.7882693133401377, + "ndcg_full@10": 0.7882693133401377, + "ndcg@20": 0.8425582888866556, + "ndcg_full@20": 0.8425582888866556, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.057803468208092484, + "recall@50": 0.23699421965317918, + "recall@100": 0.44508670520231214, + "recall@200": 0.7630057803468208, + "recall@300": 0.930635838150289, + "recall@500": 0.9826589595375722, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9263369616450166, + "ndcg_full@20": 0.9263369616450166, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 1527, + "num_ranked": 152, + "recall@10": 0.08, + "recall@50": 0.36, + "recall@100": 0.464, + "recall@200": 0.552, + "recall@300": 0.592, + "recall@500": 0.616, + "recall@1000": 0.632, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8364586133797035, + "ndcg_full@10": 0.8364586133797035, + "ndcg@20": 0.7881696871130003, + "ndcg_full@20": 0.7881696871130003, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05319148936170213, + "recall@50": 0.26063829787234044, + "recall@100": 0.5, + "recall@200": 0.776595744680851, + "recall@300": 0.8138297872340425, + "recall@500": 0.8351063829787234, + "recall@1000": 0.8404255319148937, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.7945923900143085, + "ndcg_full@20": 0.7945923900143085, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1110, + "num_ranked": 112, + "recall@10": 0.1935483870967742, + "recall@50": 0.3225806451612903, + "recall@100": 0.41935483870967744, + "recall@200": 0.45161290322580644, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.8303848902058956, + "ndcg_full@10": 0.8039700195941961, + "ndcg@20": 0.9209300938151209, + "ndcg_full@20": 0.67437156389444, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027624309392265192, + "recall@50": 0.08839779005524862, + "recall@100": 0.1712707182320442, + "recall@200": 0.27624309392265195, + "recall@300": 0.35911602209944754, + "recall@500": 0.4861878453038674, + "recall@1000": 0.6464088397790055, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.725452110735279, + "ndcg_full@10": 0.725452110735279, + "ndcg@20": 0.7263752074590025, + "ndcg_full@20": 0.7146303409881374, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.17857142857142858, + "recall@50": 0.6785714285714286, + "recall@100": 0.7857142857142857, + "recall@200": 0.9107142857142857, + "recall@300": 0.9464285714285714, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8082168163143322, + "ndcg_full@5": 0.8082168163143322, + "ndcg@10": 0.7424938263529737, + "ndcg_full@10": 0.7424938263529737, + "ndcg@20": 0.6320709054233665, + "ndcg_full@20": 0.6320709054233665, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 91, + "recall@10": 0.20689655172413793, + "recall@50": 0.6896551724137931, + "recall@100": 0.8275862068965517, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.5511391849981765, + "ndcg_full@10": 0.5511391849981765, + "ndcg@20": 0.5342973443158664, + "ndcg_full@20": 0.48142675550867364, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.07352941176470588, + "recall@50": 0.19117647058823528, + "recall@100": 0.2647058823529412, + "recall@200": 0.38235294117647056, + "recall@300": 0.4117647058823529, + "recall@500": 0.47058823529411764, + "recall@1000": 0.5588235294117647, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.2833046215732896, + "ndcg_full@5": 0.2833046215732896, + "ndcg@10": 0.32664045841248474, + "ndcg_full@10": 0.32664045841248474, + "ndcg@20": 0.3600712928437279, + "ndcg_full@20": 0.3423124410957201, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.030303030303030304, + "recall@50": 0.15151515151515152, + "recall@100": 0.24242424242424243, + "recall@200": 0.3939393939393939, + "recall@300": 0.48484848484848486, + "recall@500": 0.6363636363636364, + "recall@1000": 0.7575757575757576, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.28339660873566963, + "ndcg_full@10": 0.23365080824310533, + "ndcg@20": 0.38323149381715627, + "ndcg_full@20": 0.2515327999541444, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1099476439790576, + "recall@100": 0.20680628272251309, + "recall@200": 0.3900523560209424, + "recall@300": 0.5549738219895288, + "recall@500": 0.7617801047120419, + "recall@1000": 0.8115183246073299, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8948256838011641, + "ndcg_full@10": 0.8948256838011641, + "ndcg@20": 0.8771833055971204, + "ndcg_full@20": 0.8771833055971204, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 1522, + "num_ranked": 120, + "recall@10": 0.1282051282051282, + "recall@50": 0.5384615384615384, + "recall@100": 0.6410256410256411, + "recall@200": 0.8205128205128205, + "recall@300": 0.8205128205128205, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.21398626473452756, + "ndcg_full@5": 0.21398626473452756, + "ndcg@10": 0.3600558568883671, + "ndcg_full@10": 0.3600558568883671, + "ndcg@20": 0.3148323481484648, + "ndcg_full@20": 0.30974177807979764, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0273972602739726, + "recall@50": 0.1095890410958904, + "recall@100": 0.1780821917808219, + "recall@200": 0.2785388127853881, + "recall@300": 0.3287671232876712, + "recall@500": 0.410958904109589, + "recall@1000": 0.5251141552511416, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.34960740992542505, + "ndcg_full@5": 0.34960740992542505, + "ndcg@10": 0.4338674406457632, + "ndcg_full@10": 0.4338674406457632, + "ndcg@20": 0.5106771619112777, + "ndcg_full@20": 0.5106771619112777, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.1, + "graded_P@10": 0.3 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 137, + "recall@10": 0.05555555555555555, + "recall@50": 0.2222222222222222, + "recall@100": 0.2638888888888889, + "recall@200": 0.2638888888888889, + "recall@300": 0.3055555555555556, + "recall@500": 0.3333333333333333, + "recall@1000": 0.5, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7058075148678526, + "ndcg_full@10": 0.7058075148678526, + "ndcg@20": 0.6267749932041812, + "ndcg_full@20": 0.6267749932041812, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 687, + "num_ranked": 37, + "recall@10": 0.075, + "recall@50": 0.1375, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6827250156238165, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.7057421831526036, + "ndcg_full@10": 0.481486331854665, + "ndcg@20": 0.8696184415213486, + "ndcg_full@20": 0.4443491095298695, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 1843, + "num_ranked": 203, + "recall@10": 0.03461538461538462, + "recall@50": 0.1346153846153846, + "recall@100": 0.18461538461538463, + "recall@200": 0.2692307692307692, + "recall@300": 0.35, + "recall@500": 0.4653846153846154, + "recall@1000": 0.5961538461538461, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3344578318168005, + "ndcg_full@5": 0.3344578318168005, + "ndcg@10": 0.4818743814103431, + "ndcg_full@10": 0.4818743814103431, + "ndcg@20": 0.5799886805407521, + "ndcg_full@20": 0.5799886805407521, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.4, + "graded_P@10": 0.6 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.025974025974025976, + "recall@50": 0.11688311688311688, + "recall@100": 0.2077922077922078, + "recall@200": 0.4025974025974026, + "recall@300": 0.6623376623376623, + "recall@500": 0.935064935064935, + "recall@1000": 0.948051948051948, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.5884674004369673, + "ndcg_full@10": 0.5884674004369673, + "ndcg@20": 0.5027820617372241, + "ndcg_full@20": 0.5027820617372241, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.10256410256410256, + "recall@50": 0.5, + "recall@100": 0.7692307692307693, + "recall@200": 0.8846153846153846, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9230769230769231, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7691415988388896, + "ndcg_full@5": 0.7691415988388896, + "ndcg@10": 0.609716975895182, + "ndcg_full@10": 0.609716975895182, + "ndcg@20": 0.6553004498769627, + "ndcg_full@20": 0.6553004498769627, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 1474, + "num_ranked": 135, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6226415094339622, + "recall@200": 0.89937106918239, + "recall@300": 0.89937106918239, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9056603773584906, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8581437227514841, + "ndcg_full@10": 0.8581437227514841, + "ndcg@20": 0.9084505904897269, + "ndcg_full@20": 0.9084505904897269, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03409090909090909, + "recall@50": 0.16287878787878787, + "recall@100": 0.3333333333333333, + "recall@200": 0.5757575757575758, + "recall@300": 0.7689393939393939, + "recall@500": 0.7840909090909091, + "recall@1000": 0.8106060606060606, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9639236078740718, + "ndcg_full@10": 0.9639236078740718, + "ndcg@20": 0.9056086137940647, + "ndcg_full@20": 0.9056086137940647, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 1898, + "num_ranked": 154, + "recall@10": 0.06134969325153374, + "recall@50": 0.25766871165644173, + "recall@100": 0.36809815950920244, + "recall@200": 0.44785276073619634, + "recall@300": 0.5214723926380368, + "recall@500": 0.5644171779141104, + "recall@1000": 0.656441717791411, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.9052116356304493, + "ndcg_full@10": 0.9052116356304493, + "ndcg@20": 0.7999011016420917, + "ndcg_full@20": 0.7999011016420917, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.14285714285714285, + "recall@50": 0.7, + "recall@100": 0.9571428571428572, + "recall@200": 0.9714285714285714, + "recall@300": 0.9714285714285714, + "recall@500": 0.9714285714285714, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.8425599346661845, + "ndcg_full@10": 0.8425599346661845, + "ndcg@20": 0.8652420915180103, + "ndcg_full@20": 0.8652420915180103, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1641, + "num_ranked": 168, + "recall@10": 0.0446927374301676, + "recall@50": 0.22346368715083798, + "recall@100": 0.3743016759776536, + "recall@200": 0.46368715083798884, + "recall@300": 0.49162011173184356, + "recall@500": 0.547486033519553, + "recall@1000": 0.6536312849162011, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5878056198050307, + "ndcg_full@5": 0.5878056198050307, + "ndcg@10": 0.6993869397899498, + "ndcg_full@10": 0.6993869397899498, + "ndcg@20": 0.7003724991556767, + "ndcg_full@20": 0.7003724991556767, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.011904761904761904, + "recall@50": 0.023809523809523808, + "recall@100": 0.03571428571428571, + "recall@200": 0.11904761904761904, + "recall@300": 0.16666666666666666, + "recall@500": 0.34523809523809523, + "recall@1000": 0.6309523809523809, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.22079202960190156, + "ndcg_full@10": 0.13568544539116628, + "ndcg@20": 0.26294680986963054, + "ndcg_full@20": 0.11846072360155442, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 1781, + "num_ranked": 250, + "recall@10": 0.03759398496240601, + "recall@50": 0.17669172932330826, + "recall@100": 0.3533834586466165, + "recall@200": 0.6390977443609023, + "recall@300": 0.8609022556390977, + "recall@500": 0.943609022556391, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0000000000000002, + "ndcg_full@10": 1.0000000000000002, + "ndcg@20": 0.9642320654292361, + "ndcg_full@20": 0.9642320654292361, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 1795, + "num_ranked": 150, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.2857142857142857, + "recall@200": 0.3333333333333333, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.09843703889494596, + "ndcg_full@5": 0.08479005131840403, + "ndcg@10": 0.09198623196451756, + "ndcg_full@10": 0.05502294157452004, + "ndcg@20": 0.3668753508939135, + "ndcg_full@20": 0.1681045910666084, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.009554140127388535, + "recall@50": 0.044585987261146494, + "recall@100": 0.10828025477707007, + "recall@200": 0.25796178343949044, + "recall@300": 0.4140127388535032, + "recall@500": 0.643312101910828, + "recall@1000": 0.8662420382165605, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.2870204396558808, + "ndcg_full@5": 0.2870204396558808, + "ndcg@10": 0.4981259162371456, + "ndcg_full@10": 0.4981259162371456, + "ndcg@20": 0.5727423619772909, + "ndcg_full@20": 0.5727423619772909, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.10416666666666667, + "recall@50": 0.4895833333333333, + "recall@100": 0.8541666666666666, + "recall@200": 0.9375, + "recall@300": 0.9479166666666666, + "recall@500": 0.9791666666666666, + "recall@1000": 0.9895833333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8293271620269695, + "ndcg_full@10": 0.8293271620269695, + "ndcg@20": 0.782126732072705, + "ndcg_full@20": 0.782126732072705, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 239, + "recall@10": 0.030395136778115502, + "recall@50": 0.13677811550151975, + "recall@100": 0.2796352583586626, + "recall@200": 0.5319148936170213, + "recall@300": 0.7051671732522796, + "recall@500": 0.729483282674772, + "recall@1000": 0.78419452887538, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6069931323672637, + "ndcg_full@5": 0.6069931323672637, + "ndcg@10": 0.6446601700799531, + "ndcg_full@10": 0.6446601700799531, + "ndcg@20": 0.7145727199471759, + "ndcg_full@20": 0.7145727199471759, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.4, + "graded_P@10": 0.7 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1111111111111111, + "recall@50": 0.3888888888888889, + "recall@100": 0.6666666666666666, + "recall@200": 0.8888888888888888, + "recall@300": 0.8888888888888888, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.1610425878239398, + "ndcg_full@10": 0.1610425878239398, + "ndcg@20": 0.30774818109247026, + "ndcg_full@20": 0.2782880531363555, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.013245033112582781, + "recall@50": 0.12582781456953643, + "recall@100": 0.2052980132450331, + "recall@200": 0.3708609271523179, + "recall@300": 0.5562913907284768, + "recall@500": 0.9072847682119205, + "recall@1000": 0.9933774834437086, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.20904936749052638, + "ndcg_full@10": 0.20904936749052638, + "ndcg@20": 0.41631873631590255, + "ndcg_full@20": 0.41631873631590255, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.0781627074484694, + "recall@50": 0.28600996393064365, + "recall@100": 0.42541886913233784, + "recall@200": 0.5560649060417724, + "recall@300": 0.6276470014652813, + "recall@500": 0.6986965480131504, + "recall@1000": 0.7612156405464531, + "ndcg@5": 0.49359348519000684, + "ndcg@10": 0.4905132467678659, + "ndcg@20": 0.47923505291385365, + "ndcg_full@5": 0.49281677475913205, + "ndcg_full@10": 0.4851575514440507, + "ndcg_full@20": 0.45912070718909703, + "P@10(rel>=1)": 0.5339999999999999, + "P@10(eligible)": 0.43799999999999994, + "graded_P@10": 0.48599999999999993, + "unjudged_fraction@10": 0.2280000000000001 + }, + "median": { + "recall@10": 0.0547008547008547, + "recall@50": 0.21473805411629243, + "recall@100": 0.36262050832602977, + "recall@200": 0.5159574468085106, + "recall@300": 0.6037549407114624, + "recall@500": 0.7320885801128962, + "recall@1000": 0.8038537549407114, + "ndcg@5": 0.5825820174791262, + "ndcg@10": 0.5175254500996578, + "ndcg@20": 0.48315552973442505, + "ndcg_full@5": 0.5758506309099084, + "ndcg_full@10": 0.4977996761292215, + "ndcg_full@20": 0.4486211534081267, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5, + "unjudged_fraction@10": 0.2 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 125, + "recall@10": 0.04145077720207254, + "recall@50": 0.20725388601036268, + "recall@100": 0.38860103626943004, + "recall@200": 0.5595854922279793, + "recall@300": 0.5906735751295337, + "recall@500": 0.6062176165803109, + "recall@1000": 0.616580310880829, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7234267649959281, + "ndcg_full@5": 0.7234267649959281, + "ndcg@10": 0.6318764815110606, + "ndcg_full@10": 0.6318764815110606, + "ndcg@20": 0.6965813423433977, + "ndcg_full@20": 0.6965813423433977, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 175, + "recall@10": 0.02512562814070352, + "recall@50": 0.09547738693467336, + "recall@100": 0.16331658291457288, + "recall@200": 0.24874371859296482, + "recall@300": 0.31155778894472363, + "recall@500": 0.4296482412060301, + "recall@1000": 0.5954773869346733, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5551553341043619, + "ndcg_full@5": 0.5551553341043619, + "ndcg@10": 0.48194776504401016, + "ndcg_full@10": 0.48194776504401016, + "ndcg@20": 0.4169269796669527, + "ndcg_full@20": 0.3963639840357972, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.12658227848101267, + "recall@50": 0.5569620253164557, + "recall@100": 0.7974683544303798, + "recall@200": 0.8860759493670886, + "recall@300": 0.8987341772151899, + "recall@500": 0.9240506329113924, + "recall@1000": 0.9240506329113924, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.48955279534819107, + "ndcg_full@5": 0.48955279534819107, + "ndcg@10": 0.396084535672086, + "ndcg_full@10": 0.396084535672086, + "ndcg@20": 0.37957719101757065, + "ndcg_full@20": 0.37957719101757065, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 1778, + "num_ranked": 250, + "recall@10": 0.014577259475218658, + "recall@50": 0.0641399416909621, + "recall@100": 0.14285714285714285, + "recall@200": 0.2886297376093295, + "recall@300": 0.4110787172011662, + "recall@500": 0.6064139941690962, + "recall@1000": 0.793002915451895, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7691415988388896, + "ndcg_full@5": 0.7691415988388896, + "ndcg@10": 0.7837532828592233, + "ndcg_full@10": 0.7837532828592233, + "ndcg@20": 0.5454298450614478, + "ndcg_full@20": 0.5454298450614478, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 1748, + "num_ranked": 173, + "recall@10": 0.09090909090909091, + "recall@50": 0.36363636363636365, + "recall@100": 0.6090909090909091, + "recall@200": 0.7363636363636363, + "recall@300": 0.7727272727272727, + "recall@500": 0.7818181818181819, + "recall@1000": 0.7909090909090909, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.46692474155501895, + "ndcg_full@10": 0.46692474155501895, + "ndcg@20": 0.4416056155116293, + "ndcg_full@20": 0.4416056155116293, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.021739130434782608, + "recall@50": 0.15942028985507245, + "recall@100": 0.26811594202898553, + "recall@200": 0.463768115942029, + "recall@300": 0.6014492753623188, + "recall@500": 0.6884057971014492, + "recall@1000": 0.7971014492753623, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6917831836856678, + "ndcg_full@5": 0.6917831836856678, + "ndcg@10": 0.7215901068666986, + "ndcg_full@10": 0.7215901068666986, + "ndcg@20": 0.6810538735740798, + "ndcg_full@20": 0.6810538735740798, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 1564, + "num_ranked": 139, + "recall@10": 0.16129032258064516, + "recall@50": 0.3548387096774194, + "recall@100": 0.4032258064516129, + "recall@200": 0.5, + "recall@300": 0.5806451612903226, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5806451612903226, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.5075394027672818, + "ndcg_full@10": 0.5075394027672818, + "ndcg@20": 0.5089015672044597, + "ndcg_full@20": 0.48894415844397393, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 129, + "recall@10": 0.2, + "recall@50": 0.65, + "recall@100": 0.775, + "recall@200": 0.875, + "recall@300": 0.9, + "recall@500": 0.925, + "recall@1000": 0.975, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.16082507495339965, + "ndcg_full@10": 0.16082507495339965, + "ndcg@20": 0.30949066511365847, + "ndcg_full@20": 0.3032292818278684, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05384615384615385, + "recall@50": 0.16153846153846155, + "recall@100": 0.27692307692307694, + "recall@200": 0.33076923076923076, + "recall@300": 0.38461538461538464, + "recall@500": 0.5, + "recall@1000": 0.7153846153846154, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.4690000933206748, + "ndcg_full@10": 0.4690000933206748, + "ndcg@20": 0.3197088327818207, + "ndcg_full@20": 0.3197088327818207, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 87, + "recall@10": 0.027522935779816515, + "recall@50": 0.05963302752293578, + "recall@100": 0.0963302752293578, + "recall@200": 0.12385321100917432, + "recall@300": 0.14678899082568808, + "recall@500": 0.23853211009174313, + "recall@1000": 0.3256880733944954, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.27727342735504823, + "ndcg_full@5": 0.27727342735504823, + "ndcg@10": 0.25581257755018944, + "ndcg_full@10": 0.25581257755018944, + "ndcg@20": 0.24491270022856315, + "ndcg_full@20": 0.22044687795732618, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029605263157894735, + "recall@50": 0.1513157894736842, + "recall@100": 0.2993421052631579, + "recall@200": 0.5526315789473685, + "recall@300": 0.6973684210526315, + "recall@500": 0.7796052631578947, + "recall@1000": 0.875, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.3156484524795145, + "ndcg_full@5": 0.3156484524795145, + "ndcg@10": 0.4880599494911611, + "ndcg_full@10": 0.4880599494911611, + "ndcg@20": 0.5629941364368399, + "ndcg_full@20": 0.5629941364368399, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.19696969696969696, + "recall@100": 0.3484848484848485, + "recall@200": 0.48484848484848486, + "recall@300": 0.6060606060606061, + "recall@500": 0.7575757575757576, + "recall@1000": 0.9090909090909091, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.03406297225598251, + "ndcg_full@20": 0.03406297225598251, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 222, + "recall@10": 0.2857142857142857, + "recall@50": 0.6190476190476191, + "recall@100": 0.7619047619047619, + "recall@200": 0.8095238095238095, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.0, + "ndcg_full@20": 0.0, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 170, + "recall@10": 0.08196721311475409, + "recall@50": 0.4016393442622951, + "recall@100": 0.7622950819672131, + "recall@200": 0.9344262295081968, + "recall@300": 0.9426229508196722, + "recall@500": 0.9590163934426229, + "recall@1000": 0.9672131147540983, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.5773584151532217, + "ndcg_full@5": 0.5773584151532217, + "ndcg@10": 0.5486118189693611, + "ndcg_full@10": 0.5486118189693611, + "ndcg@20": 0.5187793355578936, + "ndcg_full@20": 0.5187793355578936, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.3, + "graded_P@10": 0.55 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1219, + "num_ranked": 175, + "recall@10": 0.05263157894736842, + "recall@50": 0.28654970760233917, + "recall@100": 0.5555555555555556, + "recall@200": 0.7017543859649122, + "recall@300": 0.7076023391812866, + "recall@500": 0.7134502923976608, + "recall@1000": 0.7134502923976608, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.9278763895387894, + "ndcg_full@20": 0.9278763895387894, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 243, + "recall@10": 0.16666666666666666, + "recall@50": 0.6458333333333334, + "recall@100": 0.8958333333333334, + "recall@200": 0.9375, + "recall@300": 0.9375, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8546876825242049, + "ndcg_full@10": 0.8546876825242049, + "ndcg@20": 0.7790143114633546, + "ndcg_full@20": 0.7790143114633546, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 144, + "recall@10": 0.0, + "recall@50": 0.03333333333333333, + "recall@100": 0.1, + "recall@200": 0.23333333333333334, + "recall@300": 0.5, + "recall@500": 0.6666666666666666, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.14300115613108022, + "ndcg_full@20": 0.07389464807379724, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 690, + "num_ranked": 116, + "recall@10": 0.12244897959183673, + "recall@50": 0.6530612244897959, + "recall@100": 0.7142857142857143, + "recall@200": 0.7346938775510204, + "recall@300": 0.7346938775510204, + "recall@500": 0.7346938775510204, + "recall@1000": 0.7346938775510204, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8981000251498876, + "ndcg_full@10": 0.8981000251498876, + "ndcg@20": 0.6497884172970241, + "ndcg_full@20": 0.6497884172970241, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 780, + "num_ranked": 135, + "recall@10": 0.21428571428571427, + "recall@50": 0.35714285714285715, + "recall@100": 0.35714285714285715, + "recall@200": 0.5, + "recall@300": 0.5714285714285714, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.12247456102170373, + "ndcg_full@10": 0.09806461755386489, + "ndcg@20": 0.30545084416036483, + "ndcg_full@20": 0.21885577565735992, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 1411, + "num_ranked": 218, + "recall@10": 0.042735042735042736, + "recall@50": 0.20512820512820512, + "recall@100": 0.4017094017094017, + "recall@200": 0.6495726495726496, + "recall@300": 0.8162393162393162, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7204111965093014, + "ndcg_full@5": 0.7204111965093014, + "ndcg@10": 0.7640319661677394, + "ndcg_full@10": 0.7640319661677394, + "ndcg@20": 0.7692651781916927, + "ndcg_full@20": 0.7692651781916927, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.7, + "graded_P@10": 0.75 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.057803468208092484, + "recall@50": 0.23699421965317918, + "recall@100": 0.44508670520231214, + "recall@200": 0.7630057803468208, + "recall@300": 0.930635838150289, + "recall@500": 0.9826589595375722, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.8588685791930554, + "ndcg_full@20": 0.8588685791930554, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 1527, + "num_ranked": 152, + "recall@10": 0.08, + "recall@50": 0.36, + "recall@100": 0.464, + "recall@200": 0.552, + "recall@300": 0.592, + "recall@500": 0.616, + "recall@1000": 0.632, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.6085019770586442, + "ndcg_full@10": 0.6085019770586442, + "ndcg@20": 0.6460541665655801, + "ndcg_full@20": 0.6460541665655801, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.05319148936170213, + "recall@50": 0.26063829787234044, + "recall@100": 0.5, + "recall@200": 0.776595744680851, + "recall@300": 0.8138297872340425, + "recall@500": 0.8351063829787234, + "recall@1000": 0.8404255319148937, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9216017310213247, + "ndcg_full@10": 0.9216017310213247, + "ndcg@20": 0.7715798280837078, + "ndcg_full@20": 0.7715798280837078, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1110, + "num_ranked": 112, + "recall@10": 0.1935483870967742, + "recall@50": 0.3225806451612903, + "recall@100": 0.41935483870967744, + "recall@200": 0.45161290322580644, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5296347172140421, + "ndcg_full@5": 0.5296347172140421, + "ndcg@10": 0.48913115794608397, + "ndcg_full@10": 0.4735717030454861, + "ndcg@20": 0.6222230641600135, + "ndcg_full@20": 0.4556366913046241, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.027624309392265192, + "recall@50": 0.08839779005524862, + "recall@100": 0.1712707182320442, + "recall@200": 0.27624309392265195, + "recall@300": 0.35911602209944754, + "recall@500": 0.4861878453038674, + "recall@1000": 0.6464088397790055, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.5404506181765133, + "ndcg_full@10": 0.5404506181765133, + "ndcg@20": 0.38984277978231713, + "ndcg_full@20": 0.38353935512498166, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 155, + "recall@10": 0.17857142857142858, + "recall@50": 0.6785714285714286, + "recall@100": 0.7857142857142857, + "recall@200": 0.9107142857142857, + "recall@300": 0.9464285714285714, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.22342676499592815, + "ndcg_full@5": 0.22342676499592815, + "ndcg@10": 0.4568579727589217, + "ndcg_full@10": 0.4568579727589217, + "ndcg@20": 0.41497385624448535, + "ndcg_full@20": 0.41497385624448535, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 91, + "recall@10": 0.20689655172413793, + "recall@50": 0.6896551724137931, + "recall@100": 0.8275862068965517, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.8620689655172413, + "recall@1000": 0.8620689655172413, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.5482339681292637, + "ndcg_full@10": 0.5482339681292637, + "ndcg@20": 0.4596124334743223, + "ndcg_full@20": 0.4141321775093432, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.4, + "graded_P@10": 0.45 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 189, + "recall@10": 0.07352941176470588, + "recall@50": 0.19117647058823528, + "recall@100": 0.2647058823529412, + "recall@200": 0.38235294117647056, + "recall@300": 0.4117647058823529, + "recall@500": 0.47058823529411764, + "recall@1000": 0.5588235294117647, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.2833046215732896, + "ndcg_full@5": 0.2833046215732896, + "ndcg@10": 0.2474661025752992, + "ndcg_full@10": 0.2474661025752992, + "ndcg@20": 0.2905556408668519, + "ndcg_full@20": 0.2762253272504836, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.030303030303030304, + "recall@50": 0.15151515151515152, + "recall@100": 0.24242424242424243, + "recall@200": 0.3939393939393939, + "recall@300": 0.48484848484848486, + "recall@500": 0.6363636363636364, + "recall@1000": 0.7575757575757576, + "unjudged_fraction@10": 0.7, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.10327064981393906, + "ndcg_full@10": 0.08514311764162098, + "ndcg@20": 0.1603583459917208, + "ndcg_full@20": 0.10525070202752546, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.02617801047120419, + "recall@50": 0.1099476439790576, + "recall@100": 0.20680628272251309, + "recall@200": 0.3900523560209424, + "recall@300": 0.5549738219895288, + "recall@500": 0.7617801047120419, + "recall@1000": 0.8115183246073299, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5952372559702129, + "ndcg_full@5": 0.5952372559702129, + "ndcg@10": 0.5618024625470325, + "ndcg_full@10": 0.5618024625470325, + "ndcg@20": 0.5752255645531686, + "ndcg_full@20": 0.5752255645531686, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 1522, + "num_ranked": 120, + "recall@10": 0.1282051282051282, + "recall@50": 0.5384615384615384, + "recall@100": 0.6410256410256411, + "recall@200": 0.8205128205128205, + "recall@300": 0.8205128205128205, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.2940945979596688, + "ndcg_full@10": 0.2940945979596688, + "ndcg@20": 0.24347412628678433, + "ndcg_full@20": 0.23953735769531193, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.0273972602739726, + "recall@50": 0.1095890410958904, + "recall@100": 0.1780821917808219, + "recall@200": 0.2785388127853881, + "recall@300": 0.3287671232876712, + "recall@500": 0.410958904109589, + "recall@1000": 0.5251141552511416, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.06560253875617089, + "ndcg_full@5": 0.06560253875617089, + "ndcg@10": 0.1894621829486417, + "ndcg_full@10": 0.1894621829486417, + "ndcg@20": 0.2995890324130263, + "ndcg_full@20": 0.2995890324130263, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.1, + "graded_P@10": 0.25 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 137, + "recall@10": 0.05555555555555555, + "recall@50": 0.2222222222222222, + "recall@100": 0.2638888888888889, + "recall@200": 0.2638888888888889, + "recall@300": 0.3055555555555556, + "recall@500": 0.3333333333333333, + "recall@1000": 0.5, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.7058075148678526, + "ndcg_full@10": 0.7058075148678526, + "ndcg@20": 0.5300708814546105, + "ndcg_full@20": 0.5300708814546105, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 687, + "num_ranked": 37, + "recall@10": 0.075, + "recall@50": 0.1375, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.5919819915518812, + "ndcg_full@5": 0.5531464700081437, + "ndcg@10": 0.5770244773978692, + "ndcg_full@10": 0.39366982114002336, + "ndcg@20": 0.6160965581493, + "ndcg_full@20": 0.3148069819208615, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.2, + "graded_P@10": 0.25 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 1843, + "num_ranked": 203, + "recall@10": 0.03461538461538462, + "recall@50": 0.1346153846153846, + "recall@100": 0.18461538461538463, + "recall@200": 0.2692307692307692, + "recall@300": 0.35, + "recall@500": 0.4653846153846154, + "recall@1000": 0.5961538461538461, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.29363847436851637, + "ndcg_full@5": 0.29363847436851637, + "ndcg@10": 0.32707788734317217, + "ndcg_full@10": 0.32707788734317217, + "ndcg@20": 0.4783742016653112, + "ndcg_full@20": 0.4783742016653112, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.3, + "graded_P@10": 0.5 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.025974025974025976, + "recall@50": 0.11688311688311688, + "recall@100": 0.2077922077922078, + "recall@200": 0.4025974025974026, + "recall@300": 0.6623376623376623, + "recall@500": 0.935064935064935, + "recall@1000": 0.948051948051948, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.5884674004369673, + "ndcg_full@10": 0.5884674004369673, + "ndcg@20": 0.4152876557835351, + "ndcg_full@20": 0.4152876557835351, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.10256410256410256, + "recall@50": 0.5, + "recall@100": 0.7692307692307693, + "recall@200": 0.8846153846153846, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9230769230769231, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.5743428466665951, + "ndcg_full@5": 0.5743428466665951, + "ndcg@10": 0.47526754193190257, + "ndcg_full@10": 0.47526754193190257, + "ndcg@20": 0.3791624973409792, + "ndcg_full@20": 0.3791624973409792, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 1474, + "num_ranked": 135, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6226415094339622, + "recall@200": 0.89937106918239, + "recall@300": 0.89937106918239, + "recall@500": 0.9056603773584906, + "recall@1000": 0.9056603773584906, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8581437227514841, + "ndcg_full@10": 0.8581437227514841, + "ndcg@20": 0.9084505904897269, + "ndcg_full@20": 0.9084505904897269, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03409090909090909, + "recall@50": 0.16287878787878787, + "recall@100": 0.3333333333333333, + "recall@200": 0.5757575757575758, + "recall@300": 0.7689393939393939, + "recall@500": 0.7840909090909091, + "recall@1000": 0.8106060606060606, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7136540102475308, + "ndcg_full@10": 0.7136540102475308, + "ndcg@20": 0.6653265262260111, + "ndcg_full@20": 0.6653265262260111, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 1898, + "num_ranked": 154, + "recall@10": 0.06134969325153374, + "recall@50": 0.25766871165644173, + "recall@100": 0.36809815950920244, + "recall@200": 0.44785276073619634, + "recall@300": 0.5214723926380368, + "recall@500": 0.5644171779141104, + "recall@1000": 0.656441717791411, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.5993294818993699, + "ndcg_full@10": 0.5993294818993699, + "ndcg@20": 0.5708272422850166, + "ndcg_full@20": 0.5708272422850166, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.14285714285714285, + "recall@50": 0.7, + "recall@100": 0.9571428571428572, + "recall@200": 0.9714285714285714, + "recall@300": 0.9714285714285714, + "recall@500": 0.9714285714285714, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7573857224418385, + "ndcg_full@5": 0.7573857224418385, + "ndcg@10": 0.8425599346661845, + "ndcg_full@10": 0.8425599346661845, + "ndcg@20": 0.8161850546773337, + "ndcg_full@20": 0.8161850546773337, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1641, + "num_ranked": 168, + "recall@10": 0.0446927374301676, + "recall@50": 0.22346368715083798, + "recall@100": 0.3743016759776536, + "recall@200": 0.46368715083798884, + "recall@300": 0.49162011173184356, + "recall@500": 0.547486033519553, + "recall@1000": 0.6536312849162011, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5878056198050307, + "ndcg_full@5": 0.5878056198050307, + "ndcg@10": 0.6993869397899498, + "ndcg_full@10": 0.6993869397899498, + "ndcg@20": 0.5829916412931193, + "ndcg_full@20": 0.5829916412931193, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.011904761904761904, + "recall@50": 0.023809523809523808, + "recall@100": 0.03571428571428571, + "recall@200": 0.11904761904761904, + "recall@300": 0.16666666666666666, + "recall@500": 0.34523809523809523, + "recall@1000": 0.6309523809523809, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.17240122738952723, + "ndcg_full@20": 0.0776688416812707, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 1781, + "num_ranked": 250, + "recall@10": 0.03759398496240601, + "recall@50": 0.17669172932330826, + "recall@100": 0.3533834586466165, + "recall@200": 0.6390977443609023, + "recall@300": 0.8609022556390977, + "recall@500": 0.943609022556391, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.7227265726449519, + "ndcg_full@5": 0.7227265726449519, + "ndcg@10": 0.5275114974320337, + "ndcg_full@10": 0.5275114974320337, + "ndcg@20": 0.4879368578035389, + "ndcg_full@20": 0.4879368578035389, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 1795, + "num_ranked": 150, + "recall@10": 0.047619047619047616, + "recall@50": 0.19047619047619047, + "recall@100": 0.2857142857142857, + "recall@200": 0.3333333333333333, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.06553231388149368, + "ndcg_full@10": 0.039199134489337664, + "ndcg@20": 0.1499845904721067, + "ndcg_full@20": 0.06872388179302048, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.009554140127388535, + "recall@50": 0.044585987261146494, + "recall@100": 0.10828025477707007, + "recall@200": 0.25796178343949044, + "recall@300": 0.4140127388535032, + "recall@500": 0.643312101910828, + "recall@1000": 0.8662420382165605, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.2870204396558808, + "ndcg_full@5": 0.2870204396558808, + "ndcg@10": 0.2872264604811076, + "ndcg_full@10": 0.2872264604811076, + "ndcg@20": 0.401388069225586, + "ndcg_full@20": 0.401388069225586, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.10416666666666667, + "recall@50": 0.4895833333333333, + "recall@100": 0.8541666666666666, + "recall@200": 0.9375, + "recall@300": 0.9479166666666666, + "recall@500": 0.9791666666666666, + "recall@1000": 0.9895833333333334, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8293271620269695, + "ndcg_full@10": 0.8293271620269695, + "ndcg@20": 0.7276938099727585, + "ndcg_full@20": 0.7276938099727585, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 239, + "recall@10": 0.030395136778115502, + "recall@50": 0.13677811550151975, + "recall@100": 0.2796352583586626, + "recall@200": 0.5319148936170213, + "recall@300": 0.7051671732522796, + "recall@500": 0.729483282674772, + "recall@1000": 0.78419452887538, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6069931323672637, + "ndcg_full@5": 0.6069931323672637, + "ndcg@10": 0.6133147750202962, + "ndcg_full@10": 0.6133147750202962, + "ndcg@20": 0.6487390619485642, + "ndcg_full@20": 0.6487390619485642, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.4, + "graded_P@10": 0.65 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.1111111111111111, + "recall@50": 0.3888888888888889, + "recall@100": 0.6666666666666666, + "recall@200": 0.8888888888888888, + "recall@300": 0.8888888888888888, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.14606834984270645, + "ndcg_full@5": 0.14606834984270645, + "ndcg@10": 0.09478836436955078, + "ndcg_full@10": 0.09478836436955078, + "ndcg@20": 0.10175336507264422, + "ndcg_full@20": 0.09201271560929407, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.013245033112582781, + "recall@50": 0.12582781456953643, + "recall@100": 0.2052980132450331, + "recall@200": 0.3708609271523179, + "recall@300": 0.5562913907284768, + "recall@500": 0.9072847682119205, + "recall@1000": 0.9933774834437086, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.20904936749052638, + "ndcg_full@10": 0.20904936749052638, + "ndcg@20": 0.34230404329137265, + "ndcg_full@20": 0.34230404329137265, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + } + } + } + } + }, + { + "name": "iimedical_medcpt", + "results_dir": "iimedical_medcpt/results_trec22", + "evaluation_input_sha256": "7eeedc0a77fa0f45790f8a677480ca10d5158b41fd05ac3cdc360ab498a0f05c", + "policies": { + "exclude": { + "unjudged_policy": "exclude", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08286938335434574, + "recall@50": 0.2968931812774538, + "recall@100": 0.4319739276995187, + "recall@200": 0.5786610843927987, + "recall@300": 0.6564142106557841, + "recall@500": 0.7294183232591044, + "recall@1000": 0.7756041353615395, + "ndcg@5": 0.6877073184363922, + "ndcg@10": 0.6907967445812107, + "ndcg@20": 0.7020203168774146, + "ndcg_full@5": 0.6807864818885743, + "ndcg_full@10": 0.6673539967750328, + "ndcg_full@20": 0.654516194394004, + "P@10(rel>=1)": 0.72, + "P@10(eligible)": 0.5980000000000001, + "graded_P@10": 0.6590000000000003, + "unjudged_fraction@10": 0.18799999999999997 + }, + "median": { + "recall@10": 0.05717272697572609, + "recall@50": 0.22552809884416103, + "recall@100": 0.3695804195804196, + "recall@200": 0.569047619047619, + "recall@300": 0.6758388856458464, + "recall@500": 0.7607747046186896, + "recall@1000": 0.8284678533067362, + "ndcg@5": 0.7765732350040719, + "ndcg@10": 0.7643278528789683, + "ndcg@20": 0.7532400072342379, + "ndcg_full@5": 0.7669794787229554, + "ndcg_full@10": 0.7373693886815849, + "ndcg_full@20": 0.7239430165807643, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 146, + "recall@10": 0.05181347150259067, + "recall@50": 0.22797927461139897, + "recall@100": 0.39378238341968913, + "recall@200": 0.5854922279792746, + "recall@300": 0.5958549222797928, + "recall@500": 0.616580310880829, + "recall@1000": 0.6217616580310881, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8976916326759747, + "ndcg_full@10": 0.8976916326759747, + "ndcg@20": 0.8881075988170513, + "ndcg_full@20": 0.8881075988170513, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 95, + "recall@10": 0.02512562814070352, + "recall@50": 0.10552763819095477, + "recall@100": 0.20351758793969849, + "recall@200": 0.35175879396984927, + "recall@300": 0.4949748743718593, + "recall@500": 0.6432160804020101, + "recall@1000": 0.8391959798994975, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.8133655231015728, + "ndcg_full@10": 0.8133655231015728, + "ndcg@20": 0.8225251674251433, + "ndcg_full@20": 0.8092256382546359, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.6, + "graded_P@10": 0.8 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.12658227848101267, + "recall@50": 0.5443037974683544, + "recall@100": 0.7974683544303798, + "recall@200": 0.8987341772151899, + "recall@300": 0.9240506329113924, + "recall@500": 0.9367088607594937, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8948256838011641, + "ndcg_full@10": 0.8948256838011641, + "ndcg@20": 0.8271270495711187, + "ndcg_full@20": 0.8271270495711187, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029154518950437316, + "recall@50": 0.12244897959183673, + "recall@100": 0.22740524781341107, + "recall@200": 0.39067055393586003, + "recall@300": 0.5335276967930029, + "recall@500": 0.7084548104956269, + "recall@1000": 0.8279883381924198, + "unjudged_fraction@10": 0.6, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9286024279819313, + "ndcg_full@10": 0.9286024279819313, + "ndcg@20": 0.9539223382602212, + "ndcg_full@20": 0.9539223382602212, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 80, + "recall@10": 0.09090909090909091, + "recall@50": 0.23636363636363636, + "recall@100": 0.35454545454545455, + "recall@200": 0.45454545454545453, + "recall@300": 0.5454545454545454, + "recall@500": 0.6363636363636364, + "recall@1000": 0.7818181818181819, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5538466623591201, + "ndcg_full@5": 0.5538466623591201, + "ndcg@10": 0.5022037311331263, + "ndcg_full@10": 0.5022037311331263, + "ndcg@20": 0.5086978628335062, + "ndcg_full@20": 0.4921134828911809, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 217, + "recall@10": 0.050724637681159424, + "recall@50": 0.21739130434782608, + "recall@100": 0.391304347826087, + "recall@200": 0.6014492753623188, + "recall@300": 0.717391304347826, + "recall@500": 0.8695652173913043, + "recall@1000": 0.927536231884058, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.9427172181013092, + "ndcg_full@10": 0.9427172181013092, + "ndcg@20": 0.8608994924120608, + "ndcg_full@20": 0.8608994924120608, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 118, + "recall@10": 0.11290322580645161, + "recall@50": 0.3225806451612903, + "recall@100": 0.4032258064516129, + "recall@200": 0.46774193548387094, + "recall@300": 0.5483870967741935, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5967741935483871, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7129795603441191, + "ndcg_full@5": 0.7129795603441191, + "ndcg@10": 0.5957269594092457, + "ndcg_full@10": 0.5957269594092457, + "ndcg@20": 0.6833372996878604, + "ndcg_full@20": 0.670095987910243, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 48, + "recall@10": 0.2, + "recall@50": 0.575, + "recall@100": 0.675, + "recall@200": 0.775, + "recall@300": 0.875, + "recall@500": 0.925, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7366165631508063, + "ndcg_full@10": 0.7366165631508063, + "ndcg@20": 0.7300426227525512, + "ndcg_full@20": 0.7021355757298436, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06153846153846154, + "recall@50": 0.2230769230769231, + "recall@100": 0.3, + "recall@200": 0.4230769230769231, + "recall@300": 0.5076923076923077, + "recall@500": 0.6307692307692307, + "recall@1000": 0.7615384615384615, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.611755947233009, + "ndcg_full@10": 0.611755947233009, + "ndcg@20": 0.5904455026455555, + "ndcg_full@20": 0.5904455026455555, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 112, + "recall@10": 0.009174311926605505, + "recall@50": 0.027522935779816515, + "recall@100": 0.045871559633027525, + "recall@200": 0.08256880733944955, + "recall@300": 0.0963302752293578, + "recall@500": 0.13302752293577982, + "recall@1000": 0.2018348623853211, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.6992148198508501, + "ndcg_full@5": 0.6992148198508501, + "ndcg@10": 0.7950560435073812, + "ndcg_full@10": 0.6302054610737258, + "ndcg@20": 0.90578152452456, + "ndcg_full@20": 0.46335608010352536, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029605263157894735, + "recall@50": 0.15460526315789475, + "recall@100": 0.29605263157894735, + "recall@200": 0.48355263157894735, + "recall@300": 0.6282894736842105, + "recall@500": 0.7664473684210527, + "recall@1000": 0.8289473684210527, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.8611375561264454, + "ndcg_full@10": 0.8611375561264454, + "ndcg@20": 0.9103827127966565, + "ndcg_full@20": 0.9103827127966565, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.2727272727272727, + "recall@100": 0.4090909090909091, + "recall@200": 0.6515151515151515, + "recall@300": 0.7878787878787878, + "recall@500": 0.9090909090909091, + "recall@1000": 0.9242424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7445940855636755, + "ndcg_full@5": 0.7445940855636755, + "ndcg@10": 0.5409530643539527, + "ndcg_full@10": 0.5409530643539527, + "ndcg@20": 0.4867555252133776, + "ndcg_full@20": 0.4867555252133776, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.2857142857142857, + "recall@50": 0.5714285714285714, + "recall@100": 0.5714285714285714, + "recall@200": 0.7619047619047619, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.055856691248982036, + "ndcg_full@5": 0.055856691248982036, + "ndcg@10": 0.0460469542673609, + "ndcg_full@10": 0.0460469542673609, + "ndcg@20": 0.28858200418820135, + "ndcg_full@20": 0.23799499884321568, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.08196721311475409, + "recall@50": 0.4016393442622951, + "recall@100": 0.7459016393442623, + "recall@200": 0.9262295081967213, + "recall@300": 0.9426229508196722, + "recall@500": 0.9508196721311475, + "recall@1000": 0.9672131147540983, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.9201370649018896, + "ndcg_full@10": 0.9201370649018896, + "ndcg@20": 0.7870012452406782, + "ndcg_full@20": 0.7870012452406782, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1202, + "num_ranked": 130, + "recall@10": 0.05847953216374269, + "recall@50": 0.26900584795321636, + "recall@100": 0.543859649122807, + "recall@200": 0.7076023391812866, + "recall@300": 0.7134502923976608, + "recall@500": 0.7134502923976608, + "recall@1000": 0.7134502923976608, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.9574284411791893, + "ndcg_full@10": 0.9574284411791893, + "ndcg@20": 0.9551506196145727, + "ndcg_full@20": 0.9551506196145727, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.20833333333333334, + "recall@50": 0.6875, + "recall@100": 0.9583333333333334, + "recall@200": 0.9583333333333334, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.5429811722745004, + "ndcg_full@10": 0.5429811722745004, + "ndcg@20": 0.5064855901765588, + "ndcg_full@20": 0.5064855901765588, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 153, + "recall@10": 0.0, + "recall@50": 0.1, + "recall@100": 0.3333333333333333, + "recall@200": 0.5666666666666667, + "recall@300": 0.7333333333333333, + "recall@500": 0.7666666666666667, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.3621576404427382, + "ndcg_full@10": 0.28997736516021183, + "ndcg@20": 0.4247388035182413, + "ndcg_full@20": 0.21948021441516624, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1826, + "num_ranked": 83, + "recall@10": 0.14285714285714285, + "recall@50": 0.6122448979591837, + "recall@100": 0.7346938775510204, + "recall@200": 0.7346938775510204, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.6968404678259545, + "ndcg_full@10": 0.6968404678259545, + "ndcg@20": 0.736735593294034, + "ndcg_full@20": 0.736735593294034, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 1417, + "num_ranked": 88, + "recall@10": 0.14285714285714285, + "recall@50": 0.35714285714285715, + "recall@100": 0.42857142857142855, + "recall@200": 0.5714285714285714, + "recall@300": 0.6428571428571429, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.2681841121184532, + "ndcg_full@10": 0.21473334682341438, + "ndcg@20": 0.4097148288839119, + "ndcg_full@20": 0.2935610046198929, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 148, + "recall@10": 0.042735042735042736, + "recall@50": 0.2094017094017094, + "recall@100": 0.38461538461538464, + "recall@200": 0.6538461538461539, + "recall@300": 0.8076923076923077, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.925524411668024, + "ndcg_full@5": 0.925524411668024, + "ndcg@10": 0.8931590350286058, + "ndcg_full@10": 0.8931590350286058, + "ndcg@20": 0.9244448185535403, + "ndcg_full@20": 0.9244448185535403, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03468208092485549, + "recall@50": 0.24277456647398843, + "recall@100": 0.4682080924855491, + "recall@200": 0.8265895953757225, + "recall@300": 0.9595375722543352, + "recall@500": 0.976878612716763, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 1.0, + "ndcg_full@10": 1.0, + "ndcg@20": 0.99191542721292, + "ndcg_full@20": 0.99191542721292, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 1949, + "num_ranked": 174, + "recall@10": 0.08, + "recall@50": 0.352, + "recall@100": 0.456, + "recall@200": 0.552, + "recall@300": 0.576, + "recall@500": 0.6, + "recall@1000": 0.616, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.77990823370192, + "ndcg_full@10": 0.77990823370192, + "ndcg@20": 0.8201142661004288, + "ndcg_full@20": 0.8201142661004288, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 151, + "recall@10": 0.05319148936170213, + "recall@50": 0.25, + "recall@100": 0.4787234042553192, + "recall@200": 0.7446808510638298, + "recall@300": 0.8138297872340425, + "recall@500": 0.8404255319148937, + "recall@1000": 0.8563829787234043, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.8551058243846009, + "ndcg_full@20": 0.8551058243846009, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1196, + "num_ranked": 61, + "recall@10": 0.1935483870967742, + "recall@50": 0.3225806451612903, + "recall@100": 0.3870967741935484, + "recall@200": 0.41935483870967744, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.8835455217521984, + "ndcg_full@10": 0.7381222142123633, + "ndcg@20": 0.9570340978529525, + "ndcg_full@20": 0.58611570609858, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.6, + "graded_P@10": 0.65 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 138, + "recall@10": 0.022099447513812154, + "recall@50": 0.08287292817679558, + "recall@100": 0.1270718232044199, + "recall@200": 0.2596685082872928, + "recall@300": 0.3756906077348066, + "recall@500": 0.5911602209944752, + "recall@1000": 0.7292817679558011, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.4, + "ndcg_full@5": 0.4, + "ndcg@10": 0.5481734541178296, + "ndcg_full@10": 0.5125764372470271, + "ndcg@20": 0.5820834248263501, + "ndcg_full@20": 0.4544762606878796, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 52, + "recall@10": 0.17857142857142858, + "recall@50": 0.7142857142857143, + "recall@100": 0.8392857142857143, + "recall@200": 0.9285714285714286, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.7803546731901579, + "ndcg_full@10": 0.7803546731901579, + "ndcg@20": 0.7525562011097645, + "ndcg_full@20": 0.7525562011097645, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 134, + "recall@10": 0.27586206896551724, + "recall@50": 0.7931034482758621, + "recall@100": 0.8275862068965517, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.896551724137931, + "recall@1000": 0.896551724137931, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.40374278939643915, + "ndcg_full@10": 0.40374278939643915, + "ndcg@20": 0.5250103170400139, + "ndcg_full@20": 0.5250103170400139, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.17647058823529413, + "recall@100": 0.23529411764705882, + "recall@200": 0.3382352941176471, + "recall@300": 0.4117647058823529, + "recall@500": 0.45588235294117646, + "recall@1000": 0.5441176470588235, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.5147714448836774, + "ndcg_full@5": 0.5147714448836774, + "ndcg@10": 0.5185628966293482, + "ndcg_full@10": 0.5185628966293482, + "ndcg@20": 0.547348181552642, + "ndcg_full@20": 0.5295037381868934, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.18181818181818182, + "recall@100": 0.2727272727272727, + "recall@200": 0.48484848484848486, + "recall@300": 0.48484848484848486, + "recall@500": 0.6666666666666666, + "recall@1000": 0.7878787878787878, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.07700824583897013, + "ndcg_full@20": 0.056572647891808124, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020942408376963352, + "recall@50": 0.0968586387434555, + "recall@100": 0.18324607329842932, + "recall@200": 0.34554973821989526, + "recall@300": 0.5052356020942408, + "recall@500": 0.7486910994764397, + "recall@1000": 0.8115183246073299, + "unjudged_fraction@10": 0.1, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9567083294488863, + "ndcg_full@10": 0.9567083294488863, + "ndcg@20": 0.958853947014088, + "ndcg_full@20": 0.958853947014088, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.1794871794871795, + "recall@50": 0.6153846153846154, + "recall@100": 0.717948717948718, + "recall@200": 0.7948717948717948, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.6396695532292432, + "ndcg_full@10": 0.6396695532292432, + "ndcg@20": 0.6684703006109134, + "ndcg_full@20": 0.657661707008333, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 202, + "recall@10": 0.0182648401826484, + "recall@50": 0.1004566210045662, + "recall@100": 0.1506849315068493, + "recall@200": 0.2602739726027397, + "recall@300": 0.3105022831050228, + "recall@500": 0.3926940639269406, + "recall@1000": 0.4977168949771689, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5248251961106588, + "ndcg_full@5": 0.5248251961106588, + "ndcg@10": 0.5356136549904646, + "ndcg_full@10": 0.5356136549904646, + "ndcg@20": 0.5483956983643569, + "ndcg_full@20": 0.5483956983643569, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 179, + "recall@10": 0.05555555555555555, + "recall@50": 0.2222222222222222, + "recall@100": 0.2638888888888889, + "recall@200": 0.3194444444444444, + "recall@300": 0.375, + "recall@500": 0.4861111111111111, + "recall@1000": 0.5277777777777778, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.639945385422766, + "ndcg_full@5": 0.639945385422766, + "ndcg@10": 0.698181652834525, + "ndcg_full@10": 0.698181652834525, + "ndcg@20": 0.6285698352514464, + "ndcg_full@20": 0.6285698352514464, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 1468, + "num_ranked": 12, + "recall@10": 0.075, + "recall@50": 0.1375, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.994773598824535, + "ndcg_full@5": 0.7072548781653097, + "ndcg@10": 0.994773598824535, + "ndcg_full@10": 0.45896002224894766, + "ndcg@20": 0.994773598824535, + "ndcg_full@20": 0.3437419564791711, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 188, + "recall@10": 0.03461538461538462, + "recall@50": 0.10384615384615385, + "recall@100": 0.16538461538461538, + "recall@200": 0.23846153846153847, + "recall@300": 0.3269230769230769, + "recall@500": 0.4807692307692308, + "recall@1000": 0.5961538461538461, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7398014973729643, + "ndcg_full@5": 0.7398014973729643, + "ndcg@10": 0.783366986659749, + "ndcg_full@10": 0.783366986659749, + "ndcg@20": 0.7538557834421996, + "ndcg_full@20": 0.7538557834421996, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 220, + "recall@10": 0.012987012987012988, + "recall@50": 0.15584415584415584, + "recall@100": 0.3246753246753247, + "recall@200": 0.6363636363636364, + "recall@300": 0.7662337662337663, + "recall@500": 0.935064935064935, + "recall@1000": 0.948051948051948, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.11004588314904008, + "ndcg_full@10": 0.11004588314904008, + "ndcg@20": 0.1462361771597033, + "ndcg_full@20": 0.1462361771597033, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 85, + "recall@10": 0.1282051282051282, + "recall@50": 0.6025641025641025, + "recall@100": 0.8717948717948718, + "recall@200": 0.9230769230769231, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5922216874835864, + "ndcg_full@5": 0.5922216874835864, + "ndcg@10": 0.6569815082938864, + "ndcg_full@10": 0.6569815082938864, + "ndcg@20": 0.6350729723655039, + "ndcg_full@20": 0.6350729723655039, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 135, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6163522012578616, + "recall@200": 0.89937106918239, + "recall@300": 0.9056603773584906, + "recall@500": 0.9182389937106918, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9450514119280142, + "ndcg_full@20": 0.9450514119280142, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03787878787878788, + "recall@50": 0.16666666666666666, + "recall@100": 0.3371212121212121, + "recall@200": 0.6060606060606061, + "recall@300": 0.7537878787878788, + "recall@500": 0.7840909090909091, + "recall@1000": 0.7954545454545454, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7828012693780855, + "ndcg_full@5": 0.7828012693780855, + "ndcg@10": 0.8272424914779678, + "ndcg_full@10": 0.8272424914779678, + "ndcg@20": 0.854160655230161, + "ndcg_full@20": 0.854160655230161, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 110, + "recall@10": 0.06134969325153374, + "recall@50": 0.25766871165644173, + "recall@100": 0.3496932515337423, + "recall@200": 0.4233128834355828, + "recall@300": 0.4785276073619632, + "recall@500": 0.5644171779141104, + "recall@1000": 0.6625766871165644, + "unjudged_fraction@10": 0.2, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9999999999999998, + "ndcg_full@10": 0.9999999999999998, + "ndcg@20": 0.9999999999999999, + "ndcg_full@20": 0.9999999999999999, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 1.0, + "graded_P@10": 1.0 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.14285714285714285, + "recall@50": 0.6857142857142857, + "recall@100": 0.9571428571428572, + "recall@200": 0.9714285714285714, + "recall@300": 0.9714285714285714, + "recall@500": 0.9857142857142858, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.7487474720560168, + "ndcg_full@10": 0.7487474720560168, + "ndcg@20": 0.7682030485523271, + "ndcg_full@20": 0.7682030485523271, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1797, + "num_ranked": 107, + "recall@10": 0.055865921787709494, + "recall@50": 0.16201117318435754, + "recall@100": 0.21787709497206703, + "recall@200": 0.40782122905027934, + "recall@300": 0.4581005586592179, + "recall@500": 0.5921787709497207, + "recall@1000": 0.6703910614525139, + "unjudged_fraction@10": 0.4, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.8945466420562732, + "ndcg_full@10": 0.8945466420562732, + "ndcg@20": 0.9137657020033905, + "ndcg_full@20": 0.9137657020033905, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 109, + "recall@10": 0.023809523809523808, + "recall@50": 0.14285714285714285, + "recall@100": 0.2261904761904762, + "recall@200": 0.38095238095238093, + "recall@300": 0.4642857142857143, + "recall@500": 0.6309523809523809, + "recall@1000": 0.7976190476190477, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.14837017375130385, + "ndcg_full@5": 0.13863671367752411, + "ndcg@10": 0.3312978630509602, + "ndcg_full@10": 0.2590398871482643, + "ndcg@20": 0.36940727378855803, + "ndcg_full@20": 0.25101535550868964, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.3, + "graded_P@10": 0.35 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 1636, + "num_ranked": 195, + "recall@10": 0.03383458646616541, + "recall@50": 0.18045112781954886, + "recall@100": 0.3458646616541353, + "recall@200": 0.6390977443609023, + "recall@300": 0.8834586466165414, + "recall@500": 0.9473684210526315, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.7765732350040719, + "ndcg_full@5": 0.7765732350040719, + "ndcg@10": 0.7892795619726267, + "ndcg_full@10": 0.7892795619726267, + "ndcg@20": 0.7585769998872213, + "ndcg_full@20": 0.7585769998872213, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.5, + "graded_P@10": 0.75 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.0, + "recall@50": 0.19047619047619047, + "recall@100": 0.23809523809523808, + "recall@200": 0.2857142857142857, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.21836974929469322, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.2026097735578014, + "ndcg_full@10": 0.11004588314904008, + "ndcg@20": 0.2026097735578014, + "ndcg_full@20": 0.08429733897617347, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.028662420382165606, + "recall@50": 0.14012738853503184, + "recall@100": 0.25477707006369427, + "recall@200": 0.4840764331210191, + "recall@300": 0.6369426751592356, + "recall@500": 0.8439490445859873, + "recall@1000": 0.9267515923566879, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.6974789857676416, + "ndcg_full@10": 0.6974789857676416, + "ndcg@20": 0.7526242310262762, + "ndcg_full@20": 0.7526242310262762, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.5, + "graded_P@10": 0.6 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.10416666666666667, + "recall@50": 0.4895833333333333, + "recall@100": 0.84375, + "recall@200": 0.9375, + "recall@300": 0.96875, + "recall@500": 0.9791666666666666, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010481, + "ndcg_full@10": 0.9363792118010481, + "ndcg@20": 0.8520369643274979, + "ndcg_full@20": 0.8520369643274979, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 224, + "recall@10": 0.030395136778115502, + "recall@50": 0.14893617021276595, + "recall@100": 0.2796352583586626, + "recall@200": 0.5379939209726444, + "recall@300": 0.6960486322188449, + "recall@500": 0.7386018237082067, + "recall@1000": 0.7933130699088146, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.7573857224418388, + "ndcg_full@5": 0.7573857224418388, + "ndcg@10": 0.807844323697796, + "ndcg_full@10": 0.807844323697796, + "ndcg@20": 0.7618827157415389, + "ndcg_full@20": 0.7618827157415389, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.7, + "graded_P@10": 0.85 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 226, + "recall@10": 0.2777777777777778, + "recall@50": 0.4444444444444444, + "recall@100": 0.5555555555555556, + "recall@200": 0.7777777777777778, + "recall@300": 0.8888888888888888, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.6917831836856678, + "ndcg_full@5": 0.6917831836856678, + "ndcg@10": 0.657969318667747, + "ndcg_full@10": 0.657969318667747, + "ndcg@20": 0.5682641285996721, + "ndcg_full@20": 0.5138653214906023, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.019867549668874173, + "recall@50": 0.1390728476821192, + "recall@100": 0.271523178807947, + "recall@200": 0.46357615894039733, + "recall@300": 0.6556291390728477, + "recall@500": 0.9470198675496688, + "recall@1000": 0.9933774834437086, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7765732350040719, + "ndcg_full@5": 0.7765732350040719, + "ndcg@10": 0.7444146109124877, + "ndcg_full@10": 0.7444146109124877, + "ndcg@20": 0.7111504398674947, + "ndcg_full@20": 0.7111504398674947, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + } + } + }, + "include_as_zero": { + "unjudged_policy": "include_as_zero", + "recall_relevance_threshold": 1, + "num_queries_scored": 50, + "num_queries_ranked": 50, + "mean": { + "recall@10": 0.08286938335434574, + "recall@50": 0.2968931812774538, + "recall@100": 0.4319739276995187, + "recall@200": 0.5786610843927987, + "recall@300": 0.6564142106557841, + "recall@500": 0.7294183232591044, + "recall@1000": 0.7756041353615395, + "ndcg@5": 0.5749697398908067, + "ndcg@10": 0.5769345362149412, + "ndcg@20": 0.5848809225603574, + "ndcg_full@5": 0.5687150522616748, + "ndcg_full@10": 0.5574648678111797, + "ndcg_full@20": 0.5462812076679711, + "P@10(rel>=1)": 0.5960000000000001, + "P@10(eligible)": 0.498, + "graded_P@10": 0.5469999999999999, + "unjudged_fraction@10": 0.18799999999999997 + }, + "median": { + "recall@10": 0.05717272697572609, + "recall@50": 0.22552809884416103, + "recall@100": 0.3695804195804196, + "recall@200": 0.569047619047619, + "recall@300": 0.6758388856458464, + "recall@500": 0.7607747046186896, + "recall@1000": 0.8284678533067362, + "ndcg@5": 0.6510928396705395, + "ndcg@10": 0.6133584875403921, + "ndcg@20": 0.6340646593007688, + "ndcg_full@5": 0.642656771457248, + "ndcg_full@10": 0.6057251395572021, + "ndcg_full@20": 0.6063092180221246, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55, + "unjudged_fraction@10": 0.1 + }, + "per_query": { + "trec-20221": { + "num_relevant": 193, + "num_retrieved": 2000, + "num_ranked": 146, + "recall@10": 0.05181347150259067, + "recall@50": 0.22797927461139897, + "recall@100": 0.39378238341968913, + "recall@200": 0.5854922279792746, + "recall@300": 0.5958549222797928, + "recall@500": 0.616580310880829, + "recall@1000": 0.6217616580310881, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.915209948681596, + "ndcg_full@5": 0.915209948681596, + "ndcg@10": 0.8388638754390225, + "ndcg_full@10": 0.8388638754390225, + "ndcg@20": 0.8086015998694486, + "ndcg_full@20": 0.8086015998694486, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-20222": { + "num_relevant": 398, + "num_retrieved": 2000, + "num_ranked": 95, + "recall@10": 0.02512562814070352, + "recall@50": 0.10552763819095477, + "recall@100": 0.20351758793969849, + "recall@200": 0.35175879396984927, + "recall@300": 0.4949748743718593, + "recall@500": 0.6432160804020101, + "recall@1000": 0.8391959798994975, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6850517971164501, + "ndcg_full@5": 0.6850517971164501, + "ndcg@10": 0.6226031276604249, + "ndcg_full@10": 0.6226031276604249, + "ndcg@20": 0.6614942752402082, + "ndcg_full@20": 0.6507984780073759, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.4, + "graded_P@10": 0.55 + }, + "trec-20223": { + "num_relevant": 79, + "num_retrieved": 2000, + "num_ranked": 150, + "recall@10": 0.12658227848101267, + "recall@50": 0.5443037974683544, + "recall@100": 0.7974683544303798, + "recall@200": 0.8987341772151899, + "recall@300": 0.9240506329113924, + "recall@500": 0.9367088607594937, + "recall@1000": 0.9367088607594937, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.6352557412688555, + "ndcg_full@10": 0.6352557412688555, + "ndcg@20": 0.6385202616574472, + "ndcg_full@20": 0.6385202616574472, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-20224": { + "num_relevant": 343, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029154518950437316, + "recall@50": 0.12244897959183673, + "recall@100": 0.22740524781341107, + "recall@200": 0.39067055393586003, + "recall@300": 0.5335276967930029, + "recall@500": 0.7084548104956269, + "recall@1000": 0.8279883381924198, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5743428466665951, + "ndcg_full@5": 0.5743428466665951, + "ndcg@10": 0.4781625662116574, + "ndcg_full@10": 0.4781625662116574, + "ndcg@20": 0.40940824153098726, + "ndcg_full@20": 0.40940824153098726, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.4, + "graded_P@10": 0.4 + }, + "trec-20225": { + "num_relevant": 110, + "num_retrieved": 2000, + "num_ranked": 80, + "recall@10": 0.09090909090909091, + "recall@50": 0.23636363636363636, + "recall@100": 0.35454545454545455, + "recall@200": 0.45454545454545453, + "recall@300": 0.5454545454545454, + "recall@500": 0.6363636363636364, + "recall@1000": 0.7818181818181819, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5538466623591201, + "ndcg_full@5": 0.5538466623591201, + "ndcg@10": 0.5022037311331263, + "ndcg_full@10": 0.5022037311331263, + "ndcg@20": 0.5086978628335062, + "ndcg_full@20": 0.4921134828911809, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-20226": { + "num_relevant": 138, + "num_retrieved": 2000, + "num_ranked": 217, + "recall@10": 0.050724637681159424, + "recall@50": 0.21739130434782608, + "recall@100": 0.391304347826087, + "recall@200": 0.6014492753623188, + "recall@300": 0.717391304347826, + "recall@500": 0.8695652173913043, + "recall@1000": 0.927536231884058, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.911727547130993, + "ndcg_full@5": 0.911727547130993, + "ndcg@10": 0.9427172181013092, + "ndcg_full@10": 0.9427172181013092, + "ndcg@20": 0.8280345218444837, + "ndcg_full@20": 0.8280345218444837, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.9, + "graded_P@10": 0.95 + }, + "trec-20227": { + "num_relevant": 62, + "num_retrieved": 2000, + "num_ranked": 118, + "recall@10": 0.11290322580645161, + "recall@50": 0.3225806451612903, + "recall@100": 0.4032258064516129, + "recall@200": 0.46774193548387094, + "recall@300": 0.5483870967741935, + "recall@500": 0.5806451612903226, + "recall@1000": 0.5967741935483871, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7129795603441191, + "ndcg_full@5": 0.7129795603441191, + "ndcg@10": 0.5957269594092457, + "ndcg_full@10": 0.5957269594092457, + "ndcg@20": 0.6238212472551757, + "ndcg_full@20": 0.6117332028411198, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-20228": { + "num_relevant": 40, + "num_retrieved": 2000, + "num_ranked": 48, + "recall@10": 0.2, + "recall@50": 0.575, + "recall@100": 0.675, + "recall@200": 0.775, + "recall@300": 0.875, + "recall@500": 0.925, + "recall@1000": 0.95, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.6608397947263839, + "ndcg_full@5": 0.6608397947263839, + "ndcg@10": 0.7366165631508063, + "ndcg_full@10": 0.7366165631508063, + "ndcg@20": 0.7300426227525512, + "ndcg_full@20": 0.7021355757298436, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-20229": { + "num_relevant": 130, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06153846153846154, + "recall@50": 0.2230769230769231, + "recall@100": 0.3, + "recall@200": 0.4230769230769231, + "recall@300": 0.5076923076923077, + "recall@500": 0.6307692307692307, + "recall@1000": 0.7615384615384615, + "unjudged_fraction@10": 0.8, + "ndcg@5": 0.11759132069648948, + "ndcg_full@5": 0.11759132069648948, + "ndcg@10": 0.13160707423862844, + "ndcg_full@10": 0.13160707423862844, + "ndcg@20": 0.2206846132717024, + "ndcg_full@20": 0.2206846132717024, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202210": { + "num_relevant": 218, + "num_retrieved": 2000, + "num_ranked": 112, + "recall@10": 0.009174311926605505, + "recall@50": 0.027522935779816515, + "recall@100": 0.045871559633027525, + "recall@200": 0.08256880733944955, + "recall@300": 0.0963302752293578, + "recall@500": 0.13302752293577982, + "recall@1000": 0.2018348623853211, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.5157051110116302, + "ndcg_full@5": 0.5157051110116302, + "ndcg@10": 0.5041217648494242, + "ndcg_full@10": 0.3995948359221323, + "ndcg@20": 0.6434025080858131, + "ndcg_full@20": 0.32913506845031215, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202211": { + "num_relevant": 304, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029605263157894735, + "recall@50": 0.15460526315789475, + "recall@100": 0.29605263157894735, + "recall@200": 0.48355263157894735, + "recall@300": 0.6282894736842105, + "recall@500": 0.7664473684210527, + "recall@1000": 0.8289473684210527, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.4703652827859579, + "ndcg_full@5": 0.4703652827859579, + "ndcg@10": 0.5884605859122715, + "ndcg_full@10": 0.5884605859122715, + "ndcg@20": 0.6970991491769998, + "ndcg_full@20": 0.6970991491769998, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202212": { + "num_relevant": 66, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.2727272727272727, + "recall@100": 0.4090909090909091, + "recall@200": 0.6515151515151515, + "recall@300": 0.7878787878787878, + "recall@500": 0.9090909090909091, + "recall@1000": 0.9242424242424242, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7445940855636755, + "ndcg_full@5": 0.7445940855636755, + "ndcg@10": 0.5409530643539527, + "ndcg_full@10": 0.5409530643539527, + "ndcg@20": 0.4541309736564895, + "ndcg_full@20": 0.4541309736564895, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.3, + "graded_P@10": 0.4 + }, + "trec-202213": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 214, + "recall@10": 0.2857142857142857, + "recall@50": 0.5714285714285714, + "recall@100": 0.5714285714285714, + "recall@200": 0.7619047619047619, + "recall@300": 0.8095238095238095, + "recall@500": 0.8095238095238095, + "recall@1000": 0.8571428571428571, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.027928345624491018, + "ndcg_full@5": 0.027928345624491018, + "ndcg@10": 0.04006536186427283, + "ndcg_full@10": 0.04006536186427283, + "ndcg@20": 0.12436951998312086, + "ndcg_full@20": 0.10256815509954902, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202214": { + "num_relevant": 122, + "num_retrieved": 2000, + "num_ranked": 142, + "recall@10": 0.08196721311475409, + "recall@50": 0.4016393442622951, + "recall@100": 0.7459016393442623, + "recall@200": 0.9262295081967213, + "recall@300": 0.9426229508196722, + "recall@500": 0.9508196721311475, + "recall@1000": 0.9672131147540983, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.9269658250786468, + "ndcg_full@5": 0.9269658250786468, + "ndcg@10": 0.9201370649018896, + "ndcg_full@10": 0.9201370649018896, + "ndcg@20": 0.7870012452406782, + "ndcg_full@20": 0.7870012452406782, + "P@10(rel>=1)": 1.0, + "P@10(eligible)": 0.8, + "graded_P@10": 0.9 + }, + "trec-202215": { + "num_relevant": 171, + "num_retrieved": 1202, + "num_ranked": 130, + "recall@10": 0.05847953216374269, + "recall@50": 0.26900584795321636, + "recall@100": 0.543859649122807, + "recall@200": 0.7076023391812866, + "recall@300": 0.7134502923976608, + "recall@500": 0.7134502923976608, + "recall@1000": 0.7134502923976608, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.934397461243829, + "ndcg_full@5": 0.934397461243829, + "ndcg@10": 0.879030172200514, + "ndcg_full@10": 0.879030172200514, + "ndcg@20": 0.8714610454766851, + "ndcg_full@20": 0.8714610454766851, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202216": { + "num_relevant": 48, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.20833333333333334, + "recall@50": 0.6875, + "recall@100": 0.9583333333333334, + "recall@200": 0.9583333333333334, + "recall@300": 0.9583333333333334, + "recall@500": 0.9583333333333334, + "recall@1000": 0.9583333333333334, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.4468535299918563, + "ndcg_full@5": 0.4468535299918563, + "ndcg@10": 0.5066770620649176, + "ndcg_full@10": 0.5066770620649176, + "ndcg@20": 0.4645333856775174, + "ndcg_full@20": 0.4645333856775174, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202217": { + "num_relevant": 30, + "num_retrieved": 2000, + "num_ranked": 153, + "recall@10": 0.0, + "recall@50": 0.1, + "recall@100": 0.3333333333333333, + "recall@200": 0.5666666666666667, + "recall@300": 0.7333333333333333, + "recall@500": 0.7666666666666667, + "recall@1000": 0.8333333333333334, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.3812509912356854, + "ndcg_full@5": 0.3812509912356854, + "ndcg@10": 0.3579457626026072, + "ndcg_full@10": 0.28660494082873905, + "ndcg@20": 0.3579457626026072, + "ndcg_full@20": 0.184965470718162, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202218": { + "num_relevant": 49, + "num_retrieved": 1826, + "num_ranked": 83, + "recall@10": 0.14285714285714285, + "recall@50": 0.6122448979591837, + "recall@100": 0.7346938775510204, + "recall@200": 0.7346938775510204, + "recall@300": 0.7551020408163265, + "recall@500": 0.7551020408163265, + "recall@1000": 0.7551020408163265, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.6300426811446146, + "ndcg_full@10": 0.6300426811446146, + "ndcg@20": 0.6197302866752329, + "ndcg_full@20": 0.6197302866752329, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202219": { + "num_relevant": 14, + "num_retrieved": 1417, + "num_ranked": 88, + "recall@10": 0.14285714285714285, + "recall@50": 0.35714285714285715, + "recall@100": 0.42857142857142855, + "recall@200": 0.5714285714285714, + "recall@300": 0.6428571428571429, + "recall@500": 0.6428571428571429, + "recall@1000": 0.6428571428571429, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.08274602130532002, + "ndcg_full@10": 0.06625422345438903, + "ndcg@20": 0.2912379818134747, + "ndcg_full@20": 0.20867224834778458, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202220": { + "num_relevant": 234, + "num_retrieved": 2000, + "num_ranked": 148, + "recall@10": 0.042735042735042736, + "recall@50": 0.2094017094017094, + "recall@100": 0.38461538461538464, + "recall@200": 0.6538461538461539, + "recall@300": 0.8076923076923077, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.925524411668024, + "ndcg_full@5": 0.925524411668024, + "ndcg@10": 0.8931590350286058, + "ndcg_full@10": 0.8931590350286058, + "ndcg@20": 0.9244448185535403, + "ndcg_full@20": 0.9244448185535403, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.8, + "graded_P@10": 0.85 + }, + "trec-202221": { + "num_relevant": 173, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03468208092485549, + "recall@50": 0.24277456647398843, + "recall@100": 0.4682080924855491, + "recall@200": 0.8265895953757225, + "recall@300": 0.9595375722543352, + "recall@500": 0.976878612716763, + "recall@1000": 0.9826589595375722, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.6164336326286644, + "ndcg_full@5": 0.6164336326286644, + "ndcg@10": 0.6114735274236563, + "ndcg_full@10": 0.6114735274236563, + "ndcg@20": 0.7492577005869105, + "ndcg_full@20": 0.7492577005869105, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202222": { + "num_relevant": 125, + "num_retrieved": 1949, + "num_ranked": 174, + "recall@10": 0.08, + "recall@50": 0.352, + "recall@100": 0.456, + "recall@200": 0.552, + "recall@300": 0.576, + "recall@500": 0.6, + "recall@1000": 0.616, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.38356636737133565, + "ndcg_full@5": 0.38356636737133565, + "ndcg@10": 0.5999767516907479, + "ndcg_full@10": 0.5999767516907479, + "ndcg@20": 0.6711547352689796, + "ndcg_full@20": 0.6711547352689796, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202223": { + "num_relevant": 188, + "num_retrieved": 2000, + "num_ranked": 151, + "recall@10": 0.05319148936170213, + "recall@50": 0.25, + "recall@100": 0.4787234042553192, + "recall@200": 0.7446808510638298, + "recall@300": 0.8138297872340425, + "recall@500": 0.8404255319148937, + "recall@1000": 0.8563829787234043, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8899541168509599, + "ndcg_full@10": 0.8899541168509599, + "ndcg@20": 0.7888033624323104, + "ndcg_full@20": 0.7888033624323104, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202224": { + "num_relevant": 31, + "num_retrieved": 1196, + "num_ranked": 61, + "recall@10": 0.1935483870967742, + "recall@50": 0.3225806451612903, + "recall@100": 0.3870967741935484, + "recall@200": 0.41935483870967744, + "recall@300": 0.45161290322580644, + "recall@500": 0.45161290322580644, + "recall@1000": 0.45161290322580644, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.7394745808335904, + "ndcg_full@10": 0.6177639991612475, + "ndcg@20": 0.8415057001010218, + "ndcg_full@20": 0.5153627323281361, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202225": { + "num_relevant": 181, + "num_retrieved": 2000, + "num_ranked": 138, + "recall@10": 0.022099447513812154, + "recall@50": 0.08287292817679558, + "recall@100": 0.1270718232044199, + "recall@200": 0.2596685082872928, + "recall@300": 0.3756906077348066, + "recall@500": 0.5911602209944752, + "recall@1000": 0.7292817679558011, + "unjudged_fraction@10": 0.6, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.11664253775060901, + "ndcg_full@10": 0.10906806227579123, + "ndcg@20": 0.19637646137636017, + "ndcg_full@20": 0.1533258568221059, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.2, + "graded_P@10": 0.2 + }, + "trec-202226": { + "num_relevant": 56, + "num_retrieved": 2000, + "num_ranked": 52, + "recall@10": 0.17857142857142858, + "recall@50": 0.7142857142857143, + "recall@100": 0.8392857142857143, + "recall@200": 0.9285714285714286, + "recall@300": 0.9821428571428571, + "recall@500": 0.9821428571428571, + "recall@1000": 0.9821428571428571, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.819972692711383, + "ndcg_full@5": 0.819972692711383, + "ndcg@10": 0.7803546731901579, + "ndcg_full@10": 0.7803546731901579, + "ndcg@20": 0.7525562011097645, + "ndcg_full@20": 0.7525562011097645, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.6, + "graded_P@10": 0.75 + }, + "trec-202227": { + "num_relevant": 29, + "num_retrieved": 2000, + "num_ranked": 134, + "recall@10": 0.27586206896551724, + "recall@50": 0.7931034482758621, + "recall@100": 0.8275862068965517, + "recall@200": 0.8620689655172413, + "recall@300": 0.8620689655172413, + "recall@500": 0.896551724137931, + "recall@1000": 0.896551724137931, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.13120507751234178, + "ndcg_full@5": 0.13120507751234178, + "ndcg@10": 0.40374278939643915, + "ndcg_full@10": 0.40374278939643915, + "ndcg@20": 0.4901442871391963, + "ndcg_full@20": 0.4901442871391963, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.6, + "graded_P@10": 0.6 + }, + "trec-202228": { + "num_relevant": 68, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.029411764705882353, + "recall@50": 0.17647058823529413, + "recall@100": 0.23529411764705882, + "recall@200": 0.3382352941176471, + "recall@300": 0.4117647058823529, + "recall@500": 0.45588235294117646, + "recall@1000": 0.5441176470588235, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.36005461457723403, + "ndcg_full@5": 0.36005461457723403, + "ndcg@10": 0.4837493984884811, + "ndcg_full@10": 0.4837493984884811, + "ndcg@20": 0.39229050911853386, + "ndcg_full@20": 0.3795011987511748, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202229": { + "num_relevant": 33, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.06060606060606061, + "recall@50": 0.18181818181818182, + "recall@100": 0.2727272727272727, + "recall@200": 0.48484848484848486, + "recall@300": 0.48484848484848486, + "recall@500": 0.6666666666666666, + "recall@1000": 0.7878787878787878, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.0, + "ndcg_full@5": 0.0, + "ndcg@10": 0.0, + "ndcg_full@10": 0.0, + "ndcg@20": 0.02634106816216367, + "ndcg_full@20": 0.01935096635428219, + "P@10(rel>=1)": 0.0, + "P@10(eligible)": 0.0, + "graded_P@10": 0.0 + }, + "trec-202230": { + "num_relevant": 382, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.020942408376963352, + "recall@50": 0.0968586387434555, + "recall@100": 0.18324607329842932, + "recall@200": 0.34554973821989526, + "recall@300": 0.5052356020942408, + "recall@500": 0.7486910994764397, + "recall@1000": 0.8115183246073299, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.830419897363192, + "ndcg_full@5": 0.830419897363192, + "ndcg@10": 0.8687471874513093, + "ndcg_full@10": 0.8687471874513093, + "ndcg@20": 0.8098898285547345, + "ndcg_full@20": 0.8098898285547345, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202231": { + "num_relevant": 39, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.1794871794871795, + "recall@50": 0.6153846153846154, + "recall@100": 0.717948717948718, + "recall@200": 0.7948717948717948, + "recall@300": 0.8461538461538461, + "recall@500": 0.8717948717948718, + "recall@1000": 0.8717948717948718, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.5711670054909295, + "ndcg_full@10": 0.5711670054909295, + "ndcg@20": 0.5116619157421688, + "ndcg_full@20": 0.5033887797417802, + "P@10(rel>=1)": 0.5, + "P@10(eligible)": 0.5, + "graded_P@10": 0.5 + }, + "trec-202232": { + "num_relevant": 219, + "num_retrieved": 2000, + "num_ranked": 202, + "recall@10": 0.0182648401826484, + "recall@50": 0.1004566210045662, + "recall@100": 0.1506849315068493, + "recall@200": 0.2602739726027397, + "recall@300": 0.3105022831050228, + "recall@500": 0.3926940639269406, + "recall@1000": 0.4977168949771689, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5, + "ndcg_full@5": 0.5, + "ndcg@10": 0.4999999999999999, + "ndcg_full@10": 0.4999999999999999, + "ndcg@20": 0.5376778191369558, + "ndcg_full@20": 0.5376778191369558, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.2, + "graded_P@10": 0.5 + }, + "trec-202233": { + "num_relevant": 72, + "num_retrieved": 2000, + "num_ranked": 179, + "recall@10": 0.05555555555555555, + "recall@50": 0.2222222222222222, + "recall@100": 0.2638888888888889, + "recall@200": 0.3194444444444444, + "recall@300": 0.375, + "recall@500": 0.4861111111111111, + "recall@1000": 0.5277777777777778, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.48522855511632257, + "ndcg_full@5": 0.48522855511632257, + "ndcg@10": 0.6161219969382546, + "ndcg_full@10": 0.6161219969382546, + "ndcg@20": 0.6008852332031293, + "ndcg_full@20": 0.6008852332031293, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + }, + "trec-202234": { + "num_relevant": 80, + "num_retrieved": 1468, + "num_ranked": 12, + "recall@10": 0.075, + "recall@50": 0.1375, + "recall@100": 0.1375, + "recall@200": 0.1375, + "recall@300": 0.1375, + "recall@500": 0.1375, + "recall@1000": 0.1375, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.8972754076830647, + "ndcg_full@5": 0.6379365213265478, + "ndcg@10": 0.9795097056968828, + "ndcg_full@10": 0.4519176995156636, + "ndcg@20": 0.9795097056968828, + "ndcg_full@20": 0.33846754982685545, + "P@10(rel>=1)": 0.4, + "P@10(eligible)": 0.2, + "graded_P@10": 0.3 + }, + "trec-202235": { + "num_relevant": 260, + "num_retrieved": 2000, + "num_ranked": 188, + "recall@10": 0.03461538461538462, + "recall@50": 0.10384615384615385, + "recall@100": 0.16538461538461538, + "recall@200": 0.23846153846153847, + "recall@300": 0.3269230769230769, + "recall@500": 0.4807692307692308, + "recall@1000": 0.5961538461538461, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7398014973729643, + "ndcg_full@5": 0.7398014973729643, + "ndcg@10": 0.783366986659749, + "ndcg_full@10": 0.783366986659749, + "ndcg@20": 0.7165583252472995, + "ndcg_full@20": 0.7165583252472995, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.7, + "graded_P@10": 0.8 + }, + "trec-202236": { + "num_relevant": 77, + "num_retrieved": 2000, + "num_ranked": 220, + "recall@10": 0.012987012987012988, + "recall@50": 0.15584415584415584, + "recall@100": 0.3246753246753247, + "recall@200": 0.6363636363636364, + "recall@300": 0.7662337662337663, + "recall@500": 0.935064935064935, + "recall@1000": 0.948051948051948, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.16958010263680806, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.11004588314904008, + "ndcg_full@10": 0.11004588314904008, + "ndcg@20": 0.1462361771597033, + "ndcg_full@20": 0.1462361771597033, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.1, + "graded_P@10": 0.1 + }, + "trec-202237": { + "num_relevant": 78, + "num_retrieved": 2000, + "num_ranked": 85, + "recall@10": 0.1282051282051282, + "recall@50": 0.6025641025641025, + "recall@100": 0.8717948717948718, + "recall@200": 0.9230769230769231, + "recall@300": 0.9230769230769231, + "recall@500": 0.9230769230769231, + "recall@1000": 0.9358974358974359, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.5922216874835864, + "ndcg_full@5": 0.5922216874835864, + "ndcg@10": 0.6569815082938864, + "ndcg_full@10": 0.6569815082938864, + "ndcg@20": 0.632229423933948, + "ndcg_full@20": 0.632229423933948, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.6, + "graded_P@10": 0.7 + }, + "trec-202238": { + "num_relevant": 159, + "num_retrieved": 2000, + "num_ranked": 135, + "recall@10": 0.06289308176100629, + "recall@50": 0.3081761006289308, + "recall@100": 0.6163522012578616, + "recall@200": 0.89937106918239, + "recall@300": 0.9056603773584906, + "recall@500": 0.9182389937106918, + "recall@1000": 0.9245283018867925, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.8687949224876582, + "ndcg_full@5": 0.8687949224876582, + "ndcg@10": 0.914856882358379, + "ndcg_full@10": 0.914856882358379, + "ndcg@20": 0.9109884396720315, + "ndcg_full@20": 0.9109884396720315, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202239": { + "num_relevant": 264, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.03787878787878788, + "recall@50": 0.16666666666666666, + "recall@100": 0.3371212121212121, + "recall@200": 0.6060606060606061, + "recall@300": 0.7537878787878788, + "recall@500": 0.7840909090909091, + "recall@1000": 0.7954545454545454, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.3304198973631919, + "ndcg_full@5": 0.3304198973631919, + "ndcg@10": 0.4201760117092417, + "ndcg_full@10": 0.4201760117092417, + "ndcg@20": 0.5532284016999461, + "ndcg_full@20": 0.5532284016999461, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.4, + "graded_P@10": 0.5 + }, + "trec-202240": { + "num_relevant": 163, + "num_retrieved": 2000, + "num_ranked": 110, + "recall@10": 0.06134969325153374, + "recall@50": 0.25766871165644173, + "recall@100": 0.3496932515337423, + "recall@200": 0.4233128834355828, + "recall@300": 0.4785276073619632, + "recall@500": 0.5644171779141104, + "recall@1000": 0.6625766871165644, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6548086577531307, + "ndcg_full@5": 0.6548086577531307, + "ndcg@10": 0.7759944384848244, + "ndcg_full@10": 0.7759944384848244, + "ndcg@20": 0.7891316627723801, + "ndcg_full@20": 0.7891316627723801, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.8, + "graded_P@10": 0.8 + }, + "trec-202241": { + "num_relevant": 70, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.14285714285714285, + "recall@50": 0.6857142857142857, + "recall@100": 0.9571428571428572, + "recall@200": 0.9714285714285714, + "recall@300": 0.9714285714285714, + "recall@500": 0.9857142857142858, + "recall@1000": 0.9857142857142858, + "unjudged_fraction@10": 0.3, + "ndcg@5": 0.6843515475204855, + "ndcg_full@5": 0.6843515475204855, + "ndcg@10": 0.5797827732044221, + "ndcg_full@10": 0.5797827732044221, + "ndcg@20": 0.5892817674198606, + "ndcg_full@20": 0.5892817674198606, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202242": { + "num_relevant": 179, + "num_retrieved": 1797, + "num_ranked": 107, + "recall@10": 0.055865921787709494, + "recall@50": 0.16201117318435754, + "recall@100": 0.21787709497206703, + "recall@200": 0.40782122905027934, + "recall@300": 0.4581005586592179, + "recall@500": 0.5921787709497207, + "recall@1000": 0.6703910614525139, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.7860137352654724, + "ndcg_full@5": 0.7860137352654724, + "ndcg@10": 0.6152434476571279, + "ndcg_full@10": 0.6152434476571279, + "ndcg@20": 0.5050877121519814, + "ndcg_full@20": 0.5050877121519814, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202243": { + "num_relevant": 84, + "num_retrieved": 2000, + "num_ranked": 109, + "recall@10": 0.023809523809523808, + "recall@50": 0.14285714285714285, + "recall@100": 0.2261904761904762, + "recall@200": 0.38095238095238093, + "recall@300": 0.4642857142857143, + "recall@500": 0.6309523809523809, + "recall@1000": 0.7976190476190477, + "unjudged_fraction@10": 0.4, + "ndcg@5": 0.07020838719835952, + "ndcg_full@5": 0.06560253875617089, + "ndcg@10": 0.14526400530847705, + "ndcg_full@10": 0.11358108740962401, + "ndcg@20": 0.2333740634754681, + "ndcg_full@20": 0.15857964275855738, + "P@10(rel>=1)": 0.2, + "P@10(eligible)": 0.1, + "graded_P@10": 0.15 + }, + "trec-202244": { + "num_relevant": 266, + "num_retrieved": 1636, + "num_ranked": 195, + "recall@10": 0.03383458646616541, + "recall@50": 0.18045112781954886, + "recall@100": 0.3458646616541353, + "recall@200": 0.6390977443609023, + "recall@300": 0.8834586466165414, + "recall@500": 0.9473684210526315, + "recall@1000": 0.9473684210526315, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.6473770215879483, + "ndcg_full@5": 0.6473770215879483, + "ndcg@10": 0.6995039277181867, + "ndcg_full@10": 0.6995039277181867, + "ndcg@20": 0.6830424928747032, + "ndcg_full@20": 0.6830424928747032, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.5, + "graded_P@10": 0.7 + }, + "trec-202245": { + "num_relevant": 21, + "num_retrieved": 2000, + "num_ranked": 107, + "recall@10": 0.0, + "recall@50": 0.19047619047619047, + "recall@100": 0.23809523809523808, + "recall@200": 0.2857142857142857, + "recall@300": 0.3333333333333333, + "recall@500": 0.3333333333333333, + "recall@1000": 0.3333333333333333, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.21836974929469322, + "ndcg_full@5": 0.16958010263680806, + "ndcg@10": 0.2026097735578014, + "ndcg_full@10": 0.11004588314904008, + "ndcg@20": 0.2026097735578014, + "ndcg_full@20": 0.08429733897617347, + "P@10(rel>=1)": 0.1, + "P@10(eligible)": 0.0, + "graded_P@10": 0.05 + }, + "trec-202246": { + "num_relevant": 314, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.028662420382165606, + "recall@50": 0.14012738853503184, + "recall@100": 0.25477707006369427, + "recall@200": 0.4840764331210191, + "recall@300": 0.6369426751592356, + "recall@500": 0.8439490445859873, + "recall@1000": 0.9267515923566879, + "unjudged_fraction@10": 0.1, + "ndcg@5": 0.8539316501572937, + "ndcg_full@5": 0.8539316501572937, + "ndcg@10": 0.664351874040447, + "ndcg_full@10": 0.664351874040447, + "ndcg@20": 0.6358998946675896, + "ndcg_full@20": 0.6358998946675896, + "P@10(rel>=1)": 0.6, + "P@10(eligible)": 0.5, + "graded_P@10": 0.55 + }, + "trec-202247": { + "num_relevant": 96, + "num_retrieved": 2000, + "num_ranked": 200, + "recall@10": 0.10416666666666667, + "recall@50": 0.4895833333333333, + "recall@100": 0.84375, + "recall@200": 0.9375, + "recall@300": 0.96875, + "recall@500": 0.9791666666666666, + "recall@1000": 0.9791666666666666, + "unjudged_fraction@10": 0.0, + "ndcg@5": 1.0, + "ndcg_full@5": 1.0, + "ndcg@10": 0.9363792118010481, + "ndcg_full@10": 0.9363792118010481, + "ndcg@20": 0.8185235921897122, + "ndcg_full@20": 0.8185235921897122, + "P@10(rel>=1)": 0.9, + "P@10(eligible)": 0.9, + "graded_P@10": 0.9 + }, + "trec-202248": { + "num_relevant": 329, + "num_retrieved": 2000, + "num_ranked": 224, + "recall@10": 0.030395136778115502, + "recall@50": 0.14893617021276595, + "recall@100": 0.2796352583586626, + "recall@200": 0.5379939209726444, + "recall@300": 0.6960486322188449, + "recall@500": 0.7386018237082067, + "recall@1000": 0.7933130699088146, + "unjudged_fraction@10": 0.2, + "ndcg@5": 0.6187490087643147, + "ndcg_full@5": 0.6187490087643147, + "ndcg@10": 0.6513525776243456, + "ndcg_full@10": 0.6513525776243456, + "ndcg@20": 0.6892655807862959, + "ndcg_full@20": 0.6892655807862959, + "P@10(rel>=1)": 0.8, + "P@10(eligible)": 0.5, + "graded_P@10": 0.65 + }, + "trec-202249": { + "num_relevant": 18, + "num_retrieved": 2000, + "num_ranked": 226, + "recall@10": 0.2777777777777778, + "recall@50": 0.4444444444444444, + "recall@100": 0.5555555555555556, + "recall@200": 0.7777777777777778, + "recall@300": 0.8888888888888888, + "recall@500": 0.9444444444444444, + "recall@1000": 1.0, + "unjudged_fraction@10": 0.5, + "ndcg@5": 0.3391602052736161, + "ndcg_full@5": 0.3391602052736161, + "ndcg@10": 0.3583931942241106, + "ndcg_full@10": 0.3583931942241106, + "ndcg@20": 0.41472192971486876, + "ndcg_full@20": 0.37502141524802374, + "P@10(rel>=1)": 0.3, + "P@10(eligible)": 0.3, + "graded_P@10": 0.3 + }, + "trec-202250": { + "num_relevant": 151, + "num_retrieved": 2000, + "num_ranked": 250, + "recall@10": 0.019867549668874173, + "recall@50": 0.1390728476821192, + "recall@100": 0.271523178807947, + "recall@200": 0.46357615894039733, + "recall@300": 0.6556291390728477, + "recall@500": 0.9470198675496688, + "recall@1000": 0.9933774834437086, + "unjudged_fraction@10": 0.0, + "ndcg@5": 0.7765732350040719, + "ndcg_full@5": 0.7765732350040719, + "ndcg@10": 0.7444146109124877, + "ndcg_full@10": 0.7444146109124877, + "ndcg@20": 0.7111504398674947, + "ndcg_full@20": 0.7111504398674947, + "P@10(rel>=1)": 0.7, + "P@10(eligible)": 0.7, + "graded_P@10": 0.7 + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/docs/paper-reproduction.md b/docs/paper-reproduction.md index 983aadac..17e98a66 100644 --- a/docs/paper-reproduction.md +++ b/docs/paper-reproduction.md @@ -30,6 +30,11 @@ It reports three separate views: evaluator to the same rankings. These values answer a different metric question and should not be substituted for the published values. +Use [`trialmatchai trec-evaluate`](trec-evaluation.md) to compare the current +evaluator with unjudged trials excluded versus retained as zero-gain results. +That comparison can reuse any completed result directory and performs no model +inference. + For an offline or CI run, supply the already-downloaded official archive and qrels: @@ -62,6 +67,11 @@ Means and medians are macro-averages across all topics in each track. | TREC 2022 nDCG@20 | 0.719456 | 0.766262 | 0.656213 | 0.632342 | | TREC 2022 graded P@10 | 0.663000 | 0.725000 | 0.661000 | 0.661000 | +For the same archived rankings, retaining unjudged trials as zero-gain results +instead of excluding them changes current tie-aware mean nDCG@10 from 0.669746 +to 0.435800 on TREC 2021 and from 0.598333 to 0.458934 on TREC 2022. The JSON +report stores both policies explicitly. + Across all 125 topics, weighting both tracks by topic count gives a stored mean nDCG@10 of 0.713276 and stored mean graded P@10 of 0.688400. These do not exactly regenerate the paper text's pooled values of 0.7232 and 0.6865 from the published diff --git a/docs/trec-evaluation.md b/docs/trec-evaluation.md new file mode 100644 index 00000000..891dd232 --- /dev/null +++ b/docs/trec-evaluation.md @@ -0,0 +1,86 @@ +# Evaluating completed TREC runs + +TrialMatchAI can recompute ranking metrics from completed result directories +without rebuilding an index or loading an LLM: + +```bash +trialmatchai trec-evaluate \ + --track 21 \ + --results-dir h100_rrf/results_trec21 \ + --results-dir qwen36_medcpt/results_trec21 \ + --qrels data/trec/qrels/qrels_21.txt \ + --include-per-query \ + --output trec21-evaluation.json +``` + +The command evaluates both supported unjudged-document policies by default: + +- `exclude` removes unjudged trials before applying each rank cutoff. This is + the condensed-list convention previously used by the current evaluator. +- `include_as_zero` preserves every ranked position and assigns unjudged trials + relevance grade zero. This is the conventional trec_eval interpretation. + +Both policies use the same tie-aware nDCG calculation. The report includes +macro means and medians, every topic when `--include-per-query` is supplied, +the qrels SHA-256, and a SHA-256 over every evaluated `ranked_trials.json` and +`nct_ids.txt`. The two policies must never be compared without their names. + +## Existing completed analysis + +Six local configurations have complete TREC 2021 and 2022 results: 75 and 50 +topic directories respectively, each with first-stage candidates, criterion +assessments, final rankings, and evaluation output. Re-running model inference +would duplicate these expensive runs. The closest current-code analogue to the +paper stack is `h100_rrf`: BGE-M3 retrieval, Gemma reranking, and Phi-4 +eligibility assessment. Its original effective configuration was not stored +beside its outputs, so it is evidence from a completed experiment rather than a +fully provenance-bound replication. + +The matching pipeline uses vLLM by default for the reranker and eligibility +reasoner. vLLM batches many criterion pairs and trial prompts within a patient. +Independent tracks and experiment arms can run as Slurm arrays on separate +GPUs; tensor parallelism is available for models that need multiple GPUs. For +these completed runs, metric evaluation is CPU-only and finishes faster than +queueing a GPU job. + +## Measured policy sensitivity + +These values were regenerated from the complete local rankings on 19 September +2026. `P@10` below is graded precision, `sum(relevance grade)/(2*10)`. + +| Track | Run | nDCG@10 exclude | nDCG@10 include-zero | P@10 exclude | P@10 include-zero | Unjudged top 10 | +| :--- | :--- | ---: | ---: | ---: | ---: | ---: | +| 2021 | h100_rrf | 0.790937 | 0.608344 | 0.775333 | 0.586667 | 26.3% | +| 2021 | h100_medcpt | 0.798727 | 0.585323 | 0.782667 | 0.572000 | 28.3% | +| 2021 | h100_baichuan | 0.822496 | 0.621061 | 0.802667 | 0.601333 | 26.1% | +| 2021 | qwen36_medcpt | 0.821458 | 0.613718 | 0.812667 | 0.601333 | 27.9% | +| 2021 | huatuo_medcpt | 0.681722 | 0.430457 | 0.676000 | 0.427333 | 39.1% | +| 2021 | iimedical_medcpt | 0.753910 | 0.545734 | 0.737333 | 0.534667 | 29.6% | +| 2022 | h100_rrf | 0.752821 | 0.653702 | 0.700000 | 0.603000 | 15.2% | +| 2022 | h100_medcpt | 0.714433 | 0.625418 | 0.671000 | 0.584000 | 17.8% | +| 2022 | h100_baichuan | 0.727787 | 0.622698 | 0.695000 | 0.589000 | 17.6% | +| 2022 | qwen36_medcpt | 0.783205 | 0.691356 | 0.731000 | 0.639000 | 14.2% | +| 2022 | huatuo_medcpt | 0.610370 | 0.490513 | 0.598000 | 0.486000 | 22.8% | +| 2022 | iimedical_medcpt | 0.690797 | 0.576935 | 0.659000 | 0.547000 | 18.8% | + +The inclusive policy lowers nDCG@10 by 0.089–0.251 across these runs. This is +expected because the systems retrieve trials outside the original judgment pool; +the inclusive policy treats those trials as incorrect even though they were +never assessed. The condensed policy avoids that assumption but measures +ordering only among judged retrieved trials. Report both, and use recall-aware +`ndcg_full@k` alongside them when retrieval coverage matters. + +The complete per-topic reports are tracked as +`benchmarks/trec/unjudged-policy-trec21.json` and +`benchmarks/trec/unjudged-policy-trec22.json`. + +## When another GPU run is warranted + +Launch a new Slurm inference job only for a configuration whose effective model, +adapter, corpus, index, prompts, candidate budgets, or ranking code differs from +an existing run. A production experiment should write that effective +configuration, Git revision, model revisions, index fingerprint, Slurm job ID, +GPU type, and output manifest into the result directory before processing the +first topic. Use one array arm per independent track/configuration and vLLM +batching within each arm. Resume incomplete patient and trial outputs rather +than passing `--rematch` unless invalidation is intentional. diff --git a/mkdocs.yml b/mkdocs.yml index 88c29cba..b1e6fe65 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,6 +62,7 @@ nav: - Registry updater: registry-updater.md - Fine-tuning: finetuning.md - Paper reproduction: paper-reproduction.md + - TREC evaluation: trec-evaluation.md - API reference: api.md - Release process: release.md - Production roadmap: production-roadmap.md diff --git a/pyproject.toml b/pyproject.toml index bf8c4209..ae64cee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ finetune = [ [project.scripts] # Single entry point. Every capability is a subcommand: `trialmatchai ` # (pipeline, healthcheck, bootstrap-data, index, build-concepts, update-registry, -# import-patient, build, run, e2e, trec, reproduce-paper, report, finetune). +# import-patient, build, run, e2e, trec, trec-evaluate, reproduce-paper, report, finetune). trialmatchai = "trialmatchai.cli.main:main" [project.urls] diff --git a/src/trialmatchai/cli/main.py b/src/trialmatchai/cli/main.py index d8c47eee..1a30e29b 100644 --- a/src/trialmatchai/cli/main.py +++ b/src/trialmatchai/cli/main.py @@ -69,6 +69,11 @@ def main() -> int: help="Verify and recalculate the published TREC 2021/2022 result artifact", add_help=False, ) + subparsers.add_parser( + "trec-evaluate", + help="Re-evaluate completed TREC rankings under explicit unjudged policies", + add_help=False, + ) subparsers.add_parser( "report", help="Render a self-contained HTML match report from a patient's results", @@ -109,6 +114,8 @@ def main() -> int: from trialmatchai.cli.trec import main as command elif args.command == "reproduce-paper": from trialmatchai.cli.reproduce_paper import main as command + elif args.command == "trec-evaluate": + from trialmatchai.cli.trec_evaluate import main as command elif args.command == "report": from trialmatchai.cli.report import main as command elif args.command == "finetune": diff --git a/src/trialmatchai/cli/reproduce_paper.py b/src/trialmatchai/cli/reproduce_paper.py index 86798591..311499e1 100644 --- a/src/trialmatchai/cli/reproduce_paper.py +++ b/src/trialmatchai/cli/reproduce_paper.py @@ -35,6 +35,9 @@ def _print_human(report: dict, output: Path) -> None: stored = track["stored_per_topic"] recalculated = track["recalculated_from_rankings"] current = track["current_evaluator"] + current_inclusive = track["current_evaluator_by_unjudged_policy"][ + "include_as_zero" + ] print(f"\n{track['track']} ({track['topics']} topics)") print( " stored per-topic aggregation: " @@ -48,7 +51,8 @@ def _print_human(report: dict, output: Path) -> None: ) print( " current-evaluator nDCG@10 mean: " - f"{current['ndcg@10']['mean']:.6f}" + f"exclude={current['ndcg@10']['mean']:.6f}, " + f"include-as-zero={current_inclusive['ndcg@10']['mean']:.6f}" ) diff --git a/src/trialmatchai/cli/trec_evaluate.py b/src/trialmatchai/cli/trec_evaluate.py new file mode 100644 index 00000000..c425052d --- /dev/null +++ b/src/trialmatchai/cli/trec_evaluate.py @@ -0,0 +1,160 @@ +"""Re-evaluate completed TREC rankings without retrieval or model inference.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +from trialmatchai.trec.metrics import UnjudgedPolicy +from trialmatchai.trec.qrels import download_qrels, evaluate, parse_qrels +from trialmatchai.utils.file_utils import write_json_file +from trialmatchai.utils.integrity import sha256_file + +POLICIES: tuple[UnjudgedPolicy, ...] = ("exclude", "include_as_zero") +PREFIXES = {"21": "trec-2021", "22": "trec-2022", "23": "trec-2023"} + + +def _evaluation_input_sha256(results_dir: Path, query_ids: Sequence[str]) -> str: + digest = hashlib.sha256() + for query_id in sorted(query_ids): + for filename in ("nct_ids.txt", "ranked_trials.json"): + path = results_dir / query_id / filename + relative = path.relative_to(results_dir).as_posix().encode() + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def evaluate_completed_runs( + *, + track: str, + results_dirs: Sequence[Path], + qrels_path: Path, + policies: Sequence[UnjudgedPolicy] = POLICIES, + include_per_query: bool = False, +) -> dict: + if track not in PREFIXES: + raise ValueError(f"Unsupported TREC track: {track}") + qrels = parse_qrels(qrels_path, PREFIXES[track]) + runs = [] + for supplied_results_dir in results_dirs: + supplied_results_dir = Path(supplied_results_dir) + results_dir = supplied_results_dir.resolve() + policy_results = {} + for policy in policies: + result = evaluate(qrels, results_dir, unjudged_policy=policy) + if not include_per_query: + result.pop("per_query", None) + policy_results[policy] = result + runs.append( + { + "name": results_dir.parent.name, + "results_dir": supplied_results_dir.as_posix(), + "evaluation_input_sha256": _evaluation_input_sha256( + results_dir, tuple(qrels) + ), + "policies": policy_results, + } + ) + return { + "schema": 1, + "track": f"TREC20{track}", + "qrels": {"path": qrels_path.as_posix(), "sha256": sha256_file(qrels_path)}, + "policies": { + "exclude": "remove unjudged trials before applying rank cutoffs", + "include_as_zero": "retain unjudged trials at their ranks with relevance grade zero", + }, + "runs": runs, + } + + +def _print_summary(report: dict, output: Path) -> None: + print(f"{report['track']} evaluation comparison") + print(f"Report: {output}") + for run in report["runs"]: + print(f"\n{run['name']} — {run['results_dir']}") + for policy, result in run["policies"].items(): + metrics = result["mean"] + print( + f" {policy:15s} " + f"nDCG@10={metrics['ndcg@10']:.6f} " + f"nDCG_full@10={metrics['ndcg_full@10']:.6f} " + f"graded_P@10={metrics['graded_P@10']:.6f} " + f"P@10(eligible)={metrics['P@10(eligible)']:.6f} " + f"unjudged@10={metrics['unjudged_fraction@10']:.3f}" + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Re-evaluate one or more completed TREC result directories under explicit " + "unjudged-document policies. No GPU, retrieval, reranking, or CoT inference is run." + ) + ) + parser.add_argument("--track", required=True, choices=tuple(PREFIXES)) + parser.add_argument( + "--results-dir", + action="append", + required=True, + type=Path, + help="Completed results_trec directory; repeat to compare runs", + ) + parser.add_argument("--qrels", type=Path, help="Official qrels file") + parser.add_argument("--data-dir", type=Path, default=Path("data")) + parser.add_argument( + "--policies", + nargs="+", + choices=POLICIES, + default=list(POLICIES), + ) + parser.add_argument( + "--include-per-query", + action="store_true", + help="Include every topic metric in the JSON report", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("trec-evaluation-comparison.json"), + ) + parser.add_argument("--json", action="store_true", help="Print the report as JSON") + args = parser.parse_args(argv) + + try: + qrels_path = args.qrels + if qrels_path is None: + qrels_path = download_qrels(args.track, args.data_dir / "trec" / "qrels") + missing = [path for path in args.results_dir if not path.is_dir()] + if missing: + raise FileNotFoundError(f"Result directories not found: {', '.join(map(str, missing))}") + report = evaluate_completed_runs( + track=args.track, + results_dirs=args.results_dir, + qrels_path=qrels_path, + policies=args.policies, + include_per_query=args.include_per_query, + ) + output = args.output.resolve() + output.parent.mkdir(parents=True, exist_ok=True) + write_json_file(report, str(output)) + except (OSError, ValueError) as exc: + print(f"trec-evaluate failed: {exc}", file=sys.stderr) + return 1 + + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + _print_summary(report, output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/trialmatchai/trec/metrics.py b/src/trialmatchai/trec/metrics.py index a5ad738e..f372d22e 100644 --- a/src/trialmatchai/trec/metrics.py +++ b/src/trialmatchai/trec/metrics.py @@ -2,15 +2,17 @@ nDCG here is tie-aware (McSherry & Najork, 2008): tied scores each get the mean positional discount over the tie group's ranks — the expected nDCG over all tie -orderings, invariant to arbitrary tie-breaking. It is also condensed (over -labeled-and-retrieved trials only, decoupling ranking quality from recall). Gain is -linear (gain = relevance grade), matching trec_eval's default. +orderings, invariant to arbitrary tie-breaking. Evaluation can either condense the +ranking to judged trials or retain unjudged trials with zero gain. Gain is linear +(gain = relevance grade), matching trec_eval's default. """ from __future__ import annotations import math -from typing import Dict, Mapping, Sequence, Set +from typing import Dict, Literal, Mapping, Sequence, Set + +UnjudgedPolicy = Literal["exclude", "include_as_zero"] def _discount(rank: int) -> float: @@ -62,7 +64,7 @@ def ndcg_at_k( *, ideal_gains: Sequence[float] | None = None, ) -> float: - """Tie-aware nDCG@k. ``ordered_ids`` should be the condensed (labeled) list. + """Tie-aware nDCG@k over the supplied evaluated ranking. ``ideal_gains`` chooses the IDCG basis: ``None`` (default) uses the gains of ``ordered_ids`` (judged-AND-ranked), making nDCG recall-independent; pass the FULL @@ -121,3 +123,33 @@ def condensed_ndcg( condensed = [nid for nid in ranked_ids if nid in grade_of] ideal = [float(g) for g in grade_of.values()] if full_ideal else None return {k: ndcg_at_k(condensed, score_of, grade_of, k, ideal_gains=ideal) for k in cutoffs} + + +def ranking_ndcg( + ranked_ids: Sequence[str], + score_of: Mapping[str, float], + grade_of: Mapping[str, int], + cutoffs: Sequence[int], + *, + unjudged_policy: UnjudgedPolicy = "exclude", + full_ideal: bool = False, +) -> Dict[int, float]: + """Tie-aware nDCG under an explicit treatment of unjudged documents. + + ``exclude`` is the condensed-list convention: unjudged documents are removed + before rank cutoffs are applied. ``include_as_zero`` retains their positions + and assigns gain zero, matching the usual trec_eval treatment. ``full_ideal`` + controls whether unretrieved judged documents remain in the ideal ranking. + """ + if unjudged_policy not in {"exclude", "include_as_zero"}: + raise ValueError(f"Unsupported unjudged policy: {unjudged_policy}") + evaluated = ( + [document for document in ranked_ids if document in grade_of] + if unjudged_policy == "exclude" + else list(ranked_ids) + ) + ideal = [float(gain) for gain in grade_of.values()] if full_ideal else None + return { + cutoff: ndcg_at_k(evaluated, score_of, grade_of, cutoff, ideal_gains=ideal) + for cutoff in cutoffs + } diff --git a/src/trialmatchai/trec/paper_reproduction.py b/src/trialmatchai/trec/paper_reproduction.py index b06fb011..e9ae4bab 100644 --- a/src/trialmatchai/trec/paper_reproduction.py +++ b/src/trialmatchai/trec/paper_reproduction.py @@ -279,18 +279,17 @@ def audit_track(track: str, results_root: Path, qrels_path: Path) -> dict: if not _close(archived_summary[key], stored_summary[key]["mean"]) } - current = evaluate(qrels, track_dir, cutoffs=RECALL_CUTOFFS) - current_keys = tuple(current["mean"]) - current_summary = { - key: { - "mean": current["mean"][key], - "median": median( - float(row[key]) for row in current["per_query"].values() if key in row - ), + current_by_policy = {} + for policy in ("exclude", "include_as_zero"): + current = evaluate( + qrels, track_dir, cutoffs=RECALL_CUTOFFS, unjudged_policy=policy + ) + current_by_policy[policy] = { + key: {"mean": value, "median": current["median"][key]} + for key, value in current["mean"].items() + if value is not None } - for key in current_keys - if current["mean"][key] is not None - } + current_summary = current_by_policy["exclude"] return { "track": f"TREC20{track}", @@ -300,6 +299,7 @@ def audit_track(track: str, results_root: Path, qrels_path: Path) -> dict: "recalculated_from_rankings": recalculated_summary, "paper_retrieval_recall": recall_summary, "current_evaluator": current_summary, + "current_evaluator_by_unjudged_policy": current_by_policy, "checks": { "archived_summary_matches_stored_topics": not summary_mismatches, "rankings_match_stored_topic_metrics": not mismatch_topics, @@ -338,6 +338,19 @@ def pooled(section: str, key: str) -> float: for key in tracks["21"]["current_evaluator"] if key in tracks["22"]["current_evaluator"] }, + "current_evaluator_by_unjudged_policy": { + policy: { + key: sum( + item["current_evaluator_by_unjudged_policy"][policy][key]["mean"] + * item["topics"] + for item in tracks.values() + ) + / total_topics + for key in tracks["21"]["current_evaluator_by_unjudged_policy"][policy] + if key in tracks["22"]["current_evaluator_by_unjudged_policy"][policy] + } + for policy in ("exclude", "include_as_zero") + }, } return { "schema": 1, diff --git a/src/trialmatchai/trec/qrels.py b/src/trialmatchai/trec/qrels.py index 94d543e1..48262f0a 100644 --- a/src/trialmatchai/trec/qrels.py +++ b/src/trialmatchai/trec/qrels.py @@ -10,14 +10,16 @@ import json from pathlib import Path +from statistics import median import requests from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from trialmatchai.trec.metrics import ( - condensed_ndcg, graded_precision_at_k, precision_at_k, + ranking_ndcg, + UnjudgedPolicy, ) from trialmatchai.utils.logging_config import setup_logging @@ -162,13 +164,18 @@ def evaluate( *, cutoffs: tuple[int, ...] = DEFAULT_CUTOFFS, threshold: int = 1, + unjudged_policy: UnjudgedPolicy = "exclude", ) -> dict: """Per-query and mean metrics over the patients in ``results_dir``. Reports recall@k (retrieval, first-level list) and tie-aware nDCG@{5,10,20} - + P@10 (ranking, condensed to judged trials). P@10 is split into "relevant" - (grade>=1) and "eligible" (grade==2). + + P@10. ``unjudged_policy='exclude'`` condenses the ranked list to judged + trials before applying cutoffs; ``'include_as_zero'`` retains unjudged trials + with gain zero. P@10 is split into "relevant" (grade>=1) and "eligible" + (grade==2). """ + if unjudged_policy not in {"exclude", "include_as_zero"}: + raise ValueError(f"Unsupported unjudged policy: {unjudged_policy}") results_dir = Path(results_dir) relevant = relevant_ncts(qrels, threshold=threshold) eligible = relevant_ncts(qrels, threshold=2) @@ -181,6 +188,7 @@ def evaluate( rank_sums[f"P@{P_CUTOFF}(rel>=1)"] = 0.0 rank_sums[f"P@{P_CUTOFF}(eligible)"] = 0.0 rank_sums[f"graded_P@{P_CUTOFF}"] = 0.0 + rank_sums[f"unjudged_fraction@{P_CUTOFF}"] = 0.0 rank_counts = {key: 0 for key in rank_sums} for query_id, judgments in qrels.items(): @@ -203,11 +211,31 @@ def evaluate( rec_counts[f"recall@{k}"] += 1 if ranked: - # Two IDCG bases: ndcg@k normalizes by the ideal over judged-AND-ranked trials - # (recall-independent ordering quality); ndcg_full@k by the ideal over the FULL judged - # pool (recall-aware, trec_eval-style). DCG numerator is condensed (ignores unjudged) in both. - ndcg = condensed_ndcg(ranked, score_of, judgments, NDCG_CUTOFFS) - ndcg_full = condensed_ndcg(ranked, score_of, judgments, NDCG_CUTOFFS, full_ideal=True) + unjudged_fraction = sum( + trial_id not in judgments for trial_id in ranked[:P_CUTOFF] + ) / float(P_CUTOFF) + row[f"unjudged_fraction@{P_CUTOFF}"] = unjudged_fraction + rank_sums[f"unjudged_fraction@{P_CUTOFF}"] += unjudged_fraction + rank_counts[f"unjudged_fraction@{P_CUTOFF}"] += 1 + # Two IDCG bases: ndcg@k normalizes by the ideal over judged-and-ranked + # trials (recall-independent ordering quality); ndcg_full@k by the ideal + # over the full judged pool. The selected policy controls whether the DCG + # ranking drops unjudged trials or retains their positions with zero gain. + ndcg = ranking_ndcg( + ranked, + score_of, + judgments, + NDCG_CUTOFFS, + unjudged_policy=unjudged_policy, + ) + ndcg_full = ranking_ndcg( + ranked, + score_of, + judgments, + NDCG_CUTOFFS, + unjudged_policy=unjudged_policy, + full_ideal=True, + ) for k in NDCG_CUTOFFS: row[f"ndcg@{k}"] = ndcg[k] row[f"ndcg_full@{k}"] = ndcg_full[k] @@ -215,13 +243,18 @@ def evaluate( rank_sums[f"ndcg_full@{k}"] += ndcg_full[k] rank_counts[f"ndcg@{k}"] += 1 rank_counts[f"ndcg_full@{k}"] += 1 - # Condense to the judged pool before the precision cutoff so unjudged trials the - # assessors never saw don't count as misses (matches condensed_ndcg above). Without - # this, a non-pooled system's many unjudged hits understated P@k and clashed with nDCG. - judged_ranked = [nid for nid in ranked if nid in judgments] - p_rel = precision_at_k(judged_ranked, rel_set, P_CUTOFF) - p_elig = precision_at_k(judged_ranked, eligible.get(query_id, set()), P_CUTOFF) - p_graded = graded_precision_at_k(judged_ranked, judgments, P_CUTOFF) + # Apply the same policy to precision: either drop unjudged trials before + # the cutoff or keep them at their original rank as nonrelevant. + evaluated_ranked = ( + [nid for nid in ranked if nid in judgments] + if unjudged_policy == "exclude" + else ranked + ) + p_rel = precision_at_k(evaluated_ranked, rel_set, P_CUTOFF) + p_elig = precision_at_k( + evaluated_ranked, eligible.get(query_id, set()), P_CUTOFF + ) + p_graded = graded_precision_at_k(evaluated_ranked, judgments, P_CUTOFF) row[f"P@{P_CUTOFF}(rel>=1)"] = p_rel row[f"P@{P_CUTOFF}(eligible)"] = p_elig row[f"graded_P@{P_CUTOFF}"] = p_graded @@ -234,10 +267,18 @@ def evaluate( per_query[query_id] = row mean = {**_mean(rec_sums, rec_counts), **_mean(rank_sums, rank_counts)} + metric_keys = tuple(mean) + medians = { + key: median(float(row[key]) for row in per_query.values() if row.get(key) is not None) + for key in metric_keys + if any(row.get(key) is not None for row in per_query.values()) + } return { + "unjudged_policy": unjudged_policy, "recall_relevance_threshold": threshold, "num_queries_scored": len(per_query), "num_queries_ranked": rank_counts[f"ndcg@{NDCG_CUTOFFS[0]}"], "mean": mean, + "median": medians, "per_query": per_query, } diff --git a/tests/test_paper_reproduction.py b/tests/test_paper_reproduction.py index 20511d7e..e05f95e9 100644 --- a/tests/test_paper_reproduction.py +++ b/tests/test_paper_reproduction.py @@ -68,13 +68,14 @@ def test_audit_track_separates_stored_metrics_from_ranking_recalculation( (topic / "ranked_trials.json").write_text( json.dumps( [ + {"TrialID": "UNJUDGED", "Score": 2.0}, {"TrialID": "NCT2", "Score": 1.0}, {"TrialID": "NCT1", "Score": 1.0}, ] ), encoding="utf-8", ) - (topic / "nct_ids.txt").write_text("NCT2\nNCT1\n", encoding="utf-8") + (topic / "nct_ids.txt").write_text("UNJUDGED\nNCT2\nNCT1\n", encoding="utf-8") for cutoff in reproduction.RECALL_CUTOFFS: (topic / f"nct_ids_{cutoff}.txt").write_text("NCT2\nNCT1\n", encoding="utf-8") @@ -88,3 +89,9 @@ def test_audit_track_separates_stored_metrics_from_ranking_recalculation( assert result["current_evaluator"]["ndcg@5"]["mean"] != pytest.approx( result["recalculated_from_rankings"]["ndcg@5"]["mean"] ) + assert ( + result["current_evaluator_by_unjudged_policy"]["include_as_zero"]["ndcg@5"][ + "mean" + ] + < result["current_evaluator_by_unjudged_policy"]["exclude"]["ndcg@5"]["mean"] + ) diff --git a/tests/test_qrels_eval.py b/tests/test_qrels_eval.py index a864fd73..2f5bbd9a 100644 --- a/tests/test_qrels_eval.py +++ b/tests/test_qrels_eval.py @@ -93,3 +93,30 @@ def test_evaluate_precision_is_condensed_to_judged_pool(tmp_path): assert mean["P@10(rel>=1)"] == pytest.approx(2 / 10) # raw would be 0/10 assert mean["P@10(eligible)"] == pytest.approx(1 / 10) # only NCT1 is grade 2 assert mean["graded_P@10"] == pytest.approx((2 + 1) / (10 * 2)) # raw would be 0 + + +def test_evaluate_can_count_unjudged_trials_as_zero_gain(tmp_path): + q = "trec-1" + pdir = tmp_path / q + pdir.mkdir() + ranked = [{"TrialID": f"NCTX{i}", "Score": 2.0 - i * 0.01} for i in range(10)] + ranked += [{"TrialID": "NCT1", "Score": 0.5}, {"TrialID": "NCT2", "Score": 0.4}] + (pdir / "ranked_trials.json").write_text(json.dumps(ranked)) + (pdir / "nct_ids.txt").write_text("\n".join(row["TrialID"] for row in ranked) + "\n") + qrels = {q: {"NCT1": 2, "NCT2": 1}} + + condensed = evaluate(qrels, tmp_path, cutoffs=(10,), unjudged_policy="exclude") + inclusive = evaluate( + qrels, tmp_path, cutoffs=(10,), unjudged_policy="include_as_zero" + ) + + assert condensed["mean"]["graded_P@10"] == pytest.approx(3 / 20) + assert condensed["mean"]["ndcg@10"] == pytest.approx(1.0) + assert inclusive["mean"]["graded_P@10"] == 0.0 + assert inclusive["mean"]["ndcg@10"] == 0.0 + assert inclusive["unjudged_policy"] == "include_as_zero" + + +def test_evaluate_rejects_unknown_unjudged_policy(tmp_path): + with pytest.raises(ValueError, match="Unsupported unjudged policy"): + evaluate({}, tmp_path, unjudged_policy="unknown") diff --git a/tests/test_trec_evaluate_cli.py b/tests/test_trec_evaluate_cli.py new file mode 100644 index 00000000..8ebd8f18 --- /dev/null +++ b/tests/test_trec_evaluate_cli.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import json + +from trialmatchai.cli.trec_evaluate import evaluate_completed_runs + + +def test_completed_run_comparison_reports_both_unjudged_policies(tmp_path): + qrels = tmp_path / "qrels.txt" + qrels.write_text("1 0 NCT1 2\n", encoding="utf-8") + results = tmp_path / "experiment" / "results_trec21" + topic = results / "trec-20211" + topic.mkdir(parents=True) + ranked = [ + {"TrialID": "UNJUDGED", "Score": 1.0}, + {"TrialID": "NCT1", "Score": 0.5}, + ] + (topic / "ranked_trials.json").write_text(json.dumps(ranked), encoding="utf-8") + (topic / "nct_ids.txt").write_text("UNJUDGED\nNCT1\n", encoding="utf-8") + + report = evaluate_completed_runs( + track="21", results_dirs=[results], qrels_path=qrels + ) + + run = report["runs"][0] + assert run["name"] == "experiment" + assert len(run["evaluation_input_sha256"]) == 64 + assert len(report["qrels"]["sha256"]) == 64 + assert run["policies"]["exclude"]["mean"]["ndcg@10"] == 1.0 + assert run["policies"]["include_as_zero"]["mean"]["ndcg@10"] < 1.0 + assert "per_query" not in run["policies"]["exclude"] From e27df8f3304fb4e34fb8b6d2c8d3ff91ad57abc7 Mon Sep 17 00:00:00 2001 From: Majd Abdallah Date: Sat, 19 Sep 2026 21:19:42 +0200 Subject: [PATCH 3/3] fix: validate complete paper audit inputs --- src/trialmatchai/trec/paper_reproduction.py | 37 ++++++++++---- tests/test_paper_reproduction.py | 54 +++++++++++++++++++++ 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/trialmatchai/trec/paper_reproduction.py b/src/trialmatchai/trec/paper_reproduction.py index e9ae4bab..449c2011 100644 --- a/src/trialmatchai/trec/paper_reproduction.py +++ b/src/trialmatchai/trec/paper_reproduction.py @@ -115,6 +115,24 @@ def _validated_member(name: str) -> PurePosixPath: return path +def _required_reproduction_paths(root: Path): + """Yield every archive file consumed by the reproduction audit.""" + for spec in TRACKS.values(): + track_dir = root / str(spec["directory"]) + yield track_dir / "average_metrics.json" + for topic in range(1, int(spec["topics"]) + 1): + topic_dir = track_dir / f"{spec['prefix']}{topic}" + yield topic_dir / "evaluation_metrics.json" + yield topic_dir / "ranked_trials.json" + yield topic_dir / "nct_ids.txt" + for cutoff in RECALL_CUTOFFS: + yield topic_dir / f"nct_ids_{cutoff}.txt" + + +def _missing_reproduction_paths(root: Path) -> list[Path]: + return [path for path in _required_reproduction_paths(root) if not path.is_file()] + + def extract_reproduction_files(archive: Path, workdir: Path) -> Path: """Extract only files needed for metric reproduction, atomically and safely.""" workdir = Path(workdir).resolve() @@ -124,13 +142,9 @@ def extract_reproduction_files(archive: Path, workdir: Path) -> Path: state = json.loads(marker.read_text(encoding="utf-8")) except (OSError, ValueError, TypeError): state = {} - cache_complete = state.get("archive_sha256") == PAPER_RESULTS_SHA256 and all( - (target / str(spec["directory"]) / "average_metrics.json").is_file() - and len(list((target / str(spec["directory"])).glob("*/evaluation_metrics.json"))) - == int(spec["topics"]) - and len(list((target / str(spec["directory"])).glob("*/ranked_trials.json"))) - == int(spec["topics"]) - for spec in TRACKS.values() + cache_complete = ( + state.get("archive_sha256") == PAPER_RESULTS_SHA256 + and not _missing_reproduction_paths(target) ) if cache_complete: return target @@ -159,8 +173,13 @@ def extract_reproduction_files(archive: Path, workdir: Path) -> Path: with bundle.open(member) as source, destination.open("wb") as output: shutil.copyfileobj(source, output) extracted += 1 - if extracted < sum(spec["topics"] for spec in TRACKS.values()) * 3: - raise ValueError("Paper archive is missing the expected TREC reproduction files") + missing = _missing_reproduction_paths(staging) + if missing: + example = missing[0].relative_to(staging) + raise ValueError( + "Paper archive is missing " + f"{len(missing)} expected TREC reproduction file(s); first missing: {example}" + ) write_json_file( {"schema": 1, "archive_sha256": PAPER_RESULTS_SHA256, "files": extracted}, str(staging / ".paper-results.json"), diff --git a/tests/test_paper_reproduction.py b/tests/test_paper_reproduction.py index e05f95e9..1465c506 100644 --- a/tests/test_paper_reproduction.py +++ b/tests/test_paper_reproduction.py @@ -36,6 +36,60 @@ def test_paper_archive_extraction_rejects_traversal(tmp_path): assert not (tmp_path / "outside.json").exists() +def test_paper_archive_extraction_requires_recall_inputs(tmp_path, monkeypatch): + monkeypatch.setattr( + reproduction, + "TRACKS", + {"21": {"directory": "TREC21", "prefix": "trec-2021", "topics": 1}}, + ) + monkeypatch.setattr(reproduction, "RECALL_CUTOFFS", (10,)) + archive = tmp_path / "incomplete.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("matching_results/TREC21/average_metrics.json", "{}") + bundle.writestr( + "matching_results/TREC21/trec-20211/evaluation_metrics.json", "{}" + ) + bundle.writestr("matching_results/TREC21/trec-20211/ranked_trials.json", "[]") + bundle.writestr("matching_results/TREC21/trec-20211/nct_ids.txt", "NCT1\n") + + with pytest.raises(ValueError, match="nct_ids_10.txt"): + reproduction.extract_reproduction_files(archive, tmp_path / "work") + + +def test_incomplete_extraction_cache_is_rebuilt(tmp_path, monkeypatch): + monkeypatch.setattr( + reproduction, + "TRACKS", + {"21": {"directory": "TREC21", "prefix": "trec-2021", "topics": 1}}, + ) + monkeypatch.setattr(reproduction, "RECALL_CUTOFFS", (10,)) + monkeypatch.setattr(reproduction, "PAPER_RESULTS_SHA256", "test-archive") + work = tmp_path / "work" + cached_topic = work / "matching_results" / "TREC21" / "trec-20211" + cached_topic.mkdir(parents=True) + (work / "matching_results" / ".paper-results.json").write_text( + json.dumps({"archive_sha256": "test-archive"}), encoding="utf-8" + ) + for name in ("evaluation_metrics.json", "ranked_trials.json", "nct_ids.txt"): + (cached_topic / name).write_text("{}", encoding="utf-8") + (cached_topic.parent / "average_metrics.json").write_text("{}", encoding="utf-8") + + archive = tmp_path / "complete.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("matching_results/TREC21/average_metrics.json", "{}") + for name, content in ( + ("evaluation_metrics.json", "{}"), + ("ranked_trials.json", "[]"), + ("nct_ids.txt", "NCT1\n"), + ("nct_ids_10.txt", "NCT1\n"), + ): + bundle.writestr(f"matching_results/TREC21/trec-20211/{name}", content) + + extracted = reproduction.extract_reproduction_files(archive, work) + + assert (extracted / "TREC21" / "trec-20211" / "nct_ids_10.txt").is_file() + + def test_paper_ranking_metric_drops_unjudged_and_preserves_archived_order(): metrics = reproduction.paper_ranking_metrics( ["UNJUDGED", "NCT_GRADE_1", "NCT_GRADE_2"],