diff --git a/.gitignore b/.gitignore index 9add94cd..01524fea 100644 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,9 @@ evaluator/workshop/results.jsonl # Large per-judge raw severity table (13 MB, regeneratable via # scripts/compute_inter_judge_agreement.py) tables/inter_judge_raw.csv + +# Partner-mode belt and braces: partner data lives outside the repo +# (~/humanebench-partners/), these patterns guard against accidents +humanebench-partners/ +partner_*.jsonl +*.eval diff --git a/docs/partner-mode.md b/docs/partner-mode.md new file mode 100644 index 00000000..855fdb1c --- /dev/null +++ b/docs/partner-mode.md @@ -0,0 +1,68 @@ +# Partner mode: re-judging external prompt/response datasets + +How to run the HumaneBench judge panel on a partner's dataset of **stored +prompt/response pairs** (no generation — the stored response is replayed and +judged). + +## Ground rules + +- **Partner data never enters git.** Keep datasets, `.eval` logs, and results + in a workspace outside the repo: `~/humanebench-partners//`. + `.eval` logs embed every prompt, response, and judge rationale — treat them + as sensitive as the raw dataset. +- **Provider egress:** judging sends each prompt + response to the three + judge-model providers via OpenRouter. Before any run, the OpenRouter + account must be set to **exclude providers that may train on inputs**, and + the partner must have signed off on third-party judging. +- Even anonymized messages can contain self-disclosed details (names, + locations). Re-check verbatim quotes before circulating analysis documents. + +## Dataset format + +HumaneBench-style JSONL, one sample per (turn, principle): + +```json +{"id": "__", + "input": "", + "target": "", + "metadata": {"ai_output": ""}} +``` + +`target` must be one of the 8 principle slugs in +`humanebench/humane_patterns.py`. `metadata.ai_output` is the contract key +read by the strict replay solver (`src/pregenerated_solver.py`) and must be a +non-empty string. `scripts/convert_partner_results.py` produces this format +from per-turn judged-results files, fanning each turn out across principles. + +## Pipeline + +1. **Curate** — `scripts/curate_production_pairs.py`: tags rows + (synthetic/QA traffic, duplicate clusters, trivial labels, language) + without dropping anything. Tags are analyst-visible only; judges never see + them. +2. **Select** (optional, for comparison runs) — + `scripts/select_comparison_subset.py`: deterministic stratified subset + incl. a repeat slice for panel self-consistency. Requires curated input. +3. **Convert** — `scripts/convert_partner_results.py`: fan out to the format + above. Turns with empty responses are skipped loudly. +4. **Judge** — run **without `--model`** (nothing can generate; the NoModel + errors loudly if anything tries), dataset path **absolute**: + + ``` + inspect eval src/partner_rejudge_task.py \ + -T dataset=/absolute/path/to/converted.jsonl \ + --log-dir ~/humanebench-partners//logs + ``` + + Pilot with `--limit 10` and verify cost before a full run. + +## Notes + +- The benchmark tasks (`baseline`, `good_persona`, `bad_persona`) also accept + `-T dataset=` for ad-hoc runs; there is deliberately **no environment + variable override** — a stale variable could silently redirect a benchmark + run. +- A HumaneScore computed over a partner dataset that does not cover all 8 + principles is **not comparable** to benchmark HumaneScores (empty + principles are zero-filled into the denominator). Report per-principle + results with their sample counts. diff --git a/scripts/convert_partner_results.py b/scripts/convert_partner_results.py new file mode 100644 index 00000000..6b82ba13 --- /dev/null +++ b/scripts/convert_partner_results.py @@ -0,0 +1,124 @@ +"""Convert a partner production-results JSONL to HumaneBench re-judge format. + +Each input row (one conversation turn with a stored assistant response and +per-principle judgments) fans out to one sample per principle: + + {"id": "__", "input": , + "target": , + "metadata": {"ai_output": , ...}} + +`metadata.ai_output` is the contract key read by +src/pregenerated_solver.py:use_pregenerated_output_strict(); the original +judgments and curation tags ride along in metadata for post-run comparison and +are never shown to judges (the overseer only sees input + ai_output). + +Usage: + python scripts/convert_partner_results.py \ + --input --output \ + [--principles {all,relevant}] +""" +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from humanebench.humane_patterns import HUMANE_PATTERNS + +KNOWN_SLUGS = frozenset(HUMANE_PATTERNS.keys()) + + +def has_judgeable_response(row: dict) -> bool: + """The strict replay solver treats falsy ai_output as missing and raises, + so only turns with a non-empty string response can be re-judged.""" + resp = row.get("assistant_response") + return isinstance(resp, str) and bool(resp) + + +def convert_row(row: dict, principles_mode: str) -> list[dict]: + unknown = set(row["principles"]) - KNOWN_SLUGS + if unknown: + raise ValueError( + f"sample {row['sample_id']}: unknown principle slug(s) {sorted(unknown)}" + ) + + if principles_mode == "relevant": + slugs = row.get("relevant_principles", []) + unknown = set(slugs) - KNOWN_SLUGS + if unknown: + raise ValueError( + f"sample {row['sample_id']}: unknown slug(s) in " + f"relevant_principles {sorted(unknown)}" + ) + else: + slugs = sorted(row["principles"]) + + samples = [] + for slug in slugs: + samples.append( + { + "id": f"{row['sample_id']}__{slug}", + "input": row["user_message"], + "target": slug, + "metadata": { + "ai_output": row["assistant_response"], + "conv": row.get("conv"), + "turn_index": row.get("turn_index"), + "ts": row.get("ts"), + "curation": row.get("curation"), + "orig_judgment": row["principles"].get(slug), + "orig_overall_severity": row.get("overall_severity"), + "orig_mean_severity": row.get("mean_severity"), + "audit": (row.get("audit") or {}).get("principles", {}).get(slug), + }, + } + ) + return samples + + +def split_sample_id(hb_id: str) -> tuple[str, str]: + """Recover (sample_id, slug) from an HB fan-out id; safe for ids containing '__'.""" + sample_id, slug = hb_id.rsplit("__", 1) + return sample_id, slug + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--principles", choices=["all", "relevant"], default="all") + args = parser.parse_args() + + n_rows = 0 + n_samples = 0 + skipped_empty: list[str] = [] + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.input) as fin, open(args.output, "w") as fout: + for line in fin: + if not line.strip(): + continue + row = json.loads(line) + n_rows += 1 + if not has_judgeable_response(row): + skipped_empty.append(row.get("sample_id", "")) + continue + for sample in convert_row(row, args.principles): + fout.write(json.dumps(sample, ensure_ascii=False) + "\n") + n_samples += 1 + + print( + f"Converted {n_rows} turns -> {n_samples} samples " + f"(mode={args.principles}) into {args.output}" + ) + if skipped_empty: + print( + f"WARNING: skipped {len(skipped_empty)} turn(s) with empty/missing " + f"assistant_response (cannot be re-judged by the strict replay " + f"solver): {', '.join(skipped_empty[:20])}" + + (" ..." if len(skipped_empty) > 20 else "") + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/curate_production_pairs.py b/scripts/curate_production_pairs.py new file mode 100644 index 00000000..87fe6e38 --- /dev/null +++ b/scripts/curate_production_pairs.py @@ -0,0 +1,143 @@ +"""Curate a partner production-results JSONL by tagging rows (never dropping them). + +Adds a "curation" object to each row: + - synthetic_test: user message matches known QA/test-traffic patterns + - dup_cluster / dup_count: exact (user_message, assistant_response) duplicate clusters + - trivial: joined from a relabeled JSONL's audit.trivial_user_message, if provided + - language: crude "en" (Latin-script) / "other" heuristic on the user message + +Tags are for analysts and downstream sampling/statistics only — judges never see them. + +Usage: + python scripts/curate_production_pairs.py \ + --input --output \ + [--relabeled ] [--report ] +""" +import argparse +import json +import re +from collections import Counter, defaultdict +from pathlib import Path + +SYNTHETIC_PATTERNS = [ + re.compile(r"automated testing", re.IGNORECASE), + re.compile(r"\btest message\b", re.IGNORECASE), +] + + +def is_synthetic(user_message: str) -> bool: + return any(p.search(user_message) for p in SYNTHETIC_PATTERNS) + + +def _is_latin(c: str) -> bool: + # Basic Latin through Latin Extended-B, plus Latin Extended Additional + # (covers Vietnamese and other precomposed Latin letters). + return ord(c) <= 0x024F or 0x1E00 <= ord(c) <= 0x1EFF + + +def language_tag(text: str) -> str: + """Rough Latin-script ("en") vs other-script ("other") heuristic — + not real language detection; accented Latin counts as "en".""" + letters = [c for c in text if c.isalpha()] + if not letters: + return "en" + non_latin = sum(1 for c in letters if not _is_latin(c)) + return "other" if non_latin / len(letters) > 0.3 else "en" + + +def curate(rows: list[dict], trivial_by_id: dict[str, bool] | None = None) -> Counter: + """Tag rows in place; returns a Counter of tag statistics.""" + stats = Counter(rows=len(rows)) + + # Non-string user/assistant fields (null in JSON, malformed exports) are + # tagged rather than crashing the run. + for row in rows: + if not isinstance(row.get("user_message"), str): + row["user_message"] = "" + stats["bad_user_message"] += 1 + if not isinstance(row.get("assistant_response"), str): + row["assistant_response"] = "" + stats["bad_assistant_response"] += 1 + + clusters: dict[tuple[str, str], list[dict]] = defaultdict(list) + for row in rows: + clusters[(row["user_message"], row["assistant_response"])].append(row) + + dup_id = 0 + for members in clusters.values(): + cluster = None + if len(members) > 1: + dup_id += 1 + cluster = f"dup-{dup_id:03d}" + stats["dup_clusters"] += 1 + stats["dup_rows"] += len(members) + for row in members: + row["curation"] = { + "synthetic_test": is_synthetic(row["user_message"]), + "dup_cluster": cluster, + "dup_count": len(members), + "trivial": (trivial_by_id or {}).get(row["sample_id"]), + "language": language_tag(row["user_message"]), + } + + for row in rows: + cur = row["curation"] + stats["synthetic_test"] += cur["synthetic_test"] + stats["trivial"] += cur["trivial"] is True + stats["trivial_unknown"] += cur["trivial"] is None + stats["non_english"] += cur["language"] != "en" + return stats + + +def load_trivial_labels(relabeled_path: Path) -> dict[str, bool]: + labels = {} + with open(relabeled_path) as f: + for line in f: + row = json.loads(line) + audit = row.get("audit") or {} + if "trivial_user_message" in audit: + labels[row["sample_id"]] = bool(audit["trivial_user_message"]) + return labels + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--relabeled", type=Path, help="relabeled JSONL to join trivial labels from") + parser.add_argument("--report", type=Path, help="write a markdown curation report here") + args = parser.parse_args() + + with open(args.input) as f: + rows = [json.loads(line) for line in f if line.strip()] + + trivial_by_id = load_trivial_labels(args.relabeled) if args.relabeled else None + stats = curate(rows, trivial_by_id) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + lines = [ + "# Curation report", + "", + f"- Input: `{args.input}`", + f"- Rows: {stats['rows']} (all kept — curation tags, never drops)", + f"- Synthetic/QA-test rows: {stats['synthetic_test']}", + f"- Exact-duplicate clusters: {stats['dup_clusters']} covering {stats['dup_rows']} rows", + f"- Trivial (from relabeled audit): {stats['trivial']}" + + (f" ({stats['trivial_unknown']} unlabeled)" if stats["trivial_unknown"] else ""), + f"- Non-English (heuristic): {stats['non_english']}", + f"- Non-string user_message/assistant_response fields: " + f"{stats['bad_user_message']}/{stats['bad_assistant_response']}", + ] + report = "\n".join(lines) + "\n" + print(report) + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(report) + + +if __name__ == "__main__": + main() diff --git a/scripts/select_comparison_subset.py b/scripts/select_comparison_subset.py new file mode 100644 index 00000000..5a5301a4 --- /dev/null +++ b/scripts/select_comparison_subset.py @@ -0,0 +1,230 @@ +"""Select a stratified subset of curated partner turns for a judge-comparison run. + +Strata (first match wins, in priority order; tagged as curation.subset_stratum): + worst — any principle judged -1.0 by the original judge (ALL included) + qa_test — synthetic/QA-test rows (ALL included). Their replies VARY, + so they probe the panel's false-alarm floor on known-mundane + traffic — they are NOT consistency probes. + repeated — assistant_response occurring >= --repeat-response-min times + (ALL occurrences included). Keyed on the response text + alone — the audit's identical-reply inconsistency exhibit — + so the *prompts* may differ across occurrences. + positive_extreme — any principle judged +1.0, none -1.0 (random sample). + Regression-to-the-mean control: an equally noisy second + judge regresses BOTH extreme tails toward the middle; + a genuinely better judge corrects asymmetrically + (concentrated on the over-escalated negative tail). + negative — some principle <= -0.5 but none -1.0 (random sample) + trivial — trivial-tagged rows not otherwise selected (random sample) + positive — no negative principle judgments (random sample) + +Because first-match-wins priority labels give wrong denominators for analysis +(e.g. most QA rows also sit in the worst stratum), every selected row also +carries boolean membership flags (curation.flags: is_worst, is_qa_test, +is_repeated_reply, is_positive_extreme, is_negative, is_trivial), and a +manifest JSON records per-stratum pool sizes and sampling fractions — required +for any reweighted/pooled estimate. Report per-stratum by default. + +A repeat slice re-judges selected turns a second time (sample_id suffixed +"__rep2") to measure panel self-consistency on identical inputs WITHIN one +eval run. Because within-run repeats share the provider load regime, they give +a lower bound on nondeterminism; a companion file (_between_run.jsonl, +"__rep3" ids) holds the same turns for a separate-day invocation to measure +between-run reliability. Selection is deterministic for a given --seed. + +The input must be the output of curate_production_pairs.py (curation tags are +required). Rows with no per-principle judgments are excluded (reported). + +Usage: + python scripts/select_comparison_subset.py \ + --input --output [--seed 42] \ + [--negative-sample 60] [--positive-sample 40] [--trivial-sample 30] \ + [--positive-extreme-sample 60] [--repeat-slice 50] \ + [--repeat-response-min 10] +""" +import argparse +import copy +import json +import random +from collections import Counter +from pathlib import Path + + +def severities(row: dict) -> list[float]: + return [p["severity"] for p in row["principles"].values()] + + +def worst_severity(row: dict) -> float: + return min(severities(row)) + + +def membership_flags(row: dict, repeated_reply: bool) -> dict: + sev = severities(row) + cur = row.get("curation") or {} + return { + "is_worst": min(sev) <= -1.0, + "is_negative": min(sev) <= -0.5, + "is_positive_extreme": max(sev) >= 1.0, + "is_qa_test": bool(cur.get("synthetic_test")), + "is_trivial": bool(cur.get("trivial")), + "is_repeated_reply": repeated_reply, + } + + +def select( + rows: list[dict], args: argparse.Namespace +) -> tuple[list[dict], Counter, dict]: + if not any("curation" in r for r in rows): + raise SystemExit( + "input has no curation tags — run curate_production_pairs.py " + "first (the qa_test and trivial strata depend on its tags)" + ) + + stats: Counter = Counter() + n_before = len(rows) + rows = [r for r in rows if r.get("principles")] + stats["excluded_no_judgments"] = n_before - len(rows) + + rng = random.Random(args.seed) + selected: dict[str, dict] = {} + manifest: dict = {"seed": args.seed, "population": len(rows), "strata": {}} + + response_counts = Counter(r["assistant_response"] for r in rows) + + def is_repeated(row: dict) -> bool: + return response_counts[row["assistant_response"]] >= args.repeat_response_min + + def take(row: dict, stratum: str) -> None: + if row["sample_id"] not in selected: + row = copy.deepcopy(row) + cur = row.setdefault("curation", {}) + cur["subset_stratum"] = stratum + cur["flags"] = membership_flags(row, is_repeated(row)) + selected[row["sample_id"]] = row + stats[stratum] += 1 + + def take_all(predicate, stratum: str) -> None: + pool = [r for r in rows if predicate(r)] + for row in pool: + take(row, stratum) + # All pool members are in the subset; some may carry a higher-priority + # stratum label — flags, not labels, give the true denominators. + manifest["strata"][stratum] = { + "pool": len(pool), + "selected": len(pool), + "labeled": stats[stratum], + "sampling_fraction": 1.0, + } + + def sample_from(predicate, n: int, stratum: str) -> None: + pool = [r for r in rows if predicate(r) and r["sample_id"] not in selected] + for row in rng.sample(pool, min(n, len(pool))): + take(row, stratum) + manifest["strata"][stratum] = { + "pool": len(pool), + "selected": stats[stratum], + "sampling_fraction": (stats[stratum] / len(pool)) if pool else 0.0, + } + + take_all(lambda r: worst_severity(r) <= -1.0, "worst") + take_all(lambda r: (r.get("curation") or {}).get("synthetic_test"), "qa_test") + take_all(is_repeated, "repeated") + sample_from( + lambda r: max(severities(r)) >= 1.0 and worst_severity(r) > -1.0, + args.positive_extreme_sample, + "positive_extreme", + ) + sample_from( + lambda r: -1.0 < worst_severity(r) <= -0.5, args.negative_sample, "negative" + ) + sample_from( + lambda r: (r.get("curation") or {}).get("trivial"), + args.trivial_sample, + "trivial", + ) + sample_from(lambda r: worst_severity(r) > -0.5, args.positive_sample, "positive") + + out = list(selected.values()) + + # Population-level flag totals (denominators for reweighted estimates). + manifest["population_flags"] = dict( + sum( + (Counter({k: int(v) for k, v in membership_flags(r, is_repeated(r)).items()}) + for r in rows), + Counter(), + ) + ) + manifest["selected_flags"] = dict( + sum( + (Counter({k: int(v) for k, v in r["curation"]["flags"].items()}) + for r in out), + Counter(), + ) + ) + + repeats = rng.sample(out, min(args.repeat_slice, len(out))) + between_run: list[dict] = [] + for row in repeats: + rep = copy.deepcopy(row) + rep["curation"]["repeat_of"] = rep["sample_id"] + rep["curation"]["subset_stratum"] = "repeat" + rep["sample_id"] = rep["sample_id"] + "__rep2" + out.append(rep) + stats["repeat"] += 1 + + between = copy.deepcopy(row) + between["curation"]["repeat_of"] = between["sample_id"] + between["curation"]["subset_stratum"] = "between_run_repeat" + between["sample_id"] = between["sample_id"] + "__rep3" + between_run.append(between) + + return out, stats, {"manifest": manifest, "between_run": between_run} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--negative-sample", type=int, default=60) + parser.add_argument("--positive-sample", type=int, default=40) + parser.add_argument("--trivial-sample", type=int, default=30) + parser.add_argument("--positive-extreme-sample", type=int, default=60) + parser.add_argument("--repeat-slice", type=int, default=50) + parser.add_argument("--repeat-response-min", type=int, default=10) + args = parser.parse_args() + + with open(args.input) as f: + rows = [json.loads(line) for line in f if line.strip()] + + out, stats, extras = select(rows, args) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + for row in out: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + manifest_path = args.output.with_suffix(".manifest.json") + manifest_path.write_text(json.dumps(extras["manifest"], indent=2) + "\n") + + between_path = args.output.with_name(args.output.stem + "_between_run.jsonl") + with open(between_path, "w") as f: + for row in extras["between_run"]: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + n_principles = 8 + n_judges = 3 + print(f"Selected {len(out)} rows (incl. repeats) from {len(rows)}:") + for stratum, count in sorted(stats.items()): + print(f" {stratum:24s} {count}") + print(f"Manifest (pools/fractions/flags): {manifest_path}") + print(f"Between-run repeat file (judge on a DIFFERENT day): {between_path}") + print( + f"Fan-out estimate: {len(out)} turns x {n_principles} principles = " + f"{len(out) * n_principles} samples; x {n_judges} judges = " + f"{len(out) * n_principles * n_judges} judge calls" + ) + + +if __name__ == "__main__": + main() diff --git a/src/bad_persona_task.py b/src/bad_persona_task.py index 14aded40..b6ee40b3 100644 --- a/src/bad_persona_task.py +++ b/src/bad_persona_task.py @@ -9,9 +9,9 @@ import textwrap from inspect_ai import Task, task -from inspect_ai.dataset import json_dataset, FieldSpec from inspect_ai.solver import system_message, generate +from dataset_source import humane_dataset from humanebench.scorer import overseer BAD_PERSONA_SYSTEM_PROMPT = textwrap.dedent("""\ @@ -85,17 +85,9 @@ """) @task -def bad_persona_eval(): +def bad_persona_eval(dataset: str | None = None): return Task( - dataset=json_dataset( - "../data/humane_bench.jsonl", - sample_fields=FieldSpec( - input="input", - target="target", - id="id", - metadata=["metadata"] - ) - ), + dataset=humane_dataset(dataset), solver=[ system_message(BAD_PERSONA_SYSTEM_PROMPT), generate() diff --git a/src/baseline_task.py b/src/baseline_task.py index 2a84de89..a7c2b41b 100644 --- a/src/baseline_task.py +++ b/src/baseline_task.py @@ -9,23 +9,15 @@ import textwrap from inspect_ai import Task, task -from inspect_ai.dataset import json_dataset, FieldSpec from inspect_ai.solver import system_message, generate +from dataset_source import humane_dataset from humanebench.scorer import overseer @task -def baseline_eval(): +def baseline_eval(dataset: str | None = None): return Task( - dataset=json_dataset( - "../data/humane_bench.jsonl", - sample_fields=FieldSpec( - input="input", - target="target", - id="id", - metadata=["metadata"] - ) - ), + dataset=humane_dataset(dataset), solver=[ generate() ], diff --git a/src/dataset_source.py b/src/dataset_source.py new file mode 100644 index 00000000..74e3f07e --- /dev/null +++ b/src/dataset_source.py @@ -0,0 +1,47 @@ +""" +Shared dataset loading for evaluation tasks. + +The default benchmark dataset (data/humane_bench.jsonl) is used unless an +explicit path is passed (e.g. `inspect eval ... -T dataset=/abs/path.jsonl`). +Explicit paths must be absolute: Inspect chdirs to src/ when loading task +files, so a relative path would silently resolve against src/ rather than +the caller's working directory. There is deliberately no environment-variable +override — a stale variable could silently redirect a benchmark run. +""" +import os + +from inspect_ai.dataset import json_dataset, FieldSpec + +# Resolves against src/ because Inspect chdirs to the task file's directory. +DEFAULT_DATASET = "../data/humane_bench.jsonl" + + +def humane_dataset(dataset: str | None = None): + """Load a HumaneBench-format JSONL dataset. + + Args: + dataset: Optional absolute path to a dataset file. When None, the + default benchmark dataset is used. + """ + if dataset is None: + path = DEFAULT_DATASET + else: + path = os.path.expanduser(dataset) + if not os.path.isabs(path): + raise ValueError( + f"dataset path must be absolute (got {dataset!r}): Inspect " + "loads task files with src/ as the working directory, so " + "relative paths resolve against src/, not where you ran the " + "command" + ) + if not os.path.exists(path): + raise FileNotFoundError(f"dataset file not found: {path}") + return json_dataset( + path, + sample_fields=FieldSpec( + input="input", + target="target", + id="id", + metadata=["metadata"] + ) + ) diff --git a/src/good_persona_task.py b/src/good_persona_task.py index aac4fb82..551fa18f 100644 --- a/src/good_persona_task.py +++ b/src/good_persona_task.py @@ -9,9 +9,9 @@ import textwrap from inspect_ai import Task, task -from inspect_ai.dataset import json_dataset, FieldSpec from inspect_ai.solver import system_message, generate +from dataset_source import humane_dataset from humanebench.scorer import overseer GOOD_PERSONA_SYSTEM_PROMPT = textwrap.dedent("""\ @@ -95,17 +95,9 @@ """) @task -def good_persona_eval(): +def good_persona_eval(dataset: str | None = None): return Task( - dataset=json_dataset( - "../data/humane_bench.jsonl", - sample_fields=FieldSpec( - input="input", - target="target", - id="id", - metadata=["metadata"] - ) - ), + dataset=humane_dataset(dataset), solver=[ system_message(GOOD_PERSONA_SYSTEM_PROMPT), generate() diff --git a/src/partner_rejudge_task.py b/src/partner_rejudge_task.py new file mode 100644 index 00000000..5ffde52c --- /dev/null +++ b/src/partner_rejudge_task.py @@ -0,0 +1,52 @@ +""" +Re-judging task for partner-provided prompt/response datasets. + +Partner datasets contain both the user prompt AND the partner assistant's +response (in metadata.ai_output). This task replays the stored response and +scores it with the standard overseer ensemble — no generation ever happens. +Run WITHOUT --model so any accidental generation attempt fails loudly: + + inspect eval src/partner_rejudge_task.py \ + -T dataset=/absolute/path/to/partner.jsonl \ + --log-dir ~/humanebench-partners//logs + +The dataset path is required and must be absolute — there is no default and +no environment-variable fallback, so the benchmark dataset can never be +judged by mistake and a stale variable can never redirect a run. +""" +import sys +from pathlib import Path + +# Add parent directory to path to import humanebench +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from inspect_ai import Task, task + +from dataset_source import humane_dataset +from humanebench.scorer import overseer +from pregenerated_solver import use_pregenerated_output_strict + + +@task +def partner_rejudge_eval(dataset: str | None = None): + if not dataset: + raise ValueError( + "partner_rejudge_eval requires a dataset path: pass " + "-T dataset=/abs/path.jsonl. See docs/partner-mode.md." + ) + return Task( + dataset=humane_dataset(dataset), + solver=[ + use_pregenerated_output_strict() # replay stored responses; never generate + ], + scorer=overseer( + humane_patterns=None, # Evaluate against all humane patterns + model=[ + "openrouter/anthropic/claude-4.5-sonnet", + "openrouter/openai/gpt-5.1", + "openrouter/google/gemini-2.5-pro" + ], + temperature=0.0, + score_attempts=3 + ) + ) diff --git a/tests/test_partner_scripts.py b/tests/test_partner_scripts.py new file mode 100644 index 00000000..d029cb6d --- /dev/null +++ b/tests/test_partner_scripts.py @@ -0,0 +1,254 @@ +"""Unit tests for the partner curation/conversion/subset scripts (synthetic data only).""" +import argparse +import copy +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) + +from curate_production_pairs import curate, is_synthetic, language_tag +from convert_partner_results import convert_row, has_judgeable_response, split_sample_id +from select_comparison_subset import select + +pytestmark = pytest.mark.unit + +SLUGS = [ + "respect-user-attention", + "enable-meaningful-choices", + "enhance-human-capabilities", + "protect-dignity-and-safety", + "foster-healthy-relationships", + "prioritize-long-term-wellbeing", + "be-transparent-and-honest", + "design-for-equity-and-inclusion", +] + + +def make_row(sample_id, user="What's 2+2?", assistant=None, severities=None): + # Unique default response per row so the repeated-response stratum only + # captures rows that deliberately share a response. + assistant = assistant if assistant is not None else f"It's 4. (reply to {sample_id})" + severities = severities or {} + return { + "sample_id": sample_id, + "ts": "2026-01-01", + "user": "u_1", + "conv": f"c_{sample_id}", + "turn_index": 1, + "user_message": user, + "assistant_response": assistant, + "principles": { + slug: { + "severity": severities.get(slug, 0.5), + "score": 0.75, + "relevant": True, + "reasoning": "fine", + } + for slug in SLUGS + }, + "relevant_principles": SLUGS[:3], + "overall_severity": 0.5, + "mean_severity": 0.5, + } + + +class TestCuration: + def test_synthetic_detection(self): + assert is_synthetic("Hello! This is a test message from automated testing.") + assert not is_synthetic("can you remind me to test my code tomorrow") + + def test_dup_clusters_and_stats(self): + rows = [ + make_row("a", user="hi", assistant="Hey!"), + make_row("b", user="hi", assistant="Hey!"), + make_row("c", user="hello", assistant="Hi there!"), + ] + stats = curate(rows) + assert stats["dup_clusters"] == 1 + assert stats["dup_rows"] == 2 + clusters = {r["sample_id"]: r["curation"]["dup_cluster"] for r in rows} + assert clusters["a"] == clusters["b"] is not None + assert clusters["c"] is None + + def test_trivial_join(self): + rows = [make_row("a"), make_row("b")] + stats = curate(rows, trivial_by_id={"a": True}) + assert rows[0]["curation"]["trivial"] is True + assert rows[1]["curation"]["trivial"] is None + assert stats["trivial"] == 1 + assert stats["trivial_unknown"] == 1 + + def test_language_tag(self): + assert language_tag("hello there") == "en" + assert language_tag("こんにちは、元気ですか") == "other" + # extended-Latin scripts are Latin, not "other" + assert language_tag("Xin chào, bạn khỏe không? Tiếng Việt đẹp") == "en" + assert language_tag("¿Qué tal? ¡Muy bien, señor!") == "en" + assert language_tag("12345 !!") == "en" + + def test_non_string_messages_tagged_not_crashed(self): + rows = [make_row("a"), make_row("b")] + rows[0]["user_message"] = None + rows[1]["assistant_response"] = None + stats = curate(rows) + assert stats["bad_user_message"] == 1 + assert stats["bad_assistant_response"] == 1 + assert rows[0]["curation"]["synthetic_test"] is False + + +class TestConversion: + def test_fanout_all_8(self): + samples = convert_row(make_row("s_1"), "all") + assert len(samples) == 8 + assert {s["target"] for s in samples} == set(SLUGS) + for s in samples: + assert s["metadata"]["ai_output"] == "It's 4. (reply to s_1)" + assert s["id"] == f"s_1__{s['target']}" + + def test_fanout_relevant_only(self): + samples = convert_row(make_row("s_1"), "relevant") + assert len(samples) == 3 + assert {s["target"] for s in samples} == set(SLUGS[:3]) + + def test_unknown_slug_rejected(self): + row = make_row("s_1") + row["principles"]["not-a-principle"] = row["principles"][SLUGS[0]] + with pytest.raises(ValueError, match="s_1"): + convert_row(row, "all") + + def test_unknown_relevant_slug_rejected(self): + row = make_row("s_1") + row["relevant_principles"] = [SLUGS[0], "be-transparent-honest"] + with pytest.raises(ValueError, match="relevant_principles"): + convert_row(row, "relevant") + + def test_empty_response_not_judgeable(self): + assert has_judgeable_response(make_row("s_1")) + for bad in ["", None, 42]: + row = make_row("s_1") + row["assistant_response"] = bad + assert not has_judgeable_response(row) + + def test_id_roundtrip_with_double_underscore(self): + for sid in ["s_1", "s_1__rep2", "weird__id__x"]: + hb_id = f"{sid}__{SLUGS[0]}" + assert split_sample_id(hb_id) == (sid, SLUGS[0]) + + +def subset_args(**overrides): + defaults = dict( + seed=42, + negative_sample=2, + positive_sample=2, + trivial_sample=2, + positive_extreme_sample=2, + repeat_slice=3, + repeat_response_min=10, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + +class TestSubsetSelection: + def make_pool(self): + rows = [] + for i in range(5): # worst stratum + rows.append(make_row(f"w{i}", severities={SLUGS[0]: -1.0})) + for i in range(10): # negative stratum + rows.append(make_row(f"n{i}", severities={SLUGS[1]: -0.5})) + for i in range(10): # positive (default severities are all +0.5) + rows.append(make_row(f"p{i}")) + for i in range(4): # positive-extreme stratum (a +1.0, no -1.0) + rows.append(make_row(f"x{i}", severities={SLUGS[2]: 1.0})) + for i in range(12): # repeated identical response + rows.append(make_row(f"r{i}", user=f"u{i}", assistant="Hey! How's it going?")) + rows.append( + make_row("t0", user="Hello! This is a test message from automated testing.") + ) + curate(rows, trivial_by_id={"p0": True, "p1": True, "p2": True}) + return rows + + def test_strata(self): + out, stats, extras = select(self.make_pool(), subset_args()) + assert stats["worst"] == 5 # all -1.0 rows included + assert stats["qa_test"] == 1 + assert stats["repeated"] == 12 + assert stats["positive_extreme"] == 2 + assert stats["negative"] == 2 + assert stats["trivial"] == 2 + assert stats["positive"] == 2 + assert stats["repeat"] == 3 + reps = [r for r in out if r["curation"]["subset_stratum"] == "repeat"] + assert all(r["sample_id"].endswith("__rep2") for r in reps) + assert all(r["curation"]["repeat_of"] + "__rep2" == r["sample_id"] for r in reps) + + def test_membership_flags_independent_of_stratum_label(self): + # A QA row that ALSO qualifies for the worst stratum gets labeled + # "worst" (priority), but its is_qa_test flag must still be true. + rows = [ + make_row( + "qa_worst", + user="Hello! This is a test message from automated testing.", + severities={SLUGS[0]: -1.0}, + ), + make_row("plain"), + ] + curate(rows) + out, stats, extras = select(rows, subset_args(repeat_slice=0)) + qa = next(r for r in out if r["sample_id"] == "qa_worst") + assert qa["curation"]["subset_stratum"] == "worst" + assert qa["curation"]["flags"]["is_qa_test"] is True + assert qa["curation"]["flags"]["is_worst"] is True + assert extras["manifest"]["selected_flags"]["is_qa_test"] == 1 + + def test_manifest_records_pools_and_fractions(self): + out, stats, extras = select(self.make_pool(), subset_args()) + m = extras["manifest"] + assert m["strata"]["worst"]["sampling_fraction"] == 1.0 + neg = m["strata"]["negative"] + assert neg["selected"] == 2 and neg["pool"] == 10 + assert abs(neg["sampling_fraction"] - 0.2) < 1e-9 + assert m["population"] == len(self.make_pool()) + assert m["population_flags"]["is_worst"] == 5 + + def test_between_run_repeats_mirror_repeat_slice(self): + out, stats, extras = select(self.make_pool(), subset_args()) + between = extras["between_run"] + assert len(between) == stats["repeat"] == 3 + assert all(r["sample_id"].endswith("__rep3") for r in between) + rep2_bases = { + r["curation"]["repeat_of"] + for r in out + if r["curation"]["subset_stratum"] == "repeat" + } + rep3_bases = {r["curation"]["repeat_of"] for r in between} + assert rep2_bases == rep3_bases + + def test_deterministic_for_seed(self): + pool = self.make_pool() + out1, _, _ = select(copy.deepcopy(pool), subset_args()) + out2, _, _ = select(copy.deepcopy(pool), subset_args()) + assert [r["sample_id"] for r in out1] == [r["sample_id"] for r in out2] + + def test_no_double_selection(self): + out, _, _ = select(self.make_pool(), subset_args()) + non_repeat = [r["sample_id"] for r in out if not r["sample_id"].endswith("__rep2")] + assert len(non_repeat) == len(set(non_repeat)) + + def test_uncurated_input_rejected(self): + rows = [make_row("a"), make_row("b")] # no curate() pass + with pytest.raises(SystemExit, match="curation"): + select(rows, subset_args()) + + def test_rows_without_judgments_excluded(self): + pool = self.make_pool() + bare = make_row("nojudge") + bare["principles"] = {} + pool.append(bare) + curate([bare]) + out, stats, _ = select(pool, subset_args()) + assert stats["excluded_no_judgments"] == 1 + assert all(r["sample_id"] != "nojudge" for r in out)