Partner mode: re-judge external prompt/response datasets

.gitignore

@@ -255,3 +255,9 @@ evaluator/workshop/results.jsonl
 # Large per-judge raw severity table (13 MB, regeneratable via
 # scripts/compute_inter_judge_agreement.py)
 tables/inter_judge_raw.csv
+
+# Partner-mode belt and braces: partner data lives outside the repo
+# (~/humanebench-partners/), these patterns guard against accidents
+humanebench-partners/
+partner_*.jsonl
+*.eval

docs/partner-mode.md

@@ -0,0 +1,68 @@
+# Partner mode: re-judging external prompt/response datasets
+
+How to run the HumaneBench judge panel on a partner's dataset of **stored
+prompt/response pairs** (no generation — the stored response is replayed and
+judged).
+
+## Ground rules
+
+- **Partner data never enters git.** Keep datasets, `.eval` logs, and results
+  in a workspace outside the repo: `~/humanebench-partners/<partner>/`.
+  `.eval` logs embed every prompt, response, and judge rationale — treat them
+  as sensitive as the raw dataset.
+- **Provider egress:** judging sends each prompt + response to the three
+  judge-model providers via OpenRouter. Before any run, the OpenRouter
+  account must be set to **exclude providers that may train on inputs**, and
+  the partner must have signed off on third-party judging.
+- Even anonymized messages can contain self-disclosed details (names,
+  locations). Re-check verbatim quotes before circulating analysis documents.
+
+## Dataset format
+
+HumaneBench-style JSONL, one sample per (turn, principle):
+
+```json
+{"id": "<sample_id>__<principle-slug>",
+ "input": "<user message>",
+ "target": "<principle-slug>",
+ "metadata": {"ai_output": "<the stored assistant response>"}}
+```
+
+`target` must be one of the 8 principle slugs in
+`humanebench/humane_patterns.py`. `metadata.ai_output` is the contract key
+read by the strict replay solver (`src/pregenerated_solver.py`) and must be a
+non-empty string. `scripts/convert_partner_results.py` produces this format
+from per-turn judged-results files, fanning each turn out across principles.
+
+## Pipeline
+
+1. **Curate** — `scripts/curate_production_pairs.py`: tags rows
+   (synthetic/QA traffic, duplicate clusters, trivial labels, language)
+   without dropping anything. Tags are analyst-visible only; judges never see
+   them.
+2. **Select** (optional, for comparison runs) —
+   `scripts/select_comparison_subset.py`: deterministic stratified subset
+   incl. a repeat slice for panel self-consistency. Requires curated input.
+3. **Convert** — `scripts/convert_partner_results.py`: fan out to the format
+   above. Turns with empty responses are skipped loudly.
+4. **Judge** — run **without `--model`** (nothing can generate; the NoModel
+   errors loudly if anything tries), dataset path **absolute**:
+
+   ```
+   inspect eval src/partner_rejudge_task.py \
+       -T dataset=/absolute/path/to/converted.jsonl \
+       --log-dir ~/humanebench-partners/<partner>/logs
+   ```
+
+   Pilot with `--limit 10` and verify cost before a full run.
+
+## Notes
+
+- The benchmark tasks (`baseline`, `good_persona`, `bad_persona`) also accept
+  `-T dataset=` for ad-hoc runs; there is deliberately **no environment
+  variable override** — a stale variable could silently redirect a benchmark
+  run.
+- A HumaneScore computed over a partner dataset that does not cover all 8
+  principles is **not comparable** to benchmark HumaneScores (empty
+  principles are zero-filled into the denominator). Report per-principle
+  results with their sample counts.

scripts/convert_partner_results.py

@@ -0,0 +1,124 @@
+"""Convert a partner production-results JSONL to HumaneBench re-judge format.
+
+Each input row (one conversation turn with a stored assistant response and
+per-principle judgments) fans out to one sample per principle:
+
+    {"id": "<sample_id>__<principle-slug>", "input": <user_message>,
+     "target": <principle-slug>,
+     "metadata": {"ai_output": <assistant_response>, ...}}
+
+`metadata.ai_output` is the contract key read by
+src/pregenerated_solver.py:use_pregenerated_output_strict(); the original
+judgments and curation tags ride along in metadata for post-run comparison and
+are never shown to judges (the overseer only sees input + ai_output).
+
+Usage:
+    python scripts/convert_partner_results.py \
+        --input <curated.jsonl> --output <hb_rejudge.jsonl> \
+        [--principles {all,relevant}]
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from humanebench.humane_patterns import HUMANE_PATTERNS
+
+KNOWN_SLUGS = frozenset(HUMANE_PATTERNS.keys())
+
+
+def has_judgeable_response(row: dict) -> bool:
+    """The strict replay solver treats falsy ai_output as missing and raises,
+    so only turns with a non-empty string response can be re-judged."""
+    resp = row.get("assistant_response")
+    return isinstance(resp, str) and bool(resp)
+
+
+def convert_row(row: dict, principles_mode: str) -> list[dict]:
+    unknown = set(row["principles"]) - KNOWN_SLUGS
+    if unknown:
+        raise ValueError(
+            f"sample {row['sample_id']}: unknown principle slug(s) {sorted(unknown)}"
+        )
+
+    if principles_mode == "relevant":
+        slugs = row.get("relevant_principles", [])
+        unknown = set(slugs) - KNOWN_SLUGS
+        if unknown:
+            raise ValueError(
+                f"sample {row['sample_id']}: unknown slug(s) in "
+                f"relevant_principles {sorted(unknown)}"
+            )
+    else:
+        slugs = sorted(row["principles"])
+
+    samples = []
+    for slug in slugs:
+        samples.append(
+            {
+                "id": f"{row['sample_id']}__{slug}",
+                "input": row["user_message"],
+                "target": slug,
+                "metadata": {
+                    "ai_output": row["assistant_response"],
+                    "conv": row.get("conv"),
+                    "turn_index": row.get("turn_index"),
+                    "ts": row.get("ts"),
+                    "curation": row.get("curation"),
+                    "orig_judgment": row["principles"].get(slug),
+                    "orig_overall_severity": row.get("overall_severity"),
+                    "orig_mean_severity": row.get("mean_severity"),
+                    "audit": (row.get("audit") or {}).get("principles", {}).get(slug),
+                },
+            }
+        )
+    return samples
+
+
+def split_sample_id(hb_id: str) -> tuple[str, str]:
+    """Recover (sample_id, slug) from an HB fan-out id; safe for ids containing '__'."""
+    sample_id, slug = hb_id.rsplit("__", 1)
+    return sample_id, slug
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--principles", choices=["all", "relevant"], default="all")
+    args = parser.parse_args()
+
+    n_rows = 0
+    n_samples = 0
+    skipped_empty: list[str] = []
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.input) as fin, open(args.output, "w") as fout:
+        for line in fin:
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            n_rows += 1
+            if not has_judgeable_response(row):
+                skipped_empty.append(row.get("sample_id", "<no id>"))
+                continue
+            for sample in convert_row(row, args.principles):
+                fout.write(json.dumps(sample, ensure_ascii=False) + "\n")
+                n_samples += 1
+
+    print(
+        f"Converted {n_rows} turns -> {n_samples} samples "
+        f"(mode={args.principles}) into {args.output}"
+    )
+    if skipped_empty:
+        print(
+            f"WARNING: skipped {len(skipped_empty)} turn(s) with empty/missing "
+            f"assistant_response (cannot be re-judged by the strict replay "
+            f"solver): {', '.join(skipped_empty[:20])}"
+            + (" ..." if len(skipped_empty) > 20 else "")
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/curate_production_pairs.py

@@ -0,0 +1,143 @@
+"""Curate a partner production-results JSONL by tagging rows (never dropping them).
+
+Adds a "curation" object to each row:
+  - synthetic_test: user message matches known QA/test-traffic patterns
+  - dup_cluster / dup_count: exact (user_message, assistant_response) duplicate clusters
+  - trivial: joined from a relabeled JSONL's audit.trivial_user_message, if provided
+  - language: crude "en" (Latin-script) / "other" heuristic on the user message
+
+Tags are for analysts and downstream sampling/statistics only — judges never see them.
+
+Usage:
+    python scripts/curate_production_pairs.py \
+        --input <partner_results.jsonl> --output <curated.jsonl> \
+        [--relabeled <relabeled.jsonl>] [--report <report.md>]
+"""
+import argparse
+import json
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+
+SYNTHETIC_PATTERNS = [
+    re.compile(r"automated testing", re.IGNORECASE),
+    re.compile(r"\btest message\b", re.IGNORECASE),
+]
+
+
+def is_synthetic(user_message: str) -> bool:
+    return any(p.search(user_message) for p in SYNTHETIC_PATTERNS)
+
+
+def _is_latin(c: str) -> bool:
+    # Basic Latin through Latin Extended-B, plus Latin Extended Additional
+    # (covers Vietnamese and other precomposed Latin letters).
+    return ord(c) <= 0x024F or 0x1E00 <= ord(c) <= 0x1EFF
+
+
+def language_tag(text: str) -> str:
+    """Rough Latin-script ("en") vs other-script ("other") heuristic —
+    not real language detection; accented Latin counts as "en"."""
+    letters = [c for c in text if c.isalpha()]
+    if not letters:
+        return "en"
+    non_latin = sum(1 for c in letters if not _is_latin(c))
+    return "other" if non_latin / len(letters) > 0.3 else "en"
+
+
+def curate(rows: list[dict], trivial_by_id: dict[str, bool] | None = None) -> Counter:
+    """Tag rows in place; returns a Counter of tag statistics."""
+    stats = Counter(rows=len(rows))
+
+    # Non-string user/assistant fields (null in JSON, malformed exports) are
+    # tagged rather than crashing the run.
+    for row in rows:
+        if not isinstance(row.get("user_message"), str):
+            row["user_message"] = ""
+            stats["bad_user_message"] += 1
+        if not isinstance(row.get("assistant_response"), str):
+            row["assistant_response"] = ""
+            stats["bad_assistant_response"] += 1
+
+    clusters: dict[tuple[str, str], list[dict]] = defaultdict(list)
+    for row in rows:
+        clusters[(row["user_message"], row["assistant_response"])].append(row)
+
+    dup_id = 0
+    for members in clusters.values():
+        cluster = None
+        if len(members) > 1:
+            dup_id += 1
+            cluster = f"dup-{dup_id:03d}"
+            stats["dup_clusters"] += 1
+            stats["dup_rows"] += len(members)
+        for row in members:
+            row["curation"] = {
+                "synthetic_test": is_synthetic(row["user_message"]),
+                "dup_cluster": cluster,
+                "dup_count": len(members),
+                "trivial": (trivial_by_id or {}).get(row["sample_id"]),
+                "language": language_tag(row["user_message"]),
+            }
+
+    for row in rows:
+        cur = row["curation"]
+        stats["synthetic_test"] += cur["synthetic_test"]
+        stats["trivial"] += cur["trivial"] is True
+        stats["trivial_unknown"] += cur["trivial"] is None
+        stats["non_english"] += cur["language"] != "en"
+    return stats
+
+
+def load_trivial_labels(relabeled_path: Path) -> dict[str, bool]:
+    labels = {}
+    with open(relabeled_path) as f:
+        for line in f:
+            row = json.loads(line)
+            audit = row.get("audit") or {}
+            if "trivial_user_message" in audit:
+                labels[row["sample_id"]] = bool(audit["trivial_user_message"])
+    return labels
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--relabeled", type=Path, help="relabeled JSONL to join trivial labels from")
+    parser.add_argument("--report", type=Path, help="write a markdown curation report here")
+    args = parser.parse_args()
+
+    with open(args.input) as f:
+        rows = [json.loads(line) for line in f if line.strip()]
+
+    trivial_by_id = load_trivial_labels(args.relabeled) if args.relabeled else None
+    stats = curate(rows, trivial_by_id)
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.output, "w") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    lines = [
+        "# Curation report",
+        "",
+        f"- Input: `{args.input}`",
+        f"- Rows: {stats['rows']} (all kept — curation tags, never drops)",
+        f"- Synthetic/QA-test rows: {stats['synthetic_test']}",
+        f"- Exact-duplicate clusters: {stats['dup_clusters']} covering {stats['dup_rows']} rows",
+        f"- Trivial (from relabeled audit): {stats['trivial']}"
+        + (f" ({stats['trivial_unknown']} unlabeled)" if stats["trivial_unknown"] else ""),
+        f"- Non-English (heuristic): {stats['non_english']}",
+        f"- Non-string user_message/assistant_response fields: "
+        f"{stats['bad_user_message']}/{stats['bad_assistant_response']}",
+    ]
+    report = "\n".join(lines) + "\n"
+    print(report)
+    if args.report:
+        args.report.parent.mkdir(parents=True, exist_ok=True)
+        args.report.write_text(report)
+
+
+if __name__ == "__main__":
+    main()