From a85da311755c575d3cd9a24a8cfde51d4d0ba81c Mon Sep 17 00:00:00 2001 From: XuanL17 Date: Sat, 8 Aug 2026 22:58:38 +0800 Subject: [PATCH 1/2] fix: load study_006 materials from the source directory study_006 overrides load_material but resolves it against the study root instead of source/, so every material lookup missed and the study could not be loaded at all. Co-Authored-By: Claude Opus 5 --- studies/study_006/scripts/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/studies/study_006/scripts/config.py b/studies/study_006/scripts/config.py index c813621..c44215e 100644 --- a/studies/study_006/scripts/config.py +++ b/studies/study_006/scripts/config.py @@ -129,13 +129,13 @@ def load_material(self, sub_study_id: str) -> Dict[str, Any]: ] for filename in possible_names: - file_path = self.study_path / "materials" / filename + file_path = self.source_path / "materials" / filename if file_path.exists(): with open(file_path, "r", encoding='utf-8') as f: return json.load(f) # If none found, raise error with all attempted paths - attempted = [str(self.study_path / "materials" / name) for name in possible_names] + attempted = [str(self.source_path / "materials" / name) for name in possible_names] raise FileNotFoundError(f"Material not found for '{sub_study_id}'. Tried: {attempted}") def create_trials(self, n_trials: int = None) -> List[Dict[str, Any]]: From dd7683e3f62b6eafdcd2b4f8aa53e2ce13268f7d Mon Sep 17 00:00:00 2001 From: XuanL17 Date: Sat, 8 Aug 2026 22:59:05 +0800 Subject: [PATCH 2/2] feat: replay a single study with any model in the playground Researchers can pick a study, a base model, and the prompt their agent takes into the experiment, then see where the agent behaved like the original participants and where it did not. The runner drives each study's own trial builder and evaluator, so a playground run is scored exactly like a benchmark run; the model, the participant prompt, and the participant count are what change. Every model is routed through OpenRouter. Results are charted and interpreted by Claude Code against the contract in playground/CLAUDE.md. Its output is validated on the runner before it is stored, and anything invalid or late falls back to deterministic charts, so a run always ends with charts a researcher can read. Runs on the shared key are capped. A researcher can supply their own OpenRouter key for a larger budget; the web app seals it with AES-256-GCM under PLAYGROUND_KEY_SECRET, so the private jobs repository only ever holds ciphertext, and the workflow drops it when the run ends. Co-Authored-By: Claude Opus 5 --- .github/workflows/run-playground.yml | 190 ++++++++++++++ README.md | 19 ++ playground/README.md | 87 +++++++ playground/__init__.py | 1 + playground/analysis.py | 219 +++++++++++++++++ playground/default_charts.py | 152 ++++++++++++ playground/progress.py | 85 +++++++ playground/run_charts.py | 157 ++++++++++++ playground/run_key.py | 49 ++++ playground/run_playground.py | 354 +++++++++++++++++++++++++++ playground/settings.py | 32 +++ playground/study_loader.py | 99 ++++++++ playground/validate_charts.py | 120 +++++++++ requirements.txt | 3 + tests/test_playground.py | 157 ++++++++++++ 15 files changed, 1724 insertions(+) create mode 100644 .github/workflows/run-playground.yml create mode 100644 playground/README.md create mode 100644 playground/__init__.py create mode 100644 playground/analysis.py create mode 100644 playground/default_charts.py create mode 100644 playground/progress.py create mode 100644 playground/run_charts.py create mode 100644 playground/run_key.py create mode 100644 playground/run_playground.py create mode 100644 playground/settings.py create mode 100644 playground/study_loader.py create mode 100644 playground/validate_charts.py create mode 100644 tests/test_playground.py diff --git a/.github/workflows/run-playground.yml b/.github/workflows/run-playground.yml new file mode 100644 index 0000000..28f8bf4 --- /dev/null +++ b/.github/workflows/run-playground.yml @@ -0,0 +1,190 @@ +name: Run a playground study + +on: + workflow_dispatch: + inputs: + run_id: + description: Private jobs repository run ID + required: true + type: string + jobs_repo: + description: Private repository containing run inputs and outputs + required: true + default: HumanStudy-Hub/humanstudy-hub-jobs + type: string + +permissions: + contents: read + +concurrency: + group: humanstudy-playground-${{ inputs.run_id }} + cancel-in-progress: true + +jobs: + run-study: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out the benchmark + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out private run storage + uses: actions/checkout@v4 + with: + repository: ${{ inputs.jobs_repo }} + token: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + path: run-storage + ref: runs/${{ inputs.run_id }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Set up Node for the charting agent + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install numpy scipy pandas tqdm openai cryptography + npm install --global @anthropic-ai/claude-code@latest + + - name: Mark run running + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json" + data = json.loads(path.read_text()) + data.update({"status": "running", "message": "Participants are running the study"}) + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + - name: Publish running state + working-directory: run-storage + env: + RUN_ID: ${{ inputs.run_id }} + TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + run: | + git config user.name "HumanStudy-Hub Playground" + git config user.email "playground@humanstudy-hub.org" + git add "runs/$RUN_ID/run.json" + git commit -m "playground: start $RUN_ID" + git push "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "HEAD:runs/$RUN_ID" + + - name: Run the study + env: + RUN_ID: ${{ inputs.run_id }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PLAYGROUND_KEY_SECRET: ${{ secrets.PLAYGROUND_KEY_SECRET }} + PIPELINE_PROGRESS_TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + run: | + python playground/run_playground.py \ + --run "$GITHUB_WORKSPACE/run-storage/runs/$RUN_ID" \ + --progress-repo "${{ inputs.jobs_repo }}" \ + --progress-branch "runs/$RUN_ID" \ + --progress-path "runs/$RUN_ID/progress.json" + + # Charts and the written reading are produced by Claude Code through + # OpenRouter, then validated. This step never fails the run: an unusable + # chart set falls back to the deterministic charts. + - name: Chart and interpret the results + if: success() + continue-on-error: true + env: + RUN_ID: ${{ inputs.run_id }} + ANTHROPIC_BASE_URL: https://openrouter.ai/api + CHARTS_AGENT_TOKEN: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: "" + ANTHROPIC_DEFAULT_OPUS_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + ANTHROPIC_DEFAULT_SONNET_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + ANTHROPIC_DEFAULT_HAIKU_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + OPENROUTER_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + CHARTS_TIMEOUT_MINUTES: ${{ vars.PLAYGROUND_CHARTS_TIMEOUT_MINUTES || '8' }} + run: | + python playground/run_charts.py \ + --run "$GITHUB_WORKSPACE/run-storage/runs/$RUN_ID" \ + --model "$OPENROUTER_MODEL" \ + --timeout-minutes "$CHARTS_TIMEOUT_MINUTES" + + - name: Mark run complete + if: success() + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + run_dir = Path("run-storage/runs") / os.environ["RUN_ID"] + charts = run_dir / "output" / "charts.json" + if not charts.exists(): + import subprocess, sys + subprocess.run([ + sys.executable, "playground/default_charts.py", + "--analysis", str(run_dir / "output" / "analysis.json"), + "--output", str(charts), + ], check=True) + path = run_dir / "run.json" + data = json.loads(path.read_text()) + data.update({ + "status": "complete", + "message": "The run finished and the results are ready", + "resultsReady": True, + }) + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + - name: Mark failed or cancelled + if: failure() || cancelled() + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json" + data = json.loads(path.read_text()) + if data.get("status") != "failed": + data.update({"status": "failed", "message": "The playground run failed or was cancelled"}) + data["resultsReady"] = False + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + # The researcher's key only ever existed as ciphertext, and it is dropped + # here so a finished run keeps no copy of it at all. + - name: Drop the supplied key + if: always() + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json" + data = json.loads(path.read_text()) + if data.pop("sealedApiKey", None) is not None: + data["usedOwnKey"] = True + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + - name: Commit run state and outputs + if: always() + working-directory: run-storage + env: + RUN_ID: ${{ inputs.run_id }} + TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + run: | + git config user.name "HumanStudy-Hub Playground" + git config user.email "playground@humanstudy-hub.org" + git add "runs/$RUN_ID" + git commit -m "playground: results for $RUN_ID" || true + git pull --rebase "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "runs/$RUN_ID" + git push "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "HEAD:runs/$RUN_ID" diff --git a/README.md b/README.md index 08a8bca..0739eba 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,30 @@ missing-information checklist. The older staged Python implementation remains in `generation_pipeline/` for reference and migration tests, but the web builder no longer dispatches it. +## Playground + +The playground replays a single study with a model of your choice and scores it +against the paper, so you can see where an agent behaves like the original +participants and where it does not. + +```bash +OPENROUTER_API_KEY=... python playground/run_playground.py --run +``` + +It drives each study's own trial builder and evaluator, so a playground run is +scored exactly like a benchmark run; the model, the participant prompt, and the +number of participants are what you change. Results are charted and interpreted +by Claude Code, with deterministic charts as the fallback. HumanStudy-Hub +dispatches the same code through +[`run-playground.yml`](.github/workflows/run-playground.yml). See +[`playground/README.md`](playground/README.md). + ## Repository structure ```text studies/ Runnable benchmark studies src/ Execution and evaluation code +playground/ On-demand single-study runs and result comparison agent_pipeline/ Claude Code package contract and validation generation_pipeline/ Legacy staged extraction implementation scripts/ Study validation and index utilities diff --git a/playground/README.md b/playground/README.md new file mode 100644 index 0000000..06f0066 --- /dev/null +++ b/playground/README.md @@ -0,0 +1,87 @@ +# Playground + +Replay one benchmark study with a model of your choice and see where the agent +matched the published human result and where it did not. + +This is the engine behind the HumanStudy-Hub playground. It does not +reimplement any study: it drives each study's own `scripts/config.py` to build +trials and its own `scripts/evaluator.py` to score the run, so a playground run +is scored exactly like a benchmark run. + +## Run it locally + +```bash +python playground/run_playground.py --run +``` + +The run directory holds `run.json` and receives `progress.json`, `logs/`, and +`output/`: + +```json +{ + "id": "local-1", + "studyId": "study_001", + "model": "openai/gpt-4o-mini", + "preset": "v3_human_plus_demo", + "systemPrompt": "", + "participantsPerScenario": 8, + "temperature": 1.0, + "seed": 42, + "demographics": { "age": 21, "gender": "female", "background": "undergraduate" } +} +``` + +`OPENROUTER_API_KEY` must be set. Every model — including Claude and Gemini +names — is routed through OpenRouter. Add `--simulate` to exercise the whole +path with simulated participants and no API calls. + +Outputs: + +| File | Contents | +|---|---| +| `output/responses.json` | Every participant response, in the shape evaluators read | +| `output/evaluation.json` | The study evaluator's own output | +| `output/analysis.json` | One row per statistical test: human result vs. agent result | +| `output/transcript_sample.json` | A few complete participant exchanges | +| `output/charts.json` | Plotly chart specifications and a written reading of the run | + +## Prompts + +`preset` selects a shipped participant prompt from +`src/agents/custom_methods/`: `v1_empty`, `v2_human`, `v3_human_plus_demo`, or +`v4_background`. Set `preset` to `custom` and supply `systemPrompt` to run a +prompt you wrote yourself. `demographics` overrides the sampled participant +profile — age, gender, education, background, population, persona — which is +what the demographic presets read. + +## Charts + +```bash +python playground/run_charts.py --run +``` + +Claude Code reads the finished run and writes `output/charts.json` following +[`CLAUDE.md`](CLAUDE.md). Its output is checked by `validate_charts.py` before +it is kept: charts must be plain JSON, restricted to an allowlist of trace +types. Anything invalid, missing, or late falls back to the deterministic charts +in `default_charts.py`, so a run always ends with readable charts. + +## Run budget + +A run on the shared HumanStudy-Hub key is capped at 60 participant sessions. +A researcher who supplies their own OpenRouter key gets 600. The web app seals +that key with AES-256-GCM under `PLAYGROUND_KEY_SECRET`, so the private jobs +repository only ever holds ciphertext, and the workflow drops it once the run +ends. + +## In CI + +[`run-playground.yml`](../.github/workflows/run-playground.yml) is dispatched by +HumanStudy-Hub. Repository settings it needs: + +| Type | Name | Value | +|---|---|---| +| Actions secret | `OPENROUTER_API_KEY` | OpenRouter key for shared runs and the charting agent | +| Actions secret | `HUMANSTUDY_PIPELINE_TOKEN` | Token with read/write access to the private jobs repository | +| Actions secret | `PLAYGROUND_KEY_SECRET` | Shared secret that opens a researcher's own key; must match the web app | +| Actions variable | `PLAYGROUND_CHARTS_MODEL` | Optional; defaults to `anthropic/claude-sonnet-5` | diff --git a/playground/__init__.py b/playground/__init__.py new file mode 100644 index 0000000..ef4c006 --- /dev/null +++ b/playground/__init__.py @@ -0,0 +1 @@ +"""Playground: run a benchmark study against a chosen model on demand.""" diff --git a/playground/analysis.py b/playground/analysis.py new file mode 100644 index 0000000..847383d --- /dev/null +++ b/playground/analysis.py @@ -0,0 +1,219 @@ +"""Turn a study evaluator's output into one comparison table the web app can plot. + +Study evaluators do not agree on a single shape. Most of them enrich their test +results through `stats_lib.add_statistical_replication_fields` and already carry +human and agent effect sizes; a few return raw t-tests. This module normalises +both into the same rows: one row per statistical test, with the human result and +the agent result side by side. +""" + +import json +import math +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +Number = Optional[float] + + +def _number(value: Any) -> Number: + if value is None or isinstance(value, bool): + return None + try: + result = float(value) + except (TypeError, ValueError): + return None + return None if math.isnan(result) or math.isinf(result) else result + + +def _flag(value: Any) -> Optional[bool]: + return bool(value) if isinstance(value, bool) else None + + +def _first(row: Dict[str, Any], *keys: str) -> Any: + for key in keys: + if row.get(key) is not None: + return row[key] + return None + + +def human_effect_from_reported(reported: str) -> Number: + """Convert a reported test statistic into a Cohen's d equivalent. + + Papers report the statistic, not the effect size, so the published + `F(1, df)`, `t(df)`, or `r(df)` is converted with the standard identities. + Anything else (chi-square, multi-numerator F) has no single-d equivalent and + is left out of the effect-size comparison. + """ + if not reported: + return None + text = str(reported) + t_match = re.search(r"\bt\s*\(\s*(\d+(?:\.\d+)?)\s*\)\s*=\s*(-?\d+(?:\.\d+)?)", text) + if t_match: + df, statistic = float(t_match.group(1)), float(t_match.group(2)) + return (2 * statistic / math.sqrt(df)) if df > 0 else None + f_match = re.search(r"\bF\s*\(\s*(\d+)\s*,\s*(\d+(?:\.\d+)?)\s*\)\s*=\s*(-?\d+(?:\.\d+)?)", text) + if f_match: + df1, df2, statistic = int(f_match.group(1)), float(f_match.group(2)), float(f_match.group(3)) + if df1 == 1 and df2 > 0 and statistic >= 0: + return 2 * math.sqrt(statistic / df2) + return None + r_match = re.search(r"\br\s*\(\s*(\d+)\s*\)\s*=\s*(-?\d+(?:\.\d+)?)", text) + if r_match: + correlation = float(r_match.group(2)) + denominator = 1 - correlation ** 2 + return (2 * correlation / math.sqrt(denominator)) if denominator > 0 else None + return None + + +def agent_effect_from_t(t_statistic: Number, n1: Number, n2: Number) -> Number: + """Cohen's d for an independent-samples t-test.""" + if t_statistic is None or not n1: + return None + if n2: + return t_statistic * math.sqrt(1 / n1 + 1 / n2) + return t_statistic / math.sqrt(n1) + + +def _ground_truth_index(study_path: Path) -> Dict[tuple, Dict[str, Any]]: + """Map (study label, finding id) to the paper's reported statistics.""" + source = study_path / "source" + path = (source if source.is_dir() else study_path) / "ground_truth.json" + if not path.exists(): + return {} + try: + with open(path, "r", encoding="utf-8", errors="replace") as handle: + ground_truth = json.load(handle) + except (OSError, json.JSONDecodeError): + return {} + index: Dict[tuple, Dict[str, Any]] = {} + for sub_study in ground_truth.get("studies", []) or []: + label = sub_study.get("study_id") + for finding in sub_study.get("findings", []) or []: + tests = finding.get("statistical_tests") or [{}] + entry = { + "hypothesis": finding.get("main_hypothesis"), + "reported_statistics": tests[0].get("reported_statistics"), + "test_name": tests[0].get("test_name"), + "expected_direction": tests[0].get("expected_direction"), + } + index[(label, finding.get("finding_id"))] = entry + index[(None, finding.get("finding_id"))] = entry + return index + + +def _label(row: Dict[str, Any], fallback: str) -> str: + parts = [str(row[key]) for key in ("study_id", "finding_id", "scenario") if row.get(key)] + if not parts: + parts = [str(row.get("test_name") or fallback)] + return " · ".join(parts).replace("_", " ") + + +def normalise_tests(evaluation: Dict[str, Any], study_path: Path) -> List[Dict[str, Any]]: + raw_tests = evaluation.get("test_results") or [] + ground_truth = _ground_truth_index(study_path) + rows: List[Dict[str, Any]] = [] + + for index, raw in enumerate(raw_tests): + if not isinstance(raw, dict): + continue + gt = ground_truth.get((raw.get("study_id"), raw.get("finding_id"))) or ground_truth.get((None, raw.get("finding_id"))) or {} + + agent_p = _number(_first(raw, "p_value_agent", "p_value", "agent_p_value")) + human_p = _number(_first(raw, "p_value_human", "human_p_value")) + agent_effect = _number(_first(raw, "agent_effect_d", "agent_effect_size")) + human_effect = _number(_first(raw, "human_effect_d", "human_effect_size")) + n_agent_1 = _number(_first(raw, "n_agent_1", "n_agent", "n_agent_extracted")) + n_agent_2 = _number(_first(raw, "n_agent_2", "n2_agent", "n2_agent_extracted")) + + if agent_effect is None: + agent_effect = agent_effect_from_t(_number(_first(raw, "t_stat", "test_statistic", "statistic")), n_agent_1, n_agent_2) + if human_effect is None: + human_effect = human_effect_from_reported(gt.get("reported_statistics") or raw.get("reported_statistics") or "") + + agent_significant = _flag(_first(raw, "is_significant_agent", "significant", "agent_significant")) + human_significant = _flag(_first(raw, "is_significant_human", "human_significant")) + direction_match = _flag(raw.get("direction_match")) + replicated = _flag(raw.get("replication")) + if replicated is None and None not in (human_significant, agent_significant, direction_match): + replicated = bool(human_significant and agent_significant and direction_match) + + rows.append({ + "test_id": f"T{index + 1}", + "study_label": raw.get("study_id"), + "sub_study_id": raw.get("sub_study_id"), + "finding_id": raw.get("finding_id"), + "scenario": raw.get("scenario"), + "label": _label(raw, f"Test {index + 1}"), + "test_type": _first(raw, "statistical_test_type", "test_type", "test"), + "hypothesis": gt.get("hypothesis"), + "reported_statistics": gt.get("reported_statistics"), + "human_effect": human_effect, + "agent_effect": agent_effect, + "human_p": human_p, + "agent_p": agent_p, + "human_significant": human_significant, + "agent_significant": agent_significant, + "direction_match": direction_match, + "replicated": replicated, + "agent_mean_1": _number(_first(raw, "mean_agent_1", "agent_mean_1")), + "agent_mean_2": _number(_first(raw, "mean_agent_2", "agent_mean_2")), + "n_agent_1": int(n_agent_1) if n_agent_1 else None, + "n_agent_2": int(n_agent_2) if n_agent_2 else None, + "pas": _number(_first(raw, "pas", "score")), + }) + return rows + + +def summarise(rows: List[Dict[str, Any]], evaluation: Dict[str, Any]) -> Dict[str, Any]: + scored = [row for row in rows if row["replicated"] is not None] + replicated = [row for row in scored if row["replicated"]] + comparable = [row for row in rows if row["human_effect"] is not None and row["agent_effect"] is not None] + directional = [row for row in rows if row["direction_match"] is not None] + + gaps = [abs(row["agent_effect"] - row["human_effect"]) for row in comparable] + human_effects = [row["human_effect"] for row in comparable] + agent_effects = [row["agent_effect"] for row in comparable] + + return { + "totalTests": len(rows), + "scoredTests": len(scored), + "replicatedTests": len(replicated), + "replicationRate": (len(replicated) / len(scored)) if scored else None, + "directionMatchRate": (sum(1 for row in directional if row["direction_match"]) / len(directional)) if directional else None, + "meanAbsoluteEffectGap": (sum(gaps) / len(gaps)) if gaps else None, + "meanHumanEffect": (sum(human_effects) / len(human_effects)) if human_effects else None, + "meanAgentEffect": (sum(agent_effects) / len(agent_effects)) if agent_effects else None, + "effectCorrelation": _correlation(human_effects, agent_effects), + "studyScore": _number(evaluation.get("score")), + } + + +def _correlation(x: List[float], y: List[float]) -> Number: + if len(x) < 3: + return None + mean_x = sum(x) / len(x) + mean_y = sum(y) / len(y) + covariance = sum((a - mean_x) * (b - mean_y) for a, b in zip(x, y)) + variance_x = sum((a - mean_x) ** 2 for a in x) + variance_y = sum((b - mean_y) ** 2 for b in y) + if variance_x <= 0 or variance_y <= 0: + return None + return covariance / math.sqrt(variance_x * variance_y) + + +def build_analysis(evaluation: Dict[str, Any], study_path: Path, run: Dict[str, Any], responses: Dict[str, Any]) -> Dict[str, Any]: + rows = normalise_tests(evaluation, study_path) + return { + "runId": run.get("id"), + "studyId": run.get("studyId"), + "studyTitle": run.get("studyTitle"), + "model": run.get("model"), + "promptPreset": run.get("preset"), + "participants": responses.get("participants"), + "trials": responses.get("trials"), + "completedTrials": responses.get("completedTrials"), + "summary": summarise(rows, evaluation), + "tests": rows, + "findings": evaluation.get("finding_results") or [], + } diff --git a/playground/default_charts.py b/playground/default_charts.py new file mode 100644 index 0000000..dc7feac --- /dev/null +++ b/playground/default_charts.py @@ -0,0 +1,152 @@ +"""Deterministic comparison charts for a playground run. + +These are the charts the playground falls back to when the charting agent does +not produce a valid set, and they are also the reference the agent is asked to +improve on. Everything here is derived from `output/analysis.json` alone. +""" + +import json +from pathlib import Path +from typing import Any, Dict, List + +SIGNIFICANT = "#0e7490" +NOT_SIGNIFICANT = "#a5e5ea" +HUMAN = "#334155" +AGENT = "#0e7490" + + +def _axis_bounds(values: List[float]) -> tuple[float, float]: + if not values: + return -1.0, 1.0 + low, high = min(values), max(values) + padding = max(0.15, (high - low) * 0.1) + return low - padding, high + padding + + +def effect_scatter(analysis: Dict[str, Any]) -> Dict[str, Any] | None: + """Agent effect size against the published human effect size.""" + points = [test for test in analysis["tests"] if test["human_effect"] is not None and test["agent_effect"] is not None] + if not points: + return None + significant = [test for test in points if test["agent_significant"]] + other = [test for test in points if not test["agent_significant"]] + low, high = _axis_bounds([test["human_effect"] for test in points] + [test["agent_effect"] for test in points]) + + def trace(rows: List[Dict[str, Any]], name: str, colour: str) -> Dict[str, Any]: + return { + "type": "scatter", + "mode": "markers", + "name": name, + "x": [row["human_effect"] for row in rows], + "y": [row["agent_effect"] for row in rows], + "text": [row["label"] for row in rows], + "hovertemplate": "%{text}
Human d=%{x:.2f}
Agent d=%{y:.2f}", + "marker": {"size": 11, "color": colour, "line": {"color": "white", "width": 1}}, + } + + data = [trace(rows, name, colour) for rows, name, colour in ( + (significant, "Agent effect significant", SIGNIFICANT), + (other, "Not significant", NOT_SIGNIFICANT), + ) if rows] + data.append({ + "type": "scatter", + "mode": "lines", + "name": "Human = agent", + "x": [low, high], + "y": [low, high], + "line": {"color": "#94a3b8", "width": 1.5, "dash": "dash"}, + "hoverinfo": "skip", + }) + return { + "id": "effect-scatter", + "title": "Agent effect size vs. published human effect size", + "description": "Each point is one statistical test from the paper. Points on the dashed line mean the agent reproduced the human effect at the same magnitude; points below it mean the agent showed a weaker effect than people did.", + "plotly": { + "data": data, + "layout": { + "xaxis": {"title": {"text": "Human effect size (d)"}, "range": [low, high], "zeroline": True}, + "yaxis": {"title": {"text": "Agent effect size (d)"}, "range": [low, high], "scaleanchor": "x", "zeroline": True}, + "legend": {"orientation": "h", "y": -0.2}, + "margin": {"t": 20, "r": 16, "b": 60, "l": 60}, + }, + }, + } + + +def effect_bars(analysis: Dict[str, Any]) -> Dict[str, Any] | None: + """Human and agent effect sizes side by side, per test.""" + points = [test for test in analysis["tests"] if test["human_effect"] is not None or test["agent_effect"] is not None][:20] + if not points: + return None + labels = [test["label"] for test in points] + return { + "id": "effect-bars", + "title": "Effect size by test", + "description": "How large the effect was for humans in the paper and for the agent in this run, test by test.", + "plotly": { + "data": [ + {"type": "bar", "name": "Human", "x": labels, "y": [test["human_effect"] for test in points], "marker": {"color": HUMAN}}, + {"type": "bar", "name": "Agent", "x": labels, "y": [test["agent_effect"] for test in points], "marker": {"color": AGENT}}, + ], + "layout": { + "barmode": "group", + "xaxis": {"tickangle": -35, "automargin": True}, + "yaxis": {"title": {"text": "Effect size (d)"}, "zeroline": True}, + "legend": {"orientation": "h", "y": -0.25}, + "margin": {"t": 20, "r": 16, "b": 80, "l": 60}, + }, + }, + } + + +def replication_breakdown(analysis: Dict[str, Any]) -> Dict[str, Any] | None: + """How many published findings the agent reproduced, missed, or reversed.""" + scored = [test for test in analysis["tests"] if test["replicated"] is not None or test["direction_match"] is not None] + if not scored: + return None + reproduced = sum(1 for test in scored if test["replicated"]) + wrong_direction = sum(1 for test in scored if not test["replicated"] and test["direction_match"] is False) + missed = len(scored) - reproduced - wrong_direction + return { + "id": "replication-breakdown", + "title": "What the agent did with each published finding", + "description": "Reproduced means the agent found the same effect, in the same direction, at the same significance level as the paper. Wrong direction means the agent showed the opposite effect.", + "plotly": { + "data": [{ + "type": "bar", + "orientation": "h", + "x": [reproduced, missed, wrong_direction], + "y": ["Reproduced", "No significant effect", "Wrong direction"], + "marker": {"color": [SIGNIFICANT, "#cbd5e1", "#f59e0b"]}, + "hovertemplate": "%{y}: %{x} tests", + }], + "layout": { + "xaxis": {"title": {"text": "Statistical tests"}, "dtick": 1}, + "yaxis": {"automargin": True}, + "showlegend": False, + "margin": {"t": 20, "r": 16, "b": 45, "l": 20}, + }, + }, + } + + +def build_charts(analysis: Dict[str, Any]) -> Dict[str, Any]: + charts = [chart for chart in (effect_scatter(analysis), effect_bars(analysis), replication_breakdown(analysis)) if chart] + return {"charts": charts, "source": "default"} + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser(description="Write the default playground charts for a run.") + parser.add_argument("--analysis", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + analysis = json.loads(args.analysis.read_text()) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(build_charts(analysis), indent=2) + "\n") + print(f"Wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/playground/progress.py b/playground/progress.py new file mode 100644 index 0000000..4a937c4 --- /dev/null +++ b/playground/progress.py @@ -0,0 +1,85 @@ +"""Publish playground run progress so the web app can show it while the run works. + +The run writes `progress.json` next to its other files and, when a token is +available, mirrors the same payload to the private jobs repository through the +GitHub contents API. The web app polls that file. +""" + +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional + + +class ProgressWriter: + def __init__(self, run_dir: Path, repo: Optional[str] = None, branch: Optional[str] = None, path: Optional[str] = None, min_interval: float = 10.0) -> None: + self.local_path = run_dir / "progress.json" + self.token = os.environ.get("PIPELINE_PROGRESS_TOKEN", "") + self.repo = repo + self.branch = branch + self.path = path + self.min_interval = min_interval + self.sha: Optional[str] = None + self._last_publish = 0.0 + if self._remote_enabled(): + self._load_sha() + + def _remote_enabled(self) -> bool: + return bool(self.token and self.repo and self.branch and self.path) + + def _request(self, method: str, body: Optional[dict] = None) -> dict: + url = f"https://api.github.com/repos/{self.repo}/contents/{urllib.parse.quote(self.path or '', safe='/')}" + if method == "GET": + url += "?ref=" + urllib.parse.quote(self.branch or "", safe="") + request = urllib.request.Request( + url, + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "HumanStudy-Hub-Playground", + }, + ) + with urllib.request.urlopen(request, timeout=20) as response: + return json.loads(response.read()) + + def _load_sha(self) -> None: + try: + self.sha = self._request("GET").get("sha") + except urllib.error.HTTPError as exc: + if exc.code != 404: + print(f"[progress] could not read remote progress: {exc}", flush=True) + except Exception as exc: + print(f"[progress] could not read remote progress: {exc}", flush=True) + + def write(self, payload: Dict[str, Any], force: bool = False) -> None: + payload = {**payload, "updatedAt": datetime.now(timezone.utc).isoformat()} + self.local_path.parent.mkdir(parents=True, exist_ok=True) + self.local_path.write_text(json.dumps(payload, indent=2) + "\n") + if not self._remote_enabled(): + return + # Publishing is a commit per call, so a long run only mirrors on an interval. + if not force and time.monotonic() - self._last_publish < self.min_interval: + return + body = { + "message": f"playground: progress {payload.get('phase', 'running')}", + "branch": self.branch, + "content": base64.b64encode((json.dumps(payload, indent=2) + "\n").encode()).decode(), + } + if self.sha: + body["sha"] = self.sha + try: + response = self._request("PUT", body) + self.sha = response["content"]["sha"] + self._last_publish = time.monotonic() + except Exception as exc: + print(f"[progress] could not publish progress: {exc}", flush=True) diff --git a/playground/run_charts.py b/playground/run_charts.py new file mode 100644 index 0000000..3fc90b9 --- /dev/null +++ b/playground/run_charts.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Have Claude Code chart and interpret a finished playground run. + +The agent is given the run directory and the contract in `playground/CLAUDE.md`, +and writes `output/charts.json`. Whatever it produces is validated before it is +kept; if it is missing, invalid, or the agent runs out of time, the deterministic +charts from `default_charts.py` are used instead. A run therefore always ends +with charts a researcher can read. +""" + +import argparse +import json +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any, Dict + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from playground.default_charts import build_charts +from playground.validate_charts import ChartError, validate + + +def build_prompt(contract: Path, run_dir: Path) -> str: + return ( + f"{contract.read_text()}\n\n" + "## This run\n\n" + f"The run directory is `{run_dir}`. Read `run.json`, `output/analysis.json`, " + "`output/evaluation.json`, `output/transcript_sample.json`, and " + "`output/charts.default.json` there.\n\n" + f"Write your charts and interpretation to `{run_dir / 'output' / 'charts.json'}`. " + "Write that file exactly once, when it is complete and valid, then stop.\n" + ) + + +def stream_output(process: "subprocess.Popen[str]", log_path: Path) -> None: + with log_path.open("a", encoding="utf-8") as log: + assert process.stdout is not None + for line in process.stdout: + sys.stdout.write(line) + sys.stdout.flush() + log.write(line) + log.flush() + + +def stop_process(process: "subprocess.Popen[str]") -> None: + if process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=20) + except (ProcessLookupError, subprocess.TimeoutExpired): + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=10) + + +def accept_charts(path: Path) -> Dict[str, Any] | None: + """Return the agent's charts when they are present and valid.""" + if not path.exists(): + return None + try: + document = json.loads(path.read_text()) + validate(document) + except (OSError, json.JSONDecodeError) as error: + print(f"[charts] agent output could not be read: {error}", flush=True) + return None + except ChartError as error: + print(f"[charts] agent output rejected: {error}", flush=True) + return None + document["source"] = "agent" + return document + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", required=True, type=Path) + parser.add_argument("--contract", type=Path, default=Path(__file__).resolve().parent / "CLAUDE.md") + parser.add_argument("--model", default=os.environ.get("OPENROUTER_MODEL", "anthropic/claude-sonnet-5")) + parser.add_argument("--timeout-minutes", type=float, default=8) + args = parser.parse_args() + + run_dir = args.run.resolve() + output_dir = run_dir / "output" + logs_dir = run_dir / "logs" + output_dir.mkdir(parents=True, exist_ok=True) + logs_dir.mkdir(parents=True, exist_ok=True) + + analysis_path = output_dir / "analysis.json" + if not analysis_path.exists(): + raise SystemExit("This run has no analysis.json, so there is nothing to chart.") + analysis = json.loads(analysis_path.read_text()) + + # The baseline is written first: it is the agent's reference and the fallback. + fallback = build_charts(analysis) + (output_dir / "charts.default.json").write_text(json.dumps(fallback, indent=2) + "\n") + + charts_path = output_dir / "charts.json" + prompt_path = run_dir / "charts-prompt.md" + prompt_path.write_text(build_prompt(args.contract, run_dir)) + + agent_env = os.environ.copy() + for secret in ("PIPELINE_PROGRESS_TOKEN", "HUMANSTUDY_PIPELINE_TOKEN", "GITHUB_TOKEN", "OPENROUTER_API_KEY", "PLAYGROUND_KEY_SECRET"): + agent_env.pop(secret, None) + agent_env["ANTHROPIC_AUTH_TOKEN"] = os.environ.get("CHARTS_AGENT_TOKEN", os.environ.get("ANTHROPIC_AUTH_TOKEN", "")) + + process = subprocess.Popen( + [ + "claude", + "--print", + "--model", args.model, + "--add-dir", str(run_dir), + "--dangerously-skip-permissions", + prompt_path.read_text(), + ], + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + start_new_session=True, + env=agent_env, + ) + output_thread = threading.Thread(target=stream_output, args=(process, logs_dir / "charts.log"), daemon=True) + output_thread.start() + deadline = time.monotonic() + args.timeout_minutes * 60 + + try: + while process.poll() is None: + if time.monotonic() >= deadline: + print("[charts] the charting agent ran out of time", flush=True) + stop_process(process) + break + time.sleep(5) + finally: + stop_process(process) + output_thread.join(timeout=5) + + document = accept_charts(charts_path) + if document is None: + print("[charts] using the deterministic charts for this run", flush=True) + document = fallback + charts_path.write_text(json.dumps(document, indent=2) + "\n") + print(f"[charts] wrote {len(document['charts'])} charts from the {document['source']} source", flush=True) + + +if __name__ == "__main__": + main() diff --git a/playground/run_key.py b/playground/run_key.py new file mode 100644 index 0000000..9e309f1 --- /dev/null +++ b/playground/run_key.py @@ -0,0 +1,49 @@ +"""Decrypt the OpenRouter key a researcher supplied for their own playground run. + +HumanStudy-Hub never stores a researcher's key in plaintext. The web app encrypts +it with AES-256-GCM under a secret shared with this workflow +(`PLAYGROUND_KEY_SECRET`), so the private jobs repository only ever holds +ciphertext. This module reverses that on the runner. +""" + +import base64 +import hashlib +import os +from typing import Any, Mapping, Optional + + +def _shared_secret() -> Optional[bytes]: + secret = os.environ.get("PLAYGROUND_KEY_SECRET", "").strip() + if not secret: + return None + # The web app derives the same 32-byte key with SHA-256 over the secret. + return hashlib.sha256(secret.encode("utf-8")).digest() + + +def decrypt_api_key(sealed: Optional[Mapping[str, Any]]) -> Optional[str]: + """Return the researcher's OpenRouter key, or None when the run has none. + + Raises ValueError when a key was supplied but cannot be opened, because + silently falling back to the shared key would spend the project's budget on + a run the researcher expected to pay for. + """ + if not sealed: + return None + key = _shared_secret() + if key is None: + raise ValueError("PLAYGROUND_KEY_SECRET is not configured, so the supplied API key cannot be opened.") + try: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + iv = base64.b64decode(sealed["iv"]) + ciphertext = base64.b64decode(sealed["ciphertext"]) + tag = base64.b64decode(sealed["tag"]) + plaintext = AESGCM(key).decrypt(iv, ciphertext + tag, None) + except KeyError as exc: + raise ValueError(f"The sealed API key is missing the {exc} field.") from exc + except Exception as exc: + raise ValueError("The supplied API key could not be decrypted with the configured secret.") from exc + opened = plaintext.decode("utf-8").strip() + if not opened: + raise ValueError("The supplied API key decrypted to an empty value.") + return opened diff --git a/playground/run_playground.py b/playground/run_playground.py new file mode 100644 index 0000000..d946624 --- /dev/null +++ b/playground/run_playground.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Replay one benchmark study with a chosen model and score it against the paper. + +This is the engine behind the HumanStudy-Hub playground. It reuses the study's +own trial builder and evaluator, so a playground run is scored exactly like a +benchmark run — the only things the researcher changes are the model, the +participant prompt, and how many participants take part. + + python playground/run_playground.py --run + +The run directory holds `run.json` (written by the web app) and receives +`progress.json`, `output/`, and `logs/`. +""" + +import argparse +import json +import os +import random +import sys +import traceback +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from playground import settings +from playground.analysis import build_analysis +from playground.progress import ProgressWriter +from playground.run_key import decrypt_api_key +from playground.study_loader import StudyNotRunnable, load_metadata, load_study + + +def now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read_run(run_dir: Path) -> Dict[str, Any]: + path = run_dir / "run.json" + if not path.exists(): + raise SystemExit(f"No run.json in {run_dir}") + return json.loads(path.read_text()) + + +def write_run(run_dir: Path, run: Dict[str, Any]) -> None: + run["updatedAt"] = now() + (run_dir / "run.json").write_text(json.dumps(run, indent=2) + "\n") + + +def resolve_prompt(run: Dict[str, Any]) -> tuple[str, Optional[str]]: + """Return (preset name passed to the registry, explicit prompt override).""" + preset = str(run.get("preset") or settings.DEFAULT_PRESET) + custom = (run.get("systemPrompt") or "").strip() + if preset == settings.CUSTOM_PRESET: + if not custom: + raise SystemExit("This run selected a custom prompt but did not include one.") + return settings.DEFAULT_PRESET, custom + if preset not in settings.PROMPT_PRESETS: + raise SystemExit(f"Unknown prompt preset: {preset}") + return preset, custom or None + + +def apply_demographics(profile: Dict[str, Any], demographics: Dict[str, Any]) -> Dict[str, Any]: + """Overlay the researcher's demographic choices on a generated profile.""" + merged = dict(profile) + for field in settings.DEMOGRAPHIC_FIELDS: + value = demographics.get(field) + if value not in (None, "", []): + merged[field] = value + return merged + + +def cap_trials(study_config: Any, run: Dict[str, Any], has_own_key: bool, log) -> List[Dict[str, Any]]: + """Build trials at the requested size, then hold the run inside its budget.""" + max_total, max_per_scenario = settings.trial_limits(has_own_key) + requested = int(run.get("participantsPerScenario") or settings.DEFAULT_PER_SCENARIO) + per_scenario = max(1, min(requested, max_per_scenario)) + if per_scenario != requested: + log(f"Requested {requested} participants per condition; capped to {per_scenario} for this run budget.") + + try: + trials = study_config.create_trials(n_trials=per_scenario) + except TypeError: + # A few studies take extra keyword arguments and ignore n_trials. + trials = study_config.create_trials() + trials = [trial for trial in trials if isinstance(trial, dict)] + if not trials: + raise SystemExit("The study produced no trials to run.") + + if len(trials) > max_total: + # Subsample evenly across conditions so every condition keeps participants. + rng = random.Random(int(run.get("seed") or 42)) + by_scenario: Dict[str, List[Dict[str, Any]]] = {} + for trial in trials: + key = str(trial.get("scenario_id") or trial.get("scenario") or trial.get("sub_study_id") or "all") + by_scenario.setdefault(key, []).append(trial) + quota = max(1, max_total // len(by_scenario)) + kept: List[Dict[str, Any]] = [] + for group in by_scenario.values(): + rng.shuffle(group) + kept.extend(group[:quota]) + trials = kept[:max_total] + log(f"Run budget allows {max_total} participant sessions; sampled {len(trials)} across {len(by_scenario)} conditions.") + return trials + + +def build_profiles(trials: List[Dict[str, Any]], specification: Dict[str, Any], run: Dict[str, Any], study_id: str) -> List[Dict[str, Any]]: + demographics = run.get("demographics") or {} + population = specification.get("participants", {}).get("population") + recruitment = specification.get("participants", {}).get("recruitment_source") + profiles = [] + for index, trial in enumerate(trials): + base = dict(trial.get("profile") or {}) + base.setdefault("participant_id", index) + base.setdefault("education", recruitment or "college student") + base.setdefault("population", population) + base["study_id"] = study_id + profiles.append(apply_demographics(base, demographics)) + return profiles + + +def evaluation_input(pool_results: Dict[str, Any], study_path: Optional[Path] = None) -> Dict[str, Any]: + """Study evaluators read participants that each carry their own responses. + + The raw API envelope for every call is dropped here: it is many times larger + than the answers themselves and no evaluator reads it. + """ + participants = [] + for summary in pool_results.get("participant_summaries", []): + participants.append({ + "participant_id": summary.get("participant_id"), + "profile": summary.get("profile"), + "responses": [ + { + "participant_id": response.get("participant_id"), + "trial_number": response.get("trial_number"), + "response": response.get("response"), + "response_text": response.get("response_text"), + "is_correct": response.get("is_correct"), + "correct_answer": response.get("correct_answer"), + "trial_info": response.get("trial_info"), + } + for response in summary.get("responses", []) or [] + ], + }) + payload: Dict[str, Any] = {"individual_data": participants} + # Some evaluators resolve the study's own ground truth from this field. + if study_path is not None: + payload["study_path"] = str(study_path) + return payload + + +def evaluate_in_study_dir(evaluator: Any, scored_input: Dict[str, Any], study_path: Path) -> Dict[str, Any]: + """Score the run from inside the study folder. + + A handful of evaluators open their ground truth through a relative path, so + they only resolve correctly when the study directory is the working + directory. + """ + previous = os.getcwd() + os.chdir(study_path) + try: + return evaluator.evaluate_study(scored_input) + finally: + os.chdir(previous) + + +def response_stats(pool_results: Dict[str, Any]) -> Dict[str, Any]: + summaries = pool_results.get("participant_summaries", []) + completed = 0 + answered = 0 + tokens = 0 + for summary in summaries: + for response in summary.get("responses", []) or []: + completed += 1 + if (response.get("response_text") or "").strip(): + answered += 1 + usage = response.get("usage") or {} + tokens += int(usage.get("total_tokens") or 0) + return { + "participants": len(summaries), + "trials": completed, + "completedTrials": completed, + "answeredTrials": answered, + "totalTokens": tokens, + } + + +def sample_transcript(pool_results: Dict[str, Any], prompt_builder: Any, limit: int = 6) -> List[Dict[str, Any]]: + """A few complete participant exchanges, so researchers can read what happened.""" + samples: List[Dict[str, Any]] = [] + for summary in pool_results.get("participant_summaries", []): + profile = summary.get("profile") or {} + for response in summary.get("responses", []) or []: + trial_info = response.get("trial_info") or {} + try: + prompt = prompt_builder.build_trial_prompt({**trial_info, "participant_profile": profile}) + except Exception: + prompt = None + samples.append({ + "participantId": summary.get("participant_id"), + "profile": {key: profile.get(key) for key in ("age", "gender", "education", "background", "persona") if profile.get(key)}, + "prompt": prompt, + "response": response.get("response_text"), + }) + if len(samples) >= limit: + return samples + return samples + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", required=True, type=Path, help="Run directory containing run.json") + parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument("--progress-repo") + parser.add_argument("--progress-branch") + parser.add_argument("--progress-path") + parser.add_argument( + "--simulate", + action="store_true", + help="Exercise the run end to end with simulated participants and no API calls.", + ) + args = parser.parse_args() + + run_dir = args.run.resolve() + output_dir = run_dir / "output" + logs_dir = run_dir / "logs" + output_dir.mkdir(parents=True, exist_ok=True) + logs_dir.mkdir(parents=True, exist_ok=True) + log_file = (logs_dir / "run.log").open("a", encoding="utf-8") + + def log(message: str) -> None: + line = f"[{now()}] {message}" + print(line, flush=True) + log_file.write(line + "\n") + log_file.flush() + + run = read_run(run_dir) + progress = ProgressWriter(run_dir, args.progress_repo, args.progress_branch, args.progress_path) + progress.write({"phase": "preparing", "completedTrials": 0, "totalTrials": 0, "message": "Loading the study"}, force=True) + + try: + api_key = decrypt_api_key(run.get("sealedApiKey")) + has_own_key = api_key is not None + if not has_own_key: + api_key = os.environ.get("OPENROUTER_API_KEY") + if not api_key and not args.simulate: + raise SystemExit("No OpenRouter API key is available for this run.") + + study_path, specification, study_config, evaluator = load_study(args.repo_root, str(run.get("studyId") or "")) + metadata = load_metadata(study_path) + run["studyTitle"] = metadata.get("title") or run.get("studyTitle") + log(f"Study {study_path.name}: {run['studyTitle']}") + + preset, override = resolve_prompt(run) + trials = cap_trials(study_config, run, has_own_key, log) + profiles = build_profiles(trials, specification, run, study_path.name) + log(f"Prepared {len(trials)} participant sessions on {run.get('model')} with prompt {run.get('preset')}.") + + from src.agents.llm_participant_agent import ParticipantPool + + prompt_builder = study_config.get_prompt_builder() + pool = ParticipantPool( + study_specification=specification, + n_participants=len(trials), + use_real_llm=not args.simulate, + model=str(run.get("model") or settings.DEFAULT_MODEL), + api_key=api_key, + # Passing the OpenRouter base explicitly is what routes every model — + # including Claude and Gemini names — through OpenRouter. + api_base=settings.OPENROUTER_API_BASE, + random_seed=int(run.get("seed") or 42), + num_workers=int(run.get("workers") or settings.DEFAULT_WORKERS), + profiles=profiles, + prompt_builder=prompt_builder, + system_prompt_override=override, + system_prompt_preset=preset, + study_id=study_path.name, + temperature=float(run.get("temperature") or 1.0), + ) + + state = {"done": 0} + + def on_trial(*_: Any) -> None: + state["done"] += 1 + progress.write({ + "phase": "running_participants", + "completedTrials": state["done"], + "totalTrials": len(trials), + "message": f"{state['done']} of {len(trials)} participant sessions complete", + }) + + progress.write({"phase": "running_participants", "completedTrials": 0, "totalTrials": len(trials), "message": "Participants are starting"}, force=True) + pool.run_experiment( + trials=trials, + instructions=study_config.get_instructions(), + prompt_builder=prompt_builder, + one_to_one=True, + save_callback=on_trial, + ) + + pool_results = pool.aggregate_results() + stats = response_stats(pool_results) + scored_input = evaluation_input(pool_results, study_path) + log(f"Collected {stats['answeredTrials']} answers from {stats['participants']} participants.") + (output_dir / "responses.json").write_text(json.dumps(scored_input, indent=2, default=str) + "\n") + (output_dir / "transcript_sample.json").write_text(json.dumps(sample_transcript(pool_results, prompt_builder), indent=2, default=str) + "\n") + + progress.write({"phase": "scoring", "completedTrials": stats["completedTrials"], "totalTrials": len(trials), "message": "Scoring against the published findings"}, force=True) + evaluation = evaluate_in_study_dir(evaluator, scored_input, study_path) + (output_dir / "evaluation.json").write_text(json.dumps(evaluation, indent=2, default=str) + "\n") + + analysis = build_analysis(evaluation, study_path, run, stats) + (output_dir / "analysis.json").write_text(json.dumps(analysis, indent=2, default=str) + "\n") + summary = analysis["summary"] + log( + "Replication: " + f"{summary['replicatedTests']}/{summary['scoredTests']} tests reproduced the published direction and significance." + ) + + run.update({ + "status": "analysing", + "message": "Building the comparison charts", + "participants": stats["participants"], + "completedTrials": stats["completedTrials"], + "answeredTrials": stats["answeredTrials"], + "totalTokens": stats["totalTokens"], + "summary": summary, + }) + write_run(run_dir, run) + progress.write({"phase": "charting", "completedTrials": stats["completedTrials"], "totalTrials": len(trials), "message": "Building the comparison charts"}, force=True) + + except StudyNotRunnable as error: + log(f"Study cannot be replayed: {error}") + run.update({"status": "failed", "message": "This study cannot be replayed in the playground", "error": str(error)}) + write_run(run_dir, run) + progress.write({"phase": "failed", "message": str(error)}, force=True) + raise SystemExit(1) + except SystemExit: + raise + except Exception as error: + log(f"Run failed: {error}\n{traceback.format_exc()}") + run.update({"status": "failed", "message": "The playground run failed", "error": str(error)}) + write_run(run_dir, run) + progress.write({"phase": "failed", "message": str(error)}, force=True) + raise SystemExit(1) + finally: + log_file.close() + + +if __name__ == "__main__": + main() diff --git a/playground/settings.py b/playground/settings.py new file mode 100644 index 0000000..545afb8 --- /dev/null +++ b/playground/settings.py @@ -0,0 +1,32 @@ +"""Shared playground constants: run limits, prompt presets, and OpenRouter routing.""" + +OPENROUTER_API_BASE = "https://openrouter.ai/api/v1" + +# A run on the shared HumanStudy-Hub key is capped so one playground session +# cannot spend the project's OpenRouter budget. A contributor who supplies their +# own key is allowed the larger budget. +SHARED_KEY_MAX_TRIALS = 60 +OWN_KEY_MAX_TRIALS = 600 + +SHARED_KEY_MAX_PER_SCENARIO = 10 +OWN_KEY_MAX_PER_SCENARIO = 80 + +DEFAULT_PER_SCENARIO = 8 +DEFAULT_MODEL = "openai/gpt-4o-mini" +DEFAULT_PRESET = "v3_human_plus_demo" +DEFAULT_WORKERS = 8 + +# Presets shipped by src/agents/custom_methods/. "custom" is the playground's own +# marker for a prompt the researcher wrote in the prompt designer. +PROMPT_PRESETS = ("v1_empty", "v2_human", "v3_human_plus_demo", "v4_background") +CUSTOM_PRESET = "custom" + +# Demographic fields the prompt designer can override on every participant. +DEMOGRAPHIC_FIELDS = ("age", "gender", "education", "background", "population", "persona") + + +def trial_limits(has_own_key: bool) -> tuple[int, int]: + """Return (max total trials, max trials per scenario) for this run.""" + if has_own_key: + return OWN_KEY_MAX_TRIALS, OWN_KEY_MAX_PER_SCENARIO + return SHARED_KEY_MAX_TRIALS, SHARED_KEY_MAX_PER_SCENARIO diff --git a/playground/study_loader.py b/playground/study_loader.py new file mode 100644 index 0000000..870484b --- /dev/null +++ b/playground/study_loader.py @@ -0,0 +1,99 @@ +"""Load a study's own trial builder and evaluator from `studies//scripts/`. + +Every study ships a self-contained `scripts/config.py` (a `BaseStudyConfig` +subclass that builds trials and knows the study's prompts) and a +`scripts/evaluator.py` exposing `evaluate_study(results)`. The playground drives +those files rather than reimplementing any study logic. +""" + +import importlib.util +import inspect +import json +import sys +from pathlib import Path +from types import ModuleType +from typing import Any, Dict, Tuple + + +class StudyNotRunnable(Exception): + """The study exists but does not expose the files the playground needs.""" + + +def _import_module(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise StudyNotRunnable(f"Could not import {path}") + module = importlib.util.module_from_spec(spec) + # A study's scripts import their sibling modules by bare name, so the study's + # own scripts directory has to be importable while the module executes. + sys.modules[name] = module + scripts_dir = str(path.parent) + if scripts_dir not in sys.path: + sys.path.insert(0, scripts_dir) + spec.loader.exec_module(module) + return module + + +def study_dir(repo_root: Path, study_id: str) -> Path: + safe = "".join(character for character in study_id if character.isalnum() or character in "_-") + if not safe or safe != study_id: + raise StudyNotRunnable(f"Invalid study id: {study_id}") + path = repo_root / "studies" / safe + if not path.is_dir(): + raise StudyNotRunnable(f"Study {safe} is not in this benchmark.") + return path + + +def source_dir(path: Path) -> Path: + source = path / "source" + return source if (source / "specification.json").exists() else path + + +def load_specification(path: Path) -> Dict[str, Any]: + spec_path = source_dir(path) / "specification.json" + if not spec_path.exists(): + raise StudyNotRunnable(f"{path.name} has no specification.json, so it cannot be replayed.") + with open(spec_path, "r", encoding="utf-8", errors="replace") as handle: + return json.load(handle) + + +def load_metadata(path: Path) -> Dict[str, Any]: + for candidate in (source_dir(path) / "metadata.json", path / "index.json"): + if candidate.exists(): + with open(candidate, "r", encoding="utf-8", errors="replace") as handle: + return json.load(handle) + return {} + + +def load_study_config(path: Path, specification: Dict[str, Any]) -> Any: + """Instantiate the study's `BaseStudyConfig` subclass.""" + config_path = path / "scripts" / "config.py" + if not config_path.exists(): + raise StudyNotRunnable(f"{path.name} has no scripts/config.py, so its trials cannot be built.") + module = _import_module(config_path, f"playground_{path.name}_config") + candidates = [ + value for _, value in inspect.getmembers(module, inspect.isclass) + if value.__module__ == module.__name__ + and hasattr(value, "create_trials") + and not inspect.isabstract(value) + and any(base.__name__ == "BaseStudyConfig" for base in value.__mro__[1:]) + ] + if not candidates: + raise StudyNotRunnable(f"{path.name} has no study config class in scripts/config.py.") + return candidates[0](path, specification) + + +def load_evaluator(path: Path) -> ModuleType: + evaluator_path = path / "scripts" / "evaluator.py" + if not evaluator_path.exists(): + raise StudyNotRunnable(f"{path.name} has no scripts/evaluator.py, so agent runs cannot be scored.") + module = _import_module(evaluator_path, f"playground_{path.name}_evaluator") + if not hasattr(module, "evaluate_study"): + raise StudyNotRunnable(f"{path.name}'s evaluator does not expose evaluate_study().") + return module + + +def load_study(repo_root: Path, study_id: str) -> Tuple[Path, Dict[str, Any], Any, ModuleType]: + path = study_dir(repo_root, study_id) + specification = load_specification(path) + return path, specification, load_study_config(path, specification), load_evaluator(path) diff --git a/playground/validate_charts.py b/playground/validate_charts.py new file mode 100644 index 0000000..847b0f7 --- /dev/null +++ b/playground/validate_charts.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check that agent-written charts are safe, plottable Plotly specifications. + +The charting agent writes `output/charts.json`. That file is rendered in the +browser, so it is validated here before it ever leaves the runner: only plain +JSON data is allowed, trace types are restricted to an allowlist, and every +numeric series must actually contain numbers. A file that fails validation is +replaced by the deterministic charts. +""" + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any, List + +ALLOWED_TYPES = {"scatter", "scattergl", "bar", "box", "violin", "histogram", "heatmap", "line"} +MAX_CHARTS = 6 +MAX_POINTS = 5000 +MAX_DEPTH = 12 +# Plotly renders these as raw markup, which would let generated text inject HTML. +FORBIDDEN_KEYS = {"meta", "customdatasrc", "xsrc", "ysrc", "zsrc", "textsrc", "idssrc"} + + +class ChartError(Exception): + pass + + +def _check_value(value: Any, path: str, depth: int = 0) -> int: + """Walk a JSON value, rejecting anything that is not plain data.""" + if depth > MAX_DEPTH: + raise ChartError(f"{path}: nested too deeply") + if value is None or isinstance(value, (bool, int, str)): + if isinstance(value, str) and len(value) > 4000: + raise ChartError(f"{path}: string is too long") + return 1 + if isinstance(value, float): + if math.isnan(value) or math.isinf(value): + raise ChartError(f"{path}: not a finite number") + return 1 + if isinstance(value, list): + if len(value) > MAX_POINTS: + raise ChartError(f"{path}: more than {MAX_POINTS} values") + return sum(_check_value(item, f"{path}[{index}]", depth + 1) for index, item in enumerate(value)) + if isinstance(value, dict): + count = 0 + for key, item in value.items(): + if not isinstance(key, str): + raise ChartError(f"{path}: non-string key") + if key.lower() in FORBIDDEN_KEYS: + raise ChartError(f"{path}.{key}: field is not allowed") + count += _check_value(item, f"{path}.{key}", depth + 1) + return count + raise ChartError(f"{path}: unsupported value of type {type(value).__name__}") + + +def validate(document: Any) -> List[str]: + if not isinstance(document, dict): + raise ChartError("charts.json must contain an object") + charts = document.get("charts") + if not isinstance(charts, list) or not charts: + raise ChartError("charts.json must contain a non-empty 'charts' list") + if len(charts) > MAX_CHARTS: + raise ChartError(f"charts.json has more than {MAX_CHARTS} charts") + + ids: List[str] = [] + for index, chart in enumerate(charts): + where = f"charts[{index}]" + if not isinstance(chart, dict): + raise ChartError(f"{where}: each chart must be an object") + for field in ("id", "title", "description"): + if not isinstance(chart.get(field), str) or not chart[field].strip(): + raise ChartError(f"{where}.{field}: required non-empty string") + if chart["id"] in ids: + raise ChartError(f"{where}.id: duplicate chart id {chart['id']}") + ids.append(chart["id"]) + + plotly = chart.get("plotly") + if not isinstance(plotly, dict): + raise ChartError(f"{where}.plotly: required object") + traces = plotly.get("data") + if not isinstance(traces, list) or not traces: + raise ChartError(f"{where}.plotly.data: required non-empty list") + for trace_index, trace in enumerate(traces): + if not isinstance(trace, dict): + raise ChartError(f"{where}.plotly.data[{trace_index}]: must be an object") + trace_type = trace.get("type") + if trace_type is not None and trace_type not in ALLOWED_TYPES: + raise ChartError(f"{where}.plotly.data[{trace_index}].type: '{trace_type}' is not an allowed chart type") + if not any(key in trace for key in ("x", "y", "z", "values", "labels")): + raise ChartError(f"{where}.plotly.data[{trace_index}]: has no data series") + if "layout" in plotly and not isinstance(plotly["layout"], dict): + raise ChartError(f"{where}.plotly.layout: must be an object") + _check_value(plotly, f"{where}.plotly") + + interpretation = document.get("interpretation") + if interpretation is not None and (not isinstance(interpretation, str) or len(interpretation) > 8000): + raise ChartError("interpretation must be a string of at most 8000 characters") + return ids + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("charts", type=Path) + args = parser.parse_args() + try: + document = json.loads(args.charts.read_text()) + ids = validate(document) + except (OSError, json.JSONDecodeError) as error: + print(f"charts.json could not be read: {error}") + raise SystemExit(1) + except ChartError as error: + print(f"charts.json is not valid: {error}") + raise SystemExit(1) + print(f"charts.json is valid: {', '.join(ids)}") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 0601981..0a40d6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,9 @@ mypy>=0.950 streamlit>=1.20.0 watchdog>=2.1.0 +# Playground +cryptography>=42.0.0 # Opens the OpenRouter key a researcher supplies for their own run + # Optional: For LLM agents openai>=1.0.0 # Required for OpenRouter and OpenAI API anthropic>=0.3.0 # For Claude (Anthropic API) diff --git a/tests/test_playground.py b/tests/test_playground.py new file mode 100644 index 0000000..5f006b7 --- /dev/null +++ b/tests/test_playground.py @@ -0,0 +1,157 @@ +import json +from pathlib import Path + +import pytest + +from playground.analysis import agent_effect_from_t, build_analysis, human_effect_from_reported, normalise_tests +from playground.default_charts import build_charts +from playground.settings import trial_limits +from playground.study_loader import StudyNotRunnable, study_dir +from playground.validate_charts import ChartError, validate + + +def write_ground_truth(tmp_path: Path) -> Path: + study = tmp_path / "study_900" + (study / "source").mkdir(parents=True) + (study / "source" / "ground_truth.json").write_text(json.dumps({ + "studies": [{ + "study_id": "Study 1", + "findings": [{ + "finding_id": "F1", + "main_hypothesis": "Choosers estimate their own choice as more common.", + "statistical_tests": [{ + "reported_statistics": "F(1, 312) = 49.1, p < .001", + "expected_direction": "positive", + }], + }], + }], + })) + return study + + +def test_human_effect_is_converted_from_the_reported_statistic() -> None: + assert human_effect_from_reported("F(1, 312) = 49.1, p < .001") == pytest.approx(0.7934, abs=1e-3) + assert human_effect_from_reported("t(78) = 2.66, p = .01") == pytest.approx(0.6023, abs=1e-3) + # A multi-numerator F has no single-d equivalent and must not be guessed at. + assert human_effect_from_reported("F(3, 120) = 4.2") is None + assert human_effect_from_reported("chi2(1) = 6.1") is None + + +def test_agent_effect_is_derived_from_the_t_statistic() -> None: + assert agent_effect_from_t(2.0, 30, 30) == pytest.approx(0.5164, abs=1e-3) + assert agent_effect_from_t(None, 30, 30) is None + + +def test_raw_evaluator_output_is_normalised_against_the_paper(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [{ + "study_id": "Study 1", + "finding_id": "F1", + "scenario": "supermarket_story", + "t_stat": 2.4, + "p_value": 0.02, + "n_agent_1": 30, + "n_agent_2": 30, + "significant": True, + "direction_match": True, + "human_significant": True, + "replication": True, + }]} + + rows = normalise_tests(evaluation, study) + + assert len(rows) == 1 + row = rows[0] + assert row["human_effect"] == pytest.approx(0.7934, abs=1e-3) + assert row["agent_effect"] == pytest.approx(0.6197, abs=1e-3) + assert row["reported_statistics"] == "F(1, 312) = 49.1, p < .001" + assert row["replicated"] is True + + +def test_enriched_evaluator_output_keeps_its_own_effect_sizes(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [{ + "study_id": "Study 1", + "finding_id": "F1", + "human_effect_d": 0.9, + "agent_effect_d": 0.2, + "p_value_agent": 0.4, + "p_value_human": 0.001, + "is_significant_agent": False, + "is_significant_human": True, + "direction_match": True, + }]} + + row = normalise_tests(evaluation, study)[0] + + assert row["human_effect"] == 0.9 + assert row["agent_effect"] == 0.2 + # Not significant for the agent, so the published finding was not reproduced. + assert row["replicated"] is False + + +def test_analysis_summarises_replication(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [ + {"study_id": "Study 1", "finding_id": "F1", "human_effect_d": 0.8, "agent_effect_d": 0.6, "is_significant_agent": True, "is_significant_human": True, "direction_match": True}, + {"study_id": "Study 1", "finding_id": "F1", "human_effect_d": 0.8, "agent_effect_d": -0.4, "is_significant_agent": True, "is_significant_human": True, "direction_match": False}, + ]} + + analysis = build_analysis(evaluation, study, {"id": "r1", "studyId": "study_900"}, {"participants": 20}) + + assert analysis["summary"]["replicatedTests"] == 1 + assert analysis["summary"]["scoredTests"] == 2 + assert analysis["summary"]["replicationRate"] == pytest.approx(0.5) + assert analysis["summary"]["directionMatchRate"] == pytest.approx(0.5) + assert analysis["participants"] == 20 + + +def test_default_charts_are_valid_and_skip_what_the_data_cannot_support(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [ + {"study_id": "Study 1", "finding_id": "F1", "human_effect_d": 0.8, "agent_effect_d": 0.6, "is_significant_agent": True, "is_significant_human": True, "direction_match": True}, + ]} + analysis = build_analysis(evaluation, study, {"id": "r1"}, {}) + + charts = build_charts(analysis) + + assert validate(charts) == ["effect-scatter", "effect-bars", "replication-breakdown"] + + # The agent produced no measurable effect here. The scatter needs both sides + # so it is dropped rather than plotted against an invented zero; the bar + # chart still shows the human effect the agent failed to produce. + one_sided = build_analysis({"test_results": [{"study_id": "Study 1", "finding_id": "F1", "direction_match": False}]}, study, {"id": "r1"}, {}) + assert [chart["id"] for chart in build_charts(one_sided)["charts"]] == ["effect-bars", "replication-breakdown"] + + # With nothing measured on either side there is no chart to draw at all. + nothing = build_analysis({"test_results": []}, study, {"id": "r1"}, {}) + assert build_charts(nothing)["charts"] == [] + + +def test_charts_from_the_agent_must_be_plain_plottable_data() -> None: + def chart(trace: dict) -> dict: + return {"charts": [{"id": "a", "title": "t", "description": "d", "plotly": {"data": [trace]}}]} + + assert validate(chart({"type": "bar", "x": ["a"], "y": [1]})) == ["a"] + with pytest.raises(ChartError): + validate(chart({"type": "pie", "values": [1, 2]})) + with pytest.raises(ChartError): + validate(chart({"type": "bar", "x": ["a"], "y": [1], "meta": ""})) + with pytest.raises(ChartError): + validate(chart({"type": "bar"})) + with pytest.raises(ChartError): + validate({"charts": []}) + + +def test_run_budget_depends_on_whose_key_pays() -> None: + shared_total, shared_per_scenario = trial_limits(has_own_key=False) + own_total, own_per_scenario = trial_limits(has_own_key=True) + assert own_total > shared_total + assert own_per_scenario > shared_per_scenario + + +def test_study_ids_cannot_escape_the_studies_directory(tmp_path: Path) -> None: + with pytest.raises(StudyNotRunnable): + study_dir(tmp_path, "../secrets") + with pytest.raises(StudyNotRunnable): + study_dir(tmp_path, "study_999")