diff --git a/.github/workflows/run-playground.yml b/.github/workflows/run-playground.yml new file mode 100644 index 0000000..28f8bf4 --- /dev/null +++ b/.github/workflows/run-playground.yml @@ -0,0 +1,190 @@ +name: Run a playground study + +on: + workflow_dispatch: + inputs: + run_id: + description: Private jobs repository run ID + required: true + type: string + jobs_repo: + description: Private repository containing run inputs and outputs + required: true + default: HumanStudy-Hub/humanstudy-hub-jobs + type: string + +permissions: + contents: read + +concurrency: + group: humanstudy-playground-${{ inputs.run_id }} + cancel-in-progress: true + +jobs: + run-study: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out the benchmark + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out private run storage + uses: actions/checkout@v4 + with: + repository: ${{ inputs.jobs_repo }} + token: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + path: run-storage + ref: runs/${{ inputs.run_id }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Set up Node for the charting agent + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install numpy scipy pandas tqdm openai cryptography + npm install --global @anthropic-ai/claude-code@latest + + - name: Mark run running + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json" + data = json.loads(path.read_text()) + data.update({"status": "running", "message": "Participants are running the study"}) + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + - name: Publish running state + working-directory: run-storage + env: + RUN_ID: ${{ inputs.run_id }} + TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + run: | + git config user.name "HumanStudy-Hub Playground" + git config user.email "playground@humanstudy-hub.org" + git add "runs/$RUN_ID/run.json" + git commit -m "playground: start $RUN_ID" + git push "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "HEAD:runs/$RUN_ID" + + - name: Run the study + env: + RUN_ID: ${{ inputs.run_id }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PLAYGROUND_KEY_SECRET: ${{ secrets.PLAYGROUND_KEY_SECRET }} + PIPELINE_PROGRESS_TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + run: | + python playground/run_playground.py \ + --run "$GITHUB_WORKSPACE/run-storage/runs/$RUN_ID" \ + --progress-repo "${{ inputs.jobs_repo }}" \ + --progress-branch "runs/$RUN_ID" \ + --progress-path "runs/$RUN_ID/progress.json" + + # Charts and the written reading are produced by Claude Code through + # OpenRouter, then validated. This step never fails the run: an unusable + # chart set falls back to the deterministic charts. + - name: Chart and interpret the results + if: success() + continue-on-error: true + env: + RUN_ID: ${{ inputs.run_id }} + ANTHROPIC_BASE_URL: https://openrouter.ai/api + CHARTS_AGENT_TOKEN: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: "" + ANTHROPIC_DEFAULT_OPUS_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + ANTHROPIC_DEFAULT_SONNET_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + ANTHROPIC_DEFAULT_HAIKU_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + OPENROUTER_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }} + CHARTS_TIMEOUT_MINUTES: ${{ vars.PLAYGROUND_CHARTS_TIMEOUT_MINUTES || '8' }} + run: | + python playground/run_charts.py \ + --run "$GITHUB_WORKSPACE/run-storage/runs/$RUN_ID" \ + --model "$OPENROUTER_MODEL" \ + --timeout-minutes "$CHARTS_TIMEOUT_MINUTES" + + - name: Mark run complete + if: success() + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + run_dir = Path("run-storage/runs") / os.environ["RUN_ID"] + charts = run_dir / "output" / "charts.json" + if not charts.exists(): + import subprocess, sys + subprocess.run([ + sys.executable, "playground/default_charts.py", + "--analysis", str(run_dir / "output" / "analysis.json"), + "--output", str(charts), + ], check=True) + path = run_dir / "run.json" + data = json.loads(path.read_text()) + data.update({ + "status": "complete", + "message": "The run finished and the results are ready", + "resultsReady": True, + }) + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + - name: Mark failed or cancelled + if: failure() || cancelled() + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json" + data = json.loads(path.read_text()) + if data.get("status") != "failed": + data.update({"status": "failed", "message": "The playground run failed or was cancelled"}) + data["resultsReady"] = False + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + # The researcher's key only ever existed as ciphertext, and it is dropped + # here so a finished run keeps no copy of it at all. + - name: Drop the supplied key + if: always() + env: + RUN_ID: ${{ inputs.run_id }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json" + data = json.loads(path.read_text()) + if data.pop("sealedApiKey", None) is not None: + data["usedOwnKey"] = True + path.write_text(json.dumps(data, indent=2) + "\n") + PY + + - name: Commit run state and outputs + if: always() + working-directory: run-storage + env: + RUN_ID: ${{ inputs.run_id }} + TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }} + run: | + git config user.name "HumanStudy-Hub Playground" + git config user.email "playground@humanstudy-hub.org" + git add "runs/$RUN_ID" + git commit -m "playground: results for $RUN_ID" || true + git pull --rebase "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "runs/$RUN_ID" + git push "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "HEAD:runs/$RUN_ID" diff --git a/README.md b/README.md index 08a8bca..0739eba 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,30 @@ missing-information checklist. The older staged Python implementation remains in `generation_pipeline/` for reference and migration tests, but the web builder no longer dispatches it. +## Playground + +The playground replays a single study with a model of your choice and scores it +against the paper, so you can see where an agent behaves like the original +participants and where it does not. + +```bash +OPENROUTER_API_KEY=... python playground/run_playground.py --run +``` + +It drives each study's own trial builder and evaluator, so a playground run is +scored exactly like a benchmark run; the model, the participant prompt, and the +number of participants are what you change. Results are charted and interpreted +by Claude Code, with deterministic charts as the fallback. HumanStudy-Hub +dispatches the same code through +[`run-playground.yml`](.github/workflows/run-playground.yml). See +[`playground/README.md`](playground/README.md). + ## Repository structure ```text studies/ Runnable benchmark studies src/ Execution and evaluation code +playground/ On-demand single-study runs and result comparison agent_pipeline/ Claude Code package contract and validation generation_pipeline/ Legacy staged extraction implementation scripts/ Study validation and index utilities diff --git a/playground/README.md b/playground/README.md new file mode 100644 index 0000000..06f0066 --- /dev/null +++ b/playground/README.md @@ -0,0 +1,87 @@ +# Playground + +Replay one benchmark study with a model of your choice and see where the agent +matched the published human result and where it did not. + +This is the engine behind the HumanStudy-Hub playground. It does not +reimplement any study: it drives each study's own `scripts/config.py` to build +trials and its own `scripts/evaluator.py` to score the run, so a playground run +is scored exactly like a benchmark run. + +## Run it locally + +```bash +python playground/run_playground.py --run +``` + +The run directory holds `run.json` and receives `progress.json`, `logs/`, and +`output/`: + +```json +{ + "id": "local-1", + "studyId": "study_001", + "model": "openai/gpt-4o-mini", + "preset": "v3_human_plus_demo", + "systemPrompt": "", + "participantsPerScenario": 8, + "temperature": 1.0, + "seed": 42, + "demographics": { "age": 21, "gender": "female", "background": "undergraduate" } +} +``` + +`OPENROUTER_API_KEY` must be set. Every model — including Claude and Gemini +names — is routed through OpenRouter. Add `--simulate` to exercise the whole +path with simulated participants and no API calls. + +Outputs: + +| File | Contents | +|---|---| +| `output/responses.json` | Every participant response, in the shape evaluators read | +| `output/evaluation.json` | The study evaluator's own output | +| `output/analysis.json` | One row per statistical test: human result vs. agent result | +| `output/transcript_sample.json` | A few complete participant exchanges | +| `output/charts.json` | Plotly chart specifications and a written reading of the run | + +## Prompts + +`preset` selects a shipped participant prompt from +`src/agents/custom_methods/`: `v1_empty`, `v2_human`, `v3_human_plus_demo`, or +`v4_background`. Set `preset` to `custom` and supply `systemPrompt` to run a +prompt you wrote yourself. `demographics` overrides the sampled participant +profile — age, gender, education, background, population, persona — which is +what the demographic presets read. + +## Charts + +```bash +python playground/run_charts.py --run +``` + +Claude Code reads the finished run and writes `output/charts.json` following +[`CLAUDE.md`](CLAUDE.md). Its output is checked by `validate_charts.py` before +it is kept: charts must be plain JSON, restricted to an allowlist of trace +types. Anything invalid, missing, or late falls back to the deterministic charts +in `default_charts.py`, so a run always ends with readable charts. + +## Run budget + +A run on the shared HumanStudy-Hub key is capped at 60 participant sessions. +A researcher who supplies their own OpenRouter key gets 600. The web app seals +that key with AES-256-GCM under `PLAYGROUND_KEY_SECRET`, so the private jobs +repository only ever holds ciphertext, and the workflow drops it once the run +ends. + +## In CI + +[`run-playground.yml`](../.github/workflows/run-playground.yml) is dispatched by +HumanStudy-Hub. Repository settings it needs: + +| Type | Name | Value | +|---|---|---| +| Actions secret | `OPENROUTER_API_KEY` | OpenRouter key for shared runs and the charting agent | +| Actions secret | `HUMANSTUDY_PIPELINE_TOKEN` | Token with read/write access to the private jobs repository | +| Actions secret | `PLAYGROUND_KEY_SECRET` | Shared secret that opens a researcher's own key; must match the web app | +| Actions variable | `PLAYGROUND_CHARTS_MODEL` | Optional; defaults to `anthropic/claude-sonnet-5` | diff --git a/playground/__init__.py b/playground/__init__.py new file mode 100644 index 0000000..ef4c006 --- /dev/null +++ b/playground/__init__.py @@ -0,0 +1 @@ +"""Playground: run a benchmark study against a chosen model on demand.""" diff --git a/playground/analysis.py b/playground/analysis.py new file mode 100644 index 0000000..847383d --- /dev/null +++ b/playground/analysis.py @@ -0,0 +1,219 @@ +"""Turn a study evaluator's output into one comparison table the web app can plot. + +Study evaluators do not agree on a single shape. Most of them enrich their test +results through `stats_lib.add_statistical_replication_fields` and already carry +human and agent effect sizes; a few return raw t-tests. This module normalises +both into the same rows: one row per statistical test, with the human result and +the agent result side by side. +""" + +import json +import math +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +Number = Optional[float] + + +def _number(value: Any) -> Number: + if value is None or isinstance(value, bool): + return None + try: + result = float(value) + except (TypeError, ValueError): + return None + return None if math.isnan(result) or math.isinf(result) else result + + +def _flag(value: Any) -> Optional[bool]: + return bool(value) if isinstance(value, bool) else None + + +def _first(row: Dict[str, Any], *keys: str) -> Any: + for key in keys: + if row.get(key) is not None: + return row[key] + return None + + +def human_effect_from_reported(reported: str) -> Number: + """Convert a reported test statistic into a Cohen's d equivalent. + + Papers report the statistic, not the effect size, so the published + `F(1, df)`, `t(df)`, or `r(df)` is converted with the standard identities. + Anything else (chi-square, multi-numerator F) has no single-d equivalent and + is left out of the effect-size comparison. + """ + if not reported: + return None + text = str(reported) + t_match = re.search(r"\bt\s*\(\s*(\d+(?:\.\d+)?)\s*\)\s*=\s*(-?\d+(?:\.\d+)?)", text) + if t_match: + df, statistic = float(t_match.group(1)), float(t_match.group(2)) + return (2 * statistic / math.sqrt(df)) if df > 0 else None + f_match = re.search(r"\bF\s*\(\s*(\d+)\s*,\s*(\d+(?:\.\d+)?)\s*\)\s*=\s*(-?\d+(?:\.\d+)?)", text) + if f_match: + df1, df2, statistic = int(f_match.group(1)), float(f_match.group(2)), float(f_match.group(3)) + if df1 == 1 and df2 > 0 and statistic >= 0: + return 2 * math.sqrt(statistic / df2) + return None + r_match = re.search(r"\br\s*\(\s*(\d+)\s*\)\s*=\s*(-?\d+(?:\.\d+)?)", text) + if r_match: + correlation = float(r_match.group(2)) + denominator = 1 - correlation ** 2 + return (2 * correlation / math.sqrt(denominator)) if denominator > 0 else None + return None + + +def agent_effect_from_t(t_statistic: Number, n1: Number, n2: Number) -> Number: + """Cohen's d for an independent-samples t-test.""" + if t_statistic is None or not n1: + return None + if n2: + return t_statistic * math.sqrt(1 / n1 + 1 / n2) + return t_statistic / math.sqrt(n1) + + +def _ground_truth_index(study_path: Path) -> Dict[tuple, Dict[str, Any]]: + """Map (study label, finding id) to the paper's reported statistics.""" + source = study_path / "source" + path = (source if source.is_dir() else study_path) / "ground_truth.json" + if not path.exists(): + return {} + try: + with open(path, "r", encoding="utf-8", errors="replace") as handle: + ground_truth = json.load(handle) + except (OSError, json.JSONDecodeError): + return {} + index: Dict[tuple, Dict[str, Any]] = {} + for sub_study in ground_truth.get("studies", []) or []: + label = sub_study.get("study_id") + for finding in sub_study.get("findings", []) or []: + tests = finding.get("statistical_tests") or [{}] + entry = { + "hypothesis": finding.get("main_hypothesis"), + "reported_statistics": tests[0].get("reported_statistics"), + "test_name": tests[0].get("test_name"), + "expected_direction": tests[0].get("expected_direction"), + } + index[(label, finding.get("finding_id"))] = entry + index[(None, finding.get("finding_id"))] = entry + return index + + +def _label(row: Dict[str, Any], fallback: str) -> str: + parts = [str(row[key]) for key in ("study_id", "finding_id", "scenario") if row.get(key)] + if not parts: + parts = [str(row.get("test_name") or fallback)] + return " · ".join(parts).replace("_", " ") + + +def normalise_tests(evaluation: Dict[str, Any], study_path: Path) -> List[Dict[str, Any]]: + raw_tests = evaluation.get("test_results") or [] + ground_truth = _ground_truth_index(study_path) + rows: List[Dict[str, Any]] = [] + + for index, raw in enumerate(raw_tests): + if not isinstance(raw, dict): + continue + gt = ground_truth.get((raw.get("study_id"), raw.get("finding_id"))) or ground_truth.get((None, raw.get("finding_id"))) or {} + + agent_p = _number(_first(raw, "p_value_agent", "p_value", "agent_p_value")) + human_p = _number(_first(raw, "p_value_human", "human_p_value")) + agent_effect = _number(_first(raw, "agent_effect_d", "agent_effect_size")) + human_effect = _number(_first(raw, "human_effect_d", "human_effect_size")) + n_agent_1 = _number(_first(raw, "n_agent_1", "n_agent", "n_agent_extracted")) + n_agent_2 = _number(_first(raw, "n_agent_2", "n2_agent", "n2_agent_extracted")) + + if agent_effect is None: + agent_effect = agent_effect_from_t(_number(_first(raw, "t_stat", "test_statistic", "statistic")), n_agent_1, n_agent_2) + if human_effect is None: + human_effect = human_effect_from_reported(gt.get("reported_statistics") or raw.get("reported_statistics") or "") + + agent_significant = _flag(_first(raw, "is_significant_agent", "significant", "agent_significant")) + human_significant = _flag(_first(raw, "is_significant_human", "human_significant")) + direction_match = _flag(raw.get("direction_match")) + replicated = _flag(raw.get("replication")) + if replicated is None and None not in (human_significant, agent_significant, direction_match): + replicated = bool(human_significant and agent_significant and direction_match) + + rows.append({ + "test_id": f"T{index + 1}", + "study_label": raw.get("study_id"), + "sub_study_id": raw.get("sub_study_id"), + "finding_id": raw.get("finding_id"), + "scenario": raw.get("scenario"), + "label": _label(raw, f"Test {index + 1}"), + "test_type": _first(raw, "statistical_test_type", "test_type", "test"), + "hypothesis": gt.get("hypothesis"), + "reported_statistics": gt.get("reported_statistics"), + "human_effect": human_effect, + "agent_effect": agent_effect, + "human_p": human_p, + "agent_p": agent_p, + "human_significant": human_significant, + "agent_significant": agent_significant, + "direction_match": direction_match, + "replicated": replicated, + "agent_mean_1": _number(_first(raw, "mean_agent_1", "agent_mean_1")), + "agent_mean_2": _number(_first(raw, "mean_agent_2", "agent_mean_2")), + "n_agent_1": int(n_agent_1) if n_agent_1 else None, + "n_agent_2": int(n_agent_2) if n_agent_2 else None, + "pas": _number(_first(raw, "pas", "score")), + }) + return rows + + +def summarise(rows: List[Dict[str, Any]], evaluation: Dict[str, Any]) -> Dict[str, Any]: + scored = [row for row in rows if row["replicated"] is not None] + replicated = [row for row in scored if row["replicated"]] + comparable = [row for row in rows if row["human_effect"] is not None and row["agent_effect"] is not None] + directional = [row for row in rows if row["direction_match"] is not None] + + gaps = [abs(row["agent_effect"] - row["human_effect"]) for row in comparable] + human_effects = [row["human_effect"] for row in comparable] + agent_effects = [row["agent_effect"] for row in comparable] + + return { + "totalTests": len(rows), + "scoredTests": len(scored), + "replicatedTests": len(replicated), + "replicationRate": (len(replicated) / len(scored)) if scored else None, + "directionMatchRate": (sum(1 for row in directional if row["direction_match"]) / len(directional)) if directional else None, + "meanAbsoluteEffectGap": (sum(gaps) / len(gaps)) if gaps else None, + "meanHumanEffect": (sum(human_effects) / len(human_effects)) if human_effects else None, + "meanAgentEffect": (sum(agent_effects) / len(agent_effects)) if agent_effects else None, + "effectCorrelation": _correlation(human_effects, agent_effects), + "studyScore": _number(evaluation.get("score")), + } + + +def _correlation(x: List[float], y: List[float]) -> Number: + if len(x) < 3: + return None + mean_x = sum(x) / len(x) + mean_y = sum(y) / len(y) + covariance = sum((a - mean_x) * (b - mean_y) for a, b in zip(x, y)) + variance_x = sum((a - mean_x) ** 2 for a in x) + variance_y = sum((b - mean_y) ** 2 for b in y) + if variance_x <= 0 or variance_y <= 0: + return None + return covariance / math.sqrt(variance_x * variance_y) + + +def build_analysis(evaluation: Dict[str, Any], study_path: Path, run: Dict[str, Any], responses: Dict[str, Any]) -> Dict[str, Any]: + rows = normalise_tests(evaluation, study_path) + return { + "runId": run.get("id"), + "studyId": run.get("studyId"), + "studyTitle": run.get("studyTitle"), + "model": run.get("model"), + "promptPreset": run.get("preset"), + "participants": responses.get("participants"), + "trials": responses.get("trials"), + "completedTrials": responses.get("completedTrials"), + "summary": summarise(rows, evaluation), + "tests": rows, + "findings": evaluation.get("finding_results") or [], + } diff --git a/playground/default_charts.py b/playground/default_charts.py new file mode 100644 index 0000000..dc7feac --- /dev/null +++ b/playground/default_charts.py @@ -0,0 +1,152 @@ +"""Deterministic comparison charts for a playground run. + +These are the charts the playground falls back to when the charting agent does +not produce a valid set, and they are also the reference the agent is asked to +improve on. Everything here is derived from `output/analysis.json` alone. +""" + +import json +from pathlib import Path +from typing import Any, Dict, List + +SIGNIFICANT = "#0e7490" +NOT_SIGNIFICANT = "#a5e5ea" +HUMAN = "#334155" +AGENT = "#0e7490" + + +def _axis_bounds(values: List[float]) -> tuple[float, float]: + if not values: + return -1.0, 1.0 + low, high = min(values), max(values) + padding = max(0.15, (high - low) * 0.1) + return low - padding, high + padding + + +def effect_scatter(analysis: Dict[str, Any]) -> Dict[str, Any] | None: + """Agent effect size against the published human effect size.""" + points = [test for test in analysis["tests"] if test["human_effect"] is not None and test["agent_effect"] is not None] + if not points: + return None + significant = [test for test in points if test["agent_significant"]] + other = [test for test in points if not test["agent_significant"]] + low, high = _axis_bounds([test["human_effect"] for test in points] + [test["agent_effect"] for test in points]) + + def trace(rows: List[Dict[str, Any]], name: str, colour: str) -> Dict[str, Any]: + return { + "type": "scatter", + "mode": "markers", + "name": name, + "x": [row["human_effect"] for row in rows], + "y": [row["agent_effect"] for row in rows], + "text": [row["label"] for row in rows], + "hovertemplate": "%{text}
Human d=%{x:.2f}
Agent d=%{y:.2f}", + "marker": {"size": 11, "color": colour, "line": {"color": "white", "width": 1}}, + } + + data = [trace(rows, name, colour) for rows, name, colour in ( + (significant, "Agent effect significant", SIGNIFICANT), + (other, "Not significant", NOT_SIGNIFICANT), + ) if rows] + data.append({ + "type": "scatter", + "mode": "lines", + "name": "Human = agent", + "x": [low, high], + "y": [low, high], + "line": {"color": "#94a3b8", "width": 1.5, "dash": "dash"}, + "hoverinfo": "skip", + }) + return { + "id": "effect-scatter", + "title": "Agent effect size vs. published human effect size", + "description": "Each point is one statistical test from the paper. Points on the dashed line mean the agent reproduced the human effect at the same magnitude; points below it mean the agent showed a weaker effect than people did.", + "plotly": { + "data": data, + "layout": { + "xaxis": {"title": {"text": "Human effect size (d)"}, "range": [low, high], "zeroline": True}, + "yaxis": {"title": {"text": "Agent effect size (d)"}, "range": [low, high], "scaleanchor": "x", "zeroline": True}, + "legend": {"orientation": "h", "y": -0.2}, + "margin": {"t": 20, "r": 16, "b": 60, "l": 60}, + }, + }, + } + + +def effect_bars(analysis: Dict[str, Any]) -> Dict[str, Any] | None: + """Human and agent effect sizes side by side, per test.""" + points = [test for test in analysis["tests"] if test["human_effect"] is not None or test["agent_effect"] is not None][:20] + if not points: + return None + labels = [test["label"] for test in points] + return { + "id": "effect-bars", + "title": "Effect size by test", + "description": "How large the effect was for humans in the paper and for the agent in this run, test by test.", + "plotly": { + "data": [ + {"type": "bar", "name": "Human", "x": labels, "y": [test["human_effect"] for test in points], "marker": {"color": HUMAN}}, + {"type": "bar", "name": "Agent", "x": labels, "y": [test["agent_effect"] for test in points], "marker": {"color": AGENT}}, + ], + "layout": { + "barmode": "group", + "xaxis": {"tickangle": -35, "automargin": True}, + "yaxis": {"title": {"text": "Effect size (d)"}, "zeroline": True}, + "legend": {"orientation": "h", "y": -0.25}, + "margin": {"t": 20, "r": 16, "b": 80, "l": 60}, + }, + }, + } + + +def replication_breakdown(analysis: Dict[str, Any]) -> Dict[str, Any] | None: + """How many published findings the agent reproduced, missed, or reversed.""" + scored = [test for test in analysis["tests"] if test["replicated"] is not None or test["direction_match"] is not None] + if not scored: + return None + reproduced = sum(1 for test in scored if test["replicated"]) + wrong_direction = sum(1 for test in scored if not test["replicated"] and test["direction_match"] is False) + missed = len(scored) - reproduced - wrong_direction + return { + "id": "replication-breakdown", + "title": "What the agent did with each published finding", + "description": "Reproduced means the agent found the same effect, in the same direction, at the same significance level as the paper. Wrong direction means the agent showed the opposite effect.", + "plotly": { + "data": [{ + "type": "bar", + "orientation": "h", + "x": [reproduced, missed, wrong_direction], + "y": ["Reproduced", "No significant effect", "Wrong direction"], + "marker": {"color": [SIGNIFICANT, "#cbd5e1", "#f59e0b"]}, + "hovertemplate": "%{y}: %{x} tests", + }], + "layout": { + "xaxis": {"title": {"text": "Statistical tests"}, "dtick": 1}, + "yaxis": {"automargin": True}, + "showlegend": False, + "margin": {"t": 20, "r": 16, "b": 45, "l": 20}, + }, + }, + } + + +def build_charts(analysis: Dict[str, Any]) -> Dict[str, Any]: + charts = [chart for chart in (effect_scatter(analysis), effect_bars(analysis), replication_breakdown(analysis)) if chart] + return {"charts": charts, "source": "default"} + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser(description="Write the default playground charts for a run.") + parser.add_argument("--analysis", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + analysis = json.loads(args.analysis.read_text()) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(build_charts(analysis), indent=2) + "\n") + print(f"Wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/playground/progress.py b/playground/progress.py new file mode 100644 index 0000000..4a937c4 --- /dev/null +++ b/playground/progress.py @@ -0,0 +1,85 @@ +"""Publish playground run progress so the web app can show it while the run works. + +The run writes `progress.json` next to its other files and, when a token is +available, mirrors the same payload to the private jobs repository through the +GitHub contents API. The web app polls that file. +""" + +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional + + +class ProgressWriter: + def __init__(self, run_dir: Path, repo: Optional[str] = None, branch: Optional[str] = None, path: Optional[str] = None, min_interval: float = 10.0) -> None: + self.local_path = run_dir / "progress.json" + self.token = os.environ.get("PIPELINE_PROGRESS_TOKEN", "") + self.repo = repo + self.branch = branch + self.path = path + self.min_interval = min_interval + self.sha: Optional[str] = None + self._last_publish = 0.0 + if self._remote_enabled(): + self._load_sha() + + def _remote_enabled(self) -> bool: + return bool(self.token and self.repo and self.branch and self.path) + + def _request(self, method: str, body: Optional[dict] = None) -> dict: + url = f"https://api.github.com/repos/{self.repo}/contents/{urllib.parse.quote(self.path or '', safe='/')}" + if method == "GET": + url += "?ref=" + urllib.parse.quote(self.branch or "", safe="") + request = urllib.request.Request( + url, + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "HumanStudy-Hub-Playground", + }, + ) + with urllib.request.urlopen(request, timeout=20) as response: + return json.loads(response.read()) + + def _load_sha(self) -> None: + try: + self.sha = self._request("GET").get("sha") + except urllib.error.HTTPError as exc: + if exc.code != 404: + print(f"[progress] could not read remote progress: {exc}", flush=True) + except Exception as exc: + print(f"[progress] could not read remote progress: {exc}", flush=True) + + def write(self, payload: Dict[str, Any], force: bool = False) -> None: + payload = {**payload, "updatedAt": datetime.now(timezone.utc).isoformat()} + self.local_path.parent.mkdir(parents=True, exist_ok=True) + self.local_path.write_text(json.dumps(payload, indent=2) + "\n") + if not self._remote_enabled(): + return + # Publishing is a commit per call, so a long run only mirrors on an interval. + if not force and time.monotonic() - self._last_publish < self.min_interval: + return + body = { + "message": f"playground: progress {payload.get('phase', 'running')}", + "branch": self.branch, + "content": base64.b64encode((json.dumps(payload, indent=2) + "\n").encode()).decode(), + } + if self.sha: + body["sha"] = self.sha + try: + response = self._request("PUT", body) + self.sha = response["content"]["sha"] + self._last_publish = time.monotonic() + except Exception as exc: + print(f"[progress] could not publish progress: {exc}", flush=True) diff --git a/playground/run_charts.py b/playground/run_charts.py new file mode 100644 index 0000000..3fc90b9 --- /dev/null +++ b/playground/run_charts.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Have Claude Code chart and interpret a finished playground run. + +The agent is given the run directory and the contract in `playground/CLAUDE.md`, +and writes `output/charts.json`. Whatever it produces is validated before it is +kept; if it is missing, invalid, or the agent runs out of time, the deterministic +charts from `default_charts.py` are used instead. A run therefore always ends +with charts a researcher can read. +""" + +import argparse +import json +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any, Dict + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from playground.default_charts import build_charts +from playground.validate_charts import ChartError, validate + + +def build_prompt(contract: Path, run_dir: Path) -> str: + return ( + f"{contract.read_text()}\n\n" + "## This run\n\n" + f"The run directory is `{run_dir}`. Read `run.json`, `output/analysis.json`, " + "`output/evaluation.json`, `output/transcript_sample.json`, and " + "`output/charts.default.json` there.\n\n" + f"Write your charts and interpretation to `{run_dir / 'output' / 'charts.json'}`. " + "Write that file exactly once, when it is complete and valid, then stop.\n" + ) + + +def stream_output(process: "subprocess.Popen[str]", log_path: Path) -> None: + with log_path.open("a", encoding="utf-8") as log: + assert process.stdout is not None + for line in process.stdout: + sys.stdout.write(line) + sys.stdout.flush() + log.write(line) + log.flush() + + +def stop_process(process: "subprocess.Popen[str]") -> None: + if process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=20) + except (ProcessLookupError, subprocess.TimeoutExpired): + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=10) + + +def accept_charts(path: Path) -> Dict[str, Any] | None: + """Return the agent's charts when they are present and valid.""" + if not path.exists(): + return None + try: + document = json.loads(path.read_text()) + validate(document) + except (OSError, json.JSONDecodeError) as error: + print(f"[charts] agent output could not be read: {error}", flush=True) + return None + except ChartError as error: + print(f"[charts] agent output rejected: {error}", flush=True) + return None + document["source"] = "agent" + return document + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", required=True, type=Path) + parser.add_argument("--contract", type=Path, default=Path(__file__).resolve().parent / "CLAUDE.md") + parser.add_argument("--model", default=os.environ.get("OPENROUTER_MODEL", "anthropic/claude-sonnet-5")) + parser.add_argument("--timeout-minutes", type=float, default=8) + args = parser.parse_args() + + run_dir = args.run.resolve() + output_dir = run_dir / "output" + logs_dir = run_dir / "logs" + output_dir.mkdir(parents=True, exist_ok=True) + logs_dir.mkdir(parents=True, exist_ok=True) + + analysis_path = output_dir / "analysis.json" + if not analysis_path.exists(): + raise SystemExit("This run has no analysis.json, so there is nothing to chart.") + analysis = json.loads(analysis_path.read_text()) + + # The baseline is written first: it is the agent's reference and the fallback. + fallback = build_charts(analysis) + (output_dir / "charts.default.json").write_text(json.dumps(fallback, indent=2) + "\n") + + charts_path = output_dir / "charts.json" + prompt_path = run_dir / "charts-prompt.md" + prompt_path.write_text(build_prompt(args.contract, run_dir)) + + agent_env = os.environ.copy() + for secret in ("PIPELINE_PROGRESS_TOKEN", "HUMANSTUDY_PIPELINE_TOKEN", "GITHUB_TOKEN", "OPENROUTER_API_KEY", "PLAYGROUND_KEY_SECRET"): + agent_env.pop(secret, None) + agent_env["ANTHROPIC_AUTH_TOKEN"] = os.environ.get("CHARTS_AGENT_TOKEN", os.environ.get("ANTHROPIC_AUTH_TOKEN", "")) + + process = subprocess.Popen( + [ + "claude", + "--print", + "--model", args.model, + "--add-dir", str(run_dir), + "--dangerously-skip-permissions", + prompt_path.read_text(), + ], + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + start_new_session=True, + env=agent_env, + ) + output_thread = threading.Thread(target=stream_output, args=(process, logs_dir / "charts.log"), daemon=True) + output_thread.start() + deadline = time.monotonic() + args.timeout_minutes * 60 + + try: + while process.poll() is None: + if time.monotonic() >= deadline: + print("[charts] the charting agent ran out of time", flush=True) + stop_process(process) + break + time.sleep(5) + finally: + stop_process(process) + output_thread.join(timeout=5) + + document = accept_charts(charts_path) + if document is None: + print("[charts] using the deterministic charts for this run", flush=True) + document = fallback + charts_path.write_text(json.dumps(document, indent=2) + "\n") + print(f"[charts] wrote {len(document['charts'])} charts from the {document['source']} source", flush=True) + + +if __name__ == "__main__": + main() diff --git a/playground/run_key.py b/playground/run_key.py new file mode 100644 index 0000000..9e309f1 --- /dev/null +++ b/playground/run_key.py @@ -0,0 +1,49 @@ +"""Decrypt the OpenRouter key a researcher supplied for their own playground run. + +HumanStudy-Hub never stores a researcher's key in plaintext. The web app encrypts +it with AES-256-GCM under a secret shared with this workflow +(`PLAYGROUND_KEY_SECRET`), so the private jobs repository only ever holds +ciphertext. This module reverses that on the runner. +""" + +import base64 +import hashlib +import os +from typing import Any, Mapping, Optional + + +def _shared_secret() -> Optional[bytes]: + secret = os.environ.get("PLAYGROUND_KEY_SECRET", "").strip() + if not secret: + return None + # The web app derives the same 32-byte key with SHA-256 over the secret. + return hashlib.sha256(secret.encode("utf-8")).digest() + + +def decrypt_api_key(sealed: Optional[Mapping[str, Any]]) -> Optional[str]: + """Return the researcher's OpenRouter key, or None when the run has none. + + Raises ValueError when a key was supplied but cannot be opened, because + silently falling back to the shared key would spend the project's budget on + a run the researcher expected to pay for. + """ + if not sealed: + return None + key = _shared_secret() + if key is None: + raise ValueError("PLAYGROUND_KEY_SECRET is not configured, so the supplied API key cannot be opened.") + try: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + iv = base64.b64decode(sealed["iv"]) + ciphertext = base64.b64decode(sealed["ciphertext"]) + tag = base64.b64decode(sealed["tag"]) + plaintext = AESGCM(key).decrypt(iv, ciphertext + tag, None) + except KeyError as exc: + raise ValueError(f"The sealed API key is missing the {exc} field.") from exc + except Exception as exc: + raise ValueError("The supplied API key could not be decrypted with the configured secret.") from exc + opened = plaintext.decode("utf-8").strip() + if not opened: + raise ValueError("The supplied API key decrypted to an empty value.") + return opened diff --git a/playground/run_playground.py b/playground/run_playground.py new file mode 100644 index 0000000..d946624 --- /dev/null +++ b/playground/run_playground.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Replay one benchmark study with a chosen model and score it against the paper. + +This is the engine behind the HumanStudy-Hub playground. It reuses the study's +own trial builder and evaluator, so a playground run is scored exactly like a +benchmark run — the only things the researcher changes are the model, the +participant prompt, and how many participants take part. + + python playground/run_playground.py --run + +The run directory holds `run.json` (written by the web app) and receives +`progress.json`, `output/`, and `logs/`. +""" + +import argparse +import json +import os +import random +import sys +import traceback +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from playground import settings +from playground.analysis import build_analysis +from playground.progress import ProgressWriter +from playground.run_key import decrypt_api_key +from playground.study_loader import StudyNotRunnable, load_metadata, load_study + + +def now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read_run(run_dir: Path) -> Dict[str, Any]: + path = run_dir / "run.json" + if not path.exists(): + raise SystemExit(f"No run.json in {run_dir}") + return json.loads(path.read_text()) + + +def write_run(run_dir: Path, run: Dict[str, Any]) -> None: + run["updatedAt"] = now() + (run_dir / "run.json").write_text(json.dumps(run, indent=2) + "\n") + + +def resolve_prompt(run: Dict[str, Any]) -> tuple[str, Optional[str]]: + """Return (preset name passed to the registry, explicit prompt override).""" + preset = str(run.get("preset") or settings.DEFAULT_PRESET) + custom = (run.get("systemPrompt") or "").strip() + if preset == settings.CUSTOM_PRESET: + if not custom: + raise SystemExit("This run selected a custom prompt but did not include one.") + return settings.DEFAULT_PRESET, custom + if preset not in settings.PROMPT_PRESETS: + raise SystemExit(f"Unknown prompt preset: {preset}") + return preset, custom or None + + +def apply_demographics(profile: Dict[str, Any], demographics: Dict[str, Any]) -> Dict[str, Any]: + """Overlay the researcher's demographic choices on a generated profile.""" + merged = dict(profile) + for field in settings.DEMOGRAPHIC_FIELDS: + value = demographics.get(field) + if value not in (None, "", []): + merged[field] = value + return merged + + +def cap_trials(study_config: Any, run: Dict[str, Any], has_own_key: bool, log) -> List[Dict[str, Any]]: + """Build trials at the requested size, then hold the run inside its budget.""" + max_total, max_per_scenario = settings.trial_limits(has_own_key) + requested = int(run.get("participantsPerScenario") or settings.DEFAULT_PER_SCENARIO) + per_scenario = max(1, min(requested, max_per_scenario)) + if per_scenario != requested: + log(f"Requested {requested} participants per condition; capped to {per_scenario} for this run budget.") + + try: + trials = study_config.create_trials(n_trials=per_scenario) + except TypeError: + # A few studies take extra keyword arguments and ignore n_trials. + trials = study_config.create_trials() + trials = [trial for trial in trials if isinstance(trial, dict)] + if not trials: + raise SystemExit("The study produced no trials to run.") + + if len(trials) > max_total: + # Subsample evenly across conditions so every condition keeps participants. + rng = random.Random(int(run.get("seed") or 42)) + by_scenario: Dict[str, List[Dict[str, Any]]] = {} + for trial in trials: + key = str(trial.get("scenario_id") or trial.get("scenario") or trial.get("sub_study_id") or "all") + by_scenario.setdefault(key, []).append(trial) + quota = max(1, max_total // len(by_scenario)) + kept: List[Dict[str, Any]] = [] + for group in by_scenario.values(): + rng.shuffle(group) + kept.extend(group[:quota]) + trials = kept[:max_total] + log(f"Run budget allows {max_total} participant sessions; sampled {len(trials)} across {len(by_scenario)} conditions.") + return trials + + +def build_profiles(trials: List[Dict[str, Any]], specification: Dict[str, Any], run: Dict[str, Any], study_id: str) -> List[Dict[str, Any]]: + demographics = run.get("demographics") or {} + population = specification.get("participants", {}).get("population") + recruitment = specification.get("participants", {}).get("recruitment_source") + profiles = [] + for index, trial in enumerate(trials): + base = dict(trial.get("profile") or {}) + base.setdefault("participant_id", index) + base.setdefault("education", recruitment or "college student") + base.setdefault("population", population) + base["study_id"] = study_id + profiles.append(apply_demographics(base, demographics)) + return profiles + + +def evaluation_input(pool_results: Dict[str, Any], study_path: Optional[Path] = None) -> Dict[str, Any]: + """Study evaluators read participants that each carry their own responses. + + The raw API envelope for every call is dropped here: it is many times larger + than the answers themselves and no evaluator reads it. + """ + participants = [] + for summary in pool_results.get("participant_summaries", []): + participants.append({ + "participant_id": summary.get("participant_id"), + "profile": summary.get("profile"), + "responses": [ + { + "participant_id": response.get("participant_id"), + "trial_number": response.get("trial_number"), + "response": response.get("response"), + "response_text": response.get("response_text"), + "is_correct": response.get("is_correct"), + "correct_answer": response.get("correct_answer"), + "trial_info": response.get("trial_info"), + } + for response in summary.get("responses", []) or [] + ], + }) + payload: Dict[str, Any] = {"individual_data": participants} + # Some evaluators resolve the study's own ground truth from this field. + if study_path is not None: + payload["study_path"] = str(study_path) + return payload + + +def evaluate_in_study_dir(evaluator: Any, scored_input: Dict[str, Any], study_path: Path) -> Dict[str, Any]: + """Score the run from inside the study folder. + + A handful of evaluators open their ground truth through a relative path, so + they only resolve correctly when the study directory is the working + directory. + """ + previous = os.getcwd() + os.chdir(study_path) + try: + return evaluator.evaluate_study(scored_input) + finally: + os.chdir(previous) + + +def response_stats(pool_results: Dict[str, Any]) -> Dict[str, Any]: + summaries = pool_results.get("participant_summaries", []) + completed = 0 + answered = 0 + tokens = 0 + for summary in summaries: + for response in summary.get("responses", []) or []: + completed += 1 + if (response.get("response_text") or "").strip(): + answered += 1 + usage = response.get("usage") or {} + tokens += int(usage.get("total_tokens") or 0) + return { + "participants": len(summaries), + "trials": completed, + "completedTrials": completed, + "answeredTrials": answered, + "totalTokens": tokens, + } + + +def sample_transcript(pool_results: Dict[str, Any], prompt_builder: Any, limit: int = 6) -> List[Dict[str, Any]]: + """A few complete participant exchanges, so researchers can read what happened.""" + samples: List[Dict[str, Any]] = [] + for summary in pool_results.get("participant_summaries", []): + profile = summary.get("profile") or {} + for response in summary.get("responses", []) or []: + trial_info = response.get("trial_info") or {} + try: + prompt = prompt_builder.build_trial_prompt({**trial_info, "participant_profile": profile}) + except Exception: + prompt = None + samples.append({ + "participantId": summary.get("participant_id"), + "profile": {key: profile.get(key) for key in ("age", "gender", "education", "background", "persona") if profile.get(key)}, + "prompt": prompt, + "response": response.get("response_text"), + }) + if len(samples) >= limit: + return samples + return samples + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", required=True, type=Path, help="Run directory containing run.json") + parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument("--progress-repo") + parser.add_argument("--progress-branch") + parser.add_argument("--progress-path") + parser.add_argument( + "--simulate", + action="store_true", + help="Exercise the run end to end with simulated participants and no API calls.", + ) + args = parser.parse_args() + + run_dir = args.run.resolve() + output_dir = run_dir / "output" + logs_dir = run_dir / "logs" + output_dir.mkdir(parents=True, exist_ok=True) + logs_dir.mkdir(parents=True, exist_ok=True) + log_file = (logs_dir / "run.log").open("a", encoding="utf-8") + + def log(message: str) -> None: + line = f"[{now()}] {message}" + print(line, flush=True) + log_file.write(line + "\n") + log_file.flush() + + run = read_run(run_dir) + progress = ProgressWriter(run_dir, args.progress_repo, args.progress_branch, args.progress_path) + progress.write({"phase": "preparing", "completedTrials": 0, "totalTrials": 0, "message": "Loading the study"}, force=True) + + try: + api_key = decrypt_api_key(run.get("sealedApiKey")) + has_own_key = api_key is not None + if not has_own_key: + api_key = os.environ.get("OPENROUTER_API_KEY") + if not api_key and not args.simulate: + raise SystemExit("No OpenRouter API key is available for this run.") + + study_path, specification, study_config, evaluator = load_study(args.repo_root, str(run.get("studyId") or "")) + metadata = load_metadata(study_path) + run["studyTitle"] = metadata.get("title") or run.get("studyTitle") + log(f"Study {study_path.name}: {run['studyTitle']}") + + preset, override = resolve_prompt(run) + trials = cap_trials(study_config, run, has_own_key, log) + profiles = build_profiles(trials, specification, run, study_path.name) + log(f"Prepared {len(trials)} participant sessions on {run.get('model')} with prompt {run.get('preset')}.") + + from src.agents.llm_participant_agent import ParticipantPool + + prompt_builder = study_config.get_prompt_builder() + pool = ParticipantPool( + study_specification=specification, + n_participants=len(trials), + use_real_llm=not args.simulate, + model=str(run.get("model") or settings.DEFAULT_MODEL), + api_key=api_key, + # Passing the OpenRouter base explicitly is what routes every model — + # including Claude and Gemini names — through OpenRouter. + api_base=settings.OPENROUTER_API_BASE, + random_seed=int(run.get("seed") or 42), + num_workers=int(run.get("workers") or settings.DEFAULT_WORKERS), + profiles=profiles, + prompt_builder=prompt_builder, + system_prompt_override=override, + system_prompt_preset=preset, + study_id=study_path.name, + temperature=float(run.get("temperature") or 1.0), + ) + + state = {"done": 0} + + def on_trial(*_: Any) -> None: + state["done"] += 1 + progress.write({ + "phase": "running_participants", + "completedTrials": state["done"], + "totalTrials": len(trials), + "message": f"{state['done']} of {len(trials)} participant sessions complete", + }) + + progress.write({"phase": "running_participants", "completedTrials": 0, "totalTrials": len(trials), "message": "Participants are starting"}, force=True) + pool.run_experiment( + trials=trials, + instructions=study_config.get_instructions(), + prompt_builder=prompt_builder, + one_to_one=True, + save_callback=on_trial, + ) + + pool_results = pool.aggregate_results() + stats = response_stats(pool_results) + scored_input = evaluation_input(pool_results, study_path) + log(f"Collected {stats['answeredTrials']} answers from {stats['participants']} participants.") + (output_dir / "responses.json").write_text(json.dumps(scored_input, indent=2, default=str) + "\n") + (output_dir / "transcript_sample.json").write_text(json.dumps(sample_transcript(pool_results, prompt_builder), indent=2, default=str) + "\n") + + progress.write({"phase": "scoring", "completedTrials": stats["completedTrials"], "totalTrials": len(trials), "message": "Scoring against the published findings"}, force=True) + evaluation = evaluate_in_study_dir(evaluator, scored_input, study_path) + (output_dir / "evaluation.json").write_text(json.dumps(evaluation, indent=2, default=str) + "\n") + + analysis = build_analysis(evaluation, study_path, run, stats) + (output_dir / "analysis.json").write_text(json.dumps(analysis, indent=2, default=str) + "\n") + summary = analysis["summary"] + log( + "Replication: " + f"{summary['replicatedTests']}/{summary['scoredTests']} tests reproduced the published direction and significance." + ) + + run.update({ + "status": "analysing", + "message": "Building the comparison charts", + "participants": stats["participants"], + "completedTrials": stats["completedTrials"], + "answeredTrials": stats["answeredTrials"], + "totalTokens": stats["totalTokens"], + "summary": summary, + }) + write_run(run_dir, run) + progress.write({"phase": "charting", "completedTrials": stats["completedTrials"], "totalTrials": len(trials), "message": "Building the comparison charts"}, force=True) + + except StudyNotRunnable as error: + log(f"Study cannot be replayed: {error}") + run.update({"status": "failed", "message": "This study cannot be replayed in the playground", "error": str(error)}) + write_run(run_dir, run) + progress.write({"phase": "failed", "message": str(error)}, force=True) + raise SystemExit(1) + except SystemExit: + raise + except Exception as error: + log(f"Run failed: {error}\n{traceback.format_exc()}") + run.update({"status": "failed", "message": "The playground run failed", "error": str(error)}) + write_run(run_dir, run) + progress.write({"phase": "failed", "message": str(error)}, force=True) + raise SystemExit(1) + finally: + log_file.close() + + +if __name__ == "__main__": + main() diff --git a/playground/settings.py b/playground/settings.py new file mode 100644 index 0000000..545afb8 --- /dev/null +++ b/playground/settings.py @@ -0,0 +1,32 @@ +"""Shared playground constants: run limits, prompt presets, and OpenRouter routing.""" + +OPENROUTER_API_BASE = "https://openrouter.ai/api/v1" + +# A run on the shared HumanStudy-Hub key is capped so one playground session +# cannot spend the project's OpenRouter budget. A contributor who supplies their +# own key is allowed the larger budget. +SHARED_KEY_MAX_TRIALS = 60 +OWN_KEY_MAX_TRIALS = 600 + +SHARED_KEY_MAX_PER_SCENARIO = 10 +OWN_KEY_MAX_PER_SCENARIO = 80 + +DEFAULT_PER_SCENARIO = 8 +DEFAULT_MODEL = "openai/gpt-4o-mini" +DEFAULT_PRESET = "v3_human_plus_demo" +DEFAULT_WORKERS = 8 + +# Presets shipped by src/agents/custom_methods/. "custom" is the playground's own +# marker for a prompt the researcher wrote in the prompt designer. +PROMPT_PRESETS = ("v1_empty", "v2_human", "v3_human_plus_demo", "v4_background") +CUSTOM_PRESET = "custom" + +# Demographic fields the prompt designer can override on every participant. +DEMOGRAPHIC_FIELDS = ("age", "gender", "education", "background", "population", "persona") + + +def trial_limits(has_own_key: bool) -> tuple[int, int]: + """Return (max total trials, max trials per scenario) for this run.""" + if has_own_key: + return OWN_KEY_MAX_TRIALS, OWN_KEY_MAX_PER_SCENARIO + return SHARED_KEY_MAX_TRIALS, SHARED_KEY_MAX_PER_SCENARIO diff --git a/playground/study_loader.py b/playground/study_loader.py new file mode 100644 index 0000000..870484b --- /dev/null +++ b/playground/study_loader.py @@ -0,0 +1,99 @@ +"""Load a study's own trial builder and evaluator from `studies//scripts/`. + +Every study ships a self-contained `scripts/config.py` (a `BaseStudyConfig` +subclass that builds trials and knows the study's prompts) and a +`scripts/evaluator.py` exposing `evaluate_study(results)`. The playground drives +those files rather than reimplementing any study logic. +""" + +import importlib.util +import inspect +import json +import sys +from pathlib import Path +from types import ModuleType +from typing import Any, Dict, Tuple + + +class StudyNotRunnable(Exception): + """The study exists but does not expose the files the playground needs.""" + + +def _import_module(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise StudyNotRunnable(f"Could not import {path}") + module = importlib.util.module_from_spec(spec) + # A study's scripts import their sibling modules by bare name, so the study's + # own scripts directory has to be importable while the module executes. + sys.modules[name] = module + scripts_dir = str(path.parent) + if scripts_dir not in sys.path: + sys.path.insert(0, scripts_dir) + spec.loader.exec_module(module) + return module + + +def study_dir(repo_root: Path, study_id: str) -> Path: + safe = "".join(character for character in study_id if character.isalnum() or character in "_-") + if not safe or safe != study_id: + raise StudyNotRunnable(f"Invalid study id: {study_id}") + path = repo_root / "studies" / safe + if not path.is_dir(): + raise StudyNotRunnable(f"Study {safe} is not in this benchmark.") + return path + + +def source_dir(path: Path) -> Path: + source = path / "source" + return source if (source / "specification.json").exists() else path + + +def load_specification(path: Path) -> Dict[str, Any]: + spec_path = source_dir(path) / "specification.json" + if not spec_path.exists(): + raise StudyNotRunnable(f"{path.name} has no specification.json, so it cannot be replayed.") + with open(spec_path, "r", encoding="utf-8", errors="replace") as handle: + return json.load(handle) + + +def load_metadata(path: Path) -> Dict[str, Any]: + for candidate in (source_dir(path) / "metadata.json", path / "index.json"): + if candidate.exists(): + with open(candidate, "r", encoding="utf-8", errors="replace") as handle: + return json.load(handle) + return {} + + +def load_study_config(path: Path, specification: Dict[str, Any]) -> Any: + """Instantiate the study's `BaseStudyConfig` subclass.""" + config_path = path / "scripts" / "config.py" + if not config_path.exists(): + raise StudyNotRunnable(f"{path.name} has no scripts/config.py, so its trials cannot be built.") + module = _import_module(config_path, f"playground_{path.name}_config") + candidates = [ + value for _, value in inspect.getmembers(module, inspect.isclass) + if value.__module__ == module.__name__ + and hasattr(value, "create_trials") + and not inspect.isabstract(value) + and any(base.__name__ == "BaseStudyConfig" for base in value.__mro__[1:]) + ] + if not candidates: + raise StudyNotRunnable(f"{path.name} has no study config class in scripts/config.py.") + return candidates[0](path, specification) + + +def load_evaluator(path: Path) -> ModuleType: + evaluator_path = path / "scripts" / "evaluator.py" + if not evaluator_path.exists(): + raise StudyNotRunnable(f"{path.name} has no scripts/evaluator.py, so agent runs cannot be scored.") + module = _import_module(evaluator_path, f"playground_{path.name}_evaluator") + if not hasattr(module, "evaluate_study"): + raise StudyNotRunnable(f"{path.name}'s evaluator does not expose evaluate_study().") + return module + + +def load_study(repo_root: Path, study_id: str) -> Tuple[Path, Dict[str, Any], Any, ModuleType]: + path = study_dir(repo_root, study_id) + specification = load_specification(path) + return path, specification, load_study_config(path, specification), load_evaluator(path) diff --git a/playground/validate_charts.py b/playground/validate_charts.py new file mode 100644 index 0000000..847b0f7 --- /dev/null +++ b/playground/validate_charts.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check that agent-written charts are safe, plottable Plotly specifications. + +The charting agent writes `output/charts.json`. That file is rendered in the +browser, so it is validated here before it ever leaves the runner: only plain +JSON data is allowed, trace types are restricted to an allowlist, and every +numeric series must actually contain numbers. A file that fails validation is +replaced by the deterministic charts. +""" + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any, List + +ALLOWED_TYPES = {"scatter", "scattergl", "bar", "box", "violin", "histogram", "heatmap", "line"} +MAX_CHARTS = 6 +MAX_POINTS = 5000 +MAX_DEPTH = 12 +# Plotly renders these as raw markup, which would let generated text inject HTML. +FORBIDDEN_KEYS = {"meta", "customdatasrc", "xsrc", "ysrc", "zsrc", "textsrc", "idssrc"} + + +class ChartError(Exception): + pass + + +def _check_value(value: Any, path: str, depth: int = 0) -> int: + """Walk a JSON value, rejecting anything that is not plain data.""" + if depth > MAX_DEPTH: + raise ChartError(f"{path}: nested too deeply") + if value is None or isinstance(value, (bool, int, str)): + if isinstance(value, str) and len(value) > 4000: + raise ChartError(f"{path}: string is too long") + return 1 + if isinstance(value, float): + if math.isnan(value) or math.isinf(value): + raise ChartError(f"{path}: not a finite number") + return 1 + if isinstance(value, list): + if len(value) > MAX_POINTS: + raise ChartError(f"{path}: more than {MAX_POINTS} values") + return sum(_check_value(item, f"{path}[{index}]", depth + 1) for index, item in enumerate(value)) + if isinstance(value, dict): + count = 0 + for key, item in value.items(): + if not isinstance(key, str): + raise ChartError(f"{path}: non-string key") + if key.lower() in FORBIDDEN_KEYS: + raise ChartError(f"{path}.{key}: field is not allowed") + count += _check_value(item, f"{path}.{key}", depth + 1) + return count + raise ChartError(f"{path}: unsupported value of type {type(value).__name__}") + + +def validate(document: Any) -> List[str]: + if not isinstance(document, dict): + raise ChartError("charts.json must contain an object") + charts = document.get("charts") + if not isinstance(charts, list) or not charts: + raise ChartError("charts.json must contain a non-empty 'charts' list") + if len(charts) > MAX_CHARTS: + raise ChartError(f"charts.json has more than {MAX_CHARTS} charts") + + ids: List[str] = [] + for index, chart in enumerate(charts): + where = f"charts[{index}]" + if not isinstance(chart, dict): + raise ChartError(f"{where}: each chart must be an object") + for field in ("id", "title", "description"): + if not isinstance(chart.get(field), str) or not chart[field].strip(): + raise ChartError(f"{where}.{field}: required non-empty string") + if chart["id"] in ids: + raise ChartError(f"{where}.id: duplicate chart id {chart['id']}") + ids.append(chart["id"]) + + plotly = chart.get("plotly") + if not isinstance(plotly, dict): + raise ChartError(f"{where}.plotly: required object") + traces = plotly.get("data") + if not isinstance(traces, list) or not traces: + raise ChartError(f"{where}.plotly.data: required non-empty list") + for trace_index, trace in enumerate(traces): + if not isinstance(trace, dict): + raise ChartError(f"{where}.plotly.data[{trace_index}]: must be an object") + trace_type = trace.get("type") + if trace_type is not None and trace_type not in ALLOWED_TYPES: + raise ChartError(f"{where}.plotly.data[{trace_index}].type: '{trace_type}' is not an allowed chart type") + if not any(key in trace for key in ("x", "y", "z", "values", "labels")): + raise ChartError(f"{where}.plotly.data[{trace_index}]: has no data series") + if "layout" in plotly and not isinstance(plotly["layout"], dict): + raise ChartError(f"{where}.plotly.layout: must be an object") + _check_value(plotly, f"{where}.plotly") + + interpretation = document.get("interpretation") + if interpretation is not None and (not isinstance(interpretation, str) or len(interpretation) > 8000): + raise ChartError("interpretation must be a string of at most 8000 characters") + return ids + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("charts", type=Path) + args = parser.parse_args() + try: + document = json.loads(args.charts.read_text()) + ids = validate(document) + except (OSError, json.JSONDecodeError) as error: + print(f"charts.json could not be read: {error}") + raise SystemExit(1) + except ChartError as error: + print(f"charts.json is not valid: {error}") + raise SystemExit(1) + print(f"charts.json is valid: {', '.join(ids)}") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 0601981..0a40d6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,9 @@ mypy>=0.950 streamlit>=1.20.0 watchdog>=2.1.0 +# Playground +cryptography>=42.0.0 # Opens the OpenRouter key a researcher supplies for their own run + # Optional: For LLM agents openai>=1.0.0 # Required for OpenRouter and OpenAI API anthropic>=0.3.0 # For Claude (Anthropic API) diff --git a/studies/study_006/scripts/config.py b/studies/study_006/scripts/config.py index c813621..c44215e 100644 --- a/studies/study_006/scripts/config.py +++ b/studies/study_006/scripts/config.py @@ -129,13 +129,13 @@ def load_material(self, sub_study_id: str) -> Dict[str, Any]: ] for filename in possible_names: - file_path = self.study_path / "materials" / filename + file_path = self.source_path / "materials" / filename if file_path.exists(): with open(file_path, "r", encoding='utf-8') as f: return json.load(f) # If none found, raise error with all attempted paths - attempted = [str(self.study_path / "materials" / name) for name in possible_names] + attempted = [str(self.source_path / "materials" / name) for name in possible_names] raise FileNotFoundError(f"Material not found for '{sub_study_id}'. Tried: {attempted}") def create_trials(self, n_trials: int = None) -> List[Dict[str, Any]]: diff --git a/tests/test_playground.py b/tests/test_playground.py new file mode 100644 index 0000000..5f006b7 --- /dev/null +++ b/tests/test_playground.py @@ -0,0 +1,157 @@ +import json +from pathlib import Path + +import pytest + +from playground.analysis import agent_effect_from_t, build_analysis, human_effect_from_reported, normalise_tests +from playground.default_charts import build_charts +from playground.settings import trial_limits +from playground.study_loader import StudyNotRunnable, study_dir +from playground.validate_charts import ChartError, validate + + +def write_ground_truth(tmp_path: Path) -> Path: + study = tmp_path / "study_900" + (study / "source").mkdir(parents=True) + (study / "source" / "ground_truth.json").write_text(json.dumps({ + "studies": [{ + "study_id": "Study 1", + "findings": [{ + "finding_id": "F1", + "main_hypothesis": "Choosers estimate their own choice as more common.", + "statistical_tests": [{ + "reported_statistics": "F(1, 312) = 49.1, p < .001", + "expected_direction": "positive", + }], + }], + }], + })) + return study + + +def test_human_effect_is_converted_from_the_reported_statistic() -> None: + assert human_effect_from_reported("F(1, 312) = 49.1, p < .001") == pytest.approx(0.7934, abs=1e-3) + assert human_effect_from_reported("t(78) = 2.66, p = .01") == pytest.approx(0.6023, abs=1e-3) + # A multi-numerator F has no single-d equivalent and must not be guessed at. + assert human_effect_from_reported("F(3, 120) = 4.2") is None + assert human_effect_from_reported("chi2(1) = 6.1") is None + + +def test_agent_effect_is_derived_from_the_t_statistic() -> None: + assert agent_effect_from_t(2.0, 30, 30) == pytest.approx(0.5164, abs=1e-3) + assert agent_effect_from_t(None, 30, 30) is None + + +def test_raw_evaluator_output_is_normalised_against_the_paper(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [{ + "study_id": "Study 1", + "finding_id": "F1", + "scenario": "supermarket_story", + "t_stat": 2.4, + "p_value": 0.02, + "n_agent_1": 30, + "n_agent_2": 30, + "significant": True, + "direction_match": True, + "human_significant": True, + "replication": True, + }]} + + rows = normalise_tests(evaluation, study) + + assert len(rows) == 1 + row = rows[0] + assert row["human_effect"] == pytest.approx(0.7934, abs=1e-3) + assert row["agent_effect"] == pytest.approx(0.6197, abs=1e-3) + assert row["reported_statistics"] == "F(1, 312) = 49.1, p < .001" + assert row["replicated"] is True + + +def test_enriched_evaluator_output_keeps_its_own_effect_sizes(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [{ + "study_id": "Study 1", + "finding_id": "F1", + "human_effect_d": 0.9, + "agent_effect_d": 0.2, + "p_value_agent": 0.4, + "p_value_human": 0.001, + "is_significant_agent": False, + "is_significant_human": True, + "direction_match": True, + }]} + + row = normalise_tests(evaluation, study)[0] + + assert row["human_effect"] == 0.9 + assert row["agent_effect"] == 0.2 + # Not significant for the agent, so the published finding was not reproduced. + assert row["replicated"] is False + + +def test_analysis_summarises_replication(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [ + {"study_id": "Study 1", "finding_id": "F1", "human_effect_d": 0.8, "agent_effect_d": 0.6, "is_significant_agent": True, "is_significant_human": True, "direction_match": True}, + {"study_id": "Study 1", "finding_id": "F1", "human_effect_d": 0.8, "agent_effect_d": -0.4, "is_significant_agent": True, "is_significant_human": True, "direction_match": False}, + ]} + + analysis = build_analysis(evaluation, study, {"id": "r1", "studyId": "study_900"}, {"participants": 20}) + + assert analysis["summary"]["replicatedTests"] == 1 + assert analysis["summary"]["scoredTests"] == 2 + assert analysis["summary"]["replicationRate"] == pytest.approx(0.5) + assert analysis["summary"]["directionMatchRate"] == pytest.approx(0.5) + assert analysis["participants"] == 20 + + +def test_default_charts_are_valid_and_skip_what_the_data_cannot_support(tmp_path: Path) -> None: + study = write_ground_truth(tmp_path) + evaluation = {"test_results": [ + {"study_id": "Study 1", "finding_id": "F1", "human_effect_d": 0.8, "agent_effect_d": 0.6, "is_significant_agent": True, "is_significant_human": True, "direction_match": True}, + ]} + analysis = build_analysis(evaluation, study, {"id": "r1"}, {}) + + charts = build_charts(analysis) + + assert validate(charts) == ["effect-scatter", "effect-bars", "replication-breakdown"] + + # The agent produced no measurable effect here. The scatter needs both sides + # so it is dropped rather than plotted against an invented zero; the bar + # chart still shows the human effect the agent failed to produce. + one_sided = build_analysis({"test_results": [{"study_id": "Study 1", "finding_id": "F1", "direction_match": False}]}, study, {"id": "r1"}, {}) + assert [chart["id"] for chart in build_charts(one_sided)["charts"]] == ["effect-bars", "replication-breakdown"] + + # With nothing measured on either side there is no chart to draw at all. + nothing = build_analysis({"test_results": []}, study, {"id": "r1"}, {}) + assert build_charts(nothing)["charts"] == [] + + +def test_charts_from_the_agent_must_be_plain_plottable_data() -> None: + def chart(trace: dict) -> dict: + return {"charts": [{"id": "a", "title": "t", "description": "d", "plotly": {"data": [trace]}}]} + + assert validate(chart({"type": "bar", "x": ["a"], "y": [1]})) == ["a"] + with pytest.raises(ChartError): + validate(chart({"type": "pie", "values": [1, 2]})) + with pytest.raises(ChartError): + validate(chart({"type": "bar", "x": ["a"], "y": [1], "meta": ""})) + with pytest.raises(ChartError): + validate(chart({"type": "bar"})) + with pytest.raises(ChartError): + validate({"charts": []}) + + +def test_run_budget_depends_on_whose_key_pays() -> None: + shared_total, shared_per_scenario = trial_limits(has_own_key=False) + own_total, own_per_scenario = trial_limits(has_own_key=True) + assert own_total > shared_total + assert own_per_scenario > shared_per_scenario + + +def test_study_ids_cannot_escape_the_studies_directory(tmp_path: Path) -> None: + with pytest.raises(StudyNotRunnable): + study_dir(tmp_path, "../secrets") + with pytest.raises(StudyNotRunnable): + study_dir(tmp_path, "study_999")