Playground: replay a study with any model and compare it to the people

.github/workflows/run-playground.yml

@@ -0,0 +1,190 @@
+name: Run a playground study
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        description: Private jobs repository run ID
+        required: true
+        type: string
+      jobs_repo:
+        description: Private repository containing run inputs and outputs
+        required: true
+        default: HumanStudy-Hub/humanstudy-hub-jobs
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: humanstudy-playground-${{ inputs.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  run-study:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Check out the benchmark
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Check out private run storage
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.jobs_repo }}
+          token: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }}
+          path: run-storage
+          ref: runs/${{ inputs.run_id }}
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Set up Node for the charting agent
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install numpy scipy pandas tqdm openai cryptography
+          npm install --global @anthropic-ai/claude-code@latest
+
+      - name: Mark run running
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json"
+          data = json.loads(path.read_text())
+          data.update({"status": "running", "message": "Participants are running the study"})
+          path.write_text(json.dumps(data, indent=2) + "\n")
+          PY
+
+      - name: Publish running state
+        working-directory: run-storage
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+          TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }}
+        run: |
+          git config user.name "HumanStudy-Hub Playground"
+          git config user.email "playground@humanstudy-hub.org"
+          git add "runs/$RUN_ID/run.json"
+          git commit -m "playground: start $RUN_ID"
+          git push "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "HEAD:runs/$RUN_ID"
+
+      - name: Run the study
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          PLAYGROUND_KEY_SECRET: ${{ secrets.PLAYGROUND_KEY_SECRET }}
+          PIPELINE_PROGRESS_TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }}
+        run: |
+          python playground/run_playground.py \
+            --run "$GITHUB_WORKSPACE/run-storage/runs/$RUN_ID" \
+            --progress-repo "${{ inputs.jobs_repo }}" \
+            --progress-branch "runs/$RUN_ID" \
+            --progress-path "runs/$RUN_ID/progress.json"
+
+      # Charts and the written reading are produced by Claude Code through
+      # OpenRouter, then validated. This step never fails the run: an unusable
+      # chart set falls back to the deterministic charts.
+      - name: Chart and interpret the results
+        if: success()
+        continue-on-error: true
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+          ANTHROPIC_BASE_URL: https://openrouter.ai/api
+          CHARTS_AGENT_TOKEN: ${{ secrets.OPENROUTER_API_KEY }}
+          ANTHROPIC_API_KEY: ""
+          ANTHROPIC_DEFAULT_OPUS_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }}
+          ANTHROPIC_DEFAULT_SONNET_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }}
+          ANTHROPIC_DEFAULT_HAIKU_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }}
+          OPENROUTER_MODEL: ${{ vars.PLAYGROUND_CHARTS_MODEL || vars.OPENROUTER_MODEL || 'anthropic/claude-sonnet-5' }}
+          CHARTS_TIMEOUT_MINUTES: ${{ vars.PLAYGROUND_CHARTS_TIMEOUT_MINUTES || '8' }}
+        run: |
+          python playground/run_charts.py \
+            --run "$GITHUB_WORKSPACE/run-storage/runs/$RUN_ID" \
+            --model "$OPENROUTER_MODEL" \
+            --timeout-minutes "$CHARTS_TIMEOUT_MINUTES"
+
+      - name: Mark run complete
+        if: success()
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          run_dir = Path("run-storage/runs") / os.environ["RUN_ID"]
+          charts = run_dir / "output" / "charts.json"
+          if not charts.exists():
+              import subprocess, sys
+              subprocess.run([
+                  sys.executable, "playground/default_charts.py",
+                  "--analysis", str(run_dir / "output" / "analysis.json"),
+                  "--output", str(charts),
+              ], check=True)
+          path = run_dir / "run.json"
+          data = json.loads(path.read_text())
+          data.update({
+              "status": "complete",
+              "message": "The run finished and the results are ready",
+              "resultsReady": True,
+          })
+          path.write_text(json.dumps(data, indent=2) + "\n")
+          PY
+
+      - name: Mark failed or cancelled
+        if: failure() || cancelled()
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json"
+          data = json.loads(path.read_text())
+          if data.get("status") != "failed":
+              data.update({"status": "failed", "message": "The playground run failed or was cancelled"})
+          data["resultsReady"] = False
+          path.write_text(json.dumps(data, indent=2) + "\n")
+          PY
+
+      # The researcher's key only ever existed as ciphertext, and it is dropped
+      # here so a finished run keeps no copy of it at all.
+      - name: Drop the supplied key
+        if: always()
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          path = Path("run-storage/runs") / os.environ["RUN_ID"] / "run.json"
+          data = json.loads(path.read_text())
+          if data.pop("sealedApiKey", None) is not None:
+              data["usedOwnKey"] = True
+              path.write_text(json.dumps(data, indent=2) + "\n")
+          PY
+
+      - name: Commit run state and outputs
+        if: always()
+        working-directory: run-storage
+        env:
+          RUN_ID: ${{ inputs.run_id }}
+          TOKEN: ${{ secrets.HUMANSTUDY_PIPELINE_TOKEN }}
+        run: |
+          git config user.name "HumanStudy-Hub Playground"
+          git config user.email "playground@humanstudy-hub.org"
+          git add "runs/$RUN_ID"
+          git commit -m "playground: results for $RUN_ID" || true
+          git pull --rebase "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "runs/$RUN_ID"
+          git push "https://x-access-token:$TOKEN@github.com/${{ inputs.jobs_repo }}.git" "HEAD:runs/$RUN_ID"

README.md

@@ -110,11 +110,30 @@ missing-information checklist.
 The older staged Python implementation remains in `generation_pipeline/` for
 reference and migration tests, but the web builder no longer dispatches it.
 
+## Playground
+
+The playground replays a single study with a model of your choice and scores it
+against the paper, so you can see where an agent behaves like the original
+participants and where it does not.
+
+```bash
+OPENROUTER_API_KEY=... python playground/run_playground.py --run <run-dir>
+```
+
+It drives each study's own trial builder and evaluator, so a playground run is
+scored exactly like a benchmark run; the model, the participant prompt, and the
+number of participants are what you change. Results are charted and interpreted
+by Claude Code, with deterministic charts as the fallback. HumanStudy-Hub
+dispatches the same code through
+[`run-playground.yml`](.github/workflows/run-playground.yml). See
+[`playground/README.md`](playground/README.md).
+
 ## Repository structure
 
 ```text
 studies/               Runnable benchmark studies
 src/                   Execution and evaluation code
+playground/            On-demand single-study runs and result comparison
 agent_pipeline/        Claude Code package contract and validation
 generation_pipeline/   Legacy staged extraction implementation
 scripts/               Study validation and index utilities

playground/README.md

@@ -0,0 +1,87 @@
+# Playground
+
+Replay one benchmark study with a model of your choice and see where the agent
+matched the published human result and where it did not.
+
+This is the engine behind the HumanStudy-Hub playground. It does not
+reimplement any study: it drives each study's own `scripts/config.py` to build
+trials and its own `scripts/evaluator.py` to score the run, so a playground run
+is scored exactly like a benchmark run.
+
+## Run it locally
+
+```bash
+python playground/run_playground.py --run <run-dir>
+```
+
+The run directory holds `run.json` and receives `progress.json`, `logs/`, and
+`output/`:
+
+```json
+{
+  "id": "local-1",
+  "studyId": "study_001",
+  "model": "openai/gpt-4o-mini",
+  "preset": "v3_human_plus_demo",
+  "systemPrompt": "",
+  "participantsPerScenario": 8,
+  "temperature": 1.0,
+  "seed": 42,
+  "demographics": { "age": 21, "gender": "female", "background": "undergraduate" }
+}
+```
+
+`OPENROUTER_API_KEY` must be set. Every model — including Claude and Gemini
+names — is routed through OpenRouter. Add `--simulate` to exercise the whole
+path with simulated participants and no API calls.
+
+Outputs:
+
+| File | Contents |
+|---|---|
+| `output/responses.json` | Every participant response, in the shape evaluators read |
+| `output/evaluation.json` | The study evaluator's own output |
+| `output/analysis.json` | One row per statistical test: human result vs. agent result |
+| `output/transcript_sample.json` | A few complete participant exchanges |
+| `output/charts.json` | Plotly chart specifications and a written reading of the run |
+
+## Prompts
+
+`preset` selects a shipped participant prompt from
+`src/agents/custom_methods/`: `v1_empty`, `v2_human`, `v3_human_plus_demo`, or
+`v4_background`. Set `preset` to `custom` and supply `systemPrompt` to run a
+prompt you wrote yourself. `demographics` overrides the sampled participant
+profile — age, gender, education, background, population, persona — which is
+what the demographic presets read.
+
+## Charts
+
+```bash
+python playground/run_charts.py --run <run-dir>
+```
+
+Claude Code reads the finished run and writes `output/charts.json` following
+[`CLAUDE.md`](CLAUDE.md). Its output is checked by `validate_charts.py` before
+it is kept: charts must be plain JSON, restricted to an allowlist of trace
+types. Anything invalid, missing, or late falls back to the deterministic charts
+in `default_charts.py`, so a run always ends with readable charts.
+
+## Run budget
+
+A run on the shared HumanStudy-Hub key is capped at 60 participant sessions.
+A researcher who supplies their own OpenRouter key gets 600. The web app seals
+that key with AES-256-GCM under `PLAYGROUND_KEY_SECRET`, so the private jobs
+repository only ever holds ciphertext, and the workflow drops it once the run
+ends.
+
+## In CI
+
+[`run-playground.yml`](../.github/workflows/run-playground.yml) is dispatched by
+HumanStudy-Hub. Repository settings it needs:
+
+| Type | Name | Value |
+|---|---|---|
+| Actions secret | `OPENROUTER_API_KEY` | OpenRouter key for shared runs and the charting agent |
+| Actions secret | `HUMANSTUDY_PIPELINE_TOKEN` | Token with read/write access to the private jobs repository |
+| Actions secret | `PLAYGROUND_KEY_SECRET` | Shared secret that opens a researcher's own key; must match the web app |
+| Actions variable | `PLAYGROUND_CHARTS_MODEL` | Optional; defaults to `anthropic/claude-sonnet-5` |

playground/__init__.py

@@ -0,0 +1 @@
+"""Playground: run a benchmark study against a chosen model on demand."""