From e5ab5b9190a93a9a19278536b6842d17cacb8c0c Mon Sep 17 00:00:00 2001 From: XuanL17 Date: Sat, 8 Aug 2026 23:02:31 +0800 Subject: [PATCH] feat: add the playground for running an agent on a study The playground lets a researcher choose a study and a base model, design the prompt the agent takes into the experiment, and see how the run compares to the people in the original paper. The prompt designer offers the four shipped participant prompts, per-run demographic overrides, or a prompt written from scratch. Results arrive as summary tiles, the charts and reading produced for the run, a test-by-test table against the paper's findings, and the transcripts the agents actually produced. Runs are dispatched to HumanStudy-Bench and stored on a run branch of the private jobs repository, the same shape as Build Study; nothing about a run executes inside Vercel. Runs use the shared OpenRouter key with a small participant cap, and a researcher can supply their own key for a larger one. That key is sealed with AES-256-GCM before it is stored and never reaches a browser. The home page now explains what Datasets, Build a study, and Playground each do, and Agent Evaluations opens with a link into the playground. Co-Authored-By: Claude Opus 5 --- .env.example | 5 + README.md | 21 + .../playground/runs/[runId]/results/route.ts | 20 + .../playground/runs/[runId]/retry/route.ts | 19 + app/api/playground/runs/[runId]/route.ts | 17 + app/api/playground/runs/route.ts | 28 + app/page.tsx | 14 + app/playground/page.tsx | 18 + app/results/page.tsx | 17 +- components/Navbar.tsx | 1 + components/PlaygroundCharts.tsx | 65 ++ components/PlaygroundStudio.tsx | 585 ++++++++++++++++++ lib/playground-runs.ts | 340 ++++++++++ 13 files changed, 1149 insertions(+), 1 deletion(-) create mode 100644 app/api/playground/runs/[runId]/results/route.ts create mode 100644 app/api/playground/runs/[runId]/retry/route.ts create mode 100644 app/api/playground/runs/[runId]/route.ts create mode 100644 app/api/playground/runs/route.ts create mode 100644 app/playground/page.tsx create mode 100644 components/PlaygroundCharts.tsx create mode 100644 components/PlaygroundStudio.tsx create mode 100644 lib/playground-runs.ts diff --git a/.env.example b/.env.example index d8fa1f6..057e5da 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,11 @@ GITHUB_JOBS_REPO=HumanStudy-Hub/humanstudy-hub-jobs GITHUB_PIPELINE_REPO=HumanStudy-Hub/HumanStudy-Bench GITHUB_PIPELINE_REF=main +# Required only for playground runs on a researcher's own OpenRouter key. The +# key is sealed with this secret before it is stored, and the same value must be +# set as the PLAYGROUND_KEY_SECRET Actions secret in HumanStudy-Bench. +PLAYGROUND_KEY_SECRET= + # Optional: enables "Save to GitHub". ZIP download works without it. GITHUB_TOKEN= GITHUB_REPO_OWNER=HumanStudy-Hub diff --git a/README.md b/README.md index 5060dbd..297700f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Private Next.js application for HumanStudy-Hub. - Dataset catalog for the initial test suite and community studies - Human-study builder with PDF/OSF intake and human review +- Playground for running a model through a study and comparing it to the people - HumanStudy-Bench agent evaluations and leaderboard - Research projects, partners, and sponsorship @@ -59,5 +60,25 @@ Actions secret in `HumanStudy-Hub/HumanStudy-Bench`. The workflow also expects the `HUMANSTUDY_PIPELINE_TOKEN` Actions secret and accepts an optional `OPENROUTER_MODEL` Actions variable, which defaults to `moonshotai/kimi-k3`. +## Playground backend + +`/playground` dispatches `run-playground.yml` in HumanStudy-Bench the same way +Build Study dispatches its workflow. Run state, progress, and results live on a +`runs/` branch of the private jobs repository, and the page polls +`/api/playground/runs/`. Nothing about a run executes inside Vercel. + +The runner replays the study with the chosen model through OpenRouter, scores it +with the study's own evaluator, and then has Claude Code chart and interpret the +result. Charts are validated on the runner before they are stored, and a run +that produces no usable chart set falls back to deterministic charts. + +Runs use the `OPENROUTER_API_KEY` Actions secret with a small participant cap. A +researcher can paste their own OpenRouter key to run at full size; the key is +sealed with AES-256-GCM before it leaves the server, so the jobs repository only +holds ciphertext and the workflow drops it when the run ends. Set +`PLAYGROUND_KEY_SECRET` here and as an Actions secret in HumanStudy-Bench with +the same value — without it, the playground still runs on the shared key and +only refuses researcher-supplied keys. + The production server only needs GitHub API access. Do not add PDF files, pipeline outputs, or tokens to the public web repository. diff --git a/app/api/playground/runs/[runId]/results/route.ts b/app/api/playground/runs/[runId]/results/route.ts new file mode 100644 index 0000000..a13e439 --- /dev/null +++ b/app/api/playground/runs/[runId]/results/route.ts @@ -0,0 +1,20 @@ +import { NextResponse } from "next/server"; +import { readResults } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function GET( + _request: Request, + context: { params: Promise<{ runId: string }> }, +) { + try { + const { runId } = await context.params; + const results = await readResults(runId); + if (!results.analysis) { + return NextResponse.json({ error: "This run has no results yet." }, { status: 404 }); + } + return NextResponse.json(results); + } catch { + return NextResponse.json({ error: "Results not found." }, { status: 404 }); + } +} diff --git a/app/api/playground/runs/[runId]/retry/route.ts b/app/api/playground/runs/[runId]/retry/route.ts new file mode 100644 index 0000000..8b4fbc8 --- /dev/null +++ b/app/api/playground/runs/[runId]/retry/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; +import { retryRun } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function POST( + _request: Request, + context: { params: Promise<{ runId: string }> }, +) { + try { + const { runId } = await context.params; + return NextResponse.json({ run: await retryRun(runId) }); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "Could not restart this run." }, + { status: 400 }, + ); + } +} diff --git a/app/api/playground/runs/[runId]/route.ts b/app/api/playground/runs/[runId]/route.ts new file mode 100644 index 0000000..22f7668 --- /dev/null +++ b/app/api/playground/runs/[runId]/route.ts @@ -0,0 +1,17 @@ +import { NextResponse } from "next/server"; +import { readLog, readRun } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function GET( + _request: Request, + context: { params: Promise<{ runId: string }> }, +) { + try { + const { runId } = await context.params; + const run = await readRun(runId); + return NextResponse.json({ run, log: await readLog(runId) }); + } catch { + return NextResponse.json({ error: "Run not found." }, { status: 404 }); + } +} diff --git a/app/api/playground/runs/route.ts b/app/api/playground/runs/route.ts new file mode 100644 index 0000000..511e4d3 --- /dev/null +++ b/app/api/playground/runs/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; +import { createRun } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function POST(request: Request) { + try { + const body = await request.json(); + const run = await createRun({ + studyId: String(body.studyId || ""), + model: String(body.model || ""), + preset: String(body.preset || "v3_human_plus_demo"), + systemPrompt: String(body.systemPrompt || ""), + demographics: body.demographics, + participantsPerScenario: Number(body.participantsPerScenario) || 8, + temperature: Number(body.temperature ?? 1), + seed: Number(body.seed ?? 42), + researcherName: String(body.researcherName || ""), + apiKey: String(body.apiKey || ""), + }); + return NextResponse.json({ run }, { status: 202 }); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "Could not start this run." }, + { status: 400 }, + ); + } +} diff --git a/app/page.tsx b/app/page.tsx index 74ad9bc..86f62d9 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -22,7 +22,21 @@ export default function Home() {
Explore datasets Build a study + Playground
+ +
+ {[ + ["Explore datasets", "Browse the human studies already turned into reusable data you can download."], + ["Build a study", "Upload a paper and our agent rebuilds it as a study an AI agent can run."], + ["Playground", "Run a model through one of those studies and see how it compares to real people."], + ].map(([term, detail]) => ( +
+
{term}
+
{detail}
+
+ ))} +
diff --git a/app/playground/page.tsx b/app/playground/page.tsx new file mode 100644 index 0000000..96d9d8f --- /dev/null +++ b/app/playground/page.tsx @@ -0,0 +1,18 @@ +import PlaygroundStudio, { type StudyOption } from "@/components/PlaygroundStudio"; +import { getStudies } from "@/lib/studies"; + +export const metadata = { + title: "Playground — HumanStudy-Hub", + description: "Run an AI agent through a published human study and compare it against the original participants.", +}; + +export default async function PlaygroundPage() { + const studies = await getStudies(); + const options: StudyOption[] = studies.map((study) => ({ + study_id: study.study_id, + title: study.title, + year: study.year, + })); + + return ; +} diff --git a/app/results/page.tsx b/app/results/page.tsx index bc814ec..09f5648 100644 --- a/app/results/page.tsx +++ b/app/results/page.tsx @@ -1,16 +1,31 @@ +import Link from "next/link"; import EffectSizePlot from "@/components/EffectSizePlot"; import LeaderboardTable from "@/components/LeaderboardTable"; export default function ResultsPage() { return (
+
+
+
+

Playground

+

Run your AI agent on one of these studies

+

+ Choose a study and a base model, design the prompt your agent takes into the experiment, and see where it + matched the original participants and where it did not. +

+
+ Open the playground +
+
+

HumanStudy-Bench

Agent Evaluations

- See how AI agents reproduce human behavioral effects across the initial test suite. + See how AI agents reproduce human behavioral effects.

Read the HumanStudy-Bench paper →
diff --git a/components/Navbar.tsx b/components/Navbar.tsx index 79349fb..20c3d58 100644 --- a/components/Navbar.tsx +++ b/components/Navbar.tsx @@ -6,6 +6,7 @@ import { usePathname } from "next/navigation"; const links = [ { name: "Datasets", href: "/dataset" }, { name: "Build Study", href: "/pipeline" }, + { name: "Playground", href: "/playground" }, { name: "Agent Evaluations", href: "/results" }, { name: "Partnerships", href: "/collaboration" }, ]; diff --git a/components/PlaygroundCharts.tsx b/components/PlaygroundCharts.tsx new file mode 100644 index 0000000..a8be5ef --- /dev/null +++ b/components/PlaygroundCharts.tsx @@ -0,0 +1,65 @@ +"use client"; + +import dynamic from "next/dynamic"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; + +const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); + +export type PlaygroundChart = { + id: string; + title: string; + description: string; + plotly: { data: Plotly.Data[]; layout?: Partial }; +}; + +export type PlaygroundChartSet = { + charts: PlaygroundChart[]; + interpretation?: string; + source?: "agent" | "default"; +}; + +const baseLayout: Partial = { + autosize: true, + paper_bgcolor: "white", + plot_bgcolor: "white", + font: { size: 11, color: "#334155" }, + margin: { t: 20, r: 16, b: 55, l: 55 }, + hovermode: "closest", +}; + +export default function PlaygroundCharts({ charts, interpretation, source }: PlaygroundChartSet) { + return ( +
+ {interpretation && ( +
+
+

What this run shows

+ + {source === "agent" ? "Written by the analysis agent" : "Generated from the run"} + +
+
+ {interpretation} +
+
+ )} + + {charts.map((chart) => ( +
+

{chart.title}

+

{chart.description}

+
+ +
+
+ ))} +
+ ); +} diff --git a/components/PlaygroundStudio.tsx b/components/PlaygroundStudio.tsx new file mode 100644 index 0000000..ca183e9 --- /dev/null +++ b/components/PlaygroundStudio.tsx @@ -0,0 +1,585 @@ +"use client"; + +import Link from "next/link"; +import { useEffect, useMemo, useRef, useState } from "react"; +import PlaygroundCharts, { type PlaygroundChartSet } from "@/components/PlaygroundCharts"; + +const STORAGE_KEY = "humanstudy-hub-playground-run"; +// A dispatched run reports back within a minute or so. Longer than this and no +// runner ever picked it up. +const STALL_MS = 5 * 60 * 1000; + +export type StudyOption = { study_id: string; title: string; year: number | null }; + +type RunStatus = "queued" | "running" | "analysing" | "complete" | "failed"; +type Progress = { + phase: "preparing" | "running_participants" | "scoring" | "charting" | "failed"; + completedTrials?: number; + totalTrials?: number; + message?: string; +}; +type Summary = { + totalTests: number; + scoredTests: number; + replicatedTests: number; + replicationRate: number | null; + directionMatchRate: number | null; + meanAbsoluteEffectGap: number | null; + meanHumanEffect: number | null; + meanAgentEffect: number | null; + effectCorrelation: number | null; + studyScore: number | null; +}; +type Run = { + id: string; + studyId: string; + studyTitle?: string; + model: string; + preset: string; + participantsPerScenario: number; + status: RunStatus; + message: string; + error?: string; + resultsReady?: boolean; + participants?: number; + answeredTrials?: number; + summary?: Summary; + progress?: Progress; + createdAt?: string; +}; +type TestRow = { + test_id: string; + label: string; + hypothesis: string | null; + reported_statistics: string | null; + human_effect: number | null; + agent_effect: number | null; + agent_p: number | null; + human_significant: boolean | null; + agent_significant: boolean | null; + direction_match: boolean | null; + replicated: boolean | null; +}; +type Analysis = { summary: Summary; tests: TestRow[] }; +type Transcript = Array<{ participantId: number; profile: Record; prompt: string | null; response: string | null }>; + +const MODELS = [ + { id: "openai/gpt-4o-mini", label: "GPT-4o mini", note: "Fast and inexpensive" }, + { id: "anthropic/claude-sonnet-5", label: "Claude Sonnet 5", note: "Strong instruction following" }, + { id: "google/gemini-2.5-flash", label: "Gemini 2.5 Flash", note: "Fast, low cost" }, + { id: "meta-llama/llama-3.3-70b-instruct", label: "Llama 3.3 70B", note: "Open weights" }, +]; + +const PRESETS = [ + { id: "v1_empty", label: "No framing", note: "The model receives the task with no participant instruction at all." }, + { id: "v2_human", label: "Human participant", note: "“You are participating in a psychology experiment as a human participant.”" }, + { id: "v3_human_plus_demo", label: "Human + demographics", note: "Adds the age, gender, and background of the person the agent is playing." }, + { id: "v4_background", label: "Generated background", note: "Uses a fuller life-history background written for this study." }, + { id: "custom", label: "Write your own", note: "Design the participant prompt yourself." }, +]; + +const PRESET_STARTERS: Record = { + v2_human: "You are participating in a psychology experiment as a human participant.", + v3_human_plus_demo: + "You are participating in a psychology experiment as a human participant.\n\nYOUR IDENTITY:\n- Age: 21 years old\n- Gender: female\n- Background: undergraduate student\n\nFollow the experimenter's instructions and answer each task in the requested format.\nBe concise. Do not add extra explanations unless explicitly asked.", +}; + +const PHASES: Record = { + preparing: "Loading the study", + running_participants: "Participants are answering", + scoring: "Scoring against the published findings", + charting: "Building the comparison charts", + failed: "The run stopped", +}; + +function percent(value: number | null | undefined) { + return value === null || value === undefined ? "—" : `${Math.round(value * 100)}%`; +} + +function decimal(value: number | null | undefined, digits = 2) { + return value === null || value === undefined ? "—" : value.toFixed(digits); +} + +function Verdict({ row }: { row: TestRow }) { + if (row.replicated) return Reproduced; + if (row.direction_match === false) return Wrong direction; + if (row.replicated === false) return No effect; + return Not scored; +} + +export default function PlaygroundStudio({ studies }: { studies: StudyOption[] }) { + const [studyId, setStudyId] = useState(studies[0]?.study_id || "study_001"); + const [model, setModel] = useState(MODELS[0].id); + const [customModel, setCustomModel] = useState(""); + const [preset, setPreset] = useState("v3_human_plus_demo"); + const [systemPrompt, setSystemPrompt] = useState(""); + const [age, setAge] = useState(""); + const [gender, setGender] = useState(""); + const [background, setBackground] = useState(""); + const [persona, setPersona] = useState(""); + const [participants, setParticipants] = useState(8); + const [temperature, setTemperature] = useState(1); + const [apiKey, setApiKey] = useState(""); + const [showKey, setShowKey] = useState(false); + + const [run, setRun] = useState(null); + const [charts, setCharts] = useState(null); + const [analysis, setAnalysis] = useState(null); + const [transcript, setTranscript] = useState([]); + const [log, setLog] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [stalled, setStalled] = useState(false); + const resultsLoaded = useRef(""); + + const maxParticipants = apiKey.trim() ? 80 : 10; + const chosenModel = model === "custom" ? customModel.trim() : model; + const study = useMemo(() => studies.find((entry) => entry.study_id === studyId), [studies, studyId]); + + useEffect(() => { + const requested = new URLSearchParams(window.location.search).get("run"); + const saved = requested || window.localStorage.getItem(STORAGE_KEY); + if (!saved) return; + window.localStorage.setItem(STORAGE_KEY, saved); + fetch(`/api/playground/runs/${saved}`, { cache: "no-store" }) + .then((response) => (response.ok ? response.json() : null)) + .then((data) => { + if (data?.run) { + setRun(data.run); + setLog(data.log || ""); + } else { + window.localStorage.removeItem(STORAGE_KEY); + } + }) + .catch(() => undefined); + }, []); + + useEffect(() => { + if (run) window.localStorage.setItem(STORAGE_KEY, run.id); + }, [run]); + + useEffect(() => { + if (!run || (run.status !== "queued" && run.status !== "running" && run.status !== "analysing")) return; + const timer = window.setInterval(async () => { + try { + const response = await fetch(`/api/playground/runs/${run.id}`, { cache: "no-store" }); + if (!response.ok) return; + const data = (await response.json()) as { run: Run; log: string }; + setRun(data.run); + setLog(data.log || ""); + } catch { + // A transient network interruption is retried on the next poll. + } + }, 3000); + return () => window.clearInterval(timer); + }, [run]); + + useEffect(() => { + if (!run?.resultsReady || resultsLoaded.current === run.id) return; + resultsLoaded.current = run.id; + fetch(`/api/playground/runs/${run.id}/results`, { cache: "no-store" }) + .then((response) => (response.ok ? response.json() : null)) + .then((data) => { + if (!data) return; + if (data.charts) setCharts(data.charts as PlaygroundChartSet); + if (data.analysis) setAnalysis(data.analysis as Analysis); + if (Array.isArray(data.transcript)) setTranscript(data.transcript as Transcript); + }) + .catch(() => undefined); + }, [run]); + + // A run that never leaves "queued" means no runner picked it up. + useEffect(() => { + setStalled(false); + if (!run || run.status !== "queued" || !run.createdAt) return; + const since = Date.parse(run.createdAt); + if (Number.isNaN(since)) return; + const remaining = since + STALL_MS - Date.now(); + if (remaining <= 0) { + setStalled(true); + return; + } + const timer = window.setTimeout(() => setStalled(true), remaining); + return () => window.clearTimeout(timer); + }, [run]); + + function choosePreset(next: string) { + setPreset(next); + if (next === "custom" && !systemPrompt.trim()) { + setSystemPrompt(PRESET_STARTERS.v3_human_plus_demo); + } + } + + async function start() { + setBusy(true); + setError(""); + try { + const response = await fetch("/api/playground/runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + studyId, + model: chosenModel, + preset, + systemPrompt: preset === "custom" ? systemPrompt : "", + demographics: { age, gender, background, persona }, + participantsPerScenario: participants, + temperature, + apiKey: apiKey.trim(), + }), + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "Could not start this run."); + setCharts(null); + setAnalysis(null); + setTranscript([]); + resultsLoaded.current = ""; + setRun(data.run); + setApiKey(""); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Could not start this run."); + } finally { + setBusy(false); + } + } + + async function retry() { + if (!run) return; + setBusy(true); + setError(""); + try { + const response = await fetch(`/api/playground/runs/${run.id}/retry`, { method: "POST" }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "Could not restart this run."); + setRun(data.run); + setStalled(false); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Could not restart this run."); + } finally { + setBusy(false); + } + } + + function reset() { + window.localStorage.removeItem(STORAGE_KEY); + window.history.replaceState({}, "", "/playground"); + setRun(null); + setCharts(null); + setAnalysis(null); + setTranscript([]); + setLog(""); + resultsLoaded.current = ""; + } + + const progress = run?.progress; + const completed = progress?.completedTrials ?? 0; + const total = progress?.totalTrials ?? 0; + const running = run?.status === "queued" || run?.status === "running" || run?.status === "analysing"; + + return ( +
+
+
+

Playground

+

Run your AI agent on a human study

+

+ Pick a study, choose a model, and design the prompt your agent takes into the experiment. The run is scored + against the paper's published findings, and the results show where the agent behaved like the original + participants and where it did not. +

+
+
+ +
+ {error &&

{error}

} + +
+
+
+

Experiment setup

+

Everything except the prompt matches how the benchmark runs this study.

+
+ +
+
+ + + {study && ( +

+ Open the study record + {study.year ? ` · published ${study.year}` : ""} +

+ )} +
+ +
+ + +

{model === "custom" ? "Any model id OpenRouter serves." : MODELS.find((entry) => entry.id === model)?.note}

+ {model === "custom" && ( + setCustomModel(event.target.value)} placeholder="provider/model-name" className="mt-2 h-10 w-full border border-gray-300 px-3 font-mono text-sm outline-none focus:border-cyan-700" /> + )} +
+ +
+ + setParticipants(Math.max(1, Math.min(maxParticipants, Number(event.target.value) || 1)))} className="mt-2 h-10 w-full border border-gray-300 px-3 text-sm outline-none focus:border-cyan-700" /> +

Up to {maxParticipants} on {apiKey.trim() ? "your own key" : "the shared key"}. More participants make small effects detectable.

+
+ +
+ + setTemperature(Number(event.target.value))} className="mt-4 w-full accent-cyan-700" /> +

{temperature.toFixed(1)} — response variability across participants.

+
+
+ +
+

Participant prompt

+

What the agent is told it is before the study begins. This is the design choice that changes results the most.

+ +
+ {PRESETS.map((entry) => ( + + ))} +
+ + {preset === "custom" ? ( +
+ +