diff --git a/.env.example b/.env.example index d8fa1f6..057e5da 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,11 @@ GITHUB_JOBS_REPO=HumanStudy-Hub/humanstudy-hub-jobs GITHUB_PIPELINE_REPO=HumanStudy-Hub/HumanStudy-Bench GITHUB_PIPELINE_REF=main +# Required only for playground runs on a researcher's own OpenRouter key. The +# key is sealed with this secret before it is stored, and the same value must be +# set as the PLAYGROUND_KEY_SECRET Actions secret in HumanStudy-Bench. +PLAYGROUND_KEY_SECRET= + # Optional: enables "Save to GitHub". ZIP download works without it. GITHUB_TOKEN= GITHUB_REPO_OWNER=HumanStudy-Hub diff --git a/README.md b/README.md index 5060dbd..297700f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Private Next.js application for HumanStudy-Hub. - Dataset catalog for the initial test suite and community studies - Human-study builder with PDF/OSF intake and human review +- Playground for running a model through a study and comparing it to the people - HumanStudy-Bench agent evaluations and leaderboard - Research projects, partners, and sponsorship @@ -59,5 +60,25 @@ Actions secret in `HumanStudy-Hub/HumanStudy-Bench`. The workflow also expects the `HUMANSTUDY_PIPELINE_TOKEN` Actions secret and accepts an optional `OPENROUTER_MODEL` Actions variable, which defaults to `moonshotai/kimi-k3`. +## Playground backend + +`/playground` dispatches `run-playground.yml` in HumanStudy-Bench the same way +Build Study dispatches its workflow. Run state, progress, and results live on a +`runs/` branch of the private jobs repository, and the page polls +`/api/playground/runs/`. Nothing about a run executes inside Vercel. + +The runner replays the study with the chosen model through OpenRouter, scores it +with the study's own evaluator, and then has Claude Code chart and interpret the +result. Charts are validated on the runner before they are stored, and a run +that produces no usable chart set falls back to deterministic charts. + +Runs use the `OPENROUTER_API_KEY` Actions secret with a small participant cap. A +researcher can paste their own OpenRouter key to run at full size; the key is +sealed with AES-256-GCM before it leaves the server, so the jobs repository only +holds ciphertext and the workflow drops it when the run ends. Set +`PLAYGROUND_KEY_SECRET` here and as an Actions secret in HumanStudy-Bench with +the same value — without it, the playground still runs on the shared key and +only refuses researcher-supplied keys. + The production server only needs GitHub API access. Do not add PDF files, pipeline outputs, or tokens to the public web repository. diff --git a/app/api/playground/runs/[runId]/results/route.ts b/app/api/playground/runs/[runId]/results/route.ts new file mode 100644 index 0000000..a13e439 --- /dev/null +++ b/app/api/playground/runs/[runId]/results/route.ts @@ -0,0 +1,20 @@ +import { NextResponse } from "next/server"; +import { readResults } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function GET( + _request: Request, + context: { params: Promise<{ runId: string }> }, +) { + try { + const { runId } = await context.params; + const results = await readResults(runId); + if (!results.analysis) { + return NextResponse.json({ error: "This run has no results yet." }, { status: 404 }); + } + return NextResponse.json(results); + } catch { + return NextResponse.json({ error: "Results not found." }, { status: 404 }); + } +} diff --git a/app/api/playground/runs/[runId]/retry/route.ts b/app/api/playground/runs/[runId]/retry/route.ts new file mode 100644 index 0000000..8b4fbc8 --- /dev/null +++ b/app/api/playground/runs/[runId]/retry/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; +import { retryRun } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function POST( + _request: Request, + context: { params: Promise<{ runId: string }> }, +) { + try { + const { runId } = await context.params; + return NextResponse.json({ run: await retryRun(runId) }); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "Could not restart this run." }, + { status: 400 }, + ); + } +} diff --git a/app/api/playground/runs/[runId]/route.ts b/app/api/playground/runs/[runId]/route.ts new file mode 100644 index 0000000..22f7668 --- /dev/null +++ b/app/api/playground/runs/[runId]/route.ts @@ -0,0 +1,17 @@ +import { NextResponse } from "next/server"; +import { readLog, readRun } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function GET( + _request: Request, + context: { params: Promise<{ runId: string }> }, +) { + try { + const { runId } = await context.params; + const run = await readRun(runId); + return NextResponse.json({ run, log: await readLog(runId) }); + } catch { + return NextResponse.json({ error: "Run not found." }, { status: 404 }); + } +} diff --git a/app/api/playground/runs/route.ts b/app/api/playground/runs/route.ts new file mode 100644 index 0000000..511e4d3 --- /dev/null +++ b/app/api/playground/runs/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; +import { createRun } from "@/lib/playground-runs"; + +export const runtime = "nodejs"; + +export async function POST(request: Request) { + try { + const body = await request.json(); + const run = await createRun({ + studyId: String(body.studyId || ""), + model: String(body.model || ""), + preset: String(body.preset || "v3_human_plus_demo"), + systemPrompt: String(body.systemPrompt || ""), + demographics: body.demographics, + participantsPerScenario: Number(body.participantsPerScenario) || 8, + temperature: Number(body.temperature ?? 1), + seed: Number(body.seed ?? 42), + researcherName: String(body.researcherName || ""), + apiKey: String(body.apiKey || ""), + }); + return NextResponse.json({ run }, { status: 202 }); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "Could not start this run." }, + { status: 400 }, + ); + } +} diff --git a/app/page.tsx b/app/page.tsx index 74ad9bc..86f62d9 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -22,7 +22,21 @@ export default function Home() {
Explore datasets Build a study + Playground
+ +
+ {[ + ["Explore datasets", "Browse the human studies already turned into reusable data you can download."], + ["Build a study", "Upload a paper and our agent rebuilds it as a study an AI agent can run."], + ["Playground", "Run a model through one of those studies and see how it compares to real people."], + ].map(([term, detail]) => ( +
+
{term}
+
{detail}
+
+ ))} +
diff --git a/app/playground/page.tsx b/app/playground/page.tsx new file mode 100644 index 0000000..96d9d8f --- /dev/null +++ b/app/playground/page.tsx @@ -0,0 +1,18 @@ +import PlaygroundStudio, { type StudyOption } from "@/components/PlaygroundStudio"; +import { getStudies } from "@/lib/studies"; + +export const metadata = { + title: "Playground — HumanStudy-Hub", + description: "Run an AI agent through a published human study and compare it against the original participants.", +}; + +export default async function PlaygroundPage() { + const studies = await getStudies(); + const options: StudyOption[] = studies.map((study) => ({ + study_id: study.study_id, + title: study.title, + year: study.year, + })); + + return ; +} diff --git a/app/results/page.tsx b/app/results/page.tsx index bc814ec..09f5648 100644 --- a/app/results/page.tsx +++ b/app/results/page.tsx @@ -1,16 +1,31 @@ +import Link from "next/link"; import EffectSizePlot from "@/components/EffectSizePlot"; import LeaderboardTable from "@/components/LeaderboardTable"; export default function ResultsPage() { return (
+
+
+
+

Playground

+

Run your AI agent on one of these studies

+

+ Choose a study and a base model, design the prompt your agent takes into the experiment, and see where it + matched the original participants and where it did not. +

+
+ Open the playground +
+
+

HumanStudy-Bench

Agent Evaluations

- See how AI agents reproduce human behavioral effects across the initial test suite. + See how AI agents reproduce human behavioral effects.

Read the HumanStudy-Bench paper →
diff --git a/components/Navbar.tsx b/components/Navbar.tsx index 79349fb..20c3d58 100644 --- a/components/Navbar.tsx +++ b/components/Navbar.tsx @@ -6,6 +6,7 @@ import { usePathname } from "next/navigation"; const links = [ { name: "Datasets", href: "/dataset" }, { name: "Build Study", href: "/pipeline" }, + { name: "Playground", href: "/playground" }, { name: "Agent Evaluations", href: "/results" }, { name: "Partnerships", href: "/collaboration" }, ]; diff --git a/components/PlaygroundCharts.tsx b/components/PlaygroundCharts.tsx new file mode 100644 index 0000000..a8be5ef --- /dev/null +++ b/components/PlaygroundCharts.tsx @@ -0,0 +1,65 @@ +"use client"; + +import dynamic from "next/dynamic"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; + +const Plot = dynamic(() => import("react-plotly.js"), { ssr: false }); + +export type PlaygroundChart = { + id: string; + title: string; + description: string; + plotly: { data: Plotly.Data[]; layout?: Partial }; +}; + +export type PlaygroundChartSet = { + charts: PlaygroundChart[]; + interpretation?: string; + source?: "agent" | "default"; +}; + +const baseLayout: Partial = { + autosize: true, + paper_bgcolor: "white", + plot_bgcolor: "white", + font: { size: 11, color: "#334155" }, + margin: { t: 20, r: 16, b: 55, l: 55 }, + hovermode: "closest", +}; + +export default function PlaygroundCharts({ charts, interpretation, source }: PlaygroundChartSet) { + return ( +
+ {interpretation && ( +
+
+

What this run shows

+ + {source === "agent" ? "Written by the analysis agent" : "Generated from the run"} + +
+
+ {interpretation} +
+
+ )} + + {charts.map((chart) => ( +
+

{chart.title}

+

{chart.description}

+
+ +
+
+ ))} +
+ ); +} diff --git a/components/PlaygroundStudio.tsx b/components/PlaygroundStudio.tsx new file mode 100644 index 0000000..ca183e9 --- /dev/null +++ b/components/PlaygroundStudio.tsx @@ -0,0 +1,585 @@ +"use client"; + +import Link from "next/link"; +import { useEffect, useMemo, useRef, useState } from "react"; +import PlaygroundCharts, { type PlaygroundChartSet } from "@/components/PlaygroundCharts"; + +const STORAGE_KEY = "humanstudy-hub-playground-run"; +// A dispatched run reports back within a minute or so. Longer than this and no +// runner ever picked it up. +const STALL_MS = 5 * 60 * 1000; + +export type StudyOption = { study_id: string; title: string; year: number | null }; + +type RunStatus = "queued" | "running" | "analysing" | "complete" | "failed"; +type Progress = { + phase: "preparing" | "running_participants" | "scoring" | "charting" | "failed"; + completedTrials?: number; + totalTrials?: number; + message?: string; +}; +type Summary = { + totalTests: number; + scoredTests: number; + replicatedTests: number; + replicationRate: number | null; + directionMatchRate: number | null; + meanAbsoluteEffectGap: number | null; + meanHumanEffect: number | null; + meanAgentEffect: number | null; + effectCorrelation: number | null; + studyScore: number | null; +}; +type Run = { + id: string; + studyId: string; + studyTitle?: string; + model: string; + preset: string; + participantsPerScenario: number; + status: RunStatus; + message: string; + error?: string; + resultsReady?: boolean; + participants?: number; + answeredTrials?: number; + summary?: Summary; + progress?: Progress; + createdAt?: string; +}; +type TestRow = { + test_id: string; + label: string; + hypothesis: string | null; + reported_statistics: string | null; + human_effect: number | null; + agent_effect: number | null; + agent_p: number | null; + human_significant: boolean | null; + agent_significant: boolean | null; + direction_match: boolean | null; + replicated: boolean | null; +}; +type Analysis = { summary: Summary; tests: TestRow[] }; +type Transcript = Array<{ participantId: number; profile: Record; prompt: string | null; response: string | null }>; + +const MODELS = [ + { id: "openai/gpt-4o-mini", label: "GPT-4o mini", note: "Fast and inexpensive" }, + { id: "anthropic/claude-sonnet-5", label: "Claude Sonnet 5", note: "Strong instruction following" }, + { id: "google/gemini-2.5-flash", label: "Gemini 2.5 Flash", note: "Fast, low cost" }, + { id: "meta-llama/llama-3.3-70b-instruct", label: "Llama 3.3 70B", note: "Open weights" }, +]; + +const PRESETS = [ + { id: "v1_empty", label: "No framing", note: "The model receives the task with no participant instruction at all." }, + { id: "v2_human", label: "Human participant", note: "“You are participating in a psychology experiment as a human participant.”" }, + { id: "v3_human_plus_demo", label: "Human + demographics", note: "Adds the age, gender, and background of the person the agent is playing." }, + { id: "v4_background", label: "Generated background", note: "Uses a fuller life-history background written for this study." }, + { id: "custom", label: "Write your own", note: "Design the participant prompt yourself." }, +]; + +const PRESET_STARTERS: Record = { + v2_human: "You are participating in a psychology experiment as a human participant.", + v3_human_plus_demo: + "You are participating in a psychology experiment as a human participant.\n\nYOUR IDENTITY:\n- Age: 21 years old\n- Gender: female\n- Background: undergraduate student\n\nFollow the experimenter's instructions and answer each task in the requested format.\nBe concise. Do not add extra explanations unless explicitly asked.", +}; + +const PHASES: Record = { + preparing: "Loading the study", + running_participants: "Participants are answering", + scoring: "Scoring against the published findings", + charting: "Building the comparison charts", + failed: "The run stopped", +}; + +function percent(value: number | null | undefined) { + return value === null || value === undefined ? "—" : `${Math.round(value * 100)}%`; +} + +function decimal(value: number | null | undefined, digits = 2) { + return value === null || value === undefined ? "—" : value.toFixed(digits); +} + +function Verdict({ row }: { row: TestRow }) { + if (row.replicated) return Reproduced; + if (row.direction_match === false) return Wrong direction; + if (row.replicated === false) return No effect; + return Not scored; +} + +export default function PlaygroundStudio({ studies }: { studies: StudyOption[] }) { + const [studyId, setStudyId] = useState(studies[0]?.study_id || "study_001"); + const [model, setModel] = useState(MODELS[0].id); + const [customModel, setCustomModel] = useState(""); + const [preset, setPreset] = useState("v3_human_plus_demo"); + const [systemPrompt, setSystemPrompt] = useState(""); + const [age, setAge] = useState(""); + const [gender, setGender] = useState(""); + const [background, setBackground] = useState(""); + const [persona, setPersona] = useState(""); + const [participants, setParticipants] = useState(8); + const [temperature, setTemperature] = useState(1); + const [apiKey, setApiKey] = useState(""); + const [showKey, setShowKey] = useState(false); + + const [run, setRun] = useState(null); + const [charts, setCharts] = useState(null); + const [analysis, setAnalysis] = useState(null); + const [transcript, setTranscript] = useState([]); + const [log, setLog] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [stalled, setStalled] = useState(false); + const resultsLoaded = useRef(""); + + const maxParticipants = apiKey.trim() ? 80 : 10; + const chosenModel = model === "custom" ? customModel.trim() : model; + const study = useMemo(() => studies.find((entry) => entry.study_id === studyId), [studies, studyId]); + + useEffect(() => { + const requested = new URLSearchParams(window.location.search).get("run"); + const saved = requested || window.localStorage.getItem(STORAGE_KEY); + if (!saved) return; + window.localStorage.setItem(STORAGE_KEY, saved); + fetch(`/api/playground/runs/${saved}`, { cache: "no-store" }) + .then((response) => (response.ok ? response.json() : null)) + .then((data) => { + if (data?.run) { + setRun(data.run); + setLog(data.log || ""); + } else { + window.localStorage.removeItem(STORAGE_KEY); + } + }) + .catch(() => undefined); + }, []); + + useEffect(() => { + if (run) window.localStorage.setItem(STORAGE_KEY, run.id); + }, [run]); + + useEffect(() => { + if (!run || (run.status !== "queued" && run.status !== "running" && run.status !== "analysing")) return; + const timer = window.setInterval(async () => { + try { + const response = await fetch(`/api/playground/runs/${run.id}`, { cache: "no-store" }); + if (!response.ok) return; + const data = (await response.json()) as { run: Run; log: string }; + setRun(data.run); + setLog(data.log || ""); + } catch { + // A transient network interruption is retried on the next poll. + } + }, 3000); + return () => window.clearInterval(timer); + }, [run]); + + useEffect(() => { + if (!run?.resultsReady || resultsLoaded.current === run.id) return; + resultsLoaded.current = run.id; + fetch(`/api/playground/runs/${run.id}/results`, { cache: "no-store" }) + .then((response) => (response.ok ? response.json() : null)) + .then((data) => { + if (!data) return; + if (data.charts) setCharts(data.charts as PlaygroundChartSet); + if (data.analysis) setAnalysis(data.analysis as Analysis); + if (Array.isArray(data.transcript)) setTranscript(data.transcript as Transcript); + }) + .catch(() => undefined); + }, [run]); + + // A run that never leaves "queued" means no runner picked it up. + useEffect(() => { + setStalled(false); + if (!run || run.status !== "queued" || !run.createdAt) return; + const since = Date.parse(run.createdAt); + if (Number.isNaN(since)) return; + const remaining = since + STALL_MS - Date.now(); + if (remaining <= 0) { + setStalled(true); + return; + } + const timer = window.setTimeout(() => setStalled(true), remaining); + return () => window.clearTimeout(timer); + }, [run]); + + function choosePreset(next: string) { + setPreset(next); + if (next === "custom" && !systemPrompt.trim()) { + setSystemPrompt(PRESET_STARTERS.v3_human_plus_demo); + } + } + + async function start() { + setBusy(true); + setError(""); + try { + const response = await fetch("/api/playground/runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + studyId, + model: chosenModel, + preset, + systemPrompt: preset === "custom" ? systemPrompt : "", + demographics: { age, gender, background, persona }, + participantsPerScenario: participants, + temperature, + apiKey: apiKey.trim(), + }), + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "Could not start this run."); + setCharts(null); + setAnalysis(null); + setTranscript([]); + resultsLoaded.current = ""; + setRun(data.run); + setApiKey(""); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Could not start this run."); + } finally { + setBusy(false); + } + } + + async function retry() { + if (!run) return; + setBusy(true); + setError(""); + try { + const response = await fetch(`/api/playground/runs/${run.id}/retry`, { method: "POST" }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "Could not restart this run."); + setRun(data.run); + setStalled(false); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Could not restart this run."); + } finally { + setBusy(false); + } + } + + function reset() { + window.localStorage.removeItem(STORAGE_KEY); + window.history.replaceState({}, "", "/playground"); + setRun(null); + setCharts(null); + setAnalysis(null); + setTranscript([]); + setLog(""); + resultsLoaded.current = ""; + } + + const progress = run?.progress; + const completed = progress?.completedTrials ?? 0; + const total = progress?.totalTrials ?? 0; + const running = run?.status === "queued" || run?.status === "running" || run?.status === "analysing"; + + return ( +
+
+
+

Playground

+

Run your AI agent on a human study

+

+ Pick a study, choose a model, and design the prompt your agent takes into the experiment. The run is scored + against the paper's published findings, and the results show where the agent behaved like the original + participants and where it did not. +

+
+
+ +
+ {error &&

{error}

} + +
+
+
+

Experiment setup

+

Everything except the prompt matches how the benchmark runs this study.

+
+ +
+
+ + + {study && ( +

+ Open the study record + {study.year ? ` · published ${study.year}` : ""} +

+ )} +
+ +
+ + +

{model === "custom" ? "Any model id OpenRouter serves." : MODELS.find((entry) => entry.id === model)?.note}

+ {model === "custom" && ( + setCustomModel(event.target.value)} placeholder="provider/model-name" className="mt-2 h-10 w-full border border-gray-300 px-3 font-mono text-sm outline-none focus:border-cyan-700" /> + )} +
+ +
+ + setParticipants(Math.max(1, Math.min(maxParticipants, Number(event.target.value) || 1)))} className="mt-2 h-10 w-full border border-gray-300 px-3 text-sm outline-none focus:border-cyan-700" /> +

Up to {maxParticipants} on {apiKey.trim() ? "your own key" : "the shared key"}. More participants make small effects detectable.

+
+ +
+ + setTemperature(Number(event.target.value))} className="mt-4 w-full accent-cyan-700" /> +

{temperature.toFixed(1)} — response variability across participants.

+
+
+ +
+

Participant prompt

+

What the agent is told it is before the study begins. This is the design choice that changes results the most.

+ +
+ {PRESETS.map((entry) => ( + + ))} +
+ + {preset === "custom" ? ( +
+ +