Playground: run your AI agent on a human study

.env.example

@@ -9,6 +9,11 @@ GITHUB_JOBS_REPO=HumanStudy-Hub/humanstudy-hub-jobs
 GITHUB_PIPELINE_REPO=HumanStudy-Hub/HumanStudy-Bench
 GITHUB_PIPELINE_REF=main
 
+# Required only for playground runs on a researcher's own OpenRouter key. The
+# key is sealed with this secret before it is stored, and the same value must be
+# set as the PLAYGROUND_KEY_SECRET Actions secret in HumanStudy-Bench.
+PLAYGROUND_KEY_SECRET=
+
 # Optional: enables "Save to GitHub". ZIP download works without it.
 GITHUB_TOKEN=
 GITHUB_REPO_OWNER=HumanStudy-Hub

README.md

@@ -6,6 +6,7 @@ Private Next.js application for HumanStudy-Hub.
 
 - Dataset catalog for the initial test suite and community studies
 - Human-study builder with PDF/OSF intake and human review
+- Playground for running a model through a study and comparing it to the people
 - HumanStudy-Bench agent evaluations and leaderboard
 - Research projects, partners, and sponsorship
 
@@ -59,5 +60,25 @@ Actions secret in `HumanStudy-Hub/HumanStudy-Bench`. The workflow also expects
 the `HUMANSTUDY_PIPELINE_TOKEN` Actions secret and accepts an optional
 `OPENROUTER_MODEL` Actions variable, which defaults to `moonshotai/kimi-k3`.
 
+## Playground backend
+
+`/playground` dispatches `run-playground.yml` in HumanStudy-Bench the same way
+Build Study dispatches its workflow. Run state, progress, and results live on a
+`runs/<id>` branch of the private jobs repository, and the page polls
+`/api/playground/runs/<id>`. Nothing about a run executes inside Vercel.
+
+The runner replays the study with the chosen model through OpenRouter, scores it
+with the study's own evaluator, and then has Claude Code chart and interpret the
+result. Charts are validated on the runner before they are stored, and a run
+that produces no usable chart set falls back to deterministic charts.
+
+Runs use the `OPENROUTER_API_KEY` Actions secret with a small participant cap. A
+researcher can paste their own OpenRouter key to run at full size; the key is
+sealed with AES-256-GCM before it leaves the server, so the jobs repository only
+holds ciphertext and the workflow drops it when the run ends. Set
+`PLAYGROUND_KEY_SECRET` here and as an Actions secret in HumanStudy-Bench with
+the same value — without it, the playground still runs on the shared key and
+only refuses researcher-supplied keys.
+
 The production server only needs GitHub API access. Do not add PDF files,
 pipeline outputs, or tokens to the public web repository.

app/api/playground/runs/[runId]/results/route.ts

@@ -0,0 +1,20 @@
+import { NextResponse } from "next/server";
+import { readResults } from "@/lib/playground-runs";
+
+export const runtime = "nodejs";
+
+export async function GET(
+  _request: Request,
+  context: { params: Promise<{ runId: string }> },
+) {
+  try {
+    const { runId } = await context.params;
+    const results = await readResults(runId);
+    if (!results.analysis) {
+      return NextResponse.json({ error: "This run has no results yet." }, { status: 404 });
+    }
+    return NextResponse.json(results);
+  } catch {
+    return NextResponse.json({ error: "Results not found." }, { status: 404 });
+  }
+}

app/api/playground/runs/[runId]/retry/route.ts

@@ -0,0 +1,19 @@
+import { NextResponse } from "next/server";
+import { retryRun } from "@/lib/playground-runs";
+
+export const runtime = "nodejs";
+
+export async function POST(
+  _request: Request,
+  context: { params: Promise<{ runId: string }> },
+) {
+  try {
+    const { runId } = await context.params;
+    return NextResponse.json({ run: await retryRun(runId) });
+  } catch (error) {
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : "Could not restart this run." },
+      { status: 400 },
+    );
+  }
+}

app/api/playground/runs/[runId]/route.ts

@@ -0,0 +1,17 @@
+import { NextResponse } from "next/server";
+import { readLog, readRun } from "@/lib/playground-runs";
+
+export const runtime = "nodejs";
+
+export async function GET(
+  _request: Request,
+  context: { params: Promise<{ runId: string }> },
+) {
+  try {
+    const { runId } = await context.params;
+    const run = await readRun(runId);
+    return NextResponse.json({ run, log: await readLog(runId) });
+  } catch {
+    return NextResponse.json({ error: "Run not found." }, { status: 404 });
+  }
+}

app/api/playground/runs/route.ts

@@ -0,0 +1,28 @@
+import { NextResponse } from "next/server";
+import { createRun } from "@/lib/playground-runs";
+
+export const runtime = "nodejs";
+
+export async function POST(request: Request) {
+  try {
+    const body = await request.json();
+    const run = await createRun({
+      studyId: String(body.studyId || ""),
+      model: String(body.model || ""),
+      preset: String(body.preset || "v3_human_plus_demo"),
+      systemPrompt: String(body.systemPrompt || ""),
+      demographics: body.demographics,
+      participantsPerScenario: Number(body.participantsPerScenario) || 8,
+      temperature: Number(body.temperature ?? 1),
+      seed: Number(body.seed ?? 42),
+      researcherName: String(body.researcherName || ""),
+      apiKey: String(body.apiKey || ""),
+    });
+    return NextResponse.json({ run }, { status: 202 });
+  } catch (error) {
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : "Could not start this run." },
+      { status: 400 },
+    );
+  }
+}

app/page.tsx

@@ -22,7 +22,21 @@ export default function Home() {
           <div className="mt-7 flex flex-wrap gap-3">
             <Link href="/dataset" className="bg-cyan-800 px-4 py-2.5 text-sm font-semibold text-white hover:bg-cyan-700">Explore datasets</Link>
             <Link href="/pipeline" className="border border-gray-400 bg-white px-4 py-2.5 text-sm font-semibold text-gray-800 hover:border-cyan-700 hover:text-cyan-800">Build a study</Link>
+            <Link href="/playground" className="border border-gray-400 bg-white px-4 py-2.5 text-sm font-semibold text-gray-800 hover:border-cyan-700 hover:text-cyan-800">Playground</Link>
           </div>
+
+          <dl className="mt-6 grid max-w-3xl gap-x-6 gap-y-3 border-t border-gray-300/70 pt-5 sm:grid-cols-3">
+            {[
+              ["Explore datasets", "Browse the human studies already turned into reusable data you can download."],
+              ["Build a study", "Upload a paper and our agent rebuilds it as a study an AI agent can run."],
+              ["Playground", "Run a model through one of those studies and see how it compares to real people."],
+            ].map(([term, detail]) => (
+              <div key={term}>
+                <dt className="text-xs font-bold uppercase text-cyan-800">{term}</dt>
+                <dd className="mt-1 text-xs leading-5 text-gray-700">{detail}</dd>
+              </div>
+            ))}
+          </dl>
         </div>
       </main>
     </div>

app/playground/page.tsx

@@ -0,0 +1,18 @@
+import PlaygroundStudio, { type StudyOption } from "@/components/PlaygroundStudio";
+import { getStudies } from "@/lib/studies";
+
+export const metadata = {
+  title: "Playground — HumanStudy-Hub",
+  description: "Run an AI agent through a published human study and compare it against the original participants.",
+};
+
+export default async function PlaygroundPage() {
+  const studies = await getStudies();
+  const options: StudyOption[] = studies.map((study) => ({
+    study_id: study.study_id,
+    title: study.title,
+    year: study.year,
+  }));
+
+  return <PlaygroundStudio studies={options} />;
+}

app/results/page.tsx

@@ -1,16 +1,31 @@
+import Link from "next/link";
 import EffectSizePlot from "@/components/EffectSizePlot";
 import LeaderboardTable from "@/components/LeaderboardTable";
 
 export default function ResultsPage() {
   return (
     <div className="min-h-screen bg-[#f7f9fa]">
+      <section className="border-b border-cyan-200 bg-cyan-50">
+        <div className="mx-auto flex max-w-7xl flex-wrap items-center justify-between gap-5 px-6 py-6 sm:px-8">
+          <div className="max-w-2xl">
+            <p className="text-xs font-bold uppercase text-cyan-800">Playground</p>
+            <h2 className="mt-2 font-serif text-2xl font-bold text-gray-950">Run your AI agent on one of these studies</h2>
+            <p className="mt-2 text-sm leading-6 text-cyan-950">
+              Choose a study and a base model, design the prompt your agent takes into the experiment, and see where it
+              matched the original participants and where it did not.
+            </p>
+          </div>
+          <Link href="/playground" className="inline-flex h-10 items-center bg-cyan-700 px-5 text-sm font-semibold text-white hover:bg-cyan-600">Open the playground</Link>
+        </div>
+      </section>
+
       <section className="border-b border-gray-200 bg-white">
         <div className="mx-auto grid max-w-7xl gap-8 px-6 py-10 sm:px-8 lg:grid-cols-[340px_minmax(0,1fr)] lg:items-center lg:py-12">
           <header>
             <p className="text-xs font-bold uppercase text-cyan-800">HumanStudy-Bench</p>
             <h1 className="mt-3 font-serif text-4xl font-bold leading-tight text-gray-950">Agent Evaluations</h1>
             <p className="mt-4 text-base leading-7 text-gray-600">
-              See how AI agents reproduce human behavioral effects across the initial test suite.
+              See how AI agents reproduce human behavioral effects.
             </p>
             <a href="https://arxiv.org/abs/2602.00685" target="_blank" rel="noopener noreferrer" className="mt-5 inline-flex text-sm font-semibold text-cyan-800 hover:underline">Read the HumanStudy-Bench paper →</a>
           </header>

components/Navbar.tsx

@@ -6,6 +6,7 @@ import { usePathname } from "next/navigation";
 const links = [
   { name: "Datasets", href: "/dataset" },
   { name: "Build Study", href: "/pipeline" },
+  { name: "Playground", href: "/playground" },
   { name: "Agent Evaluations", href: "/results" },
   { name: "Partnerships", href: "/collaboration" },
 ];

components/PlaygroundCharts.tsx

@@ -0,0 +1,65 @@
+"use client";
+
+import dynamic from "next/dynamic";
+import ReactMarkdown from "react-markdown";
+import remarkGfm from "remark-gfm";
+
+const Plot = dynamic(() => import("react-plotly.js"), { ssr: false });
+
+export type PlaygroundChart = {
+  id: string;
+  title: string;
+  description: string;
+  plotly: { data: Plotly.Data[]; layout?: Partial<Plotly.Layout> };
+};
+
+export type PlaygroundChartSet = {
+  charts: PlaygroundChart[];
+  interpretation?: string;
+  source?: "agent" | "default";
+};
+
+const baseLayout: Partial<Plotly.Layout> = {
+  autosize: true,
+  paper_bgcolor: "white",
+  plot_bgcolor: "white",
+  font: { size: 11, color: "#334155" },
+  margin: { t: 20, r: 16, b: 55, l: 55 },
+  hovermode: "closest",
+};
+
+export default function PlaygroundCharts({ charts, interpretation, source }: PlaygroundChartSet) {
+  return (
+    <div className="space-y-6">
+      {interpretation && (
+        <section className="border border-gray-200 bg-white p-6">
+          <div className="flex items-baseline justify-between gap-4">
+            <h3 className="font-serif text-lg font-bold text-gray-950">What this run shows</h3>
+            <span className="text-[11px] text-gray-400">
+              {source === "agent" ? "Written by the analysis agent" : "Generated from the run"}
+            </span>
+          </div>
+          <div className="prose prose-sm mt-3 max-w-none text-gray-700 prose-headings:font-serif prose-headings:text-gray-950">
+            <ReactMarkdown remarkPlugins={[remarkGfm]}>{interpretation}</ReactMarkdown>
+          </div>
+        </section>
+      )}
+
+      {charts.map((chart) => (
+        <section key={chart.id} className="border border-gray-200 bg-white p-6">
+          <h3 className="font-serif text-lg font-bold text-gray-950">{chart.title}</h3>
+          <p className="mt-1 max-w-3xl text-sm leading-6 text-gray-500">{chart.description}</p>
+          <div className="mt-4 overflow-x-auto">
+            <Plot
+              data={chart.plotly.data}
+              layout={{ ...baseLayout, ...(chart.plotly.layout || {}) }}
+              config={{ responsive: true, displayModeBar: false }}
+              style={{ width: "100%", minHeight: 380 }}
+              useResizeHandler
+            />
+          </div>
+        </section>
+      ))}
+    </div>
+  );
+}