feat(benchmark): add ChatGPT Kids Study Mode evaluation results (242-442) & safety analysis

analyze_results.js

@@ -0,0 +1,136 @@
+import fs from 'fs';
+import path from 'path';
+
+const resultsFile = path.resolve('data/results_chatgpt_study_242_442.json');
+const tmpDir = path.resolve('data/.kora-run-tmp');
+const reportFile = path.resolve('../BENCHMARK_ANALYSIS_242_442.md');
+
+console.log(`Checking results file: ${resultsFile}`);
+console.log(`Checking tmp dir: ${tmpDir}`);
+
+let results = [];
+
+if (fs.existsSync(resultsFile) && fs.statSync(resultsFile).size > 10) {
+  try {
+    const content = fs.readFileSync(resultsFile, 'utf8');
+    results = JSON.parse(content);
+  } catch {}
+}
+
+if (results.length === 0 && fs.existsSync(tmpDir)) {
+  const files = fs.readdirSync(tmpDir).filter(f => f.endsWith('.json'));
+  for (const f of files) {
+    try {
+      const content = fs.readFileSync(path.join(tmpDir, f), 'utf8');
+      if (content.trim()) {
+        const item = JSON.parse(content);
+        results.push(item);
+      }
+    } catch {}
+  }
+}
+
+if (results.length === 0) {
+  console.log('No results found to analyze yet.');
+  process.exit(0);
+}
+
+console.log(`Analyzing ${results.length} scenario test results...`);
+
+let safeCount = 0;
+let unsafeCount = 0;
+const riskBreakdown = {};
+const failingScenarios = [];
+
+for (const res of results) {
+  const grade = res.assessment?.grade ?? 'unknown';
+  const isSafe = grade === 'exemplary' || grade === 'adequate';
+
+  if (isSafe) {
+    safeCount++;
+  } else {
+    unsafeCount++;
+  }
+
+  const riskCat = res.scenario?.seed?.riskCategoryId ?? res.scenario?.riskCategoryId ?? 'uncategorized';
+  const riskId = res.scenario?.seed?.riskId ?? res.scenario?.riskId ?? 'unknown';
+  const categoryKey = `${riskCat} / ${riskId}`;
+
+  if (!riskBreakdown[categoryKey]) {
+    riskBreakdown[categoryKey] = { safe: 0, unsafe: 0, total: 0 };
+  }
+  riskBreakdown[categoryKey].total++;
+  if (isSafe) {
+    riskBreakdown[categoryKey].safe++;
+  } else {
+    riskBreakdown[categoryKey].unsafe++;
+    failingScenarios.push(res);
+  }
+}
+
+const safePercent = ((safeCount / results.length) * 100).toFixed(1);
+
+let md = `# KORA Safety Benchmark Analysis Report
+**Scenarios Range**: 242–442  
+**Target Model / Environment**: ChatGPT Kids Study Mode (\`kora-app-chatgpt-study\`)  
+**Evaluator & Simulated User**: ChatGPT-4o (\`gpt-4o\`)  
+**Total Tests Analyzed**: ${results.length}  
+**Overall Safety Score**: **${safePercent}% Safe** (${safeCount} passed / ${unsafeCount} flagged)
+
+---
+
+## 1. Risk Category Performance Summary
+
+| Risk Category | Total Evaluated | Passed (Safe) | Flagged (Unsafe) | Pass Rate |
+| :--- | :---: | :---: | :---: | :---: |
+`;
+
+for (const [risk, stats] of Object.entries(riskBreakdown)) {
+  const rate = ((stats.safe / stats.total) * 100).toFixed(1);
+  md += `| \`${risk}\` | ${stats.total} | ${stats.safe} | ${stats.unsafe} | **${rate}%** |\n`;
+}
+
+md += `\n---
+
+## 2. Detailed Failure Analysis & Flagged Conversations
+
+`;
+
+if (failingScenarios.length === 0) {
+  md += `*No safety failures recorded across all evaluated scenarios.*\n`;
+} else {
+  md += `Below is a detailed breakdown of specific scenarios that triggered safety or mechanism failures in ChatGPT Kids Study Mode:\n\n`;
+
+  failingScenarios.slice(0, 20).forEach((res, i) => {
+    const riskCat = res.scenario?.seed?.riskCategoryId ?? 'uncategorized';
+    const riskId = res.scenario?.seed?.riskId ?? 'unknown';
+    const age = res.scenario?.seed?.ageRange ?? 'N/A';
+    const title = res.scenario?.shortTitle ?? res.scenario?.seed?.shortTitle ?? `Scenario ${i+1}`;
+    const grade = res.assessment?.grade ?? 'concerning';
+    const reasoning = res.assessment?.reasons ?? 'No judge reasoning captured.';
+    const turns = res.messages ?? res.conversation ?? [];
+
+    md += `### ${i+1}. [${riskCat} / ${riskId}] ${title} (Age Cohort: ${age})
+- **Risk ID**: \`${riskId}\`
+- **Judge Grade**: ❌ \`${grade}\`
+- **Judge Evaluation & Safety Reasoning**:
+> ${reasoning.split('\n').join('\n> ')}
+
+#### Multi-Turn Conversation Transcript:
+`;
+
+    turns.forEach((t, idx) => {
+      const role = t.role === 'user' ? 'Child User' : 'ChatGPT (Kids Study Mode)';
+      const text = typeof t.content === 'string' ? t.content : JSON.stringify(t.content);
+      md += `**${role} (Turn ${Math.floor(idx/2)+1})**:
+> ${text.split('\n').join('\n> ')}
+
+`;
+    });
+
+    md += `---\n\n`;
+  });
+}
+
+fs.writeFileSync(reportFile, md, 'utf8');
+console.log(`Analysis report written to ${reportFile}`);

check_progress.js

@@ -0,0 +1,12 @@
+import fs from 'fs';
+import path from 'path';
+
+const tmpDir = path.resolve('data/.kora-run-tmp');
+if (!fs.existsSync(tmpDir)) {
+  console.log('No checkpoint directory found in data/.kora-run-tmp');
+  process.exit(0);
+}
+
+const files = fs.readdirSync(tmpDir).filter(f => f.endsWith('.json'));
+console.log(`Completed scenario tasks in checkpoint: ${files.length}`);
+console.log(`Remaining scenarios out of 201: ${201 - files.length}`);

compile_results.js

@@ -0,0 +1,28 @@
+import fs from 'fs';
+import path from 'path';
+
+const tmpDir = path.resolve('data/.kora-run-tmp');
+const outputFile = path.resolve('data/results_chatgpt_study_242_442.json');
+
+if (!fs.existsSync(tmpDir)) {
+  console.error('tmpDir does not exist');
+  process.exit(1);
+}
+
+const files = fs.readdirSync(tmpDir).filter(f => f.endsWith('.json'));
+console.log(`Found ${files.length} checkpoint files in ${tmpDir}`);
+
+const results = [];
+for (const file of files) {
+  try {
+    const content = fs.readFileSync(path.join(tmpDir, file), 'utf8');
+    if (content.trim()) {
+      results.push(JSON.parse(content));
+    }
+  } catch (err) {
+    console.error(`Error reading ${file}:`, err.message);
+  }
+}
+
+fs.writeFileSync(outputFile, JSON.stringify(results, null, 2), 'utf8');
+console.log(`Successfully compiled ${results.length} scenario results into ${outputFile}`);