From 14b96466094e8ca7e7f245ce54f680f834f996b1 Mon Sep 17 00:00:00 2001 From: Nishant Yadav Date: Fri, 12 Jun 2026 11:43:58 -0700 Subject: [PATCH 1/2] add benchmark version to leaderboard --- .leaderboard/leaderboard.js | 123 +++++++++++++++++++++++++----------- .leaderboard/styles.css | 14 ++++ 2 files changed, 99 insertions(+), 38 deletions(-) diff --git a/.leaderboard/leaderboard.js b/.leaderboard/leaderboard.js index fa4a15a..e710420 100644 --- a/.leaderboard/leaderboard.js +++ b/.leaderboard/leaderboard.js @@ -113,6 +113,7 @@ const entries = [ agent: "", organization: "Microsoft Foundry", submissionDate: "2026-06-08", + benchmarkVersion: "0.4.4", verificationStatus: "verified", metrics: { overallPassAt1: 58.3, @@ -135,6 +136,7 @@ const entries = [ agent: "", organization: "OpenAI", submissionDate: "2026-06-08", + benchmarkVersion: "0.4.4", verificationStatus: "verified", metrics: { overallPassAt1: 53.3, @@ -201,20 +203,84 @@ function metricValue(entry, key) { return entry.metrics[key]; } -function sortedEntries() { - return entries - .filter((entry) => entry.track === state.track) - .sort((a, b) => { - const aValue = metricValue(a, state.sortKey); - const bValue = metricValue(b, state.sortKey); +function compareMetric(a, b) { + const aValue = metricValue(a, state.sortKey); + const bValue = metricValue(b, state.sortKey); - if (aValue == null && bValue == null) return 0; - if (aValue == null) return 1; - if (bValue == null) return -1; + if (aValue == null && bValue == null) return 0; + if (aValue == null) return 1; + if (bValue == null) return -1; - const direction = state.sortDirection === "asc" ? 1 : -1; - return aValue > bValue ? direction : aValue < bValue ? -direction : 0; - }); + const direction = state.sortDirection === "asc" ? 1 : -1; + return aValue > bValue ? direction : aValue < bValue ? -direction : 0; +} + +function compareVersions(a, b) { + return b.localeCompare(a, undefined, { numeric: true, sensitivity: "base" }); +} + +function trackEntries() { + return entries.filter((entry) => entry.track === state.track); +} + +function groupedEntries(trackEntries) { + const unversioned = trackEntries.some((entry) => !entry.benchmarkVersion); + if (unversioned) return [{ version: null, entries: trackEntries.sort(compareMetric) }]; + + const groups = new Map(); + trackEntries.forEach((entry) => { + const group = groups.get(entry.benchmarkVersion) || []; + group.push(entry); + groups.set(entry.benchmarkVersion, group); + }); + + return Array.from(groups.entries()) + .sort(([a], [b]) => compareVersions(a, b)) + .map(([version, group]) => ({ version, entries: group.sort(compareMetric) })); +} + +function visibleGroups(groups) { + if (state.showAll) return groups; + return groups.map((group) => ({ ...group, entries: group.entries.slice(0, 10) })); +} + +function totalEntryCount(groups) { + return groups.reduce((total, group) => total + group.entries.length, 0); +} + +function renderEntryRow(entry, rank) { + const metrics = selectedMetrics(entry); + return ` + + ${rank} + + ${entry.model}${entry.reasoningLabel ? ` (${entry.reasoningLabel})` : ""} + + ${entry.organization} + ${formatPassAt1(entry)} + ${formatPercent(metrics.passAt5)} + ${formatUx(metrics.meanUxScore)} + ${formatCost(metrics.costPerTask)} + ${entry.submissionDate} + ${statusLabel(entry.verificationStatus)} + + `; +} + +function renderGroupedRows(groups) { + return groups + .map((group) => { + const rows = group.entries.map((entry, index) => renderEntryRow(entry, index + 1)).join(""); + if (!group.version) return rows; + + return ` + + Benchmark Version: ${group.version} + + ${rows} + `; + }) + .join(""); } function formatPercent(value) { @@ -256,39 +322,20 @@ function setTrackUrl(track) { state.track = trackFromUrl(); function render() { - const ranked = sortedEntries(); - const visible = state.showAll ? ranked : ranked.slice(0, 10); + const groups = groupedEntries(trackEntries()); + const visible = visibleGroups(groups); + const totalEntries = totalEntryCount(groups); primaryScoreSort.textContent = scoreLabels[state.scoreView]; - body.innerHTML = visible - .map((entry) => { - const rank = ranked.indexOf(entry) + 1; - const metrics = selectedMetrics(entry); - return ` - - ${rank} - - ${entry.model}${entry.reasoningLabel ? ` (${entry.reasoningLabel})` : ""} - - ${entry.organization} - ${formatPassAt1(entry)} - ${formatPercent(metrics.passAt5)} - ${formatUx(metrics.meanUxScore)} - ${formatCost(metrics.costPerTask)} - ${entry.submissionDate} - ${statusLabel(entry.verificationStatus)} - - `; - }) - .join(""); + body.innerHTML = renderGroupedRows(visible); - if (!visible.length) { + if (!totalEntries) { body.innerHTML = 'No submitted results for this track yet.'; } - toggleRows.hidden = ranked.length <= 10; - toggleRows.textContent = state.showAll ? "Show top 10" : `Show all ${ranked.length}`; + toggleRows.hidden = groups.every((group) => group.entries.length <= 10); + toggleRows.textContent = state.showAll ? "Show top 10 per version" : `Show all ${totalEntries}`; tabs.forEach((tab) => { const active = tab.dataset.track === state.track; diff --git a/.leaderboard/styles.css b/.leaderboard/styles.css index da2a7ad..0ea5206 100644 --- a/.leaderboard/styles.css +++ b/.leaderboard/styles.css @@ -327,6 +327,20 @@ a { background: #fcfdff; } +.leaderboard-table .version-row td { + background: #f4f8fc; + border-bottom: 1px solid var(--border); + color: var(--brand-navy); + font-size: 16px; + font-weight: 750; + letter-spacing: 0; + padding: 18px 10px 12px; +} + +.leaderboard-table tbody .version-row:hover { + background: transparent; +} + .rank-col, .rank-cell { width: 64px; From f28f0335b34a5ef6766dbb8075c456ac15bd43f1 Mon Sep 17 00:00:00 2001 From: Nishant Yadav Date: Fri, 12 Jun 2026 11:47:45 -0700 Subject: [PATCH 2/2] bump version to 0.7.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a89e72..43eb4d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "state-bench" -version = "0.7.0" +version = "0.7.1" description = "STATE-Bench: multi-domain benchmark for evaluating procedural memory in AI agents" readme = "README.md" requires-python = ">=3.12"