From 2c27cc43d99f7846845683a2b4d8718707631400 Mon Sep 17 00:00:00 2001 From: jkbennitt Date: Fri, 12 Jun 2026 00:52:35 -0400 Subject: [PATCH] feat: add export_site_data.py website fold-in exporter Reads results//leaderboard.json (analyze_spread.py output) and emits site/data.fragment.ts (META + MODELS blocks matching the ModelRow interface on rle.appsprout.dev) plus site/site_data.json for HF dataset / API ingestion. Round-trip verified against the live v0.3.0 site data: exact match except three half-case roundings where the hand-edit went toward zero. Editorial blocks (HERO_LOG, STORY, AGENTS, OG_*) stay hand-curated; this mechanizes the leaderboard half of a fold-in so numbers are never hand-transcribed again. Co-Authored-By: Claude Fable 5 --- scripts/export_site_data.py | 198 ++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 scripts/export_site_data.py diff --git a/scripts/export_site_data.py b/scripts/export_site_data.py new file mode 100644 index 0000000..503d2ee --- /dev/null +++ b/scripts/export_site_data.py @@ -0,0 +1,198 @@ +"""Export a spread's leaderboard.json into website-ready artifacts. + +Bridges the run-analysis pipeline to rle.appsprout.dev: reads +results//leaderboard.json (written by analyze_spread.py) and emits + + /data.fragment.ts META + MODELS blocks matching the ModelRow + interface in AppSprout-Site client/src/rle/data.ts + /site_data.json the same payload as JSON (HF dataset / API ingestion) + +Editorial blocks (HERO_LOG, STORY, AGENTS, OG_*) stay hand-curated — this +covers the mechanical half of a fold-in so leaderboard numbers are never +hand-transcribed again. + +Usage: + python scripts/export_site_data.py --spread-dir results/spread --date 2026-06-11 +""" +from __future__ import annotations + +import argparse +import json +import sys +from decimal import ROUND_HALF_UP, Decimal +from pathlib import Path + +# Display name / family / family swatch per model slug, mirroring the palette +# in AppSprout-Site client/src/rle/data.ts. Unknown slugs fall back to a +# derived family + neutral color with a warning, so a new model never blocks +# an export. +MODEL_REGISTRY: dict[str, tuple[str, str, str]] = { + "x-ai/grok-4.3": ("Grok 4.3", "xAI", "#C77DB8"), + "mistralai/mistral-medium-3-5": ("Mistral Medium 3.5", "Mistral", "#FF7000"), + "google/gemini-3.5-flash": ("Gemini 3.5 Flash", "Google", "#6B9BF0"), + "qwen/qwen3.7-max": ("Qwen3.7 Max", "Alibaba", "#A36BFF"), + "claude-fable-5": ("Claude Fable 5", "Anthropic", "#E8A06A"), + "nvidia/nemotron-3-super-120b-a12b": ("Nemotron 3 Super 120B", "NVIDIA", "#76B900"), + "claude-opus-4-8": ("Claude Opus 4.8", "Anthropic", "#D97757"), + "z-ai/glm-5.1": ("GLM-5.1", "Zhipu", "#00D4D4"), + "openai/gpt-5.5": ("GPT-5.5", "OpenAI", "#56B79A"), + "deepseek/deepseek-v4-pro": ("DeepSeek-V4 Pro", "DeepSeek", "#8A7CF0"), + "moonshotai/kimi-k2.6": ("Kimi K2.6", "Moonshot", "#9AA0A6"), + "nvidia/nemotron-3-nano-30b-a3b": ("Nemotron 3 Nano 30B", "NVIDIA", "#76B900"), + "unsloth/nvidia-nemotron-3-nano-4b": ("Nemotron 3 Nano 4B", "NVIDIA", "#76B900"), +} +FALLBACK_COLOR = "#B7AC8B" + +GITHUB_URL = "https://github.com/AppSprout-dev/RLE" +GITHUB_REPO = "AppSprout-dev/RLE" +DISCORD_URL = "https://discord.gg/vKuyjqNEea" + + +def rnd(value: float, places: int) -> float: + """Round half away from zero (matches hand-rounded site data, unlike + Python's banker's rounding).""" + q = Decimal(1).scaleb(-places) + return float(Decimal(str(value)).quantize(q, rounding=ROUND_HALF_UP)) + + +def fmt(value: float, places: int) -> str: + """Fixed-decimal literal for the TS output (keeps trailing zeros: 0.710, 4.00).""" + return f"{rnd(value, places):.{places}f}" + + +def registry_entry(slug: str) -> tuple[str, str, str]: + if slug in MODEL_REGISTRY: + return MODEL_REGISTRY[slug] + family = slug.split("/")[0] if "/" in slug else slug.split("-")[0] + print(f"WARNING: {slug} not in MODEL_REGISTRY — using derived family " + f"{family!r} + fallback color. Add it to the registry.", file=sys.stderr) + return slug, family, FALLBACK_COLOR + + +def build_model(row: dict) -> dict: + slug = row["model"] + display, family, color = registry_entry(slug) + real = row.get("real_cost_usd") + cost = real if real is not None else row["est_cost_usd"] + return { + "slug": slug, + "key": row["name"], + "model": display, + "family": family, + "color": color, + "meanComposite": rnd(row["mean_composite"], 3), + "finalComposite": rnd(row["final_composite"], 3), + "vsBaselineDelta": rnd(row["vs_baseline_mean_delta"], 3), + "ticksAboveBaseline": row["ticks_above_baseline"], + "endDay": int(row["end_day"]), + "rawActionSuccess": rnd(row["raw_action_success"], 2), + "exArtifactSuccess": rnd(row["ex_artifact_success"], 2), + "avgLatencyS": rnd(row["avg_latency_s"], 1), + "wallMin": rnd(row["wall_min"], 1), + "costUsd": rnd(cost, 2), + "costEstimated": real is None, + } + + +def build_meta(board: dict, rows: list[dict], args: argparse.Namespace) -> dict: + metered = [r["real_cost_usd"] for r in board["rows"] if r.get("real_cost_usd") is not None] + ticks = max((len(r.get("trajectory", [])) for r in board["rows"]), default=0) + return { + "scenario": args.scenario, + "seed": args.seed, + "ticks": args.ticks if args.ticks is not None else ticks, + "nRuns": args.n_runs, + "date": args.date, + "totalSpendUsd": rnd(sum(metered), 2), + "baselineMeanTteDays": int(board["baseline"]["mean_time_to_end_days"]), + "githubUrl": GITHUB_URL, + "githubRepo": GITHUB_REPO, + "discordUrl": DISCORD_URL, + } + + +def emit_ts(meta: dict, models: list[dict], spread_dir: Path) -> str: + lines = [ + "// =====================================================================", + "// GENERATED by scripts/export_site_data.py from", + f"// {spread_dir.as_posix()}/leaderboard.json", + "// Paste over the META + MODELS blocks in AppSprout-Site", + "// client/src/rle/data.ts. Editorial blocks (HERO_LOG, STORY, AGENTS,", + "// OG_*) stay hand-curated — update those from stories.md separately.", + "// =====================================================================", + "", + "export const META = {", + f' scenario: "{meta["scenario"]}",', + f" seed: {meta['seed']},", + f" ticks: {meta['ticks']},", + f" nRuns: {meta['nRuns']},", + f' date: "{meta["date"]}",', + " // OpenRouter spend for the metered models; subscription-billed models", + " // (costEstimated: true) are not included.", + f" totalSpendUsd: {fmt(meta['totalSpendUsd'], 2)},", + f" baselineMeanTteDays: {meta['baselineMeanTteDays']},", + f' githubUrl: "{meta["githubUrl"]}",', + f' githubRepo: "{meta["githubRepo"]}",', + f' discordUrl: "{meta["discordUrl"]}",', + "} as const;", + "", + "// Ranked by mean composite over the run (analyze_spread.py ordering).", + "export const MODELS: ModelRow[] = [", + ] + for m in models: + lines += [ + " {", + f' slug: "{m["slug"]}", key: "{m["key"]}", model: "{m["model"]}",' + f' family: "{m["family"]}", color: "{m["color"]}",', + f" meanComposite: {fmt(m['meanComposite'], 3)}," + f" finalComposite: {fmt(m['finalComposite'], 3)}," + f" vsBaselineDelta: {fmt(m['vsBaselineDelta'], 3)}," + f' ticksAboveBaseline: "{m["ticksAboveBaseline"]}",', + f" endDay: {m['endDay']}, rawActionSuccess: {fmt(m['rawActionSuccess'], 2)}," + f" exArtifactSuccess: {fmt(m['exArtifactSuccess'], 2)}," + f" avgLatencyS: {fmt(m['avgLatencyS'], 1)}, wallMin: {fmt(m['wallMin'], 1)},", + f" costUsd: {fmt(m['costUsd'], 2)}," + f" costEstimated: {'true' if m['costEstimated'] else 'false'},", + " },", + ] + lines += ["];", ""] + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spread-dir", type=Path, required=True, + help="run dir containing leaderboard.json (e.g. results/spread)") + parser.add_argument("--date", required=True, help="run date, YYYY-MM-DD") + parser.add_argument("--out", type=Path, default=None, + help="output dir (default: /site)") + parser.add_argument("--scenario", default="Crashlanded") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--ticks", type=int, default=None, + help="override tick count (default: longest trajectory)") + parser.add_argument("--n-runs", type=int, default=1) + args = parser.parse_args() + + board = json.loads((args.spread_dir / "leaderboard.json").read_text(encoding="utf-8")) + models = [build_model(r) for r in board["rows"]] + meta = build_meta(board, models, args) + + out = args.out or (args.spread_dir / "site") + out.mkdir(parents=True, exist_ok=True) + + ts_path = out / "data.fragment.ts" + ts_path.write_text(emit_ts(meta, models, args.spread_dir), encoding="utf-8") + + json_path = out / "site_data.json" + json_path.write_text( + json.dumps({"meta": meta, "models": models}, indent=2) + "\n", + encoding="utf-8") + + print(f"Wrote {ts_path}") + print(f"Wrote {json_path}") + print(f"{len(models)} models | total metered spend ${meta['totalSpendUsd']:.2f} " + f"| date {meta['date']}") + + +if __name__ == "__main__": + main()