diff --git a/pyproject.toml b/pyproject.toml index 2d28f41..0e709b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,3 +59,10 @@ strict = true [[tool.mypy.overrides]] module = ["wandb", "huggingface_hub"] ignore_missing_imports = true + +# wandb/huggingface_hub ship untyped APIs; when the tracking extra is +# installed locally, strict mode would otherwise flag every call into them +# (CI doesn't install the extra, so a type: ignore would be "unused" there). +[[tool.mypy.overrides]] +module = ["rle.tracking.wandb_logger", "rle.tracking.hf_logger"] +disallow_untyped_calls = false diff --git a/scripts/push_hf.py b/scripts/push_hf.py new file mode 100644 index 0000000..9b9d8ee --- /dev/null +++ b/scripts/push_hf.py @@ -0,0 +1,93 @@ +"""Push an existing spread run to the HuggingFace dataset repo. + +Standalone half of the HF integration (issue #45): pushes a completed +run dir without re-running the benchmark, regenerating the dataset card +(the public leaderboard) from that run's leaderboard.json. + +Usage: + python scripts/push_hf.py --spread-dir results/spread --date 2026-06-11 + python scripts/push_hf.py --spread-dir results/spread --date 2026-06-11 --dry-run + +Auth: HF_TOKEN in .env (RLEConfig). Repo: HF_DATASET_REPO or the default. +""" +from __future__ import annotations + +import argparse +import fnmatch +import json +from pathlib import Path + +from rle.config import RLEConfig +from rle.tracking.hf_logger import ( + SPREAD_ALLOW_PATTERNS, + HFLogger, + build_dataset_card, +) + + +def _matching_files(folder: Path) -> list[Path]: + """Files in folder that the push would include (mirror of allow_patterns).""" + return sorted( + f for f in folder.rglob("*") + if f.is_file() + and any(fnmatch.fnmatch(f.as_posix(), p) for p in SPREAD_ALLOW_PATTERNS) + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spread-dir", type=Path, required=True, + help="run dir containing leaderboard.json (e.g. results/spread)") + parser.add_argument("--date", required=True, help="run date, YYYY-MM-DD") + parser.add_argument("--repo", default=None, + help="dataset repo id (default: RLEConfig.hf_dataset_repo)") + parser.add_argument("--history", type=Path, + default=Path("results/benchmark_history.jsonl")) + parser.add_argument("--baseline-dir", type=Path, default=Path("results/baseline")) + parser.add_argument("--dry-run", action="store_true", + help="print what would be pushed, push nothing") + args = parser.parse_args() + + board_path = args.spread_dir / "leaderboard.json" + board = json.loads(board_path.read_text(encoding="utf-8")) + card = build_dataset_card(board, args.date) + + config = RLEConfig() + repo_id = args.repo or config.hf_dataset_repo + + if args.dry_run: + print(f"Would push to dataset: {repo_id}\n") + print("README.md (dataset card):") + print(" " + "\n ".join(card.splitlines()[:6]) + "\n ...") + if args.history.exists(): + print(f"benchmark_history.jsonl <- {args.history}") + for label, folder, prefix in ( + ("baseline", args.baseline_dir, "baseline/"), + ("spread", args.spread_dir, f"runs/spread-{args.date}/"), + ): + if folder.exists(): + files = _matching_files(folder) + total_kb = sum(f.stat().st_size for f in files) / 1024 + print(f"{prefix} <- {folder} ({len(files)} files, {total_kb:.0f} KB)") + else: + print(f"SKIP {label}: {folder} missing") + return + + if not config.hf_token: + raise SystemExit("HF_TOKEN not set in .env — cannot push.") + + hf = HFLogger(repo_id=repo_id, token=config.hf_token) + if not hf.enabled: + raise SystemExit("HuggingFace auth failed — check HF_TOKEN.") + + hf.push_card(card) + hf.push_results( + history_path=args.history if args.history.exists() else None, + baselines_dir=args.baseline_dir if args.baseline_dir.exists() else None, + ) + hf.push_spread(args.spread_dir, args.date) + print(f"Pushed to https://huggingface.co/datasets/{repo_id}") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index dedd3d9..e844725 100644 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -545,10 +545,13 @@ async def main(args: argparse.Namespace) -> None: ) return - # N >= 4 enforcement + # N >= 4 advisory (issue #45: N=1 content spreads are valid dataset + # entries when labeled as such — the dataset card carries the framing) if args.push_hf and num_runs < 4: - print("ERROR: Leaderboard submission requires --runs 4 or higher.") - return + print( + "WARNING: --push-hf with --runs < 4 — pushed as a content-spread " + "entry, NOT statistically valid.", + ) if num_runs < 4 and not use_mock_rimapi: print(f"WARNING: N={num_runs} runs is below minimum (4) for statistical validity.") ticks_override = _resolve_ticks(args, use_mock_rimapi) @@ -767,11 +770,13 @@ async def main(args: argparse.Namespace) -> None: # HuggingFace Hub push (optional) if args.push_hf: - hf = HFLogger(enabled=True) + hf = HFLogger( + repo_id=config.hf_dataset_repo, token=config.hf_token, + ) if hf.enabled: hf.push_results( history_path=history_path, - baselines_dir=Path("results/baselines"), + baselines_dir=Path("results/baseline"), run_dir=output_dir, ) print("Results pushed to HuggingFace Hub") diff --git a/src/rle/config.py b/src/rle/config.py index 06e4fd9..89e11ef 100644 --- a/src/rle/config.py +++ b/src/rle/config.py @@ -52,6 +52,10 @@ class RLEConfig(BaseSettings): log_level: str = "INFO" docker_image: str = "rle-headless:latest" docker_port: int = 8765 + hf_token: str | None = None + """Fine-grained HuggingFace write token (HF_TOKEN in .env) for dataset pushes.""" + hf_dataset_repo: str = "AppSprout/rle-benchmarks" + """Target HF dataset repo (HF_DATASET_REPO in .env to override).""" def get_helix_config(self) -> HelixConfig: """Return the HelixConfig preset matching ``helix_preset``.""" diff --git a/src/rle/tracking/hf_logger.py b/src/rle/tracking/hf_logger.py index de34e68..1c2feac 100644 --- a/src/rle/tracking/hf_logger.py +++ b/src/rle/tracking/hf_logger.py @@ -1,12 +1,106 @@ -"""HuggingFace Hub integration for sharing benchmark results.""" +"""HuggingFace Hub integration for sharing benchmark results. + +The dataset card (README.md) doubles as a public leaderboard: it is +regenerated from leaderboard.json on every push, so the Hub page always +shows the latest spread. Auth comes from RLEConfig.hf_token (HF_TOKEN in +.env) — pydantic-settings loads .env into the config object, not +os.environ, so the token must be passed explicitly. +""" from __future__ import annotations import logging from pathlib import Path +from typing import Any logger = logging.getLogger(__name__) +DEFAULT_REPO_ID = "AppSprout/rle-benchmarks" + +# Spread run dirs mix text artifacts with capture leftovers; only the +# analysis-grade text formats belong on the Hub. +SPREAD_ALLOW_PATTERNS = ["**/*.json", "**/*.jsonl", "**/*.csv", "**/*.md", "**/*.ts"] + +_CARD_HEADER = """--- +license: mit +pretty_name: RLE — RimWorld Learning Environment Benchmarks +tags: +- benchmark +- multi-agent +- agents +- llm +- rimworld +- game +--- + +# RLE — RimWorld Learning Environment Benchmarks + +Can 7 role-specialized LLM agents keep a RimWorld colony alive? RLE is a +multi-agent coordination benchmark: MapAnalyst + 6 domain agents +(resources, defense, research, social, construction, medical) manage a +live colony through a REST API, scored on a 10-metric weighted composite +against a no-agent baseline (RimWorld's built-in pawn AI, static +4-seed reference). + +- Site + featured runs: https://rle.appsprout.dev +- Harness: https://github.com/AppSprout-dev/RLE +""" + + +def build_dataset_card(board: dict[str, Any], date: str) -> str: + """Render the dataset card README from a spread's leaderboard.json. + + Pure function — testable without the Hub. Costs prefer the billed + OpenRouter ground truth (``real_cost_usd``); estimates are marked + with ``~`` (subscription-billed models have no metered ground truth). + """ + baseline = board.get("baseline", {}) + rows: list[dict[str, Any]] = board.get("rows", []) + + lines = [ + _CARD_HEADER, + f"## Latest spread — {date} (N=1, seed 42, Crashlanded)", + "", + "Ranked by mean composite over the run. N=1 is content-first, not", + "statistically valid — no confidence intervals. Winners advance to N=4.", + f"Baseline: no-agent, {baseline.get('n_runs', '?')} seeds, mean " + f"time-to-end {baseline.get('mean_time_to_end_days', '?')} days.", + "", + "| # | model | mean | final | vs baseline | ticks > base | action ok | cost |", + "|---|-------|------|-------|-------------|--------------|-----------|------|", + ] + for i, r in enumerate(rows, 1): + real = r.get("real_cost_usd") + cost = f"${real:.2f}" if real is not None else f"~${r.get('est_cost_usd', 0):.2f}" + lines.append( + f"| {i} | {r['model']} | {r['mean_composite']:.3f} " + f"| {r['final_composite']:.3f} | {r['vs_baseline_mean_delta']:+.3f} " + f"| {r['ticks_above_baseline']} | {r['raw_action_success']:.0%} | {cost} |" + ) + + above = [r for r in rows if r.get("vs_baseline_mean_delta", 0) > 0] + lines += [ + "", + f"**{len(above)} of {len(rows)} models beat the no-agent baseline.**" + + ( + " (" + ", ".join(r["model"] for r in above) + ")" + if above else "" + ), + "", + "## Layout", + "", + "- `benchmark_history.jsonl` — every tracked run (one JSON object per line)", + "- `baseline/` — the static no-agent reference runs (per-seed CSV + summary)", + "- `runs/spread-/` — per-model artifacts: summary JSON, per-tick CSV,", + " structured event log, full deliberation transcripts, `leaderboard.json`,", + " `site/site_data.json` (website payload)", + "", + "Costs marked `~` are token-count estimates (subscription-billed models);", + "unmarked costs are OpenRouter billed ground truth.", + "", + ] + return "\n".join(lines) + class HFLogger: """Pushes benchmark results to a HuggingFace dataset repo. @@ -16,17 +110,19 @@ class HFLogger: def __init__( self, - repo_id: str = "appsprout/rle-benchmarks", + repo_id: str = DEFAULT_REPO_ID, enabled: bool = True, + token: str | None = None, ) -> None: self._api = None self._repo_id = repo_id + self._token = token if not enabled: return try: from huggingface_hub import HfApi - self._api = HfApi() + self._api = HfApi(token=token) # Verify auth self._api.whoami() except ImportError: @@ -39,6 +135,45 @@ def __init__( def enabled(self) -> bool: return self._api is not None + def _ensure_repo(self) -> None: + assert self._api is not None + self._api.create_repo(self._repo_id, repo_type="dataset", exist_ok=True) + + def push_card(self, content: str) -> None: + """Upload the dataset card (README.md) — the public leaderboard.""" + if not self._api: + return + try: + self._ensure_repo() + self._api.upload_file( + path_or_fileobj=content.encode("utf-8"), + path_in_repo="README.md", + repo_id=self._repo_id, + repo_type="dataset", + commit_message="Update dataset card / leaderboard", + ) + logger.info("Pushed dataset card to %s", self._repo_id) + except Exception: + logger.warning("HuggingFace card push failed", exc_info=True) + + def push_spread(self, spread_dir: Path, date: str) -> None: + """Upload a spread run dir to runs/spread-/ (text artifacts only).""" + if not self._api: + return + try: + self._ensure_repo() + self._api.upload_folder( + folder_path=str(spread_dir), + path_in_repo=f"runs/spread-{date}", + repo_id=self._repo_id, + repo_type="dataset", + allow_patterns=SPREAD_ALLOW_PATTERNS, + commit_message=f"Add spread run: {date}", + ) + logger.info("Pushed spread %s to %s", date, self._repo_id) + except Exception: + logger.warning("HuggingFace spread push failed", exc_info=True) + def push_results( self, history_path: Path | None = None, @@ -50,10 +185,7 @@ def push_results( return try: - # Ensure repo exists - self._api.create_repo( - self._repo_id, repo_type="dataset", exist_ok=True, - ) + self._ensure_repo() # Push history JSONL if history_path and history_path.exists(): @@ -66,17 +198,17 @@ def push_results( ) logger.info("Pushed benchmark_history.jsonl to %s", self._repo_id) - # Push baselines + # Push baselines (static no-agent reference) if baselines_dir and baselines_dir.exists(): - for f in baselines_dir.glob("*.json"): - self._api.upload_file( - path_or_fileobj=str(f), - path_in_repo=f"baselines/{f.name}", - repo_id=self._repo_id, - repo_type="dataset", - commit_message=f"Update baseline: {f.stem}", - ) - logger.info("Pushed baselines to %s", self._repo_id) + self._api.upload_folder( + folder_path=str(baselines_dir), + path_in_repo="baseline", + repo_id=self._repo_id, + repo_type="dataset", + allow_patterns=SPREAD_ALLOW_PATTERNS, + commit_message="Update no-agent baseline", + ) + logger.info("Pushed baseline to %s", self._repo_id) # Push latest run artifacts if run_dir and run_dir.exists(): @@ -85,6 +217,7 @@ def push_results( path_in_repo=f"runs/{run_dir.name}", repo_id=self._repo_id, repo_type="dataset", + allow_patterns=SPREAD_ALLOW_PATTERNS, commit_message=f"Add run: {run_dir.name}", ) logger.info("Pushed run %s to %s", run_dir.name, self._repo_id) diff --git a/tests/unit/test_hf_logger.py b/tests/unit/test_hf_logger.py new file mode 100644 index 0000000..226927f --- /dev/null +++ b/tests/unit/test_hf_logger.py @@ -0,0 +1,79 @@ +"""Tests for the HuggingFace dataset card builder.""" + +from __future__ import annotations + +from rle.tracking.hf_logger import build_dataset_card + +_BOARD = { + "baseline": {"mean_time_to_end_days": 8.0, "n_runs": 4}, + "rows": [ + { + "model": "x-ai/grok-4.3", + "mean_composite": 0.8359, + "final_composite": 0.804, + "vs_baseline_mean_delta": -0.0035, + "ticks_above_baseline": "5/10", + "raw_action_success": 0.7874, + "est_cost_usd": 0.54, + "real_cost_usd": 0.358, + }, + { + "model": "z-ai/glm-5.1", + "mean_composite": 0.7818, + "final_composite": 0.7396, + "vs_baseline_mean_delta": 0.0276, + "ticks_above_baseline": "7/10", + "raw_action_success": 0.7111, + "est_cost_usd": 0.81, + "real_cost_usd": 0.939, + }, + { + "model": "claude-fable-5", + "mean_composite": 0.8051, + "final_composite": 0.7213, + "vs_baseline_mean_delta": -0.015, + "ticks_above_baseline": "4/10", + "raw_action_success": 0.6942, + "est_cost_usd": 5.48, + "real_cost_usd": None, + }, + ], +} + + +class TestBuildDatasetCard: + def test_has_hub_frontmatter(self) -> None: + card = build_dataset_card(_BOARD, "2026-06-11") + assert card.startswith("---\n") + assert "license: mit" in card + assert "pretty_name:" in card + + def test_all_models_in_table_with_rank(self) -> None: + card = build_dataset_card(_BOARD, "2026-06-11") + assert "| 1 | x-ai/grok-4.3 |" in card + assert "| 2 | z-ai/glm-5.1 |" in card + assert "| 3 | claude-fable-5 |" in card + + def test_real_cost_unmarked_estimate_marked(self) -> None: + card = build_dataset_card(_BOARD, "2026-06-11") + assert "$0.36" in card # billed ground truth, no marker + assert "~$5.48" in card # subscription model: estimate marker + + def test_baseline_beaters_called_out(self) -> None: + card = build_dataset_card(_BOARD, "2026-06-11") + assert "1 of 3 models beat the no-agent baseline." in card + assert "(z-ai/glm-5.1)" in card + + def test_date_and_baseline_framing(self) -> None: + card = build_dataset_card(_BOARD, "2026-06-11") + assert "2026-06-11" in card + assert "time-to-end 8.0 days" in card + assert "not" in card and "statistically valid" in card + + def test_no_baseline_beaters_omits_paren(self) -> None: + board = { + "baseline": _BOARD["baseline"], + "rows": [r for r in _BOARD["rows"] if r["model"] != "z-ai/glm-5.1"], + } + card = build_dataset_card(board, "2026-06-11") + assert "0 of 2 models beat the no-agent baseline." in card