feat: HuggingFace dataset integration

pyproject.toml

@@ -59,3 +59,10 @@ strict = true
 [[tool.mypy.overrides]]
 module = ["wandb", "huggingface_hub"]
 ignore_missing_imports = true
+
+# wandb/huggingface_hub ship untyped APIs; when the tracking extra is
+# installed locally, strict mode would otherwise flag every call into them
+# (CI doesn't install the extra, so a type: ignore would be "unused" there).
+[[tool.mypy.overrides]]
+module = ["rle.tracking.wandb_logger", "rle.tracking.hf_logger"]
+disallow_untyped_calls = false

scripts/push_hf.py

@@ -0,0 +1,93 @@
+"""Push an existing spread run to the HuggingFace dataset repo.
+
+Standalone half of the HF integration (issue #45): pushes a completed
+run dir without re-running the benchmark, regenerating the dataset card
+(the public leaderboard) from that run's leaderboard.json.
+
+Usage:
+    python scripts/push_hf.py --spread-dir results/spread --date 2026-06-11
+    python scripts/push_hf.py --spread-dir results/spread --date 2026-06-11 --dry-run
+
+Auth: HF_TOKEN in .env (RLEConfig). Repo: HF_DATASET_REPO or the default.
+"""
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import json
+from pathlib import Path
+
+from rle.config import RLEConfig
+from rle.tracking.hf_logger import (
+    SPREAD_ALLOW_PATTERNS,
+    HFLogger,
+    build_dataset_card,
+)
+
+
+def _matching_files(folder: Path) -> list[Path]:
+    """Files in folder that the push would include (mirror of allow_patterns)."""
+    return sorted(
+        f for f in folder.rglob("*")
+        if f.is_file()
+        and any(fnmatch.fnmatch(f.as_posix(), p) for p in SPREAD_ALLOW_PATTERNS)
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--spread-dir", type=Path, required=True,
+                        help="run dir containing leaderboard.json (e.g. results/spread)")
+    parser.add_argument("--date", required=True, help="run date, YYYY-MM-DD")
+    parser.add_argument("--repo", default=None,
+                        help="dataset repo id (default: RLEConfig.hf_dataset_repo)")
+    parser.add_argument("--history", type=Path,
+                        default=Path("results/benchmark_history.jsonl"))
+    parser.add_argument("--baseline-dir", type=Path, default=Path("results/baseline"))
+    parser.add_argument("--dry-run", action="store_true",
+                        help="print what would be pushed, push nothing")
+    args = parser.parse_args()
+
+    board_path = args.spread_dir / "leaderboard.json"
+    board = json.loads(board_path.read_text(encoding="utf-8"))
+    card = build_dataset_card(board, args.date)
+
+    config = RLEConfig()
+    repo_id = args.repo or config.hf_dataset_repo
+
+    if args.dry_run:
+        print(f"Would push to dataset: {repo_id}\n")
+        print("README.md (dataset card):")
+        print("  " + "\n  ".join(card.splitlines()[:6]) + "\n  ...")
+        if args.history.exists():
+            print(f"benchmark_history.jsonl  <- {args.history}")
+        for label, folder, prefix in (
+            ("baseline", args.baseline_dir, "baseline/"),
+            ("spread", args.spread_dir, f"runs/spread-{args.date}/"),
+        ):
+            if folder.exists():
+                files = _matching_files(folder)
+                total_kb = sum(f.stat().st_size for f in files) / 1024
+                print(f"{prefix}  <- {folder} ({len(files)} files, {total_kb:.0f} KB)")
+            else:
+                print(f"SKIP {label}: {folder} missing")
+        return
+
+    if not config.hf_token:
+        raise SystemExit("HF_TOKEN not set in .env — cannot push.")
+
+    hf = HFLogger(repo_id=repo_id, token=config.hf_token)
+    if not hf.enabled:
+        raise SystemExit("HuggingFace auth failed — check HF_TOKEN.")
+
+    hf.push_card(card)
+    hf.push_results(
+        history_path=args.history if args.history.exists() else None,
+        baselines_dir=args.baseline_dir if args.baseline_dir.exists() else None,
+    )
+    hf.push_spread(args.spread_dir, args.date)
+    print(f"Pushed to https://huggingface.co/datasets/{repo_id}")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_benchmark.py

@@ -545,10 +545,13 @@ async def main(args: argparse.Namespace) -> None:
         )
         return
 
-    # N >= 4 enforcement
+    # N >= 4 advisory (issue #45: N=1 content spreads are valid dataset
+    # entries when labeled as such — the dataset card carries the framing)
     if args.push_hf and num_runs < 4:
-        print("ERROR: Leaderboard submission requires --runs 4 or higher.")
-        return
+        print(
+            "WARNING: --push-hf with --runs < 4 — pushed as a content-spread "
+            "entry, NOT statistically valid.",
+        )
     if num_runs < 4 and not use_mock_rimapi:
         print(f"WARNING: N={num_runs} runs is below minimum (4) for statistical validity.")
     ticks_override = _resolve_ticks(args, use_mock_rimapi)
@@ -767,11 +770,13 @@ async def main(args: argparse.Namespace) -> None:
 
         # HuggingFace Hub push (optional)
         if args.push_hf:
-            hf = HFLogger(enabled=True)
+            hf = HFLogger(
+                repo_id=config.hf_dataset_repo, token=config.hf_token,
+            )
             if hf.enabled:
                 hf.push_results(
                     history_path=history_path,
-                    baselines_dir=Path("results/baselines"),
+                    baselines_dir=Path("results/baseline"),
                     run_dir=output_dir,
                 )
                 print("Results pushed to HuggingFace Hub")

src/rle/config.py

@@ -52,6 +52,10 @@ class RLEConfig(BaseSettings):
     log_level: str = "INFO"
     docker_image: str = "rle-headless:latest"
     docker_port: int = 8765
+    hf_token: str | None = None
+    """Fine-grained HuggingFace write token (HF_TOKEN in .env) for dataset pushes."""
+    hf_dataset_repo: str = "AppSprout/rle-benchmarks"
+    """Target HF dataset repo (HF_DATASET_REPO in .env to override)."""
 
     def get_helix_config(self) -> HelixConfig:
         """Return the HelixConfig preset matching ``helix_preset``."""

src/rle/tracking/hf_logger.py

@@ -1,12 +1,106 @@
-"""HuggingFace Hub integration for sharing benchmark results."""
+"""HuggingFace Hub integration for sharing benchmark results.
+
+The dataset card (README.md) doubles as a public leaderboard: it is
+regenerated from leaderboard.json on every push, so the Hub page always
+shows the latest spread. Auth comes from RLEConfig.hf_token (HF_TOKEN in
+.env) — pydantic-settings loads .env into the config object, not
+os.environ, so the token must be passed explicitly.
+"""
 
 from __future__ import annotations
 
 import logging
 from pathlib import Path
+from typing import Any
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_REPO_ID = "AppSprout/rle-benchmarks"
+
+# Spread run dirs mix text artifacts with capture leftovers; only the
+# analysis-grade text formats belong on the Hub.
+SPREAD_ALLOW_PATTERNS = ["**/*.json", "**/*.jsonl", "**/*.csv", "**/*.md", "**/*.ts"]
+
+_CARD_HEADER = """---
+license: mit
+pretty_name: RLE — RimWorld Learning Environment Benchmarks
+tags:
+- benchmark
+- multi-agent
+- agents
+- llm
+- rimworld
+- game
+---
+
+# RLE — RimWorld Learning Environment Benchmarks
+
+Can 7 role-specialized LLM agents keep a RimWorld colony alive? RLE is a
+multi-agent coordination benchmark: MapAnalyst + 6 domain agents
+(resources, defense, research, social, construction, medical) manage a
+live colony through a REST API, scored on a 10-metric weighted composite
+against a no-agent baseline (RimWorld's built-in pawn AI, static
+4-seed reference).
+
+- Site + featured runs: https://rle.appsprout.dev
+- Harness: https://github.com/AppSprout-dev/RLE
+"""
+
+
+def build_dataset_card(board: dict[str, Any], date: str) -> str:
+    """Render the dataset card README from a spread's leaderboard.json.
+
+    Pure function — testable without the Hub. Costs prefer the billed
+    OpenRouter ground truth (``real_cost_usd``); estimates are marked
+    with ``~`` (subscription-billed models have no metered ground truth).
+    """
+    baseline = board.get("baseline", {})
+    rows: list[dict[str, Any]] = board.get("rows", [])
+
+    lines = [
+        _CARD_HEADER,
+        f"## Latest spread — {date} (N=1, seed 42, Crashlanded)",
+        "",
+        "Ranked by mean composite over the run. N=1 is content-first, not",
+        "statistically valid — no confidence intervals. Winners advance to N=4.",
+        f"Baseline: no-agent, {baseline.get('n_runs', '?')} seeds, mean "
+        f"time-to-end {baseline.get('mean_time_to_end_days', '?')} days.",
+        "",
+        "| # | model | mean | final | vs baseline | ticks > base | action ok | cost |",
+        "|---|-------|------|-------|-------------|--------------|-----------|------|",
+    ]
+    for i, r in enumerate(rows, 1):
+        real = r.get("real_cost_usd")
+        cost = f"${real:.2f}" if real is not None else f"~${r.get('est_cost_usd', 0):.2f}"
+        lines.append(
+            f"| {i} | {r['model']} | {r['mean_composite']:.3f} "
+            f"| {r['final_composite']:.3f} | {r['vs_baseline_mean_delta']:+.3f} "
+            f"| {r['ticks_above_baseline']} | {r['raw_action_success']:.0%} | {cost} |"
+        )
+
+    above = [r for r in rows if r.get("vs_baseline_mean_delta", 0) > 0]
+    lines += [
+        "",
+        f"**{len(above)} of {len(rows)} models beat the no-agent baseline.**"
+        + (
+            " (" + ", ".join(r["model"] for r in above) + ")"
+            if above else ""
+        ),
+        "",
+        "## Layout",
+        "",
+        "- `benchmark_history.jsonl` — every tracked run (one JSON object per line)",
+        "- `baseline/` — the static no-agent reference runs (per-seed CSV + summary)",
+        "- `runs/spread-<date>/` — per-model artifacts: summary JSON, per-tick CSV,",
+        "  structured event log, full deliberation transcripts, `leaderboard.json`,",
+        "  `site/site_data.json` (website payload)",
+        "",
+        "Costs marked `~` are token-count estimates (subscription-billed models);",
+        "unmarked costs are OpenRouter billed ground truth.",
+        "",
+    ]
+    return "\n".join(lines)
+
 
 class HFLogger:
     """Pushes benchmark results to a HuggingFace dataset repo.
@@ -16,17 +110,19 @@ class HFLogger:
 
     def __init__(
         self,
-        repo_id: str = "appsprout/rle-benchmarks",
+        repo_id: str = DEFAULT_REPO_ID,
         enabled: bool = True,
+        token: str | None = None,
     ) -> None:
         self._api = None
         self._repo_id = repo_id
+        self._token = token
         if not enabled:
             return
         try:
             from huggingface_hub import HfApi
 
-            self._api = HfApi()
+            self._api = HfApi(token=token)
             # Verify auth
             self._api.whoami()
         except ImportError:
@@ -39,6 +135,45 @@ def __init__(
     def enabled(self) -> bool:
         return self._api is not None
 
+    def _ensure_repo(self) -> None:
+        assert self._api is not None
+        self._api.create_repo(self._repo_id, repo_type="dataset", exist_ok=True)
+
+    def push_card(self, content: str) -> None:
+        """Upload the dataset card (README.md) — the public leaderboard."""
+        if not self._api:
+            return
+        try:
+            self._ensure_repo()
+            self._api.upload_file(
+                path_or_fileobj=content.encode("utf-8"),
+                path_in_repo="README.md",
+                repo_id=self._repo_id,
+                repo_type="dataset",
+                commit_message="Update dataset card / leaderboard",
+            )
+            logger.info("Pushed dataset card to %s", self._repo_id)
+        except Exception:
+            logger.warning("HuggingFace card push failed", exc_info=True)
+
+    def push_spread(self, spread_dir: Path, date: str) -> None:
+        """Upload a spread run dir to runs/spread-<date>/ (text artifacts only)."""
+        if not self._api:
+            return
+        try:
+            self._ensure_repo()
+            self._api.upload_folder(
+                folder_path=str(spread_dir),
+                path_in_repo=f"runs/spread-{date}",
+                repo_id=self._repo_id,
+                repo_type="dataset",
+                allow_patterns=SPREAD_ALLOW_PATTERNS,
+                commit_message=f"Add spread run: {date}",
+            )
+            logger.info("Pushed spread %s to %s", date, self._repo_id)
+        except Exception:
+            logger.warning("HuggingFace spread push failed", exc_info=True)
+
     def push_results(
         self,
         history_path: Path | None = None,
@@ -50,10 +185,7 @@ def push_results(
             return
 
         try:
-            # Ensure repo exists
-            self._api.create_repo(
-                self._repo_id, repo_type="dataset", exist_ok=True,
-            )
+            self._ensure_repo()
 
             # Push history JSONL
             if history_path and history_path.exists():
@@ -66,17 +198,17 @@ def push_results(
                 )
                 logger.info("Pushed benchmark_history.jsonl to %s", self._repo_id)
 
-            # Push baselines
+            # Push baselines (static no-agent reference)
             if baselines_dir and baselines_dir.exists():
-                for f in baselines_dir.glob("*.json"):
-                    self._api.upload_file(
-                        path_or_fileobj=str(f),
-                        path_in_repo=f"baselines/{f.name}",
-                        repo_id=self._repo_id,
-                        repo_type="dataset",
-                        commit_message=f"Update baseline: {f.stem}",
-                    )
-                logger.info("Pushed baselines to %s", self._repo_id)
+                self._api.upload_folder(
+                    folder_path=str(baselines_dir),
+                    path_in_repo="baseline",
+                    repo_id=self._repo_id,
+                    repo_type="dataset",
+                    allow_patterns=SPREAD_ALLOW_PATTERNS,
+                    commit_message="Update no-agent baseline",
+                )
+                logger.info("Pushed baseline to %s", self._repo_id)
 
             # Push latest run artifacts
             if run_dir and run_dir.exists():
@@ -85,6 +217,7 @@ def push_results(
                     path_in_repo=f"runs/{run_dir.name}",
                     repo_id=self._repo_id,
                     repo_type="dataset",
+                    allow_patterns=SPREAD_ALLOW_PATTERNS,
                     commit_message=f"Add run: {run_dir.name}",
                 )
                 logger.info("Pushed run %s to %s", run_dir.name, self._repo_id)