From dbb59f7e6dff1e7603ff87af3727f33f58d01e57 Mon Sep 17 00:00:00 2001 From: jkbennitt Date: Thu, 11 Jun 2026 03:29:34 -0400 Subject: [PATCH] chore: set up release-please automation + doc sweep for v0.1.0 Versioning automation: - release-please GitHub Action (python release-type, manifest mode) - config + manifest seeded at 0.1.0; __init__.py annotated for sync - seed CHANGELOG.md with the 0.1.0 entry Doc sweep: - README: --dry-run -> --smoke-test; drop :free OpenRouter slug (rate-limited); replace stale 2-row Nemotron table with the 6-model spread leaderboard (N=1 caveat); add analyze_spread to commands - CONTRIBUTING: 326 -> 458 tests; felix >=0.2.1 -> >=0.3.0; add Conventional Commits section documenting how prefixes drive release-please bumps - CLAUDE.md: add the 6 new scripts to the structure Co-Authored-By: Claude Fable 5 --- .github/workflows/release-please.yml | 22 +++++++++++++++++ .release-please-manifest.json | 3 +++ CHANGELOG.md | 35 ++++++++++++++++++++++++++++ CLAUDE.md | 7 +++++- CONTRIBUTING.md | 21 +++++++++++++---- README.md | 23 +++++++++++------- release-please-config.json | 14 +++++++++++ src/rle/__init__.py | 2 +- 8 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/release-please.yml create mode 100644 .release-please-manifest.json create mode 100644 CHANGELOG.md create mode 100644 release-please-config.json diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..004887c --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,22 @@ +name: release-please + +on: + push: + branches: [master] + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: googleapis/release-please-action@v4 + with: + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + # Uses the default GITHUB_TOKEN. Note: PRs opened by this action with + # GITHUB_TOKEN do not re-trigger ci.yml. If you want CI to run on the + # release PR, create a fine-grained PAT (contents+PRs write) as the + # RELEASE_PLEASE_TOKEN secret and add `token: ${{ secrets.RELEASE_PLEASE_TOKEN }}`. diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..466df71 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.1.0" +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2d88d8b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +## 0.1.0 (2026-06-11) + +First tagged release. RLE is a multi-agent benchmark where 7 Felix Agent SDK +role-specialized LLM agents manage a RimWorld colony — multi-agent coordination +under uncertainty. + +### Features + +* **7-agent architecture** — MapAnalyst (spatial, runs first) + ResourceManager, + DefenseCommander, ResearchDirector, SocialOverseer, ConstructionPlanner, + MedicalOfficer, coordinating through Felix CentralPost hub-spoke. +* **Deterministic spatial analysis** — MAP_SUMMARY with verified build/farm/stockpile + coordinates injected into every agent; agents may not invent coordinates. +* **6 scenarios** — Crashlanded, First Winter, Toxic Fallout, Raid Defense, + Plague Response, Ship Launch, each with victory/failure conditions and weight overrides. +* **10-metric weighted composite scoring** with bootstrap confidence intervals and + per-tick CSV export. +* **Provider-agnostic** via felix-agent-sdk 0.3.0 native async — local (LM Studio), + OpenRouter, Anthropic, OpenAI, and the claude-code subscription provider. +* **RIMAPI integration** — async REST client + SSE event stream (raids, deaths, + mental breaks injected into agent context). +* **Pinned no-agent baseline** — N=4, colony dead by day 8, committed as a provenance + sidecar; every agent run is measured against it. +* **Helix phase adaptation** — exploration → analysis → synthesis as the colony progresses. +* **Docker headless mode**, React dashboard with 5 RLE widgets, W&B/HuggingFace loggers. +* **Post-run analysis & media toolkit** — cross-model leaderboard, story mining, footage + indexing, and the `/run-analysis` skill. + +### Notes + +* First public benchmark data: a 6-model spread (Crashlanded, N=1 content-first). + Grok 4.3 led on mean composite; full caveats in the README. +* 458 tests, mypy strict, ruff clean. diff --git a/CLAUDE.md b/CLAUDE.md index 908b757..790fe69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -377,9 +377,14 @@ src/rle/ scripts/ ├── run_scenario.py # Single scenario CLI (auto-loads save, unforbids items) ├── run_benchmark.py # Full benchmark suite CLI (--docker, --smoke-test, --runs) +├── run_spread_n1.sh # N=1 multi-model spread runner (sequential, continue-on-error) ├── compare_benchmarks.py # Paired statistical comparison of benchmark runs +├── analyze_spread.py # Cross-model leaderboard vs baseline + failure taxonomy ├── visualize_results.py # Matplotlib CSV plotter -└── serve_dashboard.py # CORS-enabled file server for dashboard +├── serve_dashboard.py # CORS-enabled file server for dashboard +├── replay_ticks.py # Replay a run's tick snapshots into the dashboard (for capture) +├── record_dashboard.mjs # Headless Playwright dashboard recording (run with node) +└── capture_run_media.sh # During-run game-window stills + tick snapshots docker/ ├── Dockerfile # HeadlessRim + Xvfb (debian:bookworm-slim) ├── docker-compose.yml # Volume mounts for game files, mods, saves diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 323b064..2192acd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ git clone https://github.com/AppSprout-dev/RLE.git cd RLE uv sync --extra dev -pytest # should pass 326+ tests +pytest # should pass 458+ tests ``` Requires Python 3.14+ and [uv](https://docs.astral.sh/uv/). @@ -16,11 +16,24 @@ Requires Python 3.14+ and [uv](https://docs.astral.sh/uv/). 1. Create a branch from `master` 2. Make changes 3. Run `pytest`, `ruff check src/ tests/ scripts/`, and `mypy src/` -4. Commit with a descriptive message -5. Open a PR against `master` +4. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (see below) +5. Open a PR against `master` — **regular merge commits, never squash** (preserves per-commit history) CI runs lint + type check + tests + smoke test on every push/PR. +### Commit messages drive versioning + +Versioning is automated by [release-please](https://github.com/googleapis/release-please): commit prefixes determine the version bump and changelog entry. Don't hand-edit `version` in `pyproject.toml`. + +| Prefix | Example | Effect (pre-1.0) | +|--------|---------|------------------| +| `feat:` | `feat: add toxic fallout scenario` | minor bump | +| `fix:` | `fix: correct food_security divisor` | patch bump | +| `feat!:` / `BREAKING CHANGE:` | breaking API change | minor bump (stays <1.0) | +| `docs:` `chore:` `refactor:` `test:` `ci:` `perf:` | tidy-ups | no release | + +release-please keeps an open "release PR" accumulating changes; merging it tags the version and publishes a GitHub release with generated notes. + ## Running locally ### Prerequisites @@ -130,7 +143,7 @@ docker/ # HeadlessRim Dockerfile, compose, entrypoint ## Key dependencies -- [felix-agent-sdk](https://github.com/AppSprout-dev/felix-agent-sdk) >= 0.2.1 +- [felix-agent-sdk](https://github.com/AppSprout-dev/felix-agent-sdk) >= 0.3.0 - [RIMAPI](https://github.com/IlyaChichkov/RIMAPI) C# mod - httpx, pydantic >= 2.0, pyyaml - Optional: wandb, huggingface-hub (`uv sync --extra tracking`) diff --git a/README.md b/README.md index 58ed6fc..e32c364 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ PROVIDER_BASE_URL=http://localhost:1234/v1 # OpenRouter (cloud) OPENAI_API_KEY=sk-or-v1-your-key-here PROVIDER=openai -MODEL=nvidia/nemotron-3-super-120b-a12b:free +MODEL=nvidia/nemotron-3-super-120b-a12b PROVIDER_BASE_URL=https://openrouter.ai/api/v1 ``` @@ -149,25 +149,32 @@ cd ../rimapi-dashboard && bun run start ```bash # Full benchmark (mock game state, real LLM) -python scripts/run_benchmark.py --dry-run --ticks 10 +python scripts/run_benchmark.py --smoke-test --ticks 10 # List scenarios python scripts/run_scenario.py --list # Visualize CSV results python scripts/visualize_results.py results/ --all + +# Cross-model leaderboard vs the pinned baseline (after a spread) +python scripts/analyze_spread.py --spread-dir results/spread ``` ## Benchmark Results -Live game, 10 ticks, Crashlanded scenario: +**6-model spread** — Crashlanded, 10 ticks, seed 42. **N=1, content-first — not statistically significant; top models advance to N=4.** Ranked by mean composite across the run (endpoint scores are single-event noisy): -| Model | Composite | Survival | Food | Mood | Efficiency | -|-------|----------|----------|------|------|------------| -| Nemotron 30B (OpenRouter) | **0.808** | 1.000 | 0.950 | 0.441 | 0.754 | -| Nemotron 30B (OpenRouter) | **0.794** | 1.000 | 0.850 | 0.510 | 0.894 | +| # | Model | Mean composite | Cost | +|---|-------|---------------|------| +| 1 | Grok 4.3 | **0.831** | $0.38 | +| 2 | Opus 4.8 | 0.826 | subscription | +| 3 | Gemini 3.5 Flash | 0.825 | $2.41 | +| 4 | Fable 5 | 0.803 | subscription | +| 5 | GPT-5.5 | 0.722 | $3.79 | +| 6 | DeepSeek V4 Pro | 0.704 | $0.55 | -All colonists alive, buildings on solid ground, no water placement. +Measured against a pinned no-agent baseline (N=4, colony dead by day 8). Notable finding: all six models repeatedly re-issued research targets that had just failed, despite explicit per-tick error feedback. ## Scenarios diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..74dd3d1 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "packages": { + ".": { + "release-type": "python", + "package-name": "rimworld-learning-environment", + "changelog-path": "CHANGELOG.md", + "bump-minor-pre-major": true, + "extra-files": [ + "src/rle/__init__.py" + ] + } + } +} diff --git a/src/rle/__init__.py b/src/rle/__init__.py index bd53299..dc00b5f 100644 --- a/src/rle/__init__.py +++ b/src/rle/__init__.py @@ -3,7 +3,7 @@ Multi-agent benchmark where Felix Agent SDK agents play RimWorld. """ -__version__ = "0.1.0" +__version__ = "0.1.0" # x-release-please-version from rle.agents import ( Action,