From 53995be9076bec28bc6a1003679f592a4ad4b6df Mon Sep 17 00:00:00 2001 From: jkbennitt Date: Thu, 11 Jun 2026 18:07:11 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20spread=20capture=20polish=20?= =?UTF-8?q?=E2=80=94=20model-labeled=20ticker,=20dashboard=20tick=20mirror?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - obs_studio.py: _latest_tick_file prefers per-model subdirs so the score ticker shows the model name, not the watch-root dir ("spread"). Capture URLs gain norefresh=1 (kills the dashboard's 5s background-repaint flash; the RLE widgets poll :9000 on their own). - mirror_latest_tick.py: copy the active model's latest_tick.json up to the watch root so a single :9000 tick server follows whichever model runs. Co-Authored-By: Claude Fable 5 --- scripts/mirror_latest_tick.py | 46 +++++++++++++++++++++++++++++++++++ scripts/obs_studio.py | 22 ++++++++--------- 2 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 scripts/mirror_latest_tick.py diff --git a/scripts/mirror_latest_tick.py b/scripts/mirror_latest_tick.py new file mode 100644 index 0000000..1029060 --- /dev/null +++ b/scripts/mirror_latest_tick.py @@ -0,0 +1,46 @@ +"""Mirror the most-recently-updated per-model latest_tick.json up to the watch +root, so a single dashboard tick server (serve_dashboard.py on :9000) follows +whichever model is currently running in a spread. + + python scripts/mirror_latest_tick.py results/spread + +Runs until killed. Pairs with run_spread_n1.sh (per-model output dirs) and the +OBS score ticker (obs_studio.py overlay), which use the same discovery logic. +""" + +from __future__ import annotations + +import shutil +import sys +import time +from pathlib import Path + + +def main() -> None: + root = Path(sys.argv[1] if len(sys.argv) > 1 else "results/spread") + root.mkdir(parents=True, exist_ok=True) + dest = root / "latest_tick.json" + print(f"mirroring newest {root}/*/latest_tick.json -> {dest} (ctrl-c to stop)") + last_src: Path | None = None + last_mtime = 0.0 + while True: + candidates = list(root.glob("*/latest_tick.json")) + if candidates: + newest = max(candidates, key=lambda p: p.stat().st_mtime) + mtime = newest.stat().st_mtime + if newest != last_src or mtime != last_mtime: + try: + shutil.copyfile(newest, dest) + if newest != last_src: + print(f"now following {newest.parent.name}") + last_src, last_mtime = newest, mtime + except OSError: + pass # mid-write; retry next cycle + time.sleep(1.0) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + pass diff --git a/scripts/obs_studio.py b/scripts/obs_studio.py index bae8445..8e1c28e 100644 --- a/scripts/obs_studio.py +++ b/scripts/obs_studio.py @@ -46,14 +46,9 @@ # ?api skips the dashboard's first-run setup screen; ?preset selects the # built-in capture layouts (both ship in rimapi-dashboard for fresh browser # profiles like OBS's embedded CEF). -DASHBOARD_URL = ( - "http://localhost:3000/rimapi-dashboard" - "?api=http%3A%2F%2Flocalhost%3A8765%2Fapi%2Fv1&preset=RLE%20Capture" -) -DASHBOARD_VERTICAL_URL = ( - "http://localhost:3000/rimapi-dashboard" - "?api=http%3A%2F%2Flocalhost%3A8765%2Fapi%2Fv1&preset=RLE%20Vertical" -) +_DASH_BASE = "http://localhost:3000/rimapi-dashboard?api=http%3A%2F%2Flocalhost%3A8765%2Fapi%2Fv1&norefresh=1" +DASHBOARD_URL = f"{_DASH_BASE}&preset=RLE%20Capture" +DASHBOARD_VERTICAL_URL = f"{_DASH_BASE}&preset=RLE%20Vertical" CANVAS_W, CANVAS_H = 1920, 1080 VERTICAL_W = 608 # 608x1080 is exactly 9:16 on a 1080p canvas @@ -215,10 +210,15 @@ def cmd_setup(c: obs.ReqClient) -> None: def _latest_tick_file(watch: Path) -> Path | None: + # Prefer per-model subdirs (their parent name is the model label). Only + # fall back to a root-level latest_tick.json (single-scenario --output, or + # the spread mirror file) when no subdir candidates exist — otherwise the + # mirror file, written last, always wins and the label reads "spread". candidates = list(watch.glob("*/latest_tick.json")) - direct = watch / "latest_tick.json" - if direct.exists(): - candidates.append(direct) + if not candidates: + direct = watch / "latest_tick.json" + if direct.exists(): + candidates = [direct] if not candidates: return None return max(candidates, key=lambda p: p.stat().st_mtime) From 6b33521a79c47ab7c98a62703ba8a7646191fb82 Mon Sep 17 00:00:00 2001 From: jkbennitt Date: Thu, 11 Jun 2026 23:29:06 -0400 Subject: [PATCH 2/3] fix: stockpile overlap guard, semantic priority parsing, window preflight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three harness bugs found in the 2026-06-11 v0.3.0 validation spread (drafted in results/spread/issues_to_file/, archived in rle-media): - stockpile_zone now short-circuits overlapping repeats with a clear error, mirroring the growing-zone guard — RimWorld logs one "overwriting slot group square" error PER CELL on overlap, which spammed the dev log and popped it over the fable5 footage - stockpile priority accepts RimWorld's semantic words (low/normal/ preferred/important/critical) case-insensitively; bare int() crashed on fable5's priority="important" at tick 1 - game loop probes /api/v1/ui/windows once at run start and logs a loud WARNING when the deployed RIMAPI build lacks the window endpoints — the whole 11-model spread ran with auto-dismiss silently inert because every window/close 404 was swallowed at DEBUG level Co-Authored-By: Claude Fable 5 --- src/rle/orchestration/action_executor.py | 42 +++++++++-- src/rle/orchestration/game_loop.py | 8 +++ src/rle/rimapi/client.py | 16 +++++ tests/unit/test_action_executor.py | 92 ++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 5 deletions(-) diff --git a/src/rle/orchestration/action_executor.py b/src/rle/orchestration/action_executor.py index 5f5b642..add1602 100644 --- a/src/rle/orchestration/action_executor.py +++ b/src/rle/orchestration/action_executor.py @@ -107,6 +107,11 @@ def __init__(self, client: RimAPIClient) -> None: # agent learns the zone exists instead of burning ticks on doomed calls # (issue #33). self._created_growing_zones: list[tuple[int, int, int, int]] = [] + # Same guard for stockpiles: RimWorld logs one "overwriting slot group + # square" error PER CELL when a stockpile overlaps an existing one, + # which spams the dev log (and pops it over the game when auto-open + # is enabled). + self._created_stockpile_zones: list[tuple[int, int, int, int]] = [] @staticmethod def _rects_overlap( @@ -269,11 +274,37 @@ async def _h_growing_zone(self, cid: str, params: dict[str, Any]) -> None: ) self._created_growing_zones.append(rect) + # RimWorld's StoragePriority is semantic; models reasonably emit the + # words. Map them instead of crashing in int() (found in the 2026-06-11 + # spread: fable5 sent priority="important"). + _STOCKPILE_PRIORITIES = { + "unstored": 0, "low": 1, "normal": 2, + "preferred": 3, "important": 4, "critical": 5, + } + + @classmethod + def _parse_stockpile_priority(cls, value: Any) -> int: + if isinstance(value, str): + named = cls._STOCKPILE_PRIORITIES.get(value.strip().lower()) + if named is not None: + return named + try: + return int(value) + except (TypeError, ValueError): + return 3 + async def _h_stockpile_zone(self, cid: str, params: dict[str, Any]) -> None: - x1 = params.get("x1", params.get("x", 0)) - z1 = params.get("z1", params.get("z", 0)) - x2 = params.get("x2", x1 + 5) - z2 = params.get("z2", z1 + 5) + x1 = int(params.get("x1", params.get("x", 0))) + z1 = int(params.get("z1", params.get("z", 0))) + x2 = int(params.get("x2", x1 + 5)) + z2 = int(params.get("z2", z1 + 5)) + rect = (min(x1, x2), min(z1, z2), max(x1, x2), max(z1, z2)) + if any(self._rects_overlap(rect, prev) for prev in self._created_stockpile_zones): + raise ValueError( + "A stockpile zone already covers these cells — it was created " + "earlier this run. Do NOT recreate it; pick a different, " + "non-overlapping rectangle or move on to another task." + ) await self._client.create_stockpile_zone( map_id=int(params.get("map_id", 0)), x1=x1, @@ -281,10 +312,11 @@ async def _h_stockpile_zone(self, cid: str, params: dict[str, Any]) -> None: x2=x2, z2=z2, name=params.get("name", ""), - priority=int(params.get("priority", 3)), + priority=self._parse_stockpile_priority(params.get("priority", 3)), allowed_item_defs=params.get("allowed_item_defs"), allowed_item_categories=params.get("allowed_item_categories"), ) + self._created_stockpile_zones.append(rect) async def _h_designate_area(self, cid: str, params: dict[str, Any]) -> None: x1 = params.get("x1", params.get("x", 0)) diff --git a/src/rle/orchestration/game_loop.py b/src/rle/orchestration/game_loop.py index 3f1c523..1e9bf5a 100644 --- a/src/rle/orchestration/game_loop.py +++ b/src/rle/orchestration/game_loop.py @@ -748,6 +748,14 @@ async def run_tick(self) -> TickResult: async def run(self, max_ticks: int | None = None) -> list[TickResult]: """Run the game loop for N ticks or until stopped.""" self._running = True + if not await self._client.window_endpoints_available(): + logger.warning( + "RIMAPI build lacks /api/v1/ui/window endpoints (404) — " + "auto-dismiss of force-pause dialogs is INERT this run. " + "Rebuild and deploy the rle-testing DLL (PR #77), or expect " + "the colony-name dialog and dev debug log to need manual " + "dismissal.", + ) if self._no_pause: await self._client.unpause_game() tick_count = 0 diff --git a/src/rle/rimapi/client.py b/src/rle/rimapi/client.py index e53922f..7c1c8d2 100644 --- a/src/rle/rimapi/client.py +++ b/src/rle/rimapi/client.py @@ -519,6 +519,22 @@ async def list_windows(self) -> list[dict[str, Any]]: return [] return data if isinstance(data, list) else [] + async def window_endpoints_available(self) -> bool: + """Probe whether the deployed RIMAPI build has the /ui/window endpoints. + + The 2026-06-11 spread ran an entire 11-model batch with auto-dismiss + silently inert because the Workshop DLL predated PR #77 and every + window call 404'd inside the swallow-everything error handling. A + connection error returns True (can't tell — don't warn spuriously). + """ + try: + await self._get("/api/v1/ui/windows") + except RimAPIResponseError: + return False + except RimAPIConnectionError: + return True + return True + async def get_resources( self, power_info: PowerData | None = None, ) -> ResourceData: diff --git a/tests/unit/test_action_executor.py b/tests/unit/test_action_executor.py index d942e52..9b44a64 100644 --- a/tests/unit/test_action_executor.py +++ b/tests/unit/test_action_executor.py @@ -183,6 +183,98 @@ async def test_non_overlapping_zone_allowed(self) -> None: assert client.create_growing_zone.await_count == 2 +class TestStockpileZoneIdempotency: + """Stockpile overlaps log one RimWorld error PER CELL ('overwriting slot + group square'), spamming the dev log. Same guard as growing zones.""" + + def _zone(self, x1: int, z1: int, x2: int, z2: int) -> Action: + return Action( + action_type="stockpile_zone", + parameters={"map_id": 0, "x1": x1, "z1": z1, "x2": x2, "z2": z2}, + ) + + async def test_first_stockpile_succeeds(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + result = await executor.execute(_make_plan(self._zone(132, 134, 136, 140))) + assert result.executed == 1 + client.create_stockpile_zone.assert_awaited_once() + + async def test_overlapping_repeat_is_rejected(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + await executor.execute(_make_plan(self._zone(132, 134, 136, 140))) + result = await executor.execute(_make_plan(self._zone(132, 134, 136, 140))) + assert result.failed == 1 + assert result.executed == 0 + assert result.outcomes[0].error is not None + assert "already" in result.outcomes[0].error.lower() + client.create_stockpile_zone.assert_awaited_once() + + async def test_non_overlapping_stockpile_allowed(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + await executor.execute(_make_plan(self._zone(132, 134, 136, 140))) + result = await executor.execute(_make_plan(self._zone(150, 150, 154, 154))) + assert result.executed == 1 + assert client.create_stockpile_zone.await_count == 2 + + async def test_growing_and_stockpile_guards_are_independent(self) -> None: + """A stockpile may legitimately sit where no growing zone is — the two + guards must not share state.""" + client = AsyncMock() + executor = ActionExecutor(client) + grow = Action( + action_type="growing_zone", + parameters={"x1": 10, "z1": 10, "x2": 15, "z2": 15}, + ) + await executor.execute(_make_plan(grow)) + result = await executor.execute(_make_plan(self._zone(10, 10, 15, 15))) + assert result.executed == 1 + + +class TestStockpilePriorityParsing: + """Models emit RimWorld's semantic priority words ('important'); the + handler crashed in int() during the 2026-06-11 spread.""" + + def _zone_with_priority(self, priority: object) -> Action: + return Action( + action_type="stockpile_zone", + parameters={"x1": 1, "z1": 1, "x2": 4, "z2": 4, "priority": priority}, + ) + + async def test_semantic_priority_word_mapped(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + result = await executor.execute(_make_plan(self._zone_with_priority("important"))) + assert result.executed == 1 + assert client.create_stockpile_zone.await_args.kwargs["priority"] == 4 + + async def test_priority_word_case_insensitive(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + await executor.execute(_make_plan(self._zone_with_priority(" Critical "))) + assert client.create_stockpile_zone.await_args.kwargs["priority"] == 5 + + async def test_numeric_priority_passthrough(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + await executor.execute(_make_plan(self._zone_with_priority(2))) + assert client.create_stockpile_zone.await_args.kwargs["priority"] == 2 + + async def test_numeric_string_priority(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + await executor.execute(_make_plan(self._zone_with_priority("4"))) + assert client.create_stockpile_zone.await_args.kwargs["priority"] == 4 + + async def test_garbage_priority_falls_back_to_default(self) -> None: + client = AsyncMock() + executor = ActionExecutor(client) + await executor.execute(_make_plan(self._zone_with_priority("highest"))) + assert client.create_stockpile_zone.await_args.kwargs["priority"] == 3 + + class TestDispatch: async def test_set_work_priority(self) -> None: client = AsyncMock() From 94e34a51eb922fb307687b26f51800fd2cfd2cbc Mon Sep 17 00:00:00 2001 From: jkbennitt Date: Thu, 11 Jun 2026 23:29:06 -0400 Subject: [PATCH 3/3] chore: record v0.3.0 validation spread runs in benchmark history --- results/benchmark_history.jsonl | 11 +++++++++++ uv.lock | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/results/benchmark_history.jsonl b/results/benchmark_history.jsonl index b7ea5d9..a953e7f 100644 --- a/results/benchmark_history.jsonl +++ b/results/benchmark_history.jsonl @@ -11,3 +11,14 @@ {"timestamp": "2026-06-11T01:52:11.998324+00:00", "scoring_version": "1.1", "git_commit": "d0c9a62", "git_branch": "master", "git_dirty": true, "rle_version": "0.1.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "8b26c3820e37bb21ae7227094a7e84fce52ef1099c913899c39a6c78c8f0f4e1", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "google/gemini-3.5-flash", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.8108, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 194601, "total_completion_tokens": 166376, "total_tokens": 360977, "estimated_cost_usd": 1.789286, "wall_time_s": 664.45, "num_calls": 69, "prompt_price_per_token": 1.5e-06, "completion_price_per_token": 9e-06, "pricing_source": "override"}, "event_summary": {"total_events": 753, "errors_by_type": {"parse_failure": 1}, "avg_deliberation_ms": 15690.77, "action_success_rate": 0.6957, "total_tokens": 360977, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.8108, "outcome": "timeout", "ticks": 10}]} {"timestamp": "2026-06-11T02:00:51.207361+00:00", "scoring_version": "1.1", "git_commit": "d0c9a62", "git_branch": "master", "git_dirty": true, "rle_version": "0.1.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "8b26c3820e37bb21ae7227094a7e84fce52ef1099c913899c39a6c78c8f0f4e1", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "x-ai/grok-4.3", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.7904, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 209815, "total_completion_tokens": 66013, "total_tokens": 275828, "estimated_cost_usd": 0.427301, "wall_time_s": 517.38, "num_calls": 70, "prompt_price_per_token": 1.25e-06, "completion_price_per_token": 2.5e-06, "pricing_source": "override"}, "event_summary": {"total_events": 761, "errors_by_type": {}, "avg_deliberation_ms": 8401.06, "action_success_rate": 0.7647, "total_tokens": 275828, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.7904, "outcome": "timeout", "ticks": 10}]} {"timestamp": "2026-06-11T02:29:49.825955+00:00", "scoring_version": "1.1", "git_commit": "d0c9a62", "git_branch": "master", "git_dirty": true, "rle_version": "0.1.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "8b26c3820e37bb21ae7227094a7e84fce52ef1099c913899c39a6c78c8f0f4e1", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "deepseek/deepseek-v4-pro", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.5959, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 189569, "total_completion_tokens": 121047, "total_tokens": 310616, "estimated_cost_usd": 0.187773, "wall_time_s": 1735.02, "num_calls": 68, "prompt_price_per_token": 4.35e-07, "completion_price_per_token": 8.7e-07, "pricing_source": "override"}, "event_summary": {"total_events": 895, "errors_by_type": {"deliberation_timeout": 1, "parse_failure": 1}, "avg_deliberation_ms": 45005.51, "action_success_rate": 0.7368, "total_tokens": 310616, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.5959, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T22:00:31.104898+00:00", "scoring_version": "1.1", "git_commit": "57eb188", "git_branch": "master", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "claude-fable-5", "provider": "claude-code", "base_url": null, "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.7213, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 140, "total_completion_tokens": 109645, "total_reasoning_tokens": 0, "total_tokens": 109785, "estimated_cost_usd": 5.48365, "wall_time_s": 924.1, "num_calls": 70, "prompt_price_per_token": 1e-05, "completion_price_per_token": 5e-05, "pricing_source": "override"}, "event_summary": {"total_events": 701, "errors_by_type": {}, "avg_deliberation_ms": 27858.44, "action_success_rate": 0.6667, "total_tokens": 109785, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.7213, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T22:14:20.482875+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "claude-opus-4-8", "provider": "claude-code", "base_url": null, "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.7095, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 140, "total_completion_tokens": 73053, "total_reasoning_tokens": 0, "total_tokens": 73193, "estimated_cost_usd": 1.827025, "wall_time_s": 827.23, "num_calls": 70, "prompt_price_per_token": 5e-06, "completion_price_per_token": 2.5e-05, "pricing_source": "override"}, "event_summary": {"total_events": 654, "errors_by_type": {}, "avg_deliberation_ms": 19654.52, "action_success_rate": 0.7037, "total_tokens": 73193, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.7095, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T22:33:33.848341+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "openai/gpt-5.5", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.6244, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 203635, "total_completion_tokens": 99505, "total_reasoning_tokens": 54646, "total_tokens": 357786, "estimated_cost_usd": 5.642705, "wall_time_s": 1151.6, "num_calls": 70, "prompt_price_per_token": 5e-06, "completion_price_per_token": 3e-05, "pricing_source": "override"}, "event_summary": {"total_events": 807, "errors_by_type": {}, "avg_deliberation_ms": 28803.34, "action_success_rate": 0.746, "total_tokens": 357786, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.6244, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T22:44:17.553563+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "google/gemini-3.5-flash", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.774, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 195226, "total_completion_tokens": 163767, "total_reasoning_tokens": 135419, "total_tokens": 494412, "estimated_cost_usd": 2.985513, "wall_time_s": 641.24, "num_calls": 66, "prompt_price_per_token": 1.5e-06, "completion_price_per_token": 9e-06, "pricing_source": "override"}, "event_summary": {"total_events": 553, "errors_by_type": {"parse_failure": 4}, "avg_deliberation_ms": 14407.29, "action_success_rate": 0.8182, "total_tokens": 494412, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.774, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T22:51:14.380502+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "x-ai/grok-4.3", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.804, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 210085, "total_completion_tokens": 65092, "total_reasoning_tokens": 44268, "total_tokens": 319445, "estimated_cost_usd": 0.536006, "wall_time_s": 414.84, "num_calls": 70, "prompt_price_per_token": 1.25e-06, "completion_price_per_token": 2.5e-06, "pricing_source": "override"}, "event_summary": {"total_events": 579, "errors_by_type": {}, "avg_deliberation_ms": 5522.93, "action_success_rate": 0.8136, "total_tokens": 319445, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.804, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T23:19:39.981160+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "deepseek/deepseek-v4-pro", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.6103, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 184232, "total_completion_tokens": 114464, "total_reasoning_tokens": 81725, "total_tokens": 380421, "estimated_cost_usd": 0.250825, "wall_time_s": 1703.38, "num_calls": 69, "prompt_price_per_token": 4.35e-07, "completion_price_per_token": 8.7e-07, "pricing_source": "override"}, "event_summary": {"total_events": 956, "errors_by_type": {"parse_failure": 1}, "avg_deliberation_ms": 38874.6, "action_success_rate": 0.7234, "total_tokens": 380421, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.6103, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-11T23:40:40.852789+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "qwen/qwen3.7-max", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": true, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.7829, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 234066, "total_completion_tokens": 122140, "total_reasoning_tokens": 74059, "total_tokens": 430265, "estimated_cost_usd": 1.028329, "wall_time_s": 1258.72, "num_calls": 70, "prompt_price_per_token": 1.25e-06, "completion_price_per_token": 3.75e-06, "pricing_source": "override"}, "event_summary": {"total_events": 839, "errors_by_type": {}, "avg_deliberation_ms": 33005.62, "action_success_rate": 0.7458, "total_tokens": 430265, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.7829, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-12T00:40:17.606653+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "moonshotai/kimi-k2.6", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.638, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 40735, "total_completion_tokens": 75717, "total_reasoning_tokens": 74345, "total_tokens": 190797, "estimated_cost_usd": 0.536003, "wall_time_s": 3574.7, "num_calls": 18, "prompt_price_per_token": 6.7e-07, "completion_price_per_token": 3.39e-06, "pricing_source": "override"}, "event_summary": {"total_events": 1354, "errors_by_type": {"deliberation_timeout": 23, "parse_failure": 29}, "avg_deliberation_ms": 92571.14, "action_success_rate": 0.5556, "total_tokens": 190797, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.638, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-12T01:06:13.410406+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "z-ai/glm-5.1", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.7396, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 178256, "total_completion_tokens": 123515, "total_reasoning_tokens": 81365, "total_tokens": 383136, "estimated_cost_usd": 0.805721, "wall_time_s": 1551.46, "num_calls": 63, "prompt_price_per_token": 9.8e-07, "completion_price_per_token": 3.08e-06, "pricing_source": "override"}, "event_summary": {"total_events": 811, "errors_by_type": {"parse_failure": 6, "deliberation_timeout": 1}, "avg_deliberation_ms": 37912.5, "action_success_rate": 0.7111, "total_tokens": 383136, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.7396, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-12T01:15:39.336819+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "mistralai/mistral-medium-3-5", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": false, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.8039, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 237114, "total_completion_tokens": 49824, "total_reasoning_tokens": 0, "total_tokens": 286938, "estimated_cost_usd": 0.729351, "wall_time_s": 563.67, "num_calls": 70, "prompt_price_per_token": 1.5e-06, "completion_price_per_token": 7.5e-06, "pricing_source": "override"}, "event_summary": {"total_events": 743, "errors_by_type": {}, "avg_deliberation_ms": 7397.01, "action_success_rate": 0.7792, "total_tokens": 286938, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.8039, "outcome": "timeout", "ticks": 10}]} +{"timestamp": "2026-06-12T01:44:54.353193+00:00", "scoring_version": "1.1", "git_commit": "53995be", "git_branch": "feat/obs-capture-polish", "git_dirty": true, "rle_version": "0.3.0", "felix_sdk_version": "0.3.0", "platform": "win32", "python_version": "3.14.0", "docker_mode": false, "random_seed": 42, "rimapi_dll_path": "C:\\Steam\\steamapps\\workshop\\content\\294100\\3593423732\\1.6\\Assemblies\\RIMAPI.dll", "rimapi_dll_sha256": "3a525c9d096828cfed55b17d5ba68a820dbe269dfb65eed2d0a32b7a8f26f565", "rimapi_fork_commit": "", "scenario": "Crashlanded Survival", "scenario_save_name": "rle_crashlanded_v1", "model": "nvidia/nemotron-3-super-120b-a12b", "provider": "openai", "base_url": "https://openrouter.ai/api/v1", "no_think": true, "parallel": true, "no_agent": false, "no_pause": true, "tick_interval": 30.0, "max_ticks": 10, "outcome": "timeout", "final_score": 0.7934, "ticks_run": 10, "cost_snapshot": {"total_prompt_tokens": 83996, "total_completion_tokens": 29146, "total_reasoning_tokens": 30234, "total_tokens": 143376, "estimated_cost_usd": 0.034281, "wall_time_s": 1752.76, "num_calls": 38, "prompt_price_per_token": 9e-08, "completion_price_per_token": 4.5000000000000003e-07, "pricing_source": "override"}, "event_summary": {"total_events": 878, "errors_by_type": {"parse_failure": 31, "deliberation_timeout": 1}, "avg_deliberation_ms": 27648.29, "action_success_rate": 0.8511, "total_tokens": 143376, "estimated_cost_usd": 0.0}, "run_type": "scenario", "scenarios": [{"name": "Crashlanded Survival", "difficulty": "easy", "score": 0.7934, "outcome": "timeout", "ticks": 10}]} diff --git a/uv.lock b/uv.lock index 22383ec..fc4c818 100644 --- a/uv.lock +++ b/uv.lock @@ -951,7 +951,7 @@ wheels = [ [[package]] name = "rimworld-learning-environment" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "felix-agent-sdk" },