From 337a8bfc5039402da4d73687065d18f7ebec9beb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 5 Jul 2026 23:41:30 +0900 Subject: [PATCH] fix(review): put contract-reliable reasoning models first, flagships last MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ordering the highest-quota non-reasoning models first (deepseek-v3, mistral, llama-4) surfaced that they emit reviews the publish/approve gates reject — bare APPROVE summaries missing required labels, then hallucinated non-source-backed REQUEST_CHANGES findings — and the pool records them as success instead of falling through. The mini reasoning models (o4-mini, o3-mini, gpt-5-mini/nano/ chat) reliably emit the strict contract and have adequate quota, so lead with them; keep deepseek/mistral/llama as fallbacks and the rate-starved flagships o3/gpt-5 last (first-placing gpt-5 stalled every review to timeout). Matches the pre-#295 known-good lead model (o4-mini). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RTAMs4bpSZS77Xe3RQjv9P --- .github/workflows/opencode-review.yml | 18 ++++++++++-------- tests/test_opencode_agent_contract.py | 16 ++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index ac48956..ee4d4f7 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -2280,14 +2280,16 @@ jobs: SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" - # Ordered by GitHub Models quota allowance, largest first, so the - # highest-throughput models review before the rate-starved flagships. - # gpt-5/o3 are "Reasoning" tier (8-12 req/day) — putting them first - # (as before) stalled every review until the step timed out, because - # a rate-limited/hung flagship never fell back. Non-reasoning "Low" - # tier (deepseek-v3, mistral, llama-4: 150-450 req/day) and mini - # reasoning models go first; gpt-5 stays last as a quality fallback. - OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 github-models/mistral-ai/mistral-medium-2505 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-nano github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/openai/o3 github-models/openai/gpt-5" + # Ordered contract-reliability first, then quota, with the rate-starved + # flagships last. The mini reasoning models (o4-mini, o3-mini, gpt-5- + # mini/nano/chat) reliably emit the strict review contract — every + # required label and only source-backed findings — so they lead. The + # high-quota non-reasoning models (deepseek-v3, mistral, llama-4) emit + # bare or hallucinated reviews the publish/approve gates reject, so + # they are fallbacks only. gpt-5/o3 ("Reasoning" tier, 8-12 req/day) + # stay last: first-placing them stalled every review until timeout + # because a rate-limited/hung flagship never fell back. + OPENCODE_MODEL_CANDIDATES: "github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-nano github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/deepseek/deepseek-v3-0324 github-models/mistral-ai/mistral-medium-2505 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct github-models/openai/o3 github-models/openai/gpt-5" OPENCODE_MODEL_ATTEMPTS: "5" OPENCODE_RUN_TIMEOUT_SECONDS: "20400" OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 8493dc7..e243409 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -79,9 +79,9 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_models assert set(candidate_models).issubset(set(models)) assert candidate_models[:3] == [ - "deepseek/deepseek-v3-0324", - "mistral-ai/mistral-medium-2505", - "meta/llama-4-maverick-17b-128e-instruct-fp8", + "openai/o4-mini", + "openai/o3-mini", + "openai/gpt-5-mini", ] assert { "openai/gpt-5-chat", @@ -286,17 +286,17 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert 'APPROVAL_CHECK_WAIT_ATTEMPTS: "81"' in workflow assert 'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "30"' in workflow assert ( - 'OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 ' - "github-models/mistral-ai/mistral-medium-2505 " - "github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 " - "github-models/meta/llama-4-scout-17b-16e-instruct " - "github-models/openai/o4-mini " + 'OPENCODE_MODEL_CANDIDATES: "github-models/openai/o4-mini ' "github-models/openai/o3-mini " "github-models/openai/gpt-5-mini " "github-models/openai/gpt-5-nano " 'github-models/openai/gpt-5-chat ' "github-models/deepseek/deepseek-r1-0528 " "github-models/deepseek/deepseek-r1 " + "github-models/deepseek/deepseek-v3-0324 " + "github-models/mistral-ai/mistral-medium-2505 " + "github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 " + "github-models/meta/llama-4-scout-17b-16e-instruct " "github-models/openai/o3 " 'github-models/openai/gpt-5"' ) in workflow