diff --git a/.env.example b/.env.example index 5e59cda..8a777f8 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,7 @@ AGENTFIELD_PUBLIC_URL= NODE_ID=pr-af # --- AI / harness config --- +# `aforge` selects the unreleased aforge-v2 CLI through AgentField's harness. PR_AF_PROVIDER=opencode PR_AF_MODEL=openrouter/moonshotai/kimi-k2.5 # Falls back to PR_AF_MODEL when unset @@ -24,6 +25,7 @@ PR_AF_AI_MAX_RETRIES=3 PR_AF_AI_INITIAL_BACKOFF_SECONDS=2.0 PR_AF_AI_MAX_BACKOFF_SECONDS=8.0 PR_AF_OPENCODE_BIN=opencode +PR_AF_AFORGE_BIN=aforge # Optional provider-agnostic harness executable override (leave unset to use provider defaults) # PR_AF_HARNESS_BIN= PR_AF_OPENCODE_SERVER= diff --git a/Dockerfile b/Dockerfile index ea8fc70..fff92fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY pyproject.toml README.md ./ COPY src/ src/ RUN pip install --no-cache-dir --prefix=/install \ - "agentfield>=0.1.84" \ + "agentfield @ git+https://github.com/Agent-Field/agentfield.git@60b0aa76df604575429af60fb0ccffc9aaca7745#subdirectory=sdk/python" \ "hax-sdk>=0.2.4" \ "pydantic>=2.0" \ "httpx>=0.27" \ diff --git a/README.md b/README.md index d88dff2..0b1c2c1 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,16 @@ The key knobs (see `.env.example` for the full list): |-----------------------------|----------------------------------------------------------------| | `OPENROUTER_API_KEY` | LLM provider key (OpenRouter) — required | | `GH_TOKEN` | GitHub token (`repo` scope) for reading PRs and posting reviews | -| `PR_AF_PROVIDER` | Harness provider (default `opencode`) | +| `PR_AF_PROVIDER` | Harness provider (default `opencode`; accepts `aforge`) | +| `PR_AF_AFORGE_BIN` | Path to an aforge-v2 binary (default `aforge`) | +| `PR_AF_HARNESS_BIN` | Provider-agnostic executable override | + +For an unreleased aforge-v2 source benchmark, build `./cmd/aforge`, then run +PR-AF with `PR_AF_PROVIDER=aforge` and either `PR_AF_AFORGE_BIN` (Python node) +or `PR_AF_HARNESS_BIN` (maintained Go node) set to the absolute binary path. +The draft pins the exact AgentField harness commit used for benchmarking; +replace that pin with the released SDK before merging. The current Docker +images do not bundle the unreleased Aforge binary. | `PR_AF_MODEL` | Harness model (default `openrouter/moonshotai/kimi-k2.5`) | | `PR_AF_MAX_COST_USD` | Per-run cost ceiling in USD (default `2.0`) | | `PR_AF_MAX_DURATION_SECONDS`| Per-run wall-clock ceiling in seconds (default `3600`) | diff --git a/agentfield-package.yaml b/agentfield-package.yaml index 88f20b5..36e904c 100644 --- a/agentfield-package.yaml +++ b/agentfield-package.yaml @@ -49,8 +49,10 @@ user_environment: type: secret scope: global - name: PR_AF_PROVIDER - description: Coding-agent harness provider + description: Coding-agent harness provider (aforge | claude-code | codex | gemini | opencode) default: opencode + - name: PR_AF_AFORGE_BIN + description: Optional path to the aforge-v2 binary (defaults to aforge on PATH) - name: PR_AF_MODEL description: Model the harness uses default: openrouter/moonshotai/kimi-k2.5 diff --git a/benchmark/martian-code-review-bench/scripts/README.md b/benchmark/martian-code-review-bench/scripts/README.md index 441df43..ceb360b 100644 --- a/benchmark/martian-code-review-bench/scripts/README.md +++ b/benchmark/martian-code-review-bench/scripts/README.md @@ -7,8 +7,8 @@ when present. Raw per-run transcripts and caches are written to a gitignored | script | what it does | |---|---| -| `run_node.sh` | Launches the local PR-AF runner with the whole pipeline pinned to GLM-5.2 (`openrouter/z-ai/glm-5.2` for both `.harness()` and `.ai()`), registering with the AgentField control plane on `:8080`. | -| `campaign.py` | Runs a blind `depth=deep` review for each problem in `../problems.json` (hardest-first, resumable, one review per repo at a time), LLM-judges findings against the goldens for **recall**, and writes `../scoreboard.{md,jsonl}` + `../results/.json`. | +| `run_node.sh` | Launches the local PR-AF runner with the whole pipeline pinned to GLM-5.2 (`openrouter/z-ai/glm-5.2` for both `.harness()` and `.ai()`), registering with the AgentField control plane on `:8080`. Defaults to the historical OpenCode provider; set `PR_AF_PROVIDER=aforge` for Aforge. | +| `campaign.py` | Runs a blind `depth=deep` review for each problem in `../problems.json` (hardest-first, resumable, one review per repo at a time), LLM-judges findings against the goldens for **recall**, and writes isolated result sets. It also persists wall time, summed agent time, total tokens, cost, and usage coverage from the execution DAG. | | `ensemble.py` | Self-consistency escalation: for every baseline miss, run K extra independent passes, union the findings, re-judge. Run after `campaign.py` prints `[campaign] done`. | | `all_metrics.py` | Golden-only precision/recall/F1 on the posted-comment basis, ranked against every leaderboard tool from the cloned Martian dataset. | | `honest_compare.py` | Honest scoring (Framing C, see `../RESULTS.md`): credits real non-golden bugs, applied uniformly to PR-AF and the leaders (cubic-v2, cubic-dev). | @@ -22,12 +22,29 @@ uv run python benchmark/martian-code-review-bench/scripts/campaign.py # termina uv run python benchmark/martian-code-review-bench/scripts/ensemble.py # optional: miss escalation ``` +Run the same GLM-5.2 workload through Aforge without touching the committed +OpenCode baseline: + +```bash +PR_AF_PROVIDER=aforge AFORGE_BIN=/path/to/aforge \ + bash benchmark/martian-code-review-bench/scripts/run_node.sh + +PR_AF_PROVIDER=aforge CAMPAIGN_VARIANT=aforge-glm52 \ + uv run python benchmark/martian-code-review-bench/scripts/campaign.py +``` + +The variant writes raw caches under `_glm52_bench/aforge-glm52/` and curated +outputs under `benchmark/martian-code-review-bench/variants/aforge-glm52/`. +Never leave `CAMPAIGN_VARIANT` empty for a comparison run: the empty value is +reserved for the historical OpenCode files. + `all_metrics.py` and `honest_compare.py` additionally need Martian's cloned offline dataset. Set `CRBENCH_RESULTS_DIR` to the offline `results/` directory, or set `CRBENCH_JUDGE_FILE` directly for `honest_compare.py`. ## Knobs (env) -`CAMPAIGN_CONCURRENCY` (default 3) · `CAMPAIGN_DEPTH` (deep) · `CAMPAIGN_MAX_COST` +`CAMPAIGN_VARIANT` (isolated result-set name) · `CAMPAIGN_CONCURRENCY` (default 3) +· `CAMPAIGN_DEPTH` (deep) · `CAMPAIGN_MAX_COST` · `CAMPAIGN_MAX_DURATION` · `CAMPAIGN_LIMIT` (cap unsolved problems per invocation) · `CAMPAIGN_FORCE` (comma-ids to re-run) · `ENSEMBLE_PASSES` (default 2). diff --git a/benchmark/martian-code-review-bench/scripts/campaign.py b/benchmark/martian-code-review-bench/scripts/campaign.py index a4db583..dd96afb 100644 --- a/benchmark/martian-code-review-bench/scripts/campaign.py +++ b/benchmark/martian-code-review-bench/scripts/campaign.py @@ -7,16 +7,20 @@ was FOUND). Writes an incremental scoreboard. Resumable: a problem whose raw run already exists in runs/.json is re-judged, not re-run. -Concurrency-limited so we do not thrash the local runner / opencode semaphore. -Cost is no object; quality (recall) is the metric. +Concurrency-limited so we do not thrash the local harness provider. Quality +(recall) is the primary metric; wall time, agent time, tokens, and cost are +captured for speed and efficiency comparisons. """ + from __future__ import annotations import asyncio import json import os +import re import time from pathlib import Path +from urllib.parse import quote import httpx from dotenv import load_dotenv @@ -27,21 +31,27 @@ ROOT = HERE.parents[2] load_dotenv(ROOT / ".env") -CP = "http://localhost:8080" +CP = os.getenv("AGENTFIELD_SERVER", "http://localhost:8080").rstrip("/") OR_KEY = os.environ["OPENROUTER_API_KEY"] JUDGE_MODEL = "anthropic/claude-sonnet-4.6" -REVIEW_MODEL = "openrouter/z-ai/glm-5.2" +REVIEW_MODEL = os.getenv("PR_AF_MODEL", "openrouter/z-ai/glm-5.2") +REVIEW_PROVIDER = os.getenv("PR_AF_PROVIDER", "opencode") + +VARIANT = os.getenv("CAMPAIGN_VARIANT", "").strip() +if VARIANT and not re.fullmatch(r"[a-zA-Z0-9._-]+", VARIANT): + raise ValueError("CAMPAIGN_VARIANT may contain only letters, numbers, dot, underscore, and dash") PROBLEMS = BENCH_DIR / "problems.json" # raw execution details (large) — gitignored scratch under the repo root -RUNS = ROOT / "_glm52_bench" / "runs" +RUNS = ROOT / "_glm52_bench" / VARIANT / "runs" if VARIANT else ROOT / "_glm52_bench" / "runs" RUNS.mkdir(parents=True, exist_ok=True) # Curated, committed benchmark results — organized for future readers. -RESULTS_DIR = BENCH_DIR / "results" +RESULT_SET_DIR = BENCH_DIR / "variants" / VARIANT if VARIANT else BENCH_DIR +RESULTS_DIR = RESULT_SET_DIR / "results" RESULTS_DIR.mkdir(parents=True, exist_ok=True) -SCORE_JSONL = BENCH_DIR / "scoreboard.jsonl" -SCORE_MD = BENCH_DIR / "scoreboard.md" +SCORE_JSONL = RESULT_SET_DIR / "scoreboard.jsonl" +SCORE_MD = RESULT_SET_DIR / "scoreboard.md" REVIEW_CONCURRENCY = int(os.getenv("CAMPAIGN_CONCURRENCY", "3")) DEPTH = os.getenv("CAMPAIGN_DEPTH", "deep") @@ -93,6 +103,59 @@ async def poll(client: httpx.AsyncClient, eid: str, label: str) -> dict: return data +async def collect_workflow_metrics(client: httpx.AsyncClient, details: dict) -> dict: + """Sum persisted usage and active agent time across one execution DAG.""" + execution_ids = {str(details.get("execution_id") or "")} + workflow_id = str(details.get("workflow_id") or "") + if workflow_id: + try: + response = await client.get( + f"{CP}/api/ui/v1/workflows/{quote(workflow_id, safe='')}/dag", + params={"mode": "light"}, + timeout=30, + ) + response.raise_for_status() + execution_ids.update( + str(node.get("execution_id")) + for node in response.json().get("timeline", []) + if node.get("execution_id") + ) + except Exception as exc: # noqa: BLE001 + print(f"[metrics] DAG lookup failed: {exc!r}", flush=True) + + execution_ids.discard("") + semaphore = asyncio.Semaphore(12) + + async def load(execution_id: str) -> dict | None: + async with semaphore: + try: + response = await client.get( + f"{CP}/api/ui/v1/executions/{quote(execution_id, safe='')}/details", + timeout=30, + ) + response.raise_for_status() + return response.json() + except Exception as exc: # noqa: BLE001 + print(f"[metrics] {execution_id} lookup failed: {exc!r}", flush=True) + return None + + nodes = [node for node in await asyncio.gather(*(load(eid) for eid in execution_ids)) if node] + costs = [float(node["cost"]) for node in nodes if isinstance(node.get("cost"), (int, float))] + total_tokens = sum( + int(node.get("total_tokens") or 0) for node in nodes if isinstance(node.get("total_tokens"), (int, float)) + ) + agent_duration_ms = sum( + int(node.get("duration_ms") or 0) for node in nodes if isinstance(node.get("duration_ms"), (int, float)) + ) + return { + "cost_usd": round(sum(costs), 8) if costs else None, + "total_tokens": total_tokens, + "agent_duration_seconds": round(agent_duration_ms / 1000, 3), + "execution_nodes": len(nodes), + "usage_nodes": sum(1 for node in nodes if node.get("cost") is not None or node.get("total_tokens")), + } + + # --------------------------------------------------------------------------- # # Judge: recall of golden comments (severity-agnostic) # --------------------------------------------------------------------------- # @@ -122,9 +185,7 @@ def _findings_blob(findings: list[dict]) -> str: def _goldens_blob(goldens: list[dict]) -> str: - return "\n".join( - f"[G{i}] ({g.get('severity')}) {g.get('comment')}" for i, g in enumerate(goldens) - ) + return "\n".join(f"[G{i}] ({g.get('severity')}) {g.get('comment')}" for i, g in enumerate(goldens)) async def judge(client: httpx.AsyncClient, goldens: list[dict], findings: list[dict]) -> dict: @@ -226,43 +287,85 @@ async def process(client: httpx.AsyncClient, sem: asyncio.Semaphore, prob: dict) print(f"[{pid}] fired {eid}", flush=True) details = await poll(client, eid, pid) except Exception as exc: # noqa: BLE001 - _record({"id": pid, "repo": prob["repo"], "status": "fire_error", - "error": repr(exc)[:300], "difficulty": prob.get("difficulty_score"), - "n_goldens": len(prob["goldens"]), "hits": 0, "recall": 0.0}) + _record( + { + "id": pid, + "repo": prob["repo"], + "status": "fire_error", + "error": repr(exc)[:300], + "difficulty": prob.get("difficulty_score"), + "n_goldens": len(prob["goldens"]), + "hits": 0, + "recall": 0.0, + } + ) return # Cache ONLY successful runs so a transient failure retries next time. if details.get("status") in ("succeeded", "completed") and isinstance(details.get("output_data"), dict): + details["_benchmark_metrics"] = await collect_workflow_metrics(client, details) run_file.write_text(json.dumps(details, default=str)) status = details.get("status") out = details.get("output_data") or {} findings = out.get("findings") or [] if status not in ("succeeded", "completed") or not isinstance(out, dict): - _record({"id": pid, "repo": prob["repo"], "status": f"run_{status}", - "error": (details.get("error_message") or "")[:300], - "difficulty": prob.get("difficulty_score"), "n_goldens": len(prob["goldens"]), - "n_findings": len(findings), "hits": 0, "recall": 0.0}) + _record( + { + "id": pid, + "repo": prob["repo"], + "status": f"run_{status}", + "error": (details.get("error_message") or "")[:300], + "difficulty": prob.get("difficulty_score"), + "n_goldens": len(prob["goldens"]), + "n_findings": len(findings), + "hits": 0, + "recall": 0.0, + } + ) return + metrics = details.get("_benchmark_metrics") or await collect_workflow_metrics(client, details) + # 2. judge (cheap, no repo lock / sem) try: verdict = await judge(client, prob["goldens"], findings) except Exception as exc: # noqa: BLE001 - _record({"id": pid, "repo": prob["repo"], "status": "judge_error", - "error": repr(exc)[:300], "difficulty": prob.get("difficulty_score"), - "n_goldens": len(prob["goldens"]), "n_findings": len(findings), - "hits": 0, "recall": 0.0}) + _record( + { + "id": pid, + "repo": prob["repo"], + "status": "judge_error", + "error": repr(exc)[:300], + "difficulty": prob.get("difficulty_score"), + "n_goldens": len(prob["goldens"]), + "n_findings": len(findings), + "hits": 0, + "recall": 0.0, + } + ) return rec = { - "id": pid, "repo": prob["repo"], "status": "scored", + "id": pid, + "repo": prob["repo"], + "status": "scored", "difficulty": prob.get("difficulty_score"), - "n_goldens": verdict["total"], "n_findings": len(findings), - "hits": verdict["hits"], "recall": verdict["recall"], - "missed": [prob["goldens"][m["golden_idx"]]["comment"][:120] - for m in verdict["matches"] if not m.get("hit") - and 0 <= m.get("golden_idx", -1) < len(prob["goldens"])], - "cost_usd": (out.get("summary") or {}).get("cost_usd"), + "n_goldens": verdict["total"], + "n_findings": len(findings), + "hits": verdict["hits"], + "recall": verdict["recall"], + "missed": [ + prob["goldens"][m["golden_idx"]]["comment"][:120] + for m in verdict["matches"] + if not m.get("hit") and 0 <= m.get("golden_idx", -1) < len(prob["goldens"]) + ], + "cost_usd": metrics.get("cost_usd") + if metrics.get("cost_usd") is not None + else (out.get("summary") or {}).get("cost_usd"), + "total_tokens": metrics.get("total_tokens"), + "agent_duration_s": metrics.get("agent_duration_seconds"), + "execution_nodes": metrics.get("execution_nodes"), + "usage_nodes": metrics.get("usage_nodes"), "duration_s": (out.get("summary") or {}).get("duration_seconds"), } @@ -286,14 +389,16 @@ async def process(client: httpx.AsyncClient, sem: asyncio.Semaphore, prob: dict) "n_goldens": verdict["total"], "duration_seconds": rec["duration_s"], "cost_usd": rec["cost_usd"], + "total_tokens": rec["total_tokens"], + "agent_duration_seconds": rec["agent_duration_s"], + "execution_nodes": rec["execution_nodes"], + "usage_nodes": rec["usage_nodes"], "goldens": prob["goldens"], "golden_verdicts": verdict["matches"], # EXACT, untrimmed findings — every field, full bodies/evidence/suggestions. "findings": findings, } - (RESULTS_DIR / f"{pid.replace('/', '_')}.json").write_text( - json.dumps(result_doc, indent=2, default=str) - ) + (RESULTS_DIR / f"{pid.replace('/', '_')}.json").write_text(json.dumps(result_doc, indent=2, default=str)) _record(rec) print(f"[{pid}] SCORED recall={rec['recall']} ({rec['hits']}/{rec['n_goldens']})", flush=True) @@ -319,22 +424,29 @@ def _render() -> None: micro = round(tot_h / tot_g, 3) if tot_g else 0.0 macro = round(sum(r["recall"] for r in scored) / len(scored), 3) if scored else 0.0 lines = [ - "# GLM-5.2 + PR-AF — Martian Code-Review-Bench scoreboard", + f"# GLM-5.2 + PR-AF ({REVIEW_PROVIDER}) — Martian Code-Review-Bench scoreboard", "", f"Scored {len(scored)} problems · micro-recall {tot_h}/{tot_g} = **{micro}** · " f"macro-recall **{macro}** · judge={JUDGE_MODEL} · severity-agnostic (HIT = bug found)", "", - "| problem | repo | diff | goldens | hits | recall | findings | missed |", - "|---|---|---|---|---|---|---|---|", + "| problem | repo | diff | goldens | hits | recall | findings | wall | agent time | tokens | cost | missed |", + "|---|---|---|---|---|---|---|---|---|---|---|---|", ] for r in sorted(rows, key=lambda x: -(x.get("difficulty") or 0)): if r.get("status") != "scored": - lines.append(f"| {r['id']} | {r['repo']} | {r.get('difficulty')} | " - f"— | — | _{r.get('status')}_ | — | {r.get('error','')[:60]} |") + lines.append( + f"| {r['id']} | {r['repo']} | {r.get('difficulty')} | " + f"— | — | _{r.get('status')}_ | — | — | — | — | — | {r.get('error', '')[:60]} |" + ) continue missed = "; ".join(r.get("missed", []))[:80] or "—" - lines.append(f"| {r['id']} | {r['repo']} | {r.get('difficulty')} | {r['n_goldens']} | " - f"{r['hits']} | **{r['recall']}** | {r.get('n_findings')} | {missed} |") + lines.append( + f"| {r['id']} | {r['repo']} | {r.get('difficulty')} | {r['n_goldens']} | " + f"{r['hits']} | **{r['recall']}** | {r.get('n_findings')} | " + f"{r.get('duration_s') or '—'} | {r.get('agent_duration_s') or '—'} | " + f"{r.get('total_tokens') or '—'} | " + f"{r.get('cost_usd') if r.get('cost_usd') is not None else '—'} | {missed} |" + ) SCORE_MD.write_text("\n".join(lines) + "\n") @@ -349,8 +461,12 @@ async def main() -> None: limit = int(os.getenv("CAMPAIGN_LIMIT", "0")) if limit > 0: todo = todo[:limit] - print(f"[campaign] {len(problems)} runnable, {len(done)} already scored, " - f"running {len(todo)} this batch; concurrency={REVIEW_CONCURRENCY} depth={DEPTH}", flush=True) + print( + f"[campaign] {len(problems)} runnable, {len(done)} already scored, " + f"running {len(todo)} this batch; provider={REVIEW_PROVIDER} model={REVIEW_MODEL} " + f"variant={VARIANT or 'historical-default'} concurrency={REVIEW_CONCURRENCY} depth={DEPTH}", + flush=True, + ) sem = asyncio.Semaphore(REVIEW_CONCURRENCY) async with httpx.AsyncClient() as client: await asyncio.gather(*(process(client, sem, p) for p in todo)) diff --git a/benchmark/martian-code-review-bench/scripts/run_node.sh b/benchmark/martian-code-review-bench/scripts/run_node.sh index e9a69ac..eaf5c98 100755 --- a/benchmark/martian-code-review-bench/scripts/run_node.sh +++ b/benchmark/martian-code-review-bench/scripts/run_node.sh @@ -18,11 +18,19 @@ export AGENTFIELD_SERVER="${AGENTFIELD_SERVER:-http://localhost:8080}" export AGENT_CALLBACK_URL="${AGENT_CALLBACK_URL:-http://127.0.0.1:8004}" # --- the experiment: GLM-5.2 everywhere --- -export PR_AF_PROVIDER=opencode -export PR_AF_MODEL=openrouter/z-ai/glm-5.2 # .harness() -> opencode -m +# Keep opencode as the historical default. Set PR_AF_PROVIDER=aforge to run the +# exact same campaign through Aforge without editing this script. +export PR_AF_PROVIDER="${PR_AF_PROVIDER:-opencode}" +export PR_AF_MODEL="${PR_AF_MODEL:-openrouter/z-ai/glm-5.2}" # .harness() export PR_AF_AI_MODEL=openrouter/z-ai/glm-5.2 # .ai() -> litellm (needs openrouter/ prefix too) export PR_AF_MAX_TURNS=60 +if [ "$PR_AF_PROVIDER" = "aforge" ]; then + export AFORGE_BIN="${AFORGE_BIN:-aforge}" + export AFORGE_MODEL="${AFORGE_MODEL:-z-ai/glm-5.2}" + export AFORGE_MAX_CONCURRENT="${AFORGE_MAX_CONCURRENT:-24}" +fi + # Generous budget for the hardest Martian-bench PR (keycloak/keycloak#32918). export PR_AF_MAX_COST_USD=8.0 export PR_AF_MAX_DURATION_SECONDS=2400 @@ -47,6 +55,6 @@ fi # Force HITL OFF (no human approval gate) — we want a direct dry-run. unset HAX_API_KEY || true -echo "[run_node] PR_AF_MODEL=$PR_AF_MODEL PR_AF_AI_MODEL=$PR_AF_AI_MODEL" +echo "[run_node] PR_AF_PROVIDER=$PR_AF_PROVIDER PR_AF_MODEL=$PR_AF_MODEL PR_AF_AI_MODEL=$PR_AF_AI_MODEL" echo "[run_node] server=$AGENTFIELD_SERVER budget=\$$PR_AF_MAX_COST_USD / ${PR_AF_MAX_DURATION_SECONDS}s" exec uv run python main.py diff --git a/go/README.md b/go/README.md index 854716d..87d8fc9 100644 --- a/go/README.md +++ b/go/README.md @@ -155,7 +155,15 @@ The node is configured entirely through the environment. | `AGENTFIELD_API_KEY` | Control-plane API key (if the CP has auth enabled) | | `NODE_ID` | Node ID (default `pr-af`) | | `PORT` | Listen port (default `8007`) | -| `PR_AF_PROVIDER` | Harness provider (default `opencode`) | +| `PR_AF_PROVIDER` | Harness provider (default `opencode`; accepts `aforge`) | +| `PR_AF_HARNESS_BIN` | Provider executable override (set to the aforge-v2 binary) | + +To benchmark unreleased aforge-v2 without changing PR-AF prompts, build +`aforge-v2/cmd/aforge`, then set `PR_AF_PROVIDER=aforge` and +`PR_AF_HARNESS_BIN=/absolute/path/to/aforge`. The draft pins the exact +AgentField Go SDK commit used for benchmarking; replace that pin with the +released SDK before merging. The current Docker image does not bundle the +unreleased Aforge binary. | `PR_AF_MODEL` | Harness model (default `openrouter/moonshotai/kimi-k2.5`) | | `PR_AF_LABEL` | Pull-request label that triggers a webhook review (default `pr-af`) | | `PR_AF_MAX_CONCURRENT_REVIEWERS` | Optional webhook review concurrency cap (minimum `1`) | diff --git a/go/agentfield-package.yaml b/go/agentfield-package.yaml index ed0b8de..abcadd2 100644 --- a/go/agentfield-package.yaml +++ b/go/agentfield-package.yaml @@ -41,7 +41,7 @@ user_environment: type: secret scope: global - name: PR_AF_PROVIDER - description: harness provider + description: harness provider (aforge | claude-code | codex | gemini | opencode) default: opencode - name: PR_AF_MODEL description: harness model @@ -56,7 +56,7 @@ user_environment: - name: PR_AF_MAX_COVERAGE_ITERATIONS description: optional coverage iteration cap for webhook-triggered reviews (minimum 1) - name: PR_AF_HARNESS_BIN - description: optional executable override for every harness provider (unset uses provider defaults) + description: optional executable override for every harness provider (set to an aforge-v2 binary with PR_AF_PROVIDER=aforge) - name: PR_AF_MAX_COST_USD description: per-run cost ceiling default: "2.0" diff --git a/go/cmd/pr-af/main.go b/go/cmd/pr-af/main.go index f350cf2..7fb8cec 100644 --- a/go/cmd/pr-af/main.go +++ b/go/cmd/pr-af/main.go @@ -13,7 +13,7 @@ // AGENT_CALLBACK_URL base URL the CP uses to reach this node (else localhost) // NODE_ID node id (default pr-af) // PORT listen port (default 8007) -// PR_AF_PROVIDER harness provider (default opencode) +// PR_AF_PROVIDER harness provider (default opencode; accepts aforge) // PR_AF_MODEL harness model (env wins over the code default) // PR_AF_HARNESS_BIN optional executable override for every harness provider // OPENROUTER_API_KEY LLM key — required for the .ai() gates; AIConfig is only diff --git a/go/go.mod b/go/go.mod index 339de0a..8ea39a2 100644 --- a/go/go.mod +++ b/go/go.mod @@ -5,7 +5,7 @@ module github.com/Agent-Field/pr-af/go go 1.21 require ( - github.com/Agent-Field/agentfield/sdk/go v0.0.0-20260714191100-2cc5fe2adcf4 + github.com/Agent-Field/agentfield/sdk/go v0.1.127-rc.5.0.20260810172027-60b0aa76df60 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/invopop/jsonschema v0.13.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 diff --git a/go/go.sum b/go/go.sum index 92df3d9..8451abf 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,5 +1,5 @@ -github.com/Agent-Field/agentfield/sdk/go v0.0.0-20260714191100-2cc5fe2adcf4 h1:B3uCMLZSa2rsRDRGuecBIXCgkYqBuScSK4CXKPDaCgk= -github.com/Agent-Field/agentfield/sdk/go v0.0.0-20260714191100-2cc5fe2adcf4/go.mod h1:08VZk14uw4GJH6a34psHkuLu+DcRr197Zi0IGmLlfrM= +github.com/Agent-Field/agentfield/sdk/go v0.1.127-rc.5.0.20260810172027-60b0aa76df60 h1:2D86ZSTChMaAJNk44xBIp4skutajkSwvlsa2zugzipc= +github.com/Agent-Field/agentfield/sdk/go v0.1.127-rc.5.0.20260810172027-60b0aa76df60/go.mod h1:08VZk14uw4GJH6a34psHkuLu+DcRr197Zi0IGmLlfrM= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= diff --git a/go/internal/harnessx/run.go b/go/internal/harnessx/run.go index cf0397b..fd8c911 100644 --- a/go/internal/harnessx/run.go +++ b/go/internal/harnessx/run.go @@ -80,7 +80,7 @@ func seedDefaults[T any]() T { // router.harness (system_prompt, schema, model, provider, tools, cwd, max_turns, // permission_mode). type RoleOptions struct { - // Provider is the harness ADAPTER string, e.g. "opencode" (PR-AF's default), + // Provider is the harness ADAPTER string, e.g. "aforge" or "opencode" (PR-AF's default), // "claude-code", "codex". Provider string diff --git a/go/internal/node/node_test.go b/go/internal/node/node_test.go index 0b3fe4c..9011475 100644 --- a/go/internal/node/node_test.go +++ b/go/internal/node/node_test.go @@ -16,6 +16,8 @@ func TestHarnessConfigProviderAwareBinary(t *testing.T) { }{ {"codex uses SDK default", config.AIIntegrationConfig{Provider: "codex", OpencodeBin: "C:/bin/opencode-custom"}, ""}, {"opencode uses configured binary", config.AIIntegrationConfig{Provider: "opencode", OpencodeBin: "C:/bin/opencode-custom"}, "C:/bin/opencode-custom"}, + {"aforge uses SDK default", config.AIIntegrationConfig{Provider: "aforge", OpencodeBin: "C:/bin/opencode-custom"}, ""}, + {"generic override selects aforge", config.AIIntegrationConfig{Provider: "aforge", HarnessBin: "C:/bin/aforge-custom"}, "C:/bin/aforge-custom"}, {"generic override wins", config.AIIntegrationConfig{Provider: "codex", OpencodeBin: "C:/bin/opencode-custom", HarnessBin: "C:/bin/provider-custom"}, "C:/bin/provider-custom"}, {"generic override wins for opencode", config.AIIntegrationConfig{Provider: "opencode", OpencodeBin: "C:/bin/opencode-custom", HarnessBin: "C:/bin/provider-custom"}, "C:/bin/provider-custom"}, } diff --git a/pyproject.toml b/pyproject.toml index d9c5449..5250488 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ license = "Apache-2.0" requires-python = ">=3.11" authors = [{ name = "AgentField", email = "hello@agentfield.dev" }] dependencies = [ - "agentfield>=0.1.84", + "agentfield @ git+https://github.com/Agent-Field/agentfield.git@60b0aa76df604575429af60fb0ccffc9aaca7745#subdirectory=sdk/python", "hax-sdk>=0.2.4", "pydantic>=2.0", "httpx>=0.27", @@ -36,6 +36,9 @@ pr-af = "pr_af.app:main" [tool.hatch.build.targets.wheel] packages = ["src/pr_af"] +[tool.hatch.metadata] +allow-direct-references = true + [tool.ruff] target-version = "py311" line-length = 120 diff --git a/src/pr_af/app.py b/src/pr_af/app.py index 9c22bb5..778a474 100644 --- a/src/pr_af/app.py +++ b/src/pr_af/app.py @@ -42,7 +42,8 @@ model=_ai_config.harness_model, max_turns=_ai_config.max_turns, env=_ai_config.provider_env(), - opencode_bin=_ai_config.opencode_bin, + opencode_bin=_ai_config.harness_bin or _ai_config.opencode_bin, + aforge_bin=_ai_config.harness_bin or _ai_config.aforge_bin, permission_mode="auto", ), ai_config=AIConfig( diff --git a/src/pr_af/config.py b/src/pr_af/config.py index b923d45..1dabe82 100644 --- a/src/pr_af/config.py +++ b/src/pr_af/config.py @@ -313,6 +313,13 @@ class AIIntegrationConfig(BaseModel): ) max_backoff_seconds: float = Field(default_factory=lambda: float(os.getenv("PR_AF_AI_MAX_BACKOFF_SECONDS", "8.0"))) opencode_bin: str = Field(default_factory=lambda: os.getenv("PR_AF_OPENCODE_BIN", "opencode")) + aforge_bin: str = Field( + default_factory=lambda: os.getenv( + "PR_AF_AFORGE_BIN", + os.getenv("AFORGE_BIN", "aforge"), + ) + ) + harness_bin: str = Field(default_factory=lambda: os.getenv("PR_AF_HARNESS_BIN", "")) opencode_server: str | None = Field(default_factory=lambda: os.getenv("PR_AF_OPENCODE_SERVER")) @classmethod diff --git a/tests/test_aforge_config.py b/tests/test_aforge_config.py new file mode 100644 index 0000000..6918c26 --- /dev/null +++ b/tests/test_aforge_config.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from pr_af.config import AIIntegrationConfig + + +def test_aforge_provider_and_binary_overrides(monkeypatch) -> None: + monkeypatch.setenv("PR_AF_PROVIDER", "aforge") + monkeypatch.setenv("PR_AF_AFORGE_BIN", "/opt/aforge") + + config = AIIntegrationConfig.from_env() + + assert config.provider == "aforge" + assert config.aforge_bin == "/opt/aforge" + assert config.harness_bin == "" + + +def test_generic_harness_binary_is_available_to_all_providers(monkeypatch) -> None: + monkeypatch.setenv("PR_AF_HARNESS_BIN", "/opt/harness") + + config = AIIntegrationConfig.from_env() + + assert config.harness_bin == "/opt/harness"