From 2788195ae38e47c2577cc77dc1f79fc0c1110a79 Mon Sep 17 00:00:00 2001 From: sam123ben Date: Sun, 23 Aug 2026 22:35:19 +1000 Subject: [PATCH 1/2] fix(pipeline): make dynamic crews real and no-work runs visible A composer run with Crew=Dynamic resolved no crew, so run_crew returned a successful StageResult and the pipeline reported DONE having built nothing. - select_crew picks a seed crew from the intent/tech-stack keywords, with a configurable default that is never the SRE crew - crew seeds carry tags + routing keywords - an unresolvable crew now fails the stage (ok=False) instead of passing - post_report asserts the run produced output and flags no-work runs - compose UX: states what is missing before Run, keeps a status strip (state, stage, elapsed, run link) in the card, scrolls the live area in - every shipped blueprint is now executed end to end in tests --- crews/backend_crew.yaml | 23 +++ crews/frontend_crew.yaml | 22 +++ crews/sre_crew.yaml | 21 +++ dashboard/src/app/compose/page.tsx | 154 +++++++++++++++--- dashboard/src/components/repo-picker.tsx | 7 +- src/devai/config.py | 3 + src/devai/crews/models.py | 9 + src/devai/pipeline/stages/crew_runner.py | 69 +++++++- src/devai/pipeline/stages/lifecycle.py | 57 ++++++- .../test_all_blueprints_execute.py | 122 ++++++++++++++ tests/unit/test_crews.py | 70 +++++++- tests/unit/test_run_verdict.py | 67 ++++++++ 12 files changed, 592 insertions(+), 32 deletions(-) create mode 100644 tests/integration/test_all_blueprints_execute.py create mode 100644 tests/unit/test_run_verdict.py diff --git a/crews/backend_crew.yaml b/crews/backend_crew.yaml index 13f8f2a7..6dad5e92 100644 --- a/crews/backend_crew.yaml +++ b/crews/backend_crew.yaml @@ -6,6 +6,29 @@ description: > expert reviews for vulnerabilities before the work ships. Members are equipped with terminal, git checkpoints, and web search. +tags: [alm, backend, default] + +# Intent words that make this the right crew when the composer's crew is "Dynamic". +keywords: + - api + - endpoint + - service + - backend + - server + - database + - schema + - sql + - migration + - orm + - queue + - worker + - go + - python + - fastapi + - grpc + - auth + - handler + lead: senior_developer members: diff --git a/crews/frontend_crew.yaml b/crews/frontend_crew.yaml index 48da11f3..37c282bc 100644 --- a/crews/frontend_crew.yaml +++ b/crews/frontend_crew.yaml @@ -6,6 +6,28 @@ description: > the result. Members are equipped with terminal, git checkpoints, and web search so they can build, test, and snapshot their work autonomously. +tags: [alm, frontend] + +# Intent words that make this the right crew when the composer's crew is "Dynamic". +keywords: + - ui + - frontend + - react + - next.js + - nextjs + - component + - page + - css + - tailwind + - dashboard + - button + - form + - style + - layout + - tsx + - design + - responsive + lead: senior_developer members: diff --git a/crews/sre_crew.yaml b/crews/sre_crew.yaml index 9d331041..de6193c5 100644 --- a/crews/sre_crew.yaml +++ b/crews/sre_crew.yaml @@ -8,6 +8,27 @@ description: > side-effects. Read-only investigation — remediation is handled by the blueprint's responder stage, not the crew. +tags: [sre] + +# Intent words that make this the right crew when the composer's crew is "Dynamic". +keywords: + - incident + - alert + - oncall + - kubernetes + - cluster + - pod + - latency + - slo + - outage + - postmortem + - cost + - reliability + - runbook + - rollout + - error budget + - saturation + lead: root_cause_analyst members: diff --git a/dashboard/src/app/compose/page.tsx b/dashboard/src/app/compose/page.tsx index 4abb31c0..41792c3e 100644 --- a/dashboard/src/app/compose/page.tsx +++ b/dashboard/src/app/compose/page.tsx @@ -2,7 +2,7 @@ import { Suspense, useCallback, useEffect, useRef, useState } from "react"; import { useSearchParams } from "next/navigation"; -import { ArrowUpRight, Send, Terminal, Users, Wifi, WifiOff } from "lucide-react"; +import { ArrowUpRight, CircleAlert, Send, Terminal, Users, Wifi, WifiOff } from "lucide-react"; import Link from "next/link"; import { api, @@ -36,6 +36,10 @@ import { Select } from "@/components/ui/select"; * dropped connection backfills missed frames, and the terminal/spinner * are driven off an explicit run-state poll rather than guessing from * a single "post-report" frame. + * DASH-14 — the composer states plainly what it needs before Run is armed, and + * a dispatch never disappears: a status strip (state · stage · elapsed · + * open-run link) stays pinned in the composer card and the live area + * scrolls itself into view. */ const SSE_REPLAY = 50; // frames the server replays on (re-)connect @@ -83,6 +87,12 @@ function ComposeInner() { const [conn, setConn] = useState("idle"); const [error, setError] = useState(null); + // Dispatch feedback: when the run started (for the elapsed clock) and where + // the live area is, so a Run always lands somewhere visible. + const [startedAt, setStartedAt] = useState(null); + const [nowMs, setNowMs] = useState(() => Date.now()); + const liveRef = useRef(null); + // Last server event id seen on this stream — fed back on reconnect so the // server can backfill anything emitted while we were disconnected. const lastEventIdRef = useRef(""); @@ -229,6 +239,14 @@ function ComposeInner() { }; }, [taskId, running]); + // Elapsed clock — only ticks while a run is in flight. + useEffect(() => { + if (!running || startedAt === null) return; + setNowMs(Date.now()); + const id = window.setInterval(() => setNowMs(Date.now()), 1000); + return () => window.clearInterval(id); + }, [running, startedAt]); + const parseContextRefs = useCallback( (text: string): ContextRef[] => { const refs: ContextRef[] = []; @@ -250,19 +268,17 @@ function ComposeInner() { ); async function run() { - if (!intent.trim() || !repo.trim()) { - setError("Intent and repo are required."); - return; - } - if (!blueprint) { - setError("Pick a blueprint to run."); + if (missing.length || !blueprint) { + setError(`Add ${joinList(missing)} to run.`); return; } setError(null); setRunning(true); setRunState("queued"); + setStartedAt(Date.now()); setEvents([]); setCheckpoints([]); + setTaskId(null); lastEventIdRef.current = ""; try { const result = await api.dispatchCompose({ @@ -278,10 +294,17 @@ function ComposeInner() { label: "composer", }); setTaskId(result.task_id); + // Never leave the user staring at the composer wondering if anything + // happened — bring the live terminal to them. + window.setTimeout( + () => liveRef.current?.scrollIntoView({ behavior: "smooth", block: "start" }), + 60, + ); } catch (e) { setError(e instanceof Error ? e.message : String(e)); setRunning(false); setRunState(null); + setStartedAt(null); } } @@ -292,6 +315,18 @@ function ComposeInner() { const selectedCrew = crews.find((c) => c.id === crewId); + // What the composer still needs before Run means anything. Shown up front + // rather than only after a failed click. + const missing = [ + !intent.trim() ? "what you want done" : null, + !repo.trim() ? "a repo" : null, + !blueprint ? "a blueprint" : null, + ].filter((x): x is string => x !== null); + + // Plain-language "where the run is right now", from the newest staged frame. + const currentStage = [...events].reverse().find((e) => e.stage)?.stage ?? null; + const dispatched = running || taskId !== null; + return (
@@ -316,6 +351,15 @@ function ComposeInner() { className="rounded-xl border p-4" style={{ background: "var(--surface)", borderColor: "var(--border-subtle)" }} > +
+ + 1 · What should the crew do? + + + Plain English — e.g. “add a health endpoint and a test for it”. + +
+ @@ -372,19 +416,74 @@ function ComposeInner() {
- +
+ + {missing.length > 0 && !running && ( + + Add {joinList(missing)} to run. + + )} +
+ {/* Dispatch status — stays in the composer card so the run is never + silent, even if the user doesn't scroll to the terminal. */} + {dispatched && ( +
+ {runState ? ( + + ) : ( + Dispatching… + )} + + {taskId + ? currentStage + ? `Working on ${currentStage}` + : running + ? "Crew is starting up…" + : "Run finished" + : "Sending the task to the crew…"} + + {startedAt !== null && ( + + {fmtElapsed((running ? nowMs : Math.max(nowMs, startedAt)) - startedAt)} + + )} + {taskId && ( + <> + + {taskId.slice(0, 8)} + + + Open full run + + + )} +
+ )} + {error && ( -

+

+ {error}

)} @@ -408,7 +507,7 @@ function ComposeInner() { {/* Live work area — only mounts once a run is dispatched, so the page isn't a giant empty terminal void before you hit Run. */} -
+
{/* Left: live terminal while running, friendly placeholder while idle. */}
{taskId ? ( @@ -520,6 +619,23 @@ function ComposeInner() { ); } +/** "a repo and a blueprint" — human list joining for the missing-fields hint. */ +function joinList(items: string[]): string { + if (items.length <= 1) return items[0] ?? ""; + return `${items.slice(0, -1).join(", ")} and ${items[items.length - 1]}`; +} + +/** Elapsed run time as m:ss (or h:mm:ss past an hour). */ +function fmtElapsed(ms: number): string { + const total = Math.max(0, Math.floor(ms / 1000)); + const s = total % 60; + const m = Math.floor(total / 60) % 60; + const h = Math.floor(total / 3600); + const mm = String(m).padStart(2, "0"); + const ss = String(s).padStart(2, "0"); + return h > 0 ? `${h}:${mm}:${ss}` : `${m}:${ss}`; +} + /** * Replay-safe append: the SSE stream replays recent frames on reconnect, so the * same (task_id, timestamp, stage, phase) frame can arrive twice. Drop the diff --git a/dashboard/src/components/repo-picker.tsx b/dashboard/src/components/repo-picker.tsx index 8a484de5..be8acbaa 100644 --- a/dashboard/src/components/repo-picker.tsx +++ b/dashboard/src/components/repo-picker.tsx @@ -64,7 +64,10 @@ export function RepoPicker({ // Close on outside click. useEffect(() => { function onClick(e: MouseEvent) { - if (boxRef.current && !boxRef.current.contains(e.target as Node)) setOpen(false); + if (boxRef.current && !boxRef.current.contains(e.target as Node)) { + setOpen(false); + setQuery(""); // typed-but-unpicked text isn't a selection — don't imply it is + } } document.addEventListener("mousedown", onClick); return () => document.removeEventListener("mousedown", onClick); @@ -136,7 +139,7 @@ export function RepoPicker({ setOpen(false); } }} - placeholder={value ? value : placeholder} + placeholder={value ? `${value} — type to change` : placeholder} className="w-full rounded-md py-1.5 pl-7 pr-2 text-sm outline-none focus:border-[var(--ink-strong)]" style={inputStyle} /> diff --git a/src/devai/config.py b/src/devai/config.py index f90e182d..d861ced4 100644 --- a/src/devai/config.py +++ b/src/devai/config.py @@ -596,6 +596,9 @@ def _strip_auth_bff_shared_secret(cls, value: object) -> object: # --- Crews (AI agent teams; seed catalog) --- crews_dir: str = "crews" + # Crew used when a run picks "Dynamic" and the intent matches no crew's + # keywords. Empty = fall back to a crew tagged `default` in crews/*.yaml. + default_crew: str = "backend_crew" # --- Teams (human teams that own crews) --- teams_enabled: bool = True diff --git a/src/devai/crews/models.py b/src/devai/crews/models.py index 7bdfb3e2..5acc15eb 100644 --- a/src/devai/crews/models.py +++ b/src/devai/crews/models.py @@ -42,6 +42,11 @@ class CrewSpec: lead: str display_name: str = "" description: str = "" + # Routing hints for dynamic crew selection ("Dynamic" in the composer): + # `tags` groups a crew by domain (alm / sre / default), `keywords` are the + # intent words that make this crew the right pick. + tags: list[str] = field(default_factory=list) + keywords: list[str] = field(default_factory=list) def __post_init__(self) -> None: if not self.members: @@ -72,6 +77,8 @@ def from_dict(cls, data: dict[str, Any]) -> CrewSpec: lead=str(data.get("lead", "")), display_name=str(data.get("display_name", "")), description=str(data.get("description", "")), + tags=[str(t) for t in (data.get("tags") or [])], + keywords=[str(k) for k in (data.get("keywords") or [])], ) def to_dict(self) -> dict[str, Any]: @@ -81,6 +88,8 @@ def to_dict(self) -> dict[str, Any]: "description": self.description, "lead": self.lead, "members": [m.to_dict() for m in self.members], + "tags": list(self.tags), + "keywords": list(self.keywords), } diff --git a/src/devai/pipeline/stages/crew_runner.py b/src/devai/pipeline/stages/crew_runner.py index b4bea9d9..2d33adf5 100644 --- a/src/devai/pipeline/stages/crew_runner.py +++ b/src/devai/pipeline/stages/crew_runner.py @@ -49,9 +49,16 @@ def name(self) -> str: async def execute(self, task: DevAITask) -> StageResult: crew = await self._resolve_crew(task) if crew is None: + # A crew-less run used to return a *successful* StageResult, so the + # whole pipeline reported DONE having built nothing. Signal failure + # (ok=False) so the executor honors on_failure and the run says so. + logger.error("run_crew: no crew could be resolved for task %s (repo=%s)", task.id, task.repo) return StageResult( - message="no crew resolved (set task.crew_id, config.crew, or agent_context['crew'])", - data={"crew_error": "no_crew"}, + message=( + "no crew available — no crew assigned to the run, no crew named in the blueprint, " + "and no seed crews found in the crews/ catalog" + ), + data={"ok": False, "crew_error": "no_crew"}, ) from devai.agentruntime.runner import AgentRunner @@ -127,10 +134,16 @@ async def _resolve_crew(self, task: DevAITask): seeds = load_seed_crews(crews_dir) if wanted and wanted in seeds: return seeds[wanted] - # last resort: if exactly one seed exists, use it - if len(seeds) == 1: - return next(iter(seeds.values())) - return None + if wanted: + logger.warning("crew %r not found in the seed catalog — picking dynamically", wanted) + + # 3. dynamic pick — "Dynamic" in the composer promises DevAI chooses a + # crew, so the run must never silently do nothing. + default_crew = str(getattr(self.deps.config, "default_crew", "") or "") + picked = select_crew(task, seeds, default_crew=default_crew) + if picked is not None: + logger.info("run_crew: dynamically selected crew %s for task %s", picked.name, task.id) + return picked def _resolve_member_spec(self, crew, specialization: str): spec = self._spec(specialization) @@ -223,6 +236,48 @@ async def _publish_trail(self, task: DevAITask, trail: list[dict[str, Any]]) -> logger.debug("crew trail publish failed", exc_info=True) +def select_crew(task: DevAITask, seeds: dict[str, Any], *, default_crew: str = "") -> Any | None: + """Pick the crew that best fits a task when none was explicitly chosen. + + Scores each seed crew's `keywords` (and its tags/name) against the task's + intent, repo and detected tech stack; the highest score wins. With no + signal at all it falls back to the configured `default_crew`, then to a + crew tagged `default`, then to any non-SRE crew — an ALM run must not be + handed to the SRE investigation crew just because it sorted first. + """ + if not seeds: + return None + + haystack = " ".join( + str(part).lower() + for part in ( + task.intent, + task.repo, + (task.agent_context or {}).get("tech_stack", ""), + (task.agent_context or {}).get("languages", ""), + ) + if part + ) + + def score(crew: Any) -> int: + terms = [str(k).lower() for k in (getattr(crew, "keywords", None) or [])] + terms += [str(t).lower() for t in (getattr(crew, "tags", None) or []) if t not in {"default", "alm"}] + return sum(1 for term in terms if term and term in haystack) + + ranked = sorted(seeds.values(), key=lambda c: (-score(c), c.name)) + best = ranked[0] + if score(best) > 0: + return best + + if default_crew and default_crew in seeds: + return seeds[default_crew] + tagged_default = next((c for c in ranked if "default" in (getattr(c, "tags", None) or [])), None) + if tagged_default is not None: + return tagged_default + non_sre = next((c for c in ranked if "sre" not in (getattr(c, "tags", None) or [])), None) + return non_sre or ranked[0] + + def _msg(from_agent: str, to_agent: str, kind: str, body: str) -> dict[str, Any]: return { "id": uuid.uuid4().hex[:12], @@ -246,4 +301,4 @@ def crew_runner_stage(deps: StageDeps, config: dict[str, str]) -> PipelineStage: return _CrewRunnerStage(deps, config) -__all__ = ["crew_runner_stage"] +__all__ = ["crew_runner_stage", "select_crew"] diff --git a/src/devai/pipeline/stages/lifecycle.py b/src/devai/pipeline/stages/lifecycle.py index 8be7a372..4d040596 100644 --- a/src/devai/pipeline/stages/lifecycle.py +++ b/src/devai/pipeline/stages/lifecycle.py @@ -852,21 +852,30 @@ def name(self) -> str: async def execute(self, task: DevAITask) -> StageResult: report = self._render(task) + produced_work, no_work_reason = assess_work(task) # Pull preview URLs off the handover bag so the StageResult.data # surfaces them to subscribers (dashboard SSE, task event payload). preview_url = str(task.agent_context.get("preview_url") or "") editor_url = str(task.agent_context.get("editor_url") or "") triggered_by = task.triggered_by or task.agent_context.get("trigger_actor") or "" - result_data = { + result_data: dict[str, Any] = { "report_markdown": report, "preview_url": preview_url, "editor_url": editor_url, "triggered_by": triggered_by, } + # A run where every substantive stage stubbed/skipped used to finish + # DONE with nothing to show for it. Report that honestly instead. + if not produced_work: + logger.error("post_report: run %s produced no work — %s", task.id, no_work_reason) + result_data.update({"ok": False, "run_verdict": "no_work", "no_work_reason": no_work_reason}) if self.target == "none" or self.deps.scm is None: - return StageResult(message="rendered report (not posted)", data=result_data) + return StageResult( + message=no_work_reason if not produced_work else "rendered report (not posted)", + data=result_data, + ) try: # SCMClient's comment surface is add_comment(repo, issue_id, body) @@ -881,7 +890,10 @@ async def execute(self, task: DevAITask) -> StageResult: logger.exception("post_report: failed to post comment") return StageResult(message=f"render ok, post failed: {e}", data=result_data) - return StageResult(message="report posted", data=result_data) + return StageResult( + message=no_work_reason if not produced_work else "report posted", + data=result_data, + ) def _render(self, task: DevAITask) -> str: lines = [f"## {self.title}", "", f"**Run:** `{task.id}`", f"**Blueprint:** `{task.blueprint}`"] @@ -902,6 +914,10 @@ def _render(self, task: DevAITask) -> str: lines.append(f"- Editor: <{editor_url}>") lines.append("") + produced_work, no_work_reason = assess_work(task) + if not produced_work: + lines += ["> ⚠️ **This run produced no work.** " + no_work_reason, ""] + if task.stages_completed: lines.append("### Stages completed") for s in task.stages_completed: @@ -915,11 +931,46 @@ def _render(self, task: DevAITask) -> str: return "\n".join(lines) +# Handover keys that only ever mean "a stage degraded", never "work happened". +_STUB_SUFFIXES = ("_stub", "_error", "_skipped") + + +def assess_work(task: DevAITask) -> tuple[bool, str]: + """Did this run actually produce something, or did every stage degrade? + + A run whose stages all stubbed out (no LLM, no SCM, no crew) used to end + DONE with an empty report. This is the check post_report uses to say so. + Returns ``(produced_work, reason_when_not)``. + """ + if task.pr_number is not None or (task.issue_number or 0) > 0: + return True, "" + context = task.agent_context or {} + if context.get("checkpoints") or context.get("files_changed"): + return True, "" + + for key, value in context.items(): + if key.endswith(_STUB_SUFFIXES) or key in {"report_markdown", "requirements", "context_refs", "attachments"}: + continue + if isinstance(value, dict): + # A stub payload is `{"_stub": True, ...}` — not real output. + if any(str(k).endswith("_stub") for k in value): + continue + if value: + return True, "" + elif isinstance(value, list) and value or isinstance(value, str) and value.strip(): + return True, "" + return ( + False, + "No stage produced output — every agent/crew stage degraded to a stub (check LLM provider, SCM credentials, and crew configuration).", + ) + + def post_report_stage(deps: StageDeps, config: dict[str, str]) -> PipelineStage: return _PostReportStage(deps, config) __all__ = [ + "assess_work", "alm_learn_stage", "plan_approval_stage", "await_merge_stage", diff --git a/tests/integration/test_all_blueprints_execute.py b/tests/integration/test_all_blueprints_execute.py new file mode 100644 index 00000000..0ecfad76 --- /dev/null +++ b/tests/integration/test_all_blueprints_execute.py @@ -0,0 +1,122 @@ +"""Every shipped blueprint, executed end to end against fakes. + +The registry-coverage test proves each blueprint's stage keys *exist*; this +one proves the blueprints actually RUN: the DAG resolves, every stage executes +(degrading, not exploding, when a backend is absent), and the run finishes in +a terminal state with an honest verdict rather than silently reporting DONE +having built nothing. + +Only the outside world is faked — LLM, SCM and the K8s runtime. The executor, +registry, stages and blueprint YAML are the real ones. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from devai.adapters.llm.base import LLMResponse, LLMUsage +from devai.blueprint.executor import BlueprintExecutor +from devai.blueprint.loader import discover_blueprints +from devai.blueprint.registry import StageRegistry, register_defaults +from devai.config import Settings +from devai.pipeline.interfaces import StageDeps +from devai.pipeline.stages.lifecycle import assess_work +from devai.pipeline.types import DevAITask, TaskState + +REPO_ROOT = Path(__file__).resolve().parents[2] +BLUEPRINTS_DIR = REPO_ROOT / "blueprints" + +BLUEPRINTS = sorted(discover_blueprints(BLUEPRINTS_DIR).items()) + +# Blueprints whose work happens in a backend this harness deliberately doesn't +# stand up. They still have to EXECUTE cleanly; only the "produced output" +# assertion is out of reach here and is covered by the live smoke instead. +NEEDS_RUNTIME = { + "app-scaffold": "every stage is run_as_job — needs the K8s job runner", + "sre-monitor": "SRE agents need the cluster + provider clients", +} + + +class _ScriptedLLM: + """Answers every agent turn with a well-formed, generic handover.""" + + provider_name = "scripted" + + async def generate(self, request: Any) -> LLMResponse: # noqa: ANN401 + return LLMResponse( + text='{"summary": "did the work", "findings": [], "assignments": []}', + usage=LLMUsage(prompt_tokens=10, completion_tokens=5), + ) + + async def embed(self, texts: list[str], model: str = "") -> list[list[float]]: + return [[0.0] * 8 for _ in texts] + + +def _deps() -> StageDeps: + settings = Settings( + pipeline_blueprint_dir=str(BLUEPRINTS_DIR), + crews_dir=str(REPO_ROOT / "crews"), + specializations_dir=str(REPO_ROOT / "specializations"), + pipeline_heal_on_failure=False, + ) + return StageDeps(config=settings, llm=_ScriptedLLM()) + + +@pytest.mark.parametrize("name,blueprint", BLUEPRINTS, ids=[n for n, _ in BLUEPRINTS]) +@pytest.mark.asyncio +async def test_blueprint_executes_to_a_terminal_state(name: str, blueprint: Any) -> None: + deps = _deps() + executor = BlueprintExecutor(_registry(), deps) + task = DevAITask( + intent="add a health endpoint and a test for it", + blueprint=name, + repo="tesserix/test-repo", + ) + + result = await executor.execute(blueprint, task) + + # It must not be left mid-flight — a run that never reaches a terminal + # state is exactly the "spinner forever" the dashboard used to show. + assert result.state not in (TaskState.PENDING, TaskState.QUEUED), f"{name}: run never left {result.state}" + # Every stage is accounted for: completed, failed, or explicitly skipped. + accounted = ( + set(result.stages_completed) | set(result.stages_failed) | set(getattr(result, "stages_skipped", []) or []) + ) + declared = {s.name for s in blueprint.stages} + assert declared - accounted == set() or result.state in ( + TaskState.STAGE_FAILED, + TaskState.FAILED, + TaskState.CANCELLED, + ), f"{name}: stages never ran: {sorted(declared - accounted)}" + + +@pytest.mark.parametrize("name,blueprint", BLUEPRINTS, ids=[n for n, _ in BLUEPRINTS]) +@pytest.mark.asyncio +async def test_blueprint_produces_work_when_a_model_is_available(name: str, blueprint: Any) -> None: + """With a working LLM, a run must hand over real output. + + The composer's "run finished DONE but nothing happened" bug was exactly + this: every stage degraded to a stub and the run still reported success. + """ + if name in NEEDS_RUNTIME: + pytest.skip(NEEDS_RUNTIME[name]) + deps = _deps() + task = DevAITask( + intent="add a health endpoint and a test for it", + blueprint=name, + repo="tesserix/test-repo", + ) + + result = await BlueprintExecutor(_registry(), deps).execute(blueprint, task) + + produced, reason = assess_work(result) + assert produced, f"{name}: {reason}" + + +def _registry() -> StageRegistry: + reg = StageRegistry() + register_defaults(reg) + return reg diff --git a/tests/unit/test_crews.py b/tests/unit/test_crews.py index 3408c244..f1e9b524 100644 --- a/tests/unit/test_crews.py +++ b/tests/unit/test_crews.py @@ -15,7 +15,7 @@ from devai.crews.loader import load_seed_crews from devai.crews.models import CrewMember, CrewSpec from devai.pipeline.interfaces import StageDeps -from devai.pipeline.stages.crew_runner import crew_runner_stage +from devai.pipeline.stages.crew_runner import crew_runner_stage, select_crew from devai.pipeline.types import DevAITask from devai.specializations.base import Specialization @@ -152,3 +152,71 @@ async def test_crew_runner_no_crew_resolved(): stage = crew_runner_stage(deps, {}) result = await stage.execute(DevAITask(intent="x")) assert result.data.get("crew_error") == "no_crew" + + +# ── dynamic crew selection ────────────────────────────────────────────── + + +def _real_seeds(): + return load_seed_crews(REPO_ROOT / "crews") + + +@pytest.mark.parametrize( + ("intent", "expected"), + [ + ("add a health endpoint and a test for it", "backend_crew"), + ("make the drawer component match our brand in tailwind", "frontend_crew"), + ("investigate the latency spike on the checkout pods", "sre_crew"), + ], +) +def test_select_crew_routes_by_intent(intent: str, expected: str): + picked = select_crew(DevAITask(intent=intent, repo="tesserix/test-repo"), _real_seeds()) + assert picked.name == expected + + +def test_select_crew_falls_back_to_default_never_sre(): + """A signal-free intent must land on a build crew, not the SRE crew.""" + picked = select_crew(DevAITask(intent="just do the thing"), _real_seeds(), default_crew="backend_crew") + assert picked.name == "backend_crew" + + # With no configured default, the crew tagged `default` wins. + picked = select_crew(DevAITask(intent="just do the thing"), _real_seeds()) + assert "sre" not in picked.tags + + +def test_select_crew_uses_tech_stack_signal(): + task = DevAITask(intent="ship the thing", agent_context={"tech_stack": "react, tailwind"}) + assert select_crew(task, _real_seeds()).name == "frontend_crew" + + +def test_select_crew_with_no_seeds_is_none(): + assert select_crew(DevAITask(intent="x"), {}) is None + + +@pytest.mark.asyncio +async def test_crew_runner_picks_a_crew_dynamically(): + """`Dynamic` in the composer must resolve a real crew, not silently no-op.""" + llm = FakeLLMAdapter([LLMResponse(text='```json\n{"summary": "done"}\n```')] * 6) + deps = StageDeps( + config=None, + llm=llm, + extra={"seed_crews": _real_seeds(), "specialization_registry": FakeSpecRegistry()}, + ) + stage = crew_runner_stage(deps, {}) # no config.crew, no task.crew_id + task = DevAITask(intent="add a REST endpoint to the orders service", repo="tesserix/test-repo") + + result = await stage.execute(task) + + assert result.data["crew_output"]["crew"] == "backend_crew" + assert result.data["crew_output"]["assignments"] + assert result.data.get("ok") is not False + + +@pytest.mark.asyncio +async def test_crew_runner_no_crew_is_a_failure_not_a_silent_pass(): + deps = StageDeps(config=None, llm=None, extra={"seed_crews": {}}) + result = await crew_runner_stage(deps, {}).execute(DevAITask(intent="x")) + # ok=False is what the executor reads to honor `on_failure` (a run that + # built nothing must not report DONE). + assert result.data["ok"] is False + assert result.data["crew_error"] == "no_crew" diff --git a/tests/unit/test_run_verdict.py b/tests/unit/test_run_verdict.py new file mode 100644 index 00000000..6dc65c6d --- /dev/null +++ b/tests/unit/test_run_verdict.py @@ -0,0 +1,67 @@ +"""The post_report run verdict — a run that built nothing must say so. + +Every stage in a misconfigured run (no LLM, no SCM, no crew) degrades to a +stub and returns a *successful* StageResult, so the pipeline used to finish +DONE with an empty report. `assess_work` is the check that catches it, and +post_report turns it into an `ok=False` handover the executor honors. +""" + +from __future__ import annotations + +import pytest + +from devai.pipeline.interfaces import StageDeps +from devai.pipeline.stages.lifecycle import assess_work, post_report_stage +from devai.pipeline.types import DevAITask + + +def test_stub_only_run_produced_no_work(): + task = DevAITask(intent="x", repo="tesserix/test-repo") + task.agent_context = { + "senior_developer_stub": True, + "review_code_error": "no llm", + "crew_output": {"crew_runner_stub": True}, + } + produced, reason = assess_work(task) + assert produced is False + assert "stub" in reason + + +def test_run_with_a_pr_produced_work(): + task = DevAITask(intent="x") + task.pr_number = 12 + assert assess_work(task)[0] is True + + +def test_run_with_agent_output_produced_work(): + task = DevAITask(intent="x") + task.agent_context = {"security_expert_output": {"findings": []}, "review_notes": "looks fine"} + assert assess_work(task)[0] is True + + +@pytest.mark.asyncio +async def test_post_report_flags_a_no_work_run(): + deps = StageDeps(config=None, llm=None, scm=None) + task = DevAITask(intent="add a health endpoint", repo="tesserix/test-repo") + task.agent_context = {"run_crew_stub": True} + task.stages_completed = ["create-issue", "run-crew"] + + result = await post_report_stage(deps, {"target": "none"}).execute(task) + + assert result.data["ok"] is False + assert result.data["run_verdict"] == "no_work" + assert "produced no work" in result.data["report_markdown"] + + +@pytest.mark.asyncio +async def test_post_report_stays_green_when_work_happened(): + deps = StageDeps(config=None, llm=None, scm=None) + task = DevAITask(intent="add a health endpoint", repo="tesserix/test-repo") + task.agent_context = { + "crew_output": {"crew": "backend_crew", "member_outputs": {"senior_developer": {"summary": "added"}}} + } + + result = await post_report_stage(deps, {"target": "none"}).execute(task) + + assert result.data.get("ok") is not False + assert "run_verdict" not in result.data From a9387cbcf2173c4b647d73a10a95585cd04418b6 Mon Sep 17 00:00:00 2001 From: sam123ben Date: Sun, 23 Aug 2026 22:40:17 +1000 Subject: [PATCH 2/2] fix(dashboard): label the run card agent counter --- dashboard/src/components/run-list.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dashboard/src/components/run-list.tsx b/dashboard/src/components/run-list.tsx index 71d5d729..9ff34d96 100644 --- a/dashboard/src/components/run-list.tsx +++ b/dashboard/src/components/run-list.tsx @@ -285,8 +285,12 @@ export function RunList({ > {run.repo} - - {completedAgents}/{agentCount || "?"} + + {agentCount ? `${completedAgents}/${agentCount} agents` : "no agents yet"}
{/* State on its own line; controls on a dedicated wrap-friendly row