From 9655e8d4a9e3a9eced07424798cacbe2a428a9f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 20 Nov 2026 23:06:37 +0000 Subject: [PATCH] Projects board: failed judge asks the user, remove clears board tags, coordinator never auto-resolves Three fixes from a review of #6602: - A judge that fails or times out leaves the turn `asking` (Needs you), with the reply's last paragraph as `ask`, and never resolves it. The critic fallback applies only when no judge is available. - Removing a member clears its board tags with the `project` tag, in one tag-locked update, so no stale chip stays. - The coordinator's turns are recorded but never judged or resolved. Co-authored-by: openhands Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ArwwJH94xGSYwfoyccT4E --- .../references/feature-map/F36-projects.md | 9 ++- .../openhands/agent_server/project_board.py | 68 +++++++++++++------ .../openhands/agent_server/project_service.py | 47 +++++++++---- .../openhands/agent_server/projects_router.py | 11 +-- tests/agent_server/test_project_board.py | 64 +++++++++++++++++ tests/agent_server/test_projects_router.py | 4 +- 6 files changed, 155 insertions(+), 48 deletions(-) diff --git a/.agents/skills/verify-agent-server/references/feature-map/F36-projects.md b/.agents/skills/verify-agent-server/references/feature-map/F36-projects.md index d99a536dc3..5d4853aa80 100644 --- a/.agents/skills/verify-agent-server/references/feature-map/F36-projects.md +++ b/.agents/skills/verify-agent-server/references/feature-map/F36-projects.md @@ -65,7 +65,7 @@ Routes: `GET /api/projects/{project_id}/artifacts`, - `F36.start-member`: only the coordinator starts conversations in its project: its `POST .../children` starts a member tagged `project=`, and a member's own `POST .../children` is 409. - `F36.client-tags`: a client cannot start or fork a conversation whose `project` tag names an existing project: 422, and nothing is created. The other project keys (`turn`, `ask`, `resolved`, `resolvenote`), or a `project` tag naming no project, are ordinary client tags. - `F36.attention`: conversations are grouped `needs_you` (waiting for confirmation), then `working`, `idle` (finished or never run), newest-updated first within a group; non-members are left out. -- `F36.remove-conversation`: `DELETE .../conversations/{conversation_id}` clears only the `project` tag; a second delete, or a delete through another project, is 404. +- `F36.remove-conversation`: `DELETE .../conversations/{conversation_id}` clears the `project` tag and the board tags (`resolved`, `resolvenote`, `turn`, `ask`), and no other tag; a second delete, or a delete through another project, is 404. - `F36.remove-coordinator`: removing the coordinator from the project is 409. - `F36.message-rest`: `POST /api/conversations/{id}/events` saves the message and starts a run. Without `sender_conversation_id` the message has no `sender`; a member sending to a conversation outside its project is 422. - `F36.turn`: when a member's run finishes, its `turn` tag says whether the agent's last message asked something (`asking`, attention `asking`) or not (`finished`), and an `asking` turn saves the question, in one line of at most 200 characters, as the `ask` tag (reported as `ask`); a new user message clears both. @@ -115,8 +115,11 @@ Routes: `GET /api/projects/{project_id}/artifacts`, substantive question, choice, review or check (>= 0.5) makes the turn `asking` (Needs you), with the reply's last paragraph as `ask`; work left that waits on no one stays `finished` and unresolved (Idle). There are no - text heuristics: without a verdict (no judge, or it failed) a turn is - `finished`, and a critic score >= 0.9 still resolves (`0.95 critic`). + text heuristics: without a judge a turn is `finished`, and a critic score + >= 0.9 still resolves (`0.95 critic`). When the judge fails or times out, + the turn is `asking` (Needs you) with the reply's last paragraph as `ask`, + and nothing resolves it. The coordinator's turns get `turn` (and `ask` from + an `ask_user` card) but are never judged or resolved automatically. The board shows the note as `resolve_note`. Clients resolve with `POST /api/projects/{id}/conversations/{cid}/resolve` and reopen with `.../reopen` (both 204, idempotent, and both clear diff --git a/openhands-agent-server/openhands/agent_server/project_board.py b/openhands-agent-server/openhands/agent_server/project_board.py index d92a1bfd8d..31d2c7dc63 100644 --- a/openhands-agent-server/openhands/agent_server/project_board.py +++ b/openhands-agent-server/openhands/agent_server/project_board.py @@ -7,10 +7,14 @@ when a turn leaves nothing for the user to do, marks it resolved (``resolved=auto``). Two signals can resolve a turn: the resolution judge (:mod:`openhands.sdk.conversation.resolution`), which asks whether anything is -left for the user, and a confident critic score. A turn that asked the user -something is never resolved automatically. Only the judge says a reply -waits on the user (or a question card, by construction); the end of that -reply is saved, in one short line, as ``ask``. A new +left for the user, and, only when no judge is available, a confident critic +score. A turn that asked the user something is never resolved automatically. +Only the judge says a reply waits on the user (or a question card, by +construction); the end of that reply is saved, in one short line, as ``ask``. +When the judge fails or times out, nobody has read the reply, so it waits on +the user too, and is never resolved. The coordinator's turns are recorded +but never judged or resolved: the coordinator is how the user steers the +project, so it stays open. A new user message removes these tags, since the conversation is open again. It also tells the coordinator what it needs to hear: finished turns, stops on the user, the user's own messages and answers to cards. @@ -89,7 +93,7 @@ ResolutionJudge = Callable[[ResolutionState], Awaitable[ResolutionVerdict | None]] """Judges whether a finished turn leaves anything for the user; ``None`` -when no judge model is available or the call failed.""" +when no judge model is available. Raises when the call fails.""" TurnSenderLookup = Callable[[], Awaitable[str | None]] """Reads who started the current turn from the event log, for a subscriber @@ -103,6 +107,9 @@ its one check: otherwise the board is left as it is (no outcome, judge or reopen) and the coordinator hears nothing.""" +IsCoordinator = Callable[[], Awaitable[bool]] +"""Whether the conversation is its project's coordinator right now.""" + @dataclass class ConversationOutcomeSubscriber(Subscriber[Event]): @@ -125,6 +132,7 @@ class ConversationOutcomeSubscriber(Subscriber[Event]): turn_sender_lookup: TurnSenderLookup | None = None judge: ResolutionJudge | None = None is_member: IsMember | None = None + is_coordinator: IsCoordinator | None = None tracker: TurnTracker = field(default_factory=TurnTracker) _tail: asyncio.Task | None = field(default=None, init=False) _notify_tail: asyncio.Task | None = field(default=None, init=False) @@ -249,7 +257,14 @@ async def _record(self, project: str, ended: TurnEnded) -> None: await self._set_turn(turn, ask) self._notify_finished(project, turn, ended) return + if await self._coordinator(): + # The coordinator's turns show as they ended; nothing judges or + # resolves the conversation the user steers the project from. + await self._set_turn(turn, ask) + self._notify_finished(project, turn, ended) + return verdict = None + judge_failed = False shown = False if ended.judged is not None and RESOLVED_TAG_KEY not in self.tags(): # The judge can take a while; the board shows the turn as @@ -261,9 +276,28 @@ async def _record(self, project: str, ended: TurnEnded) -> None: await asyncio.wait({judging}) self._judging = None # Cancelled by a new user message: keep it finished. - verdict = None if judging.cancelled() else judging.result() + if not judging.cancelled(): + error = judging.exception() + if error is None: + verdict = judging.result() + else: + judge_failed = True + # The error goes in the message too: some log handlers + # drop tracebacks. + logger.warning( + "Resolution judge failed for conversation %s: %s: %s", + self.conversation_id, + type(error).__name__, + error, + exc_info=error, + ) shown_turn = (turn, ask) - if verdict is not None: + if judge_failed: + # Nobody read the reply: it may hold a question, so the user + # looks at it, and nothing resolves it. + turn = "asking" + ask = ask or ask_line(text) + elif verdict is not None: # The judge alone decides whether the reply waits on the user. # A question for the user goes on the board in a turn the # coordinator started too: the coordinator gets the reply, and @@ -288,7 +322,8 @@ async def _record(self, project: str, ended: TurnEnded) -> None: if verdict is not None and verdict.resolved: note = verdict.note() elif verdict is None and score is not None and score >= AUTO_RESOLVE_THRESHOLD: - # Without a judge, a confident critic still resolves. + # Without a judge, a confident critic still resolves. (A failed + # judge left the turn asking, above.) note = f"{score:.2f} critic" if verdict is not None and note is None: logger.debug( @@ -311,19 +346,10 @@ async def _set_turn(self, turn: TurnOutcome, ask: str | None) -> None: async def _judge(self, state: ResolutionState) -> ResolutionVerdict | None: if self.judge is None: return None - try: - return await self.judge(state) - except Exception as e: - # The error goes in the message too: some log handlers drop - # tracebacks. - logger.warning( - "Resolution judge failed for conversation %s: %s: %s", - self.conversation_id, - type(e).__name__, - e, - exc_info=True, - ) - return None + return await self.judge(state) + + async def _coordinator(self) -> bool: + return self.is_coordinator is not None and await self.is_coordinator() def _notify_finished( self, project: str, turn: TurnOutcome, ended: TurnEnded diff --git a/openhands-agent-server/openhands/agent_server/project_service.py b/openhands-agent-server/openhands/agent_server/project_service.py index 1a0b415035..588165315a 100644 --- a/openhands-agent-server/openhands/agent_server/project_service.py +++ b/openhands-agent-server/openhands/agent_server/project_service.py @@ -329,6 +329,7 @@ async def watch(self, event_service: EventService) -> None: on_user_answer=functools.partial(pass_on_answer, service, conversation_id), turn_sender_lookup=functools.partial(_logged_turn_sender, event_service), is_member=functools.partial(self._is_member, event_service), + is_coordinator=functools.partial(self._is_coordinator, event_service), judge=make_resolution_judge( functools.partial(_agent_llm, event_service), lambda: self.stores, @@ -344,6 +345,13 @@ async def watch(self, event_service: EventService) -> None: async def _is_member(self, event_service: EventService) -> bool: return await self.member_of(event_service.stored.tags) is not None + async def _is_coordinator(self, event_service: EventService) -> bool: + project = await self.member_of(event_service.stored.tags) + return ( + project is not None + and project.coordinator_conversation_id == event_service.stored.id + ) + # Tags def _tag_lock(self, conversation_id: UUID) -> asyncio.Lock: @@ -415,21 +423,30 @@ async def release_members(self, project_hex: str) -> None: PROJECT_TAG_KEY, project_hex ) for stored, _ in members: - async with self._tag_lock(stored.id): - event_service = await self.conversations.get_event_service(stored.id) - if event_service is None: - continue - tags = event_service.stored.tags - if tags.get(PROJECT_TAG_KEY) != project_hex: - continue - await self.conversations.update_conversation( - stored.id, - UpdateConversationRequest( - tags={ - k: v for k, v in tags.items() if k not in RESERVED_TAG_KEYS - } - ), - ) + await self.release_member(stored.id, project_hex) + + async def release_member(self, conversation_id: UUID, project_hex: str) -> bool: + """Take a conversation out of project ``project_hex``: remove the + ``project`` tag and the board's state, so no stale chip stays. Its + other tags stay. + + Returns: + Whether the conversation was a member of that project. + """ + async with self._tag_lock(conversation_id): + event_service = await self.conversations.get_event_service(conversation_id) + if event_service is None: + return False + tags = event_service.stored.tags + if tags.get(PROJECT_TAG_KEY) != project_hex: + return False + await self.conversations.update_conversation( + conversation_id, + UpdateConversationRequest( + tags={k: v for k, v in tags.items() if k not in RESERVED_TAG_KEYS} + ), + ) + return True async def update_conversation_from_client( self, conversation_id: UUID, request: UpdateConversationRequest diff --git a/openhands-agent-server/openhands/agent_server/projects_router.py b/openhands-agent-server/openhands/agent_server/projects_router.py index 1a6dca1594..aee3a699f9 100644 --- a/openhands-agent-server/openhands/agent_server/projects_router.py +++ b/openhands-agent-server/openhands/agent_server/projects_router.py @@ -697,7 +697,8 @@ async def remove_project_conversation( stores: _Stores, service: ConversationService = Depends(get_conversation_service), ) -> DeleteResponse: - """Remove a conversation from a project by clearing its ``project`` tag. + """Remove a conversation from a project: clear its ``project`` tag and + its board state (``resolved``, ``resolvenote``, ``turn``, ``ask``). The conversation itself is not changed otherwise. Workers it started keep their own tag. Clear the coordinator before removing it. @@ -705,13 +706,7 @@ async def remove_project_conversation( project = _find(await _load(_store(stores)), project_id) if project.coordinator_conversation_id == conversation_id: raise Conflict("Conversation is the project's coordinator") - try: - tags = await service.projects.set_conversation_tag( - conversation_id, PROJECT_TAG, None, expected=project_id.hex - ) - except ValueError: - tags = None - if tags is None: + if not await service.projects.release_member(conversation_id, project_id.hex): raise NotFound("Conversation not in project") return DeleteResponse() diff --git a/tests/agent_server/test_project_board.py b/tests/agent_server/test_project_board.py index 0b36c23ba1..7dd00f3c67 100644 --- a/tests/agent_server/test_project_board.py +++ b/tests/agent_server/test_project_board.py @@ -960,6 +960,70 @@ async def test_without_a_verdict_a_confident_critic_still_resolves(): assert store.tags["resolvenote"] == "0.95 critic" +@pytest.mark.asyncio +async def test_a_failed_judge_shows_the_turn_as_needs_you_and_never_resolves(): + """Nobody read the reply, so it may hold a question: the user looks, and + even a confident critic does not resolve it.""" + store = _Tags({"project": "p1"}) + + async def judge(state: ResolutionState) -> ResolutionVerdict | None: + raise TimeoutError("judge timed out") + + subscriber = ConversationOutcomeSubscriber( + conversation_id=uuid4(), + tags=lambda: store.tags, + set_tag=store.set_tag, + judge=judge, + ) + + await _feed( + subscriber, + _message("user", "Fix it"), + _status("running"), + _finish("Fixed, but should I also bump the version?", score=0.95), + _status("finished"), + ) + + assert store.tags == { + "project": "p1", + "turn": "asking", + "ask": "Fixed, but should I also bump the version?", + } + + +@pytest.mark.asyncio +async def test_the_coordinator_is_never_judged_or_resolved(): + store = _Tags({"project": "p1"}) + seen: list[ResolutionState] = [] + verdict = ResolutionVerdict(probability=0.99, owed="nothing") + + async def judge(state: ResolutionState) -> ResolutionVerdict | None: + seen.append(state) + return verdict + + async def is_coordinator() -> bool: + return True + + subscriber = ConversationOutcomeSubscriber( + conversation_id=uuid4(), + tags=lambda: store.tags, + set_tag=store.set_tag, + judge=judge, + is_coordinator=is_coordinator, + ) + + await _feed( + subscriber, + _message("user", "Plan the release"), + _status("running"), + _finish("Two workers are on it.", score=0.95), + _status("finished"), + ) + + assert seen == [] + assert store.tags == {"project": "p1", "turn": "finished"} + + @pytest.mark.asyncio async def test_a_user_message_clears_the_resolve_note(): store = _Tags( diff --git a/tests/agent_server/test_projects_router.py b/tests/agent_server/test_projects_router.py index 5897f23295..274a15789d 100644 --- a/tests/agent_server/test_projects_router.py +++ b/tests/agent_server/test_projects_router.py @@ -391,12 +391,14 @@ async def test_tagged_children_are_conversations_with_their_parent( @pytest.mark.asyncio -async def test_remove_conversation_clears_only_the_project_tag( +async def test_remove_conversation_clears_the_project_and_board_tags( client, service, tmp_path ): project = (await client.post("/api/projects", json={"name": "P"})).json() worker = await _start(service, tmp_path, tags={"insiderrole": "worker"}) await _join(service, project["id"], worker) + await service.projects.set_conversation_tag(worker, "turn", "asking") + await service.projects.set_conversation_tag(worker, "ask", "Ship it?") removed = await client.delete( f"/api/projects/{project['id']}/conversations/{worker.hex}"