Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions apps/node-services/src/agent/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ take precedence.
cancellation, and the only Runtime execution path. Do not add a second
scheduler, execution store, or Runtime executor beside it, and do not
restore TeamHub Run admission.
- A Room Agent speaks only through TeamHub message writes, which the Computer
classifies as `message` activities (`reply-delivery.ts`). A Room Run whose
every write failed finishes failed with `reply_undelivered`. One that never
tried to write while a Human's Message that mentioned the Agent (a direct
chat always does) waits finishes failed with `reply_not_sent`. Its final
text is private scratch and is never posted for it.
- Ordinary Room/Meta execution reads effective Agent configuration before a new
Local Run. Skill saves do not activate idle Contexts. If versions changed,
reuse graceful activation to resume the same native history; running/steered
Expand Down
1 change: 1 addition & 0 deletions apps/node-services/src/agent/executor/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,7 @@ export interface RuntimeExecutionFailure {
| "execution_failed"
| "execution_outcome_unknown"
| "reply_undelivered"
| "reply_not_sent"
| "interrupted"
| "computer_unavailable"
| "cleanup_unconfirmed"
Expand Down
42 changes: 41 additions & 1 deletion apps/node-services/src/agent/executor/reply-delivery.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,30 @@
import { isRecord } from "@linger/service-kit";
import type { RunEvent } from "./provider.js";
import type { Message } from "./types.js";

/**
* The Run's TeamHub message calls, as the Computer classifies them: a
* `message` activity is a message write (`teamhub message send` and its
* siblings), `message_read` any other `teamhub message` call, and a command
* that exits non-zero finishes as failed.
* that exits non-zero finishes as failed. An emoji Reaction is a message
* write too.
*/
export class ReplyDelivery {
readonly #started = new Map<string, "message" | "message_read">();
#awaitingHuman = false;
#attemptedWrites = 0;
#failedWrites = 0;
#deliveredWrites = 0;
#failedReads = 0;

/** Records the inputs the Run took up, at its start or as steers. */
observeInputs(messages: readonly Message[]): void {
if (messages.some(awaitsReply)) this.#awaitingHuman = true;
}

observe(event: RunEvent): void {
if (event.type === "activity.started") {
if (event.kind === "message") this.#attemptedWrites += 1;
if (event.kind === "message" || event.kind === "message_read")
this.#started.set(event.activityId, event.kind);
return;
Expand All @@ -39,6 +50,15 @@ export class ReplyDelivery {
return this.#failedWrites > 0 && this.#deliveredWrites === 0;
}

/**
* A Human who mentioned the Agent is waiting and the Agent never tried to
* post. Its Runtime ended the turn in text nobody sees; small models do
* this when they answer without calling the message command.
*/
get replyNotSent(): boolean {
return this.#awaitingHuman && this.#attemptedWrites === 0;
}

get hasFailures(): boolean {
return this.#failedWrites > 0 || this.#failedReads > 0;
}
Expand All @@ -55,3 +75,23 @@ export class ReplyDelivery {
};
}
}

/**
* Whether TeamHub queued this input for a Human's Message that mentioned this
* Agent. A direct chat mentions its Agent too. Room talk the Agent only reads
* along with, a Routine wake, a Task callback and another Agent's Message may
* rightly go unanswered.
*/
function awaitsReply(message: Message): boolean {
const notification = (message as unknown as Record<string, unknown>)
.acceptedNotification;
if (!isRecord(notification) || !Array.isArray(notification.messages))
return false;
return notification.messages.some(
(queued) =>
isRecord(queued) &&
queued.mentioned === true &&
queued.authorType === "user" &&
queued.messageKind === "conversation",
);
}
52 changes: 37 additions & 15 deletions apps/node-services/src/agent/executor/run-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ interface ActiveExecution {
readonly runId: string;
readonly sourceInboxIds: Set<string>;
readonly steerOperations: Set<Promise<void>>;
readonly replyDelivery: ReplyDelivery;
acceptingSteers: boolean;
}

Expand Down Expand Up @@ -1005,6 +1006,7 @@ export class RunLoop {
);
}
const replyDelivery = new ReplyDelivery();
replyDelivery.observeInputs(execution.messages);
let resumeStartRetries = 0;
for (;;) {
controller.signal.throwIfAborted();
Expand Down Expand Up @@ -1052,7 +1054,12 @@ export class RunLoop {
execution.run = run;
const providerReadyAt = performance.now();
this.#logProviderReady(execution, run, providerStartAt, providerReadyAt);
const active = await this.#activateExecution(execution, run, input);
const active = await this.#activateExecution(
execution,
run,
input,
replyDelivery,
);

execution.runtimeStatus ??= await this.#openRuntimeStatus(
runId,
Expand Down Expand Up @@ -1291,6 +1298,7 @@ export class RunLoop {
execution: LocalRunExecution,
run: AgentRun,
input: AgentInput,
replyDelivery: ReplyDelivery,
): Promise<ActiveExecution> {
const { runId, controller } = execution;
if ((await this.#state.getLocalRun(runId))?.cancellationRequested) {
Expand Down Expand Up @@ -1322,6 +1330,7 @@ export class RunLoop {
this.#activeExecution?.sourceInboxIds ??
new Set(execution.provenance.sourceInboxIds),
steerOperations: new Set(),
replyDelivery,
};
this.#activeExecution = active;
execution.intent.resolveProviderReady(active);
Expand Down Expand Up @@ -1717,19 +1726,19 @@ export class RunLoop {
}

/**
* A Room Agent speaks only through TeamHub message writes. When it tried to
* post and none landed, the Room has nothing to show, so a finished Provider
* turn is not a succeeded Run. Task Runs report through their final
* response and keep it.
* A Room Agent speaks only through TeamHub message writes, so a finished
* Provider turn is not a succeeded Run when the Room has nothing to show:
* the Agent tried to post and none landed, or a Human who mentioned it is
* waiting and it never tried. Its final text is private scratch and is not
* posted for it. Task Runs report through their final response and keep it.
*/
#completedTerminal(
runId: string,
finalResponse: string,
replyDelivery: ReplyDelivery,
): LocalRunTerminal {
const undelivered =
this.#executionContextRef?.kind === "meta" &&
replyDelivery.replyUndelivered;
const roomRun = this.#executionContextRef?.kind === "meta";
const undelivered = roomRun && replyDelivery.replyUndelivered;
if (replyDelivery.hasFailures) {
this.#logger.log("warn", "local_run.teamhub_messages_failed", {
runId,
Expand All @@ -1738,14 +1747,26 @@ export class RunLoop {
finalResponseLength: finalResponse.length,
});
}
if (!undelivered) {
return { type: "succeeded", finalResponse };
if (undelivered) {
return {
type: "failed",
error: "Agent reply was not delivered to the Room",
failure: runFailure("reply_undelivered", this.#runtimeType),
};
}
return {
type: "failed",
error: "Agent reply was not delivered to the Room",
failure: runFailure("reply_undelivered", this.#runtimeType),
};
if (roomRun && replyDelivery.replyNotSent) {
this.#logger.log("warn", "local_run.reply_not_sent", {
runId,
...replyDelivery.counts(),
finalResponseLength: finalResponse.length,
});
return {
type: "failed",
error: "Agent ended its turn without sending a reply to the Room",
failure: runFailure("reply_not_sent", this.#runtimeType),
};
}
return { type: "succeeded", finalResponse };
}

#reportTerminal(runId: string, terminal: LocalTurnTerminal): void {
Expand Down Expand Up @@ -1839,6 +1860,7 @@ export class RunLoop {
target.sourceInboxIds.add(inboxId);
}
target.inputs.push(input);
target.replyDelivery.observeInputs(messages);
this.#reportActiveRunContext({
runId: target.runId,
sourceInboxIds: Object.freeze([...target.sourceInboxIds]),
Expand Down
134 changes: 132 additions & 2 deletions apps/node-services/tests/agent/executor/run-loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,9 @@ describe("RunLoop", () => {
const finishRun = async (
events: readonly RunEvent[],
executionContextRef?: ExecutionContextRef,
input: Message = message("introduce yourself"),
finalResponse = "I could not deliver the reply: TeamHub returned 404.",
steered?: Message,
) => {
const harness = createHarness(
undefined,
Expand All @@ -342,11 +345,19 @@ describe("RunLoop", () => {
);
const run = new FakeRun(events);
harness.provider.runs.push(run);
const runId = enqueue(harness.state, message("introduce yourself"));
const runId = enqueue(harness.state, input);
try {
harness.execution.start();
await vi.waitFor(() => expect(run.waitCalls).toBe(1));
run.complete("I could not deliver the reply: TeamHub returned 404.");
if (steered) {
const steer = selectSteer(harness.state, steered);
await harness.execution.steer(
steer.groupId,
steer.messages,
new AbortController().signal,
);
}
run.complete(finalResponse);
await vi.waitFor(() =>
expect(harness.state.getLocalRun(runId)?.status).not.toBe("running"),
);
Expand Down Expand Up @@ -424,6 +435,125 @@ describe("RunLoop", () => {
);
});

// TeamHub freezes each Inbox Message's sender, kind and whether it
// mentioned this Agent into the queued input.
const inboxInput = (
inputId: string,
queued: {
readonly authorType: "agent" | "user";
readonly messageKind: string;
readonly mentioned: boolean;
},
): Message =>
({
...message(inputId, "room:room-1"),
acceptedNotification: {
inboxId: "room:room-1",
after: "0",
through: "1",
targeted: queued.mentioned,
messages: [
{
id: `${inputId}-message`,
body: "how are you?",
author: "Leo",
authorActorId: "actor-leo",
...queued,
},
],
},
}) as Message;
const humanMention = (inputId: string) =>
inboxInput(inputId, {
authorType: "user",
messageKind: "conversation",
mentioned: true,
});

it("fails a Room Run that posted nothing while the Human who mentioned the Agent waits", async () => {
const { log, run } = await finishRun(
failedRead,
undefined,
humanMention("mention"),
"Doing well, you?",
);

expect(run).toMatchObject({
status: "failed",
terminal: {
type: "failed",
failure: { code: "reply_not_sent", runtime_type: "pi" },
},
});
expect(JSON.stringify(run?.terminal)).not.toContain("Doing well");
expect(log).toHaveBeenCalledWith(
"warn",
"local_run.reply_not_sent",
expect.objectContaining({
runId: run?.runId,
deliveredMessageWrites: 0,
finalResponseLength: "Doing well, you?".length,
}),
);
});

it("fails a Room Run that posted nothing after a Human mention was steered into it", async () => {
const { run } = await finishRun(
[],
undefined,
inboxInput("room-talk", {
authorType: "user",
messageKind: "conversation",
mentioned: false,
}),
"",
humanMention("steered-mention"),
);

expect(run?.terminal).toMatchObject({
type: "failed",
failure: { code: "reply_not_sent" },
});
});

it("keeps a Room Run succeeded when it answered a Human mention with a message", async () => {
const { run } = await finishRun(
messageWrite("send-1", "succeeded"),
undefined,
humanMention("mention"),
"",
);

expect(run?.terminal).toMatchObject({ type: "succeeded" });
});

it.each([
[
"Room talk that did not mention the Agent",
{ authorType: "user", messageKind: "conversation", mentioned: false },
],
[
"a routine wake that mentions the Agent",
{ authorType: "user", messageKind: "agent_runtime", mentioned: true },
],
[
"another Agent's mention",
{ authorType: "agent", messageKind: "conversation", mentioned: true },
],
] as const)(
"keeps a Room Run that stayed silent on %s succeeded",
async (_label, queued) => {
const { run } = await finishRun(
[],
undefined,
inboxInput("wake", queued),
"Nothing to add.",
);

expect(run?.terminal).toMatchObject({ type: "succeeded" });
},
);

it("leaves a Task Run, which reports through its final response, succeeded", async () => {
const { run } = await finishRun(messageWrite("send-1", "failed"), {
kind: "alter_ego",
Expand Down
2 changes: 1 addition & 1 deletion apps/teamhub/internal/agentruntimestate/runtime_failure.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func terminalRuntimeFailure(raw json.RawMessage) (*PublicRuntimeFailure, error)
switch failure.Code {
case "session_not_found", "protocol_incompatible", "authentication_required", "network_error", "rate_limited",
"model_unavailable", "model_not_offered", "context_too_long", "model_service_error", "timeout", "execution_failed",
"execution_outcome_unknown", "reply_undelivered", "interrupted", "computer_unavailable",
"execution_outcome_unknown", "reply_undelivered", "reply_not_sent", "interrupted", "computer_unavailable",
"cleanup_unconfirmed", "response_too_large", "unclassified":
default:
return nil, ErrInvalid
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ func TestTerminalRuntimeFailure(t *testing.T) {
{"legacy", `{"type":"failed","error":"private error"}`, true},
{"unknown execution outcome", `{"type":"failed","failure":{"code":"execution_outcome_unknown","runtime_type":"claude"}}`, true},
{"reply never reached the Room", `{"type":"failed","error":"Agent reply was not delivered to the Room","failure":{"code":"reply_undelivered","runtime_type":"pi"}}`, true},
{"no reply sent to a waiting Human", `{"type":"failed","error":"Agent ended its turn without sending a reply to the Room","failure":{"code":"reply_not_sent","runtime_type":"pi"}}`, true},
{"model refused for the account", `{"type":"failed","failure":{"code":"model_unavailable","runtime_type":"codex","runtime_version":"0.153.4"}}`, true},
{"Computer does not offer the model", `{"type":"failed","error":"Local Computer model is unavailable","failure":{"code":"model_not_offered","runtime_type":"codex"}}`, true},
{"canceled cannot claim unknown failure", `{"type":"canceled","failure":{"code":"execution_outcome_unknown","runtime_type":"codex"}}`, false},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,16 @@ describe("Runtime failure messages", () => {
expect(message).toContain("TeamHub");
});

it("says the Agent finished without sending its reply", () => {
const message = runtimeFailureMessage({
code: "reply_not_sent",
runtime_type: "pi",
});
expect(message).toContain("Pi:");
expect(message).toContain("finished without sending its reply");
expect(message).toContain("choose a stronger model");
});

it("names a failure the Agent Service observed without a Runtime prefix", () => {
expect(runtimeFailureMessage({ code: "interrupted" })).toMatch(
/^The Run was interrupted before it finished/,
Expand Down
Loading
Loading