Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 153 additions & 1 deletion app/lib/workflows/__tests__/runAgentStepStreaming.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { streamText } from "ai";
import { FatalError } from "workflow";
import { getRun } from "workflow/api";
import { pollWorkflowCancellation } from "@/lib/chat/pollWorkflowCancellation";
import { runAgentStep } from "@/app/lib/workflows/runAgentStep";
import { persistAssistantMessage } from "@/lib/chat/persistAssistantMessage";

Expand All @@ -13,7 +16,8 @@ vi.mock("@ai-sdk/gateway", () => ({
vi.mock("@/lib/chat/persistAssistantMessage", () => ({
persistAssistantMessage: vi.fn(),
}));
vi.mock("workflow", () => ({
vi.mock("workflow", async () => ({
FatalError: (await vi.importActual<typeof import("workflow")>("workflow")).FatalError,
getWorkflowMetadata: vi.fn(() => ({
workflowRunId: "test-run-id",
workflowName: "test",
Expand Down Expand Up @@ -116,3 +120,151 @@ describe("runAgentStep — streaming shape (mirrors upstream open-agents)", () =
expect(result.responseMessages).toEqual([]);
});
});

describe("retained structured stream output", () => {
const safePart = { type: "text-delta", id: "t1", delta: "already emitted independent text" };
const markedPart = {
type: "tool-output-available",
toolCallId: "source-call",
output: { rows: [{ mode: "retained_field" }], sibling: "PRIVATE-SIBLING-987654321" },
};
const markedMessage = {
id: "assistant",
role: "assistant",
parts: [
{
type: "tool-source",
toolCallId: "source-call",
state: "output-available",
input: {},
output: markedPart.output,
},
],
};

it.each(["running", "cancelled", "poller-aborted"])(
"withholds the whole marked part and refuses rather than returning output when run is %s",
async status => {
vi.mocked(getRun).mockReturnValue({ status: Promise.resolve(status) } as never);
if (status === "poller-aborted") {
Comment thread
sidneyswift marked this conversation as resolved.
Resolved
vi.mocked(pollWorkflowCancellation).mockImplementationOnce((_runId, controller) => {
controller.abort();
return { stop: vi.fn(), done: Promise.resolve() };
});
}
let signal: AbortSignal | undefined;
const finish = vi.fn();
vi.mocked(streamText).mockImplementation(options => {
signal = options.abortSignal;
// Reject after denial cleanup aborts the provider, as streamText can do.
const rejectOnAbort = () =>
new Promise<never>((_resolve, reject) => {
const fail = () =>
reject(Object.assign(new Error("provider aborted"), { name: "AbortError" }));
if (signal?.aborted) queueMicrotask(fail);
else signal?.addEventListener("abort", fail, { once: true });
});
return {
toUIMessageStream: (opts: { onFinish: (event: unknown) => void }) =>
(async function* () {
try {
yield safePart;
yield markedPart;
yield { type: "text-delta", id: "t1", delta: "later output" };
} finally {
// SDK cancellation may finish its accumulated message, including withheld data.
finish();
opts.onFinish({ responseMessage: markedMessage });
}
})(),
finishReason: rejectOnAbort(),
response: rejectOnAbort(),
} as never;
});
const { stream, written } = makeWritable();
const close = vi.spyOn(stream, "close");
const abort = vi.spyOn(stream, "abort");
const result = runAgentStep({ ...baseInput, writable: stream } as never);
await expect(result).rejects.toBeInstanceOf(FatalError);
await expect(result).rejects.toThrow(/^Retained-field model input unavailable$/);
expect(written).toEqual([safePart]);
expect(JSON.stringify(written)).not.toContain("PRIVATE-SIBLING");
expect(finish).toHaveBeenCalledOnce();
expect(stream.locked).toBe(false);
expect(close).not.toHaveBeenCalled();
expect(abort).not.toHaveBeenCalled();
expect(signal?.aborted).toBe(true);
expect(
vi.mocked(pollWorkflowCancellation).mock.results.at(-1)?.value.stop,
).toHaveBeenCalledOnce();
expect(persistAssistantMessage).not.toHaveBeenCalled();
},
);

it("refuses a marked accumulated onFinish message even during cancellation", async () => {
vi.mocked(getRun).mockReturnValue({ status: Promise.resolve("cancelled") } as never);
vi.mocked(streamText).mockReturnValue({
toUIMessageStream: (opts: { onFinish: (event: unknown) => void }) =>
(async function* () {
yield safePart;
opts.onFinish({ responseMessage: markedMessage });
throw Object.assign(new Error("stopped"), { name: "AbortError" });
})(),
finishReason: Promise.resolve("stop"),
response: Promise.resolve({ messages: [] }),
} as never);
const { stream, written } = makeWritable();
await expect(runAgentStep({ ...baseInput, writable: stream } as never)).rejects.toThrow(
"Retained-field model input unavailable",
);
expect(written).toEqual([safePart]);
});

it("preserves an unmarked tool output and its completed message", async () => {
const part = { type: "tool-output-available", toolCallId: "safe", output: { total: 7 } };
const message = {
id: "assistant",
role: "assistant",
parts: [
{
type: "tool-source",
toolCallId: "safe",
state: "output-available",
input: {},
output: part.output,
},
],
};
const modelMessages = [
{
role: "tool",
content: [
{
type: "tool-result",
toolCallId: "safe",
toolName: "source",
output: { type: "json", value: part.output },
},
],
},
];
vi.mocked(streamText).mockReturnValue({
toUIMessageStream: (opts: { onFinish: (event: unknown) => void }) =>
(async function* () {
yield part;
opts.onFinish({ responseMessage: message });
})(),
finishReason: Promise.resolve("tool-calls"),
response: Promise.resolve({ messages: modelMessages }),
} as never);
const { stream, written } = makeWritable();
const result = await runAgentStep({ ...baseInput, writable: stream } as never);
expect(written).toEqual([part]);
expect(result).toEqual({
finishReason: "tool-calls",
responseMessage: message,
responseMessages: modelMessages,
aborted: false,
});
});
});
19 changes: 19 additions & 0 deletions app/lib/workflows/runAgentStep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,17 @@ export async function runAgentStep(input: RunAgentStepInput): Promise<RunAgentSt
// from the inner stream, and missing that silently dropped every tool call
// from the transcript (chat#1918). Persistence now lives in the workflow
// body via `persistAssistantMessageStep`.
// SDK onFinish may run while iterator cancellation unwinds. Keep a privacy
// refusal authoritative even if cancellation replaces the thrown error.
let outputRefusal: unknown;
const assertOutput = (output: unknown) => {
try {
assertNoRetainedFieldModelInput(output);
} catch (error) {
outputRefusal = error;
throw error;
}
};
let responseMessage: UIMessage | undefined;
let finishReason: string;
let responseMessages: ModelMessage[] = [];
Expand All @@ -215,12 +226,14 @@ export async function runAgentStep(input: RunAgentStepInput): Promise<RunAgentSt
sendStart: false,
sendFinish: false,
onFinish: ({ responseMessage: finalMessage }) => {
assertOutput(finalMessage);
responseMessage = finalMessage;
},
})) {
// A writer lock taken inside a step applies only within that step, so
// sequential steps can share one stream. Release per part so the step's
// request can terminate.
assertOutput(part);
const writer = input.writable.getWriter();
try {
await writer.write(part);
Expand All @@ -236,6 +249,12 @@ export async function runAgentStep(input: RunAgentStepInput): Promise<RunAgentSt
finishReason = reason;
responseMessages = response.messages;
} catch (error) {
if (outputRefusal) {
// Aborting the provider in finally may reject these pending promises.
void Promise.resolve(result.finishReason).catch(() => {});
void Promise.resolve(result.response).catch(() => {});
throw outputRefusal;
}
// Three ways a user-stop surfaces here: the stream throws AbortError; the
// poller already flipped our signal; or `run.cancel()` closed the workflow
// writable underneath us and the write threw something unrelated before
Expand Down
19 changes: 19 additions & 0 deletions lib/chat/retained-field/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ source diagnostics or authority details and is not retryable by the workflow.
- `routeChatModelStep` checks before routing metadata or the Jev evaluator.
- `runAgentStep` checks both model and original UI messages before polling,
gateway construction and `streamText`, including independently resumed steps.
It also checks each structured UI stream part before acquiring a writer or
writing any of that part. A marked part is withheld in full, including sibling
payloads. The accumulated `onFinish` message is checked before capture because
the SDK can finish it during stream cancellation. A privacy refusal remains a
fatal rejection even when cancellation races it; no refused output is returned
as an ordinary aborted result for persistence. Existing poller/provider abort
cleanup runs, while shared-stream closure stays with the workflow.
- `setupChatRequest` checks original messages before `getGeneralAgent` or
conversion, then converted messages before legacy truncation/config return.
Its actual callers are Slack's `handleSlackChatMessage` and evaluation's
Expand All @@ -28,6 +35,11 @@ source diagnostics or authority details and is not retryable by the workflow.
that reconstruction intentionally drops metadata. Its authenticated API-key
account and room scope are unchanged. Existing redacted error/idle handling
remains; no automatic retry is added.
- Email's `generateEmailResponse` loads history through `getEmailRoomMessages`,
which checks raw stored content before extracting text and dropping metadata.
Marked history denies the entire generation, including sibling text; the
existing inbound handler retains its generic error response. Room lookup and
authenticated account/organization context are unchanged.
- `getGeneralAgent` checks before account/knowledge/tool setup. Its actual
`ToolLoopAgent.prepareStep` checks messages and tool results every iteration,
and checks any tool-chain replacement messages. In the installed SDK this
Expand All @@ -42,6 +54,13 @@ expansion, raw-report fallback, public route, grant or activation.

## Qualification

Stream-output regression tests invoke the actual `runAgentStep` with synthetic
stream/provider/cancellation seams. They cover running, cancelled and already
aborted runs, withheld sibling payloads, marked `onFinish` state, rejection of
pending provider promises during cleanup, and unchanged unmarked tool output.
This guard does not prevent the operation that produced the output or retract
independent prose already emitted; it denies recognized structured output only.

Tests invoke real workflow orchestration/conversion/routing/provider-step entry
points with synthetic effect seams; replay cases supply the already-completed
step outputs that those callers can receive. They are not a claim of live WDK
Expand Down
56 changes: 51 additions & 5 deletions lib/emails/inbound/__tests__/generateEmailResponse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@ import { generateEmailResponse } from "../generateEmailResponse";
import type { ChatRequestBody } from "@/lib/chat/validateChatRequest";

import getGeneralAgent from "@/lib/agents/generalAgent/getGeneralAgent";
import { getEmailRoomMessages } from "@/lib/emails/inbound/getEmailRoomMessages";
import selectMemories from "@/lib/supabase/memories/selectMemories";
import { selectRoomWithArtist } from "@/lib/supabase/rooms/selectRoomWithArtist";
import { FatalError } from "workflow";

vi.mock("@/lib/agents/generalAgent/getGeneralAgent", () => ({
default: vi.fn(),
}));

vi.mock("@/lib/emails/inbound/getEmailRoomMessages", () => ({
getEmailRoomMessages: vi.fn(),
vi.mock("@/lib/supabase/memories/selectMemories", () => ({
default: vi.fn(),
}));

vi.mock("@/lib/emails/getEmailFooter", () => ({
Expand Down Expand Up @@ -43,7 +45,10 @@ describe("generateEmailResponse", () => {
});

it("generates response with text and footer", async () => {
vi.mocked(getEmailRoomMessages).mockResolvedValue([{ role: "user", content: "Hi there" }]);
vi.mocked(selectMemories).mockResolvedValue([
{ content: { role: "user", parts: [{ type: "text", text: "Hi there" }] } },
{ content: { role: "assistant", parts: [{ type: "text", text: "Earlier reply" }] } },
] as never);

const body: ChatRequestBody = {
accountId: "acc-1",
Expand All @@ -55,10 +60,51 @@ describe("generateEmailResponse", () => {
const result = await generateEmailResponse(body);

expect(mockGenerate).toHaveBeenCalledWith({
messages: [{ role: "user", content: "Hi there" }],
messages: [
{ role: "user", content: "Hi there" },
{ role: "assistant", content: "Earlier reply" },
],
});
expect(selectMemories).toHaveBeenCalledExactlyOnceWith("room-1", { ascending: true });
expect(getGeneralAgent).toHaveBeenCalledExactlyOnceWith(body);
expect(selectRoomWithArtist).toHaveBeenCalledExactlyOnceWith("room-1");
expect(result.text).toBe("Hello from assistant");
expect(result.html).toContain("Hello from assistant");
expect(result.html).toContain("<footer>footer</footer>");
});
it.each(["part", "metadata"])(
"denies raw email history with a %s marker before text adaptation",
async location => {
const marker = { type: "data-retained-field-projection", private: "MARKER-SECRET" };
const content = {
role: "assistant",
parts: [
...(location === "part" ? [marker] : []),
{ type: "text", text: "PRIVATE-SIBLING-987654321" },
],
...(location === "metadata"
? { metadata: { contract: "catalog-retained-field-projection/v1" } }
: {}),
};
const memories = [{ content }];
const original = structuredClone(memories);
vi.mocked(selectMemories).mockResolvedValue(memories as never);
const body = {
accountId: "authenticated-account",
orgId: "authenticated-org",
roomId: "authorized-room",
messages: [],
} as ChatRequestBody;
const result = generateEmailResponse(body);
await expect(result).rejects.toBeInstanceOf(FatalError);
await expect(result).rejects.toThrow(/^Retained-field model input unavailable$/);
expect(mockGenerate).not.toHaveBeenCalled();
expect(selectRoomWithArtist).not.toHaveBeenCalled();
expect(selectMemories).toHaveBeenCalledExactlyOnceWith("authorized-room", {
ascending: true,
});
expect(getGeneralAgent).toHaveBeenCalledExactlyOnceWith(body);
expect(memories).toEqual(original);
},
);
});
3 changes: 3 additions & 0 deletions lib/emails/inbound/getEmailRoomMessages.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { assertNoRetainedFieldModelInput } from "@/lib/chat/retained-field/assertNoRetainedFieldModelInput";
import type { ModelMessage } from "ai";
import selectMemories from "@/lib/supabase/memories/selectMemories";
import { extractTextFromParts } from "./extractTextFromParts";
Expand All @@ -17,6 +18,8 @@ interface MemoryContent {
export async function getEmailRoomMessages(roomId: string): Promise<ModelMessage[]> {
const existingMemories = await selectMemories(roomId, { ascending: true });
if (!existingMemories) return [];
// Inspect stored content before text-only adaptation drops structured markers.
assertNoRetainedFieldModelInput(existingMemories.map(memory => memory.content));

const messages: ModelMessage[] = [];

Expand Down
Loading