diff --git a/app/lib/workflows/__tests__/runAgentStepStreaming.test.ts b/app/lib/workflows/__tests__/runAgentStepStreaming.test.ts index 5effcab3c..536c5cdfc 100644 --- a/app/lib/workflows/__tests__/runAgentStepStreaming.test.ts +++ b/app/lib/workflows/__tests__/runAgentStepStreaming.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { streamText } from "ai"; +import { FatalError } from "workflow"; +import { getRun } from "workflow/api"; +import { pollWorkflowCancellation } from "@/lib/chat/pollWorkflowCancellation"; import { runAgentStep } from "@/app/lib/workflows/runAgentStep"; import { persistAssistantMessage } from "@/lib/chat/persistAssistantMessage"; @@ -13,7 +16,8 @@ vi.mock("@ai-sdk/gateway", () => ({ vi.mock("@/lib/chat/persistAssistantMessage", () => ({ persistAssistantMessage: vi.fn(), })); -vi.mock("workflow", () => ({ +vi.mock("workflow", async () => ({ + FatalError: (await vi.importActual("workflow")).FatalError, getWorkflowMetadata: vi.fn(() => ({ workflowRunId: "test-run-id", workflowName: "test", @@ -116,3 +120,151 @@ describe("runAgentStep — streaming shape (mirrors upstream open-agents)", () = expect(result.responseMessages).toEqual([]); }); }); + +describe("retained structured stream output", () => { + const safePart = { type: "text-delta", id: "t1", delta: "already emitted independent text" }; + const markedPart = { + type: "tool-output-available", + toolCallId: "source-call", + output: { rows: [{ mode: "retained_field" }], sibling: "PRIVATE-SIBLING-987654321" }, + }; + const markedMessage = { + id: "assistant", + role: "assistant", + parts: [ + { + type: "tool-source", + toolCallId: "source-call", + state: "output-available", + input: {}, + output: markedPart.output, + }, + ], + }; + + it.each(["running", "cancelled", "poller-aborted"])( + "withholds the whole marked part and refuses rather than returning output when run is %s", + async status => { + vi.mocked(getRun).mockReturnValue({ status: Promise.resolve(status) } as never); + if (status === "poller-aborted") { + vi.mocked(pollWorkflowCancellation).mockImplementationOnce((_runId, controller) => { + controller.abort(); + return { stop: vi.fn(), done: Promise.resolve() }; + }); + } + let signal: AbortSignal | undefined; + const finish = vi.fn(); + vi.mocked(streamText).mockImplementation(options => { + signal = options.abortSignal; + // Reject after denial cleanup aborts the provider, as streamText can do. + const rejectOnAbort = () => + new Promise((_resolve, reject) => { + const fail = () => + reject(Object.assign(new Error("provider aborted"), { name: "AbortError" })); + if (signal?.aborted) queueMicrotask(fail); + else signal?.addEventListener("abort", fail, { once: true }); + }); + return { + toUIMessageStream: (opts: { onFinish: (event: unknown) => void }) => + (async function* () { + try { + yield safePart; + yield markedPart; + yield { type: "text-delta", id: "t1", delta: "later output" }; + } finally { + // SDK cancellation may finish its accumulated message, including withheld data. + finish(); + opts.onFinish({ responseMessage: markedMessage }); + } + })(), + finishReason: rejectOnAbort(), + response: rejectOnAbort(), + } as never; + }); + const { stream, written } = makeWritable(); + const close = vi.spyOn(stream, "close"); + const abort = vi.spyOn(stream, "abort"); + const result = runAgentStep({ ...baseInput, writable: stream } as never); + await expect(result).rejects.toBeInstanceOf(FatalError); + await expect(result).rejects.toThrow(/^Retained-field model input unavailable$/); + expect(written).toEqual([safePart]); + expect(JSON.stringify(written)).not.toContain("PRIVATE-SIBLING"); + expect(finish).toHaveBeenCalledOnce(); + expect(stream.locked).toBe(false); + expect(close).not.toHaveBeenCalled(); + expect(abort).not.toHaveBeenCalled(); + expect(signal?.aborted).toBe(true); + expect( + vi.mocked(pollWorkflowCancellation).mock.results.at(-1)?.value.stop, + ).toHaveBeenCalledOnce(); + expect(persistAssistantMessage).not.toHaveBeenCalled(); + }, + ); + + it("refuses a marked accumulated onFinish message even during cancellation", async () => { + vi.mocked(getRun).mockReturnValue({ status: Promise.resolve("cancelled") } as never); + vi.mocked(streamText).mockReturnValue({ + toUIMessageStream: (opts: { onFinish: (event: unknown) => void }) => + (async function* () { + yield safePart; + opts.onFinish({ responseMessage: markedMessage }); + throw Object.assign(new Error("stopped"), { name: "AbortError" }); + })(), + finishReason: Promise.resolve("stop"), + response: Promise.resolve({ messages: [] }), + } as never); + const { stream, written } = makeWritable(); + await expect(runAgentStep({ ...baseInput, writable: stream } as never)).rejects.toThrow( + "Retained-field model input unavailable", + ); + expect(written).toEqual([safePart]); + }); + + it("preserves an unmarked tool output and its completed message", async () => { + const part = { type: "tool-output-available", toolCallId: "safe", output: { total: 7 } }; + const message = { + id: "assistant", + role: "assistant", + parts: [ + { + type: "tool-source", + toolCallId: "safe", + state: "output-available", + input: {}, + output: part.output, + }, + ], + }; + const modelMessages = [ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "safe", + toolName: "source", + output: { type: "json", value: part.output }, + }, + ], + }, + ]; + vi.mocked(streamText).mockReturnValue({ + toUIMessageStream: (opts: { onFinish: (event: unknown) => void }) => + (async function* () { + yield part; + opts.onFinish({ responseMessage: message }); + })(), + finishReason: Promise.resolve("tool-calls"), + response: Promise.resolve({ messages: modelMessages }), + } as never); + const { stream, written } = makeWritable(); + const result = await runAgentStep({ ...baseInput, writable: stream } as never); + expect(written).toEqual([part]); + expect(result).toEqual({ + finishReason: "tool-calls", + responseMessage: message, + responseMessages: modelMessages, + aborted: false, + }); + }); +}); diff --git a/app/lib/workflows/runAgentStep.ts b/app/lib/workflows/runAgentStep.ts index 387f176b1..55dde8bce 100644 --- a/app/lib/workflows/runAgentStep.ts +++ b/app/lib/workflows/runAgentStep.ts @@ -197,6 +197,17 @@ export async function runAgentStep(input: RunAgentStepInput): Promise { + try { + assertNoRetainedFieldModelInput(output); + } catch (error) { + outputRefusal = error; + throw error; + } + }; let responseMessage: UIMessage | undefined; let finishReason: string; let responseMessages: ModelMessage[] = []; @@ -215,12 +226,14 @@ export async function runAgentStep(input: RunAgentStepInput): Promise { + assertOutput(finalMessage); responseMessage = finalMessage; }, })) { // A writer lock taken inside a step applies only within that step, so // sequential steps can share one stream. Release per part so the step's // request can terminate. + assertOutput(part); const writer = input.writable.getWriter(); try { await writer.write(part); @@ -236,6 +249,12 @@ export async function runAgentStep(input: RunAgentStepInput): Promise {}); + void Promise.resolve(result.response).catch(() => {}); + throw outputRefusal; + } // Three ways a user-stop surfaces here: the stream throws AbortError; the // poller already flipped our signal; or `run.cancel()` closed the workflow // writable underneath us and the write threw something unrelated before diff --git a/lib/chat/retained-field/README.md b/lib/chat/retained-field/README.md index 51ffece55..bbf98485f 100644 --- a/lib/chat/retained-field/README.md +++ b/lib/chat/retained-field/README.md @@ -20,6 +20,13 @@ source diagnostics or authority details and is not retryable by the workflow. - `routeChatModelStep` checks before routing metadata or the Jev evaluator. - `runAgentStep` checks both model and original UI messages before polling, gateway construction and `streamText`, including independently resumed steps. + It also checks each structured UI stream part before acquiring a writer or + writing any of that part. A marked part is withheld in full, including sibling + payloads. The accumulated `onFinish` message is checked before capture because + the SDK can finish it during stream cancellation. A privacy refusal remains a + fatal rejection even when cancellation races it; no refused output is returned + as an ordinary aborted result for persistence. Existing poller/provider abort + cleanup runs, while shared-stream closure stays with the workflow. - `setupChatRequest` checks original messages before `getGeneralAgent` or conversion, then converted messages before legacy truncation/config return. Its actual callers are Slack's `handleSlackChatMessage` and evaluation's @@ -28,6 +35,11 @@ source diagnostics or authority details and is not retryable by the workflow. that reconstruction intentionally drops metadata. Its authenticated API-key account and room scope are unchanged. Existing redacted error/idle handling remains; no automatic retry is added. +- Email's `generateEmailResponse` loads history through `getEmailRoomMessages`, + which checks raw stored content before extracting text and dropping metadata. + Marked history denies the entire generation, including sibling text; the + existing inbound handler retains its generic error response. Room lookup and + authenticated account/organization context are unchanged. - `getGeneralAgent` checks before account/knowledge/tool setup. Its actual `ToolLoopAgent.prepareStep` checks messages and tool results every iteration, and checks any tool-chain replacement messages. In the installed SDK this @@ -42,6 +54,13 @@ expansion, raw-report fallback, public route, grant or activation. ## Qualification +Stream-output regression tests invoke the actual `runAgentStep` with synthetic +stream/provider/cancellation seams. They cover running, cancelled and already +aborted runs, withheld sibling payloads, marked `onFinish` state, rejection of +pending provider promises during cleanup, and unchanged unmarked tool output. +This guard does not prevent the operation that produced the output or retract +independent prose already emitted; it denies recognized structured output only. + Tests invoke real workflow orchestration/conversion/routing/provider-step entry points with synthetic effect seams; replay cases supply the already-completed step outputs that those callers can receive. They are not a claim of live WDK diff --git a/lib/emails/inbound/__tests__/generateEmailResponse.test.ts b/lib/emails/inbound/__tests__/generateEmailResponse.test.ts index faa3b8e96..a287ed48b 100644 --- a/lib/emails/inbound/__tests__/generateEmailResponse.test.ts +++ b/lib/emails/inbound/__tests__/generateEmailResponse.test.ts @@ -3,14 +3,16 @@ import { generateEmailResponse } from "../generateEmailResponse"; import type { ChatRequestBody } from "@/lib/chat/validateChatRequest"; import getGeneralAgent from "@/lib/agents/generalAgent/getGeneralAgent"; -import { getEmailRoomMessages } from "@/lib/emails/inbound/getEmailRoomMessages"; +import selectMemories from "@/lib/supabase/memories/selectMemories"; +import { selectRoomWithArtist } from "@/lib/supabase/rooms/selectRoomWithArtist"; +import { FatalError } from "workflow"; vi.mock("@/lib/agents/generalAgent/getGeneralAgent", () => ({ default: vi.fn(), })); -vi.mock("@/lib/emails/inbound/getEmailRoomMessages", () => ({ - getEmailRoomMessages: vi.fn(), +vi.mock("@/lib/supabase/memories/selectMemories", () => ({ + default: vi.fn(), })); vi.mock("@/lib/emails/getEmailFooter", () => ({ @@ -43,7 +45,10 @@ describe("generateEmailResponse", () => { }); it("generates response with text and footer", async () => { - vi.mocked(getEmailRoomMessages).mockResolvedValue([{ role: "user", content: "Hi there" }]); + vi.mocked(selectMemories).mockResolvedValue([ + { content: { role: "user", parts: [{ type: "text", text: "Hi there" }] } }, + { content: { role: "assistant", parts: [{ type: "text", text: "Earlier reply" }] } }, + ] as never); const body: ChatRequestBody = { accountId: "acc-1", @@ -55,10 +60,51 @@ describe("generateEmailResponse", () => { const result = await generateEmailResponse(body); expect(mockGenerate).toHaveBeenCalledWith({ - messages: [{ role: "user", content: "Hi there" }], + messages: [ + { role: "user", content: "Hi there" }, + { role: "assistant", content: "Earlier reply" }, + ], }); + expect(selectMemories).toHaveBeenCalledExactlyOnceWith("room-1", { ascending: true }); + expect(getGeneralAgent).toHaveBeenCalledExactlyOnceWith(body); + expect(selectRoomWithArtist).toHaveBeenCalledExactlyOnceWith("room-1"); expect(result.text).toBe("Hello from assistant"); expect(result.html).toContain("Hello from assistant"); expect(result.html).toContain("
footer
"); }); + it.each(["part", "metadata"])( + "denies raw email history with a %s marker before text adaptation", + async location => { + const marker = { type: "data-retained-field-projection", private: "MARKER-SECRET" }; + const content = { + role: "assistant", + parts: [ + ...(location === "part" ? [marker] : []), + { type: "text", text: "PRIVATE-SIBLING-987654321" }, + ], + ...(location === "metadata" + ? { metadata: { contract: "catalog-retained-field-projection/v1" } } + : {}), + }; + const memories = [{ content }]; + const original = structuredClone(memories); + vi.mocked(selectMemories).mockResolvedValue(memories as never); + const body = { + accountId: "authenticated-account", + orgId: "authenticated-org", + roomId: "authorized-room", + messages: [], + } as ChatRequestBody; + const result = generateEmailResponse(body); + await expect(result).rejects.toBeInstanceOf(FatalError); + await expect(result).rejects.toThrow(/^Retained-field model input unavailable$/); + expect(mockGenerate).not.toHaveBeenCalled(); + expect(selectRoomWithArtist).not.toHaveBeenCalled(); + expect(selectMemories).toHaveBeenCalledExactlyOnceWith("authorized-room", { + ascending: true, + }); + expect(getGeneralAgent).toHaveBeenCalledExactlyOnceWith(body); + expect(memories).toEqual(original); + }, + ); }); diff --git a/lib/emails/inbound/getEmailRoomMessages.ts b/lib/emails/inbound/getEmailRoomMessages.ts index daf202756..8adae84e2 100644 --- a/lib/emails/inbound/getEmailRoomMessages.ts +++ b/lib/emails/inbound/getEmailRoomMessages.ts @@ -1,3 +1,4 @@ +import { assertNoRetainedFieldModelInput } from "@/lib/chat/retained-field/assertNoRetainedFieldModelInput"; import type { ModelMessage } from "ai"; import selectMemories from "@/lib/supabase/memories/selectMemories"; import { extractTextFromParts } from "./extractTextFromParts"; @@ -17,6 +18,8 @@ interface MemoryContent { export async function getEmailRoomMessages(roomId: string): Promise { const existingMemories = await selectMemories(roomId, { ascending: true }); if (!existingMemories) return []; + // Inspect stored content before text-only adaptation drops structured markers. + assertNoRetainedFieldModelInput(existingMemories.map(memory => memory.content)); const messages: ModelMessage[] = [];