diff --git a/agent-service/src/agent/prompts.spec.ts b/agent-service/src/agent/prompts.spec.ts index d4ae6e21516..7d370eef348 100644 --- a/agent-service/src/agent/prompts.spec.ts +++ b/agent-service/src/agent/prompts.spec.ts @@ -89,6 +89,19 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain("## R UDF Guide"); }); + test("describes the dataflow block that ends each step, and no format for the conversation", () => { + // The conversation reaches the model as native messages, so the prompt describes only the one + // block the service writes: the current dataflow. + const prompt = buildSystemPrompt(makeMetadataStore(), ["Filter"]); + + expect(prompt).toContain("## Current Dataflow"); + expect(prompt).toContain("### Operator `` (, executed|failed|not-executed)"); + expect(prompt).not.toContain("## Context Format"); + expect(prompt).not.toContain("# Completed Tasks"); + expect(prompt).not.toContain("# Ongoing Task"); + expect(prompt).not.toContain("### Turn"); + }); + test("reports that no operators are available when a restricted type is absent from metadata", () => { const prompt = buildSystemPrompt(makeMetadataStore(), ["MissingOperator"]); diff --git a/agent-service/src/agent/prompts.ts b/agent-service/src/agent/prompts.ts index 064eed2e3e5..da14f05db08 100644 --- a/agent-service/src/agent/prompts.ts +++ b/agent-service/src/agent/prompts.ts @@ -161,60 +161,11 @@ Dataflow represents data analysis as a DAG (directed acyclic graph) where: - Each operator receives table(s) from input operator(s), processes them, and outputs a single table - The output table can be viewed via execution, or passed to downstream operators via links -## Context Format +## Current Dataflow -Your conversation context is a single message with three top-level sections, in this order: - -- \`# Completed Tasks\` — previous tasks you've already finished (omitted if none) -- \`# Ongoing Task\` — the current task, including turns you've taken so far -- \`# Current Dataflow\` — the live DAG: every operator's current state - -**Overall layout:** +Before each of your steps, the conversation ends with a message that starts with \`# Current Dataflow\` (none while the workflow is empty). The system writes it, not the user, and it always shows the workflow as it is now: \`\`\` -# Completed Tasks - -## Task (completed) - -### User request - - - -### Turn 1 -Thought: -- (succeeded) - - Summary: - - Output: - -## Task (completed) - -### User request - - - -### Turn 1 -... - -# Ongoing Task -## Task (ongoing) - -### User request - - - -### Turn 1 -Thought: ... -- (succeeded) - - Summary: ... - - Output: ... - -### Turn 2 -Thought: ... -- (failed) - - Summary: ... - - Error: - - # Current Dataflow ## Operators @@ -237,7 +188,7 @@ Result: ## Key Principles -- **Call tools only through the native protocol**: Invoke tools using the tool-call mechanism. Never emit \`\`, \`\`, \`\`, or any other tag-like structures in your response — those shapes appear in your input to describe past turns and existing state, never in your output. +- **Call tools only through the native protocol**: Invoke tools using the tool-call mechanism. Never write a tool call, a thought or an operator as text, and never emit \`\`, \`\`, \`\`, or any other tag-like structures in your response. - **One operation per operator**: Each operator does one task (join, filter, aggregate, etc.). Use links to connect them. - **Build incrementally**: Link new operators to existing ones. Never recreate data already in the workflow. - **Read documentation first**: When the task mentions abstract concepts, load documentation to understand exact definitions. diff --git a/agent-service/src/agent/texera-agent.spec.ts b/agent-service/src/agent/texera-agent.spec.ts index a1644921cfe..90031c2c99e 100644 --- a/agent-service/src/agent/texera-agent.spec.ts +++ b/agent-service/src/agent/texera-agent.spec.ts @@ -391,6 +391,67 @@ const textModel = (text: string, i = 11, o = 7) => }), }); +/** + * The prompt one model call received, reduced to what these tests compare: the system text, each + * user message's text, the assistant's text and tool calls, and each tool result with its tool. + */ +function promptOf(call: any): any[] { + return call.prompt.map((m: any) => { + switch (m.role) { + case "system": + return { system: m.content }; + case "user": + return { user: m.content.map((p: any) => p.text).join("") }; + case "assistant": + return { + assistant: m.content.map((p: any) => (p.type === "text" ? p.text : { call: p.toolName, input: p.input })), + }; + case "tool": + return { tool: m.content.map((p: any) => ({ result: p.toolName, output: p.output })) }; + default: + return m; + } + }); +} + +/** The tool results in a list of model messages, each as its tool's name and what the model reads. */ +function toolOutputsOf(messages: any[] | undefined): any[] { + return (messages ?? []) + .filter(m => m.role === "tool") + .flatMap(m => m.content.map((p: any) => [p.toolName, p.output])); +} + +/** A model that answers each call in turn from the given list: a tool call, or a final text. */ +function scriptedModel(replies: Array<{ call: string; input: Record } | string>) { + let n = 0; + return new MockLanguageModelV4({ + doGenerate: async () => { + const reply = replies[Math.min(n++, replies.length - 1)]; + if (typeof reply === "string") + return { + content: [{ type: "text", text: reply }], + finishReason: finish("stop"), + usage: usage(1, 1), + warnings: [], + } as any; + return { + content: [{ type: "tool-call", toolCallId: "c" + n, toolName: reply.call, input: JSON.stringify(reply.input) }], + finishReason: finish("tool-calls"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); +} + +/** Lets the compile request succeed with no schemas, so a workflow with operators compiles. */ +function compileOk() { + fetchSpy.mockImplementation((async (u: any) => { + urls.push(String(u)); + return { ok: true, json: async () => ({ operatorOutputSchemas: {}, operatorErrors: {} }) } as any; + }) as any); +} + /** Sibling of makeAgent() that takes a stand-in model, so sendMessage runs with no network. * Every agent built here is tracked and destroyed from `afterEach` rather than per test: a * failing assertion would skip an in-test `destroy()`, and a pending auto-persist debounce @@ -478,7 +539,7 @@ describe("sendMessage", () => { expect(urls).toEqual([]); }); - test("pinned call options + assembled context replaces the raw message", async () => { + test("pins the call options and sends the user's message as it is", async () => { const model = textModel("ok"); const agent = makeAgentWith(model); await agent.sendMessage("raw-user-text"); @@ -492,11 +553,69 @@ describe("sendMessage", () => { expect(call.abortSignal).toBeInstanceOf(AbortSignal); expect(call.abortSignal.aborted).toBe(false); expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator", "modifyOperator"]); - expect(call.prompt[0]).toEqual({ role: "system", content: "SYS-XYZ" }); - const txt = call.prompt[1].content[0].text; - expect(txt).toContain("# Ongoing Task"); - expect(txt).toContain("raw-user-text"); - expect(agent.getAllSteps()[1].inputMessages).toEqual([{ role: "user", content: txt }]); + // An empty workflow has no dataflow block, so the user's words are the whole input. + expect(promptOf(call)).toEqual([{ system: "SYS-XYZ" }, { user: "raw-user-text" }]); + expect(agent.getAllSteps()[1].inputMessages).toEqual([{ role: "user", content: "raw-user-text" }]); + }); + + test("keeps a tool call and its whole result as native messages for the next step", async () => { + const model = scriptedModel([{ call: "deleteOperator", input: { operatorId: "ghost" } }, "done"]); + const agent = makeAgentWith(model); + await agent.sendMessage("delete it"); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([ + { system: "SYS-XYZ" }, + { user: "delete it" }, + { assistant: [{ call: "deleteOperator", input: { operatorId: "ghost" } }] }, + { + tool: [{ result: "deleteOperator", output: { type: "text", value: "[ERROR] Operator ghost not found" } }], + }, + ]); + // What the step detail shows is what the model received. + expect(agent.getAllSteps()[2].inputMessages).toEqual([ + { role: "user", content: "delete it" }, + { + role: "assistant", + content: [{ type: "tool-call", toolCallId: "c1", toolName: "deleteOperator", input: { operatorId: "ghost" } }], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "c1", + toolName: "deleteOperator", + output: { type: "text", value: "[ERROR] Operator ghost not found" }, + }, + ], + }, + ]); + }); + + test("ends every step with the current dataflow, rewritten each time and never kept", async () => { + const model = scriptedModel([{ call: "deleteOperator", input: { operatorId: "ghost" } }, "done", "again"]); + const agent = makeAgentWith(model); + agent.getWorkflowState().addOperator(srcOp("op1") as any); + compileOk(); + await agent.sendMessage("first"); + await agent.sendMessage("second"); + const calls = (model as any).doGenerateCalls.map(promptOf); + for (const prompt of calls) { + const blocks = prompt.filter((m: any) => m.user?.startsWith("# Current Dataflow")); + expect(blocks.length).toBe(1); + expect(prompt[prompt.length - 1]).toBe(blocks[0]); + expect(blocks[0].user).toContain("### Operator `op1` (CSVFileScan, not-executed)"); + } + // The second turn reads the first one as it happened, without the blocks of its steps. + expect(calls[2].slice(0, -1)).toEqual([ + { system: "SYS-XYZ" }, + { user: "first" }, + { assistant: [{ call: "deleteOperator", input: { operatorId: "ghost" } }] }, + { + tool: [{ result: "deleteOperator", output: { type: "text", value: "[ERROR] Operator ghost not found" } }], + }, + { assistant: ["done"] }, + { user: "second" }, + ]); }); test("two-step tool run: per-step + summed usage, tool projection, rolling snapshots", async () => { @@ -546,7 +665,9 @@ describe("sendMessage", () => { [2, false, true], ]); expect(steps[2].parentId).toBe(steps[1].id); - expect(steps[2].inputMessages?.[0]?.content).toContain("- deleteOperator (failed)"); + expect(toolOutputsOf(steps[2].inputMessages)).toEqual([ + ["deleteOperator", { type: "text", value: "[ERROR] Operator ghost not found" }], + ]); expect(urls).toEqual([]); }); @@ -596,50 +717,56 @@ describe("sendMessage", () => { ["agent", 0, 0], ]); expect(steps[1].toolResults).toEqual([{ toolCallId: "c1", output: "Deleted operator: op1", isError: false }]); - expect(steps[2].inputMessages?.[0]?.content).toContain("- deleteOperator (succeeded)"); + expect(toolOutputsOf(steps[2].inputMessages)).toEqual([ + ["deleteOperator", { type: "text", value: "Deleted operator: op1" }], + ]); }); - test.each([new Error("The operation was aborted."), "tool failed", null])( - "preserves a thrown tool error in stored JSON and subsequent context: %p", - async thrown => { - let n = 0; - const model = new MockLanguageModelV4({ - doGenerate: async () => { - n++; - return { - content: - n === 1 - ? [ - { - type: "tool-call", - toolCallId: "c1", - toolName: "deleteOperator", - input: JSON.stringify({ operatorId: "ghost" }), - }, - ] - : [{ type: "text", text: "done" }], - finishReason: finish(n === 1 ? "tool-calls" : "stop"), - usage: usage(1, 1), - warnings: [], - } as any; - }, - }); - const agent = makeAgentWith(model); - const throwingTool = spyOn((agent as any).tools.deleteOperator, "execute").mockImplementation(async () => { - throw thrown; - }); - try { - await agent.sendMessage("delete it"); - const steps = agent.getAllSteps(); - const expected = thrown instanceof Error ? thrown.message : String(thrown); - expect(steps[1].toolResults).toEqual([{ toolCallId: "c1", output: expected, isError: true }]); - expect(JSON.parse(JSON.stringify(steps))[1].toolResults[0].output).toBe(expected); - expect(steps[2].inputMessages?.[0]?.content).toContain(` - Error:\n ${expected}`); - } finally { - throwingTool.mockRestore(); - } + // Each thrown value, with the SDK's own wording of it, which is what the model reads. + test.each([ + [new Error("The operation was aborted."), "Error: The operation was aborted."], + ["tool failed", "tool failed"], + [null, "unknown error"], + ])("preserves a thrown tool error in stored JSON and subsequent context: %p", async (thrown, readByModel) => { + let n = 0; + const model = new MockLanguageModelV4({ + doGenerate: async () => { + n++; + return { + content: + n === 1 + ? [ + { + type: "tool-call", + toolCallId: "c1", + toolName: "deleteOperator", + input: JSON.stringify({ operatorId: "ghost" }), + }, + ] + : [{ type: "text", text: "done" }], + finishReason: finish(n === 1 ? "tool-calls" : "stop"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); + const agent = makeAgentWith(model); + const throwingTool = spyOn((agent as any).tools.deleteOperator, "execute").mockImplementation(async () => { + throw thrown; + }); + try { + await agent.sendMessage("delete it"); + const steps = agent.getAllSteps(); + const expected = thrown instanceof Error ? thrown.message : String(thrown); + expect(steps[1].toolResults).toEqual([{ toolCallId: "c1", output: expected, isError: true }]); + expect(JSON.parse(JSON.stringify(steps))[1].toolResults[0].output).toBe(expected); + expect(toolOutputsOf(steps[2].inputMessages)).toEqual([ + ["deleteOperator", { type: "error-text", value: readByModel }], + ]); + } finally { + throwingTool.mockRestore(); } - ); + }); test("a length-limited tool call is not executed and appears as failed on the next turn", async () => { let n = 0; @@ -670,8 +797,27 @@ describe("sendMessage", () => { await agent.sendMessage("delete it"); expect(execute).not.toHaveBeenCalled(); expect(agent.getAllSteps()[1].toolResults).toEqual([]); - await agent.sendMessage("continue"); - expect(agent.getAllSteps().at(-1)?.inputMessages?.[0]?.content).toContain("- deleteOperator (failed)"); + // The SDK refuses a conversation in which a call has no result, so the step is kept with a + // failed one, and the next turn runs. + const res = await agent.sendMessage("continue"); + expect(res.error).toBeUndefined(); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([ + { system: "SYS-XYZ" }, + { user: "delete it" }, + { assistant: [{ call: "deleteOperator", input: { operatorId: "ghost" } }] }, + { + tool: [ + { + result: "deleteOperator", + output: { + type: "error-text", + value: 'This call did not run: the response that made it ended with finish reason "length".', + }, + }, + ], + }, + { user: "continue" }, + ]); } finally { execute.mockRestore(); } @@ -758,7 +904,7 @@ describe("sendMessage", () => { expect(agent.getAllSteps().map(s => s.stepId)).toEqual([0, 1, 2]); }); - test("second turn chains on and reads turn 1 as completed", async () => { + test("second turn chains on and reads turn 1 as native messages", async () => { const model = textModel("a", 1, 1); const agent = makeAgentWith(model); await agent.sendMessage("one"); @@ -770,10 +916,12 @@ describe("sendMessage", () => { expect(steps[0].messageId).not.toBe(steps[2].messageId); expect(steps[2].messageId).toMatch(/^msg-agent-1-2-\d+$/); expect(steps[3].id).toMatch(/^step-agent-1-4-\d+$/); - const txt = (model as any).doGenerateCalls[1].prompt[1].content[0].text; - expect(txt).toContain("# Completed Tasks"); - expect(txt).toContain("## Task (completed)"); - expect(txt).toContain("# Ongoing Task"); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([ + { system: "SYS-XYZ" }, + { user: "one" }, + { assistant: ["a"] }, + { user: "two" }, + ]); }); test("abandoned branch is invisible to the model", async () => { @@ -784,12 +932,93 @@ describe("sendMessage", () => { await agent.sendMessage("first"); (agent as any).head = agent.getAllSteps()[0].id; await agent.sendMessage("second"); - const txt = (model as any).doGenerateCalls[1].prompt[1].content[0].text; - expect(txt).not.toContain("### Turn 1"); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([ + { system: "SYS-XYZ" }, + { user: "first" }, + { user: "second" }, + ]); expect(agent.getAllSteps().length).toBe(4); expect(agent.getVisibleReActSteps().length).toBe(3); }); + test("a stopped turn keeps its finished steps and tells the next turn it was stopped", async () => { + let agent!: TexeraAgent; + let n = 0; + const model = new MockLanguageModelV4({ + doGenerate: async () => { + n++; + if (n === 1) { + agent.stop(); + return { + content: [ + { + type: "tool-call", + toolCallId: "c1", + toolName: "deleteOperator", + input: JSON.stringify({ operatorId: "ghost" }), + }, + ], + finishReason: finish("tool-calls"), + usage: usage(1, 1), + warnings: [], + } as any; + } + return { + content: [{ type: "text", text: "ok" }], + finishReason: finish("stop"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); + agent = makeAgentWith(model); + await agent.sendMessage("go"); + await agent.sendMessage("next"); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([ + { system: "SYS-XYZ" }, + { user: "go" }, + { assistant: [{ call: "deleteOperator", input: { operatorId: "ghost" } }] }, + { + tool: [{ result: "deleteOperator", output: { type: "text", value: "[ERROR] Operator ghost not found" } }], + }, + { assistant: ["Generation stopped by user."] }, + { user: "next" }, + ]); + }); + + test("a failed turn's error is part of the next turn's history", async () => { + let n = 0; + const model = new MockLanguageModelV4({ + doGenerate: async () => { + if (n++ === 0) throw new Error("nope"); + return { + content: [{ type: "text", text: "ok" }], + finishReason: finish("stop"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); + const agent = makeAgentWith(model); + await agent.sendMessage("one"); + await agent.sendMessage("two"); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([ + { system: "SYS-XYZ" }, + { user: "one" }, + { assistant: ["Error: nope"] }, + { user: "two" }, + ]); + }); + + test("clearHistory leaves nothing of the earlier turns for the model", async () => { + const model = textModel("a", 1, 1); + const agent = makeAgentWith(model); + await agent.sendMessage("one"); + agent.clearHistory(); + await agent.sendMessage("two"); + expect(promptOf((model as any).doGenerateCalls[1])).toEqual([{ system: "SYS-XYZ" }, { user: "two" }]); + }); + test("compiles the DAG and feeds schemas + cached results into the prompt", async () => { const model = textModel("ok", 1, 1); const agent = makeAgentWith(model); @@ -808,7 +1037,10 @@ describe("sendMessage", () => { } as any; }) as any); const res = await agent.sendMessage("look"); - const txt = (model as any).doGenerateCalls[0].prompt[1].content[0].text; + const prompt = promptOf((model as any).doGenerateCalls[0]); + expect(prompt.slice(0, 2)).toEqual([{ system: "SYS-XYZ" }, { user: "look" }]); + const txt = prompt[2].user; + expect(txt).toStartWith("# Current Dataflow\n## Operators"); expect(res.error).toBeUndefined(); expect(urls).toEqual(["http://localhost:9090/api/compile"]); expect(compileHeaders.Authorization).toBe("Bearer user-token"); @@ -1238,7 +1470,12 @@ describe("delegate mode", () => { const step = agent.getAllSteps()[1]; expect(step.toolResults).toHaveLength(1); expect(step.toolResults?.[0]).toMatchObject({ toolCallId: "c1", isError: true }); - expect(agent.getAllSteps()[2].inputMessages?.[0]?.content).toContain("- modifyOperator (failed)"); + expect(toolOutputsOf(agent.getAllSteps()[2].inputMessages)).toEqual([ + [ + "modifyOperator", + { type: "error-text", value: expect.stringContaining("Invalid input for tool modifyOperator") }, + ], + ]); expect(urls.some(u => u.includes("/api/execution/"))).toBe(false); }); diff --git a/agent-service/src/agent/texera-agent.ts b/agent-service/src/agent/texera-agent.ts index 555cc2730ff..6dfdf6a08f9 100644 --- a/agent-service/src/agent/texera-agent.ts +++ b/agent-service/src/agent/texera-agent.ts @@ -17,7 +17,7 @@ * under the License. */ -import { generateText, type ModelMessage, type LanguageModel, stepCountIs } from "ai"; +import { ToolLoopAgent, isStepCount, type ModelMessage, type LanguageModel } from "ai"; import { Subscription } from "rxjs"; import { debounceTime } from "rxjs/operators"; import { WorkflowState } from "./workflow-state"; @@ -47,7 +47,7 @@ import { TOOL_NAME_EXECUTE_OPERATOR, type ExecutionConfig, } from "./tools/workflow-execution-tools"; -import { assembleContext } from "./util/context-utils"; +import { dataflowMessage, unexecutedToolCallsMessage } from "./util/context-utils"; import { compileWorkflowAsync, type WorkflowCompilationResponse } from "../api/compile-api"; import { createLogger } from "../logger"; import type { Logger } from "pino"; @@ -78,11 +78,12 @@ type ReActStepCallback = (step: ReActStep) => void; /** * A single Texera agent instance. * - * Owns the conversation (ReAct step tree with HEAD tracking), the - * workflow being edited (`WorkflowState`), cached operator execution results - * (`WorkflowResultState`), and the tool surface exposed to the LLM. Each call - * to `sendMessage` drives one multi-step generation via the Vercel AI SDK, - * streaming step updates to subscribed clients. + * Owns the conversation (ReAct step tree with HEAD tracking, each step keeping + * the model messages it added), the workflow being edited (`WorkflowState`), + * cached operator execution results (`WorkflowResultState`), and the tool + * surface exposed to the LLM. Each call to `sendMessage` drives one multi-step + * run of the AI SDK's `ToolLoopAgent` over those messages, streaming step + * updates to subscribed clients. */ export class TexeraAgent { readonly agentId: string; @@ -107,6 +108,12 @@ export class TexeraAgent { private reActStepsByMessageId: Map = new Map(); + /** + * The model messages each step added, by step id: a user step's message, an agent step's + * assistant message and tool results, or the note a stopped or failed turn ends with. + */ + private messagesByStep: Map = new Map(); + private currentMessageId: string | undefined = undefined; private delegateConfig?: { @@ -314,7 +321,8 @@ export class TexeraAgent { return `step-${this.agentId}-${++this.stepCounter}-${Date.now()}`; } - private addStep(step: ReActStep): void { + /** Records a step and the model messages it adds to the conversation. */ + private addStep(step: ReActStep, messages: ModelMessage[] = []): void { let steps = this.reActStepsByMessageId.get(step.messageId); if (!steps) { steps = []; @@ -322,11 +330,39 @@ export class TexeraAgent { } steps.push(step); this.stepsById.set(step.id, step); + this.messagesByStep.set(step.id, messages); if (this.stepCallback) { this.stepCallback(step); } } + /** + * The conversation as the model reads it: the messages of the steps on the path from the root to + * HEAD, in order. A branch HEAD has left is not on that path, so the model never sees it. + */ + private getConversation(): ModelMessage[] { + return this.getVisibleReActSteps().flatMap(step => this.messagesByStep.get(step.id) ?? []); + } + + /** + * The block that ends each step's messages: the workflow as it stands now, compiled for its + * schemas, with the results this branch has seen. None while the workflow is empty. + */ + private async currentDataflowMessage(): Promise { + let compilationResult: WorkflowCompilationResponse | null = null; + if (this.workflowState.getAllOperators().length > 0) { + try { + compilationResult = await compileWorkflowAsync( + this.workflowState.toLogicalPlan(), + this.delegateConfig?.userToken ?? this.userToken + ); + } catch (e: any) { + this.log.warn({ err: e?.message || e }, "compilation failed; proceeding without schemas"); + } + } + return dataflowMessage(this.workflowState, this.getFormattedResultsForDAG(), false, compilationResult); + } + getSystemInfo(): { systemPrompt: string; tools: Array<{ name: string; description: string; inputSchema: any; enabled: boolean }>; @@ -539,48 +575,27 @@ export class TexeraAgent { totalTokens: estimatedInputTokens, }, }; - this.addStep(userStep); + this.addStep(userStep, [{ role: "user", content: userMessage }]); this.head = userStepId; let isFirstStep = true; let lastPreparedMessages: ModelMessage[] | undefined; - // Pass only the current user turn; prepareStep rebuilds full context each step - // (historical interactions + DAG + this message). - const currentUserMessage: ModelMessage[] = [{ role: "user", content: userMessage }]; - const result = await generateText({ + const loop = new ToolLoopAgent({ model: this.model, - system: this.systemPrompt, - messages: currentUserMessage, + instructions: this.systemPrompt, tools: this.getEnabledTools(), temperature: 0.2, - stopWhen: stepCountIs(this.settings.maxSteps), - prepareStep: async ({ stepNumber, messages: currentMessages }) => { - let compilationResult: WorkflowCompilationResponse | null = null; - if (this.workflowState.getAllOperators().length > 0) { - try { - const logicalPlan = this.workflowState.toLogicalPlan(); - compilationResult = await compileWorkflowAsync( - logicalPlan, - this.delegateConfig?.userToken ?? this.userToken - ); - } catch (e: any) { - this.log.warn({ err: e?.message || e }, "compilation failed; proceeding without schemas"); - } - } - - const visibleSteps = this.getVisibleReActSteps(); - const processed = assembleContext( - visibleSteps, - this.workflowState, - this.getFormattedResultsForDAG(), - false, - compilationResult - ); - lastPreparedMessages = processed; - return { messages: processed }; + stopWhen: isStepCount(this.settings.maxSteps), + // Each step reads the conversation as it happened, then the workflow as it stands now. The + // dataflow block is written again for every step, so it is rebuilt from the turn's messages + // rather than returned on top of the previous step's, which would carry the old block along. + prepareStep: async ({ initialMessages, responseMessages }) => { + const dataflow = await this.currentDataflowMessage(); + const messages = [...initialMessages, ...responseMessages, ...(dataflow ? [dataflow] : [])]; + lastPreparedMessages = messages; + return { messages }; }, - abortSignal: this.abortController?.signal, // reasoning_effort is configured per-model in litellm-config.yaml via extra_body // to bypass LiteLLM's param validation — do not pass it here. providerOptions: { @@ -588,7 +603,12 @@ export class TexeraAgent { anthropic: { disableParallelToolUse: true }, mistral: { parallelToolCalls: false }, }, - onStepFinish: async ({ text, toolCalls, content, usage }) => { + }); + + const result = await loop.generate({ + messages: this.getConversation(), + abortSignal: this.abortController?.signal, + onStepEnd: async ({ text, toolCalls, content, usage, response, finishReason }) => { stepIndex++; const formattedToolCalls = toolCalls?.map(tc => ({ @@ -638,7 +658,8 @@ export class TexeraAgent { afterWorkflowContent: afterStepContent, }; lastPreparedMessages = undefined; - this.addStep(agentStep); + const unexecutedCalls = unexecutedToolCallsMessage(toolCalls, toolResultsByCallId, finishReason); + this.addStep(agentStep, unexecutedCalls ? [...response.messages, unexecutedCalls] : response.messages); this.head = agentStepId; const execConfig = this.buildExecutionConfig(); @@ -710,7 +731,8 @@ export class TexeraAgent { isBegin: false, isEnd: true, }; - this.addStep(stoppedStep); + // The next turn reads that this one was stopped, as the chat shows it. + this.addStep(stoppedStep, [{ role: "assistant", content: stoppedStep.content }]); this.head = stoppedStepId; return { @@ -734,7 +756,7 @@ export class TexeraAgent { isBegin: false, isEnd: true, }; - this.addStep(errorStep); + this.addStep(errorStep, [{ role: "assistant", content: errorStep.content }]); this.head = errorStepId; return { @@ -770,6 +792,7 @@ export class TexeraAgent { clearHistory(): void { this.reActStepsByMessageId.clear(); this.stepsById.clear(); + this.messagesByStep.clear(); this.currentMessageId = undefined; this.head = INITIAL_STEP_ID; const initialStep: ReActStep = { @@ -862,6 +885,7 @@ export class TexeraAgent { this.reActStepsByMessageId.clear(); this.stepsById.clear(); + this.messagesByStep.clear(); this.currentMessageId = undefined; } } diff --git a/agent-service/src/agent/util/context-utils.spec.ts b/agent-service/src/agent/util/context-utils.spec.ts index 83e82d278de..0154dfae3ad 100644 --- a/agent-service/src/agent/util/context-utils.spec.ts +++ b/agent-service/src/agent/util/context-utils.spec.ts @@ -19,26 +19,12 @@ import { describe, expect, test } from "bun:test"; import type { ModelMessage } from "ai"; -import { assembleContext } from "./context-utils"; +import { dataflowMessage } from "./context-utils"; import { WorkflowState } from "../workflow-state"; -import type { ReActStep } from "../../types/agent"; import type { OperatorLink, OperatorPredicate } from "../../types/workflow"; import type { WorkflowCompilationResponse } from "../../api/compile-api"; import { WorkflowFatalErrorType } from "../../types/execution"; -function step(messageId: string, role: "user" | "agent", stepId: number, content: string, isEnd: boolean): ReActStep { - return { - id: `${messageId}-${stepId}`, - messageId, - stepId, - timestamp: 0, - role, - content, - isBegin: true, - isEnd, - }; -} - function makeOperator(id: string, overrides: Partial = {}): OperatorPredicate { return { operatorID: id, @@ -60,138 +46,29 @@ function makeLink(source: string, sourcePort: string, target: string, targetPort }; } -// assembleContext always returns a single user message whose content is a joined string. -function contentOf(result: ModelMessage[]): string { - expect(result).toHaveLength(1); - expect(result[0].role).toBe("user"); - expect(typeof result[0].content).toBe("string"); - return result[0].content as string; +// The dataflow block is one user message whose content is a string. +function contentOf(message: ModelMessage | null): string { + expect(message).not.toBeNull(); + expect(message!.role).toBe("user"); + expect(typeof message!.content).toBe("string"); + return message!.content as string; } -describe("assembleContext", () => { - test("returns a single user message with no task sections for empty steps", () => { - const result = assembleContext([], new WorkflowState(), new Map()); - expect(result).toHaveLength(1); - expect(result[0].role).toBe("user"); - expect(contentOf(result)).toBe(""); - }); - - test("renders a completed task under a Completed Tasks heading", () => { - const steps = [step("m1", "user", 0, "do X", true), step("m1", "agent", 1, "thinking", true)]; - const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); - expect(content).toContain("# Completed Tasks"); - expect(content).toContain("## Task (completed)"); - expect(content).toContain("do X"); // the user request - expect(content).toContain("### Turn 1"); // the agent step +describe("dataflowMessage", () => { + test("is absent for an empty workflow", () => { + expect(dataflowMessage(new WorkflowState(), new Map())).toBeNull(); }); - test("renders an unfinished task under an Ongoing Task heading", () => { - const steps = [step("m1", "user", 0, "do Y", true), step("m1", "agent", 1, "working", false)]; - const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); - expect(content).toContain("# Ongoing Task"); - expect(content).toContain("## Task (ongoing)"); - }); - - test("appends a Current Dataflow section when the workflow has operators", () => { + test("is one user message headed Current Dataflow, with the operators under it", () => { const workflowState = new WorkflowState(); workflowState.addOperator(makeOperator("op1")); - const content = contentOf(assembleContext([], workflowState, new Map())); - expect(content).toContain("# Current Dataflow"); - expect(content).toContain("## Operators"); - }); - - test("groups steps sharing a messageId into a single task", () => { - const steps = [ - step("m1", "user", 0, "req", true), - step("m1", "agent", 1, "turn one", false), - step("m1", "agent", 2, "turn two", true), - ]; - const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); - expect((content.match(/## Task/g) ?? []).length).toBe(1); - expect(content).toContain("### Turn 1"); - expect(content).toContain("### Turn 2"); - }); - - test("labels each tool call with the status of its result by toolCallId", () => { - const agentStep: ReActStep = { - ...step("m1", "agent", 1, "acting", true), - toolCalls: [ - { toolName: "addOperator", toolCallId: "c1", input: {} }, - { toolName: "executeOperator", toolCallId: "c2", input: {} }, - { toolName: "deleteOperator", toolCallId: "c3", input: {} }, - ], - // Results are deliberately out of call order, and the third call was never executed. - toolResults: [ - { toolCallId: "c2", output: "bad", isError: true }, - { toolCallId: "c1", output: "ok" }, - ], - }; - const content = contentOf( - assembleContext([step("m1", "user", 0, "req", true), agentStep], new WorkflowState(), new Map()) - ); - expect(content).toContain("- addOperator (succeeded)"); - expect(content).toContain("- executeOperator (failed)"); - expect(content).toContain("- deleteOperator (failed)"); - }); - - test("writes each past call's summary and result under it, so a failed call is not repeated blindly", () => { - const agentStep: ReActStep = { - ...step("m1", "agent", 1, "acting", true), - toolCalls: [ - { toolName: "addOperator", toolCallId: "c1", input: { operatorId: "op1", summary: "Load CSV data" } }, - { toolName: "executeOperator", toolCallId: "c2", input: { operatorId: "op1" } }, - { toolName: "modifyOperator", toolCallId: "c3", input: { operatorId: "op1", summary: "Link op7" } }, - { toolName: "deleteOperator", toolCallId: "c4", input: { operatorId: "op1" } }, - ], - toolResults: [ - { toolCallId: "c1", output: "Added operator op1, input ports: 0, output ports: 1" }, - { toolCallId: "c2", output: "Executed operator op1\nOutput(table shape): (3, 2)\n| a | b |" }, - { toolCallId: "c3", output: '[ERROR] Source operator "op7" not found\n at resolveInputLinks', isError: true }, - ], - }; - const content = contentOf( - assembleContext([step("m1", "user", 0, "req", true), agentStep], new WorkflowState(), new Map()) - ); - expect(content).toContain( - [ - "- addOperator (succeeded)", - " - Summary: Load CSV data", - " - Output: Added operator op1, input ports: 0, output ports: 1", - "- executeOperator (succeeded)", - " - Output: Executed operator op1", - "- modifyOperator (failed)", - " - Summary: Link op7", - " - Error:", - ' [ERROR] Source operator "op7" not found', - " at resolveInputLinks", - // A call the SDK never executed has no result. - "- deleteOperator (failed)", - ].join("\n") - ); - // The operator's data stays in the dataflow section, not in the history. - expect(content).not.toContain("| a | b |"); - }); - - test("matches results to calls by toolCallId, not by position", () => { - // A call the SDK never executed has no result, so positions can shift. - const agentStep: ReActStep = { - ...step("m1", "agent", 1, "acting", true), - toolCalls: [ - { toolName: "addOperator", toolCallId: "c1", input: { operatorId: "op1" } }, - { toolName: "deleteOperator", toolCallId: "c2", input: { operatorId: "op0" } }, - ], - toolResults: [{ toolCallId: "c2", output: "Deleted operator op0" }], - }; - const content = contentOf( - assembleContext([step("m1", "user", 0, "req", true), agentStep], new WorkflowState(), new Map()) - ); - expect(content).toContain( - ["- addOperator (failed)", "- deleteOperator (succeeded)", " - Output: Deleted operator op0"].join("\n") - ); + const content = contentOf(dataflowMessage(workflowState, new Map())); + expect(content).toStartWith("# Current Dataflow\n## Operators\n"); + expect(content).toContain("### Operator `op1` (TestOp, not-executed)"); }); }); -describe("assembleContext — dataflow serialization", () => { +describe("dataflowMessage — serialization", () => { test("orders operators and links topologically rather than by insertion order", () => { const workflowState = new WorkflowState(); // Inserted leaf-first, so insertion order is the reverse of the dataflow order. @@ -202,7 +79,7 @@ describe("assembleContext — dataflow serialization", () => { workflowState.addLink(makeLink("a", "output-0", "c", "input-1")); workflowState.addLink(makeLink("a", "output-0", "b", "input-0")); - const content = contentOf(assembleContext([], workflowState, new Map())); + const content = contentOf(dataflowMessage(workflowState, new Map())); expect(content.indexOf("### Operator `a`")).toBeLessThan(content.indexOf("### Operator `b`")); expect(content.indexOf("### Operator `b`")).toBeLessThan(content.indexOf("### Operator `c`")); @@ -231,7 +108,7 @@ describe("assembleContext — dataflow serialization", () => { operatorErrors: {}, }; - const content = contentOf(assembleContext([], workflowState, new Map(), false, compilationResult)); + const content = contentOf(dataflowMessage(workflowState, new Map(), false, compilationResult)); expect(content).toContain("Input Schema (port 0): [a: string, n: integer]"); expect(content).toContain("Output Schema: [a: string, n: integer]"); @@ -248,7 +125,7 @@ describe("assembleContext — dataflow serialization", () => { }) ); - const content = contentOf(assembleContext([], workflowState, new Map())); + const content = contentOf(dataflowMessage(workflowState, new Map())); expect(content).toContain("Summary: op1"); // no display name -> falls back to the id expect(content).toContain(" name: alice"); @@ -263,13 +140,13 @@ describe("assembleContext — dataflow serialization", () => { const workflowState = new WorkflowState(); workflowState.addOperator(makeOperator("op1", { operatorProperties: { secret: "s3cr3t" } })); - const redacted = contentOf(assembleContext([], workflowState, new Map(), true)); + const redacted = contentOf(dataflowMessage(workflowState, new Map(), true)); expect(redacted).toContain("(TestOp, not-executed)"); expect(redacted).not.toContain("Properties:"); // A failing operator keeps its properties even when redacting, so the model // can see what caused the failure. - const failed = contentOf(assembleContext([], workflowState, new Map([["op1", "[ERROR] boom"]]), true)); + const failed = contentOf(dataflowMessage(workflowState, new Map([["op1", "[ERROR] boom"]]), true)); expect(failed).toContain("(TestOp, failed)"); expect(failed).toContain(" secret: s3cr3t"); }); @@ -278,7 +155,7 @@ describe("assembleContext — dataflow serialization", () => { const workflowState = new WorkflowState(); workflowState.addOperator(makeOperator("op1", { customDisplayName: "My Filter" })); - const content = contentOf(assembleContext([], workflowState, new Map([["op1", "header\nrow1"]]))); + const content = contentOf(dataflowMessage(workflowState, new Map([["op1", "header\nrow1"]]))); expect(content).toContain("### Operator `op1` (TestOp, executed)"); expect(content).toContain("Summary: My Filter"); @@ -302,7 +179,7 @@ describe("assembleContext — dataflow serialization", () => { }, }; - const content = contentOf(assembleContext([], workflowState, new Map(), false, compilationResult)); + const content = contentOf(dataflowMessage(workflowState, new Map(), false, compilationResult)); expect(content).toContain("Compilation Error: attribute x not found"); }); diff --git a/agent-service/src/agent/util/context-utils.ts b/agent-service/src/agent/util/context-utils.ts index f5078e27063..e4b10734b8d 100644 --- a/agent-service/src/agent/util/context-utils.ts +++ b/agent-service/src/agent/util/context-utils.ts @@ -24,139 +24,51 @@ import type { ModelMessage } from "ai"; import type { WorkflowState } from "../workflow-state"; import type { OperatorPredicate, OperatorPortSchemaMap, PortSchema } from "../../types/workflow"; -import type { ReActStep } from "../../types/agent"; import type { WorkflowCompilationResponse, WorkflowFatalError } from "../../api/compile-api"; import { extractOperatorInputPortSchemaMap } from "./workflow-utils"; -import { createLogger } from "../../logger"; -const log = createLogger("ContextAssembler"); - -export function assembleContext( - visibleSteps: ReActStep[], +/** + * The message that ends what the model reads before each step: the workflow as it stands now, with + * each operator's state, schemas and cached result, then the links. It is written again for every + * step and never kept in the conversation, so the model always sees one current copy. An empty + * workflow has none. + */ +export function dataflowMessage( workflowState: WorkflowState, operatorExecutionResults: Map, useRedact: boolean = false, compilationResult?: WorkflowCompilationResponse | null -): ModelMessage[] { - const messageIds: string[] = []; - const stepsByMessage = new Map(); - for (const step of visibleSteps) { - let group = stepsByMessage.get(step.messageId); - if (!group) { - group = []; - stepsByMessage.set(step.messageId, group); - messageIds.push(step.messageId); - } - group.push(step); - } - - const sections: string[] = []; - let completedCount = 0; - let hasOngoing = false; - - for (const msgId of messageIds) { - const steps = stepsByMessage.get(msgId)!; - // A task is completed only when an *agent* step has isEnd=true; user steps - // always have isEnd=true because they are single-step messages. - const isCompleted = steps.some(s => s.role === "agent" && s.isEnd); - - if (isCompleted) { - if (completedCount === 0) { - sections.push("# Completed Tasks"); - } - sections.push(""); - sections.push(serializeTask(steps, "completed")); - completedCount++; - } else { - hasOngoing = true; - sections.push(""); - sections.push("# Ongoing Task"); - sections.push(serializeTask(steps, "ongoing")); - sections.push(""); - sections.push( - "Above is user's request and the steps you already took. You as an assistant please keep working on solving user's request based on the progress of current workflow." - ); - } - } - +): ModelMessage | null { const dagSection = serializeDag(workflowState, operatorExecutionResults, useRedact, compilationResult); - if (dagSection) { - sections.push(""); - sections.push("# Current Dataflow"); - sections.push(dagSection); - } - - const content = sections.join("\n"); - - log.debug( - { - completed: completedCount, - ongoing: hasOngoing ? 1 : 0, - operatorResults: operatorExecutionResults.size, - useRedact, - }, - "built context" - ); - - return [{ role: "user", content }]; -} - -function serializeTask(steps: ReActStep[], status: "completed" | "ongoing"): string { - const lines: string[] = []; - lines.push(`## Task (${status})`); - lines.push(""); - - const userStep = steps.find(s => s.role === "user"); - const assistantSteps = steps.filter(s => s.role === "agent"); - - if (userStep) { - lines.push("### User request"); - lines.push(""); - lines.push(userStep.content); - lines.push(""); - } - - for (const step of assistantSteps) { - lines.push(`### Turn ${step.stepId}`); - if (step.content) { - lines.push(`Thought: ${step.content}`); - } - if (step.toolCalls && step.toolCalls.length > 0) { - const toolResultsByCallId = new Map(step.toolResults?.map(result => [result.toolCallId, result])); - for (const tc of step.toolCalls) { - const tr = toolResultsByCallId.get(tc.toolCallId); - const statusAttr = !tr || tr.isError ? "failed" : "succeeded"; - lines.push(`- ${tc.toolName} (${statusAttr})`); - lines.push(...serializeToolCallDetails(tc.input, tr)); - } - } - lines.push(""); - } - - return lines.join("\n").trimEnd(); + if (!dagSection) return null; + return { role: "user", content: `# Current Dataflow\n${dagSection}` }; } /** - * The `Summary` / `Output` / `Error` lines under a past tool call, in the layout the system prompt - * describes. Output keeps only its first line: an operator's data is already in the dataflow - * section, and the first line is the tool's one-line result. An error is kept whole, since it is - * what the model needs to avoid repeating the call. + * A failed result for each of a step's tool calls that has none, or nothing when every call has + * one. The SDK runs a step's calls only when the response ended normally, not when it was cut off + * at the output limit or by a content filter, and it refuses a conversation in which a call has no + * result. Kept as it is, such a step would fail every later turn. */ -function serializeToolCallDetails(input: any, result: { output: any; isError?: boolean } | undefined): string[] { - const lines: string[] = []; - if (typeof input?.summary === "string" && input.summary) { - lines.push(` - Summary: ${input.summary}`); - } - if (!result) return lines; - const text = typeof result.output === "string" ? result.output : JSON.stringify(result.output); - if (text === undefined || text === "") return lines; - if (result.isError) { - lines.push(" - Error:"); - lines.push(...text.split("\n").map(line => ` ${line}`)); - } else { - lines.push(` - Output: ${text.split("\n")[0]}`); - } - return lines; +export function unexecutedToolCallsMessage( + toolCalls: Array<{ toolCallId: string; toolName: string }>, + resultsByCallId: ReadonlyMap, + finishReason: string +): ModelMessage | null { + const unexecuted = toolCalls.filter(call => !resultsByCallId.has(call.toolCallId)); + if (unexecuted.length === 0) return null; + return { + role: "tool", + content: unexecuted.map(call => ({ + type: "tool-result", + toolCallId: call.toolCallId, + toolName: call.toolName, + output: { + type: "error-text", + value: `This call did not run: the response that made it ended with finish reason "${finishReason}".`, + }, + })), + }; } function serializeDag(