From 7fd7ae0e60d9c3d17f6b704b5db18b67eaff6a74 Mon Sep 17 00:00:00 2001 From: bobbai00 Date: Mon, 16 Nov 2026 01:44:35 -0800 Subject: [PATCH] feat(agent-service): look up operator schemas with a tool Every model call carried the compact schema of each allowed operator type in the system prompt: 18.8 KB by default, 189.2 KB with all 168 types. The add tool also accepted every type, whatever the allowed list said. The prompt now names each usable type in one line, and the agent reads the properties of the types it needs with get_operator_schema, several at once. The prompt's list, the lookup and the add tool share one list of operator types. --- agent-service/src/agent/prompts.spec.ts | 53 +++++----- agent-service/src/agent/prompts.ts | 40 ++----- agent-service/src/agent/texera-agent.spec.ts | 71 ++++++++++++- agent-service/src/agent/texera-agent.ts | 10 +- agent-service/src/agent/tools/index.ts | 1 + .../src/agent/tools/operator-tools.spec.ts | 100 ++++++++++++++++++ .../src/agent/tools/operator-tools.ts | 67 ++++++++++++ .../src/agent/tools/workflow-crud-tools.ts | 2 +- 8 files changed, 282 insertions(+), 62 deletions(-) create mode 100644 agent-service/src/agent/tools/operator-tools.spec.ts create mode 100644 agent-service/src/agent/tools/operator-tools.ts diff --git a/agent-service/src/agent/prompts.spec.ts b/agent-service/src/agent/prompts.spec.ts index d4ae6e21516..deebb9fef6a 100644 --- a/agent-service/src/agent/prompts.spec.ts +++ b/agent-service/src/agent/prompts.spec.ts @@ -26,12 +26,7 @@ function makeOperatorSchema(operatorType: string, description: string): Operator return { operatorType, operatorVersion: "1.0", - jsonSchema: { - properties: { - condition: { type: "string" }, - }, - required: ["condition"], - }, + jsonSchema: { properties: { condition: { type: "string" } }, required: ["condition"] }, additionalMetadata: { userFriendlyName: operatorType, operatorGroupName: "Test", @@ -56,16 +51,26 @@ function makeMetadataStore(): WorkflowSystemMetadata { } describe("buildSystemPrompt", () => { - test("renders only explicitly allowed operators with their descriptions and compact schemas", () => { + test("names each allowed operator type in one line, and holds no schema", () => { const prompt = buildSystemPrompt(makeMetadataStore(), ["Filter"]); - expect(prompt).toContain("## Filter"); - expect(prompt).toContain("Description: Keeps rows that match a condition."); - expect(prompt).toMatch(/"condition":\s*\{\s*"type":\s*"string"/); - expect(prompt).not.toContain("## PythonUDFV2"); - expect(prompt).not.toContain("## RUDF"); - expect(prompt).not.toContain("## Python UDF Guide"); - expect(prompt).not.toContain("## R UDF Guide"); + expect(prompt).toContain("- Filter: Keeps rows that match a condition."); + expect(prompt).toContain("`get_operator_schema`"); + expect(prompt).not.toContain("PythonUDFV2:"); + expect(prompt).not.toContain('"condition"'); + expect(prompt).not.toContain("```json"); + }); + + test("names every known type when no allowlist is supplied", () => { + const prompt = buildSystemPrompt(makeMetadataStore()); + + expect(prompt).toContain("- Filter: Keeps rows that match a condition."); + expect(prompt).toContain("- PythonUDFV2: Runs user-defined Python code."); + expect(prompt).toContain("- RUDF: Runs user-defined R code."); + }); + + test("says so when no allowed type is known", () => { + expect(buildSystemPrompt(makeMetadataStore(), ["MissingOperator"])).toContain("No operator types are available."); }); test("adds guidance only for the UDF language in a restricted allowlist", () => { @@ -79,21 +84,11 @@ describe("buildSystemPrompt", () => { expect(rPrompt).not.toContain("## Python UDF Guide"); }); - test("uses all metadata operators and both UDF guides when no allowlist is supplied", () => { - const prompt = buildSystemPrompt(makeMetadataStore()); - - expect(prompt).toContain("## Filter"); - expect(prompt).toContain("## PythonUDFV2"); - expect(prompt).toContain("## RUDF"); - expect(prompt).toContain("## Python UDF Guide"); - expect(prompt).toContain("## R UDF Guide"); - }); - - test("reports that no operators are available when a restricted type is absent from metadata", () => { - const prompt = buildSystemPrompt(makeMetadataStore(), ["MissingOperator"]); + test("adds both UDF guides when no allowlist is supplied, and neither for other types", () => { + const metadataStore = makeMetadataStore(); - expect(prompt).toContain("No operators available."); - expect(prompt).not.toContain("## Python UDF Guide"); - expect(prompt).not.toContain("## R UDF Guide"); + expect(buildSystemPrompt(metadataStore)).toContain("## Python UDF Guide"); + expect(buildSystemPrompt(metadataStore)).toContain("## R UDF Guide"); + expect(buildSystemPrompt(metadataStore, ["Filter"])).not.toContain("UDF Guide"); }); }); diff --git a/agent-service/src/agent/prompts.ts b/agent-service/src/agent/prompts.ts index 064eed2e3e5..b2462670b2d 100644 --- a/agent-service/src/agent/prompts.ts +++ b/agent-service/src/agent/prompts.ts @@ -17,7 +17,8 @@ * under the License. */ -import { WorkflowSystemMetadata } from "./util/workflow-system-metadata"; +import { usableOperatorTypes } from "./tools/operator-tools"; +import type { WorkflowSystemMetadata } from "./util/workflow-system-metadata"; const PYTHON_UDF_OPERATOR_TYPES = ["PythonUDFV2"]; const R_UDF_OPERATOR_TYPES = ["RUDF"]; @@ -249,40 +250,21 @@ Result: - **Handle messy data files**: Load data files directly in a single operator. Real-world data files are often malformed — they may have wrong delimiters, missing or misplaced headers, metadata/comment rows, or multiple tables in one file. After loading, inspect the result. If column names look auto-generated (e.g., \`Unnamed: 0\`) or a data value appears as a header, adjust the loading parameters (e.g., \`header=\`, \`skiprows=\`, \`sep=\`) by modifying the data loading operator. - **Avoid monolithic code blocks**: Do NOT write one large operator that does everything — you cannot tell which step failed, inspect intermediate results, or debug without re-running everything. Instead, decompose into separate operators each doing ONE thing (e.g., filter → join → aggregate → filter → join → final filter). Each can be executed and verified independently. -## Available Operators +## Operators -You have the following operators available: +These are the operator types you can use. Before you add operators of types you have not used yet, read their properties with \`get_operator_schema\`, once for all of them. -{{OPERATOR_SCHEMA}} +{{OPERATORS}} `; -function buildAllowedOperatorSchemas( - metadataStore: WorkflowSystemMetadata, - allowedOperatorTypes: string[] = [] -): string { - const schemas: string[] = []; - - const operatorTypes = - allowedOperatorTypes.length > 0 ? allowedOperatorTypes : Object.keys(metadataStore.getAllOperatorTypes()); - - for (const operatorType of operatorTypes) { - const compactSchema = metadataStore.getCompactSchema(operatorType); - const description = metadataStore.getDescription(operatorType); - - if (compactSchema) { - schemas.push( - `## ${operatorType}\n` + - (description ? `Description: ${description}\n` : "") + - `Schema:\n\`\`\`json\n${JSON.stringify(compactSchema, null, 2)}\n\`\`\`` - ); - } - } - - return schemas.length > 0 ? schemas.join("\n\n") : "No operators available."; +/** One line per operator type the agent may use: its name and what it does. */ +function listOperatorTypes(metadataStore: WorkflowSystemMetadata, allowedOperatorTypes: string[]): string { + const types = usableOperatorTypes(metadataStore, allowedOperatorTypes); + if (types.length === 0) return "No operator types are available."; + return types.map(type => `- ${type}: ${metadataStore.getDescription(type)}`).join("\n"); } export function buildSystemPrompt(metadataStore: WorkflowSystemMetadata, allowedOperatorTypes: string[] = []): string { - const operatorSchemas = buildAllowedOperatorSchemas(metadataStore, allowedOperatorTypes); const allowsAll = allowedOperatorTypes.length === 0; const pythonAllowed = allowsAll || allowedOperatorTypes.some(t => PYTHON_UDF_OPERATOR_TYPES.includes(t)); const rAllowed = allowsAll || allowedOperatorTypes.some(t => R_UDF_OPERATOR_TYPES.includes(t)); @@ -291,6 +273,6 @@ export function buildSystemPrompt(metadataStore: WorkflowSystemMetadata, allowed if (pythonAllowed) extraSections.push(PYTHON_UDF_INSTRUCTIONS); if (rAllowed) extraSections.push(R_UDF_INSTRUCTIONS); - const base = SYSTEM_PROMPT_TEMPLATE.replace("{{OPERATOR_SCHEMA}}", operatorSchemas); + const base = SYSTEM_PROMPT_TEMPLATE.replace("{{OPERATORS}}", listOperatorTypes(metadataStore, allowedOperatorTypes)); return extraSections.length > 0 ? `${base}\n${extraSections.join("\n\n")}\n` : base; } diff --git a/agent-service/src/agent/texera-agent.spec.ts b/agent-service/src/agent/texera-agent.spec.ts index a1644921cfe..1494c168e92 100644 --- a/agent-service/src/agent/texera-agent.spec.ts +++ b/agent-service/src/agent/texera-agent.spec.ts @@ -491,7 +491,12 @@ describe("sendMessage", () => { }); expect(call.abortSignal).toBeInstanceOf(AbortSignal); expect(call.abortSignal.aborted).toBe(false); - expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator", "modifyOperator"]); + expect(call.tools.map((t: any) => t.name)).toEqual([ + "get_operator_schema", + "deleteOperator", + "addOperator", + "modifyOperator", + ]); expect(call.prompt[0]).toEqual({ role: "system", content: "SYS-XYZ" }); const txt = call.prompt[1].content[0].text; expect(txt).toContain("# Ongoing Task"); @@ -683,8 +688,9 @@ describe("sendMessage", () => { agent.updateSettings({ disabledTools: new Set(["modifyOperator"]) }); await agent.sendMessage("go"); const call = (model as any).doGenerateCalls[0]; - expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator"]); + expect(call.tools.map((t: any) => t.name)).toEqual(["get_operator_schema", "deleteOperator", "addOperator"]); expect(agent.getSystemInfo().tools.map(t => [t.name, t.enabled])).toEqual([ + ["get_operator_schema", true], ["deleteOperator", true], ["addOperator", true], ["modifyOperator", false], @@ -729,6 +735,67 @@ describe("sendMessage", () => { expect(agent.getWorkflowState().getOperator("op1")).toBeDefined(); }); + test("the prompt, the lookup and the add tool share one list of operator types", async () => { + const saved = (WorkflowSystemMetadata as any).instance; + (WorkflowSystemMetadata as any).instance = undefined; + try { + const op = (operatorType: string, description: string) => ({ + operatorType, + jsonSchema: { type: "object", properties: {} }, + additionalMetadata: { + userFriendlyName: operatorType, + operatorDescription: description, + inputPorts: [], + outputPorts: [{}], + }, + }); + WorkflowSystemMetadata.getInstance().loadFromMetadata({ + operators: [op("CSVFileScan", "Read a CSV file"), op("Aggregate", "Aggregate by group")], + } as any); + let n = 0; + const calls = [ + { toolName: "get_operator_schema", input: { operatorTypes: ["Aggregate"] } }, + { + toolName: "addOperator", + input: { operatorId: "op1", operatorType: "Aggregate", properties: {}, summary: "s" }, + }, + ]; + const model = new MockLanguageModelV4({ + doGenerate: async () => { + const next = calls[n++]; + if (next) + return { + content: [ + { type: "tool-call", toolCallId: "c" + n, toolName: next.toolName, input: JSON.stringify(next.input) }, + ], + finishReason: finish("tool-calls"), + usage: usage(1, 1), + warnings: [], + } as any; + return { + content: [{ type: "text", text: "x" }], + finishReason: finish("stop"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); + const agent = makeAgentWith(model); + agent.updateSettings({ allowedOperatorTypes: ["CSVFileScan"] }); + await agent.sendMessage("add an aggregate"); + const [lookedUp, added] = agent.getAllSteps().slice(1, 3); + expect(agent.getSystemInfo().systemPrompt).toContain("- CSVFileScan: Read a CSV file"); + expect(agent.getSystemInfo().systemPrompt).not.toContain("- Aggregate:"); + expect(lookedUp.toolResults?.[0]).toMatchObject({ isError: true }); + expect(String(added.toolResults?.[0]?.output)).toStartWith( + '[ERROR] Unknown operator type: "Aggregate". Available types: CSVFileScan.' + ); + expect(agent.getWorkflowState().getAllOperators()).toEqual([]); + } finally { + (WorkflowSystemMetadata as any).instance = saved; + } + }); + test("caps the loop at maxSteps", async () => { // The cap is the only thing standing between a looping model and an unbounded run. let n = 0; diff --git a/agent-service/src/agent/texera-agent.ts b/agent-service/src/agent/texera-agent.ts index 555cc2730ff..6be0b08aedc 100644 --- a/agent-service/src/agent/texera-agent.ts +++ b/agent-service/src/agent/texera-agent.ts @@ -47,6 +47,11 @@ import { TOOL_NAME_EXECUTE_OPERATOR, type ExecutionConfig, } from "./tools/workflow-execution-tools"; +import { + createGetOperatorSchemaTool, + TOOL_NAME_GET_OPERATOR_SCHEMA, + usableOperatorTypes, +} from "./tools/operator-tools"; import { assembleContext } from "./util/context-utils"; import { compileWorkflowAsync, type WorkflowCompilationResponse } from "../api/compile-api"; import { createLogger } from "../logger"; @@ -199,8 +204,10 @@ export class TexeraAgent { } private createTools(): Record { + // One list of operator types: what the prompt names and the lookup describes is what can be added. + const usableTypes = usableOperatorTypes(this.metadataStore, this.settings.allowedOperatorTypes); const operatorSchemas = new Map(); - for (const type of Object.keys(this.metadataStore.getAllOperatorTypes())) { + for (const type of usableTypes) { const jsonSchema = this.metadataStore.getSchema(type); const additionalMetadata = this.metadataStore.getAdditionalMetadata(type); if (jsonSchema) { @@ -220,6 +227,7 @@ export class TexeraAgent { }; const tools: Record = { + [TOOL_NAME_GET_OPERATOR_SCHEMA]: createGetOperatorSchemaTool(this.metadataStore, usableTypes), [TOOL_NAME_DELETE_OPERATOR]: createDeleteOperatorTool(this.workflowState, context), [TOOL_NAME_ADD_OPERATOR]: createAddOperatorTool(this.workflowState, operatorSchemas, context), [TOOL_NAME_MODIFY_OPERATOR]: createModifyOperatorTool(this.workflowState, context), diff --git a/agent-service/src/agent/tools/index.ts b/agent-service/src/agent/tools/index.ts index 7e2d9570703..ec61c81f652 100644 --- a/agent-service/src/agent/tools/index.ts +++ b/agent-service/src/agent/tools/index.ts @@ -20,3 +20,4 @@ export * from "./tools-utility"; export * from "./workflow-crud-tools"; export * from "./workflow-execution-tools"; +export * from "./operator-tools"; diff --git a/agent-service/src/agent/tools/operator-tools.spec.ts b/agent-service/src/agent/tools/operator-tools.spec.ts new file mode 100644 index 00000000000..9e733a94a1a --- /dev/null +++ b/agent-service/src/agent/tools/operator-tools.spec.ts @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, expect, test } from "bun:test"; +import type { OperatorSchema } from "../../api/backend-api"; +import { WorkflowSystemMetadata } from "../util/workflow-system-metadata"; +import { createGetOperatorSchemaTool, usableOperatorTypes } from "./operator-tools"; + +function schema(operatorType: string, friendly: string, group: string, description: string): OperatorSchema { + return { + operatorType, + operatorVersion: "1.0", + jsonSchema: { + properties: { + groupByKeys: { type: "array", items: { type: "string" } }, + dummyPropertyList: { type: "array" }, + }, + required: ["groupByKeys"], + }, + additionalMetadata: { + userFriendlyName: friendly, + operatorGroupName: group, + operatorDescription: description, + inputPorts: [], + outputPorts: [], + }, + }; +} + +function metadata(): WorkflowSystemMetadata { + const store = new WorkflowSystemMetadata(); + store.loadFromMetadata({ + operators: [ + schema("Aggregate", "Aggregate", "Data Cleaning", "Calculate aggregates by group"), + schema("HashJoin", "Hash Join", "Join", "Join two inputs on a key"), + schema("Sort", "Sort", "Data Cleaning", "Sort rows by columns"), + ], + groups: [], + }); + return store; +} + +const run = async (agentTool: any, args: Record) => + agentTool.execute(args, { toolCallId: "test-call", messages: [] }); + +describe("usableOperatorTypes", () => { + test("is every known type when none are allowed by name", () => { + expect(usableOperatorTypes(metadata(), []).sort()).toEqual(["Aggregate", "HashJoin", "Sort"]); + }); + + test("keeps the allowed types the metadata knows, and drops the ones it does not", () => { + expect(usableOperatorTypes(metadata(), ["Sort", "Missing", "Aggregate"])).toEqual(["Sort", "Aggregate"]); + }); +}); + +describe("get_operator_schema", () => { + const store = metadata(); + const groupBy = { groupByKeys: { type: "array", items: { type: "string" } } }; + + test("gives each type's description and compact schema, without placeholder properties", async () => { + const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, [])); + expect(JSON.parse(await run(get, { operatorTypes: ["Aggregate", "Sort"] }))).toEqual({ + Aggregate: { description: "Calculate aggregates by group", properties: groupBy, required: ["groupByKeys"] }, + Sort: { description: "Sort rows by columns", properties: groupBy, required: ["groupByKeys"] }, + }); + }); + + test.each([ + ["an unknown type", ["Pivot"], [], "Pivot"], + ["a type that is not allowed", ["Sort", "HashJoin"], ["Sort"], "HashJoin"], + ["a type in another case", ["aggregate"], [], "aggregate"], + ])("refuses %s, naming it", async (_, operatorTypes, allowed, refused) => { + const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, allowed as string[])); + await expect(run(get, { operatorTypes })).rejects.toThrow( + `Not operator types you can use: ${refused}. Use the types the system prompt lists.` + ); + }); + + test("takes at least one type", async () => { + const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, [])); + expect(get.inputSchema).toBeDefined(); + expect((get.inputSchema as any).safeParse({ operatorTypes: [] }).success).toBe(false); + }); +}); diff --git a/agent-service/src/agent/tools/operator-tools.ts b/agent-service/src/agent/tools/operator-tools.ts new file mode 100644 index 00000000000..21aeaa398fe --- /dev/null +++ b/agent-service/src/agent/tools/operator-tools.ts @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// The operator lookup. The system prompt names each operator type the agent may use, one line +// each; the properties of a type are read through `get_operator_schema` when the agent needs them, +// rather than every allowed type's schema riding along on every call. + +import { tool } from "ai"; +import { z } from "zod"; +import type { WorkflowSystemMetadata } from "../util/workflow-system-metadata"; + +export const TOOL_NAME_GET_OPERATOR_SCHEMA = "get_operator_schema"; + +/** + * The operator types the agent may use: the allowed ones the metadata knows, or every type it + * knows when none are named. The prompt's list, the lookup and the add tool all go by this one list. + */ +export function usableOperatorTypes(metadataStore: WorkflowSystemMetadata, allowed: string[]): string[] { + const known = Object.keys(metadataStore.getAllOperatorTypes()); + return allowed.length === 0 ? known : allowed.filter(type => known.includes(type)); +} + +export function createGetOperatorSchemaTool(metadataStore: WorkflowSystemMetadata, usableTypes: string[]) { + return tool({ + description: + "Get the properties of one or more operator types: which are required, and each one's type, allowed " + + "values and default. Call it, once for all the types you need, before you add operators of types you " + + "have not used in this conversation.", + inputSchema: z.object({ + operatorTypes: z + .array(z.string()) + .min(1) + .describe("The operator types, as the system prompt names them, such as Aggregate"), + }), + execute: async ({ operatorTypes }) => { + const unknown = operatorTypes.filter(type => !usableTypes.includes(type)); + if (unknown.length > 0) { + throw new Error( + `Not operator types you can use: ${unknown.join(", ")}. Use the types the system prompt lists.` + ); + } + const schemas = Object.fromEntries( + operatorTypes.map(type => [ + type, + { description: metadataStore.getDescription(type), ...metadataStore.getCompactSchema(type) }, + ]) + ); + return JSON.stringify(schemas); + }, + }); +} diff --git a/agent-service/src/agent/tools/workflow-crud-tools.ts b/agent-service/src/agent/tools/workflow-crud-tools.ts index 1933b086aa1..418711c2a24 100644 --- a/agent-service/src/agent/tools/workflow-crud-tools.ts +++ b/agent-service/src/agent/tools/workflow-crud-tools.ts @@ -109,7 +109,7 @@ export function createAddOperatorTool( const workflowUtil = context?.metadataStore ? new WorkflowUtilService(context.metadataStore, workflowState) : null; return tool({ - description: `Add a new operator to the workflow. Use getOperatorSchema first to understand required properties. + description: `Add a new operator to the workflow. Use get_operator_schema first to understand required properties. Examples: 1. Add a source operator (no inputs):