diff --git a/agent-service/src/agent/prompts.spec.ts b/agent-service/src/agent/prompts.spec.ts index d4ae6e21516..deebb9fef6a 100644 --- a/agent-service/src/agent/prompts.spec.ts +++ b/agent-service/src/agent/prompts.spec.ts @@ -26,12 +26,7 @@ function makeOperatorSchema(operatorType: string, description: string): Operator return { operatorType, operatorVersion: "1.0", - jsonSchema: { - properties: { - condition: { type: "string" }, - }, - required: ["condition"], - }, + jsonSchema: { properties: { condition: { type: "string" } }, required: ["condition"] }, additionalMetadata: { userFriendlyName: operatorType, operatorGroupName: "Test", @@ -56,16 +51,26 @@ function makeMetadataStore(): WorkflowSystemMetadata { } describe("buildSystemPrompt", () => { - test("renders only explicitly allowed operators with their descriptions and compact schemas", () => { + test("names each allowed operator type in one line, and holds no schema", () => { const prompt = buildSystemPrompt(makeMetadataStore(), ["Filter"]); - expect(prompt).toContain("## Filter"); - expect(prompt).toContain("Description: Keeps rows that match a condition."); - expect(prompt).toMatch(/"condition":\s*\{\s*"type":\s*"string"/); - expect(prompt).not.toContain("## PythonUDFV2"); - expect(prompt).not.toContain("## RUDF"); - expect(prompt).not.toContain("## Python UDF Guide"); - expect(prompt).not.toContain("## R UDF Guide"); + expect(prompt).toContain("- Filter: Keeps rows that match a condition."); + expect(prompt).toContain("`get_operator_schema`"); + expect(prompt).not.toContain("PythonUDFV2:"); + expect(prompt).not.toContain('"condition"'); + expect(prompt).not.toContain("```json"); + }); + + test("names every known type when no allowlist is supplied", () => { + const prompt = buildSystemPrompt(makeMetadataStore()); + + expect(prompt).toContain("- Filter: Keeps rows that match a condition."); + expect(prompt).toContain("- PythonUDFV2: Runs user-defined Python code."); + expect(prompt).toContain("- RUDF: Runs user-defined R code."); + }); + + test("says so when no allowed type is known", () => { + expect(buildSystemPrompt(makeMetadataStore(), ["MissingOperator"])).toContain("No operator types are available."); }); test("adds guidance only for the UDF language in a restricted allowlist", () => { @@ -79,21 +84,11 @@ describe("buildSystemPrompt", () => { expect(rPrompt).not.toContain("## Python UDF Guide"); }); - test("uses all metadata operators and both UDF guides when no allowlist is supplied", () => { - const prompt = buildSystemPrompt(makeMetadataStore()); - - expect(prompt).toContain("## Filter"); - expect(prompt).toContain("## PythonUDFV2"); - expect(prompt).toContain("## RUDF"); - expect(prompt).toContain("## Python UDF Guide"); - expect(prompt).toContain("## R UDF Guide"); - }); - - test("reports that no operators are available when a restricted type is absent from metadata", () => { - const prompt = buildSystemPrompt(makeMetadataStore(), ["MissingOperator"]); + test("adds both UDF guides when no allowlist is supplied, and neither for other types", () => { + const metadataStore = makeMetadataStore(); - expect(prompt).toContain("No operators available."); - expect(prompt).not.toContain("## Python UDF Guide"); - expect(prompt).not.toContain("## R UDF Guide"); + expect(buildSystemPrompt(metadataStore)).toContain("## Python UDF Guide"); + expect(buildSystemPrompt(metadataStore)).toContain("## R UDF Guide"); + expect(buildSystemPrompt(metadataStore, ["Filter"])).not.toContain("UDF Guide"); }); }); diff --git a/agent-service/src/agent/prompts.ts b/agent-service/src/agent/prompts.ts index 064eed2e3e5..b2462670b2d 100644 --- a/agent-service/src/agent/prompts.ts +++ b/agent-service/src/agent/prompts.ts @@ -17,7 +17,8 @@ * under the License. */ -import { WorkflowSystemMetadata } from "./util/workflow-system-metadata"; +import { usableOperatorTypes } from "./tools/operator-tools"; +import type { WorkflowSystemMetadata } from "./util/workflow-system-metadata"; const PYTHON_UDF_OPERATOR_TYPES = ["PythonUDFV2"]; const R_UDF_OPERATOR_TYPES = ["RUDF"]; @@ -249,40 +250,21 @@ Result: - **Handle messy data files**: Load data files directly in a single operator. Real-world data files are often malformed — they may have wrong delimiters, missing or misplaced headers, metadata/comment rows, or multiple tables in one file. After loading, inspect the result. If column names look auto-generated (e.g., \`Unnamed: 0\`) or a data value appears as a header, adjust the loading parameters (e.g., \`header=\`, \`skiprows=\`, \`sep=\`) by modifying the data loading operator. - **Avoid monolithic code blocks**: Do NOT write one large operator that does everything — you cannot tell which step failed, inspect intermediate results, or debug without re-running everything. Instead, decompose into separate operators each doing ONE thing (e.g., filter → join → aggregate → filter → join → final filter). Each can be executed and verified independently. -## Available Operators +## Operators -You have the following operators available: +These are the operator types you can use. Before you add operators of types you have not used yet, read their properties with \`get_operator_schema\`, once for all of them. -{{OPERATOR_SCHEMA}} +{{OPERATORS}} `; -function buildAllowedOperatorSchemas( - metadataStore: WorkflowSystemMetadata, - allowedOperatorTypes: string[] = [] -): string { - const schemas: string[] = []; - - const operatorTypes = - allowedOperatorTypes.length > 0 ? allowedOperatorTypes : Object.keys(metadataStore.getAllOperatorTypes()); - - for (const operatorType of operatorTypes) { - const compactSchema = metadataStore.getCompactSchema(operatorType); - const description = metadataStore.getDescription(operatorType); - - if (compactSchema) { - schemas.push( - `## ${operatorType}\n` + - (description ? `Description: ${description}\n` : "") + - `Schema:\n\`\`\`json\n${JSON.stringify(compactSchema, null, 2)}\n\`\`\`` - ); - } - } - - return schemas.length > 0 ? schemas.join("\n\n") : "No operators available."; +/** One line per operator type the agent may use: its name and what it does. */ +function listOperatorTypes(metadataStore: WorkflowSystemMetadata, allowedOperatorTypes: string[]): string { + const types = usableOperatorTypes(metadataStore, allowedOperatorTypes); + if (types.length === 0) return "No operator types are available."; + return types.map(type => `- ${type}: ${metadataStore.getDescription(type)}`).join("\n"); } export function buildSystemPrompt(metadataStore: WorkflowSystemMetadata, allowedOperatorTypes: string[] = []): string { - const operatorSchemas = buildAllowedOperatorSchemas(metadataStore, allowedOperatorTypes); const allowsAll = allowedOperatorTypes.length === 0; const pythonAllowed = allowsAll || allowedOperatorTypes.some(t => PYTHON_UDF_OPERATOR_TYPES.includes(t)); const rAllowed = allowsAll || allowedOperatorTypes.some(t => R_UDF_OPERATOR_TYPES.includes(t)); @@ -291,6 +273,6 @@ export function buildSystemPrompt(metadataStore: WorkflowSystemMetadata, allowed if (pythonAllowed) extraSections.push(PYTHON_UDF_INSTRUCTIONS); if (rAllowed) extraSections.push(R_UDF_INSTRUCTIONS); - const base = SYSTEM_PROMPT_TEMPLATE.replace("{{OPERATOR_SCHEMA}}", operatorSchemas); + const base = SYSTEM_PROMPT_TEMPLATE.replace("{{OPERATORS}}", listOperatorTypes(metadataStore, allowedOperatorTypes)); return extraSections.length > 0 ? `${base}\n${extraSections.join("\n\n")}\n` : base; } diff --git a/agent-service/src/agent/texera-agent.spec.ts b/agent-service/src/agent/texera-agent.spec.ts index a1644921cfe..1494c168e92 100644 --- a/agent-service/src/agent/texera-agent.spec.ts +++ b/agent-service/src/agent/texera-agent.spec.ts @@ -491,7 +491,12 @@ describe("sendMessage", () => { }); expect(call.abortSignal).toBeInstanceOf(AbortSignal); expect(call.abortSignal.aborted).toBe(false); - expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator", "modifyOperator"]); + expect(call.tools.map((t: any) => t.name)).toEqual([ + "get_operator_schema", + "deleteOperator", + "addOperator", + "modifyOperator", + ]); expect(call.prompt[0]).toEqual({ role: "system", content: "SYS-XYZ" }); const txt = call.prompt[1].content[0].text; expect(txt).toContain("# Ongoing Task"); @@ -683,8 +688,9 @@ describe("sendMessage", () => { agent.updateSettings({ disabledTools: new Set(["modifyOperator"]) }); await agent.sendMessage("go"); const call = (model as any).doGenerateCalls[0]; - expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator"]); + expect(call.tools.map((t: any) => t.name)).toEqual(["get_operator_schema", "deleteOperator", "addOperator"]); expect(agent.getSystemInfo().tools.map(t => [t.name, t.enabled])).toEqual([ + ["get_operator_schema", true], ["deleteOperator", true], ["addOperator", true], ["modifyOperator", false], @@ -729,6 +735,67 @@ describe("sendMessage", () => { expect(agent.getWorkflowState().getOperator("op1")).toBeDefined(); }); + test("the prompt, the lookup and the add tool share one list of operator types", async () => { + const saved = (WorkflowSystemMetadata as any).instance; + (WorkflowSystemMetadata as any).instance = undefined; + try { + const op = (operatorType: string, description: string) => ({ + operatorType, + jsonSchema: { type: "object", properties: {} }, + additionalMetadata: { + userFriendlyName: operatorType, + operatorDescription: description, + inputPorts: [], + outputPorts: [{}], + }, + }); + WorkflowSystemMetadata.getInstance().loadFromMetadata({ + operators: [op("CSVFileScan", "Read a CSV file"), op("Aggregate", "Aggregate by group")], + } as any); + let n = 0; + const calls = [ + { toolName: "get_operator_schema", input: { operatorTypes: ["Aggregate"] } }, + { + toolName: "addOperator", + input: { operatorId: "op1", operatorType: "Aggregate", properties: {}, summary: "s" }, + }, + ]; + const model = new MockLanguageModelV4({ + doGenerate: async () => { + const next = calls[n++]; + if (next) + return { + content: [ + { type: "tool-call", toolCallId: "c" + n, toolName: next.toolName, input: JSON.stringify(next.input) }, + ], + finishReason: finish("tool-calls"), + usage: usage(1, 1), + warnings: [], + } as any; + return { + content: [{ type: "text", text: "x" }], + finishReason: finish("stop"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); + const agent = makeAgentWith(model); + agent.updateSettings({ allowedOperatorTypes: ["CSVFileScan"] }); + await agent.sendMessage("add an aggregate"); + const [lookedUp, added] = agent.getAllSteps().slice(1, 3); + expect(agent.getSystemInfo().systemPrompt).toContain("- CSVFileScan: Read a CSV file"); + expect(agent.getSystemInfo().systemPrompt).not.toContain("- Aggregate:"); + expect(lookedUp.toolResults?.[0]).toMatchObject({ isError: true }); + expect(String(added.toolResults?.[0]?.output)).toStartWith( + '[ERROR] Unknown operator type: "Aggregate". Available types: CSVFileScan.' + ); + expect(agent.getWorkflowState().getAllOperators()).toEqual([]); + } finally { + (WorkflowSystemMetadata as any).instance = saved; + } + }); + test("caps the loop at maxSteps", async () => { // The cap is the only thing standing between a looping model and an unbounded run. let n = 0; diff --git a/agent-service/src/agent/texera-agent.ts b/agent-service/src/agent/texera-agent.ts index 555cc2730ff..6be0b08aedc 100644 --- a/agent-service/src/agent/texera-agent.ts +++ b/agent-service/src/agent/texera-agent.ts @@ -47,6 +47,11 @@ import { TOOL_NAME_EXECUTE_OPERATOR, type ExecutionConfig, } from "./tools/workflow-execution-tools"; +import { + createGetOperatorSchemaTool, + TOOL_NAME_GET_OPERATOR_SCHEMA, + usableOperatorTypes, +} from "./tools/operator-tools"; import { assembleContext } from "./util/context-utils"; import { compileWorkflowAsync, type WorkflowCompilationResponse } from "../api/compile-api"; import { createLogger } from "../logger"; @@ -199,8 +204,10 @@ export class TexeraAgent { } private createTools(): Record { + // One list of operator types: what the prompt names and the lookup describes is what can be added. + const usableTypes = usableOperatorTypes(this.metadataStore, this.settings.allowedOperatorTypes); const operatorSchemas = new Map(); - for (const type of Object.keys(this.metadataStore.getAllOperatorTypes())) { + for (const type of usableTypes) { const jsonSchema = this.metadataStore.getSchema(type); const additionalMetadata = this.metadataStore.getAdditionalMetadata(type); if (jsonSchema) { @@ -220,6 +227,7 @@ export class TexeraAgent { }; const tools: Record = { + [TOOL_NAME_GET_OPERATOR_SCHEMA]: createGetOperatorSchemaTool(this.metadataStore, usableTypes), [TOOL_NAME_DELETE_OPERATOR]: createDeleteOperatorTool(this.workflowState, context), [TOOL_NAME_ADD_OPERATOR]: createAddOperatorTool(this.workflowState, operatorSchemas, context), [TOOL_NAME_MODIFY_OPERATOR]: createModifyOperatorTool(this.workflowState, context), diff --git a/agent-service/src/agent/tools/index.ts b/agent-service/src/agent/tools/index.ts index 7e2d9570703..ec61c81f652 100644 --- a/agent-service/src/agent/tools/index.ts +++ b/agent-service/src/agent/tools/index.ts @@ -20,3 +20,4 @@ export * from "./tools-utility"; export * from "./workflow-crud-tools"; export * from "./workflow-execution-tools"; +export * from "./operator-tools"; diff --git a/agent-service/src/agent/tools/operator-tools.spec.ts b/agent-service/src/agent/tools/operator-tools.spec.ts new file mode 100644 index 00000000000..9e733a94a1a --- /dev/null +++ b/agent-service/src/agent/tools/operator-tools.spec.ts @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, expect, test } from "bun:test"; +import type { OperatorSchema } from "../../api/backend-api"; +import { WorkflowSystemMetadata } from "../util/workflow-system-metadata"; +import { createGetOperatorSchemaTool, usableOperatorTypes } from "./operator-tools"; + +function schema(operatorType: string, friendly: string, group: string, description: string): OperatorSchema { + return { + operatorType, + operatorVersion: "1.0", + jsonSchema: { + properties: { + groupByKeys: { type: "array", items: { type: "string" } }, + dummyPropertyList: { type: "array" }, + }, + required: ["groupByKeys"], + }, + additionalMetadata: { + userFriendlyName: friendly, + operatorGroupName: group, + operatorDescription: description, + inputPorts: [], + outputPorts: [], + }, + }; +} + +function metadata(): WorkflowSystemMetadata { + const store = new WorkflowSystemMetadata(); + store.loadFromMetadata({ + operators: [ + schema("Aggregate", "Aggregate", "Data Cleaning", "Calculate aggregates by group"), + schema("HashJoin", "Hash Join", "Join", "Join two inputs on a key"), + schema("Sort", "Sort", "Data Cleaning", "Sort rows by columns"), + ], + groups: [], + }); + return store; +} + +const run = async (agentTool: any, args: Record) => + agentTool.execute(args, { toolCallId: "test-call", messages: [] }); + +describe("usableOperatorTypes", () => { + test("is every known type when none are allowed by name", () => { + expect(usableOperatorTypes(metadata(), []).sort()).toEqual(["Aggregate", "HashJoin", "Sort"]); + }); + + test("keeps the allowed types the metadata knows, and drops the ones it does not", () => { + expect(usableOperatorTypes(metadata(), ["Sort", "Missing", "Aggregate"])).toEqual(["Sort", "Aggregate"]); + }); +}); + +describe("get_operator_schema", () => { + const store = metadata(); + const groupBy = { groupByKeys: { type: "array", items: { type: "string" } } }; + + test("gives each type's description and compact schema, without placeholder properties", async () => { + const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, [])); + expect(JSON.parse(await run(get, { operatorTypes: ["Aggregate", "Sort"] }))).toEqual({ + Aggregate: { description: "Calculate aggregates by group", properties: groupBy, required: ["groupByKeys"] }, + Sort: { description: "Sort rows by columns", properties: groupBy, required: ["groupByKeys"] }, + }); + }); + + test.each([ + ["an unknown type", ["Pivot"], [], "Pivot"], + ["a type that is not allowed", ["Sort", "HashJoin"], ["Sort"], "HashJoin"], + ["a type in another case", ["aggregate"], [], "aggregate"], + ])("refuses %s, naming it", async (_, operatorTypes, allowed, refused) => { + const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, allowed as string[])); + await expect(run(get, { operatorTypes })).rejects.toThrow( + `Not operator types you can use: ${refused}. Use the types the system prompt lists.` + ); + }); + + test("takes at least one type", async () => { + const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, [])); + expect(get.inputSchema).toBeDefined(); + expect((get.inputSchema as any).safeParse({ operatorTypes: [] }).success).toBe(false); + }); +}); diff --git a/agent-service/src/agent/tools/operator-tools.ts b/agent-service/src/agent/tools/operator-tools.ts new file mode 100644 index 00000000000..21aeaa398fe --- /dev/null +++ b/agent-service/src/agent/tools/operator-tools.ts @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// The operator lookup. The system prompt names each operator type the agent may use, one line +// each; the properties of a type are read through `get_operator_schema` when the agent needs them, +// rather than every allowed type's schema riding along on every call. + +import { tool } from "ai"; +import { z } from "zod"; +import type { WorkflowSystemMetadata } from "../util/workflow-system-metadata"; + +export const TOOL_NAME_GET_OPERATOR_SCHEMA = "get_operator_schema"; + +/** + * The operator types the agent may use: the allowed ones the metadata knows, or every type it + * knows when none are named. The prompt's list, the lookup and the add tool all go by this one list. + */ +export function usableOperatorTypes(metadataStore: WorkflowSystemMetadata, allowed: string[]): string[] { + const known = Object.keys(metadataStore.getAllOperatorTypes()); + return allowed.length === 0 ? known : allowed.filter(type => known.includes(type)); +} + +export function createGetOperatorSchemaTool(metadataStore: WorkflowSystemMetadata, usableTypes: string[]) { + return tool({ + description: + "Get the properties of one or more operator types: which are required, and each one's type, allowed " + + "values and default. Call it, once for all the types you need, before you add operators of types you " + + "have not used in this conversation.", + inputSchema: z.object({ + operatorTypes: z + .array(z.string()) + .min(1) + .describe("The operator types, as the system prompt names them, such as Aggregate"), + }), + execute: async ({ operatorTypes }) => { + const unknown = operatorTypes.filter(type => !usableTypes.includes(type)); + if (unknown.length > 0) { + throw new Error( + `Not operator types you can use: ${unknown.join(", ")}. Use the types the system prompt lists.` + ); + } + const schemas = Object.fromEntries( + operatorTypes.map(type => [ + type, + { description: metadataStore.getDescription(type), ...metadataStore.getCompactSchema(type) }, + ]) + ); + return JSON.stringify(schemas); + }, + }); +} diff --git a/agent-service/src/agent/tools/workflow-crud-tools.ts b/agent-service/src/agent/tools/workflow-crud-tools.ts index 1933b086aa1..418711c2a24 100644 --- a/agent-service/src/agent/tools/workflow-crud-tools.ts +++ b/agent-service/src/agent/tools/workflow-crud-tools.ts @@ -109,7 +109,7 @@ export function createAddOperatorTool( const workflowUtil = context?.metadataStore ? new WorkflowUtilService(context.metadataStore, workflowState) : null; return tool({ - description: `Add a new operator to the workflow. Use getOperatorSchema first to understand required properties. + description: `Add a new operator to the workflow. Use get_operator_schema first to understand required properties. Examples: 1. Add a source operator (no inputs):