diff --git a/agent-service/LICENSE-binary b/agent-service/LICENSE-binary index 0e628cc877c..bef3bbfa01e 100644 --- a/agent-service/LICENSE-binary +++ b/agent-service/LICENSE-binary @@ -215,6 +215,7 @@ Agent service npm packages: - @ai-sdk/openai@4.0.118 - @ai-sdk/provider-utils@5.0.81 - @ai-sdk/provider@4.0.38 + - @texera/workflow-macros@0.0.0 - @vercel/oidc@3.2.0 - @workflow/serde@4.1.0 - ai@7.0.173 @@ -249,6 +250,7 @@ Agent service npm packages: - file-type@21.1.1 - graphlib@2.1.8 - json-schema-traverse@1.0.0 + - lodash-es@4.18.1 - lodash@4.18.1 - memoirist@0.4.0 - ms@2.1.3 @@ -268,6 +270,7 @@ Agent service npm packages: - token-types@6.1.1 - uint8array-extras@1.5.0 - undici-types@7.16.0 + - uuid@14.0.1 - zod@3.25.76 -------------------------------------------------------------------------------- diff --git a/agent-service/bun.lock b/agent-service/bun.lock index b989fdf6a09..c7c50497405 100644 --- a/agent-service/bun.lock +++ b/agent-service/bun.lock @@ -7,17 +7,21 @@ "dependencies": { "@ai-sdk/openai": "4.0.118", "@elysiajs/cors": "1.4.3", + "@texera/workflow-macros": "file:../common/workflow-core/src/main/typescript/workflow-macros", "ai": "7.0.173", "ajv": "8.20.0", "dagre": "0.8.5", "elysia": "1.4.32", + "lodash-es": "4.18.1", "pino": "10.4.0", "rxjs": "7.8.2", + "uuid": "14.0.1", "zod": "3.25.76", }, "devDependencies": { "@types/bun": "1.4.4", "@types/dagre": "0.7.54", + "@types/lodash-es": "4.17.12", "pino-pretty": "13.2.0", "prettier": "3.10.1", "tsx": "4.23.15", @@ -96,6 +100,8 @@ "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "@texera/workflow-macros": ["@texera/workflow-macros@file:../common/workflow-core/src/main/typescript/workflow-macros", { "dependencies": { "lodash-es": "4.18.1", "uuid": "14.0.1" }, "devDependencies": { "@types/lodash-es": "4.17.12" } }], + "@tokenizer/inflate": ["@tokenizer/inflate@0.4.1", "", { "dependencies": { "debug": "^4.4.3", "token-types": "^6.1.1" } }, "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA=="], "@tokenizer/token": ["@tokenizer/token@0.3.0", "", {}, "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="], @@ -104,6 +110,10 @@ "@types/dagre": ["@types/dagre@0.7.54", "", {}, "sha512-QjcRY+adGbYvBFS7cwv5txhVIwX1XXIUswWl+kSQTbI6NjgZydrZkEKX/etzVd7i+bCsCb40Z/xlBY5eoFuvWQ=="], + "@types/lodash": ["@types/lodash@4.17.26", "", {}, "sha512-48NWfiY6oNcKOh47Z4h+sIvwcQR91h6KPhaYKgeMIrkT2jV82Gy90SErj60vW8jQbeMSOSbhIv3bZ7JoqPPVhA=="], + + "@types/lodash-es": ["@types/lodash-es@4.17.12", "", { "dependencies": { "@types/lodash": "*" } }, "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ=="], + "@types/node": ["@types/node@24.10.1", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ=="], "@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="], @@ -166,6 +176,8 @@ "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], + "lodash-es": ["lodash-es@4.18.1", "", {}, "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A=="], + "memoirist": ["memoirist@0.4.0", "", {}, "sha512-zxTgA0mSYELa66DimuNQDvyLq36AwDlTuVRbnQtB+VuTcKWm5Qc4z3WkSpgsFWHNhexqkIooqpv4hdcqrX5Nmg=="], "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], @@ -226,6 +238,8 @@ "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + "uuid": ["uuid@14.0.1", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], diff --git a/agent-service/package.json b/agent-service/package.json index 9e9eb874b34..f32428aabba 100644 --- a/agent-service/package.json +++ b/agent-service/package.json @@ -17,17 +17,21 @@ "dependencies": { "@ai-sdk/openai": "4.0.118", "@elysiajs/cors": "1.4.3", + "@texera/workflow-macros": "file:../common/workflow-core/src/main/typescript/workflow-macros", "ai": "7.0.173", "ajv": "8.20.0", "dagre": "0.8.5", "elysia": "1.4.32", + "lodash-es": "4.18.1", "pino": "10.4.0", "rxjs": "7.8.2", + "uuid": "14.0.1", "zod": "3.25.76" }, "devDependencies": { "@types/bun": "1.4.4", "@types/dagre": "0.7.54", + "@types/lodash-es": "4.17.12", "pino-pretty": "13.2.0", "prettier": "3.10.1", "tsx": "4.23.15", diff --git a/agent-service/src/agent/macro-test-data.ts b/agent-service/src/agent/macro-test-data.ts new file mode 100644 index 00000000000..0a6773888f9 --- /dev/null +++ b/agent-service/src/agent/macro-test-data.ts @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import fixture from "../../../common/workflow-compiler/src/test/resources/macros/nested-expansion.json"; + +export function macroWorkflow(leafProperty = "value") { + const content = structuredClone(fixture.content); + macroLeaf(content).formBinding.fields[0].propertyKey = leafProperty; + return content; +} + +export function macroLeaf(content: typeof fixture.content) { + return content.macroDependencies[0].content.macroDependencies[0].content.macroDependencies[0].content; +} + +export function macroExpansionResponse() { + return structuredClone(fixture.expected); +} diff --git a/agent-service/src/agent/texera-agent.spec.ts b/agent-service/src/agent/texera-agent.spec.ts index a1644921cfe..3077b42effa 100644 --- a/agent-service/src/agent/texera-agent.spec.ts +++ b/agent-service/src/agent/texera-agent.spec.ts @@ -834,6 +834,31 @@ describe("sendMessage", () => { expect(res.error).toBeUndefined(); }); + test("uses authoritative macro boundary schemas and nested errors in the model's context", async () => { + const { macroWorkflow, macroExpansionResponse } = await import("./macro-test-data"); + const content = macroWorkflow(); + const before = JSON.stringify(content); + const model = textModel("ok", 1, 1); + const agent = makeAgentWith(model); + agent.getWorkflowState().setWorkflowContent(content); + fetchSpy.mockResolvedValue({ + ok: true, + json: async () => ({ + macroExpansion: macroExpansionResponse(), + operatorOutputSchemas: { one: { "0_false": [{ attributeName: "boundary", attributeType: "integer" }] } }, + operatorErrors: { one: { operatorId: "one", message: "Outer / Child / Scan: invalid column", type: "schema" } }, + }), + } as any); + + const result = await agent.sendMessage("inspect the macro"); + const prompt = (model as any).doGenerateCalls[0].prompt[1].content[0].text; + expect(result.error).toBeUndefined(); + expect(prompt).toContain("Output Schema: [boundary: integer]"); + expect(prompt).toContain("Compilation Error: Outer / Child / Scan: invalid column"); + expect(agent.getWorkflowState().getWorkflowContent()).toEqual(content); + expect(JSON.stringify(content)).toBe(before); + }); + test("reports a model failure as an error step", async () => { const model = new MockLanguageModelV4({ doGenerate: async () => { @@ -1087,6 +1112,8 @@ describe("delegate mode", () => { await agent.sendMessage("one"); const retrieves = () => urls.filter(u => u.includes("/api/workflow/7")).length; expect(retrieves()).toBe(1); + const compileCall = fetchSpy.mock.calls.find(([url]: [unknown]) => String(url).includes("/api/compile")); + expect(compileCall?.[1]?.headers).toEqual({ Authorization: "Bearer tok", "Content-Type": "application/json" }); expect( agent .getWorkflowState() diff --git a/agent-service/src/agent/texera-agent.ts b/agent-service/src/agent/texera-agent.ts index 555cc2730ff..d47866fa78b 100644 --- a/agent-service/src/agent/texera-agent.ts +++ b/agent-service/src/agent/texera-agent.ts @@ -211,6 +211,7 @@ export class TexeraAgent { const getExecutionConfig = this.delegateConfig ? () => this.buildExecutionConfig()! : undefined; const context: ToolContext = { + userToken: this.delegateConfig?.userToken, metadataStore: this.metadataStore, settings: { maxOperatorResultCharLimit: this.settings.maxOperatorResultCharLimit, diff --git a/agent-service/src/agent/tools/workflow-crud-tools.spec.ts b/agent-service/src/agent/tools/workflow-crud-tools.spec.ts index 44c6953de10..de423b1fe5a 100644 --- a/agent-service/src/agent/tools/workflow-crud-tools.spec.ts +++ b/agent-service/src/agent/tools/workflow-crud-tools.spec.ts @@ -17,13 +17,16 @@ * under the License. */ -import { describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test, spyOn } from "bun:test"; import { createAddOperatorTool, createModifyOperatorTool, createDeleteOperatorTool } from "./workflow-crud-tools"; +import { macroWorkflow } from "../macro-test-data"; import { WorkflowState } from "../workflow-state"; import { WorkflowSystemMetadata } from "../util/workflow-system-metadata"; import type { OperatorMetadata } from "../../api/backend-api"; import type { OperatorPredicate } from "../../types/workflow"; +const fetchTarget: { fetch: (...args: Parameters) => ReturnType } = globalThis; + // A single synthetic operator type with one input and one output port, backed // by a real (in-memory, no backend fetch) WorkflowSystemMetadata so the tools // exercise the genuine schema-validation and predicate-construction paths. @@ -782,3 +785,59 @@ describe("createAddOperatorTool", () => { expect(state.getAllOperators()).toHaveLength(0); }); }); + +describe("macro property edits", () => { + const content = macroWorkflow("count"); + let state: WorkflowState; + let fetchSpy: ReturnType>; + const run = (id: string, value: unknown, other = {}) => + runTool(createModifyOperatorTool(state, { ...context, userToken: "token" }), { + operatorId: id, + properties: { value }, + ...other, + }); + + beforeEach(() => { + state = new WorkflowState(); + state.setWorkflowContent({ + ...content, + operators: [content.operators[0], { ...content.operators[1], isDisabled: true }], + }); + fetchSpy = spyOn(fetchTarget, "fetch").mockResolvedValue(new Response(null, { status: 204 })); + }); + afterEach(() => { + fetchSpy.mockRestore(); + state.destroy(); + }); + + test("submits the proposed pin with its bearer token and commits only accepted edits", async () => { + expect(await run("one", 42)).not.toContain("[ERROR]"); + expect(state.getOperator("one")!.operatorProperties.value).toBe(42); + const before = structuredClone(state.getWorkflowContent()); + for (const operator of before.operators) { + const message = "Outer / Inner / Leaf: count must be an integer"; + fetchSpy.mockResolvedValueOnce(Response.json({ message }, { status: 400 })); + expect(await run(operator.operatorID, "bad", { summary: "Rename" })).toContain(message); + const [url, init] = fetchSpy.mock.calls.at(-1)!; + expect(String(url)).toEndWith("/api/compile/validate-properties"); + expect(init!.headers).toMatchObject({ Authorization: "Bearer token" }); + expect(JSON.parse(init!.body as string)).toEqual({ + operator: { ...operator, operatorProperties: { ...operator.operatorProperties, value: "bad" } }, + macroDependencies: before.macroDependencies, + }); + } + expect(await run("one", 7, { summary: "Rename", inputOperatorIds: { "9": ["two"] } })).toContain("out of range"); + expect(state.getWorkflowContent()).toEqual(before); + }); + + test("keeps a concurrent change made while validation is pending", async () => { + const validation = Promise.withResolvers(); + fetchSpy.mockReturnValueOnce(validation.promise); + const pending = run("one", 42); + expect(fetchSpy).toHaveBeenCalledTimes(1); + state.updateOperatorProperties("one", { value: 20 }); + validation.resolve(new Response(null, { status: 204 })); + expect(await pending).toContain("workflow changed"); + expect(state.getOperator("one")!.operatorProperties.value).toBe(20); + }); +}); diff --git a/agent-service/src/agent/tools/workflow-crud-tools.ts b/agent-service/src/agent/tools/workflow-crud-tools.ts index 1933b086aa1..f4e5bb50047 100644 --- a/agent-service/src/agent/tools/workflow-crud-tools.ts +++ b/agent-service/src/agent/tools/workflow-crud-tools.ts @@ -17,6 +17,8 @@ * under the License. */ +import { isMacro } from "@texera/workflow-macros"; +import { validateMacroPropertiesAsync } from "../../api/compile-api"; import { z } from "zod"; import { tool } from "ai"; import { WorkflowState } from "../workflow-state"; @@ -37,6 +39,7 @@ import { } from "../util/workflow-system-metadata"; export interface ToolContext { + userToken?: string; metadataStore?: WorkflowSystemMetadata; settings?: { maxOperatorResultCharLimit?: number; @@ -57,6 +60,19 @@ function formatInputArgs(args: Record): string { return `Input: ${JSON.stringify(compact)}`; } +function propertyError( + metadata: WorkflowSystemMetadata, + type: string, + properties: Record, + inputInfo: string +): string | undefined { + const validation = metadata.validateOperatorProperties(type, properties); + if (validation.isValid) return; + const schema = metadata.getCompactSchema(type); + const expected = schema ? ` Expected: ${formatCompactSchemaForError(schema)}.` : ""; + return `Invalid properties for "${type}": ${formatValidationErrors(validation)}.${expected} ${inputInfo}`; +} + /** * Checks every port index and source operator in inputOperatorIds and returns the links to create, * or the first error. It doesn't change the workflow, so callers must run it before any mutation. @@ -101,6 +117,10 @@ function resolveInputLinks( return { links }; } +function linkPair(link: Omit) { + return { source: link.source.operatorID, target: link.target.operatorID }; +} + export function createAddOperatorTool( workflowState: WorkflowState, operatorSchemas: Map, @@ -153,14 +173,8 @@ Examples: } if (context?.metadataStore && args.properties) { - const validation = context.metadataStore.validateOperatorProperties(args.operatorType, args.properties); - if (!validation.isValid) { - const compactSchema = context.metadataStore.getCompactSchema(args.operatorType); - const schemaStr = compactSchema ? ` Expected: ${formatCompactSchemaForError(compactSchema)}.` : ""; - return createErrorResult( - `Invalid properties for "${args.operatorType}": ${formatValidationErrors(validation)}.${schemaStr} ${inputInfo}` - ); - } + const error = propertyError(context.metadataStore, args.operatorType, args.properties, inputInfo); + if (error) return createErrorResult(error); } if (!workflowUtil) { @@ -189,20 +203,15 @@ Examples: const resolved = resolveInputLinks( workflowState, - args.operatorId, + operator.operatorID, operator.inputPorts, args.inputOperatorIds, inputInfo ); if ("error" in resolved) return createErrorResult(resolved.error); - + const { links } = resolved; workflowState.addOperator(operator); - - const createdLinkPairs: { source: string; target: string }[] = []; - for (const link of resolved.links) { - workflowState.addLink({ linkID: workflowState.generateLinkId(), ...link }); - createdLinkPairs.push({ source: link.source.operatorID, target: args.operatorId }); - } + links.forEach(link => workflowState.addLink({ ...link, linkID: workflowState.generateLinkId() })); autoLayoutWorkflow(workflowState); @@ -214,7 +223,7 @@ Examples: operator.operatorID, numInputPorts, numOutputPorts, - createdLinkPairs.length > 0 ? createdLinkPairs : undefined + links.map(linkPair) ); return createToolResult(resultMsg); @@ -263,31 +272,31 @@ Examples: const operator = workflowState.getOperator(args.operatorId); if (!operator) return createErrorResult(`Operator ${args.operatorId} not found. ${inputInfo}`); - if (args.properties && context?.metadataStore) { - const mergedProperties = { ...operator.operatorProperties, ...args.properties }; - const validation = context.metadataStore.validateOperatorProperties(operator.operatorType, mergedProperties); - if (!validation.isValid) { - const compactSchema = context.metadataStore.getCompactSchema(operator.operatorType); - const schemaStr = compactSchema ? ` Expected: ${formatCompactSchemaForError(compactSchema)}.` : ""; - return createErrorResult( - `Invalid properties for "${operator.operatorType}": ${formatValidationErrors(validation)}.${schemaStr} ${inputInfo}` - ); - } - } - const resolved = resolveInputLinks( workflowState, - args.operatorId, + operator.operatorID, operator.inputPorts, args.inputOperatorIds, inputInfo ); if ("error" in resolved) return createErrorResult(resolved.error); - - const createdLinkPairs: { source: string; target: string }[] = []; - const deletedLinkPairs: { source: string; target: string }[] = []; - + const { links } = resolved; if (args.properties) { + const proposed = { ...operator.operatorProperties, ...args.properties }; + if (isMacro(operator)) { + const content = workflowState.getWorkflowContent(); + const before = JSON.stringify(content); + await validateMacroPropertiesAsync( + { ...operator, operatorProperties: proposed }, + content.macroDependencies, + context?.userToken + ); + if (JSON.stringify(workflowState.getWorkflowContent()) !== before) + return createErrorResult("The workflow changed while validating properties. Try again."); + } else if (context?.metadataStore) { + const error = propertyError(context.metadataStore, operator.operatorType, proposed, inputInfo); + if (error) return createErrorResult(error); + } workflowState.updateOperatorProperties(args.operatorId, args.properties); } @@ -295,29 +304,18 @@ Examples: workflowState.updateOperatorDisplayName(args.operatorId, args.summary); } + const removed = args.inputOperatorIds + ? workflowState + .getLinksConnectedToOperator(args.operatorId) + .filter(link => link.target.operatorID === args.operatorId) + : []; if (args.inputOperatorIds) { - const currentLinks = workflowState - .getLinksConnectedToOperator(args.operatorId) - .filter(link => link.target.operatorID === args.operatorId); - for (const link of currentLinks) { - deletedLinkPairs.push({ source: link.source.operatorID, target: link.target.operatorID }); - workflowState.deleteLink(link.linkID); - } - - for (const link of resolved.links) { - workflowState.addLink({ linkID: workflowState.generateLinkId(), ...link }); - createdLinkPairs.push({ source: link.source.operatorID, target: args.operatorId }); - } - + removed.forEach(link => workflowState.deleteLink(link.linkID)); + links.forEach(link => workflowState.addLink({ ...link, linkID: workflowState.generateLinkId() })); autoLayoutWorkflow(workflowState); } - let resultMsg = formatModifyOperatorResult( - args.operatorId, - createdLinkPairs.length > 0 ? createdLinkPairs : undefined, - deletedLinkPairs.length > 0 ? deletedLinkPairs : undefined - ); - + const resultMsg = formatModifyOperatorResult(args.operatorId, links.map(linkPair), removed.map(linkPair)); return createToolResult(resultMsg); } catch (error: any) { return createErrorResult(formatOperatorError(args.operatorId, error.message || String(error))); diff --git a/agent-service/src/agent/tools/workflow-execution-tools.spec.ts b/agent-service/src/agent/tools/workflow-execution-tools.spec.ts index 1ade5bc6480..ca7e36b3962 100644 --- a/agent-service/src/agent/tools/workflow-execution-tools.spec.ts +++ b/agent-service/src/agent/tools/workflow-execution-tools.spec.ts @@ -31,7 +31,10 @@ import { type WorkflowExecutionSummary, type WorkflowFatalError, } from "../../types/execution"; -import type { OperatorLink, OperatorPredicate } from "../../types/workflow"; +import type { OperatorLink, OperatorPredicate, WorkflowContent } from "../../types/workflow"; +import { macroInstance } from "@texera/workflow-macros"; +import { macroGraph } from "../../../../common/workflow-core/src/test/typescript/macro-test-graph"; +import { macroLeaf, macroWorkflow } from "../macro-test-data"; // --- fixtures ------------------------------------------------------------- @@ -873,3 +876,124 @@ describe("createExecuteOperatorTool", () => { expect(onResult).toHaveBeenCalledWith(target, expect.anything()); }); }); + +describe("nested macro execution", () => { + function loadWorkflow(content: WorkflowContent): WorkflowState { + const state = new WorkflowState(); + state.setWorkflowContent(content); + return state; + } + + function tableResult(row: Record) { + return { state: "Completed", inputTuples: 0, outputTuples: 1, resultMode: "table", result: [row] }; + } + + function complete(operators: Record>): void { + setFetchResolving(legacyJsonResponse({ success: true, state: "Completed", operators })); + } + + test("submits native defaults intact and returns the selected nonzero output of the pinned instance", async () => { + const content = macroWorkflow("code"); + content.operators[0].operatorProperties.value = "# override"; + const inner: { operators: OperatorPredicate[]; links: OperatorLink[] } = macroLeaf(content); + inner.operators[0] = { + ...inner.operators[0], + operatorType: "PythonUDFSourceV2", + operatorProperties: { code: "# source" }, + }; + const split = makeOperator("split", { + inputPorts: [{ portID: "input-0" }], + operatorType: "Split", + outputPorts: [{ portID: "output-0" }, { portID: "output-1" }], + }); + inner.operators.push(split); + const output = inner.links[0].target; + inner.links = [ + makeLink("leaf", "output-0", "split", "input-0"), + makeLink("split", "output-1", output.operatorID, output.portID), + ]; + const before = structuredClone(content); + const state = loadWorkflow(content); + complete({ + "runtime-split": tableResult({ branch: "native output zero" }), + one: tableResult({ branch: "café 東京" }), + }); + const onResult = mock(() => {}); + const result = await executeOperatorAndFormat(state, makeConfig(), "one", { onResult }); + const plan = requestBody().logicalPlan; + expect(String(fetchSpy.mock.calls[0][0])).toMatch(/\/api\/execution\/1\/0\/run$/); + expect(plan.operators).toEqual([]); + expect(plan.macroContent).toEqual(before); + expect(state.getWorkflowContent()).toEqual(before); + expect(plan.executeOperatorIds).toEqual(["one"]); + expect(plan.opsToViewResult).toEqual(["one"]); + expect(result).toContain("café 東京"); + expect(result).not.toContain("native output zero"); + expect(onResult).toHaveBeenCalledWith( + "one", + expect.objectContaining({ resultSummary: expect.objectContaining({ totalTuplesCount: 1 }) }) + ); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(validateSpy).not.toHaveBeenCalled(); + state.destroy(); + }); + + test.each(["Missing macro version", "Recursive macro reference"])("surfaces backend rejection: %s", async message => { + const content = macroWorkflow(); + if (message === "Missing macro version") content.macroDependencies = []; + else content.macroDependencies[0].content.operators[0].operatorType = "WorkflowMacro:outer:1"; + const state = loadWorkflow(content); + const diagnostic = `Outer / Child: ${message}`; + setFetchResolving( + legacyJsonResponse({ + success: false, + state: "CompilationFailed", + operators: {}, + compilationErrors: { one: diagnostic }, + }) + ); + expect(await executeOperatorAndFormat(state, makeConfig(), "one")).toContain(diagnostic); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(String(fetchSpy.mock.calls[0][0])).toMatch(/\/api\/execution\/1\/0\/run$/); + state.destroy(); + }); + + test("keeps the requested passthrough result and visible source callbacks without marking a sibling executed", async () => { + const state = new WorkflowState(); + const definition = { + id: "identity", + name: "Identity", + version: 1, + content: macroGraph({ + ...state.getWorkflowContent(), + operators: [], + links: [], + macroInterface: { + id: "identity", + inputs: [{ portID: "rows", targets: [{ operatorID: "identity__outputs", portID: "rows" }] }], + outputs: [{ portID: "rows", targets: [] }], + }, + }), + }; + const content = { + ...state.getWorkflowContent(), + operators: [ + makeOperator("source", { outputPorts: [{ portID: "output-0" }] }), + macroInstance(definition, "A"), + macroInstance(definition, "B"), + ], + links: [makeLink("source", "output-0", "A", "rows"), makeLink("source", "output-0", "B", "rows")], + macroDependencies: [definition], + }; + state.setWorkflowContent(content); + const before = JSON.stringify(content); + complete({ source: tableResult({ value: "native" }), A: tableResult({ value: "requested A" }) }); + const onResult = mock((_id: string, _info: OperatorExecutionSummary) => {}); + const result = await executeOperatorAndFormat(state, makeConfig(), "A", { onResult }); + expect(result).toContain("requested A"); + expect(onResult.mock.calls.map(([id]) => id).sort()).toEqual(["A", "source"]); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(JSON.stringify(content)).toBe(before); + state.destroy(); + }); +}); diff --git a/agent-service/src/agent/tools/workflow-execution-tools.ts b/agent-service/src/agent/tools/workflow-execution-tools.ts index 829479ba77f..a9db1c42b4f 100644 --- a/agent-service/src/agent/tools/workflow-execution-tools.ts +++ b/agent-service/src/agent/tools/workflow-execution-tools.ts @@ -17,6 +17,7 @@ * under the License. */ +import type { WorkflowContent, OperatorPredicate, OperatorLink } from "../../types/workflow"; import { z } from "zod"; import { tool } from "ai"; import { createErrorResult, formatOperatorOutput, getOperatorErrorText } from "./tools-utility"; @@ -103,14 +104,9 @@ function validateOperatorSchema(operatorType: string, operatorProperties: Record return validation.isValid ? { isValid: true, messages: {} } : { isValid: false, messages: validation.messages }; } -function validateOperatorConnection(operatorId: string, workflowState: WorkflowState): OperatorValidation { - const operator = workflowState.getOperator(operatorId); - if (!operator) { - return { isValid: false, messages: { error: `Operator ${operatorId} not found` } }; - } - +function validateOperatorConnection(operator: OperatorPredicate, allLinks: OperatorLink[]): OperatorValidation { + const operatorId = operator.operatorID; const numInputLinksByPort = new Map(); - const allLinks = workflowState.getAllLinks(); for (const link of allLinks) { if (link.target.operatorID === operatorId) { @@ -157,12 +153,12 @@ function combineValidations(...validations: OperatorValidation[]): OperatorValid return { isValid, messages }; } -function validateWorkflow(workflowState: WorkflowState): WorkflowValidationResult { +function validateWorkflow(content: WorkflowContent): WorkflowValidationResult { const errors: Record> = {}; - for (const operator of workflowState.getAllEnabledOperators()) { + for (const operator of content.operators) { const schemaValidation = validateOperatorSchema(operator.operatorType, operator.operatorProperties); - const connectionValidation = validateOperatorConnection(operator.operatorID, workflowState); + const connectionValidation = validateOperatorConnection(operator, content.links); const combined = combineValidations(schemaValidation, connectionValidation); if (!combined.isValid) { @@ -176,78 +172,6 @@ function validateWorkflow(workflowState: WorkflowState): WorkflowValidationResul }; } -function buildLogicalPlan(workflowState: WorkflowState, opsToViewResult?: string[]): LogicalPlan { - const useSubDAG = opsToViewResult && opsToViewResult.length === 1; - const targetOperatorId = useSubDAG ? opsToViewResult[0] : undefined; - - let operatorsList: { operatorID: string; operatorType: string; [key: string]: any }[]; - let linksList: LogicalLink[]; - - const getInputPortOrdinal = (operatorID: string, inputPortID: string): number => { - const op = workflowState.getOperator(operatorID); - if (!op) return 0; - const idx = op.inputPorts.findIndex(port => port.portID === inputPortID); - return idx >= 0 ? idx : 0; - }; - - const getOutputPortOrdinal = (operatorID: string, outputPortID: string): number => { - const op = workflowState.getOperator(operatorID); - if (!op) return 0; - const idx = op.outputPorts.findIndex(port => port.portID === outputPortID); - return idx >= 0 ? idx : 0; - }; - - if (targetOperatorId) { - const subDAG = workflowState.getSubDAG(targetOperatorId); - - operatorsList = subDAG.operators.map(op => ({ - ...op.operatorProperties, - operatorID: op.operatorID, - operatorType: op.operatorType, - inputPorts: op.inputPorts, - outputPorts: op.outputPorts, - })); - - linksList = subDAG.links.map(link => ({ - fromOpId: link.source.operatorID, - fromPortId: { id: getOutputPortOrdinal(link.source.operatorID, link.source.portID), internal: false }, - toOpId: link.target.operatorID, - toPortId: { id: getInputPortOrdinal(link.target.operatorID, link.target.portID), internal: false }, - })); - } else { - operatorsList = workflowState.getAllEnabledOperators().map(op => ({ - ...op.operatorProperties, - operatorID: op.operatorID, - operatorType: op.operatorType, - inputPorts: op.inputPorts, - outputPorts: op.outputPorts, - })); - - linksList = workflowState.getAllLinks().map(link => ({ - fromOpId: link.source.operatorID, - fromPortId: { id: getOutputPortOrdinal(link.source.operatorID, link.source.portID), internal: false }, - toOpId: link.target.operatorID, - toPortId: { id: getInputPortOrdinal(link.target.operatorID, link.target.portID), internal: false }, - })); - } - - let allOpsToView: string[]; - if (opsToViewResult && opsToViewResult.length > 0) { - const operatorIds = new Set(operatorsList.map(op => op.operatorID)); - allOpsToView = opsToViewResult.filter(id => operatorIds.has(id)); - } else { - allOpsToView = operatorsList - .filter(op => !linksList.some(link => link.fromOpId === op.operatorID)) - .map(op => op.operatorID); - } - - return { - operators: operatorsList, - links: linksList, - opsToViewResult: allOpsToView, - }; -} - async function executeWorkflowHttp( config: ExecutionConfig, logicalPlan: LogicalPlan, @@ -272,8 +196,7 @@ async function executeWorkflowHttp( const request = { executionName: "agent-execution", logicalPlan: { - operators: logicalPlan.operators, - links: logicalPlan.links, + ...logicalPlan, opsToViewResult: logicalPlan.opsToViewResult || [], opsToReuseResult: [], }, @@ -384,16 +307,15 @@ export async function executeOperatorAndFormat( const release = await getWorkflowMutex(config.workflowId).acquire(); try { - const logicalPlan = buildLogicalPlan(workflowState, [operatorId]); + const logicalPlan = workflowState.toLogicalPlan(operatorId); - if (logicalPlan.operators.length === 0) { + if ((logicalPlan.executeOperatorIds ?? logicalPlan.operators).length === 0) { return createErrorResult("Cannot execute: workflow has no operators."); } - // Only block on the target operator's validation errors; upstream issues will - // surface as runtime errors that correctly identify the failing operator. - const validationResult = validateWorkflow(workflowState); - if (!validationResult.isValid) { + // Macro validation and result projection belong to the backend compiler. + if (!logicalPlan.macroContent) { + const validationResult = validateWorkflow(workflowState.getWorkflowContent()); const targetErrors = validationResult.errors[operatorId]; if (targetErrors) { const lines = [`Operator ${operatorId}:`]; diff --git a/agent-service/src/agent/util/context-utils.ts b/agent-service/src/agent/util/context-utils.ts index f5078e27063..bc3cf6b65cc 100644 --- a/agent-service/src/agent/util/context-utils.ts +++ b/agent-service/src/agent/util/context-utils.ts @@ -206,7 +206,7 @@ function serializeDag( lines.push(""); for (const op of sortedOps) { - const inputSchemaMap = extractOperatorInputPortSchemaMap(op.operatorID, op, outputSchemas, allLinks); + const inputSchemaMap = extractOperatorInputPortSchemaMap(op.operatorID, op, outputSchemas, allLinks, allOperators); const outputSchemaMap = outputSchemas[op.operatorID]; const compilationError = compilationErrors[op.operatorID]; lines.push( diff --git a/agent-service/src/agent/util/workflow-utils.spec.ts b/agent-service/src/agent/util/workflow-utils.spec.ts index b8fd82dfd97..442b8f0add5 100644 --- a/agent-service/src/agent/util/workflow-utils.spec.ts +++ b/agent-service/src/agent/util/workflow-utils.spec.ts @@ -45,23 +45,70 @@ function link(linkID: string, sourceOp: string, sourcePort: string, targetPort: const schemaA: PortSchema = [{ attributeName: "a", attributeType: "string" }]; const schemaB: PortSchema = [{ attributeName: "b", attributeType: "integer" }]; +const sourceOperator = (operatorID: string): OperatorPredicate => ({ + ...operatorWithInputPorts(0), + operatorID, + outputPorts: [{ portID: "output-0" }], +}); describe("extractOperatorInputPortSchemaMap", () => { test("returns undefined when the operator has no input links", () => { - expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(1), {}, [])).toBeUndefined(); + expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(1), {}, [], [])).toBeUndefined(); }); test("maps a single input port to its resolved upstream schema", () => { const links = [link("l0", "src", "output-0", "input-0")]; const outputSchemas = { src: { "0_false": schemaA } }; - expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(1), outputSchemas, links)).toEqual({ + expect( + extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(1), outputSchemas, links, [ + sourceOperator("src"), + ]) + ).toEqual({ "0_false": schemaA, }); }); + test("uses saved native port positions after removing middle input and output ports", () => { + const target = { ...operatorWithInputPorts(2), inputPorts: [{ portID: "input-0" }, { portID: "input-2" }] }; + const source = { + ...operatorWithInputPorts(0), + operatorID: "src", + outputPorts: [{ portID: "output-0" }, { portID: "output-2" }], + }; + const outputSchemas = { src: { "0_false": schemaA, "1_false": schemaB } }; + const links = [link("gap", "src", "output-2", "input-2")]; + const operators = [source, target]; + const before = JSON.stringify({ operators, links, outputSchemas }); + + expect(extractOperatorInputPortSchemaMap(TARGET, target, outputSchemas, links, operators)).toEqual({ + "0_false": undefined, + "1_false": schemaB, + }); + expect(JSON.stringify({ operators, links, outputSchemas })).toBe(before); + }); + test("returns undefined when the only source operator is absent from outputSchemas", () => { const links = [link("l0", "missing", "output-0", "input-0")]; - expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(1), {}, links)).toBeUndefined(); + expect( + extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(1), {}, links, [sourceOperator("missing")]) + ).toBeUndefined(); + }); + + test("does not infer a missing source or an undeclared target port from its numeric suffix", () => { + const target = operatorWithInputPorts(1); + const schemas = { src: { "0_false": schemaA } }; + expect( + extractOperatorInputPortSchemaMap(TARGET, target, schemas, [link("missing", "src", "output-0", "input-0")], []) + ).toBeUndefined(); + expect( + extractOperatorInputPortSchemaMap( + TARGET, + target, + schemas, + [link("unknown", "src", "output-0", "input-1")], + [sourceOperator("src")] + ) + ).toBeUndefined(); }); test("keys each input port separately and routes links by target port number", () => { @@ -70,7 +117,12 @@ describe("extractOperatorInputPortSchemaMap", () => { srcA: { "0_false": schemaA }, srcB: { "0_false": schemaB }, }; - expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(2), outputSchemas, links)).toEqual({ + expect( + extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(2), outputSchemas, links, [ + sourceOperator("srcA"), + sourceOperator("srcB"), + ]) + ).toEqual({ "0_false": schemaA, "1_false": schemaB, }); @@ -79,7 +131,12 @@ describe("extractOperatorInputPortSchemaMap", () => { test("leaves a port undefined when its source operator is absent from outputSchemas", () => { const links = [link("l0", "srcA", "output-0", "input-0"), link("l1", "missing", "output-0", "input-1")]; const outputSchemas = { srcA: { "0_false": schemaA } }; - expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(2), outputSchemas, links)).toEqual({ + expect( + extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(2), outputSchemas, links, [ + sourceOperator("srcA"), + sourceOperator("missing"), + ]) + ).toEqual({ "0_false": schemaA, "1_false": undefined, }); @@ -91,7 +148,12 @@ describe("extractOperatorInputPortSchemaMap", () => { srcA: { "0_false": schemaA }, srcB: { "0_false": schemaB }, }; - expect(extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(2), outputSchemas, links)).toEqual({ + expect( + extractOperatorInputPortSchemaMap(TARGET, operatorWithInputPorts(2), outputSchemas, links, [ + sourceOperator("srcA"), + sourceOperator("srcB"), + ]) + ).toEqual({ "0_false": schemaA, "1_false": undefined, }); diff --git a/agent-service/src/agent/util/workflow-utils.ts b/agent-service/src/agent/util/workflow-utils.ts index d723f61eff0..8cc525dd6a2 100644 --- a/agent-service/src/agent/util/workflow-utils.ts +++ b/agent-service/src/agent/util/workflow-utils.ts @@ -27,24 +27,13 @@ import type { } from "../../types/workflow"; import type { WorkflowSystemMetadata } from "./workflow-system-metadata"; import type { WorkflowState } from "../workflow-state"; +import { portOrdinal } from "@texera/workflow-macros"; // Format "{id}_{internal}" must align with the backend port-identity serializer. function serializePortIdentity(id: number, internal: boolean = false): string { return `${id}_${internal}`; } -function parseLogicalOperatorPortID(portId: string): { portNumber: number; portType: "input" | "output" } | undefined { - const match = portId.match(/^(input|output)-(\d+)$/); - if (!match) { - return undefined; - } - - const portType = match[1] as "input" | "output"; - const portNumber = parseInt(match[2]); - - return { portNumber, portType }; -} - function getInputLinksByOperatorId(operatorId: string, links: OperatorLink[]): OperatorLink[] { return links.filter(link => link.target.operatorID === operatorId); } @@ -53,7 +42,8 @@ export function extractOperatorInputPortSchemaMap( operatorId: string, operator: OperatorPredicate, outputSchemas: Record, - links: OperatorLink[] + links: OperatorLink[], + operators: readonly OperatorPredicate[] ): OperatorPortSchemaMap | undefined { const inputLinks = getInputLinksByOperatorId(operatorId, links); if (!inputLinks.length) return undefined; @@ -64,11 +54,9 @@ export function extractOperatorInputPortSchemaMap( const portId = serializePortIdentity(portIndex, false); inputPortSchemaMap[portId] = undefined; - const linksToThisPort = inputLinks.filter(link => { - const parsedPort = parseLogicalOperatorPortID(link.target.portID); - if (!parsedPort) return false; - return parsedPort.portNumber === portIndex; - }); + const linksToThisPort = inputLinks.filter( + link => portOrdinal(operator, "inputPorts", link.target.portID) === portIndex + ); if (linksToThisPort.length > 0) { const schemas: (PortSchema | undefined)[] = linksToThisPort.map(link => { @@ -77,12 +65,13 @@ export function extractOperatorInputPortSchemaMap( return undefined; } - const outputPort = parseLogicalOperatorPortID(link.source.portID); - if (!outputPort) { + const source = operators.find(op => op.operatorID === link.source.operatorID); + if (!source) { return undefined; } - - return sourcePortSchemaMap[serializePortIdentity(outputPort.portNumber, false)]; + return sourcePortSchemaMap[ + serializePortIdentity(portOrdinal(source, "outputPorts", link.source.portID), false) + ]; }); // Unlike the frontend, we don't flag mismatched schemas as a compilation diff --git a/agent-service/src/agent/workflow-state.spec.ts b/agent-service/src/agent/workflow-state.spec.ts index 4e8abc3aa08..f053736d7d3 100644 --- a/agent-service/src/agent/workflow-state.spec.ts +++ b/agent-service/src/agent/workflow-state.spec.ts @@ -18,6 +18,7 @@ */ import { describe, expect, test } from "bun:test"; +import { macroWorkflow } from "./macro-test-data"; import { WorkflowState } from "./workflow-state"; import type { OperatorPredicate, OperatorLink, WorkflowContent } from "../types/workflow"; @@ -43,6 +44,23 @@ function makeLink(linkId: string, sourceId: string, targetId: string): OperatorL } describe("WorkflowState - operators", () => { + test("rejects colliding additions and imports without replacing the current graph", () => { + const state = new WorkflowState(); + state.addOperator(makeOperator("a")); + state.addOperator(makeOperator("b")); + const before = state.getWorkflowContent(); + expect(() => state.addLink(makeLink("a", "a", "b"))).toThrow(/Duplicate graph element ID/); + expect(() => state.setWorkflowContent({ ...before, links: [makeLink("a", "a", "b")] })).toThrow( + /Duplicate graph element ID/ + ); + expect(state.getWorkflowContent()).toEqual(before); + state.addLink(makeLink("edge", "a", "b")); + expect(() => state.addOperator(makeOperator("edge"))).toThrow(/Duplicate graph element ID/); + state.addOperator(makeOperator("link-1")); + expect(state.generateLinkId()).toBe("link-2"); + state.addOperator(makeOperator("TestOp-operator-1")); + expect(state.generateOperatorId("TestOp")).toBe("TestOp-operator-2"); + }); test("add and get operator round-trips", () => { const state = new WorkflowState(); const op = makeOperator("op1"); @@ -176,6 +194,27 @@ describe("WorkflowState - getSubDAG", () => { }); describe("WorkflowState - toLogicalPlan", () => { + test("keeps the empty plan and legacy missing-port fallback without serializing dangling links", () => { + const state = new WorkflowState(); + expect(state.toLogicalPlan()).toEqual({ operators: [], links: [], opsToViewResult: [], opsToReuseResult: [] }); + state.addOperator(makeOperator("café", { operatorProperties: { operatorID: "user value", value: "東京" } })); + state.addOperator(makeOperator("dst")); + state.addLink({ + linkID: "missing ports", + source: { operatorID: "café", portID: "missing" }, + target: { operatorID: "dst", portID: "missing" }, + }); + state.addLink(makeLink("dangling", "absent", "dst")); + const before = JSON.stringify(state.getWorkflowContent()); + const plan = state.toLogicalPlan(); + expect(plan.operators[0]).toMatchObject({ operatorID: "café", value: "東京" }); + expect(plan.links).toEqual([ + { fromOpId: "café", fromPortId: { id: 0, internal: false }, toOpId: "dst", toPortId: { id: 0, internal: false } }, + ]); + expect(JSON.stringify(state.getWorkflowContent())).toBe(before); + state.destroy(); + }); + test("produces operators, port-indexed links, and an empty reuse list", () => { const state = new WorkflowState(); state.addOperator(makeOperator("op1")); @@ -310,3 +349,41 @@ describe("WorkflowState - workflow content round-trip", () => { expect("formBinding" in state.getWorkflowContent()).toBe(false); }); }); + +describe("macro workflow compatibility", () => { + test("sends nested pins intact for full and run-up-to selections", () => { + const content = macroWorkflow(); + const before = JSON.stringify(content); + const state = new WorkflowState(); + state.setWorkflowContent(content); + for (const target of [undefined, "one"]) { + const plan = state.toLogicalPlan(target); + const ids = target ? [target] : ["one", "two"]; + expect(plan).toMatchObject({ operators: [], links: [], macroContent: content, executeOperatorIds: ids }); + if (target) expect(plan.opsToViewResult).toEqual([target]); + expect(state.toLogicalPlan(target)).toEqual(plan); + } + expect(state.toLogicalPlan("absent").operators).toEqual([]); + expect(state.getWorkflowContent()).toEqual(content); + expect(JSON.stringify(content)).toBe(before); + state.destroy(); + }); + + test("preserves pinned snapshots and definition interfaces through agent edits and reset", () => { + const original = macroWorkflow(); + const state = new WorkflowState(); + state.setWorkflowContent(original); + state.updateOperatorDisplayName("one", "Renamed"); + expect(state.getWorkflowContent()).toEqual({ + ...original, + operators: [{ ...original.operators[0], customDisplayName: "Renamed" }, original.operators[1]], + }); + const definition = original.macroDependencies[0].content; + state.setWorkflowContent(definition); + expect(state.getWorkflowContent()).toEqual(definition); + state.reset(); + expect(state.getWorkflowContent()).not.toHaveProperty("macroDependencies"); + expect(state.getWorkflowContent()).not.toHaveProperty("macroInterface"); + state.destroy(); + }); +}); diff --git a/agent-service/src/agent/workflow-state.ts b/agent-service/src/agent/workflow-state.ts index e91f78e97a4..36141666714 100644 --- a/agent-service/src/agent/workflow-state.ts +++ b/agent-service/src/agent/workflow-state.ts @@ -17,14 +17,13 @@ * under the License. */ +import { assertWorkflowIDs, isMacro, toLogicalGraph } from "@texera/workflow-macros"; import { Subject, Observable, merge, Subscription } from "rxjs"; import type { OperatorPredicate, OperatorLink, WorkflowContent, LogicalPlan, - LogicalOperator, - LogicalLink, Point, CommentBox, WorkflowSettings, @@ -48,9 +47,7 @@ export class WorkflowState { private operatorPositions: Map = new Map(); private commentBoxes: CommentBox[] = []; private settings: WorkflowSettings = { ...DEFAULT_WORKFLOW_SETTINGS }; - // Opaque Form View definition carried through unchanged (the agent never touches it); - // see WorkflowContent.formBinding. - private formBinding: unknown = undefined; + private contentMeta: Pick = {}; private operatorsToViewResult: Set = new Set(); private operatorIdCounter: number = 0; @@ -84,14 +81,27 @@ export class WorkflowState { } generateOperatorId(operatorType: string): string { - return `${operatorType}-operator-${++this.operatorIdCounter}`; + let id: string; + do { + id = `${operatorType}-operator-${++this.operatorIdCounter}`; + } while (this.hasGraphID(id)); + return id; } generateLinkId(): string { - return `link-${++this.linkIdCounter}`; + let id: string; + do { + id = `link-${++this.linkIdCounter}`; + } while (this.hasGraphID(id)); + return id; + } + + private hasGraphID(id: string): boolean { + return this.operators.has(id) || this.links.has(id) || this.commentBoxes.some(box => box.commentBoxID === id); } addOperator(operator: OperatorPredicate, position?: Point): void { + assertWorkflowIDs({ ...this.getWorkflowContent(), operators: [...this.getAllOperators(), operator] }); this.operators.set(operator.operatorID, operator); const defaultPosition: Point = position || { x: 100 + (this.operators.size - 1) * 200, @@ -173,6 +183,7 @@ export class WorkflowState { } addLink(link: OperatorLink): void { + assertWorkflowIDs({ ...this.getWorkflowContent(), links: [...this.getAllLinks(), link] }); this.links.set(link.linkID, link); this.linkAddSubject.next(link); } @@ -242,71 +253,56 @@ export class WorkflowState { links: this.getAllLinks(), commentBoxes: [...this.commentBoxes], settings: { ...this.settings }, - // Only re-emit the key when the loaded workflow carried one, so plain workflows - // stay byte-identical (JSON.stringify would otherwise add "formBinding":null). - ...(this.formBinding !== undefined ? { formBinding: this.formBinding } : {}), + ...this.contentMeta, }; } setWorkflowContent(content: WorkflowContent): void { + assertWorkflowIDs(content); + const { operators, links, operatorPositions, commentBoxes, settings, ...contentMeta } = content; this.operators.clear(); this.links.clear(); this.operatorPositions.clear(); - for (const op of content.operators) { + for (const op of operators) { this.operators.set(op.operatorID, op); } - for (const link of content.links) { + for (const link of links) { this.links.set(link.linkID, link); } - if (content.operatorPositions) { - for (const [id, pos] of Object.entries(content.operatorPositions)) { + if (operatorPositions) { + for (const [id, pos] of Object.entries(operatorPositions)) { this.operatorPositions.set(id, pos); } } - this.commentBoxes = content.commentBoxes ? [...content.commentBoxes] : []; + this.commentBoxes = commentBoxes ? [...commentBoxes] : []; - this.settings = content.settings ? { ...content.settings } : { ...DEFAULT_WORKFLOW_SETTINGS }; + this.settings = settings ? { ...settings } : { ...DEFAULT_WORKFLOW_SETTINGS }; - this.formBinding = content.formBinding; + this.contentMeta = contentMeta; } - toLogicalPlan(): LogicalPlan { - const enabledOperators = this.getAllEnabledOperators(); - - const operators: LogicalOperator[] = enabledOperators.map(op => ({ - operatorID: op.operatorID, - operatorType: op.operatorType, - ...op.operatorProperties, - inputPorts: op.inputPorts, - outputPorts: op.outputPorts, - })); - - const operatorIds = new Set(operators.map(op => op.operatorID)); - - const links: LogicalLink[] = this.getAllLinks() - .filter(link => operatorIds.has(link.source.operatorID) && operatorIds.has(link.target.operatorID)) - .map(link => { - const sourceOp = this.getOperator(link.source.operatorID)!; - const targetOp = this.getOperator(link.target.operatorID)!; - - const fromPortIdx = sourceOp.outputPorts.findIndex(p => p.portID === link.source.portID); - const toPortIdx = targetOp.inputPorts.findIndex(p => p.portID === link.target.portID); - - return { - fromOpId: link.source.operatorID, - fromPortId: { id: fromPortIdx >= 0 ? fromPortIdx : 0, internal: false }, - toOpId: link.target.operatorID, - toPortId: { id: toPortIdx >= 0 ? toPortIdx : 0, internal: false }, - }; - }); - + toLogicalPlan(targetOperatorId?: string): LogicalPlan { + let graph = targetOperatorId + ? this.getSubDAG(targetOperatorId) + : { operators: this.getAllEnabledOperators(), links: this.getAllLinks() }; + const results = targetOperatorId !== undefined ? [targetOperatorId] : [...this.operatorsToViewResult]; + if (this.contentMeta.macroInterface || this.getAllOperators().some(isMacro)) + return { + operators: [], + links: [], + macroContent: this.getWorkflowContent(), + executeOperatorIds: graph.operators.map(op => op.operatorID), + opsToViewResult: results, + opsToReuseResult: [], + }; + const ids = new Set(graph.operators.map(op => op.operatorID)); + const links = graph.links.filter(link => ids.has(link.source.operatorID) && ids.has(link.target.operatorID)); return { - operators, - links, - opsToViewResult: Array.from(this.operatorsToViewResult).filter(id => operatorIds.has(id)), + ...toLogicalGraph({ ...graph, links }, 0), + opsToViewResult: results.filter(id => ids.has(id)), opsToReuseResult: [], }; } @@ -321,7 +317,7 @@ export class WorkflowState { this.operatorPositions.clear(); this.commentBoxes = []; this.settings = { ...DEFAULT_WORKFLOW_SETTINGS }; - this.formBinding = undefined; + this.contentMeta = {}; this.operatorsToViewResult.clear(); } diff --git a/agent-service/src/api/compile-api.spec.ts b/agent-service/src/api/compile-api.spec.ts index 498f22b54be..27267c30db7 100644 --- a/agent-service/src/api/compile-api.spec.ts +++ b/agent-service/src/api/compile-api.spec.ts @@ -17,8 +17,10 @@ * under the License. */ +import { WorkflowFatalErrorType } from "../types/execution"; import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import { compileWorkflowAsync, type WorkflowCompilationResponse } from "./compile-api"; +import { macroWorkflow } from "../agent/macro-test-data"; import type { LogicalPlan } from "../types/workflow"; const plan = { @@ -53,6 +55,38 @@ describe("compileWorkflowAsync", () => { }); }); + test.each([ + [500, undefined], + [401, "expired-token"], + ] as const)("returns null for HTTP %d", async (status, token) => { + spyOn(globalThis, "fetch").mockResolvedValue(new Response(null, { status })); + expect(await compileWorkflowAsync(plan, token)).toBeNull(); + }); + + test("sends the authenticated pinned bundle and preserves the compiler's boundary schemas and nested diagnostics", async () => { + const macroContent = macroWorkflow(); + const compilation: WorkflowCompilationResponse = { + operatorOutputSchemas: { one: { "0_false": [{ attributeName: "value", attributeType: "string" }] } }, + operatorErrors: { + one: { + operatorId: "one", + message: "Outer / Inner / Scan: invalid column", + type: { name: WorkflowFatalErrorType.COMPILATION_ERROR }, + timestamp: { seconds: 0, nanos: 0 }, + workerId: "", + details: "", + }, + }, + macroExpansion: { content: { ...macroContent, operators: [], links: [] }, paths: {}, ports: {} }, + }; + const fetchSpy = spyOn(globalThis, "fetch").mockResolvedValue(Response.json(compilation)); + const request = { ...plan, macroContent, executeOperatorIds: ["one"] }; + expect(await compileWorkflowAsync(request, "user-token")).toEqual(compilation); + const [, init] = fetchSpy.mock.calls[0]; + expect(init?.headers).toEqual({ Authorization: "Bearer user-token", "Content-Type": "application/json" }); + expect(JSON.parse(init!.body as string)).toMatchObject({ macroContent, executeOperatorIds: ["one"] }); + }); + test("sends the user's token as a Bearer Authorization header when given", async () => { const fetchSpy = spyOn(globalThis, "fetch").mockResolvedValue( new Response(JSON.stringify({ operatorOutputSchemas: {}, operatorErrors: {} }), { status: 200 }) diff --git a/agent-service/src/api/compile-api.ts b/agent-service/src/api/compile-api.ts index 7b11bd2cfc1..aecc42fec03 100644 --- a/agent-service/src/api/compile-api.ts +++ b/agent-service/src/api/compile-api.ts @@ -22,6 +22,8 @@ import { createAuthHeaders } from "./auth-api"; import type { LogicalPlan, OperatorPortSchemaMap } from "../types/workflow"; import type { WorkflowFatalError } from "../types/execution"; import { createLogger } from "../logger"; +import type { MacroExpansion } from "@texera/workflow-macros"; +import type { OperatorPredicate, WorkflowContent } from "../types/workflow"; // WorkflowFatalError is defined in types/execution.ts (shared by compile and // execution errors); re-exported here for existing importers of this module. @@ -37,6 +39,7 @@ export interface SchemaAttribute { export type PortSchema = ReadonlyArray; export interface WorkflowCompilationResponse { + macroExpansion?: MacroExpansion; physicalPlan?: any; operatorOutputSchemas: Record; operatorErrors: Record; @@ -50,8 +53,7 @@ export async function compileWorkflowAsync( const url = `${config.compileEndpoint}/api/compile`; const body = { - operators: logicalPlan.operators, - links: logicalPlan.links, + ...logicalPlan, opsToReuseResult: [], opsToViewResult: [], }; @@ -75,3 +77,19 @@ export async function compileWorkflowAsync( return null; } } + +export async function validateMacroPropertiesAsync( + operator: OperatorPredicate, + macroDependencies: WorkflowContent["macroDependencies"], + token?: string +): Promise { + const response = await fetch(`${getBackendConfig().compileEndpoint}/api/compile/validate-properties`, { + method: "POST", + headers: token ? createAuthHeaders(token) : { "Content-Type": "application/json" }, + body: JSON.stringify({ operator, macroDependencies }), + }); + if (!response.ok) { + const error = (await response.json().catch(() => ({}))) as { message?: string }; + throw new Error(error.message ?? `Macro validation failed (${response.status})`); + } +} diff --git a/agent-service/src/api/execution-api.ts b/agent-service/src/api/execution-api.ts index ad40758f39e..8cb21cab904 100644 --- a/agent-service/src/api/execution-api.ts +++ b/agent-service/src/api/execution-api.ts @@ -17,6 +17,8 @@ * under the License. */ +export type { LogicalLink, LogicalPlan } from "../types/workflow"; + import { z } from "zod"; import { ConsoleMessageType, @@ -31,26 +33,6 @@ import { type WorkflowFatalError, } from "../types/execution"; -export interface LogicalLink { - fromOpId: string; - fromPortId: { id: number; internal: boolean }; - toOpId: string; - toPortId: { id: number; internal: boolean }; -} - -interface LogicalOperator { - operatorID: string; - operatorType: string; - [key: string]: any; -} - -export interface LogicalPlan { - operators: LogicalOperator[]; - links: LogicalLink[]; - opsToViewResult?: string[]; - opsToReuseResult?: string[]; -} - /** * TEMPORARY BACKEND COMPATIBILITY LAYER. * diff --git a/agent-service/src/types/workflow.ts b/agent-service/src/types/workflow.ts index cc564db6cff..a469527acc8 100644 --- a/agent-service/src/types/workflow.ts +++ b/agent-service/src/types/workflow.ts @@ -17,51 +17,21 @@ * under the License. */ -interface LogicalPort { - readonly operatorID: string; - readonly portID: string; -} +import type { + MacroDefinition, + OperatorPredicate as CoreOperator, + OperatorLink, + PortDescription, +} from "@texera/workflow-macros"; +export type { OperatorLink, PortDescription } from "@texera/workflow-macros"; interface PortIdentity { readonly id: number; readonly internal: boolean; } -type PartitionInfo = - | { readonly type: "hash"; readonly hashAttributeNames: string[] } - | { - readonly type: "range"; - readonly rangeAttributeNames: string[]; - readonly rangeMin: number; - readonly rangeMax: number; - } - | { readonly type: "single" } - | { readonly type: "broadcast" } - | { readonly type: "none" }; - -export interface PortDescription { - readonly portID: string; - readonly displayName?: string; - readonly disallowMultiInputs?: boolean; - readonly isDynamicPort?: boolean; - readonly partitionRequirement?: PartitionInfo; - readonly dependencies?: { id: number; internal: boolean }[]; -} - -export interface OperatorPredicate { - readonly operatorID: string; - readonly operatorType: string; - readonly operatorVersion: string; +export interface OperatorPredicate extends CoreOperator { readonly operatorProperties: Record; - readonly inputPorts: PortDescription[]; - readonly outputPorts: PortDescription[]; - readonly dynamicInputPorts?: boolean; - readonly dynamicOutputPorts?: boolean; - readonly showAdvanced: boolean; - readonly isDisabled?: boolean; - readonly viewResult?: boolean; - readonly markedForReuse?: boolean; - readonly customDisplayName?: string; } export interface LogicalOperator { @@ -70,12 +40,6 @@ export interface LogicalOperator { readonly [key: string]: any; } -export interface OperatorLink { - readonly linkID: string; - readonly source: LogicalPort; - readonly target: LogicalPort; -} - export interface LogicalLink { readonly fromOpId: string; readonly fromPortId: PortIdentity; @@ -84,6 +48,8 @@ export interface LogicalLink { } export interface LogicalPlan { + readonly macroContent?: import("@texera/workflow-macros").MacroContent; + readonly executeOperatorIds?: string[]; readonly operators: LogicalOperator[]; readonly links: LogicalLink[]; readonly opsToViewResult?: string[]; @@ -119,6 +85,8 @@ export interface WorkflowContent { // save untouched -- rebuilding content without it would leave a workflow whose default_view // is form with a form that has no fields. Carried through opaquely. readonly formBinding?: unknown; + readonly macroInterface?: { id: string }; + readonly macroDependencies?: MacroDefinition[]; } type AttributeType = "string" | "integer" | "double" | "boolean" | "long" | "timestamp" | "binary"; diff --git a/agent-service/tsconfig.json b/agent-service/tsconfig.json index 5f0da1e5bee..768a709c60e 100644 --- a/agent-service/tsconfig.json +++ b/agent-service/tsconfig.json @@ -3,6 +3,7 @@ "target": "ESNext", "module": "Preserve", "moduleResolution": "bundler", + "paths": { "@texera/workflow-macros": ["../common/workflow-core/src/main/typescript/workflow-macros/index.ts"] }, "lib": ["ESNext"], "types": ["bun-types"], diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala index 6030e6748e9..5768885f7e9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/Coordinator.scala @@ -150,9 +150,7 @@ class Coordinator( (region.id.id, region.physicalOps.map(_.id.logicalOpId.id).toList) } - SessionState.getAllSessionStates.foreach { state => - state.send(RegionUpdateEvent(regions)) - } + SessionState.sendToWorkflow(workflowContext.workflowId, RegionUpdateEvent(regions)) val coordinatorRestoreConf = coordinatorConfig.stateRestoreConfOpt if (coordinatorRestoreConf.isDefined) { diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala index 303fba9a1f7..99bcc293051 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/coordinator/execution/ExecutionUtils.scala @@ -60,23 +60,12 @@ object ExecutionUtils { aggregatePortMetrics(filteredMetrics) } - val inputMetricsSum = sumMetrics(_.operatorStatistics.inputMetrics) - val outputMetricsSum = sumMetrics(_.operatorStatistics.outputMetrics) - - val numWorkersSum = metrics.map(_.operatorStatistics.numWorkers).sum - val dataProcessingTimeSum = metrics.map(_.operatorStatistics.dataProcessingTime).sum - val controlProcessingTimeSum = metrics.map(_.operatorStatistics.controlProcessingTime).sum - val idleTimeSum = metrics.map(_.operatorStatistics.idleTime).sum - OperatorMetrics( aggregatedState, - OperatorStatistics( - inputMetricsSum, - outputMetricsSum, - numWorkersSum, - dataProcessingTimeSum, - controlProcessingTimeSum, - idleTimeSum + aggregateStatistics( + metrics, + sumMetrics(_.operatorStatistics.inputMetrics), + sumMetrics(_.operatorStatistics.outputMetrics) ), // Fully-reused semantics: partial reuse is possible (HashJoin's build and // probe sit in different regions), and a partially reused operator reports @@ -88,6 +77,21 @@ object ExecutionUtils { ) } + /** Sum worker costs once per operator; callers supply the ports visible at their boundary. */ + def aggregateStatistics( + metrics: Iterable[OperatorMetrics], + inputs: Seq[PortTupleMetricsMapping], + outputs: Seq[PortTupleMetricsMapping] + ): OperatorStatistics = + OperatorStatistics( + inputs, + outputs, + metrics.iterator.map(_.operatorStatistics.numWorkers).sum, + metrics.iterator.map(_.operatorStatistics.dataProcessingTime).sum, + metrics.iterator.map(_.operatorStatistics.controlProcessingTime).sum, + metrics.iterator.map(_.operatorStatistics.idleTime).sum + ) + def aggregateStates[T]( states: Iterable[T], completedState: T, diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala index a012538a5bb..af026d7e1b3 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManager.scala @@ -725,8 +725,8 @@ class RegionExecutionManager( private def setPhase(phase: RegionExecutionPhase): Unit = { currentPhaseRef.set(phase) - SessionState.getAllSessionStates.foreach { state => - state.send(RegionStateEvent(region.id.id, phase.toString)) + region.physicalOps.headOption.foreach { op => + SessionState.sendToWorkflow(op.workflowId, RegionStateEvent(region.id.id, phase.toString)) } } diff --git a/amber/src/main/scala/org/apache/texera/web/SessionState.scala b/amber/src/main/scala/org/apache/texera/web/SessionState.scala index b50df866afe..05cc0597a27 100644 --- a/amber/src/main/scala/org/apache/texera/web/SessionState.scala +++ b/amber/src/main/scala/org/apache/texera/web/SessionState.scala @@ -21,6 +21,7 @@ package org.apache.texera.web import io.reactivex.rxjava3.disposables.Disposable import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.core.virtualidentity.WorkflowIdentity import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum import org.apache.texera.web.model.websocket.event.TexeraWebSocketEvent import org.apache.texera.web.service.WorkflowService @@ -48,6 +49,12 @@ object SessionState { sessionIdToSessionState.values } + def sendToWorkflow(workflowId: WorkflowIdentity, msg: TexeraWebSocketEvent): Unit = { + getAllSessionStates + .filter(_.getCurrentWorkflowState.exists(_.workflowId == workflowId)) + .foreach(_.send(msg)) + } + } class SessionState(session: Session) { diff --git a/amber/src/main/scala/org/apache/texera/web/model/websocket/event/OperatorStatisticsUpdateEvent.scala b/amber/src/main/scala/org/apache/texera/web/model/websocket/event/OperatorStatisticsUpdateEvent.scala index af91f657843..7820d88a33f 100644 --- a/amber/src/main/scala/org/apache/texera/web/model/websocket/event/OperatorStatisticsUpdateEvent.scala +++ b/amber/src/main/scala/org/apache/texera/web/model/websocket/event/OperatorStatisticsUpdateEvent.scala @@ -19,6 +19,9 @@ package org.apache.texera.web.model.websocket.event +import org.apache.texera.amber.engine.common.Utils.aggregatedStateToString +import org.apache.texera.amber.engine.common.executionruntimestate.OperatorMetrics + case class OperatorAggregatedMetrics( operatorState: String, aggregatedInputRowCount: Long, @@ -34,8 +37,34 @@ case class OperatorAggregatedMetrics( // Provenance: the operator completed by reusing cached results (no workers ran). // Deliberately no default: a new construction site must decide the flag // explicitly instead of silently sending false. - reusedFromCache: Boolean + reusedFromCache: Boolean, + inputPortSizes: Map[String, Long] = Map.empty, + outputPortSizes: Map[String, Long] = Map.empty ) +object OperatorAggregatedMetrics { + def from(metrics: OperatorMetrics): OperatorAggregatedMetrics = { + val stats = metrics.operatorStatistics + val inputs = stats.inputMetrics.map(p => p.portId.id.toString -> p.tupleMetrics).toMap + val outputs = stats.outputMetrics.map(p => p.portId.id.toString -> p.tupleMetrics).toMap + OperatorAggregatedMetrics( + aggregatedStateToString(metrics.operatorState), + stats.inputMetrics.map(_.tupleMetrics.count).sum, + stats.inputMetrics.map(_.tupleMetrics.size).sum, + inputs.view.mapValues(_.count).toMap, + stats.outputMetrics.map(_.tupleMetrics.count).sum, + stats.outputMetrics.map(_.tupleMetrics.size).sum, + outputs.view.mapValues(_.count).toMap, + stats.numWorkers, + stats.dataProcessingTime, + stats.controlProcessingTime, + stats.idleTime, + metrics.reusedFromCache, + inputs.view.mapValues(_.size).toMap, + outputs.view.mapValues(_.size).toMap + ) + } +} + case class OperatorStatisticsUpdateEvent(operatorStatistics: Map[String, OperatorAggregatedMetrics]) extends TexeraWebSocketEvent diff --git a/amber/src/main/scala/org/apache/texera/web/model/websocket/event/WorkflowStateEvent.scala b/amber/src/main/scala/org/apache/texera/web/model/websocket/event/WorkflowStateEvent.scala index 3bad65234fe..ba543c30efa 100644 --- a/amber/src/main/scala/org/apache/texera/web/model/websocket/event/WorkflowStateEvent.scala +++ b/amber/src/main/scala/org/apache/texera/web/model/websocket/event/WorkflowStateEvent.scala @@ -19,4 +19,5 @@ package org.apache.texera.web.model.websocket.event -case class WorkflowStateEvent(state: String) extends TexeraWebSocketEvent +case class WorkflowStateEvent(state: String, executionId: Option[Long] = None) + extends TexeraWebSocketEvent diff --git a/amber/src/main/scala/org/apache/texera/web/model/websocket/request/TexeraWebSocketRequest.scala b/amber/src/main/scala/org/apache/texera/web/model/websocket/request/TexeraWebSocketRequest.scala index dc0cba77fbb..6b39b9b903b 100644 --- a/amber/src/main/scala/org/apache/texera/web/model/websocket/request/TexeraWebSocketRequest.scala +++ b/amber/src/main/scala/org/apache/texera/web/model/websocket/request/TexeraWebSocketRequest.scala @@ -57,7 +57,8 @@ case class ResultPaginationRequest( columnLimit: Int = Int.MaxValue, columnSearch: Option[String] = None, // which of the operator's output ports to read; an operator with one output has only port 0 - portIndex: Int = 0 + portIndex: Int = 0, + executionId: Option[Long] = None ) extends TexeraWebSocketRequest case class RetryRequest(workers: Seq[String]) extends TexeraWebSocketRequest case class SkipTupleRequest(workerIds: Array[String]) extends TexeraWebSocketRequest diff --git a/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala index 400d7bbc877..045b36c276a 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala @@ -154,6 +154,18 @@ class SyncExecutionResource extends LazyLogging { ) try { + // Validate the complete pinned graph before touching an existing execution. + require( + !request.logicalPlan.macroContent.exists(_.hasNonNull("macroInterface")), + "Run macro definitions inside a workflow" + ) + val normalized = request.logicalPlan.normalized + val targetOperatorIds = + if (request.logicalPlan.macroContent.isDefined) normalized.opsToViewResult + else request.targetOperatorIds + val resultPorts = request.logicalPlan.macroResultPorts + val effectiveLogicalPlan = computeSubDAGIfNeeded(normalized, targetOperatorIds) + val workflowService = WorkflowService.getOrCreate( WorkflowIdentity(workflowId), computingUnitId @@ -168,10 +180,6 @@ class SyncExecutionResource extends LazyLogging { shutdownPreviousExecution(workflowService) - // "Execute To" semantics: when a single target is given, run only its upstream sub-DAG. - val effectiveLogicalPlan = - computeSubDAGIfNeeded(request.logicalPlan, request.targetOperatorIds) - val executeRequest = WorkflowExecuteRequest( executionName = request.executionName, engineVersion = "1.0", @@ -189,7 +197,8 @@ class SyncExecutionResource extends LazyLogging { workflowService.initExecutionService( executeRequest, Some(user.getUser), - new URI(s"sync-execution://$workflowId") + new URI(s"sync-execution://$workflowId"), + macroSnapshot = request.logicalPlan.macroContent ) val executionService = workflowService.executionService.getValue @@ -216,14 +225,14 @@ class SyncExecutionResource extends LazyLogging { // input port to be present in the operator's input metrics — port-1 stats only appear // once probe actually starts consuming, which closes the race. val targetExpectedExternalInputs: Map[String, Int] = effectiveLogicalPlan.operators - .filter(op => request.targetOperatorIds.contains(op.operatorIdentifier.id)) + .filter(op => targetOperatorIds.contains(op.operatorIdentifier.id)) .map(op => op.operatorIdentifier.id -> op.operatorInfo.inputPorts.count(!_.id.internal)) .toMap // Require COMPLETED, not just "has output", so upstream operators finish flushing // their data downstream before we tear the execution down. def allTargetsCompleted(stats: ExecutionStatsStore): Boolean = { - request.targetOperatorIds.nonEmpty && request.targetOperatorIds.forall { opId => + targetOperatorIds.nonEmpty && targetOperatorIds.forall { opId => stats.operatorInfo.get(opId).exists { metrics => val externalInputPortsReporting = metrics.operatorStatistics.inputMetrics.count(!_.portId.internal) @@ -326,10 +335,11 @@ class SyncExecutionResource extends LazyLogging { val operatorInfos = collectOperatorInfos( executionId, executionService, - request.targetOperatorIds, + targetOperatorIds, maxOperatorResultCharLimit, maxOperatorResultCellCharLimit, - inMemoryConsoleState + inMemoryConsoleState, + resultPorts ) val fatalErrors = finalState.fatalErrors @@ -399,7 +409,8 @@ class SyncExecutionResource extends LazyLogging { targetOperatorIds: List[String], maxOperatorResultCharLimit: Int, maxOperatorResultCellCharLimit: Int, - inMemoryConsoleState: Option[ExecutionConsoleStore] = None + inMemoryConsoleState: Option[ExecutionConsoleStore] = None, + resultPorts: Map[String, (OperatorIdentity, PortIdentity)] = Map.empty ): Map[String, OperatorInfo] = { val operatorInfos = mutable.Map[String, OperatorInfo]() @@ -420,14 +431,20 @@ class SyncExecutionResource extends LazyLogging { } .getOrElse(List.empty) - val targetOps = (baseTargetOps ++ consoleErrorOps).distinct + val targetOps = (baseTargetOps ++ consoleErrorOps ++ resultPorts.keys).distinct - for (opId <- targetOps) { + for (resultId <- targetOps) { + val (operator, port) = + resultPorts.getOrElse(resultId, (OperatorIdentity(resultId), PortIdentity())) + val opId = operator.id val stats = operatorStats.get(opId) val (state, inputTuples, outputTuples): (String, Long, Long) = stats match { case Some(s) => val inputCount = s.operatorStatistics.inputMetrics.map(_.tupleMetrics.count).sum - val outputCount = s.operatorStatistics.outputMetrics.map(_.tupleMetrics.count).sum + val outputCount = s.operatorStatistics.outputMetrics + .filter(pm => !resultPorts.contains(resultId) || pm.portId == port) + .map(_.tupleMetrics.count) + .sum (stateToString(s.operatorState), inputCount, outputCount) case None => ("Unknown", 0L, 0L) } @@ -445,7 +462,8 @@ class SyncExecutionResource extends LazyLogging { executionId, opId, maxOperatorResultCharLimit, - maxOperatorResultCellCharLimit + maxOperatorResultCellCharLimit, + port ) // DB is authoritative once written; fall back to in-memory state for in-flight runs @@ -482,7 +500,7 @@ class SyncExecutionResource extends LazyLogging { .map(_.filter(_.title.startsWith("WARNING: ")).map(_.title)) .filter(_.nonEmpty) - operatorInfos(opId) = OperatorInfo( + operatorInfos(resultId) = OperatorInfo( state = state, inputTuples = inputTuples, outputTuples = outputTuples, @@ -503,7 +521,7 @@ class SyncExecutionResource extends LazyLogging { private def handleExecutionError(e: Exception): SyncExecutionResult = { val errorMsg = e.getMessage - val isCompilationError = errorMsg != null && ( + val isCompilationError = e.isInstanceOf[IllegalArgumentException] || errorMsg != null && ( errorMsg.contains("compilation") || errorMsg.contains("Compilation") || errorMsg.contains("operator") || @@ -538,7 +556,8 @@ class SyncExecutionResource extends LazyLogging { executionId: ExecutionIdentity, opId: String, maxOperatorResultCharLimit: Int, - maxOperatorResultCellCharLimit: Int + maxOperatorResultCellCharLimit: Int, + port: PortIdentity ): (String, Option[Any], Option[Int], Option[Int], Option[Boolean]) = { import com.fasterxml.jackson.databind.node.ObjectNode @@ -546,7 +565,7 @@ class SyncExecutionResource extends LazyLogging { val storageUriOption = ExecutionResultUris.getResultUriByLogicalPortId( executionId, OperatorIdentity(opId), - PortIdentity() + port ) storageUriOption match { @@ -922,7 +941,7 @@ class SyncExecutionResource extends LazyLogging { dfs(targetOpId) - LogicalPlanPojo( + logicalPlan.copy( operators = subDagOperators.toList, links = subDagLinks.toList, opsToViewResult = targetOperatorIds.filter(id => visited.contains(id)), diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala index febca649a9f..edebda5995e 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResource.scala @@ -43,6 +43,8 @@ import org.apache.texera.dao.jooq.generated.enums.UserRoleEnum import org.apache.texera.dao.jooq.generated.tables.daos.WorkflowExecutionsDao import org.apache.texera.dao.jooq.generated.tables.pojos.{WorkflowExecutions, User => UserPojo} import org.apache.texera.web.model.http.request.result.ResultExportRequest +import org.apache.texera.common.compiler.MacroExpansion +import org.apache.texera.web.service.ExecutionStatsService import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource._ import org.apache.texera.web.service.ResultExportService import org.jooq.DSLContext @@ -58,6 +60,21 @@ import scala.collection.mutable import scala.jdk.CollectionConverters._ object WorkflowExecutionsResource { + def ownsExecution(wid: WorkflowIdentity, cuid: Int, eid: Long): Boolean = + eid > 0 && eid <= Int.MaxValue && context.fetchExists( + context + .selectOne() + .from(WORKFLOW_EXECUTIONS) + .join(WORKFLOW_VERSION) + .on(WORKFLOW_VERSION.VID.eq(WORKFLOW_EXECUTIONS.VID)) + .where( + WORKFLOW_VERSION.WID + .eq(wid.id.toInt) + .and(WORKFLOW_EXECUTIONS.CUID.eq(cuid)) + .and(WORKFLOW_EXECUTIONS.EID.eq(eid.toInt)) + ) + ) + private def context: DSLContext = SqlServer .getInstance() @@ -673,6 +690,32 @@ class WorkflowExecutionsResource { } } + @GET + @Produces(Array(MediaType.APPLICATION_JSON)) + @Path("/{wid}/macro-content/{eid}") + @RolesAllowed(Array("REGULAR", "ADMIN")) + def macroContent( + @PathParam("wid") wid: Integer, + @PathParam("eid") eid: Integer, + @Auth user: SessionUser + ): com.fasterxml.jackson.databind.JsonNode = { + validateUserCanAccessWorkflow(user.getUid, wid) + val visible = + if (WorkflowAccess.hasGrantedAccess(wid, user.getUid)) + org.jooq.impl.DSL.trueCondition() + else WORKFLOW_EXECUTIONS.UID.eq(user.getUid) + val content = context + .select(WORKFLOW_EXECUTIONS.MACRO_CONTENT) + .from(WORKFLOW_EXECUTIONS) + .join(WORKFLOW_VERSION) + .on(WORKFLOW_VERSION.VID.eq(WORKFLOW_EXECUTIONS.VID)) + .where(WORKFLOW_VERSION.WID.eq(wid).and(WORKFLOW_EXECUTIONS.EID.eq(eid)).and(visible)) + .fetchOne(WORKFLOW_EXECUTIONS.MACRO_CONTENT) + Option(content) + .map(objectMapper.readTree(_)) + .getOrElse(throw new javax.ws.rs.NotFoundException()) + } + @GET @Produces(Array(MediaType.APPLICATION_JSON)) @Path("/{wid}/stats/{eid}") @@ -682,6 +725,49 @@ class WorkflowExecutionsResource { @PathParam("eid") eid: Integer, @Auth sessionUser: SessionUser ): List[WorkflowRuntimeStatistics] = { + val (document, _) = statisticsDocument(wid, eid, sessionUser) + + // Read all records from Iceberg and convert to WorkflowRuntimeStatistics + document + .get() + .map(tuple => { + val record = tuple.asInstanceOf[Tuple] + WorkflowRuntimeStatistics( + operatorId = record.getField(0).asInstanceOf[String], + timestamp = record.getField(1).asInstanceOf[Timestamp], + inputTupleCount = record.getField(2).asInstanceOf[Long], + inputTupleSize = record.getField(3).asInstanceOf[Long], + outputTupleCount = record.getField(4).asInstanceOf[Long], + outputTupleSize = record.getField(5).asInstanceOf[Long], + dataProcessingTime = record.getField(6).asInstanceOf[Long], + controlProcessingTime = record.getField(7).asInstanceOf[Long], + idleTime = record.getField(8).asInstanceOf[Long], + numWorkers = record.getField(9).asInstanceOf[Int], + status = record.getField(10).asInstanceOf[Int] + ) + }) + .toList + } + + @GET + @Produces(Array(MediaType.APPLICATION_JSON)) + @Path("/{wid}/stats/{eid}/canvas") + @RolesAllowed(Array("REGULAR", "ADMIN")) + def canvasStatistics( + @PathParam("wid") wid: Integer, + @PathParam("eid") eid: Integer, + @Auth user: SessionUser + ): Response = { + val stored = statisticsDocument(wid, eid, user) + if (stored._2.exists(_.containsAttribute(ExecutionStatsService.metricsColumn))) { + val expansion = MacroExpansion.expand(macroContent(wid, eid, user)) + Response + .ok(ExecutionStatsService.restore(expansion, stored._1.get().map(_.asInstanceOf[Tuple]))) + .build() + } else Response.noContent().build() + } + + private def statisticsDocument(wid: Integer, eid: Integer, sessionUser: SessionUser) = { validateUserCanAccessWorkflow(sessionUser.getUser.getUid, wid) // Create URI for runtime statistics @@ -711,28 +797,7 @@ class WorkflowExecutionsResource { val uri: URI = new URI(uriString) // Refuse to read per-user-warehouse statistics while the feature is off (#6930). WarehouseReadGuard.assertReadable(uri) - val document = DocumentFactory.openDocument(uri)._1 - - // Read all records from Iceberg and convert to WorkflowRuntimeStatistics - document - .get() - .map(tuple => { - val record = tuple.asInstanceOf[Tuple] - WorkflowRuntimeStatistics( - operatorId = record.getField(0).asInstanceOf[String], - timestamp = record.getField(1).asInstanceOf[Timestamp], - inputTupleCount = record.getField(2).asInstanceOf[Long], - inputTupleSize = record.getField(3).asInstanceOf[Long], - outputTupleCount = record.getField(4).asInstanceOf[Long], - outputTupleSize = record.getField(5).asInstanceOf[Long], - dataProcessingTime = record.getField(6).asInstanceOf[Long], - controlProcessingTime = record.getField(7).asInstanceOf[Long], - idleTime = record.getField(8).asInstanceOf[Long], - numWorkers = record.getField(9).asInstanceOf[Int], - status = record.getField(10).asInstanceOf[Int] - ) - }) - .toList + DocumentFactory.openDocument(uri) } /** Sets a group of executions' bookmarks to the payload passed in the body. */ diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowPublishService.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowPublishService.scala index 62d15fde230..2a4decf6a66 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowPublishService.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowPublishService.scala @@ -240,6 +240,9 @@ object WorkflowPublishService extends LazyLogging { workflow.getPublishedDefaultView ) + private[workflow] def visibleCopy(workflow: Workflow, granted: Boolean): PublicCopy = + if (granted) workingCopyOf(workflow) else publicCopyOf(workflow) + /** The author's own copy, in the same shape. */ private def workingCopyOf(workflow: Workflow): PublicCopy = PublicCopy( diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala index 0918f3b4b2d..287015cf641 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala @@ -19,7 +19,8 @@ package org.apache.texera.web.resource.dashboard.user.workflow -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.typesafe.scalalogging.LazyLogging import io.dropwizard.auth.Auth @@ -32,6 +33,9 @@ import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity import org.apache.texera.auth.SessionUser import org.apache.texera.auth.util.OwnerAccess import org.apache.texera.auth.util.WorkflowAccess +import org.apache.texera.amber.core.workflow.WorkflowGraphIDs +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.common.compiler.MacroExpansion import org.apache.texera.dao.SqlServer import org.apache.texera.dao.jooq.generated.Tables._ import org.apache.texera.dao.jooq.generated.enums.{DefaultViewEnum, PrivilegeEnum} @@ -46,8 +50,9 @@ import org.apache.texera.service.util.LargeBinaryManager import org.apache.texera.web.resource.dashboard.hub.EntityType import org.apache.texera.web.resource.dashboard.hub.HubResource.recordCloneAction import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowResource._ +import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowVersionResource.MacroPin +import org.jooq.impl.DSL.{coalesce, count, lateral, max, select} import org.jooq.impl.DSL -import org.jooq.impl.DSL.max import org.jooq.{DSLContext, Record10, Result, SelectOnConditionStep, TableField} import java.sql.Timestamp @@ -68,6 +73,13 @@ import scala.util.control.NonFatal */ object WorkflowResource { + private def validatedContent(workflow: Workflow): ObjectNode = + try { + val graph = objectMapper.readTree(workflow.getContent) + WorkflowGraphIDs.validate(graph) + graph.asInstanceOf[ObjectNode] + } catch { case NonFatal(error) => throw new BadRequestException(error.getMessage, error) } + private def context: DSLContext = SqlServer .getInstance() @@ -141,6 +153,17 @@ object WorkflowResource { case class WorkflowIDs(wids: List[Integer]) + case class MacroHead(version: Int, name: String, description: String) + case class MacroSaveStep(workflow: Workflow, head: Option[MacroHead], operatorID: Option[String]) + case class MacroPathSave(steps: List[MacroSaveStep], rootBefore: Workflow) + case class SavedRevision(version: Int, revision: Int, content: JsonNode) + case class MacroUsage(id: String, wid: Integer, path: List[String], label: String) + case class MacroCatalog( + definitions: List[MacroPin], + usages: List[MacroUsage], + errors: List[String] + ) + /** Clears every column that describes a pinned public copy. */ private def clearPublishState(workflow: Workflow): Unit = { workflow.setPublishedVersionId(null) @@ -178,16 +201,10 @@ object WorkflowResource { * up carrying the published graph under a title the author never published. */ private def copyVisibleTo(workflow: Workflow, uid: Integer): WorkflowPublishService.PublicCopy = - if (uid != null && WorkflowAccess.hasGrantedAccess(workflow.getWid, uid)) { - WorkflowPublishService.PublicCopy( - workflow.getName, - workflow.getDescription, - workflow.getContent, - workflow.getDefaultView - ) - } else { - WorkflowPublishService.publicCopyOf(workflow) - } + WorkflowPublishService.visibleCopy( + workflow, + uid != null && WorkflowAccess.hasGrantedAccess(workflow.getWid, uid) + ) /** * One column as the public sees it: the frozen value while a pin is in place, the live one @@ -241,22 +258,54 @@ object WorkflowResource { */ def assignNewOperatorIds(workflowContent: String): String = { val objectMapper = new ObjectMapper().registerModule(DefaultScalaModule) - val operatorIdMap = objectMapper - .readValue(workflowContent, classOf[Map[String, List[Map[String, String]]]])("operators") + val canvas = objectMapper.readTree(workflowContent).asInstanceOf[ObjectNode] + val operators = Option(canvas.get("operators")) + .filter(_.isArray) + .getOrElse(throw new IllegalArgumentException("Missing operators array")) + val operatorIdMap = operators + .elements() + .asScala .map(operator => { - val oldOperatorId = operator("operatorID") - val operatorType = operator("operatorType") + require(operator.path("operatorID").isTextual && operator.path("operatorType").isTextual) + val oldOperatorId = operator.get("operatorID").asText() + val operatorType = operator.get("operatorType").asText() // operator id in frontend: operatorSchema.operatorType + "-operator-" + uuid(); // v4 = UUID.randomUUID().toString val newOperatorId = s"$operatorType-operator-${UUID.randomUUID()}" + operator.asInstanceOf[ObjectNode].put("operatorID", newOperatorId) oldOperatorId -> newOperatorId }) .toMap - // replace all old operator ids with new operator ids - operatorIdMap.foldLeft(workflowContent) { - case (updatedContent, (oldId, newId)) => - updatedContent.replace(oldId, newId) + // Only graph references change; pinned snapshots and user values stay untouched. + Seq("/links", "/formBinding/fields", "/macroInterface/inputs", "/macroInterface/outputs") + .foreach { path => + canvas.at(path).findParents("operatorID").asScala.foreach { node => + val ref = node.asInstanceOf[ObjectNode] + operatorIdMap + .get(ref.path("operatorID").asText()) + .foreach(id => ref.put("operatorID", id)) + } + } + Option(canvas.get("operatorPositions")).collect { case p: ObjectNode => p }.foreach { + positions => + operatorIdMap.foreach { + case (oldId, newId) => + Option(positions.remove(oldId)).foreach(value => positions.set[JsonNode](newId, value)) + } + } + val shown = canvas.at("/formBinding/shownResultIds") + shown.elements().asScala.zipWithIndex.foreach { + case (entry, index) => + operatorIdMap + .get(entry.asText()) + .foreach(id => + shown.asInstanceOf[ArrayNode].set(index, objectMapper.getNodeFactory.textNode(id)) + ) } + Option(canvas.get("macroInterface")).filter(_.isObject).foreach { boundary => + boundary.asInstanceOf[ObjectNode].put("id", UUID.randomUUID().toString) + } + canvas.toString } /** The access join is narrowed to `uid`, so each row carries only the viewer's grant. */ @@ -397,6 +446,60 @@ class WorkflowResource extends LazyLogging { mapWorkflowEntries(workflowEntries, user.getUid) } + @GET + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/macros") + def macroCatalog(@Auth user: SessionUser): MacroCatalog = { + val definitions = List.newBuilder[MacroPin] + val usages = List.newBuilder[MacroUsage] + val errors = List.newBuilder[String] + val version = max(WORKFLOW_VERSION.VID).as("version") + val ordinal = count().as("ordinal") + val head = lateral( + select(version, ordinal).from(WORKFLOW_VERSION).where(WORKFLOW_VERSION.WID.eq(WORKFLOW.WID)) + ).as("head") + context + .select( + WORKFLOW.WID, + WORKFLOW.NAME, + WORKFLOW.CONTENT, + head.field(version), + head.field(ordinal) + ) + .from(WORKFLOW) + .join(WORKFLOW_USER_ACCESS) + .on(WORKFLOW_USER_ACCESS.WID.eq(WORKFLOW.WID)) + .crossJoin(head) + .where( + WORKFLOW_USER_ACCESS.UID + .eq(user.getUid) + .and(WORKFLOW_USER_ACCESS.PRIVILEGE.ne(PrivilegeEnum.NONE)) + ) + .fetch() + .asScala + .foreach { row => + val name = row.value2() + try { + val body = objectMapper.readTree(row.value3()) + val wid = row.value1() + MacroExpansion.references(body).foreach { ref => + usages += MacroUsage(ref.id, wid, ref.path, (name :: ref.names).mkString(" / ")) + } + if (body.hasNonNull("macroInterface")) { + val id = body.path("macroInterface").path("id").textValue() + require(id != null && id.nonEmpty, "Malformed macro definition") + require(row.value4() != null, "Macro has no saved version") + definitions += MacroPin(id, wid, name, row.value4(), row.value5(), body) + } + } catch { case NonFatal(error) => errors += s"$name: ${error.getMessage}" } + } + MacroCatalog( + definitions.result().sortBy(_.name), + usages.result().sortBy(_.label), + errors.result() + ) + } + /** * This method handles the client request to get a specific workflow to be displayed in canvas * at current design, it only takes the workflowID and searches within the database for the matching workflow @@ -448,11 +551,11 @@ class WorkflowResource extends LazyLogging { @RolesAllowed(Array("REGULAR", "ADMIN")) @Path("/persist") def persistWorkflow(workflow: Workflow, @Auth sessionUser: SessionUser): Workflow = { + validatedContent(workflow) val user = sessionUser.getUser if (WorkflowAccess.isOwner(workflow.getWid, user.getUid)) { - WorkflowVersionResource.insertVersion(workflow, insertingNewWorkflow = false) - saveWorkflowFields(workflow) + saveExisting(workflow) } else { if (!WorkflowAccess.hasReadAccess(workflow.getWid, user.getUid)) { // Check if this workflow exists in the database @@ -467,9 +570,7 @@ class WorkflowResource extends LazyLogging { insertWorkflow(workflow, user) WorkflowVersionResource.insertVersion(workflow, insertingNewWorkflow = true) } else if (WorkflowAccess.hasWriteAccess(workflow.getWid, user.getUid)) { - WorkflowVersionResource.insertVersion(workflow, insertingNewWorkflow = false) - // not owner but has write access - saveWorkflowFields(workflow) + saveExisting(workflow) } else { // not owner and no write access -> rejected throw new ForbiddenException("No sufficient access privilege.") @@ -480,6 +581,138 @@ class WorkflowResource extends LazyLogging { workflowDao.fetchOneByWid(wid) } + /** Commit the edited leaf and its parent usages in order, including repeated occurrences of the same workflow. */ + @POST + @Consumes(Array(MediaType.APPLICATION_JSON)) + @Produces(Array(MediaType.APPLICATION_JSON)) + @RolesAllowed(Array("REGULAR", "ADMIN")) + @Path("/persist/macro-path") + def persistMacroPath( + request: MacroPathSave, + @Auth sessionUser: SessionUser + ): List[SavedRevision] = { + if (request == null) throw new BadRequestException("Missing macro save path") + val steps = request.steps + if ( + steps == null || steps.length < 2 || steps.length > 33 || request.rootBefore == null || + steps.exists(s => s == null || s.workflow == null || s.workflow.getWid == null) || + steps.head.operatorID.nonEmpty || steps.tail.exists(_.operatorID.isEmpty) || + steps.init.exists(_.head.isEmpty) || request.rootBefore.getWid != steps.last.workflow.getWid + ) + throw new BadRequestException("Invalid macro save path") + val graphs = steps.map(s => s -> validatedContent(s.workflow)) + val original = validatedContent(request.rootBefore) + val uid = sessionUser.getUser.getUid + def conflict(): Nothing = + throw new WebApplicationException( + "The workflow has a newer saved revision. Your draft is still open.", + 409 + ) + context.transactionResult { config => + val db = config.dsl() + val ids = steps.map(_.workflow.getWid).distinct + val rows = db + .selectFrom(WORKFLOW) + .where(WORKFLOW.WID.in(ids.asJava)) + .orderBy(WORKFLOW.WID) + .forUpdate() + .fetchMap(WORKFLOW.WID) + if ( + ids.exists(id => + !rows + .containsKey(id) || !(WorkflowAccess.getPrivilege(id, uid, db) == PrivilegeEnum.WRITE) + ) + ) + throw new ForbiddenException("The workflow or macro source is read-only.") + def revision(wid: Integer) = + db + .select(coalesce(max(WORKFLOW_VERSION.VID), Int.box(0)), count()) + .from(WORKFLOW_VERSION) + .where(WORKFLOW_VERSION.WID.eq(wid)) + .fetchSingle() + + val before = request.rootBefore + val root = rows.get(before.getWid) + if ( + root.getName != before.getName || root.getDescription != before.getDescription || + objectMapper.readTree(root.getContent) != original + ) conflict() + graphs.init.foreach { + case (s, graph) => + val wid = s.workflow.getWid + val row = rows.get(wid) + val expectedId = objectMapper.readTree(row.getContent).path("macroInterface").path("id") + if ( + !expectedId.isTextual || expectedId != graph.path("macroInterface").path("id") || + !s.head.contains( + MacroHead(revision(wid).value1(), row.getName, row.getDescription) + ) + ) conflict() + } + var child: Option[(Workflow, SavedRevision)] = None + graphs.map { + case (step, graph) => + val workflow = step.workflow + child.foreach { + case (saved, pin) => + val body = pin.content.deepCopy[ObjectNode]() + val id = body.path("macroInterface").path("id").asText() + def operatorType(version: Long): String = s"WorkflowMacro:$id:$version" + val op = graph + .path("operators") + .elements() + .asScala + .collectFirst { + case value: ObjectNode + if value.path("operatorID").asText() == step.operatorID.get => + value + } + .getOrElse(conflict()) + val dependencies = graph.withArray("macroDependencies") + val currentType = op.path("operatorType").asText() + val pinned = dependencies.elements().asScala.exists { d => + d.path("id").asText() == id && operatorType( + d.path("version").asLong() + ) == currentType + } + if (id.isEmpty || !pinned) conflict() + op.put("operatorType", operatorType(pin.version)) + .put("operatorVersion", pin.version.toString) + val nested = + Option(body.remove("macroDependencies")).toList.flatMap(_.elements().asScala) + val definition = objectMapper.valueToTree[ObjectNode]( + MacroPin(id, saved.getWid, saved.getName, pin.version, pin.revision, body) + ) + val merged = scala.collection.mutable.LinkedHashMap.empty[(String, Long), JsonNode] + (dependencies.elements().asScala ++ Iterator.single(definition) ++ nested).foreach { + d => + val key = (d.path("id").asText(), d.path("version").asLong()) + if (merged.get(key).exists(_.path("content") != d.path("content"))) conflict() + merged(key) = d + } + dependencies.removeAll().addAll(merged.values.toList.asJava) + workflow.setContent(graph.toString) + } + saveRevision(workflow, db) + val row = revision(workflow.getWid) + val pin = SavedRevision(row.value1(), row.value2(), graph) + child = Some(workflow -> pin) + pin + } + } + } + + private def saveExisting(workflow: Workflow): Unit = + context.transaction { config => + val db = config.dsl() + db.select(WORKFLOW.WID) + .from(WORKFLOW) + .where(WORKFLOW.WID.eq(workflow.getWid)) + .forUpdate() + .fetch() + saveRevision(workflow, db) + } + /** * Persists a plain save by updating only what a save is: name, description and content. * `is_public` is not written here. Publishing has its own endpoints (/public, /private), and @@ -489,8 +722,9 @@ class WorkflowResource extends LazyLogging { * save failed with 500. `default_view` is likewise owned by /set-default-view alone, and the * timestamps are not rewritten here, so a save can never clobber a concurrent change to either. */ - private def saveWorkflowFields(workflow: Workflow): Unit = { - context + private def saveRevision(workflow: Workflow, db: DSLContext): Unit = { + WorkflowVersionResource.insertVersion(workflow, insertingNewWorkflow = false, db = db) + db .update(WORKFLOW) .set(WORKFLOW.NAME, workflow.getName) .set(WORKFLOW.DESCRIPTION, workflow.getDescription) @@ -597,6 +831,7 @@ class WorkflowResource extends LazyLogging { @RolesAllowed(Array("REGULAR", "ADMIN")) @Path("/create") def createWorkflow(workflow: Workflow, @Auth sessionUser: SessionUser): DashboardWorkflow = { + validatedContent(workflow) val user = sessionUser.getUser if (workflow.getWid != null) { throw new BadRequestException("Cannot create a new workflow with a provided id.") diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResource.scala index dc3984efc0c..40e4afafd2a 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResource.scala @@ -20,13 +20,15 @@ package org.apache.texera.web.resource.dashboard.user.workflow import com.flipkart.zjsonpatch.{JsonDiff, JsonPatch} +import com.fasterxml.jackson.databind.JsonNode import io.dropwizard.auth.Auth import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.texera.auth.SessionUser import org.apache.texera.auth.util.WorkflowAccess import org.apache.texera.common.config.UserSystemConfig import org.apache.texera.dao.SqlServer -import org.apache.texera.dao.jooq.generated.Tables.WORKFLOW_VERSION +import org.apache.texera.dao.jooq.generated.Tables.{WORKFLOW, WORKFLOW_VERSION} +import org.apache.texera.dao.jooq.generated.enums.{DefaultViewEnum, PrivilegeEnum} import org.apache.texera.dao.jooq.generated.tables.daos.{WorkflowDao, WorkflowVersionDao} import org.apache.texera.dao.jooq.generated.tables.pojos.{Workflow, WorkflowVersion} import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowResource.{ @@ -53,9 +55,38 @@ object WorkflowVersionResource { SqlServer .getInstance() .createDSLContext() - private def workflowVersionDao = new WorkflowVersionDao(context.configuration) private def workflowDao = new WorkflowDao(context.configuration) + case class MacroPin( + id: String, + wid: Integer, + name: String, + version: Integer, + revision: Int, + content: JsonNode + ) + + case class MacroMetadata( + wid: Integer, + name: String, + description: String, + readonly: Boolean, + creationTime: Timestamp, + lastModifiedTime: Timestamp, + isPublished: Boolean, + defaultView: DefaultViewEnum + ) + case class MacroSource(metadata: MacroMetadata, revisions: List[Integer], head: Option[MacroPin]) + + private def revisionIDs(wid: Integer, db: DSLContext): List[Integer] = + db.select(WORKFLOW_VERSION.VID) + .from(WORKFLOW_VERSION) + .where(WORKFLOW_VERSION.WID.eq(wid)) + .orderBy(WORKFLOW_VERSION.VID) + .fetchInto(classOf[Integer]) + .asScala + .toList + /** * Whether this user may read the workflow's revision history. * @@ -64,10 +95,14 @@ object WorkflowVersionResource { * version folds deltas back from the author's *current* content, so handing the history to a * public viewer would hand them the edits the pin is holding back. */ - private def canReadHistory(wid: Integer, uid: Integer): Boolean = - WorkflowAccess.hasGrantedAccess(wid, uid) || - (WorkflowAccess.hasReadAccess(wid, uid) && - Option(workflowDao.fetchOneByWid(wid)).forall(_.getPublishedContent == null)) + private def canReadHistory(wid: Integer, uid: Integer, db: DSLContext = context): Boolean = + WorkflowAccess.getPrivilege(wid, uid, db) != PrivilegeEnum.NONE || + db.fetchExists( + WORKFLOW, + WORKFLOW.WID.eq(wid), + WORKFLOW.IS_PUBLIC.isTrue, + WORKFLOW.PUBLISHED_CONTENT.isNull + ) // constant to indicate versions should be aggregated if they are within the specified time limit private final val AGGREGATE_TIME_LIMIT_MILLSEC = UserSystemConfig.workflowVersionCollapseIntervalInMinutes * 60000 @@ -81,10 +116,14 @@ object WorkflowVersionResource { * @param workflow * @param insertingNewWorkflow indicates if the workflow didn't exist before */ - def insertVersion(workflow: Workflow, insertingNewWorkflow: Boolean): Unit = { + def insertVersion( + workflow: Workflow, + insertingNewWorkflow: Boolean, + db: DSLContext = context + ): Unit = { val wid = workflow.getWid // retrieve current workflow from DB - val currentWorkflow = workflowDao.fetchOneByWid(wid) + val currentWorkflow = new WorkflowDao(db.configuration).fetchOneByWid(wid) // if the workflow is new then previous workflow is empty val existingWorkflowContent = if (insertingNewWorkflow) "{}" else currentWorkflow.getContent // compute diff @@ -96,7 +135,7 @@ object WorkflowVersionResource { // if we are creating a new workflow, even if it is empty, create a new version // otherwise, only when there is a diff we would create a new version if (insertingNewWorkflow || !patch.isEmpty) { - insertNewVersion(wid, patch.toString) + insertNewVersion(wid, patch.toString, db) } } @@ -128,11 +167,15 @@ object WorkflowVersionResource { * * @param wid */ - def insertNewVersion(wid: Integer, content: String = "[]"): WorkflowVersion = { + def insertNewVersion( + wid: Integer, + content: String = "[]", + db: DSLContext = context + ): WorkflowVersion = { val workflowVersion = new WorkflowVersion() workflowVersion.setContent(content) workflowVersion.setWid(wid) - workflowVersionDao.insert(workflowVersion) + new WorkflowVersionDao(db.configuration).insert(workflowVersion) workflowVersion } @@ -336,6 +379,88 @@ object WorkflowVersionResource { @Produces(Array(MediaType.APPLICATION_JSON)) class WorkflowVersionResource { + @GET + @Path("/{wid}/macro-source") + @RolesAllowed(Array("REGULAR", "ADMIN")) + def macroSource( + @PathParam("wid") wid: Integer, + @QueryParam("id") id: String, + @QueryParam("head") head: Boolean, + @Auth user: SessionUser + ): MacroSource = + readMacro(wid, id, null, user, includeHead = head, requirePin = false) + + /** A pinned revision names the result of its save. Its own inverse patch must never be applied. */ + @GET + @Path("/{wid}/macro") + @RolesAllowed(Array("REGULAR", "ADMIN")) + def pinMacro( + @PathParam("wid") wid: Integer, + @QueryParam("id") id: String, + @QueryParam("version") selected: Integer, + @Auth user: SessionUser + ): MacroPin = + readMacro(wid, id, selected, user, includeHead = true, requirePin = true).head.get + + private def readMacro( + wid: Integer, + id: String, + selected: Integer, + user: SessionUser, + includeHead: Boolean, + requirePin: Boolean + ): MacroSource = + context.transactionResult { config => + val db = config.dsl() + val workflow = Option( + db.selectFrom(WORKFLOW) + .where(WORKFLOW.WID.eq(wid)) + .forShare() + .fetchOneInto(classOf[Workflow]) + ) + .getOrElse(throw new NotFoundException("Macro source unavailable")) + val privilege = WorkflowAccess.getPrivilege(wid, user.getUid, db) + val granted = privilege != PrivilegeEnum.NONE + val historyVisible = granted || workflow.getPublishedContent == null + if (!granted && (!workflow.getIsPublic || (requirePin && !historyVisible))) + throw new ForbiddenException("No sufficient access privilege.") + val visible = WorkflowPublishService.visibleCopy(workflow, granted) + val body = objectMapper.readTree(visible.content) + def verify(graph: JsonNode): Unit = { + if (!graph.hasNonNull("macroInterface")) + throw new BadRequestException("Saved version is not a macro definition") + if (id == null || id.isEmpty || graph.path("macroInterface").path("id").textValue() != id) + throw new BadRequestException("Macro source identity differs") + } + verify(body) + val versions = if (historyVisible) revisionIDs(wid, db) else Nil + if (requirePin && versions.isEmpty) throw new NotFoundException("Macro has no saved version") + val head = if (includeHead && versions.nonEmpty) { + val version = Option(selected).getOrElse(versions.last) + val ordinal = versions.indexOf(version) + 1 + if (ordinal == 0) throw new NotFoundException("Macro revision is not in its source history") + val snapshot = + if (ordinal == versions.size) body + else { + val patches = fetchSubsequentVersions(wid, version, db).filterNot(_.getVid == version) + objectMapper.readTree(applyPatch(patches, workflow).getContent) + } + verify(snapshot) + Some(MacroPin(id, wid, visible.name, version, ordinal, snapshot)) + } else None + val metadata = MacroMetadata( + wid, + visible.name, + visible.description, + privilege != PrivilegeEnum.WRITE, + workflow.getCreationTime, + workflow.getLastModifiedTime, + workflow.getIsPublic, + visible.defaultView + ) + MacroSource(metadata, versions, head) + } + /** * This method returns the versions of a workflow given by its ID * diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala index b29d6dce275..07690de340c 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionResultService.scala @@ -404,12 +404,22 @@ class ExecutionResultService( def handleResultPagination(request: ResultPaginationRequest): TexeraWebSocketEvent = { // calculate from index (pageIndex starts from 1 instead of 0) val from = request.pageSize * (request.pageIndex - 1) - val latestExecutionId = getLatestExecutionId(workflowIdentity, computingUnitId).getOrElse( - throw new IllegalStateException("No execution is recorded") - ) + val executionId = request.executionId match { + case Some(id) + if WorkflowExecutionsResource.ownsExecution(workflowIdentity, computingUnitId, id) => + ExecutionIdentity(id) + case Some(_) => + throw new IllegalArgumentException( + "Execution is unavailable for this workflow and computing unit" + ) + case None => + getLatestExecutionId(workflowIdentity, computingUnitId).getOrElse( + throw new IllegalStateException("No execution is recorded") + ) + } val storageUriOption = ExecutionResultUris.getResultUriByLogicalPortId( - latestExecutionId, + executionId, OperatorIdentity(request.operatorID), PortIdentity(request.portIndex) ) diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala index 7e470a38b43..2ad557ab0b5 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionStatsService.scala @@ -24,8 +24,8 @@ import com.typesafe.scalalogging.LazyLogging import org.apache.texera.amber.core.storage.model.BufferedItemWriter import org.apache.texera.amber.core.storage.result.ResultSchema import org.apache.texera.amber.core.storage.{DocumentFactory, VFSURIFactory} -import org.apache.texera.amber.core.tuple.Tuple -import org.apache.texera.amber.core.workflow.WorkflowContext +import org.apache.texera.amber.core.tuple.{AttributeType, Tuple} +import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} import org.apache.texera.amber.core.workflowruntimestate.FatalErrorType.EXECUTION_FAILURE import org.apache.texera.amber.core.workflowruntimestate.WorkflowFatalError import org.apache.texera.amber.engine.architecture.coordinator.{ @@ -42,8 +42,12 @@ import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAg FAILED, KILLED } -import org.apache.texera.amber.engine.common.Utils -import org.apache.texera.amber.engine.architecture.worker.statistics.PortTupleMetricsMapping +import org.apache.texera.amber.engine.architecture.coordinator.execution.ExecutionUtils.aggregateStatistics +import org.apache.texera.amber.engine.architecture.worker.statistics.{ + PortTupleMetricsMapping, + TupleMetrics +} +import org.apache.texera.common.compiler.{MacroExpansion, MacroResultTarget} import org.apache.texera.amber.engine.common.Utils.maptoStatusCode import org.apache.texera.amber.engine.common.client.AmberClient import org.apache.texera.amber.engine.common.executionruntimestate.{ @@ -68,12 +72,103 @@ import org.apache.texera.web.storage.ExecutionStateStore.updateWorkflowState import java.time.Instant import java.util.concurrent.Executors +object ExecutionStatsService { + val metricsColumn = "operatorMetrics" + + private[service] def canvasStatistics( + expansion: Option[MacroExpansion], + native: Map[String, OperatorMetrics] + ) = { + val boundaries = expansion.iterator.flatMap(_.boundaries).flatMap { + case (id, boundary) => + val owned = boundary.members + val members = + if (owned.nonEmpty) owned + else + (boundary.outputs.flatten ++ (if (boundary.terminal) boundary.inputs.flatten else Nil)) + .map(_.operatorID) + .distinct + val updates = members.flatMap(native.get) + Option.when(updates.nonEmpty) { + def ports(bindings: List[List[MacroResultTarget]], input: Boolean) = + bindings.zipWithIndex.map { + case (targets, ordinal) => + // A broadcast sends the same stream to several consumers; use the furthest progress. + val tuples = targets + .flatMap { target => + native.get(target.operatorID).flatMap { m => + val stats = m.operatorStatistics + (if (input) stats.inputMetrics else stats.outputMetrics) + .find(_.portId == PortIdentity(target.portIndex)) + .map(_.tupleMetrics) + } + } + .maxByOption(m => (m.count, m.size)) + .getOrElse(TupleMetrics.defaultInstance) + PortTupleMetricsMapping(PortIdentity(ordinal), tuples) + } + val states = updates.map(_.operatorState) + import WorkflowAggregatedState._ + val complete = updates.size == members.size + val state = + if (complete && states.forall(s => s == COMPLETED || s == TERMINATED)) COMPLETED + else + List(FAILED, KILLED, PAUSING, PAUSED, RESUMING, RUNNING, READY) + .find(states.contains) + .getOrElse(UNINITIALIZED) + id -> OperatorMetrics( + state, + aggregateStatistics( + if (owned.nonEmpty) updates else Nil, + ports(boundary.inputs, !boundary.terminal), + ports(boundary.outputs, input = false) + ), + complete && updates.forall(_.reusedFromCache) + ) + } + } + (native ++ boundaries).view.mapValues(OperatorAggregatedMetrics.from).toMap + } + + def restore( + expansion: MacroExpansion, + rows: Iterator[Tuple] + ): Map[String, OperatorAggregatedMetrics] = { + val latest = scala.collection.mutable.Map.empty[String, Tuple] + rows.foreach { row => + val id = row.getField[String]("operatorId") + if ( + latest + .get(id) + .forall(r => + !r.getField[java.sql.Timestamp]("time").after(row.getField[java.sql.Timestamp]("time")) + ) + ) + latest(id) = row + } + canvasStatistics( + Some(expansion), + latest.view + .mapValues(row => OperatorMetrics.parseFrom(row.getField[Array[Byte]](metricsColumn))) + .toMap + ) + } +} + class ExecutionStatsService( client: AmberClient, stateStore: ExecutionStateStore, - workflowContext: WorkflowContext + workflowContext: WorkflowContext, + expansion: Option[MacroExpansion] = None ) extends SubscriptionManager with LazyLogging { + private val schema = + if (expansion.nonEmpty) + ResultSchema.runtimeStatisticsSchema.add( + ExecutionStatsService.metricsColumn, + AttributeType.BINARY + ) + else ResultSchema.runtimeStatisticsSchema private val (metricsPersistThread, runtimeStatsWriter) = { val thread = Executors.newSingleThreadExecutor() val uri = VFSURIFactory.createRuntimeStatisticsURI( @@ -82,7 +177,7 @@ class ExecutionStatsService( warehouse = workflowContext.warehouse ) val writer = DocumentFactory - .createDocument(uri, ResultSchema.runtimeStatisticsSchema) + .createDocument(uri, schema) .writer("runtime_statistics") .asInstanceOf[BufferedItemWriter[Tuple]] WorkflowExecutionsResource.updateRuntimeStatsUri( @@ -94,6 +189,7 @@ class ExecutionStatsService( (thread, writer) } + private var lastSampleTime = 0L private var lastPersistedMetrics: Map[String, OperatorMetrics] = Map.empty[String, OperatorMetrics] @@ -104,29 +200,9 @@ class ExecutionStatsService( // Update operator stats if any operator updates its stat if (newState.operatorInfo.toSet != oldState.operatorInfo.toSet) { Iterable( - OperatorStatisticsUpdateEvent(newState.operatorInfo.map { - case (opId, metrics) => - val (inCount, inSize, inMap) = - summarizePortMetrics(metrics.operatorStatistics.inputMetrics) - val (outCount, outSize, outMap) = - summarizePortMetrics(metrics.operatorStatistics.outputMetrics) - - val res = OperatorAggregatedMetrics( - Utils.aggregatedStateToString(metrics.operatorState), - inCount, - inSize, - inMap, - outCount, - outSize, - outMap, - metrics.operatorStatistics.numWorkers, - metrics.operatorStatistics.dataProcessingTime, - metrics.operatorStatistics.controlProcessingTime, - metrics.operatorStatistics.idleTime, - reusedFromCache = metrics.reusedFromCache - ) - (opId, res) - }) + OperatorStatisticsUpdateEvent( + ExecutionStatsService.canvasStatistics(expansion, newState.operatorInfo) + ) ) } else { Iterable.empty @@ -186,7 +262,7 @@ class ExecutionStatsService( client .registerCallback[ExecutionStatsUpdate]((evt: ExecutionStatsUpdate) => { stateStore.statsStore.updateState { statsStore => - statsStore.withOperatorInfo(evt.operatorMetrics) + statsStore.withOperatorInfo(statsStore.operatorInfo ++ evt.operatorMetrics) } }) ) @@ -196,8 +272,8 @@ class ExecutionStatsService( client .registerCallback[RuntimeStatisticsPersist]((evt: RuntimeStatisticsPersist) => { metricsPersistThread.execute(() => { - storeRuntimeStatistics(computeStatsDiff(evt.operatorMetrics)) - lastPersistedMetrics = evt.operatorMetrics + lastPersistedMetrics ++= evt.operatorMetrics + storeRuntimeStatistics(lastPersistedMetrics) }) }) ) @@ -209,7 +285,7 @@ class ExecutionStatsService( if state == COMPLETED || state == FAILED || state == KILLED => logger.info("Workflow execution terminated. Commit runtime statistics.") try { - runtimeStatsWriter.close() + metricsPersistThread.submit((() => runtimeStatsWriter.close()): Runnable).get() } catch { case e: Exception => logger.error("Failed to close runtime statistics writer", e) @@ -218,28 +294,8 @@ class ExecutionStatsService( } ) - private def computeStatsDiff( - newMetrics: Map[String, OperatorMetrics] - ): Map[String, OperatorMetrics] = { - val previousMetrics = lastPersistedMetrics - val missingKeys = previousMetrics.keySet.diff(newMetrics.keySet) - newMetrics ++ missingKeys.map(key => key -> previousMetrics(key)) - } - - private def summarizePortMetrics( - portMetrics: Seq[PortTupleMetricsMapping] - ): (Long, Long, Map[String, Long]) = { - val perPortCounts = Map.newBuilder[String, Long] - val (totalCount, totalSize) = aggregatePortMetrics( - portMetrics, - (portId, count) => perPortCounts += (portId -> count) - ) - (totalCount, totalSize, perPortCounts.result()) - } - private def aggregatePortMetrics( - portMetrics: Seq[PortTupleMetricsMapping], - onPort: (String, Long) => Unit = (_, _) => () + portMetrics: Seq[PortTupleMetricsMapping] ): (Long, Long) = { var totalCount = 0L var totalSize = 0L @@ -247,7 +303,6 @@ class ExecutionStatsService( val count = pm.tupleMetrics.count totalCount += count totalSize += pm.tupleMetrics.size - onPort(pm.portId.id.toString, count) } (totalCount, totalSize) } @@ -259,15 +314,15 @@ class ExecutionStatsService( return } try { - val schema = ResultSchema.runtimeStatisticsSchema - val timestamp = new java.sql.Timestamp(System.currentTimeMillis()) + lastSampleTime = math.max(System.currentTimeMillis(), lastSampleTime + 1) + val timestamp = new java.sql.Timestamp(lastSampleTime) operatorStatistics.foreach { case (operatorId, stat) => val (inCount, inSize) = aggregatePortMetrics(stat.operatorStatistics.inputMetrics) val (outCount, outSize) = aggregatePortMetrics(stat.operatorStatistics.outputMetrics) val runtimeStats = new Tuple( schema, - Array( + Array[Any]( operatorId, timestamp, inCount, @@ -279,7 +334,7 @@ class ExecutionStatsService( stat.operatorStatistics.idleTime, stat.operatorStatistics.numWorkers, maptoStatusCode(stat.operatorState).toInt - ) + ) ++ expansion.map(_ => stat.toByteArray) ) runtimeStatsWriter.putOne(runtimeStats) } diff --git a/amber/src/main/scala/org/apache/texera/web/service/ExecutionsMetadataPersistService.scala b/amber/src/main/scala/org/apache/texera/web/service/ExecutionsMetadataPersistService.scala index f7e5b86ac83..a371195e47d 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/ExecutionsMetadataPersistService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/ExecutionsMetadataPersistService.scala @@ -57,7 +57,8 @@ object ExecutionsMetadataPersistService extends LazyLogging { executionName: String, environmentVersion: String, computingUnitId: Integer, - warehouseId: Option[Int] = None + warehouseId: Option[Int] = None, + macroContent: Option[com.fasterxml.jackson.databind.JsonNode] = None ): ExecutionIdentity = { // first retrieve the latest version of this workflow val vid = getLatestVersion(workflowId.id.toInt) @@ -66,6 +67,7 @@ object ExecutionsMetadataPersistService extends LazyLogging { newExecution.setName(executionName) } newExecution.setVid(vid) + macroContent.foreach(content => newExecution.setMacroContent(content.toString)) newExecution.setUid(uid) newExecution.setStartingTime(new Timestamp(System.currentTimeMillis())) newExecution.setEnvironmentVersion(environmentVersion) diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala index b1c738e51c6..9a48e0293c7 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala @@ -93,11 +93,10 @@ class WorkflowExecutionService( val wsInput = new WebsocketInput(errorHandler) private def createStateEvent(state: ExecutionMetadataStore): WorkflowStateEvent = { - if (state.isRecovering && state.state != COMPLETED) { - WorkflowStateEvent("Recovering") - } else { - WorkflowStateEvent(Utils.aggregatedStateToString(state.state)) - } + val label = + if (state.isRecovering && state.state != COMPLETED) "Recovering" + else Utils.aggregatedStateToString(state.state) + WorkflowStateEvent(label, Option(workflowContext.executionId).map(_.id).filter(_ > 0)) } var workflow: Workflow = _ @@ -136,7 +135,12 @@ class WorkflowExecutionService( ) executionReconfigurationService = new ExecutionReconfigurationService(client, executionStateStore, workflow) - executionStatsService = new ExecutionStatsService(client, executionStateStore, workflow.context) + executionStatsService = new ExecutionStatsService( + client, + executionStateStore, + workflow.context, + request.logicalPlan.macroExpansion + ) executionRuntimeService = new ExecutionRuntimeService( client, executionStateStore, diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index 5d9b064af43..19321b85745 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -216,7 +216,8 @@ class WorkflowService( def initExecutionService( req: WorkflowExecuteRequest, userOpt: Option[User], - sessionUri: URI + sessionUri: URI, + macroSnapshot: Option[com.fasterxml.jackson.databind.JsonNode] = None ): Unit = { val span = TexeraTracer.tracer .spanBuilder("WorkflowService.initExecutionService") @@ -258,7 +259,8 @@ class WorkflowService( req.executionName, convertToJson(req.engineVersion), req.computingUnitId, - req.warehouseId + req.warehouseId, + macroSnapshot.orElse(req.logicalPlan.macroContent) ) span.setAttribute(SpanAttrs.ExecutionId, Long.box(workflowContext.executionId.id)) // A run has started: record the start counter and stamp its start time. diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala index 792116bea90..edda9be8df4 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/TestUtils.scala @@ -147,7 +147,8 @@ object TestUtils { workflow: Workflow, operatorIds: Iterable[OperatorIdentity], extract: VirtualDocument[Tuple] => T, - completionTimeout: Duration = Duration.fromMinutes(1) + completionTimeout: Duration = Duration.fromMinutes(1), + configureClient: AmberClient => Unit = _ => () ): Map[OperatorIdentity, T] = { // The Promise carries the result so completing the run and handing back the // value are atomic. Every terminal path uses `updateIfEmpty`, so a second @@ -175,6 +176,7 @@ object TestUtils { // post-completion `readMaterializedResults` lookup via // `getResultUriByLogicalPortId` finds the rows. registerResultUriPersistence(client, workflow.context.executionId) + configureClient(client) client.registerCallback[ExecutionStateUpdate](evt => { if (evt.state == COMPLETED) { completion.updateIfEmpty( diff --git a/amber/src/test/scala/org/apache/texera/web/SessionStateSpec.scala b/amber/src/test/scala/org/apache/texera/web/SessionStateSpec.scala index 34fe6250f57..164dddb94de 100644 --- a/amber/src/test/scala/org/apache/texera/web/SessionStateSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/SessionStateSpec.scala @@ -24,6 +24,7 @@ import org.apache.texera.amber.core.virtualidentity.WorkflowIdentity import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum import org.apache.texera.web.model.websocket.event.{TexeraWebSocketEvent, WorkflowStateEvent} +import org.apache.texera.web.model.websocket.response.RegionUpdateEvent import org.apache.texera.web.service.WorkflowService import org.scalamock.scalatest.MockFactory import org.scalatest.flatspec.AnyFlatSpec @@ -126,6 +127,32 @@ class SessionStateSpec extends AnyFlatSpec with Matchers with MockFactory { state.getCurrentWorkflowState shouldBe None } + it should "send region events only to sessions subscribed to their workflow" in { + val entries = (1 to 4).map { _ => + val (session, messages) = messageCollectingSession() + val state = new SessionState(session) + val id = UUID.randomUUID().toString + SessionState.setState(id, state) + (id, state, messages) + } + try { + val first = new TestWorkflowService(94L) + entries(0)._2.subscribe(first) + entries(1)._2.subscribe(first) + entries(2)._2.subscribe(new TestWorkflowService(106L)) + val event = RegionUpdateEvent(List((0L, List("operator")))) + SessionState.sendToWorkflow(WorkflowIdentity(94L), event) + entries.take(2).foreach(_._3 should have size 1) + entries.drop(2).foreach(_._3 shouldBe empty) + + entries(0)._2.subscribe(new TestWorkflowService(106L)) + SessionState.sendToWorkflow(WorkflowIdentity(94L), event) + entries(0)._3 should have size 1 + entries(1)._3 should have size 2 + entries.drop(2).foreach(_._3 shouldBe empty) + } finally entries.foreach(entry => removeStateIfPresent(entry._1)) + } + it should "remove registered sessions, cleaning up subscriptions, and retain their computing-unit access level" in { val sessionId = UUID.randomUUID().toString val state = new SessionState(stub[Session]) diff --git a/amber/src/test/scala/org/apache/texera/web/resource/SyncExecutionResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/SyncExecutionResourceSpec.scala index 7c505881e5b..7c8c63f8c1a 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/SyncExecutionResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/SyncExecutionResourceSpec.scala @@ -256,20 +256,23 @@ class SyncExecutionResourceSpec .execute() } - private def globalPortIdOf(opId: String): GlobalPortIdentity = + private def globalPortIdOf( + opId: String, + port: PortIdentity = PortIdentity() + ): GlobalPortIdentity = GlobalPortIdentity( PhysicalOpIdentity(OperatorIdentity(opId), "main"), - PortIdentity(), + port, input = false ) /** The external-output result URI shape `getResultUriByLogicalPortId` decodes and matches. */ - private def resultUriOf(opId: String): URI = + private def resultUriOf(opId: String, port: PortIdentity = PortIdentity()): URI = VFSURIFactory.resultURI( VFSURIFactory.createPortBaseURI( WorkflowIdentity(testWid.toLong), ExecutionIdentity(resultEid.toLong), - globalPortIdOf(opId) + globalPortIdOf(opId, port) ) ) @@ -302,8 +305,13 @@ class SyncExecutionResourceSpec * workflow 9317, because an Iceberg table's identity derives from its URI path and sbt runs * amber's suites in one unforked JVM: an id another suite also uses would truncate its table. */ - private def storeResult(opId: String, tuples: Seq[Tuple], schema: Schema = rowSchema): Unit = { - val uri = resultUriOf(opId) + private def storeResult( + opId: String, + tuples: Seq[Tuple], + schema: Schema = rowSchema, + port: PortIdentity = PortIdentity() + ): Unit = { + val uri = resultUriOf(opId, port) val writer = DocumentFactory .createDocument(uri, schema) .writer(opId) @@ -319,7 +327,8 @@ class SyncExecutionResourceSpec ExecutionIdentity(resultEid.toLong), opId, charLimit, - cellCharLimit + cellCharLimit, + PortIdentity() ) /** @@ -741,7 +750,51 @@ class SyncExecutionResourceSpec )) should be theSameInstanceAs plan } - "collectOperatorInfos" should "report per-operator stats, summed across ports" in { + "collectOperatorInfos" should "preserve selected macro output ports and native aggregate statistics" in { + val opId = "serc-macro-selected-port" + storeResult(opId, Seq(row(0))) + storeResult(opId, Seq(row(4), row(5)), port = PortIdentity(1)) + val state = new ExecutionStateStore() + state.statsStore.updateState( + _.withOperatorInfo( + Map( + opId -> metrics( + COMPLETED, + inputs = Seq(0 -> 3L), + outputs = Seq(0 -> 1L, 1 -> 2L) + ) + ) + ) + ) + val infos = resource invokePrivate collectOperatorInfos( + ExecutionIdentity(resultEid.toLong), + newExecution(state), + List(opId), + 100000, + 20000, + None, + Map( + "selected" -> (OperatorIdentity(opId), PortIdentity(1)), + "left" -> (OperatorIdentity(opId), PortIdentity()) + ) + ) + def values(id: String) = + infos(id).result.get + .asInstanceOf[java.util.List[ObjectNode]] + .asScala + .toList + .map(_.path("v").asText()) + infos.keySet shouldBe Set(opId, "selected", "left") + values(opId) shouldBe List("r0") + values("selected") shouldBe List("r4", "r5") + values("left") shouldBe List("r0") + infos(opId).outputTuples shouldBe 3L + infos("selected").outputTuples shouldBe 2L + infos("left").outputTuples shouldBe 1L + infos("selected").totalRowCount shouldBe Some(2) + } + + it should "report per-operator stats, summed across ports" in { val stateStore = new ExecutionStateStore() stateStore.statsStore.updateState( _.withOperatorInfo( @@ -763,7 +816,8 @@ class SyncExecutionResourceSpec List("op-with-stats", "op-no-ports", "op-never-ran"), 100000, 20000, - None + None, + Map.empty ) infos.keySet shouldBe Set("op-with-stats", "op-no-ports", "op-never-ran") @@ -820,7 +874,8 @@ class SyncExecutionResourceSpec List.empty, 100000, 20000, - None + None, + Map.empty ) // Two operators, not one: "all of them" has to mean the whole stats map. @@ -860,7 +915,8 @@ class SyncExecutionResourceSpec List("target"), 100000, 20000, - Some(consoleState) + Some(consoleState), + Map.empty ) infos.keySet shouldBe Set("target", "upstream", "chatty", "silent") @@ -918,7 +974,8 @@ class SyncExecutionResourceSpec List("guarded"), 100000, 20000, - None + None, + Map.empty ) } } diff --git a/amber/src/test/scala/org/apache/texera/web/resource/SyncMacroExecutionSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/SyncMacroExecutionSpec.scala new file mode 100644 index 00000000000..1ad976c04af --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/resource/SyncMacroExecutionSpec.scala @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.web.resource + +import org.apache.texera.amber.operator.TestOperators +import org.apache.texera.amber.util.JSONUtils.objectMapper +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.texera.common.compiler.model.LogicalPlanPojo +import org.apache.texera.common.compiler.MacroTestUtils._ +import org.apache.texera.common.compiler.WorkflowCompiler +import org.apache.texera.amber.core.virtualidentity.{OperatorIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} +import org.scalatest.PrivateMethodTester +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class SyncMacroExecutionSpec extends AnyFlatSpec with Matchers with PrivateMethodTester { + private val resource = new SyncExecutionResource() + + private def selectedPortPlan: LogicalPlanPojo = { + val leaf = forwardingBody( + "leaf", + operator("split", "Split", List("input-0"), List("output-0", "output-2")), + List("rows" -> "input-0"), + List("selected" -> "output-2") + ) + val child = instance("child", "leaf", List("rows"), List("selected")) + val parent = + forwardingBody("parent", child, List("rows" -> "rows"), List("result" -> "selected")) + val ids = List("selected", "sibling") + val root = graph( + textSource("source", "one") :: ids.map(instance(_, "parent", List("rows"), List("result"))), + ids.map(id => link(s"$id-in", end("source", "output-0"), end(id, "rows"))) + ) + macroPlan( + pinned(root, definition("parent", "Parent", parent), definition("leaf", "Leaf", leaf)), + List("source", "selected") + ) + } + + "Macro result ports" should "retain a nested selected endpoint's dense ordinal and leave sibling pins unchanged" in { + val plan = selectedPortPlan + val before = plan.macroContent.get.deepCopy[JsonNode]() + val leaf = plan.macroExpansion.get.paths.find(_._2 == List("selected", "child", "split")).get._1 + plan.macroResultPorts shouldBe Map("selected" -> (OperatorIdentity(leaf), PortIdentity(1))) + val browserTarget = + objectMapper.valueToTree[JsonNode](plan.macroExpansion.get).at("/resultTargets/selected") + browserTarget.path("operatorID").asText() shouldBe leaf + browserTarget.path("portIndex").asInt(-1) shouldBe 1 + plan.macroContent.get shouldBe before + objectMapper + .valueToTree[JsonNode](plan) + .has("macroResultPorts") shouldBe false + plan.copy(opsToViewResult = List("sibling")).macroResultPorts.keys.toList shouldBe List( + "sibling" + ) + } + + it should "leave native selections unchanged and reject missing pinned definitions" in { + LogicalPlanPojo(Nil, Nil, List("native"), Nil).macroResultPorts shouldBe empty + val plan = selectedPortPlan + val malformed = plan.macroContent.get.deepCopy[ObjectNode]() + malformed.remove("macroDependencies") + intercept[IllegalArgumentException]( + plan.copy(macroContent = Some(malformed)).macroResultPorts + ).getMessage should include("Missing macro version") + } + + private def reject(plan: LogicalPlanPojo): String = { + val result = resource.executeWorkflowSync( + Long.MaxValue, + Int.MaxValue, + SyncExecutionRequest( + "macro preflight", + plan, + None, + Nil, + 1, + 1000, + 1000 + ), + // A rejected graph must not reach session, warehouse, database or runtime setup. + null + ) + result.success shouldBe false + result.state shouldBe "CompilationFailed" + result.errors.get.mkString + } + + "Execution preflight" should "reject duplicate native IDs before accessing any session or runtime" in { + val plan = objectMapper.readValue( + """{"operators":[ + {"operatorID":"same","operatorType":"TextInput","textInput":"one", + "attributeName":"line","attributeType":"string"}, + {"operatorID":"same","operatorType":"TextInput","textInput":"two", + "attributeName":"line","attributeType":"string"} + ],"links":[],"opsToViewResult":[],"opsToReuseResult":[]}""", + classOf[LogicalPlanPojo] + ) + reject(plan) should include("Duplicate graph element ID 'same'") + } + + "Macro execution preflight" should "reject missing pins before initializing or replacing an execution" in { + reject(macroPlan(graph(List(instance("instance", "missing"))))) should include( + "Missing macro version" + ) + } + + it should "reject standalone definitions before initializing or replacing an execution" in { + reject(macroPlan(macroGraph("definition", Nil))) should include( + "Run macro definitions inside a workflow" + ) + } + + it should "reject invalid nested settings before accessing the session or replacing an execution" in { + val sorting = operator("sort", "StableMergeSort", List("input-0"), List("output-0")) + val pin = definition("sort", "Sorting", macroGraph("sort", List(sorting))) + val error = reject(macroPlan(pinned(graph(List(instance("instance", "sort"))), pin))) + error should include("Sorting / Stable Merge Sort") + error should include("keys") + } + + it should "retain nested validation paths when execution selects an upstream sub-DAG" in { + val projection = operator( + "project", + "Projection", + List("input-0"), + List("output-0"), + json("isDrop" -> false, "attributes" -> List(json("originalAttribute" -> "missing"))) + ) + val body = macroGraph( + "projection", + List(textSource("text", "one"), projection), + List(link("link", end("text", "output-0"), end("project", "input-0"))) + ) + val root = pinned( + graph(List(instance("instance", "projection"))), + definition("projection", "Projection macro", body) + ) + val normalized = macroPlan(root).normalized + val target = normalized.macroExpansion.get.paths.find(_._2.last == "project").get._1 + val select = PrivateMethod[LogicalPlanPojo](Symbol("computeSubDAGIfNeeded")) + val subDag = resource invokePrivate select(normalized, List(target)) + val compiler = new WorkflowCompiler( + new WorkflowContext(WorkflowIdentity(0)), + TestOperators.fixtureFileResolver + ) + val errors = compiler.compile(subDag).operatorIdToError + errors(OperatorIdentity("instance")).message should include("Projection macro / Projection") + errors(OperatorIdentity("instance")).message should include("missing") + subDag.macroContent shouldBe None + objectMapper + .valueToTree[JsonNode](subDag) + .has("expandedMacros") shouldBe false + } +} diff --git a/amber/src/test/scala/org/apache/texera/web/resource/WorkflowWebsocketResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/WorkflowWebsocketResourceSpec.scala index cd2c77d22f3..d6c1b653353 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/WorkflowWebsocketResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/WorkflowWebsocketResourceSpec.scala @@ -249,7 +249,8 @@ class WorkflowWebsocketResourceSpec override def initExecutionService( req: WorkflowExecuteRequest, userOpt: Option[User], - sessionUri: URI + sessionUri: URI, + macroSnapshot: Option[com.fasterxml.jackson.databind.JsonNode] ): Unit = { initCalls = initCalls :+ ((req, userOpt, sessionUri)) publishThenFail.foreach { execution => diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/file/WorkflowResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/file/WorkflowResourceSpec.scala index 501c4e3f050..3fe9837e092 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/file/WorkflowResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/file/WorkflowResourceSpec.scala @@ -19,6 +19,8 @@ package org.apache.texera.web.resource.dashboard.file +import com.fasterxml.jackson.databind.JsonNode +import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.texera.auth.SessionUser import org.apache.texera.dao.MockTexeraDB import org.apache.texera.dao.jooq.generated.Tables.{ @@ -38,6 +40,9 @@ import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowResource import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowResource.{ DashboardWorkflow, DefaultViewRequest, + MacroHead, + MacroPathSave, + MacroSaveStep, WorkflowIDs, WorkflowWithPrivilege } @@ -920,6 +925,99 @@ class WorkflowResourceSpec workflowResource.createWorkflow(workflow, user) } + private def macroHead(workflow: Workflow): MacroHead = + MacroHead( + WorkflowVersionResource.getLatestVersion(workflow.getWid), + workflow.getName, + workflow.getDescription + ) + + private def macroPath( + leaf: Workflow, + head: MacroHead, + parent: Workflow, + instance: String + ): MacroPathSave = + MacroPathSave( + List( + MacroSaveStep(leaf, Some(head), None), + MacroSaveStep(new Workflow(parent), None, Some(instance)) + ), + parent + ) + + "Saving a macro path" should "commit exact revisions and roll back the leaf when a parent cannot be updated" in { + val source = seedWorkflow( + sessionUser1, + "Child", + content = """{"macroInterface":{"id":"child"},"operators":[]}""" + ).workflow + val head = macroHead(source) + val old = head.version + val root = seedWorkflow( + sessionUser1, + "Parent", + content = + s"""{"operators":[{"operatorID":"use","operatorType":"WorkflowMacro:child:$old"}],"macroDependencies":[{"id":"child","name":"Child","version":$old,"content":${source.getContent}}]}""" + ).workflow + val edit = new Workflow(source) + edit.setContent("""{"macroInterface":{"id":"child"},"operators":[],"operatorPositions":{}}""") + def save(op: String, user: SessionUser = sessionUser1) = + workflowResource.persistMacroPath(macroPath(edit, head, root, op), user) + assert(intercept[WebApplicationException](save("missing")).getResponse.getStatus == 409) + assert(WorkflowVersionResource.getLatestVersion(source.getWid) == old) + intercept[ForbiddenException](save("use", sessionUser2)) + import scala.concurrent.{Await, ExecutionContext, Future} + import scala.concurrent.duration._ + implicit val ec: ExecutionContext = ExecutionContext.global + val gate = new java.util.concurrent.CountDownLatch(1) + val writers = List.fill(2)(Future { gate.await(); scala.util.Try(save("use")) }) + gate.countDown() + val results = Await.result(Future.sequence(writers), 15.seconds) + assert(results.collect { + case scala.util.Failure(e: WebApplicationException) => e.getResponse.getStatus + } == List(409)) + val receipt = results.flatMap(_.toOption).head + assert(receipt.head.version > old) + assert(receipt.head.revision == 2) + assert(WorkflowVersionResource.getLatestVersion(root.getWid) == receipt.last.version) + val stored = workflowResource.retrieveWorkflow(root.getWid, sessionUser1) + val response = objectMapper.valueToTree[JsonNode](receipt) + assert(response.at("/0/content") == objectMapper.readTree(edit.getContent)) + assert(response.at("/1/content") == objectMapper.readTree(stored.content)) + assert(stored.content.contains(s"WorkflowMacro:child:${receipt.head.version}")) + assert(intercept[WebApplicationException](save("use")).getResponse.getStatus == 409) + assert(WorkflowVersionResource.getLatestVersion(source.getWid) == receipt.head.version) + } + + it should "save two occurrences of the same workflow in order and preserve the original revision" in { + val leaf = seedWorkflow( + sessionUser1, + "A", + content = """{"macroInterface":{"id":"a"},"operators":[]}""" + ).workflow + val old = WorkflowVersionResource.getLatestVersion(leaf.getWid) + val outer = new Workflow(leaf) + outer.setContent( + s"""{"macroInterface":{"id":"a"},"operators":[{"operatorID":"inner","operatorType":"WorkflowMacro:a:$old"}],"macroDependencies":[{"id":"a","name":"A","version":$old,"content":${leaf.getContent}}]}""" + ) + workflowResource.persistWorkflow(outer, sessionUser1) + val receipts = workflowResource.persistMacroPath( + macroPath(leaf, macroHead(leaf), outer, "inner"), + sessionUser1 + ) + assert(receipts.map(_.revision) == List(3, 4)) + val saved = + objectMapper.readTree(workflowResource.retrieveWorkflow(leaf.getWid, sessionUser1).content) + assert( + saved.at("/operators/0/operatorType").asText() == s"WorkflowMacro:a:${receipts.head.version}" + ) + assert(receipts.last.content == saved) + assert(receipts.head.content == saved.at("/macroDependencies/1/content")) + assert(saved.at("/macroDependencies/0/version").asInt() == old) + assert(saved.at("/macroDependencies/1/content").toString == leaf.getContent) + } + "WorkflowResource.retrieveWorkflow" should "return the workflow for a user with access" in { val wid = seedWorkflow(sessionUser1, "retrieve-me", "the-desc", "{\"a\":1}").workflow.getWid val result: WorkflowWithPrivilege = workflowResource.retrieveWorkflow(wid, sessionUser1) @@ -1246,7 +1344,7 @@ class WorkflowResourceSpec // not a ForbiddenException/BadRequestException, which the same catch swallows assert(thrown.getClass == classOf[WebApplicationException]) - assert(thrown.getCause.isInstanceOf[NoSuchElementException]) + assert(thrown.getCause.getMessage == "Missing operators array") assert(workflowNamesOf(sessionUser1) == List("dup-no-operators")) } diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala index 30e0acc185e..6bc49a04697 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowExecutionsResourceSpec.scala @@ -1053,6 +1053,34 @@ class WorkflowExecutionsResourceSpec u } + "A macro execution snapshot" should "belong to the requested run and workflow and require access" in { + grantReadAccess() + val first = insertExecution() + val second = insertExecution() + for ((row, value) <- List(first -> "old", second -> "new")) { + row.setMacroContent(s"""{"value":"$value"}""") + workflowExecutionsDao.update(row) + } + assert( + resource + .macroContent(testWorkflowWid, first.getEid, session(testUser)) + .path("value") + .asText() == "old" + ) + assert( + resource + .macroContent(testWorkflowWid, second.getEid, session(testUser)) + .path("value") + .asText() == "new" + ) + intercept[javax.ws.rs.NotFoundException]( + resource.macroContent(testWorkflowWid, -1, session(testUser)) + ) + intercept[javax.ws.rs.WebApplicationException]( + resource.macroContent(testWorkflowWid, first.getEid, session(userWithoutAccess())) + ) + } + "retrieveExecutionsOfWorkflow" should "return an empty list when the user lacks read access" in { val result = resource.retrieveExecutionsOfWorkflow(testWorkflowWid, session(userWithoutAccess()), null) diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowGraphIDsResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowGraphIDsResourceSpec.scala new file mode 100644 index 00000000000..ebafea5967a --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowGraphIDsResourceSpec.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.resource.dashboard.user.workflow + +import javax.ws.rs.BadRequestException +import org.apache.texera.auth.SessionUser +import org.apache.texera.dao.jooq.generated.tables.pojos.{User, Workflow} +import org.scalatest.flatspec.AnyFlatSpec + +class WorkflowGraphIDsResourceSpec extends AnyFlatSpec { + for (save <- List(false, true)) { + "Workflow writes" should s"reject malformed JSON before ${if (save) "save" else "create"}" in { + for (content <- List(null, "null", "not-json")) { + val workflow = new Workflow() + workflow.setContent(content) + val resource = new WorkflowResource() + val session = new SessionUser(new User()) + intercept[BadRequestException] { + if (save) resource.persistWorkflow(workflow, session) + else resource.createWorkflow(workflow, session) + } + assert(workflow.getWid == null) + } + } + "Workflow writes" should s"reject duplicate IDs before any ${if (save) "save" else "create"} writes" in { + val workflow = new Workflow() + workflow.setContent("""{"operators":[{"operatorID":"same"}],"links":[{"linkID":"same"}]}""") + val resource = new WorkflowResource() + val session = new SessionUser(new User()) + val error = intercept[BadRequestException] { + if (save) resource.persistWorkflow(workflow, session) + else resource.createWorkflow(workflow, session) + } + assert(error.getMessage.contains("Duplicate graph element ID 'same'")) + assert(workflow.getWid == null) + } + } +} diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala new file mode 100644 index 00000000000..f1942276f95 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.resource.dashboard.user.workflow + +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.flatspec.AnyFlatSpec + +class WorkflowMacroCloneSpec extends AnyFlatSpec { + private val mapper = new ObjectMapper() + + "Cloning a workflow" should "rewrite canvas references and preserve pinned dependencies" in { + val content = mapper.readTree( + """{"operators":[{"operatorID":"leaf","operatorType":"Filter"}],"operatorPositions":{"leaf":{"x":0,"y":0}},"formBinding":{"fields":[{"operatorID":"leaf","propertyKey":"value"}]},"macroInterface":{"id":"definition","inputs":[{"targets":[{"operatorID":"leaf","portID":"input-0"}]}],"outputs":[]},"macroDependencies":[{"id":"child","version":8,"content":{"operators":[{"operatorID":"leaf","operatorType":"Filter"}]}}]}""" + ) + val cloned = mapper.readTree(WorkflowResource.assignNewOperatorIds(content.toString)) + val id = cloned.at("/operators/0/operatorID").asText() + assert(id != "leaf") + assert(cloned.path("operatorPositions").has(id)) + assert(cloned.at("/formBinding/fields/0/operatorID").asText() == id) + assert(cloned.at("/macroInterface/inputs/0/targets/0/operatorID").asText() == id) + assert(cloned.at("/macroInterface/id").asText() != "definition") + assert(cloned.path("macroDependencies") == content.path("macroDependencies")) + } + + it should "keep ordinary and empty workflows valid and reject malformed content" in { + assert( + mapper + .readTree(WorkflowResource.assignNewOperatorIds("""{"operators":[]}""")) + .path("operators") + .isEmpty + ) + intercept[Exception](WorkflowResource.assignNewOperatorIds("{}")) + intercept[Exception](WorkflowResource.assignNewOperatorIds("invalid")) + } + + it should "rewrite only graph references when IDs occur in macro types and user values" in { + val content = mapper.readTree("""{ + "operators":[{"operatorID":"leaf","operatorType":"WorkflowMacro:leaf-definition:8","operatorProperties":{"code":"leaf + 1","value":"leaf"},"customDisplayName":"leaf"}], + "operatorPositions":{"leaf":{"x":0,"y":0}}, + "links":[{"source":{"operatorID":"leaf","portID":"output-0"},"target":{"operatorID":"external","portID":"input-0"}}], + "formBinding":{"fields":[{"id":"leaf-field","operatorID":"leaf","propertyKey":"leaf-key"}],"shownResultIds":["leaf"]}, + "macroDependencies":[{"id":"leaf-definition","version":8,"content":{"operators":[]}}] + }""") + val cloned = mapper.readTree(WorkflowResource.assignNewOperatorIds(content.toString)) + val id = cloned.at("/operators/0/operatorID").asText() + assert(id != "leaf") + assert(cloned.at("/operators/0/operatorType") == content.at("/operators/0/operatorType")) + assert( + cloned.at("/operators/0/operatorProperties") == content.at("/operators/0/operatorProperties") + ) + assert(cloned.at("/operators/0/customDisplayName").asText() == "leaf") + assert(cloned.at("/formBinding/fields/0/id").asText() == "leaf-field") + assert(cloned.at("/formBinding/fields/0/propertyKey").asText() == "leaf-key") + assert(cloned.at("/formBinding/shownResultIds/0").asText() == id) + assert(cloned.at("/links/0/source/operatorID").asText() == id) + assert(cloned.at("/links/0/target/operatorID").asText() == "external") + assert(cloned.path("macroDependencies") == content.path("macroDependencies")) + } +} diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResourceSpec.scala index 91b0b1c1be0..26fd2a60060 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowVersionResourceSpec.scala @@ -24,30 +24,15 @@ import org.apache.texera.auth.SessionUser import org.apache.texera.dao.MockTexeraDB import org.apache.texera.dao.jooq.generated.Tables import org.apache.texera.dao.jooq.generated.enums.{DefaultViewEnum, PrivilegeEnum} -import org.apache.texera.dao.jooq.generated.tables.daos.{ - UserDao, - WorkflowDao, - WorkflowOfUserDao, - WorkflowUserAccessDao, - WorkflowVersionDao -} -import org.apache.texera.dao.jooq.generated.tables.pojos.{ - User, - Workflow, - WorkflowOfUser, - WorkflowUserAccess, - WorkflowVersion -} -import org.jooq.impl.DSL +import org.apache.texera.dao.jooq.generated.tables.daos.{UserDao, WorkflowDao, WorkflowVersionDao} +import org.apache.texera.dao.jooq.generated.tables.pojos.{User, Workflow, WorkflowVersion} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, PrivateMethodTester} import java.sql.Timestamp -import java.util.UUID import java.util.concurrent.TimeUnit import javax.ws.rs.ForbiddenException -import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ class WorkflowVersionResourceSpec @@ -63,95 +48,46 @@ class WorkflowVersionResourceSpec private val strangerUid = 6000 + scala.util.Random.nextInt(1000) private var testWorkflow: Workflow = _ - private var owner: User = _ - private var stranger: User = _ - private var workflowDao: WorkflowDao = _ - private var workflowVersionDao: WorkflowVersionDao = _ - private var userDao: UserDao = _ - private var workflowOfUserDao: WorkflowOfUserDao = _ - private var workflowUserAccessDao: WorkflowUserAccessDao = _ - private var resource: WorkflowVersionResource = _ - - private val capturedVersions = ArrayBuffer.empty[Integer] - - override protected def beforeAll(): Unit = { - initializeDBAndReplaceDSLContext() - } + private val owner = makeUser(ownerUid, "wfv_owner") + private val stranger = makeUser(strangerUid, "wfv_stranger") + private lazy val workflowDao = new WorkflowDao(getDSLContext.configuration()) + private lazy val workflowVersionDao = new WorkflowVersionDao(getDSLContext.configuration()) + private lazy val resource = new WorkflowVersionResource() + + override protected def beforeAll(): Unit = initializeDBAndReplaceDSLContext() + override protected def afterAll(): Unit = closeConnectionPool() override protected def beforeEach(): Unit = { - workflowDao = new WorkflowDao(getDSLContext.configuration()) - workflowVersionDao = new WorkflowVersionDao(getDSLContext.configuration()) - userDao = new UserDao(getDSLContext.configuration()) - workflowOfUserDao = new WorkflowOfUserDao(getDSLContext.configuration()) - workflowUserAccessDao = new WorkflowUserAccessDao(getDSLContext.configuration()) - resource = new WorkflowVersionResource() - - testWorkflow = new Workflow - testWorkflow.setWid(Integer.valueOf(testWorkflowWid)) - testWorkflow.setName("test_workflow_" + UUID.randomUUID().toString.substring(0, 8)) - testWorkflow.setContent(createWorkflowContent("initial")) - testWorkflow.setDescription("test description") - testWorkflow.setIsPublic(false) - testWorkflow.setCreationTime(new Timestamp(System.currentTimeMillis())) - testWorkflow.setLastModifiedTime(new Timestamp(System.currentTimeMillis())) - - owner = makeUser(ownerUid, "wfv_owner") - stranger = makeUser(strangerUid, "wfv_stranger") - - cleanupTestData() - userDao.insert(owner) - userDao.insert(stranger) - workflowDao.insert(testWorkflow) - - val ownership = new WorkflowOfUser - ownership.setUid(Integer.valueOf(ownerUid)) - ownership.setWid(Integer.valueOf(testWorkflowWid)) - workflowOfUserDao.insert(ownership) + val db = getDSLContext + new UserDao(db.configuration()).insert(owner, stranger) + val w = Tables.WORKFLOW + testWorkflow = db + .insertInto(w) + .set(w.WID, Int.box(testWorkflowWid)) + .set(w.NAME, s"test_workflow_$testWorkflowWid") + .set(w.DESCRIPTION, "test description") + .set(w.CONTENT, createWorkflowContent("initial")) + .returning() + .fetchOneInto(classOf[Workflow]) + val ownership = Tables.WORKFLOW_OF_USER + db.insertInto(ownership, ownership.UID, ownership.WID) + .values(owner.getUid, testWorkflow.getWid) + .execute() grantAccess(ownerUid, PrivilegeEnum.WRITE) - - capturedVersions.clear() } override protected def afterEach(): Unit = { - cleanupTestData() - } - - private def cleanupTestData(): Unit = { - val ctx = getDSLContext - // Purge the fixed test workflow plus anything our test users came to own - // (cloneVersion creates a fresh workflow), children before parents so the - // user rows can be removed without violating foreign keys. - val wids = (ctx - .select(Tables.WORKFLOW_OF_USER.WID) - .from(Tables.WORKFLOW_OF_USER) - .where( - Tables.WORKFLOW_OF_USER.UID.in(Integer.valueOf(ownerUid), Integer.valueOf(strangerUid)) - ) - .fetchInto(classOf[Integer]) - .asScala - .toSet + Integer.valueOf(testWorkflowWid)).toList - - ctx - .deleteFrom(Tables.WORKFLOW_VERSION) - .where(Tables.WORKFLOW_VERSION.WID.in(wids: _*)) - .execute() - ctx - .deleteFrom(Tables.WORKFLOW_USER_ACCESS) - .where(Tables.WORKFLOW_USER_ACCESS.WID.in(wids: _*)) - .execute() - ctx - .deleteFrom(Tables.WORKFLOW_OF_USER) - .where(Tables.WORKFLOW_OF_USER.WID.in(wids: _*)) - .execute() - ctx.deleteFrom(Tables.WORKFLOW).where(Tables.WORKFLOW.WID.in(wids: _*)).execute() - ctx - .deleteFrom(Tables.USER) - .where(Tables.USER.UID.in(Integer.valueOf(ownerUid), Integer.valueOf(strangerUid))) + // The fixture owns the source and any revision clones. Their access and version rows cascade. + val db = getDSLContext + val ownership = Tables.WORKFLOW_OF_USER + val owned = db + .select(ownership.WID) + .from(ownership) + .where(ownership.UID.in(owner.getUid, stranger.getUid)) + db.deleteFrom(Tables.WORKFLOW) + .where(Tables.WORKFLOW.WID.eq(testWorkflowWid).or(Tables.WORKFLOW.WID.in(owned))) .execute() - } - - override protected def afterAll(): Unit = { - closeConnectionPool() + db.deleteFrom(Tables.USER).where(Tables.USER.UID.in(owner.getUid, stranger.getUid)).execute() } private def makeUser(uid: Int, name: String): User = { @@ -163,42 +99,24 @@ class WorkflowVersionResourceSpec } private def grantAccess(uid: Int, privilege: PrivilegeEnum): Unit = { - val access = new WorkflowUserAccess - access.setUid(Integer.valueOf(uid)) - access.setWid(Integer.valueOf(testWorkflowWid)) - access.setPrivilege(privilege) - workflowUserAccessDao.insert(access) + val access = Tables.WORKFLOW_USER_ACCESS + getDSLContext + .insertInto(access, access.UID, access.WID, access.PRIVILEGE) + .values(Int.box(uid), testWorkflow.getWid, privilege) + .execute() } private def session(user: User): SessionUser = new SessionUser(user) - private def createWorkflowContent(value: String): String = { - val jsonNode = objectMapper.createObjectNode() - jsonNode.put("value", value) - jsonNode.toString - } - - private def createVersionDiff(oldValue: String, newValue: String): String = { - val oldJson = objectMapper.createObjectNode() - oldJson.put("value", oldValue) - - val newJson = objectMapper.createObjectNode() - newJson.put("value", newValue) + private def versionValue(value: String) = objectMapper.createObjectNode().put("value", value) + private def createWorkflowContent(value: String): String = versionValue(value).toString + private def createVersionDiff(oldValue: String, newValue: String): String = + com.flipkart.zjsonpatch.JsonDiff.asJson(versionValue(oldValue), versionValue(newValue)).toString - val patch = com.flipkart.zjsonpatch.JsonDiff.asJson( - oldJson, - newJson - ) - patch.toString - } - - /** A patch whose only op is an `add`, i.e. a semantically important change. */ - private def createAddDiff(): String = { - val before = objectMapper.createObjectNode() - val after = objectMapper.createObjectNode() - after.put("addedKey", "x") - com.flipkart.zjsonpatch.JsonDiff.asJson(before, after).toString - } + private def createAddDiff(): String = + com.flipkart.zjsonpatch.JsonDiff + .asJson(objectMapper.createObjectNode(), versionValue("added")) + .toString private def seedVersion(content: String, minutesAgo: Int): Unit = { val version = new WorkflowVersion @@ -224,91 +142,115 @@ class WorkflowVersionResourceSpec private def versionCount(): Int = versionVids().size - "WorkflowVersionResource" should "return versions in descending order from fetchSubsequentVersions and apply patches correctly" in { - var currentContent = "initial" - for (i <- 1 to 10) { - val newContent = s"version_$i" - val diffContent = createVersionDiff(currentContent, newContent) - - val version = new WorkflowVersion - version.setWid(testWorkflow.getWid) - version.setContent(diffContent) - version.setCreationTime( - new Timestamp(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(10 - i)) - ) - workflowVersionDao.insert(version) - - currentContent = newContent - } - - testWorkflow.setContent(createWorkflowContent(currentContent)) + it should "pin the result of the selected save and protect the author's history" in { + val workflows = new WorkflowResource + val author = session(owner) + val guest = session(stranger) + intercept[javax.ws.rs.BadRequestException]( + resource.pinMacro(testWorkflowWid, "m", null, author) + ).getMessage should include("not a macro") + testWorkflow.setContent("""{"macroInterface":{"id":"m"},"operators":[]}""") workflowDao.update(testWorkflow) - - val midVersionId = 5 - val versions = WorkflowVersionResource.fetchSubsequentVersions( - testWorkflow.getWid, - midVersionId, - getDSLContext - ) - - assert(versions.nonEmpty, "No versions were returned") - - for (i <- 0 until versions.length - 1) { - assert( - versions(i).getVid > versions(i + 1).getVid, - s"Versions not in descending order: ${versions(i).getVid} should be > ${versions(i + 1).getVid}" - ) + intercept[javax.ws.rs.NotFoundException]( + resource.pinMacro(testWorkflowWid, "m", null, author) + ).getMessage should include("no saved version") + def save(value: String): Unit = { + testWorkflow.setContent(s"""{"macroInterface":{"id":"m"},"operators":[],"value":"$value"}""") + workflows.persistWorkflow(testWorkflow, author) } - - val highestVersionId = getDSLContext - .select(DSL.max(Tables.WORKFLOW_VERSION.VID)) - .from(Tables.WORKFLOW_VERSION) - .where(Tables.WORKFLOW_VERSION.WID.eq(testWorkflowWid)) - .fetchOneInto(classOf[Integer]) - - assert(versions.head.getVid === highestVersionId, "First version should have the highest VID") - - capturedVersions.clear() - versions.foreach(v => capturedVersions.append(v.getVid)) - - val workflowFromDb = workflowDao.fetchOneByWid(testWorkflow.getWid) - - val workflowVersionDirect = WorkflowVersionResource.applyPatch(versions, workflowFromDb) - val directVersionContent = - objectMapper.readTree(workflowVersionDirect.getContent).get("value").asText() - - assert( - directVersionContent === s"version_$midVersionId", - s"Workflow content from direct applyPatch should be 'version_$midVersionId' but was '$directVersionContent'" + save("first") + save("second") + val ids = versionVids() + val summary = resource.macroSource(testWorkflowWid, "m", false, author) + summary.revisions shouldBe ids + summary.head shouldBe empty + summary.metadata.readonly shouldBe false + val older = resource.pinMacro(testWorkflowWid, "m", ids.head, author) + older.content.path("value").asText() shouldBe "first" + older.revision shouldBe 1 + val latest = resource.pinMacro(testWorkflowWid, "m", null, author) + resource.macroSource(testWorkflowWid, "m", true, author).head shouldBe Some(latest) + latest.version shouldBe ids.last + latest.revision shouldBe 2 + latest.content.path("value").asText() shouldBe "second" + intercept[javax.ws.rs.NotFoundException](resource.pinMacro(testWorkflowWid, "m", -1, author)) + intercept[javax.ws.rs.BadRequestException]( + resource.pinMacro(testWorkflowWid, "foreign", null, author) ) - - val combinedVersions = WorkflowVersionResource.fetchSubsequentVersions( - testWorkflow.getWid, - midVersionId, - getDSLContext + testWorkflow.setIsPublic(true) + testWorkflow.setPublishedContent(older.content.toString) + testWorkflow.setPublishedName("Public title") + testWorkflow.setPublishedDefaultView(DefaultViewEnum.CANVAS) + workflowDao.update(testWorkflow) + val visible = resource.macroSource(testWorkflowWid, "m", true, guest) + visible.metadata.name shouldBe "Public title" + visible.metadata.readonly shouldBe true + visible.revisions shouldBe empty + visible.head shouldBe empty + intercept[ForbiddenException](resource.pinMacro(testWorkflowWid, "m", ids.last, guest)) + grantAccess(strangerUid, PrivilegeEnum.READ) + resource.pinMacro(testWorkflowWid, "m", ids.head, guest) shouldBe older + resource + .macroSource(testWorkflowWid, "m", false, guest) + .metadata + .name shouldBe testWorkflow.getName + } + + it should "list only granted workflows and their reachable pinned macro occurrences" in { + val workflows = new WorkflowResource + val author = session(owner) + val guest = session(stranger) + val inner = + """{"name":"Inner","id":"i","version":3,"content":{"macroInterface":{"id":"i"},"operators":[]}}""" + val child = + """{"operatorID":"child","operatorType":"WorkflowMacro:i:3","customDisplayName":"Renamed"}""" + val outer = + s"""{"name":"Outer","id":"o","version":2,"content":{"macroInterface":{"id":"o"},"operators":[$child]}}""" + val occurrence = """{"operatorID":"selected","operatorType":"WorkflowMacro:o:2"}""" + val unused = inner.replace("\"i\"", "\"unused\"") + testWorkflow.setContent( + s"""{"macroInterface":{"id":"root"},"operators":[$occurrence,$child],"macroDependencies":[$inner,$outer,$unused]}""" ) - val currentWorkflowForCombined = workflowDao.fetchOneByWid(testWorkflow.getWid) - val workflowVersion = - WorkflowVersionResource.applyPatch(combinedVersions, currentWorkflowForCombined) - - assert(capturedVersions.nonEmpty, "No versions were captured") - assert( - capturedVersions.length === versions.length, - "Captured versions length doesn't match fetched versions" + workflows.persistWorkflow(testWorkflow, author) + val catalog = workflows.macroCatalog(author) + catalog.errors shouldBe empty + catalog.definitions.map(_.id) should contain only "root" + catalog.usages.map(u => u.id -> u.path).toSet shouldBe Set( + "o" -> List("selected"), + "i" -> List("selected", "child"), + "i" -> List("child") ) + catalog.usages + .find(_.path.size == 2) + .get + .label shouldBe s"${testWorkflow.getName} / Outer / Renamed" + workflows.macroCatalog(guest).usages shouldBe empty + grantAccess(strangerUid, PrivilegeEnum.READ) + workflows.macroCatalog(guest) shouldBe catalog + testWorkflow.setContent(s"""{"operators":[$occurrence]}""") + workflowDao.update(testWorkflow) + workflows.macroCatalog(guest).errors.head should include("Missing referenced macro definition") + } - for (i <- versions.indices) { - assert( - capturedVersions(i) === versions(i).getVid, - s"Captured version ${capturedVersions(i)} doesn't match fetched version ${versions(i).getVid} at index $i" - ) + "WorkflowVersionResource" should "return versions in descending order from fetchSubsequentVersions and apply patches correctly" in { + val values = "initial" +: (1 to 10).map(i => s"version_$i") + values.sliding(2).zipWithIndex.foreach { + case (pair, index) => + seedVersion(createVersionDiff(pair.head, pair.last), 10 - index) } - - val midVersionContent = objectMapper.readTree(workflowVersion.getContent).get("value").asText() - assert( - midVersionContent === s"version_$midVersionId", - s"Workflow content should be 'version_$midVersionId' but was '$midVersionContent'" - ) + testWorkflow.setContent(createWorkflowContent(values.last)) + workflowDao.update(testWorkflow) + val ids = versionVids() + val fifth = ids(4) + val versions = + WorkflowVersionResource.fetchSubsequentVersions(testWorkflowWid, fifth, getDSLContext) + versions.map(_.getVid) shouldBe ids.drop(4).reverse + val restored = + WorkflowVersionResource.applyPatch(versions, workflowDao.fetchOneByWid(testWorkflowWid)) + objectMapper.readTree(restored.getContent).path("value").asText() shouldBe "version_5" + resource + .retrieveWorkflowVersion(testWorkflowWid, fifth, session(owner)) + .getContent shouldBe restored.getContent } it should "reconstruct an older content by applying a stored delta (applyPatch, no DB)" in { @@ -433,59 +375,23 @@ class WorkflowVersionResourceSpec } } - "cloneVersion" should "create a new workflow from an existing version" in { - // cloneVersion re-ids operators, so the version must reconstruct to a real - // workflow document; an empty delta keeps the workflow's own content. - val workflowContent = - """{"operators":[{"operatorID":"CSVFileScan-operator-a","operatorType":"CSVFileScan"}],"links":[]}""" - testWorkflow.setContent(workflowContent) - workflowDao.update(testWorkflow) - val version = WorkflowVersionResource.insertNewVersion(testWorkflowWid, "[]") - - val newWid = resource.cloneVersion( - version.getVid, - session(owner), - Map("displayedVersionId" -> 1).asJava - ) - - newWid should not be null - newWid should not be Integer.valueOf(testWorkflowWid) - val cloned = workflowDao.fetchOneByWid(newWid) - cloned should not be null - cloned.getName should include("_copy") - } - - it should "inherit the source workflow's default view" in { - val workflowContent = - """{"operators":[{"operatorID":"CSVFileScan-operator-a","operatorType":"CSVFileScan"}],"links":[]}""" - testWorkflow.setContent(workflowContent) - testWorkflow.setDefaultView(DefaultViewEnum.FORM) - workflowDao.update(testWorkflow) - val version = WorkflowVersionResource.insertNewVersion(testWorkflowWid, "[]") - - val newWid = resource.cloneVersion( - version.getVid, - session(owner), - Map("displayedVersionId" -> 1).asJava - ) - - workflowDao.fetchOneByWid(newWid).getDefaultView shouldBe DefaultViewEnum.FORM - } - - it should "leave the clone defaulting to canvas when the source does" in { - val workflowContent = - """{"operators":[{"operatorID":"CSVFileScan-operator-a","operatorType":"CSVFileScan"}],"links":[]}""" - testWorkflow.setContent(workflowContent) - workflowDao.update(testWorkflow) - val version = WorkflowVersionResource.insertNewVersion(testWorkflowWid, "[]") - - val newWid = resource.cloneVersion( - version.getVid, - session(owner), - Map("displayedVersionId" -> 1).asJava - ) - - workflowDao.fetchOneByWid(newWid).getDefaultView shouldBe DefaultViewEnum.CANVAS + for (view <- List(DefaultViewEnum.CANVAS, DefaultViewEnum.FORM)) { + "cloneVersion" should s"create an independent workflow with the source's $view default" in { + testWorkflow.setContent( + """{"operators":[{"operatorID":"source","operatorType":"CSVFileScan"}],"links":[]}""" + ) + if (view == DefaultViewEnum.FORM) testWorkflow.setDefaultView(view) + workflowDao.update(testWorkflow) + val saved = WorkflowVersionResource.insertNewVersion(testWorkflowWid, "[]") + val wid = + resource.cloneVersion(saved.getVid, session(owner), Map("displayedVersionId" -> 1).asJava) + wid should not be null + wid should not be testWorkflow.getWid + val cloned = workflowDao.fetchOneByWid(wid) + cloned should not be null + cloned.getName should include("_copy") + cloned.getDefaultView shouldBe view + } } // ─── version-importance helpers (pure JSON/timestamp logic) ──────────────── diff --git a/amber/src/test/scala/org/apache/texera/web/service/ExecutionResultServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/ExecutionResultServiceSpec.scala index 7842fb20962..8b55a9f8c64 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/ExecutionResultServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/ExecutionResultServiceSpec.scala @@ -357,6 +357,24 @@ class ExecutionResultServiceSpec event.schema.map(_.getName) shouldBe List("id", "label") } + it should "read the requested execution after another run and reject an unavailable execution" in { + val old = newExecution() + val schema = new Schema(List(new Attribute("id", AttributeType.INTEGER))) + for ((run, value) <- List(old -> 1, newExecution() -> 2)) { + storeResult( + run, + "run-specific", + schema, + Seq(Tuple.builder(schema).add("id", AttributeType.INTEGER, value).build()) + ) + } + val request = paginationRequest.copy(operatorID = "run-specific", executionId = Some(old.id)) + paginate(request).table.map(_.get("id").asInt()) shouldBe List(1) + intercept[IllegalArgumentException] { + paginate(request.copy(executionId = Some(-1L))) + }.getMessage should include("unavailable for this workflow and computing unit") + } + it should "clamp the last page to the rows that exist" in { val executionId = newExecution() val schema = new Schema(List(new Attribute("id", AttributeType.INTEGER))) diff --git a/amber/src/test/scala/org/apache/texera/web/service/ExecutionStatsServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/ExecutionStatsServiceSpec.scala index 39502a278ad..cad43eb00a3 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/ExecutionStatsServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/ExecutionStatsServiceSpec.scala @@ -19,7 +19,12 @@ package org.apache.texera.web.service -import org.apache.pekko.actor.ActorSystem +import org.apache.texera.amber.operator.TestOperators +import io.reactivex.rxjava3.disposables.Disposable +import org.apache.pekko.actor.{ActorSystem, Props} +import org.apache.texera.amber.clustering.SingleNodeListener +import org.apache.texera.amber.engine.e2e.TestUtils +import org.apache.texera.common.compiler.{CompilationErrorHandling, WorkflowCompiler} import org.apache.pekko.testkit.TestKit import org.apache.texera.amber.core.storage.model.VirtualDocument import org.apache.texera.amber.core.storage.{DocumentFactory, VFSURIFactory} @@ -49,8 +54,13 @@ import org.apache.texera.amber.engine.common.executionruntimestate.{ OperatorMetrics, OperatorStatistics } +import org.apache.texera.common.compiler.{MacroExpansion, MacroTestUtils} +import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.texera.dao.MockTexeraDB -import org.apache.texera.dao.jooq.generated.Tables.WORKFLOW_EXECUTIONS +import org.apache.texera.auth.SessionUser +import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource +import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum +import org.apache.texera.dao.jooq.generated.Tables.{WORKFLOW_USER_ACCESS, WORKFLOW_EXECUTIONS} import org.apache.texera.dao.jooq.generated.tables.daos.{ UserDao, WorkflowDao, @@ -81,12 +91,8 @@ import java.sql.Timestamp import java.util.concurrent.ExecutorService /** - * The service class was entirely uncovered. It owns two independent outputs: the websocket - * diff handlers that tell the frontend what each operator is doing, and the runtime-statistics - * iceberg table that the dashboard replays after the execution is over. - * - * Everything here runs against an empty-plan AmberClient, so no engine is involved: the test - * fires the client events straight into the callbacks the service registered. + * The callback tests inject Amber events. The nested macro regression runs real workers and + * compares their websocket statistics with the execution history reopened from Iceberg. * * Two things about the ids below are deliberate. They must not collide with the * runtime-statistics URI of any other suite, because the constructor's `createDocument` @@ -132,6 +138,13 @@ class ExecutionStatsServiceSpec workflow.setLastModifiedTime(new Timestamp(System.currentTimeMillis())) new WorkflowDao(getDSLContext.configuration()).insert(workflow) + getDSLContext + .insertInto(WORKFLOW_USER_ACCESS) + .set(WORKFLOW_USER_ACCESS.WID, testWid) + .set(WORKFLOW_USER_ACCESS.UID, testUid) + .set(WORKFLOW_USER_ACCESS.PRIVILEGE, PrivilegeEnum.READ) + .execute() + val version = new WorkflowVersion version.setWid(testWid) version.setContent("{}") @@ -300,6 +313,182 @@ class ExecutionStatsServiceSpec } } + private val instancePaths = List( + List("left"), + List("right"), + List("wrapper", "child"), + List("another-wrapper", "child"), + List("deep", "middle", "child") + ) + private def repeatedWorkflow = { + import MacroTestUtils._ + val ports = List("accepted", "rejected") + def usage(id: String, definition: String) = instance(id, definition, List("in"), ports) + def body(id: String, child: com.fasterxml.jackson.databind.node.ObjectNode) = + forwardingBody(id, child, List("in" -> "in"), ports.map(port => port -> port)) + val levels = List("inner", "outer", "top") + val nodes = instancePaths.zipWithIndex.flatMap { + case (path, index) => + val source = textSource(s"src-${path.head}", (1 to (index + 1) * 10).mkString("\n")) + List(source, usage(path.head, levels(path.size - 1))) + } + val wires = instancePaths + .map(_.head) + .map(id => link(s"wire-$id", end(s"src-$id", "output-0"), end(id, "in"))) + val split = operator("split", "Split", List("in"), ports, json("random" -> false, "k" -> 60)) + pinned( + graph(nodes, wires), + definition("inner", "Inner", body("inner", split)), + definition("outer", "Outer", body("outer", usage("child", "inner"))), + definition("top", "Top", body("top", usage("middle", "outer"))) + ) + } + + it should "stream and reopen the port counts emitted by real split workers for every occurrence" in { + val root = repeatedWorkflow + val snapshot = root.toString + val plan = MacroTestUtils.macroPlan(root, instancePaths.map(_.head)) + val ctx = new WorkflowContext( + workflowId = WorkflowIdentity(testWid.toLong), + executionId = ExecutionIdentity(testEid.toLong) + ) + val compiled = new WorkflowCompiler(ctx, TestOperators.fixtureFileResolver) + .compile(plan, CompilationErrorHandling.Strict) + val workflow = org.apache.texera.amber.engine.architecture.coordinator.Workflow + .fromCompilationResult(ctx, compiled) + val store = new ExecutionStateStore + system.actorOf(Props[SingleNodeListener](), "cluster-info") + val batches = recordBatches(store.statsStore) { + TestUtils.runWorkflowAndReadResults( + system, + workflow, + Nil, + _.get().size, + configureClient = + client => new ExecutionStatsService(client, store, ctx, plan.macroExpansion) + ) + } + val live = statisticsEvents(batches.flatten) + val rows = persistedRows() + val expansion = plan.macroExpansion.get + rows.map(_.getField[String]("operatorId")).toSet shouldBe expansion.paths.keySet + ExecutionStatsService.restore(expansion, rows.reverseIterator) shouldBe live + getDSLContext + .update(WORKFLOW_EXECUTIONS) + .set(WORKFLOW_EXECUTIONS.MACRO_CONTENT, snapshot) + .where(WORKFLOW_EXECUTIONS.EID.eq(testEid)) + .execute() + val user = new SessionUser(new UserDao(getDSLContext.configuration()).fetchOneByUid(testUid)) + val endpoint = new WorkflowExecutionsResource + endpoint.canvasStatistics(testWid, testEid, user).getEntity shouldBe live + val reopened = MacroExpansion.expand(endpoint.macroContent(testWid, testEid, user)) + reopened.inspectors shouldBe expansion.inspectors + endpoint + .retrieveWorkflowRuntimeStatistics(testWid, testEid, user) + .map(_.operatorId) + .toSet shouldBe expansion.paths.keySet + val denied = new User + denied.setUid(-1) + intercept[javax.ws.rs.WebApplicationException]( + endpoint.canvasStatistics(testWid, testEid, new SessionUser(denied)) + ).getResponse.getStatus shouldBe 401 + instancePaths.zipWithIndex.foreach { + case (path, index) => + val expected = (index + 1) * 10L + val ids = reopened.inspectors(objectMapper.writeValueAsString(path)).ids + val parent = reopened.inspectors(objectMapper.writeValueAsString(path.init)) + val usage = live(parent.ids(path.last)) + val split = live(ids("split")) + val inputs = live(ids("inner__inputs")) + val outputs = live(ids("inner__outputs")) + usage shouldBe split + live(path.head) shouldBe usage + usage.inputPortMetrics shouldBe Map("0" -> expected) + usage.outputPortMetrics.keySet shouldBe Set("0", "1") + usage.outputPortMetrics.values.sum shouldBe expected + all(usage.outputPortMetrics.values) should be > 0L + inputs.outputPortMetrics shouldBe usage.inputPortMetrics + outputs.inputPortMetrics shouldBe usage.outputPortMetrics + outputs.inputPortSizes shouldBe usage.outputPortSizes + outputs.numWorkers shouldBe 0 + } + root.toString shouldBe snapshot + } + + it should "count broadcasts once, keep partial state, and give passthroughs no workers" in { + import MacroTestUtils._ + val a = operator("a", "Split", List("in"), List("hidden", "out")) + val b = changed(a, "operatorID" -> "b") + val off = changed(a, "operatorID" -> "off", "isDisabled" -> true) + val branches = List("a", "b", "off") + val fan = macroGraph( + "fan", + List(a, b, off), + branches.map(id => link(s"to-$id", end("fan__inputs", "in"), end(id, "in"))) ++ + branches.map(id => link(s"from-$id", end(id, "out"), end("fan__outputs", id))), + List("in"), + branches + ) + val identity = macroGraph( + "pass", + Nil, + List(link("through", end("pass__inputs", "in"), end("pass__outputs", "out"))), + List("in"), + List("out") + ) + val snapshot = changed( + graph( + List( + textSource("src", "x"), + operator("identity", "WorkflowMacro:pass:1", List("in"), List("out")), + operator("fork", "WorkflowMacro:fan:1", List("in"), branches) + ), + List( + link("source", end("src", "output-0"), end("identity", "in")), + link("fork-in", end("identity", "out"), end("fork", "in")) + ) + ), + "macroDependencies" -> List( + definition("pass", "Pass", identity), + definition("fan", "Fan", fan) + ) + ) + val exp = Some(MacroExpansion.expand(snapshot)) + val first = MacroExpansion.operatorID(List("fork"), "a") + val second = MacroExpansion.operatorID(List("fork"), "b") + val completed = + metrics(COMPLETED, Seq((0, 8, 80)), Seq((0, 5, 50), (1, 3, 30)), dataProcessingTime = 100) + .withReusedFromCache(true) + val partial = Map("src" -> metrics(COMPLETED, outputs = Seq((0, 3, 30))), first -> completed) + def project(reports: Map[String, OperatorMetrics]) = + ExecutionStatsService.canvasStatistics(exp, reports) + project(Map.empty) shouldBe empty + project(partial.updated(second, completed))("fork").reusedFromCache shouldBe true + val one = project(partial) + one("identity").aggregatedOutputRowCount shouldBe 3 + one("identity").numWorkers shouldBe 0 + one("identity").aggregatedDataProcessingTime shouldBe 0 + one("fork").operatorState shouldBe "Uninitialized" + one("fork").reusedFromCache shouldBe false + val report = + metrics(COMPLETED, Seq((0, 7, 70)), Seq((0, 7, 70), (1, 0, 0)), dataProcessingTime = 20) + val fork = project(partial.updated(second, report))("fork") + fork.inputPortMetrics shouldBe Map("0" -> 8) + fork.inputPortSizes shouldBe Map("0" -> 80) + fork.outputPortSizes shouldBe Map("0" -> 30, "1" -> 0, "2" -> 0) + fork.outputPortMetrics shouldBe Map("0" -> 3, "1" -> 0, "2" -> 0) + fork.numWorkers shouldBe 2 + fork.aggregatedDataProcessingTime shouldBe 120 + fork.operatorState shouldBe "Completed" + fork.reusedFromCache shouldBe false + List(PAUSED, PAUSING, RESUMING, RUNNING, READY).foreach { state => + project(partial.updated(second, report.withOperatorState(state)))( + "fork" + ).operatorState shouldBe + org.apache.texera.amber.engine.common.Utils.aggregatedStateToString(state) + } + } + it should "publish nothing when the reported metrics repeat" in { // The frontend redraws the whole statistics panel per event, so re-announcing identical // numbers is pure churn on every heartbeat of a long-running execution. @@ -481,6 +670,10 @@ class ExecutionStatsServiceSpec .execute() withService { _ => + val user = new SessionUser(new UserDao(getDSLContext.configuration()).fetchOneByUid(testUid)) + new WorkflowExecutionsResource() + .canvasStatistics(testWid, testEid, user) + .getStatus shouldBe 204 getDSLContext .select(WORKFLOW_EXECUTIONS.RUNTIME_STATS_URI) .from(WORKFLOW_EXECUTIONS) @@ -508,11 +701,9 @@ class ExecutionStatsServiceSpec Map("opA" -> metrics(RUNNING, inputs = Seq((0, 1L, 10L))), "opB" -> opBFinal) ) ) - awaitPersist(f.service) // opB is gone from the second report, as happens once an operator completes and stops // being polled. f.client.fire(RuntimeStatisticsPersist(Map("opA" -> metrics(COMPLETED)))) - awaitPersist(f.service) f.client.fire(ExecutionStateUpdate(COMPLETED)) val rows = persistedRows() diff --git a/amber/src/test/scala/org/apache/texera/web/service/ExecutionsMetadataPersistServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/ExecutionsMetadataPersistServiceSpec.scala index 1bb151507c6..fab346fc0ac 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/ExecutionsMetadataPersistServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/ExecutionsMetadataPersistServiceSpec.scala @@ -97,13 +97,18 @@ class ExecutionsMetadataPersistServiceSpec testUid, executionName = "named-execution", environmentVersion = "env-1", - computingUnitId = seededCuid + computingUnitId = seededCuid, + macroContent = Some( + org.apache.texera.amber.util.JSONUtils.objectMapper + .readTree("""{"operators":[],"draft":true}""") + ) ) id should not be ExecutionIdentity(0L) val stored = workflowExecutionsDao.fetchOneByEid(id.id.toInt) stored should not be null stored.getVid shouldBe seededVid + stored.getMacroContent shouldBe """{"operators":[],"draft":true}""" stored.getUid shouldBe testUid stored.getName shouldBe "named-execution" stored.getEnvironmentVersion shouldBe "env-1" diff --git a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala index d84a1cd88c2..f0896728ed8 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala @@ -313,9 +313,9 @@ class WorkflowExecutionServiceSpec events.collect { case e: WorkflowStateEvent => e } shouldBe Seq( - WorkflowStateEvent("Running"), - WorkflowStateEvent("Recovering"), - WorkflowStateEvent("Running") + WorkflowStateEvent("Running", Some(1L)), + WorkflowStateEvent("Recovering", Some(1L)), + WorkflowStateEvent("Running", Some(1L)) ) } @@ -336,7 +336,7 @@ class WorkflowExecutionServiceSpec store.metadataStore.updateState(_.withState(COMPLETED).withIsRecovering(true)) events.collect { case e: WorkflowStateEvent => e } shouldBe - Seq(WorkflowStateEvent("Completed")) + Seq(WorkflowStateEvent("Completed", Some(1L))) } it should "report a compilation failure and leave the workflow and the client unset" in { diff --git a/amber/src/test/scala/org/apache/texera/web/service/WorkflowServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/WorkflowServiceSpec.scala index c26dd76d0b0..9e4ebcd0c5d 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/WorkflowServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/WorkflowServiceSpec.scala @@ -680,7 +680,15 @@ class WorkflowServiceSpec portRowCount(previousEid) shouldBe 2 executionsOutsideFixture shouldBe 0 - service.initExecutionService(request, Some(executingUser), new URI("vfs:///session")) + val snapshot = org.apache.texera.amber.util.JSONUtils.objectMapper + .readTree(getClass.getResource("/macros/nested-expansion.json")) + .get("content") + service.initExecutionService( + request, + Some(executingUser), + new URI("vfs:///session"), + Some(snapshot) + ) val execution = service.executionService.getValue // Stopped exactly where the header says it stops: one fatal error, the compiler's. A second @@ -725,6 +733,7 @@ class WorkflowServiceSpec // from the request throughout. executionsOutsideFixture shouldBe 1 val newRow = insertedExecution() + newRow.getMacroContent shouldBe snapshot.toString newRow.getName shouldBe "cleanup-spec-run" newRow.getCuid.intValue() shouldBe testCuid newRow.getUid.intValue() shouldBe testUid diff --git a/bin/dockerfiles/agent-service.dockerfile b/bin/dockerfiles/agent-service.dockerfile index 6deaa5005d9..b0655d9da94 100644 --- a/bin/dockerfiles/agent-service.dockerfile +++ b/bin/dockerfiles/agent-service.dockerfile @@ -28,6 +28,7 @@ FROM docker.io/oven/bun:1.3.3-alpine@sha256:d2bc1fbc3afcd3d70afc2bb2544235bf559c WORKDIR /app +COPY common/workflow-core/src/main/typescript/workflow-macros /common/workflow-core/src/main/typescript/workflow-macros COPY agent-service/package.json agent-service/bun.lock ./ # The download cache holds registry manifests fetched at build time; node_modules diff --git a/bin/dockerfiles/texera-web-application.dockerfile b/bin/dockerfiles/texera-web-application.dockerfile index 2d58c5c9052..84e42d14763 100644 --- a/bin/dockerfiles/texera-web-application.dockerfile +++ b/bin/dockerfiles/texera-web-application.dockerfile @@ -31,6 +31,7 @@ RUN --mount=type=bind,source=bin/dockerfiles/snapshot,target=/snapshot \ bash /snapshot/install.sh apt --no-install-recommends python3 build-essential git ca-certificates WORKDIR /frontend +COPY common/workflow-core/src/main/typescript/workflow-macros /common/workflow-core/src/main/typescript/workflow-macros COPY frontend /frontend COPY common/workflow-operator/src/main/resources/udf/ui-parameter-types.json /common/workflow-operator/src/main/resources/udf/ui-parameter-types.json # The repo's .yarnrc.yml pins Yarn (yarnPath) and the node-modules linker; diff --git a/bin/k8s/templates/base/gateway/gateway-routes.yaml b/bin/k8s/templates/base/gateway/gateway-routes.yaml index e498090b820..cee14b69da0 100644 --- a/bin/k8s/templates/base/gateway/gateway-routes.yaml +++ b/bin/k8s/templates/base/gateway/gateway-routes.yaml @@ -172,7 +172,7 @@ spec: value: /wsapi - path: type: RegularExpression - value: "^/api/executions/\\d+/stats/\\d+$" + value: "^/api/executions/\\d+/stats/\\d+(/canvas)?$" - path: type: PathPrefix value: /api/executions/result/export diff --git a/build.sbt b/build.sbt index 92fdec3a837..0ffb07f3f30 100644 --- a/build.sbt +++ b/build.sbt @@ -255,7 +255,7 @@ lazy val WorkflowCompiler = (project in file("common/workflow-compiler")) .configs(Test) .dependsOn(WorkflowOperator) lazy val WorkflowCompilingService = (project in file("workflow-compiling-service")) - .dependsOn(WorkflowCompiler, Auth, Config, Resource, Observability) + .dependsOn(WorkflowCompiler % "compile->compile;test->test", Auth, Config, Resource, Observability) .configs(Test) .dependsOn(DAO % "test->test") // reuse MockTexeraDB embedded Postgres in tests .dependsOn(WorkflowOperator % "test->test") // reuse PythonWorkerPool in verify tests @@ -273,7 +273,7 @@ lazy val WorkflowCompilingService = (project in file("workflow-compiling-service ) lazy val WorkflowExecutionService = (project in file("amber")) - .dependsOn(WorkflowCompiler, Auth, Config, Observability) + .dependsOn(WorkflowCompiler % "compile->compile;test->test", Auth, Config, Observability) .settings(commonModuleSettings) .settings( dependencyOverrides ++= Seq( diff --git a/common/auth/src/main/scala/org/apache/texera/auth/util/WorkflowAccess.scala b/common/auth/src/main/scala/org/apache/texera/auth/util/WorkflowAccess.scala index 8dc9f92fc1c..8f2d881d0ff 100644 --- a/common/auth/src/main/scala/org/apache/texera/auth/util/WorkflowAccess.scala +++ b/common/auth/src/main/scala/org/apache/texera/auth/util/WorkflowAccess.scala @@ -57,10 +57,10 @@ object WorkflowAccess { * @param uid user id, works with workflow id as primary keys in database * @return PrivilegeEnum value indicating NONE/READ/WRITE */ - def getPrivilege(wid: Integer, uid: Integer): PrivilegeEnum = { + def getPrivilege(wid: Integer, uid: Integer, db: DSLContext = context): PrivilegeEnum = { // One round trip: the caller's own ownership row and grant, read together. Both joins are // narrowed to the caller, so a workflow with several owners still yields one row. - val row = context + val row = db .select(WORKFLOW_OF_USER.UID, WORKFLOW_USER_ACCESS.PRIVILEGE) .from(WORKFLOW) .leftJoin(WORKFLOW_OF_USER) diff --git a/common/workflow-compiler/build.sbt b/common/workflow-compiler/build.sbt index b5224529668..b2b62fdb9ba 100644 --- a/common/workflow-compiler/build.sbt +++ b/common/workflow-compiler/build.sbt @@ -36,6 +36,7 @@ Compile / scalacOptions ++= Seq( ////////////////////////////////////////////////////////////////////////////// libraryDependencies ++= Seq( + "com.networknt" % "json-schema-validator" % "1.5.9", "org.scalatest" %% "scalatest" % "3.2.15" % Test ) diff --git a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/MacroExpansion.scala b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/MacroExpansion.scala new file mode 100644 index 00000000000..566c09a5e25 --- /dev/null +++ b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/MacroExpansion.scala @@ -0,0 +1,813 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.common.compiler + +import org.apache.texera.amber.core.workflow.WorkflowGraphIDs + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.annotation.{JsonIgnoreProperties, JsonProperty} +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlanPojo} +import org.apache.texera.amber.core.virtualidentity.OperatorIdentity +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.LogicalOp + +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets.UTF_8 +import java.security.MessageDigest +import java.util.UUID +import scala.collection.mutable +import scala.jdk.CollectionConverters._ +import scala.util.Try + +/** An unsaved body applies only to the named occurrence in a compile preview. */ +case class MacroDraft(path: List[String], content: JsonNode) + +case class MacroResultTarget(operatorID: String, portIndex: Int) + +case class MacroInspectorInput(link: Int, input: Int, output: Int) +case class MacroInspectorPort(direction: String, portID: String, links: List[Int]) +case class MacroInspector( + ids: Map[String, String], + inputs: Map[String, List[MacroInspectorInput]], + terminals: List[MacroInspectorPort] +) + +case class MacroBoundary( + members: List[String], + inputs: List[List[MacroResultTarget]], + outputs: List[List[MacroResultTarget]], + terminal: Boolean +) + +@JsonIgnoreProperties(Array("names", "boundaries")) +case class MacroExpansion( + content: JsonNode, + paths: Map[String, List[String]], + ports: Map[String, List[JsonNode]], + names: Map[String, List[String]] = Map.empty, + resultTargets: Map[String, MacroResultTarget] = Map.empty, + boundaries: Map[String, MacroBoundary] = Map.empty, + inspectors: Map[String, MacroInspector] = Map.empty +) { + + /** A pinned result tab selects the native port behind that particular macro output. */ + @JsonProperty("resultPorts") + def resultPorts: Map[String, List[Option[MacroResultTarget]]] = + boundaries.collect { + case (id, boundary) if !boundary.terminal && boundary.outputs.size > 1 => + id -> boundary.outputs.map(_.headOption) + } +} + +/** Expands immutable, self-contained workflow snapshots before ordinary logical compilation. */ +object MacroExpansion { + class InvalidMacro(val root: String, labels: List[String], cause: Throwable) + extends IllegalArgumentException(s"${labels.mkString(" / ")}: ${cause.getMessage}", cause) + private val namespace = UUID.fromString("33269d11-7448-5d6d-9aeb-68826068e94d") + private def json(value: Any): JsonNode = objectMapper.valueToTree(value) + private def text(node: JsonNode, key: String): String = node.path(key).asText("") + private def values(node: JsonNode): List[JsonNode] = node.elements().asScala.toList + private def array(node: JsonNode, key: String, message: String): List[JsonNode] = { + require(node.path(key).isArray, message) + values(node.path(key)) + } + private def fields(node: JsonNode): List[JsonNode] = { + val value = node.path("formBinding").path("fields") + if (value.isMissingNode || value.isNull) Nil + else { + require(value.isArray, "Invalid exposed macro properties") + values(value) + } + } + private def isMacro(op: JsonNode): Boolean = text(op, "operatorType").startsWith("WorkflowMacro:") + private def isTerminal(op: JsonNode): Boolean = + Set("MacroInput", "MacroOutput")(text(op, "operatorType")) + + /** The public interface is derived from the saved terminal ports. */ + private def boundary(body: JsonNode, direction: String): List[JsonNode] = { + val input = direction == "inputs" + val terminal = values(body.path("operators")).filter(op => + text(op, "operatorType") == (if (input) "MacroInput" else "MacroOutput") + ) + require(terminal.size == 1, "A macro must contain exactly one Inputs and one Outputs operator") + values(terminal.head.path(if (input) "outputPorts" else "inputPorts")).map { port => + port + .deepCopy[ObjectNode]() + .set[JsonNode]( + "targets", + json( + List( + Map( + "operatorID" -> text(terminal.head, "operatorID"), + "portID" -> text(port, "portID") + ) + ) + ) + ) + } + } + + private def portKey(id: String, direction: String, port: String): String = + json(List(id, direction, port)).toString + private def runtimeID(value: JsonNode): String = { + val bytes = ByteBuffer + .allocate(16) + .putLong(namespace.getMostSignificantBits) + .putLong(namespace.getLeastSignificantBits) + .array() + val digest = MessageDigest.getInstance("SHA-1") + digest.update(bytes) + val hash = digest.digest(value.toString.getBytes(UTF_8)) + hash(6) = ((hash(6) & 0x0f) | 0x50).toByte + hash(8) = ((hash(8) & 0x3f) | 0x80).toByte + val buffer = ByteBuffer.wrap(hash) + new UUID(buffer.getLong(), buffer.getLong()).toString + } + def operatorID(chain: List[String], id: String): String = + if (chain.isEmpty) id else runtimeID(json(chain :+ id)) + + private def nativeDescriptors(expansion: MacroExpansion): Map[String, Try[LogicalOp]] = + values(expansion.content.path("operators")).map { op => + val descriptor = op.path("operatorProperties").deepCopy[ObjectNode]() + List("operatorID", "operatorType", "inputPorts", "outputPorts").foreach(k => + descriptor.set[JsonNode](k, op.path(k)) + ) + text(op, "operatorID") -> Try(objectMapper.treeToValue(descriptor, classOf[LogicalOp])) + }.toMap + + def validateProperties(operator: JsonNode, dependencies: List[JsonNode]): Unit = { + require(operator != null && isMacro(operator), "Expected a macro operator") + val root = json( + Map( + "operators" -> List(operator.deepCopy[ObjectNode]().put("isDisabled", false)), + "links" -> Nil, + "macroDependencies" -> dependencies + ) + ) + val expanded = expand(root, propertiesOnly = true) + MacroValidation.validate(expanded, nativeDescriptors(expanded), propertiesOnly = true) + } + + def logicalPlan( + pojo: LogicalPlanPojo, + expansion: MacroExpansion, + errorList: Option[mutable.Buffer[(OperatorIdentity, Throwable)]] = None + ): LogicalPlanPojo = { + val all = values(expansion.content.path("operators")) + val descriptors = nativeDescriptors(expansion) + try MacroValidation.validate(expansion, descriptors) + catch { + case error: MacroValidation.ValidationErrors if errorList.isDefined => + errorList.foreach(_ ++= error.errors) + } + val decoded = all + .filter(op => + pojo.executeOperatorIds.forall(_.contains(expansion.paths(text(op, "operatorID")).head)) + ) + .flatMap { op => + try Some(text(op, "operatorID") -> descriptors(text(op, "operatorID")).get) + catch { + case scala.util.control.NonFatal(error) if errorList.isDefined => + errorList.foreach(_ += OperatorIdentity(text(op, "operatorID")) -> error) + None + } + } + .toMap + val operators = all.filter(op => decoded.contains(text(op, "operatorID"))) + val byID = operators.map(op => text(op, "operatorID") -> op).toMap + val links = values(expansion.content.path("links")) + .filter(link => + byID.contains(text(link.path("source"), "operatorID")) && + byID.contains(text(link.path("target"), "operatorID")) + ) + .map { link => + def ordinal(end: JsonNode, direction: String): PortIdentity = + PortIdentity( + values(byID(text(end, "operatorID")).path(direction)) + .indexWhere(text(_, "portID") == text(end, "portID")) + ) + LogicalLink( + OperatorIdentity(text(link.path("source"), "operatorID")), + ordinal(link.path("source"), "outputPorts"), + OperatorIdentity(text(link.path("target"), "operatorID")), + ordinal(link.path("target"), "inputPorts") + ) + } + val results = pojo.opsToViewResult.flatMap { id => + if (byID.contains(id)) List(id) + else + pojo.macroContent.toList + .flatMap(c => values(c.path("operators"))) + .find(text(_, "operatorID") == id) + .toList + .flatMap(op => values(op.path("outputPorts"))) + .flatMap(port => + expansion.ports.getOrElse(portKey(id, "outputs", text(port, "portID")), Nil) + ) + .map(text(_, "operatorID")) + } ++ operators.filter(_.path("viewResult").asBoolean()).map(text(_, "operatorID")) + LogicalPlanPojo( + operators.map(op => decoded(text(op, "operatorID"))), + links, + results.filter(byID.contains).distinct, + operators + .filter(op => + op.path("markedForReuse").asBoolean() || pojo.opsToReuseResult + .contains(expansion.paths(text(op, "operatorID")).head) + ) + .map(text(_, "operatorID")), + expandedMacros = Some(expansion) + ) + } + + private case class Definition(id: String, name: String, content: JsonNode) + + /** Index immutable captures without requiring a runnable graph. Drafts still have usages. */ + private def indexDefinitions( + content: JsonNode, + additional: Iterable[JsonNode] = Nil + ): Map[String, Definition] = { + val definitions = mutable.Map.empty[String, Definition] + def collect(definition: JsonNode): Unit = { + val version = definition.path("version") + require( + definition.path("id").isTextual && text(definition, "id").nonEmpty && + definition.path("name").isTextual && version.isNumber && + version.decimalValue().stripTrailingZeros().scale() <= 0 && + version.decimalValue().signum() > 0 && + version.decimalValue().compareTo(java.math.BigDecimal.valueOf(9007199254740991L)) <= 0 && + definition.path("content").path("macroInterface").path("id") == definition.path("id"), + "Malformed macro definition" + ) + val id = text(definition, "id") + val body = objectMapper + .createObjectNode() + .setAll[ObjectNode](definition.get("content").asInstanceOf[ObjectNode]) + body.remove("macroDependencies") + val key = s"WorkflowMacro:$id:${version.asLong()}" + require(definitions.get(key).forall(_.content == body), "Conflicting macro version") + definitions(key) = Definition(id, text(definition, "name"), body) + collectBundle(definition.path("content")) + } + def collectBundle(body: JsonNode): Unit = { + if (body.hasNonNull("macroDependencies")) + array(body, "macroDependencies", "Malformed macro definitions").foreach(collect) + } + collectBundle(content) + additional.foreach(collectBundle) + definitions.toMap + } + + case class Reference(id: String, path: List[String], names: List[String]) + def references(content: JsonNode): List[Reference] = { + val captures = indexDefinitions(content) + def visit( + body: JsonNode, + path: List[String], + names: List[String], + ancestors: Set[String] + ): List[Reference] = { + require(path.size <= 32, "Macro nesting exceeds 32 levels") + array(body, "operators", "Malformed workflow operators").filter(isMacro).flatMap { op => + val key = text(op, "operatorType") + val definition = captures.getOrElse( + key, + throw new IllegalArgumentException(s"Missing referenced macro definition: $key") + ) + val next = Reference( + definition.id, + path :+ text(op, "operatorID"), + names :+ op.path("customDisplayName").asText(definition.name) + ) + next :: (if (ancestors(key)) Nil + else visit(definition.content, next.path, next.names, ancestors + key)) + } + } + visit(content, Nil, Nil, Set.empty) + } + + def expand( + content: JsonNode, + drafts: List[MacroDraft] = Nil, + propertiesOnly: Boolean = false + ): MacroExpansion = { + require(content != null && content.isObject, "Malformed macro graph") + WorkflowGraphIDs.validate(content) + val draftBodies = drafts.map { draft => + require( + draft != null && draft.path != null && draft.path.nonEmpty && draft.path.length <= 32 && + draft.path.forall(id => id != null && id.trim.nonEmpty), + "Invalid macro draft path" + ) + WorkflowGraphIDs.validate(draft.content) + draft.path -> draft.content + }.toMap + require(draftBodies.size == drafts.size, "Duplicate macro draft path") + val definitions = indexDefinitions(content, drafts.map(_.content)) + + def bodyFor(op: JsonNode, path: List[String]): Option[JsonNode] = + definitions + .get(text(op, "operatorType")) + .map(definition => draftBodies.getOrElse(path, definition.content)) + + // Check even disabled occurrences, which the executable graph deliberately skips. + drafts.foreach { draft => + var parent = content + var path = List.empty[String] + draft.path.foreach { id => + path = path :+ id + val op = values(parent.path("operators")).find(text(_, "operatorID") == id) + val pinned = op.flatMap(o => definitions.get(text(o, "operatorType"))).getOrElse { + throw new IllegalArgumentException(s"Unknown macro draft: ${path.mkString(" / ")}") + } + parent = draftBodies.getOrElse(path, pinned.content) + require( + parent.at("/macroInterface/id") == pinned.content.at("/macroInterface/id"), + "Macro draft identity differs from its pinned definition" + ) + } + } + val operators = mutable.ListBuffer.empty[JsonNode] + val links = mutable.ListBuffer.empty[JsonNode] + val paths = mutable.LinkedHashMap.empty[String, List[String]] + val names = mutable.LinkedHashMap.empty[String, List[String]] + val ports = mutable.LinkedHashMap.empty[String, List[JsonNode]] + val runtimeIDs = mutable.HashSet.empty[String] + val terminals = mutable.Map.empty[String, String] + val boundaryOps = mutable.LinkedHashMap.empty[String, (List[String], JsonNode)] + val sinkTargets = mutable.Map.empty[String, MacroResultTarget] + + def visit( + body: JsonNode, + chain: List[String], + stack: Set[String], + labels: List[String], + overrides: JsonNode = objectMapper.createObjectNode() + ): Map[String, List[JsonNode]] = { + val ops = array(body, "operators", "Malformed macro graph") + val edges = if (propertiesOnly) Nil else array(body, "links", "Malformed macro graph") + require(chain.size <= 32, "Macro nesting exceeds 32 levels") + val byID = mutable.LinkedHashMap.empty[String, JsonNode] + def checkPorts(group: List[JsonNode], message: String): Unit = + require( + group.forall(p => p.path("portID").isTextual && text(p, "portID").nonEmpty) && + group.map(text(_, "portID")).distinct.size == group.size, + message + ) + ops.foreach { op => + val id = text(op, "operatorID") + require( + op.path("operatorType").isTextual && op.path("operatorProperties").isObject, + "Malformed operator" + ) + List("inputPorts", "outputPorts").foreach { direction => + checkPorts( + array(op, direction, "Malformed operator"), + "Duplicate or invalid operator port" + ) + } + byID(id) = op + } + val savedTerminals = ops.filter(isTerminal) + if (savedTerminals.nonEmpty) { + require(body.hasNonNull("macroInterface"), "Inputs/Outputs belong in a macro definition") + require( + !body.path("macroInterface").has("inputs") && !body.path("macroInterface").has("outputs"), + "The macro interface must be stored in terminal operators" + ) + List("inputs", "outputs").foreach(boundary(body, _)) + savedTerminals.foreach { op => + require(!op.path("isDisabled").asBoolean(), "Macro terminals cannot be disabled") + val input = text(op, "operatorType") == "MacroInput" + require( + op.path(if (input) "inputPorts" else "outputPorts").isEmpty, + "Invalid macro terminal direction" + ) + if (!propertiesOnly) values(op.path(if (input) "outputPorts" else "inputPorts")).foreach { + port => + val matches = edges.count(link => { + val end = link.path(if (input) "source" else "target") + text(end, "operatorID") == text(op, "operatorID") && text(end, "portID") == text( + port, + "portID" + ) + }) + val name = port.path("displayName").asText(text(port, "portID")) + if (matches == 0) + throw new IllegalArgumentException( + s"Connect or remove macro ${if (input) "input" else "output"} '$name' before running." + ) + if (!input && matches != 1) + throw new IllegalArgumentException( + s"Macro output '$name' must connect to exactly one operator output." + ) + } + } + } + if (body.hasNonNull("macroInterface")) { + val boundary = body.path("macroInterface") + require( + boundary.path("id").isTextual && text(boundary, "id").nonEmpty, + "Invalid macro interface ID" + ) + List("inputs", "outputs").foreach(MacroExpansion.boundary(body, _)) + val exposed = fields(body) + require( + exposed.map(text(_, "id")).distinct.size == exposed.size, + "Duplicate exposed macro property" + ) + val targets = exposed.map { field => + require( + field.path("id").isTextual && text(field, "id").nonEmpty && + field.path("propertyKey").isTextual && byID.contains(text(field, "operatorID")), + "Invalid exposed macro property" + ) + val target = byID(text(field, "operatorID")) + require( + !isMacro(target) || bodyFor(target, chain :+ text(target, "operatorID")) + .exists(d => fields(d).exists(text(_, "id") == text(field, "propertyKey"))), + s"Missing exposed macro property: ${field.path("displayName").asText(text(field, "id"))}" + ) + text(field, "id") -> (field, target) + }.toMap + overrides.fields().asScala.foreach { property => + val (binding, target) = targets.getOrElse( + property.getKey, + throw new IllegalArgumentException(s"Unknown macro property: ${property.getKey}") + ) + target + .path("operatorProperties") + .asInstanceOf[ObjectNode] + .set[JsonNode](text(binding, "propertyKey"), property.getValue.deepCopy[JsonNode]()) + } + } + val resolved = mutable.LinkedHashMap.empty[String, List[JsonNode]] + def endpoint(port: JsonNode, direction: String): List[JsonNode] = { + val id = text(port, "operatorID") + val portID = text(port, "portID") + require( + port.path("operatorID").isTextual && port.path("portID").isTextual && + byID + .get(id) + .exists( + _.path(if (direction == "inputs") "inputPorts" else "outputPorts") + .elements() + .asScala + .exists(text(_, "portID") == portID) + ), + "Missing operator or port in macro graph" + ) + if (byID(id).path("isDisabled").asBoolean()) Nil + else + resolved.getOrElse( + portKey(id, direction, portID), + throw new IllegalArgumentException("Missing exposed macro port") + ) + } + ops.filterNot(_.path("isDisabled").asBoolean()).foreach { op => + val id = text(op, "operatorID") + val label = Option(text(op, "customDisplayName")) + .filter(_.trim.nonEmpty) + .getOrElse( + definitions + .get(text(op, "operatorType")) + .map(_.name) + .getOrElse(MacroValidation.operatorName(text(op, "operatorType"))) + ) + val runtime = operatorID(chain, id) + require(runtimeIDs.add(runtime), "Duplicate runtime operator ID") + if (isTerminal(op) || isMacro(op)) boundaryOps(runtime) = (chain :+ id, op) + if (isTerminal(op)) { + terminals(runtime) = chain.headOption.getOrElse(id) + names(runtime) = labels :+ label + values( + op.path(if (text(op, "operatorType") == "MacroInput") "outputPorts" else "inputPorts") + ).foreach { port => + val end = json(Map("operatorID" -> runtime, "portID" -> text(port, "portID"))) + List("inputs", "outputs").foreach { direction => + resolved(portKey(id, direction, text(port, "portID"))) = List(end) + ports(portKey(runtime, direction, text(port, "portID"))) = List(end) + } + } + } else if (!isMacro(op)) { + operators += op.deepCopy[ObjectNode]().put("operatorID", runtime) + paths(runtime) = chain :+ id + if (text(op, "operatorType").toLowerCase(java.util.Locale.ROOT).contains("sink")) + chain.inits + .filter(_.nonEmpty) + .foreach(path => + sinkTargets + .getOrElseUpdate(operatorID(path.init, path.last), MacroResultTarget(runtime, 0)) + ) + names(runtime) = labels :+ label + List("inputs", "outputs").foreach { direction => + op.path(if (direction == "inputs") "inputPorts" else "outputPorts") + .elements() + .asScala + .foreach { port => + resolved(portKey(id, direction, text(port, "portID"))) = List( + objectMapper + .createObjectNode() + .put("operatorID", runtime) + .put("portID", text(port, "portID")) + ) + } + } + } else { + try { + val key = text(op, "operatorType") + val path = chain :+ id + val inner = bodyFor(op, path) + .getOrElse(throw new IllegalArgumentException(s"Missing macro version: $key")) + .deepCopy[ObjectNode]() + require( + op.path("operatorVersion") + .isTextual && text(op, "operatorVersion") == key.split(":").last, + "Macro version does not match its pinned definition" + ) + // An occurrence override is a distinct body. Its children may legitimately use the saved pin. + val ancestors = + if (draftBodies.contains(path)) Set.empty[String] + else { + require(!stack(key), "Recursive macro reference") + stack + key + } + val innerPorts = + visit(inner, path, ancestors, labels :+ label, op.path("operatorProperties")) + List("inputs", "outputs").foreach { direction => + val boundary = + MacroExpansion.boundary(inner, direction) + val declared = array( + op, + if (direction == "inputs") "inputPorts" else "outputPorts", + "Malformed operator" + ) + require( + declared.size == boundary.size && declared + .forall(p => boundary.exists(text(_, "portID") == text(p, "portID"))), + "Macro instance interface differs from its pinned version" + ) + boundary.foreach { port => + val targets = port + .path("targets") + .elements() + .asScala + .toList + .flatMap(t => + innerPorts + .getOrElse(portKey(text(t, "operatorID"), direction, text(t, "portID")), Nil) + ) + resolved(portKey(id, direction, text(port, "portID"))) = targets + // Nested boundaries let every inspector project this same authoritative expansion. + ports(portKey(runtime, direction, text(port, "portID"))) = targets + } + } + } catch { + case error: InvalidMacro => throw error + case scala.util.control.NonFatal(error) => + throw new InvalidMacro( + chain.headOption.getOrElse(id), + labels :+ label, + error + ) + } + } + } + edges.foreach { link => + val sources = endpoint(link.path("source"), "outputs") + val targets = endpoint(link.path("target"), "inputs") + for (source <- sources; target <- targets) { + val id = + if (chain.nonEmpty || sources.size * targets.size > 1) + runtimeID(json(List(json(chain), link.path("linkID"), source, target))) + else text(link, "linkID") + links += objectMapper + .createObjectNode() + .put("linkID", id) + .set[ObjectNode]("source", source) + .set[ObjectNode]("target", target) + } + } + resolved.toMap + } + visit(content, Nil, Set.empty, Nil) + // Draft property checks end at native descriptors. Links, regions and runtime mappings belong to compilation. + if (propertiesOnly) + return MacroExpansion( + json(Map("operators" -> operators.toList)), + paths.toMap, + Map.empty, + names.toMap + ) + // Terminals are junctions, never workers. Collapse them only after the entire parent graph + // is wired, so passthrough macros and mixed old/new definitions use the same lowering. + if (terminals.nonEmpty) { + def key(end: JsonNode): String = + portKey(text(end, "operatorID"), "inputs", text(end, "portID")) + val forward = links.toList.groupBy(link => key(link.path("source"))) + val backward = links.toList.groupBy(link => key(link.path("target"))) + def follow( + end: JsonNode, + downstream: Boolean, + visiting: Set[String] = Set.empty + ): List[JsonNode] = { + if (!terminals.contains(text(end, "operatorID"))) List(end) + else { + val id = key(end) + if (visiting(id)) + throw new InvalidMacro( + terminals(text(end, "operatorID")), + names(text(end, "operatorID")), + new IllegalArgumentException("Workflow contains a cycle through macro terminals") + ) + (if (downstream) forward else backward) + .getOrElse(id, Nil) + .flatMap(link => + follow(link.path(if (downstream) "target" else "source"), downstream, visiting + id) + ) + .distinct + } + } + ports.keys.toList.foreach { key => + val direction = objectMapper.readTree(key).get(1).asText() + ports(key) = ports(key).flatMap(follow(_, direction == "inputs")).distinct + } + val lowered = links.toList + .filterNot(link => terminals.contains(text(link.path("source"), "operatorID"))) + .flatMap { link => + follow(link.path("target"), downstream = true).map { target => + if (target == link.path("target")) link + else + link + .deepCopy[ObjectNode]() + .put("linkID", runtimeID(json(List(link.path("source"), target)))) + .set[JsonNode]("target", target) + } + } + links.clear() + links ++= lowered.distinct + } + val children = links.toList.groupBy(_.path("source").path("operatorID").asText()) + val visiting = mutable.Set.empty[String] + val visited = mutable.Set.empty[String] + def checkCycle(id: String): Unit = { + if (visiting(id)) + throw new InvalidMacro( + paths(id).head, + names(id), + new IllegalArgumentException("Workflow contains a cycle") + ) + if (!visited(id)) { + visiting += id + children + .getOrElse(id, Nil) + .foreach(link => checkCycle(text(link.path("target"), "operatorID"))) + visiting -= id + visited += id + } + } + paths.keys.foreach(checkCycle) + val expanded = content.deepCopy[ObjectNode]() + expanded.set[JsonNode]("operators", json(operators.toList)) + expanded.set[JsonNode]("links", json(links.toList)) + expanded.remove(List("macroInterface", "macroDependencies").asJava) + val native = operators.iterator.map(op => text(op, "operatorID") -> op).toMap + val members = + paths.toList.flatMap { case (id, path) => path.init.inits.map(_ -> id) }.groupMap(_._1)(_._2) + val ordinals = native.toList.flatMap { + case (id, op) => + List(true, false).flatMap { input => + values(op.path(if (input) "inputPorts" else "outputPorts")).zipWithIndex.map { + case (port, index) => + (id, input, text(port, "portID")) -> index + } + } + }.toMap + def nativePort(end: JsonNode, input: Boolean): MacroResultTarget = { + val id = text(end, "operatorID") + MacroResultTarget(id, ordinals.getOrElse((id, input, text(end, "portID")), -1)) + } + val boundaries = boundaryOps.map { + case (id, (path, op)) => + val terminal = isTerminal(op) + def resolve(publicPorts: List[String], input: Boolean): List[List[MacroResultTarget]] = + publicPorts.map { port => + ports + .getOrElse(portKey(id, if (input) "inputs" else "outputs", port), Nil) + .map(nativePort(_, input)) + } + id -> MacroBoundary( + members.getOrElse(path, Nil), + resolve(values(op.path("inputPorts")).map(text(_, "portID")), !terminal), + resolve(values(op.path("outputPorts")).map(text(_, "portID")), input = false), + terminal + ) + }.toMap + val targets = boundaries + .collect { + case (id, b) if !b.terminal => + b.outputs.headOption.flatMap(_.headOption).orElse(sinkTargets.get(id)).map(id -> _) + } + .flatten + .toMap + // Index each occurrence against one native graph. Ancestors share its operator bodies and links. + val nativePaths = paths.iterator.map { case (id, path) => id -> (path, native(id)) }.toMap + val direct = (nativePaths ++ boundaryOps).toList.groupBy(_._2._1.init) + val endpoints = links.iterator + .map(link => + nativePort(link.path("source"), input = false) -> nativePort( + link.path("target"), + input = true + ) + ) + .toVector + val byTarget = endpoints.indices.groupBy(i => endpoints(i)._2) + val bySource = endpoints.indices.groupBy(i => endpoints(i)._1) + val terminalsByScope = boundaryOps.collect { + case (id, (path, op)) if isTerminal(op) => + (path.init, text(op, "operatorType") == "MacroInput") -> id + }.toMap + def between(from: Set[MacroResultTarget], to: Set[MacroResultTarget]): List[Int] = { + // At a fan-in or fan-out, visit only the cheaper set of candidate links. + val incoming = to.toList.flatMap(byTarget.get) + val outgoing = from.toList.flatMap(bySource.get) + val indices = + if (outgoing.iterator.map(_.size).sum <= incoming.iterator.map(_.size).sum) + outgoing.iterator.flatten.filter(i => to(endpoints(i)._2)) + else incoming.iterator.flatten.filter(i => from(endpoints(i)._1)) + indices.toList.sorted + } + val crossings = boundaryOps.values.collect { + case (scope, op) if isMacro(op) => + scope -> List(true, false).flatMap { input => + val direction = if (input) "inputs" else "outputs" + val terminal = terminalsByScope((scope, input)) + values(op.path(if (input) "inputPorts" else "outputPorts")).map { port => + val portID = text(port, "portID") + val from = ports + .getOrElse(portKey(terminal, "outputs", portID), Nil) + .map(nativePort(_, input = false)) + .toSet + val to = ports + .getOrElse(portKey(terminal, "inputs", portID), Nil) + .map(nativePort(_, input = true)) + .toSet + MacroInspectorPort(direction, portID, between(from, to)) + } + } + }.toMap + val inspectors = (List.empty[String] :: crossings.keys.toList).map { scope => + val canvas = direct.getOrElse(scope, Nil) + val leaves = members.getOrElse(scope, Nil) + val ids = leaves.map { runtime => + val relative = paths(runtime).drop(scope.size) + operatorID(relative.init, relative.last) -> runtime + }.toMap ++ canvas.map { case (runtime, (path, _)) => path.last -> runtime } + val inputs = canvas.flatMap { + case (runtime, (path, op)) => + val refs = if (isMacro(op)) crossings(path).collect { + case p if p.direction == "inputs" => p.links + } + else + values(op.path("inputPorts")).indices.toList.map(input => + byTarget.getOrElse(MacroResultTarget(runtime, input), Nil) + ) + val incoming = refs.zipWithIndex.flatMap { + case (links, input) => + links.map(i => MacroInspectorInput(i, input, endpoints(i)._1.portIndex)) + } + Option.when(incoming.nonEmpty)(path.last -> incoming) + }.toMap + val visible = leaves.toSet + val external = crossings.getOrElse(scope, Nil).map { port => + port.copy(links = + port.links.filterNot(i => + visible( + (if (port.direction == "inputs") endpoints(i)._1 else endpoints(i)._2).operatorID + ) + ) + ) + } + json(scope).toString -> MacroInspector(ids, inputs, external) + }.toMap + MacroExpansion(expanded, paths.toMap, ports.toMap, names.toMap, targets, boundaries, inspectors) + } +} diff --git a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/MacroValidation.scala b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/MacroValidation.scala new file mode 100644 index 00000000000..139be253c4a --- /dev/null +++ b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/MacroValidation.scala @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.common.compiler + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import com.networknt.schema.{JsonSchema, JsonSchemaFactory, SpecVersion} +import org.apache.texera.amber.core.virtualidentity.OperatorIdentity +import org.apache.texera.amber.core.state.StateReferencing.referencedVariable +import org.apache.texera.amber.core.workflowruntimestate.WorkflowFatalError +import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.util.JSONUtils.objectMapper + +import scala.collection.concurrent.TrieMap +import scala.jdk.CollectionConverters._ +import scala.util.Try + +/** Validate effective properties with the same schemas used by Texera's property editor. */ +private[compiler] object MacroValidation { + private lazy val metadata = OperatorMetadataGenerator.allOperatorMetadata.operators + .map(op => op.operatorType -> op) + .toMap + private val factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7) + private val schemas = TrieMap.empty[String, JsonSchema] + + def operatorName(operatorType: String): String = + metadata.get(operatorType).map(_.additionalMetadata.userFriendlyName).getOrElse(operatorType) + + private def path(expansion: MacroExpansion, id: String): String = + expansion.names.getOrElse(id, expansion.paths.getOrElse(id, List(id))).mkString(" / ") + + class ValidationErrors(val errors: List[(OperatorIdentity, Throwable)], expansion: MacroExpansion) + extends IllegalArgumentException( + errors + .map { case (id, error) => s"${path(expansion, id.id)}: ${error.getMessage}" } + .mkString("\n") + ) + + def validate( + expansion: MacroExpansion, + descriptors: Map[String, Try[LogicalOp]], + propertiesOnly: Boolean = false + ): Unit = { + lazy val incoming = expansion.content + .path("links") + .elements() + .asScala + .toList + .groupBy(link => + ( + link.path("target").path("operatorID").asText(), + link.path("target").path("portID").asText() + ) + ) + val errors = expansion.content.path("operators").elements().asScala.toList.flatMap { op => + val id = op.path("operatorID").asText() + val kind = op.path("operatorType").asText() + val messages = metadata.get(kind).fold(List(s"Unknown operator type: $kind")) { meta => + val descriptor = descriptors(id).toOption + val schema = schemas.getOrElseUpdate(kind, factory.getSchema(meta.jsonSchema)) + // Declared defaults use descriptor initializers; explicit nulls and invalid types stay intact. + val effective = descriptor.fold(op.path("operatorProperties")) { descriptor => + objectMapper + .valueToTree[ObjectNode](descriptor) + .retain( + meta.jsonSchema + .path("properties") + .fields() + .asScala + .filter(_.getValue.has("default")) + .map(_.getKey) + .toList + .asJava + ) + .setAll[JsonNode](op.path("operatorProperties").asInstanceOf[ObjectNode]) + } + // Typed loop references are checked by the existing state-reference compiler after parsing. + val properties = schema + .validate(effective) + .asScala + .toList + .filterNot { error => + error.getType == "type" && error.getInstanceNode.isTextual && + referencedVariable(error.getInstanceNode.asText()).nonEmpty && + Set("integer", "number", "boolean").contains(error.getSchemaNode.asText()) + } + .map(_.getMessage) + if (propertiesOnly) { + if (properties.nonEmpty) properties + else descriptors(id).failed.toOption.toList.map(_.getMessage) + } else { + val info = descriptor.map(_.operatorInfo).getOrElse(meta.additionalMetadata) + val inputs = op + .path("inputPorts") + .elements() + .asScala + .zipWithIndex + .flatMap { + case (port, index) => + val portID = port.path("portID").asText() + val count = incoming.getOrElse((id, portID), Nil).size + val single = info.inputPorts.lift(index).exists(_.disallowMultiLinks) + Option.when(count == 0 || (single && count != 1))( + s"Input '${port.path("displayName").asText(portID)}' ($portID) requires ${if (single) "1 input" + else "at least 1 input"}, has $count" + ) + } + .toList + val declarations = Option + .when( + !info.dynamicInputPorts && op.path("inputPorts").size() != info.inputPorts.size + )( + s"Expected ${info.inputPorts.size} input ports, found ${op.path("inputPorts").size()}" + ) + .toList + properties ++ inputs ++ declarations + } + } + Option.when(messages.nonEmpty)( + OperatorIdentity(id) -> new IllegalArgumentException( + messages.distinct.sorted.mkString("; ") + ) + ) + } + if (errors.nonEmpty) throw new ValidationErrors(errors, expansion) + } + + /** Keep leaf errors for inspectors and aggregate all descendants on each enclosing macro. */ + def propagate( + expansion: MacroExpansion, + errors: Map[OperatorIdentity, WorkflowFatalError] + ): Map[OperatorIdentity, WorkflowFatalError] = { + val named = errors.map { + case (id, error) => + id -> error.copy(message = s"${path(expansion, id.id)}: ${error.message}") + } + val ancestors = named.toList + .flatMap { + case (id, error) => + val ids = expansion.paths.getOrElse(id.id, Nil) + ids.indices + .dropRight(1) + .map(index => + OperatorIdentity(MacroExpansion.operatorID(ids.take(index), ids(index))) -> error + ) + } + .groupBy(_._1) + .map { + case (id, children) => + id -> children.head._2.copy( + operatorId = id.id, + message = children.map(_._2.message).distinct.sorted.mkString("\n") + ) + } + named ++ ancestors + } +} diff --git a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/WorkflowCompiler.scala b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/WorkflowCompiler.scala index e8d162e9abf..376824abe6f 100644 --- a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/WorkflowCompiler.scala +++ b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/WorkflowCompiler.scala @@ -60,7 +60,8 @@ object WorkflowCompiler { // util function to convert the error list to an error map, and report the errors in the log private def convertErrorListToWorkflowFatalErrorMap( logger: Logger, - errorList: List[(OperatorIdentity, Throwable)] + errorList: List[(OperatorIdentity, Throwable)], + message: Throwable => String = _.toString ): Map[OperatorIdentity, WorkflowFatalError] = { val opIdToError = mutable.Map[OperatorIdentity, WorkflowFatalError]() errorList.foreach { @@ -74,7 +75,7 @@ object WorkflowCompiler { opIdToError += (opId -> WorkflowFatalError( COMPILATION_ERROR, Timestamp(Instant.now), - err.toString, + message(err), getStackTraceWithAllCauses(err), opId.id )) @@ -365,23 +366,79 @@ class WorkflowCompiler( logicalPlanPojo: LogicalPlanPojo, errorHandling: CompilationErrorHandling = CompilationErrorHandling.Lenient ): WorkflowCompilationResult = { - // Lenient collects into a buffer; Strict passes None so the first error is thrown. + val hasMacros = + logicalPlanPojo.macroContent.isDefined || logicalPlanPojo.expandedMacros.isDefined + // Macros collect errors so Strict failures can name their enclosing instances too. val errorList: Option[ArrayBuffer[(OperatorIdentity, Throwable)]] = errorHandling match { - case CompilationErrorHandling.Lenient => + case CompilationErrorHandling.Strict if !hasMacros => None + case _ => Some(new ArrayBuffer[(OperatorIdentity, Throwable)]()) - case CompilationErrorHandling.Strict => None } + def errors( + failures: List[(OperatorIdentity, Throwable)] + ): Map[OperatorIdentity, WorkflowFatalError] = { + val result = convertErrorListToWorkflowFatalErrorMap( + logger, + failures, + error => + if (hasMacros) + Option(error.getMessage).getOrElse(error.toString) + else error.toString + ) + Try(logicalPlanPojo.macroExpansion).toOption.flatten + .map(MacroValidation.propagate(_, result)) + .getOrElse(result) + } + val normalized = + try { + require( + errorHandling != CompilationErrorHandling.Strict || + !logicalPlanPojo.macroContent.exists(_.hasNonNull("macroInterface")), + "Run macro definitions inside a workflow" + ) + if ( + errorHandling == CompilationErrorHandling.Lenient && logicalPlanPojo.macroContent.isDefined + ) + MacroExpansion.logicalPlan(logicalPlanPojo, logicalPlanPojo.macroExpansion.get, errorList) + else logicalPlanPojo.normalized + } catch { + case scala.util.control.NonFatal(error) + if errorHandling == CompilationErrorHandling.Lenient => + val operators = logicalPlanPojo.macroContent.toList + .flatMap(_.path("operators").elements().asScala) + val failures = error match { + case invalid: MacroValidation.ValidationErrors => invalid.errors + case _ => + val id = error match { + case invalid: MacroExpansion.InvalidMacro => invalid.root + case _ => + operators + .find(_.path("operatorType").asText().startsWith("WorkflowMacro:")) + .orElse(operators.headOption) + .map(_.path("operatorID").asText()) + .getOrElse("workflow") + } + List(OperatorIdentity(id) -> error) + } + return WorkflowCompilationResult( + LogicalPlan(Nil, Nil), + None, + Map.empty, + errors(failures), + Set.empty + ) + } // 1. convert the pojo to logical plan - val logicalPlan: LogicalPlan = LogicalPlan(logicalPlanPojo) + val logicalPlan: LogicalPlan = LogicalPlan(normalized) // 2. resolve the file name in each scan source operator logicalPlan.resolveScanSourceOpFileName(fileResolver, errorList) // 3. expand the logical plan to the physical plan, and get the output ports that need storage val (physicalPlan, outputPortsNeedingStorage) = - expandLogicalPlan(logicalPlan, logicalPlanPojo.opsToViewResult, errorList) + expandLogicalPlan(logicalPlan, normalized.opsToViewResult, errorList) // 4. collect the output schema for each logical op // even if an error is encountered during logical => physical expansion, we still want to @@ -392,7 +449,9 @@ class WorkflowCompiler( val schemaErrorList = errorList.getOrElse(new ArrayBuffer[(OperatorIdentity, Throwable)]()) val opIdToOutputSchema = collectOutputSchemaFromPhysicalPlan(physicalPlan, schemaErrorList) if (errorHandling == CompilationErrorHandling.Strict && schemaErrorList.nonEmpty) { - throw schemaErrorList.head._2 + throw logicalPlanPojo.macroExpansion + .map(new MacroValidation.ValidationErrors(schemaErrorList.toList, _)) + .getOrElse(schemaErrorList.head._2) } val hasErrors = errorList.exists(_.nonEmpty) @@ -401,10 +460,7 @@ class WorkflowCompiler( physicalPlan = if (hasErrors) None else Some(physicalPlan), operatorIdToOutputSchemas = opIdToOutputSchema, // map each error from OpId to WorkflowFatalError, and report them via logger - operatorIdToError = convertErrorListToWorkflowFatalErrorMap( - logger, - errorList.map(_.toList).getOrElse(List.empty) - ), + operatorIdToError = errors(errorList.map(_.toList).getOrElse(List.empty)), outputPortsNeedingStorage = outputPortsNeedingStorage ) } diff --git a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlan.scala b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlan.scala index 7bdb85cd696..ff092775ef4 100644 --- a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlan.scala +++ b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlan.scala @@ -60,7 +60,7 @@ object LogicalPlan { def apply( pojo: LogicalPlanPojo ): LogicalPlan = { - LogicalPlan(pojo.operators, pojo.links) + LogicalPlan(pojo.normalized.operators, pojo.normalized.links) } } diff --git a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlanPojo.scala b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlanPojo.scala index efda9ceda79..77f3a2c652a 100644 --- a/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlanPojo.scala +++ b/common/workflow-compiler/src/main/scala/org/apache/texera/common/compiler/model/LogicalPlanPojo.scala @@ -20,10 +20,43 @@ package org.apache.texera.common.compiler.model import org.apache.texera.amber.operator.LogicalOp +import com.fasterxml.jackson.annotation.{JsonIgnore, JsonIgnoreProperties, JsonInclude} +import com.fasterxml.jackson.databind.JsonNode +import org.apache.texera.common.compiler.{MacroDraft, MacroExpansion} +import org.apache.texera.amber.core.workflow.WorkflowGraphIDs +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.core.virtualidentity.OperatorIdentity +@JsonIgnoreProperties(Array("expandedMacros", "drafts")) case class LogicalPlanPojo( operators: List[LogicalOp], links: List[LogicalLink], opsToViewResult: List[String], - opsToReuseResult: List[String] -) + opsToReuseResult: List[String], + @JsonInclude(JsonInclude.Include.NON_ABSENT) macroContent: Option[JsonNode] = None, + @JsonInclude(JsonInclude.Include.NON_ABSENT) executeOperatorIds: Option[List[String]] = None, + // Preserve diagnostic paths through normalization and sub-DAG selection; never persist them. + expandedMacros: Option[MacroExpansion] = None, + drafts: List[MacroDraft] = Nil +) { + @JsonIgnore lazy val macroExpansion: Option[MacroExpansion] = + expandedMacros.orElse(macroContent.map(MacroExpansion.expand(_, drafts))) + + /** The first public output is the result of a selected macro; keep its native port identity. */ + @JsonIgnore lazy val macroResultPorts: Map[String, (OperatorIdentity, PortIdentity)] = + macroExpansion.toList + .flatMap(_.resultTargets) + .collect { + case (id, target) if opsToViewResult.contains(id) => + id -> (OperatorIdentity(target.operatorID), PortIdentity(target.portIndex)) + } + .toMap + + @JsonIgnore lazy val normalized: LogicalPlanPojo = { + WorkflowGraphIDs.validateIDs(operators.map(_.operatorIdentifier.id)) + macroContent + .flatMap(_ => macroExpansion) + .map(MacroExpansion.logicalPlan(this, _)) + .getOrElse(this) + } +} diff --git a/common/workflow-compiler/src/test/resources/macros/nested-expansion.json b/common/workflow-compiler/src/test/resources/macros/nested-expansion.json new file mode 100644 index 00000000000..711efa53129 --- /dev/null +++ b/common/workflow-compiler/src/test/resources/macros/nested-expansion.json @@ -0,0 +1,421 @@ +{ + "name": "legacy nesting, repeated IDs, Unicode overrides", + "content": { + "operators": [ + { + "operatorID": "one", + "operatorType": "WorkflowMacro:outer:1", + "operatorVersion": "1", + "operatorProperties": { + "value": "café 東京" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + }, + { + "operatorID": "two", + "operatorType": "WorkflowMacro:outer:1", + "operatorVersion": "1", + "operatorProperties": { + "value": "other" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + } + ], + "links": [], + "formBinding": { + "fields": [ + { + "id": "value", + "operatorID": "leaf", + "displayName": "Value", + "propertyKey": "value" + } + ] + }, + "operatorPositions": { + "leaf": { + "x": 0, + "y": 0 + } + }, + "commentBoxes": [], + "settings": { + "dataTransferBatchSize": 10 + }, + "macroDependencies": [ + { + "name": "Outer", + "id": "outer", + "version": 1, + "content": { + "operators": [ + { + "operatorID": "child", + "operatorType": "WorkflowMacro:middle:1", + "operatorVersion": "1", + "operatorProperties": { + "value": "outer" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + }, + { + "operatorID": "outer__inputs", + "operatorType": "MacroInput", + "operatorVersion": "1", + "operatorProperties": {}, + "inputPorts": [], + "outputPorts": [], + "dynamicInputPorts": false, + "dynamicOutputPorts": true, + "showAdvanced": false + }, + { + "operatorID": "outer__outputs", + "operatorType": "MacroOutput", + "operatorVersion": "1", + "operatorProperties": {}, + "inputPorts": [ + { + "portID": "output-0" + } + ], + "outputPorts": [], + "dynamicInputPorts": true, + "dynamicOutputPorts": false, + "showAdvanced": false + } + ], + "links": [ + { + "linkID": "outer__1_output-0_0", + "source": { + "operatorID": "child", + "portID": "output-0" + }, + "target": { + "operatorID": "outer__outputs", + "portID": "output-0" + } + } + ], + "macroInterface": { + "id": "outer" + }, + "formBinding": { + "fields": [ + { + "id": "value", + "operatorID": "child", + "displayName": "Value", + "propertyKey": "value" + } + ] + }, + "operatorPositions": { + "child": { + "x": 0, + "y": 0 + }, + "outer__inputs": { + "x": -100, + "y": 100 + }, + "outer__outputs": { + "x": 600, + "y": 100 + } + }, + "commentBoxes": [], + "settings": { + "dataTransferBatchSize": 10 + }, + "macroDependencies": [ + { + "name": "Middle", + "id": "middle", + "version": 1, + "content": { + "operators": [ + { + "operatorID": "child", + "operatorType": "WorkflowMacro:inner:1", + "operatorVersion": "1", + "operatorProperties": { + "value": "middle" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + }, + { + "operatorID": "middle__inputs", + "operatorType": "MacroInput", + "operatorVersion": "1", + "operatorProperties": {}, + "inputPorts": [], + "outputPorts": [], + "dynamicInputPorts": false, + "dynamicOutputPorts": true, + "showAdvanced": false + }, + { + "operatorID": "middle__outputs", + "operatorType": "MacroOutput", + "operatorVersion": "1", + "operatorProperties": {}, + "inputPorts": [ + { + "portID": "output-0" + } + ], + "outputPorts": [], + "dynamicInputPorts": true, + "dynamicOutputPorts": false, + "showAdvanced": false + } + ], + "links": [ + { + "linkID": "middle__1_output-0_0", + "source": { + "operatorID": "child", + "portID": "output-0" + }, + "target": { + "operatorID": "middle__outputs", + "portID": "output-0" + } + } + ], + "macroInterface": { + "id": "middle" + }, + "formBinding": { + "fields": [ + { + "id": "value", + "operatorID": "child", + "displayName": "Value", + "propertyKey": "value" + } + ] + }, + "operatorPositions": { + "child": { + "x": 0, + "y": 0 + }, + "middle__inputs": { + "x": -100, + "y": 100 + }, + "middle__outputs": { + "x": 600, + "y": 100 + } + }, + "commentBoxes": [], + "settings": { + "dataTransferBatchSize": 10 + }, + "macroDependencies": [ + { + "name": "Inner", + "id": "inner", + "version": 1, + "content": { + "operators": [ + { + "operatorID": "leaf", + "operatorType": "TestOp", + "operatorVersion": "1", + "operatorProperties": { + "value": "original" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + }, + { + "operatorID": "inner__inputs", + "operatorType": "MacroInput", + "operatorVersion": "1", + "operatorProperties": {}, + "inputPorts": [], + "outputPorts": [], + "dynamicInputPorts": false, + "dynamicOutputPorts": true, + "showAdvanced": false + }, + { + "operatorID": "inner__outputs", + "operatorType": "MacroOutput", + "operatorVersion": "1", + "operatorProperties": {}, + "inputPorts": [ + { + "portID": "output-0" + } + ], + "outputPorts": [], + "dynamicInputPorts": true, + "dynamicOutputPorts": false, + "showAdvanced": false + } + ], + "links": [ + { + "linkID": "inner__1_output-0_0", + "source": { + "operatorID": "leaf", + "portID": "output-0" + }, + "target": { + "operatorID": "inner__outputs", + "portID": "output-0" + } + } + ], + "macroInterface": { + "id": "inner" + }, + "formBinding": { + "fields": [ + { + "id": "value", + "operatorID": "leaf", + "displayName": "Value", + "propertyKey": "value" + } + ] + }, + "operatorPositions": { + "inner__inputs": { + "x": -100, + "y": 100 + }, + "inner__outputs": { + "x": 600, + "y": 100 + }, + "leaf": { + "x": 0, + "y": 0 + } + }, + "commentBoxes": [], + "settings": { + "dataTransferBatchSize": 10 + } + } + } + ] + } + } + ] + } + } + ] + }, + "expected": { + "content": { + "operators": [ + { + "operatorID": "2f4a4c9e-b586-5c8b-b7d1-a71a203acf22", + "operatorType": "TestOp", + "operatorVersion": "1", + "operatorProperties": { + "value": "café 東京" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + }, + { + "operatorID": "667fa1e1-9a78-550b-9c3a-23839baa105e", + "operatorType": "TestOp", + "operatorVersion": "1", + "operatorProperties": { + "value": "other" + }, + "inputPorts": [], + "outputPorts": [ + { + "portID": "output-0" + } + ], + "showAdvanced": false + } + ], + "links": [], + "formBinding": { + "fields": [ + { + "id": "value", + "operatorID": "leaf", + "displayName": "Value", + "propertyKey": "value" + } + ] + }, + "operatorPositions": { + "leaf": { + "x": 0, + "y": 0 + } + }, + "commentBoxes": [], + "settings": { + "dataTransferBatchSize": 10 + } + }, + "paths": { + "2f4a4c9e-b586-5c8b-b7d1-a71a203acf22": ["one", "child", "child", "leaf"], + "667fa1e1-9a78-550b-9c3a-23839baa105e": ["two", "child", "child", "leaf"] + }, + "ports": { + "[\"one\",\"outputs\",\"output-0\"]": [ + { + "operatorID": "2f4a4c9e-b586-5c8b-b7d1-a71a203acf22", + "portID": "output-0" + } + ], + "[\"two\",\"outputs\",\"output-0\"]": [ + { + "operatorID": "667fa1e1-9a78-550b-9c3a-23839baa105e", + "portID": "output-0" + } + ] + } + } +} diff --git a/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroExpansionCases.scala b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroExpansionCases.scala new file mode 100644 index 00000000000..9d4f51f08ef --- /dev/null +++ b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroExpansionCases.scala @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.common.compiler + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.texera.common.compiler.MacroTestUtils._ +import org.apache.texera.amber.util.JSONUtils.objectMapper + +object MacroExpansionCases { + private val output = "output-0" + private val legacy = objectAt( + objectMapper.readTree(getClass.getResource("/macros/nested-expansion.json")) + ) + private val content = objectAt(legacy, "/content") + private def unpack(body: ObjectNode): List[ObjectNode] = { + val definition = objectAt(body, "/macroDependencies/0").deepCopy[ObjectNode]() + val child = objectAt(definition, "/content") + val nested = if (child.has("macroDependencies")) unpack(child) else Nil + child.remove("macroDependencies") + definition :: nested + } + private val pins = unpack(content) + private val List(outer, _, inner) = pins + private val one = objectAt(content, "/operators/0") + private val two = objectAt(content, "/operators/1") + private val unicode = one.at("/operatorProperties/value").asText() + private val nested = changed(content, "macroDependencies" -> pins) + private val expected = objectAt(legacy, "/expected") + + private def snapshot( + content: ObjectNode, + paths: Map[String, List[String]], + ports: (String, List[ObjectNode])* + ): ObjectNode = + json("content" -> content, "paths" -> paths, "ports" -> json(ports: _*)) + + private def nestedSnapshot( + value: Any, + includeTwo: Boolean = true, + includeLeaves: Boolean = true + ): ObjectNode = { + val result = expected.deepCopy[ObjectNode]() + val operators = arrayAt(result, "/content/operators") + objectAt(operators.get(0), "/operatorProperties") + .set[JsonNode]("value", objectMapper.valueToTree(value)) + for (i <- (operators.size() - 1 to 0 by -1) if !includeLeaves || (!includeTwo && i == 1)) { + val id = operators.remove(i).path("operatorID").asText() + val instance = objectAt(result, "/paths").remove(id).get(0).asText() + val key = s"""["$instance","outputs","$output"]""" + if (includeLeaves) objectAt(result, "/ports").remove(key) + else arrayAt(result, s"/ports/$key").removeAll() + } + result + } + + private def filter(id: String): ObjectNode = operator(id, "Filter", List("in"), List("out")) + private val outputs = List("a", "b") + private def fanoutBody(id: String): ObjectNode = { + val inputs = outputs.zipWithIndex.map { + case (leaf, i) => + link(s"${id}__0_in_$i", end(s"${id}__inputs", "in"), end(leaf, "in")) + } + val outgoing = outputs.map(leaf => + link(s"${id}__1_${leaf}_0", end(leaf, "out"), end(s"${id}__outputs", leaf)) + ) + macroGraph(id, outputs.map(filter), inputs ++ outgoing, List("in"), outputs) + } + + private def fanoutInstance( + id: String, + macroID: String, + name: String, + version: Int = 1 + ): ObjectNode = + instance(id, macroID, List("in"), outputs, version).put("customDisplayName", name) + + private val incoming = link("incoming", end("source", "out"), end("m", "in")) + private val outgoing = link("outgoing", end("m", "b"), end("sink", "in")) + private val fanout = pinned( + graph( + List(filter("source"), fanoutInstance("m", "fanout", "Fanout"), filter("sink")), + List(incoming, outgoing) + ), + definition("fanout", "Fanout", fanoutBody("fanout")) + ) + private val fanoutA = "da88d5c7-b748-5512-9a56-20a153a7d515" + private val fanoutB = "b43e157f-d0f5-5383-9584-b62c40558563" + private def fanoutPorts(id: String, a: String, b: String): List[(String, List[ObjectNode])] = + List( + s"""["$id","inputs","in"]""" -> List(end(a, "in"), end(b, "in")), + s"""["$id","outputs","a"]""" -> List(end(a, "out")), + s"""["$id","outputs","b"]""" -> List(end(b, "out")) + ) + private val fanoutExpected = snapshot( + graph( + List("source", fanoutA, fanoutB, "sink").map(filter), + List( + link("653afb35-c8c5-530c-8cf4-07ab881c6ab5", end(fanoutB, "out"), end("sink", "in")), + link("74334672-93bf-5467-aebe-0953ab46813b", end("source", "out"), end(fanoutA, "in")), + link("17318951-ffc5-591e-85f8-8f8772ac33df", end("source", "out"), end(fanoutB, "in")) + ) + ), + Map( + "source" -> List("source"), + fanoutA -> List("m", "a"), + fanoutB -> List("m", "b"), + "sink" -> List("sink") + ), + fanoutPorts("m", fanoutA, fanoutB): _* + ) + + private val oldSelf = forwardingBody( + "self", + fanoutInstance("old-self", "self", "Self"), + List("in" -> "in"), + outputs.map(port => port -> port) + ) + private val self = pinned( + graph(List(fanoutInstance("self", "self", "Self", version = 4))), + definition("self", "Self", fanoutBody("self")), + definition("self", "Self", oldSelf, version = 4) + ) + private val selfA = "dd8f47a9-6940-55f2-859f-b840f0c18dad" + private val selfB = "be1a66b4-093c-5a81-9f6b-14059540081b" + private val selfExpected = snapshot( + graph(List(filter(selfA), filter(selfB))), + Map(selfA -> List("self", "old-self", "a"), selfB -> List("self", "old-self", "b")), + fanoutPorts("self", selfA, selfB): _* + ) + + private def positive(name: String, content: ObjectNode, expected: ObjectNode): ObjectNode = + json("name" -> name, "content" -> content, "expected" -> expected) + private def negative(name: String, content: ObjectNode, error: String): ObjectNode = + json("name" -> name, "content" -> content, "error" -> error) + private def editOne(fields: (String, Any)*): ObjectNode = + changed(nested, "operators" -> List(changed(one, fields: _*), two)) + private def editPin(pin: ObjectNode)(edit: ObjectNode => Unit): ObjectNode = { + val copy = pin.deepCopy[ObjectNode]() + edit(objectAt(copy, "/content")) + changed(nested, "macroDependencies" -> pins.map(p => if (p eq pin) copy else p)) + } + + val cases: List[ObjectNode] = List( + legacy, + positive("flat bundle", nested, expected), + positive("portable bundle", nested, expected), + positive( + "disabled instance", + changed(nested, "operators" -> List(one, changed(two, "isDisabled" -> true))), + nestedSnapshot(unicode, includeTwo = false) + ), + positive( + "disabled leaf", + editPin(inner)(body => objectAt(body, "/operators/0").put("isDisabled", true)), + nestedSnapshot(unicode, includeLeaves = false) + ), + positive( + "null override", + editOne("operatorProperties" -> json("value" -> null)), + nestedSnapshot(null) + ), + positive( + "empty override", + editOne("operatorProperties" -> json("value" -> "")), + nestedSnapshot("") + ), + negative( + "missing definition", + changed(nested, "macroDependencies" -> Nil), + "Missing macro version: WorkflowMacro:outer:1" + ), + negative( + "pin mismatch", + editOne("operatorVersion" -> "2"), + "Macro version does not match its pinned definition" + ), + negative( + "unknown property", + editOne("operatorProperties" -> json("value" -> unicode, "unknown" -> 1)), + "Unknown macro property: unknown" + ), + negative( + "duplicate operator ID", + changed(nested, "operators" -> List(one, two, one)), + "Duplicate graph element ID 'one'" + ), + negative( + "duplicate port ID", { + val content = nested.deepCopy[ObjectNode]() + val operators = List(content.at("/operators/0"), content.at("/operators/1")) ++ + pins.indices.map(index => content.at(s"/macroDependencies/$index/content/operators/0")) + operators.foreach(op => arrayAt(op, "/outputPorts").add(json("portID" -> output))) + content + }, + "Duplicate or invalid operator port" + ), + negative( + "interface mismatch", + editOne("inputPorts" -> List(json("portID" -> "unexpected"))), + "Macro instance interface differs from its pinned version" + ), + negative( + "missing endpoint", + changed( + nested, + "links" -> List(link("missing", end("missing", "out"), end("one", "input-0"))) + ), + "Missing operator or port in macro graph" + ), + negative( + "conflicting definition", + changed( + nested, + "macroDependencies" -> (pins :+ changed( + outer, + "content" -> + changed( + outer.path("content"), + "links" -> List(json("linkID" -> "changed"), outer.at("/content/links/0")) + ) + )) + ), + "Conflicting macro version" + ), + negative( + "version cycle", + editPin(outer)(body => + objectAt(body, "/operators/0").put("operatorType", "WorkflowMacro:outer:1") + ), + "Recursive macro reference" + ), + negative( + "unconnected boundary", + editPin(inner)(_.putArray("links")), + "Connect or remove macro output 'output-0' before running." + ), + negative( + "missing boundary target", + editPin(inner)(body => objectAt(body, "/links/0/source").put("portID", "missing")), + "Missing operator or port in macro graph" + ), + positive("input fanout and multiple outputs", fanout, fanoutExpected), + negative( + "duplicate link ID", + changed(fanout, "links" -> List(incoming, outgoing, incoming)), + "Duplicate graph element ID 'incoming'" + ), + negative( + "expanded graph cycle", + graph( + List(filter("a"), filter("b")), + List( + link("ab", end("a", "out"), end("b", "in")), + link("ba", end("b", "out"), end("a", "in")) + ) + ), + "Workflow contains a cycle" + ), + positive("finite older self revision", self, selfExpected) + ) +} diff --git a/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroExpansionSpec.scala b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroExpansionSpec.scala new file mode 100644 index 00000000000..55d2bb710e9 --- /dev/null +++ b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroExpansionSpec.scala @@ -0,0 +1,473 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.common.compiler + +import org.apache.texera.amber.operator.TestOperators +import org.apache.texera.amber.util.JSONUtils.objectMapper +import com.fasterxml.jackson.databind.JsonNode +import org.apache.texera.common.compiler.MacroTestUtils._ +import com.fasterxml.jackson.databind.node.ObjectNode +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ +import org.apache.texera.common.compiler.model.{LogicalPlan, LogicalPlanPojo} +import org.apache.texera.amber.core.virtualidentity.WorkflowIdentity +import org.apache.texera.amber.core.workflow.WorkflowContext + +class MacroExpansionSpec extends AnyFlatSpec with Matchers { + private val cases = MacroExpansionCases.cases + + cases.foreach { example => + "Backend macro expansion" should example.get("name").asText() in { + val content = example.get("content") + val original = content.deepCopy[JsonNode]() + if (example.has("error")) { + val error = intercept[IllegalArgumentException](MacroExpansion.expand(content)) + error.getMessage should include(example.get("error").asText()) + } else { + val expanded = MacroExpansion.expand(content) + expanded.content shouldBe example.at("/expected/content") + objectMapper.valueToTree[JsonNode](expanded.paths) shouldBe + example.at("/expected/paths") + example.at("/expected/ports").fields().asScala.foreach { port => + objectMapper.valueToTree[JsonNode]( + expanded.ports(port.getKey) + ) shouldBe port.getValue + } + } + content shouldBe original + } + } + + private def pinnedSource: ObjectNode = { + val body = + forwardingBody("source", textSource("leaf", "original"), Nil, List("out" -> "output-0")) + body.set[JsonNode]("formBinding", formBinding(field("text", "leaf", "textInput"))) + val usage = instance("macro", "source", outputs = List("out")) + objectAt(usage, "/operatorProperties").put("text", "changed") + pinned(graph(List(usage)), definition("source", "Source", body)) + } + + private def firstBody(content: JsonNode): ObjectNode = + objectAt(content, "/macroDependencies/0/content") + + private def expansionError(content: JsonNode, drafts: List[MacroDraft] = Nil): String = { + val before = content.deepCopy[JsonNode]() + val error = intercept[IllegalArgumentException](MacroExpansion.expand(content, drafts)) + content shouldBe before + error.getMessage + } + + it should "apply a nested preview only at its occurrence, even at the largest saved revision" in { + val version = 9007199254740991L + val leaf = instance("child", "source", outputs = List("out"), version = version) + val parent = macroGraph("parent", List(leaf)) + val original = firstBody(pinnedSource) + val root = pinned( + graph(List(instance("one", "parent"), instance("two", "parent"))), + definition("parent", "Parent", parent), + definition("source", "Source", original).put("version", version) + ) + val before = root.deepCopy[JsonNode]() + val edited = original.deepCopy[ObjectNode]() + objectAt(edited, "/operators/0/operatorProperties").put("textInput", "draft") + val path = List("one", "child") + val preview = MacroDraft(path, edited) + val expanded = MacroExpansion.expand(root, List(preview)) + val operators = expanded.content.path("operators").elements().asScala.toList + operators.map(_.at("/operatorProperties/textInput").asText()) shouldBe List("draft", "original") + operators.map(op => expanded.paths(op.path("operatorID").asText()).head) shouldBe List( + "one", + "two" + ) + val self = macroGraph( + "source", + List(changed(leaf, "operatorID" -> "again")), + outputs = List("out"), + links = List(link("self-out", end("again", "out"), end("source__outputs", "out"))) + ) + val nested = MacroExpansion.expand(root, List(preview.copy(content = self))) + nested.paths.values should contain(path ++ List("again", "leaf")) + root shouldBe before + for ( + (drafts, error) <- List( + List(preview, preview) -> "Duplicate macro draft", + List(preview.copy(path = List("missing"))) -> "Unknown macro draft", + List(preview.copy(content = parent)) -> "Macro draft identity" + ) + ) + expansionError(root, drafts) should include(error) + } + + private def boundary(body: JsonNode, input: Boolean): ObjectNode = + body + .path("operators") + .elements() + .asScala + .find(_.path("operatorType").asText() == (if (input) "MacroInput" else "MacroOutput")) + .get + .asInstanceOf[ObjectNode] + + List("null", "content", "version", "name", "macroInterface").foreach { malformed => + "External macro definitions" should s"reject malformed $malformed without changing the submitted snapshot" in { + val root = pinnedSource + val definition = objectAt(root, "/macroDependencies/0") + malformed match { + case "null" => + arrayAt(root, "/macroDependencies") + .set(0, objectMapper.nullNode()) + case "content" => definition.putNull("content") + case "version" => definition.put("version", 0) + case "name" => definition.put("name", 42) + case "macroInterface" => + objectAt(definition, "/content") + .putNull("macroInterface") + } + expansionError(root) should include("Malformed") + } + } + + "Native and empty graphs" should "keep their complete content and deterministic native identities" in { + for (ids <- List(Nil, List("native"))) { + val content = graph( + ids.map(id => + json( + "operatorID" -> id, + "operatorType" -> "Filter", + "operatorProperties" -> json(), + "inputPorts" -> Nil, + "outputPorts" -> Nil + ) + ) + ) + val expanded = MacroExpansion.expand(content) + expanded.content shouldBe content + expanded.paths shouldBe ids.map(id => id -> List(id)).toMap + } + } + + "Exposed optional fields" should "accept an override without a stored native value" in { + val root = pinnedSource + objectAt(root, "/macroDependencies/0/content/operators/0") + .putObject("operatorProperties") + MacroExpansion + .expand(root) + .content + .at("/operators/0/operatorProperties/textInput") + .asText() shouldBe "changed" + } + + List("inputs" -> "input", "outputs" -> "output").foreach { + case (direction, label) => + "Macro boundaries" should s"identify an unfinished $label without changing its snapshot" in { + val content = pinnedSource + boundary(firstBody(content), input = direction == "inputs") + .withArray(if (direction == "inputs") "outputPorts" else "inputPorts") + .add(objectMapper.readTree("""{"portID":"pending","displayName":"Rows"}""")) + expansionError(content) shouldBe + s"Source: Connect or remove macro $label 'Rows' before running." + } + } + + "Macro boundaries" should "reject multiple targets on one output with an actionable error" in { + val content = pinnedSource + val edges = arrayAt(content, "/macroDependencies/0/content/links") + edges.add(changed(edges.get(0), "linkID" -> "duplicate-output")) + expansionError(content) shouldBe + "Source: Macro output 'out' must connect to exactly one operator output." + } + + "Macro logical plans" should "deserialize pinned graphs and normalize result and reuse selections" in { + val pojo = LogicalPlanPojo(Nil, Nil, List("macro"), List("macro"), Some(pinnedSource)) + val decoded = + objectMapper.readValue(objectMapper.writeValueAsString(pojo), classOf[LogicalPlanPojo]) + val normalized = decoded.normalized + val leaf = normalized.operators.head + leaf.operatorIdentifier.id shouldBe decoded.macroExpansion.get.paths.keys.head + normalized.opsToViewResult shouldBe List(leaf.operatorIdentifier.id) + normalized.opsToReuseResult shouldBe normalized.opsToViewResult + normalized.macroContent shouldBe None + LogicalPlan(decoded).operators.size shouldBe 1 + } + + it should "compile macros through the ordinary compiler and leave the snapshot untouched" in { + val content = pinnedSource + val original = content.deepCopy[JsonNode]() + val pojo = macroPlan(content) + val result = new WorkflowCompiler( + new WorkflowContext(WorkflowIdentity(0)), + TestOperators.fixtureFileResolver + ) + .compile(pojo, CompilationErrorHandling.Strict) + result.physicalPlan should not be empty + result.logicalPlan.operators.size shouldBe 1 + content shouldBe original + } + + it should "filter selected canvas nodes after validating the whole graph" in { + val pojo = LogicalPlanPojo(Nil, Nil, List("macro"), Nil, Some(pinnedSource), Some(Nil)) + pojo.normalized.operators shouldBe empty + pojo.normalized.opsToViewResult shouldBe empty + val invalid = pinnedSource + objectAt(invalid) + .putArray("macroDependencies") + intercept[IllegalArgumentException]( + pojo.copy(macroContent = Some(invalid)).normalized + ).getMessage should include("Missing macro version") + } + + it should "report malformed macros as compilation errors in lenient mode and reject them in strict mode" in { + val invalid = pinnedSource + objectAt(invalid) + .putArray("macroDependencies") + arrayAt(invalid, "/operators") + .insert( + 0, + pinnedSource.at("/macroDependencies/0/content/operators/0") + ) + val pojo = macroPlan(invalid) + val compiler = new WorkflowCompiler( + new WorkflowContext(WorkflowIdentity(0)), + TestOperators.fixtureFileResolver + ) + val result = compiler.compile(pojo) + result.physicalPlan shouldBe None + result.operatorIdToError.keys.map(_.id).toList shouldBe List("macro") + result.operatorIdToError.values.head.message should include("Missing macro version") + intercept[IllegalArgumentException](compiler.compile(pojo, CompilationErrorHandling.Strict)) + } + + it should "preserve the declared output order when selecting results" in { + val content = pinnedSource + val body = firstBody(content) + val second = changed(body.at("/operators/0"), "operatorID" -> "second") + arrayAt(body, "/operators").add(second) + val terminal = boundary(body, input = false) + terminal.withArray("inputPorts").add(objectMapper.readTree("""{"portID":"second"}""")) + arrayAt(body, "/links") + .add( + link( + "second-output", + end("second", "output-0"), + end(terminal.path("operatorID").asText(), "second") + ) + ) + arrayAt(content, "/operators/0/outputPorts") + .insert(0, objectMapper.readTree("""{"portID":"second"}""")) + val pojo = macroPlan(content, List("macro")) + val ids = pojo.macroExpansion.get.paths.map { case (id, path) => path.last -> id } + pojo.normalized.opsToViewResult shouldBe List(ids("second"), ids("leaf")) + val ports = objectMapper.valueToTree[JsonNode](pojo.macroExpansion.get).at("/resultPorts/macro") + ports.get(0).path("operatorID").asText() shouldBe ids("second") + ports.get(1).path("operatorID").asText() shouldBe ids("leaf") + ports.get(1).path("portIndex").asInt(-1) shouldBe 0 + } + + it should "reject numeric pins and malformed bundles instead of accepting coerced values" in { + val content = pinnedSource + objectAt(content, "/operators/0") + .put("operatorVersion", 1) + intercept[IllegalArgumentException](MacroExpansion.expand(content)) + val invalid = objectMapper.readTree("""{"operators":[],"links":[],"macroDependencies":{}}""") + intercept[IllegalArgumentException](MacroExpansion.expand(invalid)) + } + + "Nested macro expansion" should "return resolved boundaries at every depth for the live inspector" in { + val example = cases.head + val expansion = MacroExpansion.expand(example.path("content")) + val inspectors = objectMapper.valueToTree[JsonNode](expansion).path("inspectors") + for ((runtime, path) <- expansion.paths if path.length > 1) { + val scope = objectMapper.writeValueAsString(path.init) + inspectors.path(scope).path("ids").path(path.last).asText() shouldBe runtime + } + expansion.ports.size should be > example.at("/expected/ports").size() + expansion.ports.values.flatten.foreach { port => + expansion.paths should contain key port.path("operatorID").asText() + expansion.content + .path("operators") + .elements() + .asScala + .find(_.path("operatorID") == port.path("operatorID")) should not be empty + } + } + + it should "keep boundary links and native source ordinals on the correct occurrence" in { + val example = cases.find(_.path("name").asText() == "input fanout and multiple outputs").get + val root = example.path("content").deepCopy[ObjectNode]() + val source = + root.path("operators").elements().asScala.find(_.path("operatorID").asText() == "source").get + arrayAt(source, "/outputPorts").insert(0, json("portID" -> "unused")) + val expansion = MacroExpansion.expand(root) + val view = expansion.inspectors("""["m"]""") + val links = expansion.content.path("links") + List("a", "b").foreach { local => + view.inputs(local) should have size 1 + val input = view.inputs(local).head + input.input shouldBe 0 + input.output shouldBe 1 + links.get(input.link).at("/target/operatorID").asText() shouldBe view.ids(local) + } + val exposed = view.terminals.map(p => p.portID -> p.links.map(i => links.get(i))).toMap + exposed("in").map(_.at("/source/operatorID").asText()) shouldBe List("source", "source") + exposed("a") shouldBe empty + exposed("b").map(_.at("/target/operatorID").asText()) shouldBe List("sink") + } + + it should "distinguish two exposed inputs wired to the same native port" in { + val ports = List("left", "right") + val body = + forwardingBody("join", operator("leaf", "Filter", List("in")), ports.map(_ -> "in"), Nil) + val root = pinned( + graph( + ports.map(id => textSource(id, id)) :+ instance("m", "join", ports), + ports.map(id => link(s"wire-$id", end(id, "output-0"), end("m", id))) + ), + definition("join", "Join", body) + ) + val expanded = MacroExpansion.expand(root) + val links = expanded.content.path("links") + val inputs = expanded.inspectors("[]").inputs("m") + inputs.map(i => + ports(i.input) -> links.get(i.link).at("/source/operatorID").asText() + ) shouldBe ports.zip(ports) + expanded.inspectors("""["m"]""").terminals.foreach { port => + port.links.map(i => links.get(i).at("/source/operatorID").asText()) shouldBe List(port.portID) + } + } + + it should "scale the occurrence index linearly for repeated sibling definitions" in { + def index(width: Int) = { + val root = cases.head.path("content") + val sibling = root.at("/operators/0") + val usages = (0 until width).map(i => changed(sibling, "operatorID" -> s"copy-$i")) + MacroExpansion.expand(changed(root, "operators" -> usages)).inspectors + } + val one = index(1) + val many = index(32) + many.size - 1 shouldBe 32 * (one.size - 1) + many.values.map(_.ids.size).sum shouldBe 32 * one.values.map(_.ids.size).sum + } + + it should "allow 32 pinned levels and reject the next level without changing the snapshots" in { + def nested(depth: Int): JsonNode = { + val root = pinnedSource + val definitions = root.withArray("macroDependencies") + var child = instance("macro", "source", outputs = List("out")) + for (level <- 1 until depth) { + val id = s"level-$level" + val body = macroGraph(id, List(child)) + definitions.add(definition(id, id, body)) + child = instance(id, id) + } + root.withArray("operators").removeAll().add(child) + root + } + MacroExpansion.expand(nested(32)).content.path("operators").size() shouldBe 1 + expansionError(nested(33)) should include("32 levels") + } + private def terminalBody: ObjectNode = firstBody(passthroughWorkflow) + + "Saved macro terminals" should "lower passthrough links without adding runtime operators or changing pins" in { + val content = passthroughWorkflow + val before = content.toString + val expanded = MacroExpansion.expand(content) + expanded.paths.keySet shouldBe Set("source", "sink") + expanded.content.path("operators").size() shouldBe 2 + expanded.content.path("links").size() shouldBe 1 + expanded.content + .at("/links/0/source/operatorID") + .asText() shouldBe "source" + expanded.content.at("/links/0/target/operatorID").asText() shouldBe "sink" + expanded + .ports("[\"macro\",\"inputs\",\"rows\"]") + .head + .path("operatorID") + .asText() shouldBe "sink" + expanded + .ports("[\"macro\",\"outputs\",\"out\"]") + .head + .path("operatorID") + .asText() shouldBe "source" + val pojo = macroPlan(content, List("macro")) + pojo.normalized.operators.size shouldBe 2 + pojo.normalized.opsToViewResult shouldBe List("source") + new WorkflowCompiler( + new WorkflowContext(WorkflowIdentity(0)), + TestOperators.fixtureFileResolver + ) + .compile(pojo, CompilationErrorHandling.Strict) + .physicalPlan should not be empty + content.toString shouldBe before + } + + it should "deduplicate repeated input routes independently of endpoint JSON key order" in { + val example = cases.find(_.path("name").asText() == "input fanout and multiple outputs").get + val content = example.path("content").deepCopy[JsonNode]() + val body = firstBody(content) + val input = boundary(body, input = true) + val links = arrayAt(body, "/links") + val original = + links.elements().asScala.find(_.at("/source/operatorID") == input.path("operatorID")).get + val duplicate = original.deepCopy[ObjectNode]().put("linkID", "again") + for (direction <- List("source", "target")) + duplicate.set[JsonNode]( + direction, + json( + "portID" -> original.at(s"/$direction/portID"), + "operatorID" -> original.at(s"/$direction/operatorID") + ) + ) + links.add(duplicate) + val before = content.toString + val result = MacroExpansion.expand(content) + result.content shouldBe example.at("/expected/content") + content.toString shouldBe before + } + + it should "reject missing, duplicate, disabled and out-of-context terminals with clear errors" in { + val disconnected = passthroughWorkflow + arrayAt(firstBody(disconnected), "/links").removeAll() + expansionError(disconnected) should include("Connect or remove") + val duplicate = terminalBody + arrayAt(duplicate, "/operators").add( + changed(duplicate.at("/operators/0"), "operatorID" -> "another-input") + ) + expansionError(duplicate) should include("exactly one") + val disabled = terminalBody + objectAt(disabled, "/operators/0").put("isDisabled", true) + expansionError(disabled) should include("cannot be disabled") + val outside = terminalBody + outside.remove("macroInterface") + expansionError(outside) should include("macro definition") + } + + it should "detect cycles made entirely of compile-time boundaries" in { + val root = passthroughWorkflow + val first = root.at("/operators/1") + val second = changed(first, "operatorID" -> "second") + root.putArray("operators").add(first).add(second) + root + .putArray("links") + .add(link("forward", end("macro", "out"), end("second", "rows"))) + .add(link("back", end("second", "out"), end("macro", "rows"))) + expansionError(root).toLowerCase should include("cycle") + } + +} diff --git a/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroTestUtils.scala b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroTestUtils.scala new file mode 100644 index 00000000000..69d14fe7887 --- /dev/null +++ b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroTestUtils.scala @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.common.compiler + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.common.compiler.model.LogicalPlanPojo + +/** Typed access to mutable fixture nodes; pointers stay explicit at each mutation. */ +object MacroTestUtils { + def objectAt(node: JsonNode, pointer: String = ""): ObjectNode = + node.at(pointer).asInstanceOf[ObjectNode] + + def arrayAt(node: JsonNode, pointer: String): ArrayNode = + node.at(pointer).asInstanceOf[ArrayNode] + + def json(fields: (String, Any)*): ObjectNode = { + val node = objectMapper.createObjectNode() + fields.foreach { case (key, value) => node.set[JsonNode](key, objectMapper.valueToTree(value)) } + node + } + + def changed(node: JsonNode, fields: (String, Any)*): ObjectNode = + node.deepCopy[ObjectNode]().setAll[ObjectNode](json(fields: _*)) + + def operator( + id: String, + kind: String, + inputs: List[String] = Nil, + outputs: List[String] = Nil, + properties: JsonNode = json(), + version: Long = 1 + ): ObjectNode = + json( + "operatorID" -> id, + "operatorType" -> kind, + "operatorVersion" -> version.toString, + "operatorProperties" -> properties, + "inputPorts" -> inputs.map(id => json("portID" -> id)), + "outputPorts" -> outputs.map(id => json("portID" -> id)), + "showAdvanced" -> false + ) + + def terminal(id: String, input: Boolean, ports: List[String]): ObjectNode = + changed( + if (input) operator(id, "MacroInput", outputs = ports) + else operator(id, "MacroOutput", inputs = ports), + "dynamicInputPorts" -> !input, + "dynamicOutputPorts" -> input + ) + + def textSource(id: String, text: String): ObjectNode = + operator( + id, + "TextInput", + outputs = List("output-0"), + properties = json("textInput" -> text, "attributeName" -> "line", "attributeType" -> "string") + ) + + def field(id: String, operatorID: String, property: String): ObjectNode = + json("id" -> id, "operatorID" -> operatorID, "propertyKey" -> property) + + def formBinding(fields: ObjectNode*): ObjectNode = json("fields" -> fields) + + def end(id: String, port: String): ObjectNode = json("operatorID" -> id, "portID" -> port) + + def link(id: String, source: ObjectNode, target: ObjectNode): ObjectNode = + json("linkID" -> id, "source" -> source, "target" -> target) + + def graph(operators: List[JsonNode], links: List[JsonNode] = Nil): ObjectNode = + json("operators" -> operators, "links" -> links) + + def macroGraph( + id: String, + operators: List[JsonNode], + links: List[JsonNode] = Nil, + inputs: List[String] = Nil, + outputs: List[String] = Nil + ): ObjectNode = { + val terminals = List( + terminal(s"${id}__inputs", input = true, inputs), + terminal(s"${id}__outputs", input = false, outputs) + ) + changed(graph(operators ++ terminals, links), "macroInterface" -> json("id" -> id)) + } + + def pinned(graph: ObjectNode, definitions: ObjectNode*): ObjectNode = + changed(graph, "macroDependencies" -> definitions) + + def macroPlan(content: JsonNode, results: List[String] = Nil): LogicalPlanPojo = + LogicalPlanPojo(Nil, Nil, results, Nil, Some(content)) + + def instance( + id: String, + macroID: String, + inputs: List[String] = Nil, + outputs: List[String] = Nil, + version: Long = 1 + ): ObjectNode = + operator(id, s"WorkflowMacro:$macroID:$version", inputs, outputs, version = version) + + /** Expose named ports of a single child through the saved terminal nodes. */ + def forwardingBody( + id: String, + child: ObjectNode, + inputs: List[(String, String)], + outputs: List[(String, String)] + ): ObjectNode = { + val childID = child.get("operatorID").asText() + val incoming = inputs.map { + case (publicPort, port) => + link(s"${id}__0_${publicPort}_0", end(s"${id}__inputs", publicPort), end(childID, port)) + } + val outgoing = outputs.map { + case (publicPort, port) => + link(s"${id}__1_${publicPort}_0", end(childID, port), end(s"${id}__outputs", publicPort)) + } + macroGraph(id, List(child), incoming ++ outgoing, inputs.map(_._1), outputs.map(_._1)) + } + + def passthroughWorkflow: ObjectNode = { + val inputs = List("rows") + val outputs = List("out") + val body = macroGraph( + "identity", + Nil, + List(link("passthrough", end("identity__inputs", "rows"), end("identity__outputs", "out"))), + inputs, + outputs + ) + objectAt(body, "/operators/0/outputPorts/0").put("displayName", "Rows 東京") + pinned( + graph( + List( + textSource("source", "3\n1"), + instance("macro", "identity", inputs, outputs), + operator("sink", "Limit", List("input-0"), List("output-0"), json("limit" -> 1)) + ), + List( + link("in", end("source", "output-0"), end("macro", "rows")), + link("out", end("macro", "out"), end("sink", "input-0")) + ) + ), + definition("identity", "Identity", body) + ) + } + + def definition(id: String, name: String, content: ObjectNode, version: Int = 1): ObjectNode = + json("id" -> id, "version" -> version, "name" -> name, "content" -> content) +} diff --git a/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroValidationSpec.scala b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroValidationSpec.scala new file mode 100644 index 00000000000..a642dfbfd5c --- /dev/null +++ b/common/workflow-compiler/src/test/scala/org/apache/texera/common/compiler/MacroValidationSpec.scala @@ -0,0 +1,442 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.texera.common.compiler + +import org.apache.texera.amber.operator.TestOperators +import com.fasterxml.jackson.databind.JsonNode +import org.apache.texera.common.compiler.MacroTestUtils._ +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.core.virtualidentity.{OperatorIdentity, WorkflowIdentity} +import org.apache.texera.amber.core.workflow.WorkflowContext +import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 +import org.apache.texera.common.compiler.model.LogicalPlanPojo +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import scala.jdk.CollectionConverters._ + +class MacroValidationSpec extends AnyFlatSpec with Matchers { + private val validKeys = """{"keys":[{"attribute":"line","sortPreference":"ASC"}]}""" + private def transform(id: String, kind: String, properties: String): ObjectNode = + operator(id, kind, List("input-0"), List("output-0"), objectMapper.readTree(properties)) + + private def workflow(properties: String = "{}", kind: String = "StableMergeSort"): ObjectNode = { + val out = List("out") + val inner = macroGraph( + "inner", + List(textSource("text", "b\na"), transform("sort", kind, properties)), + List( + link("text-sort", end("text", "output-0"), end("sort", "input-0")), + link("inner__1_out_0", end("sort", "output-0"), end("inner__outputs", "out")) + ), + outputs = out + ) + def named(id: String, macroID: String, name: String): ObjectNode = + instance(id, macroID, outputs = out).put("customDisplayName", name) + val outer = macroGraph( + "outer", + List(named("first", "inner", "Nested α"), named("second", "inner", "Nested β")), + List(link("outer__1_out_0", end("first", "out"), end("outer__outputs", "out"))), + outputs = out + ) + pinned( + graph(List(named("outer", "outer", "Reusable 東京"))), + definition("outer", "Outer", outer), + definition("inner", "Inner", inner) + ) + } + private def body(content: JsonNode) = objectAt(content, "/macroDependencies/1/content") + private def compile( + request: LogicalPlanPojo, + mode: CompilationErrorHandling = CompilationErrorHandling.Lenient + ): WorkflowCompilationResult = { + val snapshot = request.macroContent.map(_.toString) + try new WorkflowCompiler( + new WorkflowContext(WorkflowIdentity(0)), + TestOperators.fixtureFileResolver + ).compile(request, mode) + finally request.macroContent.map(_.toString) shouldBe snapshot + } + private def sortBinding = + formBinding(field("sort-keys", "sort", "keys").put("displayName", "Sort Keys")) + private val outerID = OperatorIdentity("outer") + + private def invalidCompilation(content: JsonNode): WorkflowCompilationResult = { + val result = compile(macroPlan(content)) + result.physicalPlan shouldBe None + result + } + + private def rootError(content: JsonNode): String = + invalidCompilation(content).operatorIdToError(outerID).message + + private def normalizationError(request: LogicalPlanPojo): String = + intercept[IllegalArgumentException](request.normalized).getMessage + + private def strict(request: LogicalPlanPojo) = + compile(request, CompilationErrorHandling.Strict) + + private def validCompilation(request: LogicalPlanPojo): WorkflowCompilationResult = { + val result = strict(request) + result.physicalPlan should not be empty + result.operatorIdToError shouldBe empty + result + } + + private def checkModes(request: LogicalPlanPojo, valid: Boolean): WorkflowCompilationResult = { + val editing = compile(request) + if (valid) { + editing.operatorIdToError shouldBe empty + validCompilation(request) + } else { + editing.physicalPlan shouldBe None + intercept[IllegalArgumentException](strict(request)) + } + editing + } + + private def checkPaths(message: String, leaf: String): Unit = + for (instance <- List("Nested α", "Nested β")) + message should include(s"Reusable 東京 / $instance / $leaf") + + for (single <- List(false, true); count <- List(1, 2)) { + "Nested macro input validation" should s"honor configured UDF multiplicity single=$single with $count links" in { + val content = workflow("""{"code":"from pytexera import *"}""", "PythonUDFV2") + val inner = body(content) + objectAt(inner, "/operators/1/inputPorts/0") + .put("disallowMultiInputs", single) + if (count == 2) { + arrayAt(inner, "/operators").add(textSource("second-source", "b\na")) + arrayAt(inner, "/links").add( + link("second-input", end("second-source", "output-0"), end("sort", "input-0")) + ) + } + val valid = !single || count == 1 + val result = checkModes(macroPlan(content), valid) + if (!valid) { + val message = result.operatorIdToError(outerID).message + checkPaths(message, "Python UDF") + message should include("requires 1 input, has 2") + } + } + } + + for (exposed <- List(false, true); overridden <- List(false, true)) { + "Forwarded macro properties" should s"validate the declared binding exposed=$exposed overridden=$overridden" in { + val content = workflow(validKeys) + body(content).set[JsonNode]("formBinding", if (exposed) sortBinding else formBinding()) + val forwarded = field("forwarded", "first", "sort-keys").put("displayName", "Sort Keys") + objectAt(content, "/macroDependencies/0/content") + .set[JsonNode]("formBinding", formBinding(forwarded)) + if (overridden) + objectAt(content, "/operators/0/operatorProperties") + .set[JsonNode]("forwarded", objectMapper.readTree(validKeys).path("keys")) + val before = content.toString + val imported = macroPlan(objectMapper.readTree(before)) + val result = checkModes(imported, exposed) + if (!exposed) { + result.operatorIdToError(outerID).message should include("Missing exposed macro property") + intercept[IllegalArgumentException](MacroExpansion.expand(content)) + } + content.toString shouldBe before + imported.macroContent.get.toString shouldBe before + } + } + + "Macro property defaults" should "use the descriptor defaults and honor explicitly configured values" in { + val defaults = new PythonUDFOpDescV2 + defaults.retainInputColumns shouldBe false // the schema annotation instead advertises true + val explicit = + json("code" -> "# 東京", "workers" -> 2, "retainInputColumns" -> true, "defaultEnv" -> false) + for (properties <- List(json("code" -> "# default"), explicit)) { + val content = workflow(properties.toString, "PythonUDFV2") + val before = content.toString + val operators = macroPlan(content).normalized.operators.collect { + case op: PythonUDFOpDescV2 => op + } + operators should have size 2 + operators.foreach { op => + op.workers shouldBe properties.path("workers").asInt(defaults.workers) + op.retainInputColumns shouldBe properties + .path("retainInputColumns") + .asBoolean(defaults.retainInputColumns) + op.defaultEnv shouldBe properties.path("defaultEnv").asBoolean(defaults.defaultEnv) + op.code shouldBe properties.get("code").asText() + } + content.toString shouldBe before + } + } + + List( + "workers" -> "\"1\"", + "workers" -> "null", + "workers" -> "{}", + "retainInputColumns" -> "\"true\"", + "retainInputColumns" -> "null", + "defaultEnv" -> "\"true\"", + "defaultEnv" -> "null" + ).foreach { + case (key, value) => + it should s"reject the explicit invalid value $key=$value in both compilation modes" in { + val content = workflow( + """{"code":"from pytexera import *","workers":1,"retainInputColumns":false,"defaultEnv":true}""", + "PythonUDFV2" + ) + objectAt(body(content), "/operators/1/operatorProperties") + .set[JsonNode](key, objectMapper.readTree(value)) + val result = invalidCompilation(content) + val message = result.operatorIdToError(outerID).message + checkPaths(message, "Python UDF") + message should include(key) + val schemas = result.operatorIdToOutputSchemas.values.flatMap(_.values.flatten) + schemas.flatMap(_.getAttributeNames) should contain("line") + intercept[IllegalArgumentException](strict(macroPlan(content))).getMessage should include( + key + ) + } + } + + private def castingWorkflow(properties: String): JsonNode = { + val content = workflow(kind = "Sort") + val inner = body(content) + arrayAt(inner, "/operators").add(transform("cast", "TypeCasting", properties)) + arrayAt(inner, "/links").add(link("sort-cast", end("sort", "output-0"), end("cast", "input-0"))) + objectAt(inner, "/links/1/source").put("operatorID", "cast") + content + } + + List( + "{}", + """{"typeCastingUnits":null}""", + """{"typeCastingUnits":"bad"}""", + """{"typeCastingUnits":[{"attribute":"line","resultType":""}]}""" + ).foreach { properties => + "Macro editing schemas" should s"preserve upstream schemas through incomplete Sort and Type Casting settings $properties" in { + val content = castingWorkflow(properties) + val before = content.toString + val request = macroPlan(content) + val result = compile(request) + result.physicalPlan shouldBe None + val upstream = request.macroExpansion.get.paths.filter { + case (_, path) => path.last == "sort" + } + upstream should have size 2 + upstream.keys.foreach { id => + result + .operatorIdToOutputSchemas(OperatorIdentity(id)) + .values + .flatten + .flatMap(_.getAttributeNames) + .toList shouldBe List("line") + } + result.operatorIdToError(outerID).message should include("Type Casting") + intercept[IllegalArgumentException](strict(request)) + intercept[IllegalArgumentException](request.normalized) + content.toString shouldBe before + } + } + + "Macro editing schemas" should "produce complete schemas once the same nested operators are configured" in { + val content = + castingWorkflow("""{"typeCastingUnits":[{"attribute":"line","resultType":"string"}]}""") + objectAt(body(content), "/operators/1") + .set[JsonNode]( + "operatorProperties", + objectMapper.readTree("""{"attributes":[{"attribute":"line","sortPreference":"ASC"}]}""") + ) + validCompilation(macroPlan(content)).operatorIdToOutputSchemas should have size 6 + } + + "Nested macro validation" should "report every missing property at the root with readable instance paths" in { + val content = workflow() + val before = content.toString + val result = invalidCompilation(content) + val message = result.operatorIdToError(outerID).message + checkPaths(message, "Stable Merge Sort") + message should include("keys") + message should not include "java.lang." + result.operatorIdToError.size should be >= 5 // both leaves, both nested instances, root + for (selection <- List(None, Some(Nil))) { + val request = macroPlan(content).copy(executeOperatorIds = selection) + val error = normalizationError(request) + error should include("Nested α") + error should include("keys") + intercept[IllegalArgumentException](strict(request)) + } + content.toString shouldBe before + } + + List( + """{"keys":null}""", + """{"keys":"bad"}""", + """{"keys":[{}]}""", + """{"keys":[{"attribute":"line","sortPreference":"INVALID"}]}""" + ).foreach { properties => + it should s"reject malformed nested settings $properties" in { + rootError(workflow(properties)) should include("Stable Merge Sort") + } + } + + it should "compile valid settings and preserve the complete pinned bundle" in { + val content = workflow(validKeys) + validCompilation(macroPlan(content)) + objectMapper + .valueToTree[JsonNode](macroPlan(content).macroExpansion.get) + .has("names") shouldBe false + } + + it should "validate effective exposed values after overrides without editing the stored definition" in { + val content = workflow() + body(content).set[JsonNode]("formBinding", sortBinding) + val children = content.at("/macroDependencies/0/content/operators") + val keys = objectMapper.readTree(validKeys).path("keys") + children + .elements() + .asScala + .foreach(child => objectAt(child, "/operatorProperties").set[JsonNode]("sort-keys", keys)) + val before = content.toString + validCompilation(macroPlan(content)) + objectAt(children, "/0/operatorProperties").putNull("sort-keys") + val message = rootError(content) + message should include("Nested α") + message should not include "Nested β" + body(content).at("/operators/1/operatorProperties").has("keys") shouldBe false + content.toString should not be before // only the deliberate override changed + } + + it should "report disconnected inputs with the same nested paths" in { + val content = workflow(validKeys) + arrayAt(body(content), "/links") + .remove(0) + rootError(content) should include("input-0") + } + + it should "ignore disabled invalid operators and keep structural expansion available for editing" in { + val content = workflow() + MacroExpansion.expand(content).content.path("operators").size() shouldBe 4 + objectAt(body(content), "/operators/1").put("isDisabled", true) + macroPlan(content).normalized.operators.size shouldBe 2 + } + + it should "use instance and definition names when custom labels are empty" in { + val content = workflow() + objectAt(content, "/operators/0").put("customDisplayName", " ") + objectAt(content, "/macroDependencies/0/content/operators/0") + .remove("customDisplayName") + rootError(content) should include("Outer / Inner / Stable Merge Sort") + } + + it should "propagate ordinary schema failures through every ancestor in both compilation modes" in { + val content = + workflow("""{"isDrop":false,"attributes":[{"originalAttribute":"missing"}]}""", "Projection") + val before = content.toString + val pinned = macroPlan(content) + val normalized = pinned.normalized + for (request <- List(pinned, normalized)) { + val result = compile(request) + result.physicalPlan shouldBe None + result.operatorIdToError.size shouldBe 5 + val message = result.operatorIdToError(outerID).message + checkPaths(message, "Projection") + message should include("missing") + intercept[Throwable](strict(request)).getMessage should include("Nested α / Projection") + } + objectMapper.valueToTree[JsonNode](normalized).has("expandedMacros") shouldBe false + content.toString shouldBe before + } + + it should "defer typed loop references to Texera's existing state-reference validation" in { + val content = workflow("""{"limit":"$n"}""", "Limit") + macroPlan(content).normalized.operators.count(_.stateReferences.nonEmpty) shouldBe 2 + rootError(content) should include("not inside a loop block") + objectAt(body(content), "/operators/1/operatorProperties") + .put("limit", "bad") + normalizationError(macroPlan(content)) should include( + "integer" + ) + } + + it should "reject unknown native types and missing required input declarations with readable paths" in { + val content = workflow(validKeys) + val sort = objectAt(body(content), "/operators/1") + sort.put("operatorType", "MissingOperator") + rootError(content) should include("Nested α / MissingOperator") + sort.put("operatorType", "StableMergeSort") + sort.putArray("inputPorts") + arrayAt(body(content), "/links") + .remove(0) + rootError(content) should include("input ports") + } + + it should "attribute an unfinished nested boundary to its own root instead of an unrelated macro" in { + val content = workflow(validKeys) + val ignored = + changed(content.at("/operators/0"), "operatorID" -> "ignored", "isDisabled" -> true) + arrayAt(content, "/operators") + .insert(0, ignored) + val edges = arrayAt(body(content), "/links") + edges.remove(edges.elements().asScala.indexWhere(_.path("linkID").asText() == "inner__1_out_0")) + val result = invalidCompilation(content) + result.operatorIdToError.keySet shouldBe Set(OperatorIdentity("outer")) + val message = result.operatorIdToError(outerID).message + message should startWith("Reusable 東京 / Nested α") + message should include("Connect or remove") + } + + it should "identify the operator path when a nested graph contains a cycle" in { + val content = workflow(validKeys) + arrayAt(body(content), "/links") + .add(link("cycle", end("sort", "output-0"), end("sort", "input-0"))) + val error = rootError(content) + error should include("Reusable 東京 / Nested α / Stable Merge Sort") + error should include("cycle") + } + + it should "validate shared snapshots concurrently without changing pins or runtime identities" in { + import scala.concurrent.{Await, ExecutionContext, Future} + import scala.concurrent.duration._ + implicit val executionContext: ExecutionContext = ExecutionContext.global + val content = workflow(validKeys) + val before = content.toString + val results = + Await.result( + Future.sequence((1 to 8).map(_ => Future(macroPlan(content).normalized))), + 10.seconds + ) + results.map(_.operators.map(_.operatorIdentifier)).distinct.size shouldBe 1 + content.toString shouldBe before + } + + it should "reuse conditional operator schemas for COUNT and aggregations that require an attribute" in { + val content = + workflow("""{"aggregations":[{"aggFunction":"count","result attribute":"n"}]}""", "Aggregate") + validCompilation(macroPlan(content)) + val aggregation = objectAt(body(content), "/operators/1/operatorProperties/aggregations/0") + aggregation.put("aggFunction", "sum") + normalizationError(macroPlan(content)) should include( + "attribute" + ) + aggregation.put("attribute", " ") + normalizationError(macroPlan(content)) should include( + "pattern" + ) + aggregation.put("attribute", "line") + macroPlan(content).normalized.operators.size shouldBe 4 + } +} diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowGraphIDs.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowGraphIDs.scala new file mode 100644 index 00000000000..45fb076612a --- /dev/null +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowGraphIDs.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.workflow + +import com.fasterxml.jackson.databind.JsonNode +import scala.collection.mutable +import scala.jdk.CollectionConverters._ + +/** Canvas cells share an ID namespace; each pinned graph owns a separate namespace. */ +object WorkflowGraphIDs { + def validateIDs(ids: Iterable[String], path: String = "Workflow"): Unit = { + val seen = mutable.HashSet.empty[String] + ids.foreach { id => + require(id != null && id.trim.nonEmpty, s"$path: Invalid graph element ID") + require(seen.add(id), s"$path: Duplicate graph element ID '$id'") + } + } + + def validate(content: JsonNode, path: String = "Workflow", depth: Int = 0): Unit = { + require(content != null && content.isObject && depth <= 32, s"$path: Malformed workflow graph") + val ids = + List("operators" -> "operatorID", "links" -> "linkID", "commentBoxes" -> "commentBoxID") + .flatMap { + case (field, key) => + val records = content.path(field) + require( + records.isMissingNode || records.isNull || records.isArray, + s"$path: Malformed workflow graph" + ) + records.elements().asScala.map(_.path(key).textValue()).toList + } + validateIDs(ids, path) + val dependencies = content.path("macroDependencies") + require( + dependencies.isMissingNode || dependencies.isNull || dependencies.isArray, + s"$path: Malformed workflow graph" + ) + dependencies.elements().asScala.foreach { definition => + validate( + definition.path("content"), + s"$path / ${definition.path("name").asText("Macro")}", + depth + 1 + ) + } + } +} diff --git a/common/workflow-core/src/main/typescript/workflow-macros/index.ts b/common/workflow-core/src/main/typescript/workflow-macros/index.ts new file mode 100644 index 00000000000..017affc9934 --- /dev/null +++ b/common/workflow-core/src/main/typescript/workflow-macros/index.ts @@ -0,0 +1,304 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** The graph contract shared by browser and agent execution; presentation content passes through untouched. */ +export interface OperatorPredicate { + readonly operatorID: string; + readonly operatorType: string; + readonly operatorVersion: string; + readonly operatorProperties: Readonly>; + readonly inputPorts: PortDescription[]; + readonly outputPorts: PortDescription[]; + readonly dynamicInputPorts?: boolean; + readonly dynamicOutputPorts?: boolean; + readonly showAdvanced: boolean; + readonly isDisabled?: boolean; + readonly viewResult?: boolean; + readonly markedForReuse?: boolean; + readonly customDisplayName?: string; +} +export interface LogicalPort { + readonly operatorID: string; + readonly portID: string; +} +export type PartitionInfo = + | Readonly<{ type: "hash"; hashAttributeNames: string[] }> + | Readonly<{ + type: "range"; + rangeAttributeNames: string[]; + rangeMin: number; + rangeMax: number; + }> + | Readonly<{ type: "single" }> + | Readonly<{ type: "broadcast" }> + | Readonly<{ type: "none" }>; +export interface PortDescription { + readonly portID: string; + readonly displayName?: string; + readonly disallowMultiInputs?: boolean; + readonly isDynamicPort?: boolean; + readonly partitionRequirement?: PartitionInfo; + readonly dependencies?: { id: number; internal: boolean }[]; +} +export interface OperatorLink { + readonly linkID: string; + readonly source: LogicalPort; + readonly target: LogicalPort; +} +export interface MacroPort extends PortDescription { + targets: LogicalPort[]; +} +export interface MacroInterface { + id: string; + inputs: MacroPort[]; + outputs: MacroPort[]; +} +export interface MacroContent { + operators: OperatorPredicate[]; + links: OperatorLink[]; + commentBoxes?: { commentBoxID: string }[]; + macroInterface?: { id: string }; + macroDependencies?: MacroDefinition[]; + formBinding?: unknown; +} + +/** Canvas cells share an ID namespace; each pinned graph owns a separate namespace. */ +export function assertWorkflowIDs( + content: unknown, + path = "Workflow", + depth = 0, +): void { + if ( + !content || + typeof content !== "object" || + Array.isArray(content) || + depth > 32 + ) + throw new Error(`${path}: Malformed workflow graph`); + const graph = content as Record; + const seen = new Set(); + for (const [field, key] of [ + ["operators", "operatorID"], + ["links", "linkID"], + ["commentBoxes", "commentBoxID"], + ]) { + const records = graph[field] ?? []; + if (!Array.isArray(records)) + throw new Error(`${path}: Malformed workflow graph`); + for (const record of records) { + const id: unknown = record?.[key]; + if (typeof id !== "string" || !id.trim()) + throw new Error(`${path}: Invalid graph element ID`); + if (seen.has(id)) + throw new Error(`${path}: Duplicate graph element ID '${id}'`); + seen.add(id); + } + } + const dependencies = graph.macroDependencies ?? []; + if (!Array.isArray(dependencies)) + throw new Error(`${path}: Malformed workflow graph`); + for (const definition of dependencies) + assertWorkflowIDs( + definition?.content, + `${path} / ${definition?.name ?? "Macro"}`, + depth + 1, + ); +} +export interface MacroDefinition { + id: string; + version: number; + /** Display ordinal within this workflow's history; version remains the immutable storage key. */ + revision?: number; + name: string; + wid?: number; + content: C; +} + +export const isMacro = (op: OperatorPredicate): boolean => + op.operatorType.startsWith("WorkflowMacro:"); +export const isMacroTerminal = ( + op: Pick, +): boolean => + op.operatorType === "MacroInput" || op.operatorType === "MacroOutput"; + +/** Port records own the public identity and name; ordinary links own the mapping. */ +export function readMacroInterface( + content: MacroContent, +): MacroInterface | undefined { + if (!content.macroInterface) return undefined; + const group = (input: boolean): MacroPort[] => { + const nodes = content.operators.filter( + (op) => op.operatorType === (input ? "MacroInput" : "MacroOutput"), + ); + return nodes.flatMap((node) => + (input ? node.outputPorts : node.inputPorts).map((port) => ({ + ...port, + targets: content.links + .filter((link) => { + const end = input ? link.source : link.target; + return ( + end.operatorID === node.operatorID && end.portID === port.portID + ); + }) + .map((link) => (input ? link.target : link.source)), + })), + ); + }; + return { + id: content.macroInterface.id, + inputs: group(true), + outputs: group(false), + }; +} + +export const macroType = (d: MacroDefinition): string => + `WorkflowMacro:${d.id}:${d.version}`; +export const portKey = ( + id: string, + direction: "inputs" | "outputs", + port: string, +): string => JSON.stringify([id, direction, port]); +const require = (condition: unknown, message: string): void => { + if (!condition) throw new Error(message); +}; +export function macroInstance( + d: MacroDefinition, + operatorID: string, +): OperatorPredicate { + require(d.content?.macroInterface, "Missing macro interface"); + const boundary = readMacroInterface(d.content)!; + return { + operatorID, + operatorType: macroType(d), + operatorVersion: String(d.version), + operatorProperties: {}, + inputPorts: boundary.inputs.map(({ targets, ...p }) => p), + outputPorts: boundary.outputs.map(({ targets, ...p }) => p), + showAdvanced: false, + customDisplayName: d.name, + }; +} + +export interface ResultTarget { + operatorID: string; + portIndex: number; +} + +/** Index into the single expanded native graph for one breadcrumb document. */ +export interface MacroInspector { + ids: Record; + inputs: Record; + terminals: { + direction: "inputs" | "outputs"; + portID: string; + links: number[]; + }[]; +} + +export interface MacroExpansion { + content: C; + /** Runtime id -> instance chain followed by the definition's leaf id. */ + paths: Record; + resultTargets?: Record; + resultPorts?: Record; + inspectors?: Record; + /** Canvas boundary -> runtime ports, for result and schema lookup. */ + ports: Record; +} + +/** Stable saved port IDs are distinct from their current position in the compiler's wire graph. */ +export const portOrdinal = ( + operator: OperatorPredicate, + direction: "inputPorts" | "outputPorts", + portID: string, +): number => operator[direction].findIndex((port) => port.portID === portID); + +/** Texera's wire graph uses port ordinals; the agent retains its fallback to port zero. */ +export function toLogicalGraph( + graph: Pick, + minimumPortOrdinal = -1, +) { + const byID = new Map(graph.operators.map((op) => [op.operatorID, op])); + const ordinal = (end: LogicalPort, direction: "inputPorts" | "outputPorts") => + Math.max( + minimumPortOrdinal, + portOrdinal(byID.get(end.operatorID)!, direction, end.portID), + ); + return { + operators: graph.operators.map((op) => ({ + ...op.operatorProperties, + operatorID: op.operatorID, + operatorType: op.operatorType, + inputPorts: op.inputPorts, + outputPorts: op.outputPorts, + })), + links: graph.links.map((link) => ({ + fromOpId: link.source.operatorID, + fromPortId: { id: ordinal(link.source, "outputPorts"), internal: false }, + toOpId: link.target.operatorID, + toPortId: { id: ordinal(link.target, "inputPorts"), internal: false }, + })), + }; +} + +/** Hosts supply their existing UUID v5 and equality implementations; the graph core has no runtime dependencies. */ +export function createMacroRuntime( + uuid: (value: string, namespace: string) => string, + isEqual: (a: unknown, b: unknown) => boolean, +) { + const namespace = "33269d11-7448-5d6d-9aeb-68826068e94d"; + const macroRuntimeID = (chain: string[], id: string): string => + chain.length ? uuid(JSON.stringify([...chain, id]), namespace) : id; + + /** One immutable snapshot per identity/version, shared by imports, libraries and clipboard operations. */ + function mergeMacroDefinitions( + ...bundles: MacroDefinition[][] + ): MacroDefinition[] { + const definitions = new Map>(); + const seen = new WeakSet(); + const collect = (d: MacroDefinition): void => { + require(d && + typeof d.id === "string" && + typeof d.name === "string" && + d.id.length && + Number.isSafeInteger(d.version) && + d.version > 0 && + d.content?.macroInterface?.id === d.id, "Malformed macro definition"); + if (seen.has(d)) return; + seen.add(d); + // A bundle is packaging, not part of the immutable definition body. Older saves nested it. + const content = { ...d.content }; + delete content.macroDependencies; + const previous = definitions.get(macroType(d)); + require(!previous || + isEqual(previous.content, content), "Conflicting macro version"); + definitions.set(macroType(d), { ...d, content }); + (d.content.macroDependencies ?? []).forEach((child) => + collect(child as MacroDefinition), + ); + }; + bundles.flat().forEach(collect); + return [...definitions.values()]; + } + + return { + mergeMacroDefinitions, + macroRuntimeID, + }; +} diff --git a/common/workflow-core/src/main/typescript/workflow-macros/package.json b/common/workflow-core/src/main/typescript/workflow-macros/package.json new file mode 100644 index 00000000000..452bac31f2b --- /dev/null +++ b/common/workflow-core/src/main/typescript/workflow-macros/package.json @@ -0,0 +1,8 @@ +{ + "name": "@texera/workflow-macros", + "version": "0.0.0", + "private": true, + "license": "Apache-2.0", + "type": "module", + "exports": "./index.ts" +} diff --git a/common/workflow-core/src/test/resources/workflow-graph-ids.json b/common/workflow-core/src/test/resources/workflow-graph-ids.json new file mode 100644 index 00000000000..6ad6f12b580 --- /dev/null +++ b/common/workflow-core/src/test/resources/workflow-graph-ids.json @@ -0,0 +1,135 @@ +{ + "collections": { + "operators": "operatorID", + "links": "linkID", + "commentBoxes": "commentBoxID" + }, + "invalidIDs": [ + { + "value": "", + "label": "''" + }, + { + "value": null, + "label": "None" + }, + { + "value": 3, + "label": "3" + }, + { + "value": " ", + "label": "' '" + } + ], + "errors": { + "collision": "Duplicate graph element ID 'same'", + "id": "Invalid graph element ID", + "graph": "Malformed workflow graph" + }, + "positive": [ + { + "name": "empty graph", + "content": {} + }, + { + "name": "unicode identities and untouched values", + "content": { + "operators": [ + { + "operatorID": "节点/東京", + "operatorProperties": { + "value": "same" + } + } + ], + "links": [ + { + "linkID": "edge" + } + ], + "commentBoxes": [ + { + "commentBoxID": "💬" + } + ] + } + }, + { + "name": "IDs scoped to each saved revision", + "content": { + "operators": [ + { + "operatorID": "same" + } + ], + "macroDependencies": [ + { + "name": "Revision one", + "content": { + "operators": [ + { + "operatorID": "same" + } + ] + } + }, + { + "name": "Revision two", + "content": { + "links": [ + { + "linkID": "same" + } + ] + } + } + ] + } + } + ], + "negative": [ + { + "name": "reject missing ID", + "content": { + "operators": [ + {} + ] + }, + "error": "Invalid graph element ID" + }, + { + "name": "reject null root", + "content": null, + "error": "Malformed workflow graph" + }, + { + "name": "reject array root", + "content": [], + "error": "Malformed workflow graph" + }, + { + "name": "reject duplicate in an unused pin", + "content": { + "macroDependencies": [ + { + "name": "Nested", + "content": { + "links": [ + { + "linkID": "same" + } + ], + "commentBoxes": [ + { + "commentBoxID": "same" + } + ] + } + } + ] + }, + "error": "Nested: Duplicate graph element ID 'same'" + } + ] +} diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowGraphIDsSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowGraphIDsSpec.scala new file mode 100644 index 00000000000..fbfbd61b72b --- /dev/null +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/workflow/WorkflowGraphIDsSpec.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.core.workflow + +import com.fasterxml.jackson.databind.JsonNode +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.flatspec.AnyFlatSpec +import scala.jdk.CollectionConverters._ + +class WorkflowGraphIDsSpec extends AnyFlatSpec { + private val matrix = objectMapper.readTree(getClass.getResource("/workflow-graph-ids.json")) + private val collections = + matrix.path("collections").fields().asScala.map(e => e.getKey -> e.getValue.asText()).toList + + private def check(name: String, content: JsonNode, expected: Option[String]): Unit = + "Workflow graph identities" should name in { + val before = content.toString + expected match { + case Some(message) => + val error = intercept[IllegalArgumentException](WorkflowGraphIDs.validate(content)) + assert(error.getMessage.contains(message)) + case None => WorkflowGraphIDs.validate(content) + } + assert(content.toString == before) + } + + for (group <- List("positive", "negative"); example <- matrix.path(group).elements().asScala) + check( + example.path("name").asText(), + example.path("content"), + Option(example.get("error")).map(_.asText()) + ) + + private def reject(name: String, graph: Map[String, Any], error: String): Unit = + check( + name, + objectMapper.valueToTree[JsonNode](graph), + Some(matrix.path("errors").path(error).asText()) + ) + + for (((field, key), i) <- collections.zipWithIndex) { + for ((other, otherKey) <- collections.drop(i)) { + val ids = List(field -> Map(key -> "same"), other -> Map(otherKey -> "same")) + reject(s"reject $field/$other collisions", ids.groupMap(_._1)(_._2), "collision") + } + for (invalid <- matrix.path("invalidIDs").elements().asScala) + reject( + s"reject $field ID ${invalid.path("label").asText()}", + Map(field -> List(Map(key -> invalid.path("value")))), + "id" + ) + reject(s"reject malformed $field", Map(field -> Map.empty[String, String]), "graph") + } +} diff --git a/common/workflow-core/src/test/typescript/macro-test-graph.ts b/common/workflow-core/src/test/typescript/macro-test-graph.ts new file mode 100644 index 00000000000..3cb9f957e80 --- /dev/null +++ b/common/workflow-core/src/test/typescript/macro-test-graph.ts @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + readMacroInterface, + isMacroTerminal, +} from "../../main/typescript/workflow-macros"; +import type { + MacroContent, + MacroInterface, +} from "../../main/typescript/workflow-macros"; +import graphIDs from "../resources/workflow-graph-ids.json"; + +const collections = Object.entries(graphIDs.collections); +export const graphIDCases: { + name: string; + content: unknown; + error?: string; +}[] = [ + ...graphIDs.positive, + ...collections.flatMap(([field, key], i) => [ + ...collections.slice(i).map(([other, otherKey]) => ({ + name: `reject ${field}/${other} collisions`, + error: graphIDs.errors.collision, + content: + field === other + ? { [field]: [{ [key]: "same" }, { [key]: "same" }] } + : { [field]: [{ [key]: "same" }], [other]: [{ [otherKey]: "same" }] }, + })), + ...graphIDs.invalidIDs.map(({ value, label }) => ({ + name: `reject ${field} ID ${label}`, + content: { [field]: [{ [key]: value }] }, + error: graphIDs.errors.id, + })), + { + name: `reject malformed ${field}`, + content: { [field]: {} }, + error: graphIDs.errors.graph, + }, + ]), + ...graphIDs.negative, +]; + +/** Concise fixture builder: specify the exposed ports and generate their saved graph nodes/links. */ +export function macroGraph< + C extends Omit & { + macroInterface: MacroInterface; + operatorPositions?: Record; + }, +>(graph: C): Omit & { macroInterface: { id: string } } { + const boundary = graph.macroInterface; + const oldIDs = new Set( + graph.operators.filter(isMacroTerminal).map((op) => op.operatorID), + ); + const operators = graph.operators.filter((op) => !isMacroTerminal(op)); + const links = graph.links.filter( + (link) => + !oldIDs.has(link.source.operatorID) && + !oldIDs.has(link.target.operatorID), + ); + if (!boundary.id) + return { ...graph, operators, links, macroInterface: { id: boundary.id } }; + const terminals = [true, false].map((input) => ({ + operatorID: `${boundary.id}__${input ? "inputs" : "outputs"}`, + operatorType: input ? "MacroInput" : "MacroOutput", + operatorVersion: "1", + operatorProperties: {}, + inputPorts: input + ? [] + : boundary.outputs.map(({ targets, ...port }) => port), + outputPorts: input + ? boundary.inputs.map(({ targets, ...port }) => port) + : [], + dynamicInputPorts: !input, + dynamicOutputPorts: input, + showAdvanced: false, + })); + return { + ...graph, + macroInterface: { id: boundary.id }, + operators: [...operators, ...terminals], + links: [ + ...links, + ...[boundary.inputs, boundary.outputs].flatMap((ports, index) => + ports.flatMap((port) => + port.targets.map((target, n) => { + const end = { + operatorID: terminals[index].operatorID, + portID: port.portID, + }; + return { + linkID: `${boundary.id}__${index}_${port.portID}_${n}`, + source: index === 0 ? end : target, + target: index === 0 ? target : end, + }; + }), + ), + ), + ], + ...(graph.operatorPositions + ? { + operatorPositions: { + ...graph.operatorPositions, + [terminals[0].operatorID]: { x: -100, y: 100 }, + [terminals[1].operatorID]: { x: 600, y: 100 }, + }, + } + : {}), + }; +} + +/** Edit a fixture's public interface by rebuilding its normal graph wiring. */ +export function editMacroGraph( + graph: C, + edit: (boundary: MacroInterface) => void, +): void { + const boundary = structuredClone(readMacroInterface(graph)!); + edit(boundary); + Object.assign(graph, macroGraph({ ...graph, macroInterface: boundary })); +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala index 1b5a8b666b5..1e846a8b391 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala @@ -40,6 +40,7 @@ import org.apache.texera.amber.operator.cartesianProduct.CartesianProductOpDesc import org.apache.texera.amber.operator.dictionary.DictionaryMatcherOpDesc import org.apache.texera.amber.operator.set.difference.DifferenceOpDesc import org.apache.texera.amber.operator.distinct.DistinctOpDesc +import org.apache.texera.amber.operator.macroterminal.{MacroInputOpDesc, MacroOutputOpDesc} import org.apache.texera.amber.operator.dummy.DummyOpDesc import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc @@ -268,6 +269,8 @@ trait StateTransferFunc new Type(value = classOf[ImageVisualizerOpDesc], name = "ImageVisualizer"), new Type(value = classOf[HierarchyChartOpDesc], name = "HierarchyChart"), new Type(value = classOf[DumbbellPlotOpDesc], name = "DumbbellPlot"), + new Type(value = classOf[MacroInputOpDesc], name = "MacroInput"), + new Type(value = classOf[MacroOutputOpDesc], name = "MacroOutput"), new Type(value = classOf[DummyOpDesc], name = "Dummy"), new Type(value = classOf[BoxViolinPlotOpDesc], name = "BoxViolinPlot"), new Type(value = classOf[NetworkGraphOpDesc], name = "NetworkGraph"), diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/macroterminal/MacroTerminalOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/macroterminal/MacroTerminalOpDesc.scala new file mode 100644 index 00000000000..313211cf52b --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/macroterminal/MacroTerminalOpDesc.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.macroterminal + +import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} +import org.apache.texera.amber.operator.{LogicalOp, PortDescriptor} +import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} + +/** Saved graph boundaries. MacroExpansion removes these before physical compilation. */ +abstract class MacroTerminalOpDesc(input: Boolean) extends LogicalOp with PortDescriptor { + operatorVersion = "1" + override def operatorInfo: OperatorInfo = + OperatorInfo( + if (input) "Inputs" else "Outputs", + "Connect this macro boundary to the operators it exposes.", + OperatorGroupConstants.UTILITY_GROUP, + Option(inputPorts).getOrElse(Nil).zipWithIndex.map { + case (port, index) => + InputPort(PortIdentity(index), displayName = port.displayName, disallowMultiLinks = true) + }, + Option(outputPorts).getOrElse(Nil).zipWithIndex.map { + case (port, index) => + OutputPort(PortIdentity(index), displayName = port.displayName) + }, + dynamicInputPorts = !input, + dynamicOutputPorts = input + ) +} + +class MacroInputOpDesc extends MacroTerminalOpDesc(true) +class MacroOutputOpDesc extends MacroTerminalOpDesc(false) diff --git a/frontend/LICENSE-binary b/frontend/LICENSE-binary index 66bc0d1b79b..990967c6915 100644 --- a/frontend/LICENSE-binary +++ b/frontend/LICENSE-binary @@ -216,6 +216,7 @@ Angular / npm packages: - @ai-sdk/provider@2.0.0 - @ai-sdk/provider-utils@3.0.17 - @opentelemetry/api@1.9.0 + - @texera/workflow-macros@0.0.0 - @vercel/oidc@3.0.3 - ai@5.0.290 - dompurify@3.4.15 diff --git a/frontend/README.md b/frontend/README.md index d1f97bbcd3d..b99b0d75b3b 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -53,6 +53,44 @@ Tests come first — write the failing test before the source change. The full testing reference (Vitest stack, recipes, anti-patterns, coverage troubleshooting) is in [`TESTING.md`](TESTING.md). +## Workflow macros + +Select operators, open **Operators → Macros**, enter a name, and choose **Create from selection**. +Drag saved macros from that panel into any workflow. Macros may contain other macros. +Dragging a macro captures its latest saved revision. Each usage keeps that immutable snapshot until explicitly updated. +Double-click a usage to open its pinned graph on the main canvas; title-bar breadcrumbs return to its parents. +Any pinned revision is editable with write access to its source and parent; saving creates new revisions without +changing historical snapshots. Older revisions use isolated drafts, including an older revision of the same macro. +Unexpected source changes stop the save so they can be reviewed before retrying. The property panel's revision +selector and **Use latest** action preserve connections to surviving ports and values of surviving parameters. +Removed ports disconnect their links; removed parameters stop forwarding through ancestors. Normal validation reports +disconnected inputs or incompatible retained values. Undo restores the previous pin, wiring and forwarded interface. +Saving through a parent creates normal workflow revisions along that breadcrumb path and updates only that usage. +Editing the definition directly at `/user/macros/:id` leaves all existing usages pinned. +The regular history panel supports preview and restore; restoring creates a new latest revision. +The canvas shows the selected instance's live row counts, workers, execution state and runtime, including nested macros. +Changing the viewed graph clears its execution statistics. **Save editable copy** creates an independent definition, +including for imported snapshots without accessible sources. Macro details beside the title list accessible direct and nested usages. +In a definition, two saved graph operators show Inputs and Outputs. Select a node and use + to add a port, then draw ordinary connections; +select a port and use − to remove it. Select a port to edit its name in the normal Property panel. +If a boundary operator is removed, add it again from Utilities; each role is offered only while missing. +Unconnected ports remain visible and must be wired or removed before expansion. +The port's Property panel also exposes an existing operator port. Replacing an internal input connection requires an explicit action. +Inputs can fan out; each output port has one source. Ports, links and positions use the normal shared graph and Undo/Redo. +The backend compiles these boundary operators away; they never create runtime workers. +Use **Choose Form View fields** in the Property panel, or **Edit** in Form View, to select a definition's public +parameters. The same fields appear on macro usages; a parent can select them for its own Form View. +Removing a macro parameter also removes its forwarding Form View fields and saved overrides along the edited parent path. +Undo/Redo restores those bindings and values, including after autosave; other usages keep their saved revisions. +A macro definition runs through a workflow. Compilation recursively expands it into ordinary operators. +Workflow JSON exports include pinned definitions, so importing does not depend on the original deployment. +External datasets and connections still need access on the destination deployment. +The editor and agent send pinned graphs to the shared Scala compiler in `common/workflow-compiler`. +Compilation and execution use the same expansion; its returned paths and port mappings drive nested schemas, results and statistics. +The TypeScript macro package retains the snapshot contract and presentation helpers. +Agent edits preserve pinned snapshots, and agent compilation and execution use the same expanded operators as the editor. +Exports share one immutable definition body per version, including the saved boundary operators and connections. + ## Project layout | Path | What lives here | diff --git a/frontend/package.json b/frontend/package.json index cb55140783b..fddbee011bd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -40,6 +40,7 @@ "@ngneat/until-destroy": "10.0.0", "@ngx-formly/core": "7.1.0", "@ngx-formly/ng-zorro-antd": "7.1.0", + "@texera/workflow-macros": "file:../common/workflow-core/src/main/typescript/workflow-macros", "ai": "5.0.290", "ajv": "8.18.0", "concaveman": "2.0.0", diff --git a/frontend/src/app/app-routing.constant.ts b/frontend/src/app/app-routing.constant.ts index 417e5753368..44e8e5aa229 100644 --- a/frontend/src/app/app-routing.constant.ts +++ b/frontend/src/app/app-routing.constant.ts @@ -41,6 +41,7 @@ export const USER_WORKFLOW = `${USER}/workflow`; /** One workflow is open in the workspace under two views: the operator canvas and the Form View. */ export const workspaceCanvasUrl = (wid: number): string => `${USER_WORKSPACE}/${wid}`; +export const macroCanvasUrl = (wid: number): string => `${USER}/macros/${wid}`; export const workspaceFormUrl = (wid: number): string => `${workspaceCanvasUrl(wid)}/form`; /** @@ -52,7 +53,7 @@ function isWorkspaceViewOf(url: string, wid: number | undefined): boolean { return false; } const path = url.split(/[?#]/)[0]; - return path === workspaceCanvasUrl(wid) || path === workspaceFormUrl(wid); + return path === workspaceCanvasUrl(wid) || path === workspaceFormUrl(wid) || path === macroCanvasUrl(wid); } /** diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index bab01f2dd88..7b6a0902ee4 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -148,6 +148,12 @@ routes.push({ { path: "workflow/:id", component: WorkspaceComponent, + canDeactivate: [(component: WorkspaceComponent) => component.canLeave()], + }, + { + path: "macros/:id", + component: WorkspaceComponent, + canDeactivate: [(component: WorkspaceComponent) => component.canLeave()], }, { path: "dataset", diff --git a/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.html b/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.html index 09f9d29d8ca..0e7fc774708 100644 --- a/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.html +++ b/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.html @@ -20,11 +20,12 @@
diff --git a/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.spec.ts b/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.spec.ts index 8374b314ff3..5258f95af08 100644 --- a/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.spec.ts +++ b/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.spec.ts @@ -19,6 +19,7 @@ import { ComponentFixture, TestBed } from "@angular/core/testing"; import { FormlyFieldConfig } from "@ngx-formly/core"; +import { FormControl } from "@angular/forms"; import { ExposePropertyWrapperComponent } from "./expose-property-wrapper.component"; describe("ExposePropertyWrapperComponent", () => { @@ -34,50 +35,67 @@ describe("ExposePropertyWrapperComponent", () => { }); describe("decorate", () => { - it("keeps form-field outermost and appends the wrapper when no wrappers are set", () => { - const config: FormlyFieldConfig = { key: "k" }; - ExposePropertyWrapperComponent.decorate(config, true, () => {}); - expect(config.wrappers).toEqual(["form-field", "expose-property-wrapper"]); - }); - - it("appends after existing wrappers without dropping them", () => { - const config: FormlyFieldConfig = { key: "k", wrappers: ["custom"] }; - ExposePropertyWrapperComponent.decorate(config, false, () => {}); - expect(config.wrappers).toEqual(["custom", "expose-property-wrapper"]); + it.each([ + { wrappers: undefined, first: "form-field" }, + { wrappers: ["custom"], first: "custom" }, + ])("preserves $first as the outer wrapper", ({ wrappers, first }) => { + const config: FormlyFieldConfig = { key: "k", wrappers }; + ExposePropertyWrapperComponent.decorate(config, () => true, vi.fn()); + expect(config.wrappers).toEqual([first, "expose-property-wrapper"]); }); it("carries the exposed state and toggle callback in props, preserving existing props", () => { const toggle = vi.fn(); + let exposed = true; const config: FormlyFieldConfig = { key: "k", props: { label: "kept" } }; - ExposePropertyWrapperComponent.decorate(config, true, toggle); + ExposePropertyWrapperComponent.decorate(config, () => exposed, toggle); expect(config.props?.["label"]).toBe("kept"); - expect(config.props?.["exposed"]).toBe(true); + expect(config.props?.["isExposed"]()).toBe(true); + exposed = false; + expect(config.props?.["isExposed"]()).toBe(false); expect(config.props?.["toggleExposed"]).toBe(toggle); }); }); - describe("onToggle", () => { - it("forwards the checkbox checked state to toggleExposed", () => { - const toggle = vi.fn(); - component.field = { props: { exposed: false, toggleExposed: toggle } } as unknown as FormlyFieldConfig; - component.onToggle({ target: { checked: true } } as unknown as Event); - expect(toggle).toHaveBeenCalledWith(true); + describe("template", () => { + const checkbox = (): HTMLInputElement => fixture.nativeElement.querySelector("input[type=checkbox]"); + + it.each([ + { label: "Text", expected: "Show Text on the Form View" }, + { label: undefined, expected: "Show this property on the Form View" }, + ])("uses the accessible label '$expected' and follows the form control", ({ label, expected }) => { + const formControl = new FormControl({ value: "text", disabled: true }); + component.field = { formControl, props: { label, isExposed: () => false } }; + fixture.detectChanges(); + expect(checkbox().getAttribute("aria-label")).toBe(expected); + expect(checkbox().disabled).toBe(true); + formControl.enable(); + fixture.detectChanges(); + expect(checkbox().disabled).toBe(false); }); - }); - describe("template", () => { - it("renders a checkbox reflecting props.exposed and drives onToggle on change", () => { + it("reflects binding changes without remounting and drives onToggle on change", () => { const toggle = vi.fn(); - component.field = { props: { exposed: true, toggleExposed: toggle } } as unknown as FormlyFieldConfig; + let exposed = true; + component.field = { formControl: new FormControl(), props: { isExposed: () => exposed, toggleExposed: toggle } }; fixture.detectChanges(); - const box = fixture.nativeElement.querySelector("input[type=checkbox]") as HTMLInputElement; + const box = checkbox(); expect(box).toBeTruthy(); expect(box.checked).toBe(true); - box.checked = false; - box.dispatchEvent(new Event("change")); - expect(toggle).toHaveBeenCalledWith(false); + exposed = false; + fixture.detectChanges(); + expect(box.checked).toBe(false); + exposed = true; + fixture.detectChanges(); + expect(box.checked).toBe(true); + + for (const checked of [false, true]) { + box.checked = checked; + box.dispatchEvent(new Event("change")); + expect(toggle).toHaveBeenLastCalledWith(checked); + } }); }); }); diff --git a/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.ts b/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.ts index ade80e1e0fb..ac4a891a134 100644 --- a/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.ts +++ b/frontend/src/app/common/formly/expose-property-wrapper/expose-property-wrapper.component.ts @@ -22,8 +22,7 @@ import { FieldWrapper, FormlyFieldConfig } from "@ngx-formly/core"; import { merge } from "lodash-es"; /** - * A tick box beside an operator property, in the property editor, for an author to choose - * whether it appears on the Form View. Rendered only while choosing. + * A tick box beside an operator property for an author to choose whether it appears on the Form View. */ @Component({ selector: "texera-expose-property-wrapper", @@ -31,12 +30,20 @@ import { merge } from "lodash-es"; styleUrls: ["./expose-property-wrapper.component.scss"], }) export class ExposePropertyWrapperComponent extends FieldWrapper { + public get exposureLabel(): string { + const label = this.props.label || "this property"; + return `Show ${label} on the Form View`; + } /** Add this wrapper to a field, carrying state + callback in `props`; `form-field` stays * outermost so label/error rendering is untouched. */ - public static decorate(config: FormlyFieldConfig, exposed: boolean, toggle: (checked: boolean) => void): void { + public static decorate( + config: FormlyFieldConfig, + isExposed: () => boolean, + toggle: (checked: boolean) => void + ): void { merge(config, { wrappers: [...(config.wrappers ?? ["form-field"]), "expose-property-wrapper"], - props: { ...config.props, exposed, toggleExposed: toggle }, + props: { ...config.props, isExposed, toggleExposed: toggle }, }); } diff --git a/frontend/src/app/common/formly/formly-utils.spec.ts b/frontend/src/app/common/formly/formly-utils.spec.ts index 6f8feea682f..08ab8b21636 100644 --- a/frontend/src/app/common/formly/formly-utils.spec.ts +++ b/frontend/src/app/common/formly/formly-utils.spec.ts @@ -173,6 +173,22 @@ describe("createOutputFormChangeEventStream", () => { vi.useRealTimers(); }); + it("flushes pending valid edits before an editor document changes, without duplicates", () => { + const source = new Subject>(), + flush = new Subject(); + const values: Record[] = []; + createOutputFormChangeEventStream(source, data => data["valid"] === true, flush).subscribe(v => values.push(v)); + flush.next(); + source.next({ valid: true, value: "last keystroke" }); + flush.next(); + flush.next(); + vi.advanceTimersByTime(FORM_DEBOUNCE_TIME_MS); + expect(values).toEqual([{ valid: true, value: "last keystroke" }]); + source.next({ valid: false }); + flush.next(); + expect(values).toHaveLength(1); + }); + it("debounces bursts, emitting only the latest value once the debounce window elapses", () => { const source = new Subject>(); const emissions: Record[] = []; diff --git a/frontend/src/app/common/formly/formly-utils.ts b/frontend/src/app/common/formly/formly-utils.ts index 15c25fb071a..fdaaa6b728b 100644 --- a/frontend/src/app/common/formly/formly-utils.ts +++ b/frontend/src/app/common/formly/formly-utils.ts @@ -20,26 +20,76 @@ import { FormlyFieldConfig } from "@ngx-formly/core"; import { isDefined } from "../util/predicate"; -import { Observable } from "rxjs"; +import { Observable, EMPTY, merge, timer } from "rxjs"; import { FORM_DEBOUNCE_TIME_MS } from "../../workspace/service/execute-workflow/execute-workflow.service"; -import { debounceTime, distinctUntilChanged, filter, share, tap } from "rxjs/operators"; -import { HideType, ValueRuleSet } from "../../workspace/types/custom-json-schema.interface"; +import { debounce, distinctUntilChanged, filter, share, tap } from "rxjs/operators"; +import { + CustomJSONSchema7, + HideType, + PropertyContext, + ValueRuleSet, +} from "../../workspace/types/custom-json-schema.interface"; import { PortSchema } from "../../workspace/types/workflow-compiling.interface"; import { AbstractControl } from "@angular/forms"; +import { propertyModel, ConditionalRequiredRule } from "../../workspace/util/custom-formly-type"; +import { applyHuggingFaceFieldBindings } from "../../workspace/util/hugging-face-field-bindings"; export function getFieldByName(fieldName: string, fields: FormlyFieldConfig[]): FormlyFieldConfig | undefined { return fields.filter((field, _, __) => field.key === fieldName)[0]; } export function setHideExpression(toggleHidden: string[], fields: FormlyFieldConfig[], hiddenBy: string): void { + const context = getFieldByName(hiddenBy, fields)?.props?.["propertyContext"] as PropertyContext | undefined; toggleHidden.forEach(hiddenFieldName => { - const fieldToBeHidden = getFieldByName(hiddenFieldName, fields); + const fieldToBeHidden = getFieldByName(context?.bindings[hiddenFieldName] ?? hiddenFieldName, fields); if (isDefined(fieldToBeHidden)) { - fieldToBeHidden.expressions = { hide: "!field.parent.model." + hiddenBy }; + fieldToBeHidden.expressions = { + ...fieldToBeHidden.expressions, + hide: context + ? (field: FormlyFieldConfig) => !propertyModel(field)?.[context.propertyKey] + : "!field.parent.model." + hiddenBy, + }; } }); } +export function setFieldDependencies( + schema: CustomJSONSchema7, + fields: FormlyFieldConfig[], + attributes?: Readonly> +): void { + Object.entries(schema.properties ?? {}).forEach(([key, property]) => { + if (typeof property === "boolean") return; + if (property.toggleHidden) setHideExpression(property.toggleHidden, fields, key); + if (property.dependOn) setChildTypeDependency(attributes, property.dependOn, fields, key); + }); +} + +/** Apply the same schema-driven behaviour in the property panel and Form View. */ +export function setFieldBehavior( + field: FormlyFieldConfig, + schema: CustomJSONSchema7, + required?: ConditionalRequiredRule +): void { + const context = field.props?.["propertyContext"] as PropertyContext | undefined; + if (context?.operatorType === "HuggingFace") applyHuggingFaceFieldBindings(field); + if (schema.hideExpectedValue !== undefined && schema.hideTarget && schema.hideType) { + field.expressions = { + ...field.expressions, + hide: createShouldHideFieldFunc(schema.hideTarget, schema.hideType, schema.hideExpectedValue, schema.hideOnNull), + }; + } + if (context?.hiddenBy) + field.expressions = { ...field.expressions, hide: f => !propertyModel(f)?.[context.hiddenBy!] }; + if (schema.valueRules) setValueRules(field, schema.valueRules); + const rule = context?.required ?? required; + if (rule) + field.expressions = { + ...field.expressions, + "props.required": f => (propertyModel(f)?.[rule.sibling] === rule.value) === rule.requiredOnMatch, + }; +} + type ValueRule = ValueRuleSet["allOf"][number]["then"]; /** @@ -69,7 +119,7 @@ export function matchingValueRule(rules: ValueRuleSet | undefined, rowModel: any */ export function createValueRulesValidator(rules: ValueRuleSet) { return (control: AbstractControl, field: FormlyFieldConfig): boolean => { - const rule = matchingValueRule(rules, field?.parent?.model); + const rule = matchingValueRule(rules, propertyModel(field)); if (!isDefined(rule)) { return true; } @@ -108,7 +158,7 @@ export function createValueRulesValidator(rules: ValueRuleSet) { /** Says what the field will take, naming the branch rather than the rule that rejected it. */ export function valueRulesValidationMessage(_err: unknown, field: FormlyFieldConfig): string { - const rule = matchingValueRule(field?.props?.valueRules, field?.parent?.model); + const rule = matchingValueRule(field?.props?.valueRules, propertyModel(field)); if (isDefined(rule?.enum)) { return `must be one of ${rule.enum.join(", ")}`; } @@ -139,7 +189,10 @@ export function valueRulesValidationMessage(_err: unknown, field: FormlyFieldCon * model is what picks the branch. */ export function setValueRules(field: FormlyFieldConfig, rules: ValueRuleSet): void { - const siblings = new Set(rules.allOf.flatMap(branch => Object.keys(branch.if))); + const context = field.props?.["propertyContext"] as PropertyContext | undefined; + const siblings = new Set( + rules.allOf.flatMap(branch => Object.keys(branch.if)).map(key => context?.bindings[key] ?? key) + ); field.type = "constrainedvalue"; // written into the existing object rather than over it: `props` and `templateOptions` are two // names for one object, and replacing it leaves them pointing at different ones @@ -178,7 +231,7 @@ export function createShouldHideFieldFunc( let shared_regex: RegExp | null = null; const hideFunc = (field?: FormlyFieldConfig | undefined) => { - const model = field?.parent?.model; + const model = propertyModel(field); if (model === null || model === undefined) { console.debug("Formly main model not detected. Hiding will fail."); return false; @@ -219,14 +272,19 @@ export function setChildTypeDependency( const childField = getFieldByName(childName, fields); if (isDefined(childField)) { childField.expressions = { + ...childField.expressions, // 'type': 'string', // 'templateOptions.type': JSON.stringify(timestampFieldNames) + '.includes(model.' + parentName + ')? \'string\' : \'number\'', - "templateOptions.description": - JSON.stringify(timestampFieldNames) + - ".includes(model." + - parentName + - ")? 'Input a datetime string' : 'Input a positive number'", + "templateOptions.description": childField.props?.["propertyContext"] + ? (field: FormlyFieldConfig) => + timestampFieldNames.includes(propertyModel(field)?.[parentName]) + ? "Input a datetime string" + : "Input a positive number" + : JSON.stringify(timestampFieldNames) + + ".includes(model." + + parentName + + ")? 'Input a datetime string' : 'Input a positive number'", }; } } @@ -243,14 +301,15 @@ export function setChildTypeDependency( */ export function createOutputFormChangeEventStream( formChangeEvent: Observable>, - modelCheck: (formData: Record) => boolean + modelCheck: (formData: Record) => boolean, + flush: Observable = EMPTY ): Observable> { return ( formChangeEvent // set a debounce time to avoid events triggering too often // and to circumvent a bug of the library - each action triggers event twice .pipe( - debounceTime(FORM_DEBOUNCE_TIME_MS), + debounce(() => merge(timer(FORM_DEBOUNCE_TIME_MS), flush)), // .do(evt => console.log(evt)) // don't emit the event until the data is changed distinctUntilChanged(), diff --git a/frontend/src/app/common/service/workflow-persist/workflow-persist.service.spec.ts b/frontend/src/app/common/service/workflow-persist/workflow-persist.service.spec.ts index 7773458b5f1..61599fa8462 100644 --- a/frontend/src/app/common/service/workflow-persist/workflow-persist.service.spec.ts +++ b/frontend/src/app/common/service/workflow-persist/workflow-persist.service.spec.ts @@ -89,6 +89,21 @@ describe("WorkflowPersistService", () => { expect(service).toBeTruthy(); }); + it("does not POST invalid IDs through either create or persist", () => { + const content = jsonCast(testContent); + const invalid = { ...content, links: [{ ...content.links[0], linkID: content.operators[0].operatorID }] }; + const errors: unknown[] = []; + service.createWorkflow(invalid).subscribe({ error: (error: unknown) => errors.push(error) }); + service + .persistWorkflow({ name: "bad", content: invalid } as Workflow) + .subscribe({ error: (error: unknown) => errors.push(error) }); + expect(errors).toHaveLength(2); + expect(errors.every(error => error instanceof Error && error.message.includes("Duplicate graph element ID"))).toBe( + true + ); + httpTestingController.expectNone(request => request.method === "POST"); + }); + it("should send http post request once", () => { service .createWorkflow(jsonCast(testContent), "testname") @@ -111,6 +126,40 @@ describe("WorkflowPersistService", () => { const API = AppSettings.getApiEndpoint(); + it("submits the edited path once and returns the server's committed revisions in order", () => { + const root = { wid: 7, name: "Root", content: jsonCast(testContent) } as Workflow; + const child = { ...root, wid: 11, name: "Child" }; + const head = { version: 19, name: "Child" }; + const next = vi.fn(); + const save = service.persistMacroPath( + [ + { workflow: child, head }, + { workflow: root, operatorID: "use" }, + ], + root + ); + save.subscribe(next); + const request = httpTestingController.expectOne(`${API}/workflow/persist/macro-path`); + expect(request.request.method).toBe("POST"); + expect(request.request.body.steps).toMatchObject([ + { head, workflow: { wid: 11, content: JSON.stringify(child.content) } }, + { operatorID: "use", workflow: { wid: 7 } }, + ]); + expect(request.request.body.rootBefore.content).toBe(JSON.stringify(root.content)); + const saved = [ + { version: 31, revision: 3, content: child.content }, + { version: 34, revision: 8, content: root.content }, + ]; + request.flush(saved); + expect(next).toHaveBeenCalledWith(saved); + const error = vi.fn(); + save.subscribe({ error }); + httpTestingController + .expectOne(`${API}/workflow/persist/macro-path`) + .flush({ message: "The workflow has a newer saved revision." }, { status: 409, statusText: "Conflict" }); + expect(error).toHaveBeenCalledWith(new Error("The workflow has a newer saved revision.")); + }); + it("retrieveWorkflow issues a GET and parses the workflow content", () => { const raw = { wid: 5, name: "wf", content: '{"operators":[]}' } as unknown as Workflow; let result: Workflow | undefined; @@ -182,7 +231,9 @@ describe("WorkflowPersistService", () => { const brokenContent = { operators: [], operatorPositions: {}, - links: [{ source: { operatorID: "does-not-exist" }, target: { operatorID: "also-missing" } }], + links: [ + { linkID: "broken-link", source: { operatorID: "does-not-exist" }, target: { operatorID: "also-missing" } }, + ], } as unknown as WorkflowContent; beforeEach(() => { diff --git a/frontend/src/app/common/service/workflow-persist/workflow-persist.service.ts b/frontend/src/app/common/service/workflow-persist/workflow-persist.service.ts index fa61b112008..a5423434dd6 100644 --- a/frontend/src/app/common/service/workflow-persist/workflow-persist.service.ts +++ b/frontend/src/app/common/service/workflow-persist/workflow-persist.service.ts @@ -17,12 +17,13 @@ * under the License. */ -import { HttpClient, HttpParams } from "@angular/common/http"; +import { HttpClient, HttpErrorResponse, HttpParams } from "@angular/common/http"; import { Injectable, Injector } from "@angular/core"; import { EMPTY, Observable, of, ReplaySubject, Subject, throwError } from "rxjs"; import { catchError, concatMap, filter, finalize, map, take, tap } from "rxjs/operators"; import { AppSettings } from "../../app-setting"; import { Workflow, WorkflowContent } from "../../type/workflow"; + import { DashboardWorkflow } from "../../../dashboard/type/dashboard-workflow.interface"; import { DefaultView } from "../../../dashboard/type/workflow-metadata.interface"; import { WorkflowUtilService } from "../../../workspace/service/workflow-graph/util/workflow-util.service"; @@ -31,6 +32,24 @@ import { NotificationService } from "../notification/notification.service"; import { SearchFilterParameters, toQueryStrings } from "../../../dashboard/type/search-filter-parameters"; import { User } from "../../type/user"; import { checkIfWorkflowBroken } from "../../util/workflow-check"; +import { assertWorkflowIDs, MacroDefinition } from "../../util/workflow-macro"; + +export interface MacroSaveStep { + workflow: Workflow; + head?: { version: number; name: string; description?: string }; + operatorID?: string; +} +export interface MacroCatalog { + definitions: MacroDefinition[]; + usages: { id: string; wid: number; label: string; path: string[] }[]; + errors: string[]; +} +const workflowPayload = ({ wid, name, description, content }: Workflow) => ({ + wid, + name, + description, + content: JSON.stringify(content), +}); export const WORKFLOW_BASE_URL = "workflow"; export const WORKFLOW_PERSIST_URL = WORKFLOW_BASE_URL + "/persist"; @@ -153,6 +172,11 @@ export class WorkflowPersistService { * @param workflow */ public persistWorkflow(workflow: Workflow): Observable { + try { + assertWorkflowIDs(workflow.content); + } catch (error) { + return throwError(() => error); + } if (checkIfWorkflowBroken(workflow)) { this.notificationService.error( "Sorry! The workflow is broken and cannot be persisted. Please contact the system admin." @@ -164,12 +188,7 @@ export class WorkflowPersistService { // and it is not reliably known here anyway, since the metadata fed back after a save names // it differently (see WorkflowUtilService.parseWorkflowInfo). const send = this.http - .post(`${AppSettings.getApiEndpoint()}/${WORKFLOW_PERSIST_URL}`, { - wid: workflow.wid, - name: workflow.name, - description: workflow.description, - content: JSON.stringify(workflow.content), - }) + .post(`${AppSettings.getApiEndpoint()}/${WORKFLOW_PERSIST_URL}`, workflowPayload(workflow)) .pipe( filter((updatedWorkflow: Workflow) => updatedWorkflow != null), map(WorkflowUtilService.parseWorkflowInfo) @@ -182,6 +201,23 @@ export class WorkflowPersistService { return result.asObservable(); } + public persistMacroPath(steps: MacroSaveStep[], rootBefore: Workflow) { + return this.http + .post<{ version: number; revision: number; content: WorkflowContent }[]>( + `${AppSettings.getApiEndpoint()}/${WORKFLOW_PERSIST_URL}/macro-path`, + { + steps: steps.map(s => ({ ...s, workflow: workflowPayload(s.workflow) })), + rootBefore: workflowPayload(rootBefore), + } + ) + .pipe( + catchError((error: unknown) => { + if (error instanceof HttpErrorResponse && error.error?.message) throw new Error(error.error.message); + throw error; + }) + ); + } + /** * creates a workflow and insert it to backend database and return its information * @param newWorkflowName @@ -192,6 +228,11 @@ export class WorkflowPersistService { newWorkflowName: string = DEFAULT_WORKFLOW_NAME, defaultView?: DefaultView ): Observable { + try { + assertWorkflowIDs(newWorkflowContent); + } catch (error) { + return throwError(() => error); + } return this.http .post(`${AppSettings.getApiEndpoint()}/${WORKFLOW_CREATE_URL}`, { name: newWorkflowName, @@ -242,6 +283,10 @@ export class WorkflowPersistService { /** * retrieves a list of workflows from backend database that belongs to the user in the session. */ + public retrieveMacroCatalog(): Observable { + return this.http.get(`${AppSettings.getApiEndpoint()}/${WORKFLOW_BASE_URL}/macros`); + } + public retrieveWorkflowsBySessionUser(): Observable { return this.makeRequestAndFormatWorkflowResponse(`${AppSettings.getApiEndpoint()}/${WORKFLOW_LIST_URL}`); } diff --git a/frontend/src/app/common/testing/macro-graph.ts b/frontend/src/app/common/testing/macro-graph.ts new file mode 100644 index 00000000000..d9399af3290 --- /dev/null +++ b/frontend/src/app/common/testing/macro-graph.ts @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { macroGraph } from "../../../../../common/workflow-core/src/test/typescript/macro-test-graph"; +import { + isMacroTerminal, + MacroInterface, + MacroExpansion, + MacroDefinition, + MacroPort, + macroRuntimeID, + macroSchema, +} from "../util/workflow-macro"; +import type { WorkflowActionService } from "../../workspace/service/workflow-graph/model/workflow-action.service"; +import { WorkflowContent } from "../type/workflow"; +import { testField, testOperator, testWorkflow } from "./workflow-graph"; + +import { OperatorLink, OperatorPredicate } from "../../workspace/types/workflow-common.interface"; +import { OperatorSchema } from "../../workspace/types/operator-schema.interface"; + +export const testPort = ( + portID: string, + operatorID: string, + internalPort = portID, + displayName?: string +): MacroPort => ({ + portID, + ...(displayName === undefined ? {} : { displayName }), + targets: [{ operatorID, portID: internalPort }], +}); + +type MacroFixtureOptions = Partial & + Partial> & + Partial>; +export function testMacro( + id: string, + operators: OperatorPredicate[], + options?: C +): MacroDefinition & { content: WorkflowContent & Omit }; +export function testMacro( + id: string, + operators: OperatorPredicate[], + options: MacroFixtureOptions = {} +): MacroDefinition { + const { version = 1, name = id, wid, inputs = [], outputs = [], ...body } = options; + return { + id, + version, + name, + ...(wid === undefined ? {} : { wid }), + content: macroGraph( + testWorkflow(operators, body.links ?? [], { ...body, operators, macroInterface: { id, inputs, outputs } }) + ), + }; +} + +/** Expose every native field through two saved macro levels for rendered Formly tests. */ +export function testNestedMacroSchema(native: OperatorSchema, values = {}): OperatorSchema { + let schema = native; + for (const depth of [1, 2]) { + const previous = schema; + const inner = testOperator("inner", { + operatorType: previous.operatorType, + operatorProperties: depth === 1 ? values : {}, + }); + const fields = Object.keys(previous.jsonSchema.properties ?? {}).map((key, index) => + testField(inner.operatorID, key, `field-${depth}-${index}`, key) + ); + const definition = testMacro(`macro-${depth}`, [inner], { version: depth, formBinding: { fields } }); + schema = macroSchema(definition, () => previous); + } + return schema; +} + +/** Construct a backend response from explicit native leaves; never resolve or validate a macro graph. */ +export function expansionFixture( + content: WorkflowContent, + leaves: [string[], OperatorPredicate][], + links: OperatorLink[] = [], + ports: MacroExpansion["ports"] = {} +): MacroExpansion & { inspectors: NonNullable } { + const entries = leaves.map(([path, op]) => [macroRuntimeID(path.slice(0, -1), path.at(-1)!), path, op] as const); + const inspectors: NonNullable = { "[]": { ids: {}, inputs: {}, terminals: [] } }; + for (const [runtime, path] of entries) { + for (let depth = 0; depth < path.length; depth++) { + const scope = path.slice(0, depth); + const view = (inspectors[JSON.stringify(scope)] ??= { ids: {}, inputs: {}, terminals: [] }); + view.ids[path[depth]] = macroRuntimeID(scope, path[depth]); + view.ids[macroRuntimeID(path.slice(depth, -1), path.at(-1)!)] = runtime; + } + } + return structuredClone({ + inspectors, + content: { + ...content, + operators: entries.map(([operatorID, , op]) => ({ ...op, operatorID })), + links, + macroInterface: undefined, + macroDependencies: undefined, + }, + paths: Object.fromEntries(entries.map(([id, path]) => [id, path])), + ports, + }); +} + +/** Build a test interface using the same normal graph operations as the editor. */ +export function setMacroGraph(actions: WorkflowActionService, boundary: MacroInterface): void { + const old = actions.getWorkflowContent(); + const terminalIDs = new Set(old.operators.filter(isMacroTerminal).map(op => op.operatorID)); + const graph = macroGraph({ + ...old, + macroInterface: boundary, + }); + const terminals = graph.operators.filter(isMacroTerminal); + actions.bundleMacroActions(() => { + actions.deleteOperatorsAndLinks([...terminalIDs]); + actions.setMacroInterface({ id: boundary.id }); + actions.addOperatorsAndLinks( + terminals.map(op => ({ op, pos: graph.operatorPositions[op.operatorID] })), + graph.links.filter(link => + terminals.some(op => op.operatorID === link.source.operatorID || op.operatorID === link.target.operatorID) + ) + ); + }); +} diff --git a/frontend/src/app/common/testing/workflow-graph.ts b/frontend/src/app/common/testing/workflow-graph.ts new file mode 100644 index 00000000000..8572b268774 --- /dev/null +++ b/frontend/src/app/common/testing/workflow-graph.ts @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ExecutionMode, FormBindingConfig, WorkflowContent } from "../type/workflow"; +import { OperatorLink, OperatorPredicate } from "../../workspace/types/workflow-common.interface"; + +export const testOperator = (operatorID: string, extra: Partial = {}): OperatorPredicate => ({ + operatorID, + operatorType: "Filter", + operatorProperties: {}, + operatorVersion: "1", + showAdvanced: false, + inputPorts: [{ portID: "input-0" }], + outputPorts: [{ portID: "output-0" }], + ...extra, +}); + +export const testLink = ( + source: string, + target: string, + output = "output-0", + input = "input-0", + linkID = `${source}-${target}` +): OperatorLink => ({ + linkID, + source: { operatorID: source, portID: output }, + target: { operatorID: target, portID: input }, +}); + +export function testWorkflow>( + operators: OperatorPredicate[], + links: OperatorLink[], + extra: C +): WorkflowContent & C; +export function testWorkflow(operators?: OperatorPredicate[], links?: OperatorLink[]): WorkflowContent; +export function testWorkflow( + operators: OperatorPredicate[] = [], + links: OperatorLink[] = [], + extra: Partial = {} +): WorkflowContent { + return { + operators, + links, + operatorPositions: Object.fromEntries(operators.map(op => [op.operatorID, { x: 0, y: 0 }])), + commentBoxes: [], + settings: { dataTransferBatchSize: 10, executionMode: ExecutionMode.PIPELINED }, + ...extra, + }; +} + +export const testField = (operatorID: string, propertyKey: string, id = propertyKey, displayName = propertyKey) => ({ + id, + operatorID, + propertyKey, + displayName, +}); + +export const testBinding = (...args: Parameters): FormBindingConfig => ({ + fields: [testField(...args)], +}); diff --git a/frontend/src/app/common/type/workflow.ts b/frontend/src/app/common/type/workflow.ts index 701086e2a71..da40788d39d 100644 --- a/frontend/src/app/common/type/workflow.ts +++ b/frontend/src/app/common/type/workflow.ts @@ -19,6 +19,7 @@ import { DefaultView, WorkflowMetadata } from "../../dashboard/type/workflow-metadata.interface"; import { CommentBox, OperatorLink, OperatorPredicate, Point } from "../../workspace/types/workflow-common.interface"; +import type { MacroDefinition } from "../util/workflow-macro"; export enum ExecutionMode { PIPELINED = "PIPELINED", @@ -101,6 +102,10 @@ export interface WorkflowContent /** Present once an author set up the Form View. Rides in the content (like `settings`), * so it is saved/cloned/versioned/published with the workflow for free. */ formBinding?: FormBindingConfig; + /** A reusable definition; this graph runs only as part of a workflow. */ + macroInterface?: { id: string }; + /** Immutable, transitively bundled versions, keyed by definition id and version. */ + macroDependencies?: MacroDefinition[]; }> {} export type Workflow = { content: WorkflowContent } & WorkflowMetadata; diff --git a/frontend/src/app/common/util/assert.ts b/frontend/src/app/common/util/assert.ts index 93695b61b9b..0528efdf7d9 100644 --- a/frontend/src/app/common/util/assert.ts +++ b/frontend/src/app/common/util/assert.ts @@ -22,6 +22,12 @@ * They are used to provide type hints to help Typescript analyze code. */ +export function assert(condition: unknown, message?: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + interface Primitives { number: number; boolean: boolean; diff --git a/frontend/src/app/common/util/workflow-check.spec.ts b/frontend/src/app/common/util/workflow-check.spec.ts index e58e1df0b07..f84d422002a 100644 --- a/frontend/src/app/common/util/workflow-check.spec.ts +++ b/frontend/src/app/common/util/workflow-check.spec.ts @@ -16,80 +16,33 @@ * specific language governing permissions and limitations * under the License. */ - -import { ExecutionMode } from "../type/workflow"; +import { testWorkflow, testLink, testOperator } from "../testing/workflow-graph"; +import { DEFAULT_WORKFLOW } from "../../workspace/service/workflow-graph/model/workflow-action.service"; import { checkIfWorkflowBroken } from "./workflow-check"; - -type WorkflowInput = Parameters[0]; - -function createWorkflow( - operatorIDs: ReadonlyArray, - links: ReadonlyArray<{ - sourceOperatorID: string; - targetOperatorID: string; - }> -): WorkflowInput { - return { - name: "test-workflow", - description: undefined, - wid: undefined, - creationTime: undefined, - lastModifiedTime: undefined, - isPublished: 0, - readonly: false, - content: { - operators: operatorIDs.map(operatorID => ({ - operatorID, - operatorType: "TestOperator", - operatorVersion: "0", - operatorProperties: {}, - inputPorts: [], - outputPorts: [], - showAdvanced: false, - })), - operatorPositions: {}, - links: links.map(({ sourceOperatorID, targetOperatorID }, index) => ({ - linkID: `link-${index}`, - source: { operatorID: sourceOperatorID, portID: "output-0" }, - target: { operatorID: targetOperatorID, portID: "input-0" }, - })), - commentBoxes: [], - settings: { dataTransferBatchSize: 1, executionMode: ExecutionMode.PIPELINED }, - }, - }; -} +import { assertWorkflowIDs } from "@texera/workflow-macros"; +import { graphIDCases } from "../../../../../common/workflow-core/src/test/typescript/macro-test-graph"; + +describe("workflow graph identities", () => { + for (const example of graphIDCases) { + it(example.name, () => { + const before = JSON.stringify(example.content); + if (example.error) expect(() => assertWorkflowIDs(example.content)).toThrow(example.error); + else expect(() => assertWorkflowIDs(example.content)).not.toThrow(); + expect(JSON.stringify(example.content)).toBe(before); + }); + } +}); describe("checkIfWorkflowBroken", () => { - it("should return false when every link references existing operators", () => { - const workflow = createWorkflow( - ["operator-1", "operator-2"], - [{ sourceOperatorID: "operator-1", targetOperatorID: "operator-2" }] - ); - - expect(checkIfWorkflowBroken(workflow)).toBe(false); - }); - - it("should return true when a link source references a missing operator", () => { - const workflow = createWorkflow( - ["operator-2"], - [{ sourceOperatorID: "missing-operator", targetOperatorID: "operator-2" }] - ); - - expect(checkIfWorkflowBroken(workflow)).toBe(true); - }); - - it("should return true when a link target references a missing operator", () => { - const workflow = createWorkflow( - ["operator-1"], - [{ sourceOperatorID: "operator-1", targetOperatorID: "missing-operator" }] - ); - - expect(checkIfWorkflowBroken(workflow)).toBe(true); - }); - - it("should return false when the workflow has no links", () => { - const workflow = createWorkflow(["operator-1"], []); - - expect(checkIfWorkflowBroken(workflow)).toBe(false); + const operators = ["source", "target"].map(id => testOperator(id)); + + it.each([ + { scenario: "connected operators", links: [testLink("source", "target")], broken: false }, + { scenario: "missing source", links: [testLink("missing", "target")], broken: true }, + { scenario: "missing target", links: [testLink("source", "missing")], broken: true }, + { scenario: "no links", links: [], broken: false }, + ])("checks links for $scenario", ({ links, broken }) => { + const content = testWorkflow(operators, links); + expect(checkIfWorkflowBroken({ ...DEFAULT_WORKFLOW, content })).toBe(broken); }); }); diff --git a/frontend/src/app/common/util/workflow-macro.spec.ts b/frontend/src/app/common/util/workflow-macro.spec.ts new file mode 100644 index 00000000000..0ec79b0cc9c --- /dev/null +++ b/frontend/src/app/common/util/workflow-macro.spec.ts @@ -0,0 +1,496 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { testPort, testMacro, expansionFixture } from "../testing/macro-graph"; +import { + testField, + testBinding, + testLink as link, + testOperator, + testWorkflow as graph, +} from "../testing/workflow-graph"; + +import { editMacroGraph } from "../../../../../common/workflow-core/src/test/typescript/macro-test-graph"; + +import { WorkflowContent, exportedWorkflow } from "../type/workflow"; +import { DefaultView } from "../../dashboard/type/workflow-metadata.interface"; +import { OperatorPredicate } from "../../workspace/types/workflow-common.interface"; +import { + mergeMacroDefinitions, + extractMacro, + macroInstance, + macroType, + macroSchema, + adoptMacroVersion, + replaceMacroVersion, + macroRevisionLabel, + macroRuntimeID, + projectMacroExpansion, + compiledInputLinks, + portKey, + MacroDefinition, + isMacroTerminal, + readMacroInterface, +} from "./workflow-macro"; + +import { OperatorSchema } from "../../workspace/types/operator-schema.interface"; +import { mockScanSourceSchema } from "../../workspace/service/operator-metadata/mock-operator-metadata.data"; +import { setFieldBehavior, setHideExpression } from "../formly/formly-utils"; +import { propertyModel } from "../../workspace/util/custom-formly-type"; +import { FormlyFieldConfig } from "@ngx-formly/core"; + +const op = (id: string) => testOperator(id, { operatorProperties: { value: "default" } }); +const nativeSchema = (jsonSchema: OperatorSchema["jsonSchema"]): OperatorSchema => ({ + ...mockScanSourceSchema, + jsonSchema, +}); +const definition = (id = "clean", version = 5) => + testMacro(id, [op("a"), op("b")], { + version, + name: "清理", + wid: 12, + links: [link("a", "b")], + inputs: [testPort("input-0", "a", "input-0", "Rows")], + outputs: [testPort("output-0", "b", "output-0", "Clean")], + formBinding: testBinding("a", "value", "value", "Value"), + }); + +const configuredInstance = ( + d: MacroDefinition, + id: string, + operatorProperties: OperatorPredicate["operatorProperties"] +) => ({ ...macroInstance(d, id), operatorProperties }); +const macroWithFields = (id: string, operator: OperatorPredicate, ...fields: ReturnType[]) => + testMacro(id, [operator], { formBinding: { fields } }); + +function conflictingBody(definition: MacroDefinition): MacroDefinition { + const copy = structuredClone(definition); + copy.content.operators[0] = { ...copy.content.operators[0], operatorProperties: { value: "different" } }; + return copy; +} + +describe("workflow macros", () => { + it("keeps native sibling defaults and required rules through nested exposure and Form View rekeying", () => { + const leaf = macroWithFields( + "leaf", + testOperator("a", { operatorProperties: { fileName: "old", unexposed: "東京" } }), + testField("a", "enabled", "flag", "Enabled"), + testField("a", "fileName", "name", "File") + ); + const native = nativeSchema({ + type: "object", + properties: { enabled: { type: "boolean", default: true }, fileName: { type: "string" } }, + allOf: [{ if: { properties: { enabled: { const: true } } }, then: { required: ["fileName"] } }], + }); + const inner = macroSchema(leaf, () => native); + const innerProperty = inner.jsonSchema.properties!.name as any; + expect(propertyModel({ key: "name", ...innerProperty.widget.formlyConfig, parent: { model: {} } })).toMatchObject({ + enabled: true, + }); + const parent = macroWithFields( + "parent", + configuredInstance(leaf, "child", { flag: false, name: "child" }), + testField("child", "name", "public-file", "File") + ); + const pins = JSON.stringify([leaf, parent]); + const schema = macroSchema(parent, () => inner).jsonSchema; + const property = schema.properties!["public-file"] as any; + const field: FormlyFieldConfig = { + key: "public-file", + ...property.widget.formlyConfig, + parent: { model: { "public-file": "" } }, + }; + setFieldBehavior(field, property); + const required = field.expressions!["props.required"] as (f: FormlyFieldConfig) => boolean; + expect(required(field)).toBe(false); + expect(propertyModel(field)).toEqual({ enabled: false, fileName: "", unexposed: "東京" }); + field.key = "form-view-binding"; + field.parent!.model["form-view-binding"] = null; + expect(propertyModel(field).fileName).toBeNull(); + expect(propertyModel(field).enabled).toBe(false); + expect(JSON.stringify([leaf, parent])).toBe(pins); + }); + it.each([undefined, { fields: [] }])("removes unavailable forwarding only on the adopted usage: %j", formBinding => { + const child = definition(); + const parent = testMacro( + "parent", + [configuredInstance(child, "child", { value: "mine" }), configuredInstance(child, "sibling", { value: "東京" })], + { + macroDependencies: [child], + formBinding: { + fields: [testField("child", "value", "forward"), testField("sibling", "value", "kept")], + shownResultIds: ["child"], + }, + } + ).content; + const before = JSON.stringify(parent); + const removed = { ...child, version: 6, content: { ...child.content, formBinding } }; + const saved = replaceMacroVersion(parent, "child", removed); + expect(saved.formBinding).toEqual({ fields: [parent.formBinding.fields[1]], shownResultIds: ["child"] }); + expect(saved.operators[0].operatorProperties).toEqual({}); + expect(saved.operators[1]).toEqual(parent.operators[1]); + expect(saved.macroDependencies!.find(d => macroType(d) === macroType(child))).toEqual(child); + expect(JSON.stringify(parent)).toBe(before); + }); + + it("keeps native widgets and sibling visibility through public field IDs", () => { + const scan = testOperator("a", { + operatorType: "FileScanOp", + operatorProperties: { progressive: false, fileName: "a.csv", min: 1 }, + }); + const d = macroWithFields( + "scan", + scan, + testField("a", "fileName", "field-file", "File"), + testField("a", "progressive", "field-progressive", "Progressive"), + testField("a", "min", "field-min", "Minimum") + ); + const native = nativeSchema({ + properties: { + fileName: { type: "string" }, + progressive: { type: "boolean", toggleHidden: ["min"] }, + min: { type: "number" }, + }, + }); + const schema = macroSchema(d, () => native).jsonSchema; + const properties = schema.properties! as Record; + expect(properties["field-file"].widget?.formlyConfig?.type).toBe("inputautocomplete"); + const model = { "field-progressive": false, "field-min": 2 }; + const fields = Object.entries(properties).map( + ([key, value]) => + ({ key, props: value.widget?.formlyConfig?.props ?? {}, parent: { model } }) as FormlyFieldConfig + ); + setHideExpression(["min"], fields, "field-progressive"); + const hide = fields.find(f => f.key === "field-min")!.expressions!.hide as (f: any) => boolean; + expect(hide(fields[2])).toBe(true); + model["field-progressive"] = true; + expect(hide(fields[2])).toBe(false); + }); + + it.each([DefaultView.CANVAS, DefaultView.FORM])( + "exports selected and nested usages with their captured revisions in %s", + view => { + const previous = definition(), + next = definition("clean", 64); + next.content.operators[0] = { ...next.content.operators[0], operatorProperties: { value: "new default" } }; + const parent = testMacro("outer", [macroInstance(previous, "nested")], { + version: 8, + wid: 12, + inputs: [testPort("input-0", "nested")], + outputs: [testPort("output-0", "nested")], + }); + const pins = [previous, parent, definition("unused")]; + const content = graph( + [macroInstance(previous, "chosen"), macroInstance(previous, "sibling"), macroInstance(parent, "outer")], + [], + { macroDependencies: pins } + ); + const before = structuredClone(content); + const updated = replaceMacroVersion(content, "chosen", next); + expect(updated.operators.map(o => o.operatorType)).toEqual([ + macroType(next), + macroType(previous), + macroType(parent), + ]); + expect(updated.operatorPositions).toEqual(content.operatorPositions); + expect(() => replaceMacroVersion(content, "missing", next)).toThrow(/missing/i); + const { defaultView, ...uploaded } = JSON.parse(JSON.stringify(exportedWorkflow(updated, view))); + expect(defaultView).toBe(view); + expect(uploaded).toEqual(updated); + expect(uploaded.macroDependencies).toEqual(expect.arrayContaining([...pins, next])); + for (const d of uploaded.macroDependencies) delete d.wid; + expect(content).toEqual(before); + } + ); + + it("removes only obsolete connections and forwards surviving boundaries when an interface shrinks", () => { + const previous = definition(), + next = definition("clean", 6); + editMacroGraph(next.content, boundary => { + boundary.inputs = []; + boundary.outputs = []; + }); + next.content.formBinding.fields = []; + const chosen = configuredInstance(previous, "chosen", { value: "mine" }); + const sibling = macroInstance(previous, "sibling"); + const shared = { + portID: "shared", + targets: [chosen, sibling].map(({ operatorID }) => ({ operatorID, portID: "input-0" })), + }; + const content = testMacro("outer", [op("source"), chosen, sibling, op("sink")], { + links: [link("source", "chosen"), link("chosen", "sink"), link("source", "sibling")], + macroDependencies: [previous], + inputs: [shared], + outputs: [testPort("removed", "chosen", "output-0")], + formBinding: { + fields: [testField("chosen", "value", "removed", "Removed"), testField("sibling", "value", "kept", "Kept")], + }, + }).content; + const snapshot = JSON.stringify(content); + const updated = replaceMacroVersion(content, "chosen", next); + expect(updated.links.filter(link => ["source", "chosen", "sibling"].includes(link.source.operatorID))).toEqual([ + link("source", "sibling"), + ]); + expect(readMacroInterface(updated)).toEqual({ + id: "outer", + inputs: [testPort("shared", "sibling", "input-0")], + outputs: [{ portID: "removed", targets: [] }], + }); + expect(updated.formBinding).toEqual({ fields: [content.formBinding!.fields[1]] }); + expect(updated.operators[1].operatorProperties).toEqual({}); + expect(updated.operators[2]).toEqual(sibling); + expect(JSON.stringify(content)).toBe(snapshot); + const { defaultView } = JSON.parse(JSON.stringify(exportedWorkflow(updated, DefaultView.FORM))); + expect(defaultView).toBe(DefaultView.FORM); + + const root = { ...content, macroInterface: undefined }; + expect(replaceMacroVersion(root, "chosen", next).formBinding).toEqual(updated.formBinding); + }); + + it("labels per-workflow revisions without exposing a global history ID as a version number", () => { + expect(macroRevisionLabel({ ...definition("clean", 64), revision: 3 })).toBe("Revision 3"); + expect(macroRevisionLabel(definition("clean", 64))).toBe("Saved revision"); + }); + + it("deduplicates equivalent definitions regardless of JSON key order but rejects differing content", () => { + const d = definition(); + const equivalent = { ...d, content: Object.fromEntries(Object.entries(d.content).reverse()) as WorkflowContent }; + expect(mergeMacroDefinitions([d, equivalent])).toEqual([d]); + expect(() => mergeMacroDefinitions([d, conflictingBody(d)])).toThrow(/conflicting/i); + }); + + it("carries referenced property schemas into exposed macro properties", () => { + const d = definition(); + const schema = nativeSchema({ + properties: { value: { $ref: "#/definitions/value" } }, + definitions: { value: { type: "string", minLength: 1 } }, + }); + expect(macroSchema(d, () => schema).jsonSchema.properties?.value).toMatchObject({ type: "string", minLength: 1 }); + expect(macroSchema(d, () => nativeSchema({ properties: {} })).jsonSchema.properties).toEqual({}); + }); + + it("keeps compiled column choices specific to each exposed operator and reuses its schema across fields", () => { + const d = definition(); + d.content.formBinding.fields.push( + testField("a", "value", "same", "Again"), + testField("b", "value", "other", "Other") + ); + const before = JSON.stringify(d); + const compiled = vi.fn((op: OperatorPredicate) => + nativeSchema({ properties: { value: { type: "string", enum: [op.operatorID] } } }) + ); + const result = macroSchema(d, () => nativeSchema({}), compiled).jsonSchema.properties; + expect(result).toMatchObject({ value: { enum: ["a"] }, same: { enum: ["a"] }, other: { enum: ["b"] } }); + expect(compiled).toHaveBeenCalledTimes(2); + expect(JSON.stringify(d)).toBe(before); + }); + + it("resolves shared property schemas once through deep nesting and rereads changed metadata", () => { + const definitions = new Map>(); + let d = definition("depth-0"); + d.content.formBinding.fields.push(testField("b", "value", "other", "Other")); + definitions.set(macroType(d), d); + for (let depth = 1; depth <= 10; depth++) { + const child = d; + d = testMacro(`depth-${depth}`, [macroInstance(child, "child")], { + inputs: [testPort("input-0", "child", "input-0", "Rows")], + outputs: [testPort("output-0", "child", "output-0", "Clean")], + formBinding: { fields: ["value", "other"].map(id => testField("child", id)) }, + }); + definitions.set(macroType(d), d); + } + const leaf = vi.fn(() => nativeSchema({ properties: { value: { type: "string", default: "original" } } })); + const schema = (type: string): OperatorSchema => + definitions.has(type) ? macroSchema(definitions.get(type)!, schema) : leaf(); + expect(schema(macroType(d)).jsonSchema.properties?.other).toMatchObject({ type: "string", default: "default" }); + expect(leaf).toHaveBeenCalledTimes(1); + leaf.mockReturnValue(nativeSchema({ properties: {} })); + expect(schema(macroType(d)).jsonSchema.properties).toEqual({}); + }); + + it("maps exposed attribute pickers to the macro's boundary port instead of the inner port ordinal", () => { + const d = definition(); + d.content.operators[0].inputPorts.push({ portID: "input-1" }); + editMacroGraph(d.content, boundary => { + boundary.inputs = [testPort("input-0", "a", "input-1", "Right"), testPort("input-1", "a", "input-0", "Left")]; + }); + const schema = nativeSchema({ + properties: { value: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 1 } }, + }); + expect((macroSchema(d, () => schema).jsonSchema.properties!.value as any).autofillAttributeOnPort).toBe(0); + editMacroGraph(d.content, boundary => { + boundary.inputs = []; + }); + expect((macroSchema(d, () => schema).jsonSchema.properties!.value as any).autofillAttributeOnPort).toBe(-1); + }); + + it("extracts a connected selection with a portable terminal graph and its Form View bindings", () => { + const a = { ...op("a"), inputPorts: [{ portID: "in", displayName: "Rows 東京" }] }; + const b = { ...op("b"), outputPorts: [{ portID: "out" }] }; + const content = graph( + [op("s"), a, b, op("t")], + [link("s", "a", "output-0", "in"), link("a", "b"), link("b", "t", "out")], + { + formBinding: testBinding("a", "value", "value", "Value"), + } + ); + const extracted = extractMacro(content, ["a", "b"], "clean"); + expect(extracted.operators.filter(op => !isMacroTerminal(op)).map(o => o.operatorID)).toEqual(["a", "b"]); + expect(extracted.links.filter(link => link.source.operatorID === "a")).toEqual([link("a", "b")]); + expect(extracted.operators).toHaveLength(4); + expect(extracted.links).toHaveLength(3); + expect(extracted.macroInterface).toEqual({ id: "clean" }); + const before = JSON.stringify(extracted); + expect(readMacroInterface(extracted)?.inputs[0].targets).toEqual([{ operatorID: "a", portID: "in" }]); + expect(readMacroInterface(extracted)?.outputs[0].targets).toEqual([{ operatorID: "b", portID: "out" }]); + const copy = { id: "clean", version: 2, name: "M", content: JSON.parse(before) }; + expect(macroInstance(copy, "usage").inputPorts[0]).toMatchObject({ portID: "input-0", displayName: "Rows 東京" }); + expect(JSON.stringify(extracted)).toBe(before); + expect(extracted.formBinding).toEqual(content.formBinding); + expect(() => extractMacro(content, [], "empty")).toThrow(/select/i); + expect(() => extractMacro(content, ["missing"], "bad")).toThrow(/missing/i); + expect(() => extractMacro(content, ["s", "b"], "nonconvex")).toThrow(/selection/i); + }); + + it("keeps port identity through rename and disconnection, including duplicate or missing terminal drafts", () => { + const body = definition().content; + const input = body.operators.find(op => op.operatorType === "MacroInput")!; + input.outputPorts[0] = { ...input.outputPorts[0], displayName: "New" }; + const renamed = { ...body, links: [] }; + expect(readMacroInterface(renamed)?.inputs[0]).toMatchObject({ + portID: "input-0", + displayName: "New", + targets: [], + }); + expect( + readMacroInterface({ ...body, operators: [...body.operators, { ...input, operatorID: "other" }] })?.inputs.length + ).toBe(2); + expect(readMacroInterface({ ...body, operators: body.operators.filter(op => op !== input) })?.inputs).toEqual([]); + expect(readMacroInterface({ ...body, macroInterface: undefined })).toBeUndefined(); + }); + + it("adopts interface changes explicitly while preserving values and checking macro identity", () => { + const old = definition(); + const instance = configuredInstance(old, "m", { value: "mine" }); + const next = definition("clean", 6); + const updated = adoptMacroVersion(instance, old, next); + expect(updated.operatorType).toBe(macroType(next)); + expect(updated.operatorProperties).toEqual({ value: "mine" }); + expect(instance.operatorType).toBe(macroType(old)); + expect(() => adoptMacroVersion(instance, old, definition("other", 6))).toThrow(/definition/i); + editMacroGraph(next.content, boundary => { + boundary.outputs = []; + }); + expect(adoptMacroVersion(instance, old, next).outputPorts).toEqual([]); + }); + + it("adopts additive interface edits while keeping existing ports, values and snapshots", () => { + const previous = definition(), + before = JSON.stringify(previous); + const next = structuredClone(previous); + next.version++; + editMacroGraph(next.content, boundary => { + boundary.inputs.push(testPort("additional", "b", "input-0", "Additional")); + }); + next.content.formBinding.fields.push(testField("b", "value", "extra", "Extra")); + const instance = { ...macroInstance(previous, "m"), operatorProperties: { value: "mine" }, viewResult: true }; + const updated = adoptMacroVersion(instance, previous, next); + expect(updated.inputPorts.map(p => p.portID)).toEqual(["input-0", "additional"]); + expect(updated.outputPorts).toEqual(instance.outputPorts); + expect(updated.operatorProperties).toEqual({ value: "mine" }); + expect(updated.viewResult).toBe(true); + expect(JSON.stringify(previous)).toBe(before); + expect(instance.operatorType).toBe(macroType(previous)); + const reordered = structuredClone(next); + reordered.version++; + reordered.content.formBinding.fields.reverse(); + editMacroGraph(reordered.content, boundary => boundary.inputs.reverse()); + const reorderedUsage = adoptMacroVersion(updated, next, reordered); + expect(reorderedUsage.operatorProperties).toEqual({ value: "mine" }); + expect(reorderedUsage.inputPorts.map(p => p.portID)).toEqual(["additional", "input-0"]); + next.content.formBinding.fields = []; + expect(adoptMacroVersion(instance, previous, next).operatorProperties).toEqual({}); + }); + + it("rejects a compiler response missing its root index even when a nested scope is requested", () => { + const content = graph(); + const incomplete = { content, paths: {}, ports: {}, inspectors: {} }; + for (const path of [[], ["missing"]]) + expect(() => projectMacroExpansion(content, incomplete, path)).toThrow(/missing.*root/i); + }); + + it("keeps the compiler's native result port when projecting a nested usage", () => { + const d = definition(); + const graphWithUsage = graph([macroInstance(d, "inner")], [], { macroDependencies: [d] }); + const compiled = expansionFixture(graphWithUsage, [[["outer", "inner", "b"], op("b")]]); + const localID = macroRuntimeID(["inner"], "b"); + const runtimeMacro = macroRuntimeID(["outer"], "inner"); + const runtimeLeaf = compiled.content.operators[0].operatorID; + compiled.resultTargets = { [runtimeMacro]: { operatorID: runtimeLeaf, portIndex: 1 } }; + compiled.ports[portKey(runtimeMacro, "outputs", "output-0")] = [{ operatorID: runtimeLeaf, portID: "out" }]; + const view = projectMacroExpansion(graphWithUsage, compiled, ["outer"]).local; + expect(view.resultTargets).toEqual({ inner: { operatorID: localID, portIndex: 1 } }); + expect(view.ports[portKey("inner", "outputs", "output-0")]).toEqual([{ operatorID: localID, portID: "out" }]); + }); + it("reads the compiler's distinct sources for two exposed inputs that converge on one native port", () => { + const ports = ["left", "right"]; + const usage = testOperator("usage", { inputPorts: ports.map(portID => ({ portID })) }); + const compiled = expansionFixture( + graph([usage]), + [], + ports.map(id => link(id, "same-native-target")) + ); + compiled.inspectors["[]"].inputs.usage = ports.map((_, i) => ({ link: i, input: i, output: i })); + expect(compiledInputLinks(compiled, [], usage)).toMatchObject( + ports.map((id, i) => ({ + source: { operatorID: id, portIndex: i }, + target: { operatorID: "usage", portID: id, portIndex: i }, + })) + ); + }); +}); + +describe("macro bundle normalization", () => { + it("reads legacy nested bundles and compares immutable bodies independently of packaging", () => { + const child = definition("child", 1); + const parent = testMacro("parent", [macroInstance(child, "nested")], { macroDependencies: [child] }); + const before = structuredClone(parent); + const flat = mergeMacroDefinitions([ + parent, + { ...parent, content: { ...parent.content, macroDependencies: undefined } }, + ]); + expect(flat).toHaveLength(2); + expect(flat.every(d => !d.content.macroDependencies)).toBe(true); + + expect(parent).toEqual(before); + expect(() => mergeMacroDefinitions([parent, conflictingBody(child)])).toThrow(/conflicting/i); + }); + it("keeps repeated extraction linear with a flat snapshot bundle", () => { + let content = graph([op("leaf")]); + for (let depth = 1; depth <= 12; depth++) { + const id = `level-${depth}`; + const body = extractMacro(content, [content.operators[0].operatorID], id); + const d = { id, version: 1, name: `Level ${depth}`, content: body }; + content = graph([macroInstance(d, `instance-${depth}`)], [], { + macroDependencies: mergeMacroDefinitions(content.macroDependencies ?? [], [d]), + }); + expect(content.macroDependencies!.every(d => !d.content.macroDependencies)).toBe(true); + } + expect(JSON.stringify(content).length).toBeLessThan(25000); + }); +}); diff --git a/frontend/src/app/common/util/workflow-macro.ts b/frontend/src/app/common/util/workflow-macro.ts new file mode 100644 index 00000000000..4b4977cec66 --- /dev/null +++ b/frontend/src/app/common/util/workflow-macro.ts @@ -0,0 +1,376 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { cloneDeep, groupBy, isEqual, memoize } from "lodash-es"; +import { WorkflowContent, FormFieldBinding } from "../type/workflow"; +import { LogicalPort, OperatorLink, OperatorPredicate } from "../../workspace/types/workflow-common.interface"; +import { OperatorSchema } from "../../workspace/types/operator-schema.interface"; +import { CustomJSONSchema7, PropertyContext } from "../../workspace/types/custom-json-schema.interface"; +import { conditionalRequiredRules, customFormlyFieldType } from "../../workspace/util/custom-formly-type"; +import { assert } from "./assert"; + +export { + assertWorkflowIDs, + isMacro, + isMacroTerminal, + readMacroInterface, + macroType, + portKey, + macroInstance, +} from "@texera/workflow-macros"; +export type { MacroInterface, MacroPort } from "@texera/workflow-macros"; +import { + isMacro, + isMacroTerminal, + readMacroInterface, + macroType, + portKey, + macroInstance, + createMacroRuntime, +} from "@texera/workflow-macros"; +import { v5 as uuid } from "uuid"; +import type { MacroDefinition as Definition, MacroExpansion as Expansion, MacroPort } from "@texera/workflow-macros"; +const macros = createMacroRuntime(uuid, isEqual); +export const { macroRuntimeID } = macros; +export type MacroDefinition = Definition; +export type MacroExpansion = Expansion; +export const mergeMacroDefinitions = macros.mergeMacroDefinitions; +const fields = (content?: WorkflowContent): FormFieldBinding[] => content?.formBinding?.fields ?? []; + +/** Project a backend expansion into the breadcrumb document without expanding its definitions again. */ +export function projectMacroExpansion(content: WorkflowContent, root: MacroExpansion, prefix: string[]) { + assert(root.inspectors?.["[]"], "Missing root compiler mapping"); + // Disabled occurrences and their descendants are absent from the executable graph. + const inspector = root.inspectors[JSON.stringify(prefix)]; + const ids = inspector?.ids ?? {}; + if (!prefix.length) return { local: root, ids, inspector }; + const boundaries = inspector ? content.operators.filter(op => isMacro(op) || isMacroTerminal(op)) : []; + const localIDs = new Map(Object.entries(ids).map(([local, runtime]) => [runtime, local])); + const endpoint = (port: LogicalPort) => ({ ...port, operatorID: localIDs.get(port.operatorID) ?? port.operatorID }); + const local: MacroExpansion = { + content: { + ...content, + macroInterface: undefined, + macroDependencies: undefined, + operators: root.content.operators + .filter(op => localIDs.has(op.operatorID)) + .map(op => ({ ...op, operatorID: localIDs.get(op.operatorID)! })), + links: root.content.links + .filter(link => localIDs.has(link.source.operatorID) && localIDs.has(link.target.operatorID)) + .map(link => ({ ...link, source: endpoint(link.source), target: endpoint(link.target) })), + }, + paths: Object.fromEntries( + Object.entries(ids).flatMap(([localID, runtime]) => + root.paths[runtime] ? [[localID, root.paths[runtime].slice(prefix.length)]] : [] + ) + ), + resultTargets: Object.fromEntries( + boundaries.flatMap(op => { + const runtime = ids[op.operatorID]; + const target = runtime && root.resultTargets?.[runtime]; + return target + ? [[op.operatorID, { ...target, operatorID: localIDs.get(target.operatorID) ?? target.operatorID }]] + : []; + }) + ), + ports: Object.fromEntries( + boundaries.flatMap(op => + (["inputs", "outputs"] as const).flatMap(direction => + (direction === "inputs" ? op.inputPorts : op.outputPorts).map(port => [ + portKey(op.operatorID, direction, port.portID), + (root.ports[portKey(ids[op.operatorID], direction, port.portID)] ?? []).map(endpoint), + ]) + ) + ) + ), + }; + return { local, ids, inspector }; +} +/** Read compiler-issued ordinals even when the source lives outside the visible canvas. */ +export function compiledInputLinks(root: MacroExpansion, path: string[], op: OperatorPredicate) { + const references = root.inspectors?.[JSON.stringify(path)]?.inputs[op.operatorID] ?? []; + return references.map(({ link, input, output }) => { + const edge = root.content.links[link]; + return { + ...edge, + source: { ...edge.source, portIndex: output }, + target: { operatorID: op.operatorID, portID: op.inputPorts[input].portID, portIndex: input }, + }; + }); +} +export const macroRevisionLabel = (d: MacroDefinition): string => + Number.isSafeInteger(d.revision) && d.revision! > 0 ? `Revision ${d.revision}` : "Saved revision"; + +/** Retarget one usage; retain every captured body, including the revision it used before. */ +export function replaceMacroVersion(content: WorkflowContent, id: string, next: MacroDefinition): WorkflowContent { + return { + ...retargetMacro(content, id, next), + macroDependencies: mergeMacroDefinitions(content.macroDependencies ?? [], [next]), + }; +} + +export function linkFitsMacroPorts(link: OperatorLink, macro: OperatorPredicate): boolean { + const fits = (end: LogicalPort, ports: OperatorPredicate["inputPorts"]) => + end.operatorID !== macro.operatorID || ports.some(port => port.portID === end.portID); + return fits(link.source, macro.outputPorts) && fits(link.target, macro.inputPorts); +} + +/** Update one interface; persistence attaches the committed definition after allocating its revision. */ +export function retargetMacro(content: WorkflowContent, id: string, next: MacroDefinition): WorkflowContent { + const op = content.operators.find(o => o.operatorID === id); + const previous = content.macroDependencies?.find(d => macroType(d) === op?.operatorType); + assert(op && previous, "Missing pinned macro definition"); + const updated = adoptMacroVersion(op, previous, next); + return { + ...content, + links: content.links.filter(link => linkFitsMacroPorts(link, updated)), + operators: content.operators.map(o => (o === op ? updated : o)), + ...(content.formBinding && { + formBinding: { + ...content.formBinding, + fields: fields(content).filter( + f => f.operatorID !== id || fields(next.content).some(p => p.id === f.propertyKey) + ), + }, + }), + }; +} + +/** Capture a convex selection; re-entering it through unselected nodes would create a macro cycle. */ +export function extractMacro(content: WorkflowContent, selected: readonly string[], id: string): WorkflowContent { + assert(selected.length > 0, "Select at least one operator"); + const ids = new Set(selected); + assert( + selected.every(id => content.operators.some(o => o.operatorID === id)), + "Missing selected operator" + ); + const escaped = new Set(); + const follow = (from: string): void => { + assert(!ids.has(from), "Selection must not leave and re-enter the macro"); + if (escaped.has(from)) return; + escaped.add(from); + content.links.filter(l => l.source.operatorID === from).forEach(l => follow(l.target.operatorID)); + }; + content.links + .filter(l => ids.has(l.source.operatorID) && !ids.has(l.target.operatorID)) + .forEach(l => follow(l.target.operatorID)); + const operators = content.operators.filter(o => ids.has(o.operatorID)); + const links = content.links.filter(l => ids.has(l.source.operatorID) && ids.has(l.target.operatorID)); + const boundary = (direction: "inputs" | "outputs"): MacroPort[] => { + const connections = groupBy(content.links, link => { + const end = direction === "inputs" ? link.target : link.source; + return portKey(end.operatorID, direction, end.portID); + }); + return operators + .flatMap(op => + (direction === "inputs" ? op.inputPorts : op.outputPorts) + .filter(p => { + const edges = connections[portKey(op.operatorID, direction, p.portID)] ?? []; + return ( + !edges.length || edges.some(l => !ids.has((direction === "inputs" ? l.source : l.target).operatorID)) + ); + }) + .map(p => ({ ...p, targets: [{ operatorID: op.operatorID, portID: p.portID }] })) + ) + .map((p, i) => ({ ...p, portID: `${direction === "inputs" ? "input" : "output"}-${i}` })); + }; + const inputPorts = boundary("inputs"), + outputPorts = boundary("outputs"); + const terminals: OperatorPredicate[] = [true, false].map(input => ({ + operatorID: macroRuntimeID([id], input ? "inputs" : "outputs"), + operatorType: input ? "MacroInput" : "MacroOutput", + operatorVersion: "1", + operatorProperties: {}, + inputPorts: input + ? [] + : outputPorts.map(({ targets, ...port }) => ({ ...port, disallowMultiInputs: true, isDynamicPort: true })), + outputPorts: input ? inputPorts.map(({ targets, ...port }) => ({ ...port, isDynamicPort: true })) : [], + dynamicInputPorts: !input, + dynamicOutputPorts: input, + showAdvanced: false, + })); + const points = operators.map(op => content.operatorPositions[op.operatorID]); + const terminalLinks = [inputPorts, outputPorts].flatMap((ports, index) => + ports.flatMap(port => + port.targets.map((target, n) => { + const end = { operatorID: terminals[index].operatorID, portID: port.portID }; + return { + linkID: macroRuntimeID([id, port.portID], String(n)), + source: index === 0 ? end : target, + target: index === 0 ? target : end, + }; + }) + ) + ); + return cloneDeep({ + ...content, + ...(content.macroDependencies ? { macroDependencies: mergeMacroDefinitions(content.macroDependencies) } : {}), + operators: [...operators, ...terminals], + links: [...links, ...terminalLinks], + operatorPositions: { + ...Object.fromEntries(operators.map(o => [o.operatorID, content.operatorPositions[o.operatorID]])), + [terminals[0].operatorID]: { x: Math.min(...points.map(p => p.x)) - 180, y: Math.min(...points.map(p => p.y)) }, + [terminals[1].operatorID]: { x: Math.max(...points.map(p => p.x)) + 180, y: Math.min(...points.map(p => p.y)) }, + }, + commentBoxes: [], + macroInterface: { id }, + formBinding: { fields: fields(content).filter(f => ids.has(f.operatorID)) }, + }); +} + +export function adoptMacroVersion( + op: OperatorPredicate, + previous: MacroDefinition, + next: MacroDefinition +): OperatorPredicate { + assert(previous.id === next.id && op.operatorType === macroType(previous), "Different macro definition"); + const updated = macroInstance(next, op.operatorID); + return { + ...op, + operatorType: updated.operatorType, + operatorVersion: updated.operatorVersion, + inputPorts: updated.inputPorts, + outputPorts: updated.outputPorts, + operatorProperties: cloneDeep( + Object.fromEntries( + Object.entries(op.operatorProperties).filter(([key]) => fields(next.content).some(f => f.id === key)) + ) + ), + }; +} + +/** Exposed Form View fields become the macro's ordinary operator properties, including nested macros. */ +export function macroSchema( + d: MacroDefinition, + getSchema: (type: string) => OperatorSchema, + getOperatorSchema?: (op: OperatorPredicate) => OperatorSchema +): OperatorSchema { + const boundary = readMacroInterface(d.content)!; + const schemaFor = memoize( + (op: OperatorPredicate) => (getOperatorSchema ? getOperatorSchema(op) : getSchema(op.operatorType)).jsonSchema, + op => (getOperatorSchema ? op : op.operatorType) + ); + const properties: Record = Object.create(null); + const required: string[] = []; + const resolve = (value: any, root: CustomJSONSchema7, op: OperatorPredicate, refs: string[] = []): any => { + if (Array.isArray(value)) return value.map(v => resolve(v, root, op, refs)); + if (!value || typeof value !== "object") return value; + if (value.$ref) { + const { $ref, ...rest } = value; + assert($ref.startsWith("#/") && !refs.includes($ref), "Missing or recursive exposed property schema reference"); + const target = $ref + .slice(2) + .split("/") + .reduce((node: any, key: string) => node?.[key.replace(/~1/g, "/").replace(/~0/g, "~")], root); + assert(target, "Missing exposed property schema reference"); + return resolve({ ...target, ...rest }, root, op, [...refs, $ref]); + } + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => key !== "$id") + .map(([key, v]) => [ + key, + key === "autofillAttributeOnPort" && Number.isInteger(v) + ? boundary.inputs.findIndex(p => + p.targets.some(t => t.operatorID === op.operatorID && t.portID === op.inputPorts[v as number]?.portID) + ) + : resolve(v, root, op, refs), + ]) + ); + }; + for (const f of fields(d.content)) { + const op = d.content.operators.find(o => o.operatorID === f.operatorID); + if (!op) continue; + const root = schemaFor(op); + const schema = root.properties?.[f.propertyKey]; + if (!schema || typeof schema === "boolean") continue; + const aliases = Object.fromEntries( + fields(d.content) + .filter(p => p.operatorID === f.operatorID) + .map(p => [p.propertyKey, p.id]) + ); + const inherited = schema.widget?.formlyConfig?.props?.["propertyContext"] as PropertyContext | undefined; + const context: PropertyContext = inherited + ? { + ...inherited, + properties: { + ...inherited.properties, + ...Object.fromEntries( + Object.entries(inherited.bindings) + .filter(([, key]) => Object.hasOwn(op.operatorProperties, key)) + .map(([key, id]) => [key, op.operatorProperties[id]]) + ), + }, + bindings: Object.fromEntries( + Object.entries(inherited.bindings) + .filter(([, id]) => aliases[id]) + .map(([key, id]) => [key, aliases[id]]) + ), + } + : { + operatorType: op.operatorType, + propertyKey: f.propertyKey, + properties: cloneDeep({ + ...Object.fromEntries( + Object.entries(root.properties ?? {}) + .filter(([, p]) => typeof p !== "boolean" && p.default !== undefined) + .map(([key, p]) => [key, (p as CustomJSONSchema7).default]) + ), + ...op.operatorProperties, + }), + bindings: aliases, + required: conditionalRequiredRules(root).get(f.propertyKey), + hiddenBy: Object.entries(root.properties ?? {}).find( + ([, p]) => typeof p !== "boolean" && p.toggleHidden?.includes(f.propertyKey) + )?.[0], + }; + const { widget: schemaWidget, ...valueSchema } = schema; + const widget = schemaWidget?.formlyConfig; + if (root.required?.includes(f.propertyKey)) required.push(f.id); + const type = customFormlyFieldType({ + key: context.propertyKey, + operatorType: context.operatorType, + description: schema.description, + currentType: schema.type, + }); + properties[f.id] = { + ...resolve(valueSchema, root, op), + title: f.displayName, + description: f.helpText, + default: cloneDeep( + Object.hasOwn(op.operatorProperties, f.propertyKey) ? op.operatorProperties[f.propertyKey] : schema.default + ), + widget: { + formlyConfig: { ...widget, ...(type ? { type } : {}), props: { ...widget?.props, propertyContext: context } }, + }, + }; + } + return { + operatorType: macroType(d), + operatorVersion: String(d.version), + jsonSchema: { type: "object", properties, ...(required.length ? { required } : {}), additionalProperties: false }, + additionalMetadata: { + userFriendlyName: `${d.name} (${macroRevisionLabel(d)})`, + operatorGroupName: "Macros", + operatorDescription: "Reusable macro. Double-click to open this revision on the canvas.", + inputPorts: boundary.inputs.map(p => ({ displayName: p.displayName })), + outputPorts: boundary.outputs.map(p => ({ displayName: p.displayName })), + }, + }; +} diff --git a/frontend/src/app/dashboard/component/dashboard.component.spec.ts b/frontend/src/app/dashboard/component/dashboard.component.spec.ts index b15cee14e75..9f1593494f6 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.spec.ts +++ b/frontend/src/app/dashboard/component/dashboard.component.spec.ts @@ -261,6 +261,27 @@ describe("DashboardComponent", () => { expect(component.isNavbarEnabled("/user/workflow")).toBe(true); }); + it.each([ + { url: "/user/macros/33", hidden: true }, + { url: "/user/macros/33?macro=outer¯o=inner#operator", hidden: true }, + { url: "/user/workflow/42", hidden: true }, + { url: "/user/workflow/42/form?macro=child", hidden: true }, + { url: "/user/workflow/42#operator", hidden: true }, + { url: "", hidden: false }, + { url: "/user/workflow", hidden: false }, + { url: "/user/workflow?search=33", hidden: false }, + { url: "/user/macros", hidden: false }, + { url: "/user/macros/new", hidden: false }, + { url: "/user/macros/33x", hidden: false }, + { url: "/user/workflow/42x", hidden: false }, + { url: "/user/dataset", hidden: false }, + ])("sets search header hidden=$hidden for '$url'", ({ url, hidden }) => { + Object.assign(routerMock, { url }); + component.checkRoute(); + fixture.detectChanges(); + expect(fixture.debugElement.query(By.css("#nav")).nativeElement.hidden).toBe(hidden); + }); + it("exposes route constants without the legacy /dashboard prefix", () => { expect(USER_WORKFLOW).toBe("/user/workflow"); expect(USER_DATASET).toBe("/user/dataset"); diff --git a/frontend/src/app/dashboard/component/dashboard.component.ts b/frontend/src/app/dashboard/component/dashboard.component.ts index ee52ab65a7f..f1ff5a23125 100644 --- a/frontend/src/app/dashboard/component/dashboard.component.ts +++ b/frontend/src/app/dashboard/component/dashboard.component.ts @@ -267,11 +267,8 @@ export class DashboardComponent implements OnInit { } isNavbarEnabled(currentRoute: string) { - // Hide navbar for workflow workspace pages (with numeric ID) - if (currentRoute.match(/\/user\/workflow\/\d+/)) { - return false; - } - return true; + // Workflow and macro editors provide their own header. + return !/^\/user\/(?:workflow|macros)\/\d+(?:[/?#]|$)/.test(currentRoute); } handleCollapseChange(collapsed: boolean) { diff --git a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts index 4a305fb9062..3b6a1745591 100644 --- a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts @@ -16,6 +16,9 @@ * specific language governing permissions and limitations * under the License. */ +import { testOperator } from "../../../../common/testing/workflow-graph"; +import { macroWorkspaceTestProviders } from "../../../../workspace/service/macro/macro-workspace.spec-harness"; +import { MacroExpansionService } from "../../../../workspace/service/macro/macro-expansion.service"; import { ComponentFixture, TestBed } from "@angular/core/testing"; import { By } from "@angular/platform-browser"; @@ -44,9 +47,7 @@ import { NzPopoverModule } from "ng-zorro-antd/popover"; import { NzDatePickerModule } from "ng-zorro-antd/date-picker"; import { en_US, NZ_I18N } from "ng-zorro-antd/i18n"; import { NoopAnimationsModule } from "@angular/platform-browser/animations"; -import { OperatorMetadataService } from "../../../../workspace/service/operator-metadata/operator-metadata.service"; -import { StubOperatorMetadataService } from "../../../../workspace/service/operator-metadata/stub-operator-metadata.service"; -import { NzUploadModule } from "ng-zorro-antd/upload"; +import { NzUploadModule, NzUploadFile } from "ng-zorro-antd/upload"; import { ScrollingModule } from "@angular/cdk/scrolling"; import { NzAvatarModule } from "ng-zorro-antd/avatar"; import { NzTooltipModule } from "ng-zorro-antd/tooltip"; @@ -65,7 +66,6 @@ import JSZip from "jszip"; import { ModalOptions, NzModalRef, NzModalService } from "ng-zorro-antd/modal"; import { NzButtonModule } from "ng-zorro-antd/button"; import { DownloadService } from "../../../service/user/download/download.service"; -import { commonTestProviders } from "../../../../common/testing/test-utils"; import { Router } from "@angular/router"; import { USER_WORKSPACE } from "../../../../app-routing.constant"; import { GuiConfigService } from "../../../../common/service/gui-config.service"; @@ -80,7 +80,6 @@ import { NotebookImportModalComponent, NotebookImportModalData, } from "../../../../workspace/component/notebook-import-modal/notebook-import-modal.component"; -import { NzUploadFile } from "ng-zorro-antd/upload"; import type { Mocked } from "vitest"; describe("SavedWorkflowSectionComponent", () => { let component: UserWorkflowComponent; @@ -96,7 +95,6 @@ describe("SavedWorkflowSectionComponent", () => { NzModalService, { provide: WorkflowPersistService, useValue: new StubWorkflowPersistService(testWorkflowEntries) }, ShareAccessService, - { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, { provide: NZ_I18N, useValue: en_US }, { provide: UserService, useClass: StubUserService }, { @@ -104,7 +102,7 @@ describe("SavedWorkflowSectionComponent", () => { useValue: new StubSearchService(testWorkflowEntries, mockUserInfo), }, { provide: DownloadService, useValue: downloadServiceSpy }, - ...commonTestProviders, + ...macroWorkspaceTestProviders, ], imports: [ UserWorkflowComponent, @@ -1091,6 +1089,36 @@ describe("SavedWorkflowSectionComponent", () => { }); describe("workflow uploads", () => { + it.each([ + { + error: "Duplicate graph element ID", + graph: { operators: [{ operatorID: "same" }], links: [{ linkID: "same" }] }, + }, + { + error: "Missing macro version", + graph: { operators: [testOperator("missing", { operatorType: "WorkflowMacro:missing:1" })], links: [] }, + }, + ])("rejects $error before saving and keeps the list", async ({ error, graph }) => { + const expansion = vi + .spyOn(TestBed.inject(MacroExpansionService), "expand") + .mockRejectedValue(new Error("Missing macro version")); + const create = vi.fn(); + TestBed.inject(WorkflowPersistService).createWorkflow = create; + const notify = vi + .spyOn(TestBed.inject(NotificationService), "error") + .mockImplementation(() => undefined as any); + const entries = component.searchResultsComponent.entries; + const file = new File([JSON.stringify({ ...testWorkflowContent([]), ...graph })], "broken.json"); + + await expect(firstValueFrom(component.onClickUploadExistingWorkflowFromLocal(file as any))).rejects.toThrow( + error + ); + expect(expansion).toHaveBeenCalledTimes(error === "Missing macro version" ? 1 : 0); + expect(create).not.toHaveBeenCalled(); + expect(component.searchResultsComponent.entries).toBe(entries); + expect(notify).toHaveBeenCalledWith(expect.stringContaining(error)); + }); + it("imports a valid .json workflow, appending an entry and toasting success", async () => { const persist = TestBed.inject(WorkflowPersistService) as any; persist.createWorkflow = vi.fn().mockReturnValue(of(makeDashboardWorkflow(42, "wf"))); diff --git a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts index 22946324803..6fd4cbe85dd 100644 --- a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts +++ b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts @@ -30,6 +30,7 @@ import { UserService } from "../../../../common/service/user/user.service"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { NotificationService } from "../../../../common/service/notification/notification.service"; import { ExecutionMode, ExportedWorkflow, WorkflowContent } from "../../../../common/type/workflow"; +import { isMacro, assertWorkflowIDs } from "../../../../common/util/workflow-macro"; import { NzUploadFile, NzUploadComponent } from "ng-zorro-antd/upload"; import JSZip from "jszip"; import { FiltersComponent } from "../filters/filters.component"; @@ -116,6 +117,8 @@ const SCRIPT_INPUTS: Record(observer => { let reader = new FileReader(); reader.readAsText(file); - reader.onload = () => { + reader.onload = async () => { + let validatingGraph = false; try { const result = reader.result; if (typeof result !== "string") { @@ -685,6 +690,16 @@ export class UserWorkflowComponent implements AfterViewInit, OnDestroy { // The landing view rides as a sibling key next to the content (see exportedWorkflow); // pull it back out so it is stored on the workflow row, not inside content. const { defaultView, ...workflowContent } = JSON.parse(result) as ExportedWorkflow; + validatingGraph = true; + assertWorkflowIDs(workflowContent); + if ( + workflowContent.macroInterface || + workflowContent.macroDependencies || + workflowContent.operators?.some(isMacro) + ) { + await this.macroExpansion.expand(workflowContent); + } + validatingGraph = false; this.workflowPersistService .createWorkflow(workflowContent, this.deriveWorkflowName(name), defaultView) .pipe(untilDestroyed(this)) @@ -703,7 +718,9 @@ export class UserWorkflowComponent implements AfterViewInit, OnDestroy { }); } catch (error) { this.notificationService.error( - "An error occurred when importing the workflow. Please import a workflow json file." + validatingGraph && error instanceof Error + ? error.message + : "An error occurred when importing the workflow. Please import a workflow json file." ); observer.error(error); } diff --git a/frontend/src/app/dashboard/service/user/download/download.service.spec.ts b/frontend/src/app/dashboard/service/user/download/download.service.spec.ts index 0d1c3ac734f..1ee54f283ba 100644 --- a/frontend/src/app/dashboard/service/user/download/download.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/download/download.service.spec.ts @@ -246,6 +246,17 @@ describe("DownloadService", () => { reader.readAsText(blob); }); + it("includes pinned macro dependencies in the downloaded file", async () => { + const content = { + operators: [], + links: [], + macroDependencies: [{ id: "nested", version: 8, content: { operators: [] } }], + }; + workflowPersistServiceSpy.retrieveWorkflow.mockReturnValue(of({ content } as any)); + const result = await firstValueFrom(downloadService.downloadWorkflow(42, "Macros")); + expect(JSON.parse(await readBlob(result.blob)).macroDependencies).toEqual(content.macroDependencies); + }); + it("carries the default view into the downloaded workflow json so it round-trips on upload", async () => { const content = { operators: [], links: [] }; workflowPersistServiceSpy.retrieveWorkflow.mockReturnValue(of({ content, defaultView: "FORM" } as any)); diff --git a/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.spec.ts b/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.spec.ts index 66d89ece5c5..e0ce7e7e11a 100644 --- a/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.spec.ts @@ -97,6 +97,12 @@ describe("WorkflowExecutionsService", () => { req.flush({}); }); + it("requests the executed canvas on the selected computing unit", () => { + service.retrieveMacroStatistics(1, 10, 5).subscribe(); + const req = httpMock.expectOne(`${WORKFLOW_EXECUTIONS_API_BASE_URL}/1/stats/10/canvas?cuid=5`); + req.flush(null, { status: 204, statusText: "No Content" }); + }); + it("retrieveWorkflowRuntimeStatistics issues GET with the cuid param", () => { let result: WorkflowRuntimeStatistics[] | undefined; service.retrieveWorkflowRuntimeStatistics(1, 10, 5).subscribe(res => (result = res)); diff --git a/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.ts b/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.ts index b15f668bff3..174aea591f5 100644 --- a/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.ts +++ b/frontend/src/app/dashboard/service/user/workflow-executions/workflow-executions.service.ts @@ -22,8 +22,9 @@ import { Observable } from "rxjs"; import { AppSettings } from "../../../../common/app-setting"; import { HttpClient, HttpParams } from "@angular/common/http"; import { WorkflowExecutionsEntry } from "../../../type/workflow-executions-entry"; +import type { WorkflowContent } from "../../../../common/type/workflow"; import { WorkflowRuntimeStatistics } from "../../../type/workflow-runtime-statistics"; -import { ExecutionState } from "../../../../workspace/types/execute-workflow.interface"; +import { ExecutionState, OperatorRuntimeStatus } from "../../../../workspace/types/execute-workflow.interface"; export const WORKFLOW_EXECUTIONS_API_BASE_URL = `${AppSettings.getApiEndpoint()}/executions`; @@ -85,6 +86,21 @@ export class WorkflowExecutionsService { }); } + retrieveMacroContent(wid: number, eid: number): Observable { + return this.http.get(`${WORKFLOW_EXECUTIONS_API_BASE_URL}/${wid}/macro-content/${eid}`); + } + + retrieveMacroStatistics( + wid: number, + eid: number, + cuid: number + ): Observable | null> { + return this.http.get | null>( + `${WORKFLOW_EXECUTIONS_API_BASE_URL}/${wid}/stats/${eid}/canvas`, + { params: { cuid } } + ); + } + retrieveWorkflowRuntimeStatistics(wid: number, eId: number, cuid: number): Observable { const params = new HttpParams().set("cuid", cuid.toString()); return this.http.get(`${WORKFLOW_EXECUTIONS_API_BASE_URL}/${wid}/stats/${eId}`, { diff --git a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts index 301515d9568..8768146a69f 100644 --- a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts +++ b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts @@ -39,8 +39,7 @@ export class WorkflowToPythonService { public convertToPython(logicalPlan: LogicalPlan): Observable { const body = { - operators: logicalPlan.operators, - links: logicalPlan.links, + ...logicalPlan, opsToReuseResult: [], opsToViewResult: [], }; diff --git a/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.spec.ts b/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.spec.ts index aa29f455903..df76362dec7 100644 --- a/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.spec.ts @@ -16,74 +16,58 @@ * specific language governing permissions and limitations * under the License. */ +import { testLink, testOperator } from "../../../../common/testing/workflow-graph"; +import { testWorkflowContent } from "../../../component/user-dashboard-test-fixtures"; import { TestBed } from "@angular/core/testing"; import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; -import { firstValueFrom } from "rxjs"; +import { firstValueFrom, Subject } from "rxjs"; import { take, toArray } from "rxjs/operators"; import { WORKFLOW_VERSIONS_API_BASE_URL, WorkflowVersionService } from "./workflow-version.service"; -import { WorkflowActionService } from "../../../../workspace/service/workflow-graph/model/workflow-action.service"; +import { + DEFAULT_WORKFLOW, + WorkflowActionService, +} from "../../../../workspace/service/workflow-graph/model/workflow-action.service"; import { WorkflowPersistService } from "../../../../common/service/workflow-persist/workflow-persist.service"; import { UndoRedoService } from "../../../../workspace/service/undo-redo/undo-redo.service"; import { AppSettings } from "../../../../common/app-setting"; import { commonTestProviders } from "../../../../common/testing/test-utils"; -import { ExecutionMode, Workflow, WorkflowContent } from "../../../../common/type/workflow"; -import { CommentBox, OperatorLink, OperatorPredicate } from "../../../../workspace/types/workflow-common.interface"; +import { Workflow, WorkflowContent } from "../../../../common/type/workflow"; +import { OperatorPredicate } from "../../../../workspace/types/workflow-common.interface"; import { WorkflowVersionEntry } from "../../../type/workflow-version-entry"; const API = "api"; function buildOperator(overrides: Partial = {}): OperatorPredicate { - return { - operatorID: "op-1", - operatorType: "Filter", - operatorVersion: "v1", - operatorProperties: {}, - inputPorts: [], - outputPorts: [], - showAdvanced: false, - ...overrides, - }; + return testOperator("op-1", { operatorVersion: "v1", inputPorts: [], outputPorts: [], ...overrides }); } -function buildLink(source: string, target: string): OperatorLink { - return { - linkID: `${source}->${target}`, - source: { operatorID: source, portID: "out-0" }, - target: { operatorID: target, portID: "in-0" }, - }; -} +const buildLink = (source: string, target: string) => testLink(source, target, "out-0", "in-0", `${source}->${target}`); function buildContent(overrides: Partial = {}): WorkflowContent { - return { - operators: [], - operatorPositions: {}, - links: [], - commentBoxes: [] as CommentBox[], - settings: { dataTransferBatchSize: 400, executionMode: ExecutionMode.PIPELINED }, - ...overrides, - }; + return { ...testWorkflowContent([]), ...overrides }; } function buildWorkflow(overrides: Partial = {}): Workflow { return { + ...DEFAULT_WORKFLOW, name: "wf", - description: undefined, wid: 1, creationTime: 0, lastModifiedTime: 0, - isPublished: 0, - readonly: false, content: buildContent(), ...overrides, }; } describe("WorkflowVersionService", () => { + let metadataChanged: Subject; let service: WorkflowVersionService; let http: HttpTestingController; let actionSpy: { + getWorkflowMetadata: ReturnType; + workflowMetaDataChanged: ReturnType; getJointGraphWrapper: ReturnType; getWorkflow: ReturnType; getWorkflowContent: ReturnType; @@ -108,6 +92,7 @@ describe("WorkflowVersionService", () => { let getMainJointPaper: ReturnType; beforeEach(() => { + metadataChanged = new Subject(); modelAttr = vi.fn(); paperGetModelById = vi.fn().mockReturnValue({ attr: modelAttr }); getMainJointPaper = vi.fn().mockReturnValue({ getModelById: paperGetModelById }); @@ -116,6 +101,8 @@ describe("WorkflowVersionService", () => { unhighlightElements: vi.fn(), }; actionSpy = { + getWorkflowMetadata: vi.fn().mockReturnValue(buildWorkflow()), + workflowMetaDataChanged: vi.fn(() => metadataChanged), getJointGraphWrapper: vi.fn().mockReturnValue({ getCurrentHighlights: highlightedSpy.getCurrentHighlights, unhighlightElements: highlightedSpy.unhighlightElements, @@ -181,6 +168,28 @@ describe("WorkflowVersionService", () => { // ─── setDisplayParticularVersion + streams ──────────────────────────────── describe("setDisplayParticularVersion", () => { + it("keeps a parent's preview and revision selection scoped to that document while inspecting a child", () => { + const actions = TestBed.inject(WorkflowActionService); + const parent = buildWorkflow(); + actionSpy.getWorkflowMetadata.mockReturnValue(parent); + metadataChanged.next(); + actionSpy.getTempWorkflow.mockReturnValue(parent); + service.setDisplayParticularVersion(true, 7, 2); + const flags: boolean[] = []; + const subscription = service.getDisplayParticularVersionStream().subscribe(value => flags.push(value)); + actionSpy.getWorkflowMetadata.mockReturnValue(buildWorkflow({ wid: 2 })); + metadataChanged.next(); + expect(flags.at(-1)).toBe(false); + expect(service.canPreviewVersion).toBe(false); + expect(actions.getTempWorkflow()).toBe(parent); + expect(service.selectedVersionId.value).toBe(7); + actionSpy.getWorkflowMetadata.mockReturnValue(parent); + metadataChanged.next(); + expect(flags.at(-1)).toBe(true); + expect(service.canPreviewVersion).toBe(true); + subscription.unsubscribe(); + }); + it("publishes both ids and the flag when entering a particular version", async () => { const flag = firstValueFrom(service.getDisplayParticularVersionStream().pipe(take(2), toArray())); service.setDisplayParticularVersion(true, 7, 9); @@ -530,6 +539,26 @@ describe("WorkflowVersionService", () => { expect(await pending).toEqual(entries); }); + it.each([true, false])("requests macro metadata with head=%s", async head => { + const pending = firstValueFrom(service.macroSource(42, "definition", head)); + const req = http.expectOne(request => request.url === `${API}/version/42/macro-source`); + expect(req.request.params.get("id")).toBe("definition"); + expect(req.request.params.get("head")).toBe(String(head)); + const summary = { metadata: DEFAULT_WORKFLOW, revisions: [3, 64] }; + req.flush(summary); + expect(await pending).toEqual(summary); + }); + + it.each([undefined, 5])("requests the pinned body by source identity and selected save (%s)", async version => { + const pending = firstValueFrom(service.pinMacro(42, "definition", version)); + const request = http.expectOne(req => req.url === `${API}/${WORKFLOW_VERSIONS_API_BASE_URL}/42/macro`); + expect(request.request.params.get("id")).toBe("definition"); + expect(request.request.params.get("version")).toBe(version?.toString() ?? null); + const body = { id: "definition", wid: 42, version: version ?? 64, revision: 2, content: buildContent() }; + request.flush(body); + expect(await pending).toEqual(body); + }); + it("retrieveWorkflowByVersion parses a string `content` into an object", async () => { const pending = firstValueFrom(service.retrieveWorkflowByVersion(42, 7)); const req = http.expectOne(`${API}/${WORKFLOW_VERSIONS_API_BASE_URL}/42/7`); diff --git a/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.ts b/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.ts index cf2db135ee2..4e8fe390ea5 100644 --- a/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.ts +++ b/frontend/src/app/dashboard/service/user/workflow-version/workflow-version.service.ts @@ -18,22 +18,30 @@ */ import { Injectable } from "@angular/core"; -import { BehaviorSubject, Observable } from "rxjs"; +import { BehaviorSubject, Observable, merge } from "rxjs"; import { WorkflowActionService } from "../../../../workspace/service/workflow-graph/model/workflow-action.service"; import { Workflow, WorkflowContent } from "../../../../common/type/workflow"; +import type { WorkflowMetadata } from "../../../type/workflow-metadata.interface"; +import type { MacroDefinition } from "../../../../common/util/workflow-macro"; import { WorkflowPersistService } from "../../../../common/service/workflow-persist/workflow-persist.service"; import { UndoRedoService } from "../../../../workspace/service/undo-redo/undo-redo.service"; import { isEqual } from "lodash"; import { OperatorLink, OperatorPredicate, Point } from "../../../../workspace/types/workflow-common.interface"; import { WorkflowVersionEntry } from "../../../type/workflow-version-entry"; import { AppSettings } from "../../../../common/app-setting"; -import { filter, map } from "rxjs/operators"; +import { filter, map, startWith, distinctUntilChanged, skip } from "rxjs/operators"; import { WorkflowUtilService } from "../../../../workspace/service/workflow-graph/util/workflow-util.service"; import { HttpClient } from "@angular/common/http"; export const WORKFLOW_VERSIONS_API_BASE_URL = "version"; +export interface MacroSource { + metadata: WorkflowMetadata; + revisions: number[]; + head?: MacroDefinition; +} + type WorkflowContentKeys = keyof WorkflowContent; type Element = OperatorLink | OperatorPredicate | Point; type DifferentOpIDsList = { @@ -88,7 +96,20 @@ export class WorkflowVersionService { } public getDisplayParticularVersionStream(): Observable { - return this.displayParticularWorkflowVersion.asObservable(); + const documentChanges = this.workflowActionService.workflowMetaDataChanged().pipe( + map(() => this.canPreviewVersion), + startWith(this.canPreviewVersion), + distinctUntilChanged(), + skip(1) + ); + return merge(this.displayParticularWorkflowVersion, documentChanges).pipe( + map(() => this.displayParticularWorkflowVersion.value && this.canPreviewVersion) + ); + } + + public get canPreviewVersion(): boolean { + const owner = this.workflowActionService.getTempWorkflow()?.wid; + return owner === undefined || owner === this.workflowActionService.getWorkflowMetadata().wid; } public get canRestoreVersion(): boolean { @@ -313,6 +334,22 @@ export class WorkflowVersionService { ); } + macroSource(wid: number, id: string, head: boolean): Observable { + return this.http.get( + `${AppSettings.getApiEndpoint()}/${WORKFLOW_VERSIONS_API_BASE_URL}/${wid}/macro-source`, + { params: { id, head } } + ); + } + + pinMacro(wid: number, id: string, version?: number): Observable { + return this.http.get( + `${AppSettings.getApiEndpoint()}/${WORKFLOW_VERSIONS_API_BASE_URL}/${wid}/macro`, + { + params: version === undefined ? { id } : { id, version }, + } + ); + } + /** * retrieves a version of the workflow from backend database */ diff --git a/frontend/src/app/workspace/component/constrained-value/constrained-value.component.ts b/frontend/src/app/workspace/component/constrained-value/constrained-value.component.ts index 87e305458e8..24b4c592305 100644 --- a/frontend/src/app/workspace/component/constrained-value/constrained-value.component.ts +++ b/frontend/src/app/workspace/component/constrained-value/constrained-value.component.ts @@ -23,6 +23,7 @@ import { FormsModule } from "@angular/forms"; import { FieldType, FieldTypeConfig, FormlyModule } from "@ngx-formly/core"; import { NzInputModule } from "ng-zorro-antd/input"; import { NzSelectModule } from "ng-zorro-antd/select"; +import { propertyModel } from "../../util/custom-formly-type"; import { matchingValueRule } from "../../../common/formly/formly-utils"; /** @@ -65,7 +66,7 @@ import { matchingValueRule } from "../../../common/formly/formly-utils"; export class ConstrainedValueComponent extends FieldType { /** The branch of the rules that the sibling's current value selects, if any. */ private get rule() { - return matchingValueRule(this.props.valueRules, this.field?.parent?.model); + return matchingValueRule(this.props.valueRules, propertyModel(this.field)); } get acceptedValues(): ReadonlyArray { diff --git a/frontend/src/app/workspace/component/hugging-face/hugging-face.component.html b/frontend/src/app/workspace/component/hugging-face/hugging-face.component.html index bcaaab30038..c93ddd9464c 100644 --- a/frontend/src/app/workspace/component/hugging-face/hugging-face.component.html +++ b/frontend/src/app/workspace/component/hugging-face/hugging-face.component.html @@ -25,6 +25,7 @@ (ngModelChange)="onTaskSelected($event)" nzPlaceHolder="Select a task" [nzLoading]="tasksLoading" + [nzDisabled]="!canSelectTask" style="width: 100%; margin-bottom: 4px"> showError, fieldChanges: new Subject() }; } +function buildField(form: FormGroup, model: object, key = "modelId"): FieldTypeConfig { + return { + key, + formControl: form.get(key)!, + form, + model, + props: {}, + parent: { fieldGroup: [] }, + options: buildFormlyOptions(), + } as unknown as FieldTypeConfig; +} + /** * Build a minimal FormlyFieldConfig with a FormGroup backing it, * similar to what Formly provides at runtime. @@ -81,22 +93,37 @@ function buildFieldWithFormGroup(taskValue = "", modelIdValue = ""): { field: Fi temperature: new FormControl(0.7), }); - const model: Record = { - task: taskValue, - modelId: modelIdValue, - }; - - const field = { - key: "modelId", - formControl: formGroup.get("modelId")! as FormControl, - form: formGroup, - model, - props: {}, - parent: { fieldGroup: [] }, - options: buildFormlyOptions(), - } as unknown as FieldTypeConfig; + return { field: buildField(formGroup, { task: taskValue, modelId: modelIdValue }), formGroup }; +} - return { field, formGroup }; +function projectedField(exposeTask: boolean, singleCard = false) { + const bindings = { + modelId: "macro-model", + promptColumn: "macro-prompt", + ...(exposeTask ? { task: "macro-task" } : {}), + }; + const model = { + "macro-model": "saved/model", + "macro-prompt": "prompt", + ...(exposeTask ? { "macro-task": "text-generation" } : {}), + ...(singleCard ? { "form-model": "saved/model" } : {}), + }; + const key = singleCard ? "form-model" : "macro-model"; + const formGroup = new FormGroup( + Object.fromEntries( + Object.entries(model) + .filter(([name]) => !singleCard || name === key) + .map(([name, value]) => [name, new FormControl(value)]) + ) + ); + const context = { + operatorType: "HuggingFace", + propertyKey: "modelId", + properties: { task: exposeTask ? "text-generation" : "image-classification", systemPrompt: "Pinned prompt" }, + bindings, + }; + const field = { ...buildField(formGroup, model, key), parent: { model }, props: { propertyContext: context } }; + return { field, formGroup, context }; } // ── Pure unit tests (no TestBed) ── @@ -209,8 +236,11 @@ describe("HuggingFaceComponent (TestBed)", () => { } /** Set up field + trigger ngOnInit, then flush the two startup HTTP requests. */ - function initComponent(taskTag = "text-generation", models: HuggingFaceModelOption[] = buildModels(3)) { - const { field } = buildFieldWithFormGroup(taskTag); + function initComponent( + taskTag = "text-generation", + models: HuggingFaceModelOption[] = buildModels(3), + field = buildFieldWithFormGroup(taskTag).field + ) { component.field = field; fixture.detectChanges(); // triggers ngOnInit flushIconRequests(); @@ -219,7 +249,7 @@ describe("HuggingFaceComponent (TestBed)", () => { const tasksReq = http.expectOne(`${API}/huggingface/tasks`); tasksReq.flush(buildTaskResponse()); - const modelsReq = http.expectOne(req => req.url.startsWith(`${API}/huggingface/models`)); + const modelsReq = http.expectOne(`${API}/huggingface/models?task=${taskTag}`); modelsReq.flush(models); flushIconRequests(); } @@ -236,6 +266,47 @@ describe("HuggingFaceComponent (TestBed)", () => { expect(component.selectedTaskTag).toBe("text-generation"); }); + it("keeps an unexposed macro task pinned without adding native properties on initialization", () => { + const { field, context } = projectedField(false); + const before = JSON.stringify(context); + initComponent("image-classification", [], field); + + expect(Object.keys(field.model!)).toEqual(["macro-model", "macro-prompt"]); + expect(fixture.debugElement.query(By.css("nz-select")).componentInstance.nzDisabled).toBe(true); + component.onTaskSelected("text-generation"); + expect(component.selectedTaskTag).toBe("image-classification"); + expect(JSON.stringify(context)).toBe(before); + component.onModelSelected("other/image-model"); + expect(field.formControl.value).toBe("other/image-model"); + }); + + it("changes exposed task aliases and restores only exposed task-specific values", () => { + const { field, formGroup, context } = projectedField(true); + const before = JSON.stringify(context); + initComponent("text-generation", [], field); + component.onTaskSelected("image-classification"); + http.expectOne(`${API}/huggingface/models?task=image-classification`).flush([]); + expect(formGroup.get("macro-task")!.value).toBe("image-classification"); + expect(Object.keys(field.model!)).toEqual(["macro-model", "macro-prompt", "macro-task"]); + expect(formGroup.get("macro-model")!.value).toBe(""); + + component.onTaskSelected("text-generation"); + expect(formGroup.get("macro-model")!.value).toBe("saved/model"); + expect(formGroup.get("macro-prompt")!.value).toBe("prompt"); + expect(JSON.stringify(context)).toBe(before); + }); + + it("keeps task changes read-only in a Form View card while preserving its model control", () => { + const { field, formGroup } = projectedField(true, true); + initComponent("text-generation", [], field); + expect(fixture.debugElement.query(By.css("nz-select")).componentInstance.nzDisabled).toBe(true); + component.onTaskSelected("image-classification"); + expect(component.selectedTaskTag).toBe("text-generation"); + expect(field.model!["macro-task"]).toBe("text-generation"); + component.onModelSelected("new/model"); + expect(formGroup.get("form-model")!.value).toBe("new/model"); + }); + // ── Task loading ── describe("task loading", () => { diff --git a/frontend/src/app/workspace/component/hugging-face/hugging-face.component.ts b/frontend/src/app/workspace/component/hugging-face/hugging-face.component.ts index 4925656425c..4b5ed8adebc 100644 --- a/frontend/src/app/workspace/component/hugging-face/hugging-face.component.ts +++ b/frontend/src/app/workspace/component/hugging-face/hugging-face.component.ts @@ -28,6 +28,8 @@ import { NzSpinModule } from "ng-zorro-antd/spin"; import { NzButtonModule } from "ng-zorro-antd/button"; import { NzIconModule } from "ng-zorro-antd/icon"; import { AppSettings } from "../../../common/app-setting"; +import { propertyModel } from "../../util/custom-formly-type"; +import { huggingFacePropertyKey } from "../../util/hugging-face-field-bindings"; import { of, Subject, Subscription } from "rxjs"; import { catchError, debounceTime, finalize, switchMap, takeUntil } from "rxjs/operators"; @@ -310,6 +312,10 @@ export class HuggingFaceComponent extends FieldType implements // ── Task selection ── onTaskSelected(tag: string): void { + if (!this.canSelectTask) { + this.selectedTaskTag = this.getCurrentTaskTag() ?? this.selectedTaskTag; + return; + } const previousTask = this.getCurrentTaskTag() ?? this.selectedTaskTag; this.snapshotTaskState(previousTask); this.syncTaskSelection(tag, true); @@ -536,41 +542,20 @@ export class HuggingFaceComponent extends FieldType implements // ── Private helpers ── private getCurrentTaskTag(): string | undefined { - const fromModel = this.model?.task; + const fromModel = (propertyModel(this.field) ?? this.model)?.task; if (typeof fromModel === "string" && fromModel.trim().length > 0) { return fromModel; } - const fromParentControl = this.formControl?.parent?.get("task")?.value; - if (typeof fromParentControl === "string" && fromParentControl.trim().length > 0) { - return fromParentControl; - } - const fromFieldForm = this.field.form?.get("task")?.value; - if (typeof fromFieldForm === "string" && fromFieldForm.trim().length > 0) { - return fromFieldForm; + const fromControl = this.controlFor("task")?.value; + if (typeof fromControl === "string" && fromControl.trim().length > 0) { + return fromControl; } return undefined; } private persistTaskSelection(tag: string): void { - // 1. Update the backing model FIRST so expression functions read the new value. - if (this.model) { - this.model.task = tag; - } - - // 2. Update the hidden task form control. Using emitEvent: true (default) - // ensures formly picks up the change and re-evaluates all sibling expressions. - const taskControlFromField = this.field.form?.get("task"); - if (taskControlFromField) { - taskControlFromField.setValue(tag); - } - - const taskControlFromParent = this.formControl?.parent?.get("task"); - if (taskControlFromParent && taskControlFromParent !== taskControlFromField) { - taskControlFromParent.setValue(tag); - } - - // 3. Force formly to re-evaluate ALL field expressions (not just this field's subtree). - // this.field is the modelId field; its parent covers all sibling fields. + this.writeFieldValue("task", tag, false); + // Re-evaluate sibling expressions after updating an exposed task control. const rootField = this.field.parent ?? this.field; this.field.options?.detectChanges?.(rootField); } @@ -601,7 +586,7 @@ export class HuggingFaceComponent extends FieldType implements "temperature", ]; for (const key of keys) { - const control = this.field.form?.get(key) ?? this.formControl?.parent?.get(key); + const control = this.controlFor(key); control?.updateValueAndValidity({ emitEvent: false }); } this.field.form?.updateValueAndValidity({ emitEvent: false }); @@ -659,23 +644,35 @@ export class HuggingFaceComponent extends FieldType implements } } - private readFieldValue(key: (typeof this.taskScopedKeys)[number]): unknown { - const control = this.field.form?.get(key) ?? this.formControl?.parent?.get(key); + public get canSelectTask(): boolean { + return !this.props.disabled && (!this.props["propertyContext"] || !!this.controlFor("task")); + } + + private controlFor(key: string) { + const id = huggingFacePropertyKey(this.field, key); + return id ? this.field.form?.get(id) ?? this.formControl?.parent?.get(id) : undefined; + } + + private readFieldValue(key: string): unknown { + const control = this.controlFor(key); if (control) { return control.value; } - return this.model?.[key]; + return (propertyModel(this.field) ?? this.model)?.[key]; } - private writeFieldValue(key: (typeof this.taskScopedKeys)[number], value: unknown): void { - const control = this.field.form?.get(key) ?? this.formControl?.parent?.get(key); - if (control) { - control.setValue(value, { emitEvent: false }); - control.markAsDirty(); - control.updateValueAndValidity({ emitEvent: false }); + private writeFieldValue(key: string, value: unknown, markDirty = true): void { + const id = huggingFacePropertyKey(this.field, key); + const control = this.controlFor(key); + if (!id || (this.props["propertyContext"] && !control)) return; + for (const target of new Set([control, this.formControl?.parent?.get(id)])) { + if (!target) continue; + target.setValue(value, { emitEvent: false }); + if (markDirty) target.markAsDirty(); + target.updateValueAndValidity({ emitEvent: false }); } if (this.model) { - (this.model as Record)[key] = value; + (this.model as Record)[id] = value; } } } diff --git a/frontend/src/app/workspace/component/left-panel/left-panel.component.spec.ts b/frontend/src/app/workspace/component/left-panel/left-panel.component.spec.ts index 57e65c8c843..b29be07ce51 100644 --- a/frontend/src/app/workspace/component/left-panel/left-panel.component.spec.ts +++ b/frontend/src/app/workspace/component/left-panel/left-panel.component.spec.ts @@ -17,6 +17,7 @@ * under the License. */ +import { setMacroGraph } from "../../../common/testing/macro-graph"; import { ComponentFixture, fakeAsync, TestBed, tick } from "@angular/core/testing"; import { By } from "@angular/platform-browser"; import { DebugElement } from "@angular/core"; @@ -42,6 +43,8 @@ const PANEL_LOCAL_STORAGE_KEYS = [ "left-panel-style", ]; +import { of } from "rxjs"; + describe("LeftPanelComponent", () => { let component: LeftPanelComponent; @@ -78,6 +81,25 @@ describe("LeftPanelComponent", () => { clearPanelStorage(); }); + it("keeps macro editing on the canvas without an Interface dock", () => { + component.openFrame(1); + setMacroGraph(workflowActionService, { id: "m", inputs: [], outputs: [] }); + fixture.detectChanges(); + expect(component.items.some(i => i.title === "Interface")).toBe(false); + expect(fixture.nativeElement.querySelector('#dock [nztype="api"], #docked-buttons [nztype="api"]')).toBeNull(); + expect(component.currentComponent).toBe(OperatorMenuComponent); + }); + + it("restores the remaining dock order when Interface was previously selected", () => { + localStorage.setItem("left-panel-order", "5,6,4,3,2,1"); + localStorage.setItem("left-panel-index", "6"); + const fresh = TestBed.createComponent(LeftPanelComponent); + fresh.detectChanges(); + expect(fresh.componentInstance.order).toEqual([5, 4, 3, 2, 1]); + expect(fresh.componentInstance.currentComponent).toBe(OperatorMenuComponent); + fresh.destroy(); + }); + it("should create", () => { expect(component).toBeTruthy(); }); @@ -96,7 +118,11 @@ describe("LeftPanelComponent", () => { // click on versions display component.openFrame(2); - new VersionsListComponent(workflowActionService, {} as never, { snapshot: { params: {} } } as never).ngOnInit(); + new VersionsListComponent( + workflowActionService, + { retrieveVersionsOfWorkflow: () => of([]) } as never, + { snapshot: { params: {} } } as never + ).ngOnInit(); // all the elements shall be un-highlighted expect(workflowActionService.getJointGraphWrapper().getCurrentHighlightedOperatorIDs().length).toBe(0); @@ -305,6 +331,13 @@ describe("LeftPanelComponent", () => { freshFixture.destroy(); }); + it.each(["-1", "NaN", "999"])("ignores an invalid saved dock index %s", saved => { + localStorage.setItem("left-panel-index", saved); + const fresh = TestBed.createComponent(LeftPanelComponent); + expect(fresh.componentInstance.currentIndex).toBe(1); + fresh.destroy(); + }); + it("constructor ignores a saved tab order whose unique-entry count differs from the default", () => { // the guard compares Set sizes only: a duplicate/short set has a different unique count -> fall back to the default order localStorage.setItem("left-panel-order", "1,1,1"); diff --git a/frontend/src/app/workspace/component/left-panel/left-panel.component.ts b/frontend/src/app/workspace/component/left-panel/left-panel.component.ts index d0e8e786502..89362d676d3 100644 --- a/frontend/src/app/workspace/component/left-panel/left-panel.component.ts +++ b/frontend/src/app/workspace/component/left-panel/left-panel.component.ts @@ -109,11 +109,16 @@ export class LeftPanelComponent implements OnDestroy, OnInit, AfterViewInit { // Initialize items array with config values this.updateItemsWithConfig(); - const savedOrder = localStorage.getItem("left-panel-order")?.split(",").map(Number); - this.order = savedOrder && new Set(savedOrder).size === new Set(this.order).size ? savedOrder : this.order; + const savedOrder = localStorage + .getItem("left-panel-order") + ?.split(",") + .map(Number) + .filter(i => this.order.includes(i)); + if (savedOrder?.length === this.order.length && new Set(savedOrder).size === savedOrder.length) + this.order = savedOrder; const savedIndex = Number(localStorage.getItem("left-panel-index")); - this.openFrame(savedIndex < this.items.length && this.items[savedIndex].enabled ? savedIndex : 1); + this.openFrame(this.items[savedIndex]?.enabled ? savedIndex : 1); this.width = Number(localStorage.getItem("left-panel-width")) || this.width; this.height = Number(localStorage.getItem("left-panel-height")) || this.height; diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-breadcrumbs.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-breadcrumbs.component.spec.ts new file mode 100644 index 00000000000..39cd78fe6f4 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-breadcrumbs.component.spec.ts @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { provideRouter, Router } from "@angular/router"; +import { MacroBreadcrumbsComponent } from "./macro-breadcrumbs.component"; +import { MacroNavigationService } from "../../../service/macro/macro-navigation.service"; +import { MacroService } from "../../../service/macro/macro.service"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { testMacro } from "../../../../common/testing/macro-graph"; +import { commonTestImports } from "../../../../common/testing/test-utils"; +import { macroWorkspaceTestProviders } from "../../../service/macro/macro-workspace.spec-harness"; + +describe("MacroBreadcrumbsComponent", () => { + let navigation: MacroNavigationService, macros: MacroService, router: Router; + const error = vi.fn(); + let fixture: ComponentFixture; + const definition = testMacro("m", [], { version: 64, name: "Macro" }); + function addBreadcrumb(instanceName: string, revision: number, readonly = false) { + navigation.parents.push({ + operatorID: instanceName, + instanceName, + session: navigation.actions.suspendWorkflow(), + definition: { ...definition, revision }, + source: { wid: 12, readonly }, + saved: { ...navigation.actions.getWorkflow(), readonly }, + }); + } + beforeEach(async () => { + error.mockClear(); + await TestBed.configureTestingModule({ + imports: [MacroBreadcrumbsComponent, ...commonTestImports], + providers: [ + provideRouter([]), + ...macroWorkspaceTestProviders, + { provide: NotificationService, useValue: { error } }, + ], + }).compileComponents(); + navigation = TestBed.inject(MacroNavigationService); + navigation.actions.suspendWorkflow().shared.wsProvider.disconnect(); + navigation.actions.setWorkflowMetadata({ ...navigation.actions.getWorkflowMetadata(), name: "Parent" }); + navigation.actions.setMacroInterface({ id: definition.id }); + addBreadcrumb("Macro", 2); + vi.spyOn(navigation, "back").mockResolvedValue(); + macros = TestBed.inject(MacroService); + vi.spyOn(macros, "copyDefinition").mockResolvedValue({ ...definition, wid: 77 }); + router = TestBed.inject(Router); + vi.spyOn(router, "navigateByUrl").mockResolvedValue(true); + fixture = TestBed.createComponent(MacroBreadcrumbsComponent); + }); + afterEach(() => navigation.actions.destroySharedModel()); + it("distinguishes a renamed instance from its definition and exposes the reason editing is locked", () => { + navigation.parents.splice(0); + addBreadcrumb("Macro2", 4); + addBreadcrumb("Macro3", 1, true); + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("Macro2 · Revision 4"); + expect(fixture.nativeElement.querySelector(".instance-name")?.textContent).toContain("Macro3"); + expect(fixture.nativeElement.querySelector('[aria-label="Why this macro is read-only"]')).toBeTruthy(); + expect(fixture.nativeElement.textContent).toContain("Revision 1"); + }); + + it("puts accessible nested usages beside macro details and reports refresh failures", async () => { + macros.usages$.next([ + { id: "m", wid: 55, label: "Parent / nested", path: ["outer", "inner"] }, + { id: "other", wid: 56, label: "Other", path: [] }, + ]); + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector('[aria-label="Macro details"]')).toBeTruthy(); + expect(fixture.componentInstance.usages).toEqual([macros.usages$.value[0]]); + vi.spyOn(macros, "refresh").mockRejectedValueOnce(new Error("refresh failed")); + await fixture.componentInstance.refreshUsages(); + expect(error).toHaveBeenCalledWith("refresh failed"); + vi.spyOn(navigation.actions, "getMacroInterface").mockReturnValue(undefined); + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector('[aria-label="Macro details"]')).toBeNull(); + }); + + it("uses the parent hierarchy and truthful revision labels without opening a second canvas", async () => { + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("Parent"); + expect(fixture.nativeElement.textContent).toContain("Revision 2"); + expect(fixture.nativeElement.textContent).not.toContain("64"); + expect(fixture.nativeElement.querySelector("svg.joint-paper")).toBeNull(); + await fixture.componentInstance.back(0); + expect(navigation.back).toHaveBeenCalledWith(0, false); + navigation.parents.splice(0); + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector("nz-breadcrumb")).toBeNull(); + }); + it.each(["nested draft", "portable snapshot"])("retries saving a %s as an independent macro", async kind => { + navigation.dirty = kind === "nested draft"; + if (!navigation.dirty) { + navigation.parents.splice(0); + navigation.actions.disableWorkflowModification(); + } + const before = navigation.actions.getWorkflow(); + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("Save editable copy"); + vi.mocked(macros.copyDefinition).mockRejectedValueOnce(new Error("save failed")); + await fixture.componentInstance.copy(); + expect(error).toHaveBeenCalledWith("save failed"); + expect(fixture.componentInstance.copying).toBe(false); + expect(router.navigateByUrl).not.toHaveBeenCalled(); + expect(navigation.actions.getWorkflow()).toEqual(before); + await fixture.componentInstance.copy(); + expect(macros.copyDefinition).toHaveBeenLastCalledWith(before, []); + if (kind === "nested draft") expect(navigation.back).toHaveBeenCalledWith(0, true); + else expect(navigation.back).not.toHaveBeenCalled(); + expect(router.navigateByUrl).toHaveBeenCalledWith("/user/macros/77"); + expect(navigation.actions.getWorkflow()).toEqual(before); + }); +}); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-breadcrumbs.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-breadcrumbs.component.ts new file mode 100644 index 00000000000..4c0a363daed --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-breadcrumbs.component.ts @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Component } from "@angular/core"; +import { NgFor, NgIf } from "@angular/common"; +import { NzBreadCrumbModule } from "ng-zorro-antd/breadcrumb"; +import { NzIconModule } from "ng-zorro-antd/icon"; +import { NzButtonModule } from "ng-zorro-antd/button"; +import { MacroNavigationService } from "../../../service/macro/macro-navigation.service"; +import { macroRevisionLabel } from "../../../../common/util/workflow-macro"; +import { NzPopoverModule } from "ng-zorro-antd/popover"; +import { Router, RouterLink } from "@angular/router"; +import { macroCanvasUrl, workspaceCanvasUrl } from "../../../../app-routing.constant"; +import { MacroService } from "../../../service/macro/macro.service"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; + +@Component({ + selector: "texera-macro-breadcrumbs", + imports: [NgFor, NgIf, NzBreadCrumbModule, NzButtonModule, NzIconModule, NzPopoverModule, RouterLink], + template: ` `, + styles: [ + ` + :host { + display: inline-block; + vertical-align: middle; + max-width: 100%; + } + nav { + display: flex; + align-items: center; + gap: 4px; + white-space: nowrap; + } + nz-breadcrumb { + max-width: 38vw; + overflow-x: auto; + scrollbar-width: thin; + } + nz-breadcrumb button { + padding: 0 4px; + max-width: 190px; + overflow: hidden; + text-overflow: ellipsis; + } + .separator, + small { + color: #8c8c8c; + } + .instance-name { + margin-right: 4px; + } + .read-only-help { + max-width: 360px; + white-space: normal; + } + .revision { + color: #595959; + font-size: 12px; + margin-right: 8px; + } + `, + ], +}) +export class MacroBreadcrumbsComponent { + public label = macroRevisionLabel; + public copying = false; + public refreshing = false; + public canvasURL = workspaceCanvasUrl; + public sourceURL = macroCanvasUrl; + public get usages() { + return this.macros.usages$.value.filter(u => u.id === this.navigation.actions.getMacroInterface()?.id); + } + public async refreshUsages(): Promise { + if (this.refreshing) return; + this.refreshing = true; + try { + await this.macros.refresh(); + } catch (error) { + this.notification.error(error instanceof Error ? error.message : String(error)); + } finally { + this.refreshing = false; + } + } + constructor( + public navigation: MacroNavigationService, + private notification: NotificationService, + private macros: MacroService, + private router: Router + ) {} + public async copy(): Promise { + if (this.copying) return; + this.copying = true; + try { + const copy = await this.macros.copyDefinition( + this.navigation.actions.getWorkflow(), + this.navigation.actions.getMacroDependencies() + ); + if (this.navigation.parents.length) await this.navigation.back(0, true); + await this.router.navigateByUrl(macroCanvasUrl(copy.wid!)); + } catch (error) { + this.notification.error(error instanceof Error ? error.message : String(error)); + } finally { + this.copying = false; + } + } + public async back(index: number, discard = false): Promise { + try { + await this.navigation.back(index, discard); + } catch (error) { + this.notification.error(error instanceof Error ? error.message : String(error)); + } + } +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html new file mode 100644 index 00000000000..23526051566 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html @@ -0,0 +1,82 @@ + + + +
+ Saved macros + +
+

+ No saved macros yet. Create one from a selection below. +

+

+ Drag a macro onto the canvas. +

+
+ +
+
+ +

+ {{ selectionHint }} +

+ + +
+
+
diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts new file mode 100644 index 00000000000..dea3af5e036 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Component } from "@angular/core"; +import { NgFor, NgIf } from "@angular/common"; +import { FormsModule } from "@angular/forms"; +import { NzButtonModule } from "ng-zorro-antd/button"; +import { NzInputModule } from "ng-zorro-antd/input"; +import { NzCollapseModule } from "ng-zorro-antd/collapse"; +import { MacroService } from "../../../service/macro/macro.service"; +import { WorkflowActionService } from "../../../service/workflow-graph/model/workflow-action.service"; +import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; +import { OperatorLabelComponent } from "./operator-label/operator-label.component"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { MacroDefinition, macroType, macroRevisionLabel } from "../../../../common/util/workflow-macro"; + +@Component({ + selector: "texera-macro-panel", + imports: [NgIf, NgFor, FormsModule, NzButtonModule, NzInputModule, NzCollapseModule, OperatorLabelComponent], + templateUrl: "./macro-panel.component.html", + styleUrls: ["./macro.component.scss"], +}) +export class MacroPanelComponent { + public name = ""; + public busy = false; + public readonly type = macroType; + public readonly revisionLabel = macroRevisionLabel; + public readonly trackDefinition = (_: number, d: MacroDefinition) => macroType(d); + public get canModify(): boolean { + return this.actions.checkWorkflowModificationEnabled(); + } + public get selected(): readonly string[] { + return this.actions.getJointGraphWrapper().getCurrentHighlightedOperatorIDs(); + } + public get selectionHint(): string { + if (!this.canModify) return "Read-only workflow"; + const count = this.selected.length; + return count ? `${count} operator${count === 1 ? "" : "s"} selected` : "Select operators on the canvas"; + } + constructor( + public macros: MacroService, + public actions: WorkflowActionService, + public metadata: OperatorMetadataService, + private notification: NotificationService + ) {} + public async refresh(): Promise { + await this.perform(() => this.macros.refresh()); + } + public async create(): Promise { + await this.perform(async () => { + await this.macros.createFromSelection(this.name); + this.name = ""; + this.notification.success("Macro created"); + }); + } + private async perform(action: () => Promise): Promise { + if (this.busy) return; + this.busy = true; + try { + await action(); + } catch (e) { + this.notification.error(e instanceof Error ? e.message : String(e)); + } finally { + this.busy = false; + } + } +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-version.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-version.component.spec.ts new file mode 100644 index 00000000000..b91d96a55c7 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-version.component.spec.ts @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { testMacro } from "../../../../common/testing/macro-graph"; +import { testBinding } from "../../../../common/testing/workflow-graph"; +import { commonTestImports } from "../../../../common/testing/test-utils"; +import { provideRouter } from "@angular/router"; +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { firstValueFrom, Subject } from "rxjs"; +import { MacroVersionComponent } from "./macro-version.component"; +import { MacroService } from "../../../service/macro/macro.service"; +import { DEFAULT_WORKFLOW, WorkflowActionService } from "../../../service/workflow-graph/model/workflow-action.service"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { MacroDefinition, macroInstance } from "../../../../common/util/workflow-macro"; +import { UndoRedoService } from "../../../service/undo-redo/undo-redo.service"; +import { macroWorkspaceTestProviders } from "../../../service/macro/macro-workspace.spec-harness"; + +describe("MacroVersionComponent", () => { + const definition = testMacro("m", [], { + formBinding: testBinding("leaf", "text", "kept"), + version: 3, + name: "Macro", + wid: 12, + }); + const source = { metadata: { ...DEFAULT_WORKFLOW, name: definition.name, wid: 12 }, revisions: [3, 64] }; + let actions: WorkflowActionService, macros: MacroService; + const error = vi.fn(); + let fixture: ComponentFixture, component: MacroVersionComponent; + const setValues = (properties: Record) => { + actions.setOperatorProperty("selected", properties); + vi.mocked(actions.setOperatorProperty).mockClear(); + }; + const repairButton = () => + Array.from(fixture.nativeElement.querySelectorAll("button")).find(button => + button.textContent?.includes("Remove unavailable values") + ); + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [MacroVersionComponent, ...commonTestImports], + providers: [ + provideRouter([]), + ...macroWorkspaceTestProviders, + { provide: NotificationService, useValue: { error } }, + ], + }).compileComponents(); + error.mockClear(); + macros = TestBed.inject(MacroService); + vi.spyOn(macros, "sourceFor").mockResolvedValue(source); + vi.spyOn(macros, "pin").mockResolvedValue({ ...definition, version: 64, revision: 2 }); + vi.spyOn(macros, "adoptVersion"); + actions = TestBed.inject(WorkflowActionService); + actions.suspendWorkflow().shared.wsProvider.disconnect(); + actions.setMacroDependencies([definition]); + actions.addOperator(macroInstance(definition, "selected"), { x: 0, y: 0 }); + actions.addOperator(macroInstance(definition, "sibling"), { x: 100, y: 0 }); + vi.spyOn(actions, "setOperatorProperty"); + fixture = TestBed.createComponent(MacroVersionComponent); + component = fixture.componentInstance; + component.operatorID = "selected"; + }); + afterEach(() => actions.destroySharedModel()); + + it("shows history ordinals and explicitly switches only the selected usage", async () => { + await component.ngOnChanges(); + fixture.detectChanges(); + expect(component.versions).toEqual([ + { id: 3, label: "Revision 1" }, + { id: 64, label: "Revision 2" }, + ]); + expect(fixture.nativeElement.textContent).toContain("A newer revision is available"); + expect(fixture.nativeElement.textContent).not.toContain("v64"); + component.disabled = true; + fixture.detectChanges(); + await component.use(3); + expect(macros.adoptVersion).not.toHaveBeenCalled(); + expect(fixture.nativeElement.querySelector("button:last-child").disabled).toBe(true); + component.disabled = false; + await component.use(64); + expect(macros.adoptVersion).toHaveBeenCalledExactlyOnceWith("selected", expect.objectContaining({ version: 64 })); + expect(actions.getTexeraGraph().getOperator("selected").operatorType).toBe("WorkflowMacro:m:64"); + expect(actions.getTexeraGraph().getOperator("sibling").operatorType).toBe("WorkflowMacro:m:3"); + }); + + it("keeps portable snapshots viewable when a foreign source id belongs to another definition", async () => { + vi.mocked(macros.sourceFor).mockRejectedValueOnce(new Error("Macro source identity differs")); + await component.ngOnChanges(); + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("Source unavailable"); + expect(macros.pin).not.toHaveBeenCalled(); + await component.use(64); + expect(macros.adoptVersion).not.toHaveBeenCalled(); + }); + + it("does not apply an in-flight revision to another selected operator", async () => { + await component.ngOnChanges(); + fixture.detectChanges(); + const pin = new Subject(); + vi.mocked(macros.pin).mockReturnValue(firstValueFrom(pin)); + const using = component.use(64); + component.operatorID = "another"; + pin.next({ ...definition, version: 64 }); + await using; + expect(macros.adoptVersion).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringMatching(/changed/i)); + }); + + it.each(["history response", "source failure"])("ignores a late %s after selection is cleared", async kind => { + const response = new Subject(); + vi.mocked(macros.sourceFor).mockReturnValueOnce(firstValueFrom(response)); + const loading = component.ngOnChanges(); + component.operatorID = undefined; + await component.ngOnChanges(); + if (kind === "history response") response.next(source); + else response.error(new Error("source unavailable")); + await loading; + fixture.detectChanges(); + + expect(component.versions).toEqual([]); + expect(component.source).toBeUndefined(); + expect(component.latest).toBeUndefined(); + expect(component.busy).toBe(false); + expect(component.unavailable).toBe(false); + }); + + it.each([ + { before: { kept: { text: "東京" }, removed: null }, after: { kept: { text: "東京" } } }, + { before: { removed: "東京" }, after: {} }, + ])("repairs unavailable values and restores the saved graph with Undo/Redo (%j)", ({ before, after }) => { + const undo = TestBed.inject(UndoRedoService); + setValues(before); + undo.clearUndoStack(); + const original = structuredClone(actions.getWorkflowContent()); + fixture.detectChanges(); + repairButton()!.click(); + fixture.detectChanges(); + expect(actions.setOperatorProperty).toHaveBeenCalledWith("selected", after); + expect(repairButton()).toBeUndefined(); + const repaired = actions.getWorkflowContent(); + expect(repaired.operators[0].operatorProperties).toEqual(after); + expect(repaired.macroDependencies).toEqual(original.macroDependencies); + undo.undoAction(); + expect(actions.getWorkflowContent()).toEqual(original); + undo.redoAction(); + expect(actions.getWorkflowContent()).toEqual(repaired); + }); + + it.each(["disabled", "busy"] as const)("keeps unavailable values while %s", guard => { + setValues({ removed: "preserve" }); + component[guard] = true; + fixture.detectChanges(); + expect(repairButton()?.disabled).toBe(true); + component.removeUnavailableValues(); + expect(actions.setOperatorProperty).not.toHaveBeenCalled(); + }); + + it("does not offer or perform repair without unavailable values or a selected macro", () => { + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).not.toContain("Remove unavailable values"); + component.removeUnavailableValues(); + component.operatorID = "missing"; + fixture.detectChanges(); + component.removeUnavailableValues(); + expect(actions.setOperatorProperty).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-version.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-version.component.ts new file mode 100644 index 00000000000..b20e3a92cf2 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-version.component.ts @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Component, Input, OnChanges } from "@angular/core"; +import { NgFor, NgIf } from "@angular/common"; +import { FormsModule } from "@angular/forms"; +import { RouterLink } from "@angular/router"; +import { macroCanvasUrl } from "../../../../app-routing.constant"; +import { NzSelectModule } from "ng-zorro-antd/select"; +import { NzButtonModule } from "ng-zorro-antd/button"; +import { MacroService } from "../../../service/macro/macro.service"; +import { WorkflowActionService } from "../../../service/workflow-graph/model/workflow-action.service"; +import { MacroDefinition, macroRevisionLabel, macroType } from "../../../../common/util/workflow-macro"; +import type { WorkflowMetadata } from "../../../../dashboard/type/workflow-metadata.interface"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; + +@Component({ + selector: "texera-macro-version", + imports: [NgIf, NgFor, FormsModule, NzSelectModule, NzButtonModule, RouterLink], + template: `
+ + + + + {{ label(d) }} + Definition: {{ s.name }} +

Source unavailable. The saved snapshot still runs and exports with this workflow.

+

A newer revision is available.

+
+ + +
+ +
`, + styles: [ + ` + .macro-version { + display: grid; + gap: 8px; + margin-bottom: 16px; + padding-bottom: 12px; + border-bottom: 1px solid #f0f0f0; + } + label { + font-weight: 500; + } + p { + color: #595959; + margin: 0; + font-size: 12px; + } + button + button { + margin-left: 8px; + } + `, + ], +}) +export class MacroVersionComponent implements OnChanges { + @Input() operatorID?: string; + @Input() disabled = false; + public versions: { id: number; label: string }[] = []; + public latest?: number; + public busy = false; + public unavailable = false; + public label = macroRevisionLabel; + public sourceURL = macroCanvasUrl; + public source?: WorkflowMetadata; + private request = 0; + public get definition(): MacroDefinition | undefined { + const graph = this.actions.getTexeraGraph(); + const op = this.operatorID && graph.hasOperator(this.operatorID) ? graph.getOperator(this.operatorID) : undefined; + return this.actions.getMacroDependencies().find(d => macroType(d) === op?.operatorType); + } + + public get unavailableValues(): string[] { + const definition = this.definition; + if (!definition || !this.operatorID) return []; + return Object.keys(this.actions.getTexeraGraph().getOperator(this.operatorID).operatorProperties).filter( + key => !definition.content.formBinding?.fields.some(field => field.id === key) + ); + } + + public removeUnavailableValues(): void { + if (this.disabled || this.busy || !this.operatorID) return; + const unavailable = new Set(this.unavailableValues); + if (!unavailable.size) return; + const properties = this.actions.getTexeraGraph().getOperator(this.operatorID).operatorProperties; + this.actions.setOperatorProperty( + this.operatorID, + Object.fromEntries(Object.entries(properties).filter(([key]) => !unavailable.has(key))) + ); + } + constructor( + public macros: MacroService, + private actions: WorkflowActionService, + private notification: NotificationService + ) {} + + public async ngOnChanges(): Promise { + const request = ++this.request; + const d = this.definition; + this.versions = []; + this.latest = undefined; + this.source = undefined; + this.unavailable = false; + this.busy = !!d; + if (!d) return; + try { + const { metadata, revisions } = await this.macros.sourceFor(d); + if (request !== this.request) return; + this.source = metadata; + this.versions = revisions.map((id, i) => ({ id, label: `Revision ${i + 1}` })); + this.latest = this.versions.at(-1)?.id; + } catch { + if (request === this.request) this.unavailable = true; + } finally { + if (request === this.request) this.busy = false; + } + } + + public async use(version?: number): Promise { + if ( + this.disabled || + this.busy || + !this.source || + !this.operatorID || + (version !== undefined && !this.versions.some(v => v.id === version)) + ) + return; + this.busy = true; + const before = JSON.stringify(this.actions.getWorkflow()); + const operatorID = this.operatorID; + try { + const next = await this.macros.pin(this.definition, version); + if (this.operatorID !== operatorID || JSON.stringify(this.actions.getWorkflow()) !== before) + throw new Error("The workflow changed. Select the revision again."); + await this.macros.adoptVersion(operatorID, next); + await this.ngOnChanges(); + } catch (error) { + this.notification.error(error instanceof Error ? error.message : String(error)); + } finally { + this.busy = false; + } + } +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss b/frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss new file mode 100644 index 00000000000..43ed9a31b24 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +:host { + display: block; + min-width: 0; +} +.macro-interface { + padding: 12px; + border-bottom: 1px solid #f0f0f0; +} +summary { + cursor: pointer; + font-weight: 600; +} +.badge { + display: inline-block; + padding: 1px 8px; + border-radius: 12px; + background: #f0f5ff; + color: #597ef7; + font-size: 12px; + font-weight: 400; + white-space: nowrap; +} +.section-header, +.actions { + display: flex; + align-items: center; + gap: 8px; +} +.section-header { + justify-content: space-between; +} +.actions { + flex-wrap: wrap; +} +.content-section { + margin-top: 16px; + padding-top: 12px; + border-top: 1px solid #f0f0f0; +} +.muted, +.empty-state { + color: #8c8c8c; + font-size: 12px; + line-height: 1.6; +} +p { + margin: 8px 0; +} +.empty-state { + padding: 12px; + border: 1px dashed #d9d9d9; + border-radius: 6px; + background: #fafafa; +} +.operator-label { + padding-left: 7px; + height: 32px; +} +label { + display: block; + font-weight: 600; +} +input, +nz-select { + width: 100%; + min-width: 0; + margin: 8px 0; +} +form button { + margin-top: 4px; +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.html index eaaf4feeb84..4eaa9d82739 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.html +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.html @@ -20,6 +20,7 @@
{ // the label is visibly marked undraggable ... const label = fixture.debugElement.query(By.css(".operator-label")).nativeElement; expect(label.classList.contains("disable-drag-drop")).toBe(true); + expect(cdkDrag.disabled).toBe(true); + const drop = vi.spyOn(dragDropService, "dragDropped").mockImplementation(() => {}); + cdkDrag.dropped.emit({ dropPoint: { x: 137, y: 421 } } as unknown as CdkDragDrop); + expect(drop).not.toHaveBeenCalled(); // ... and a drag that fires anyway is dropped on the floor cdkDrag.started.emit({ source: cdkDrag } as unknown as CdkDragStart); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.ts index 56eba9d7d24..4ee3da86528 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-label/operator-label.component.ts @@ -58,6 +58,6 @@ export class OperatorLabelComponent implements AfterContentInit { } dragDropped(dropPoint: Point) { - this.dragDropService.dragDropped(dropPoint); + if (this.draggable) this.dragDropService.dragDropped(dropPoint); } } diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html index ff8bb97a296..8ed380d9c99 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html @@ -29,7 +29,7 @@ [nzBackfill]="false" (selectionChange)="onSelectionChange($event)"> @@ -39,6 +39,7 @@
+ @@ -59,7 +60,7 @@
diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.spec.ts index a9e133827f6..fe7a22b6b36 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.spec.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.spec.ts @@ -17,6 +17,7 @@ * under the License. */ +import { testMacro } from "../../../../common/testing/macro-graph"; import { mockOperatorGroup, mockScanSourceSchema, @@ -24,6 +25,7 @@ import { import { UndoRedoService } from "../../../service/undo-redo/undo-redo.service"; import { DragDropService } from "../../../service/drag-drop/drag-drop.service"; import { ComponentFixture, fakeAsync, TestBed, tick } from "@angular/core/testing"; +import { By } from "@angular/platform-browser"; import { BrowserAnimationsModule } from "@angular/platform-browser/animations"; import { OperatorMenuComponent } from "./operator-menu.component"; import { OperatorLabelComponent } from "./operator-label/operator-label.component"; @@ -39,6 +41,11 @@ import type { NzAutocompleteOptionComponent } from "ng-zorro-antd/auto-complete" import type * as joint from "jointjs"; import { commonTestProviders } from "../../../../common/testing/test-utils"; +import { MacroService } from "../../../service/macro/macro.service"; +import { macroType } from "../../../../common/util/workflow-macro"; +import { MacroPanelComponent } from "./macro-panel.component"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; + describe("OperatorPanelComponent", () => { let component: OperatorMenuComponent; let fixture: ComponentFixture; @@ -74,6 +81,110 @@ describe("OperatorPanelComponent", () => { component = fixture.componentInstance; }); + it("finds loaded saved macros through the existing operator search and selection flow", () => { + const actions = TestBed.inject(WorkflowActionService); + const definition = testMacro("saved", actions.getWorkflowContent().operators, { + ...actions.getWorkflowContent(), + name: "Saved 清理", + }); + component.searchInputValue = "Saved 清理"; + component.onInput({ target: { value: "Saved 清理" } } as unknown as Event); + expect(component.autocompleteOptions).toEqual([]); + TestBed.inject(OperatorMetadataService).registerMacros([definition]); + TestBed.inject(MacroService).library$.next([definition]); + expect(component.autocompleteOptions.map(op => op.operatorType)).toContain(macroType(definition)); + component.onSelectionChange({ nzValue: component.autocompleteOptions[0] } as NzAutocompleteOptionComponent); + expect( + actions + .getTexeraGraph() + .getAllOperators() + .some(op => op.operatorType === macroType(definition)) + ).toBe(true); + TestBed.inject(MacroService).library$.next([]); + expect(component.autocompleteOptions).toEqual([]); + }); + + it("offers boundary operators only inside a macro and only while that role is missing", () => { + const actions = TestBed.inject(WorkflowActionService); + const types = ["MacroInput", "MacroOutput"]; + const schemas = types.map(type => TestBed.inject(OperatorMetadataService).getOperatorSchema(type)); + expect(component.availableOperators(schemas)).toEqual([]); + actions.setMacroInterface({ id: "definition" }); + expect(component.availableOperators(schemas).map(op => op.operatorType)).toEqual(types); + const operator = TestBed.inject(WorkflowUtilService).getNewOperatorPredicate("MacroInput"); + actions.addOperator(operator, { x: 0, y: 0 }); + expect(component.availableOperators(schemas).map(op => op.operatorType)).toEqual(["MacroOutput"]); + actions.deleteOperatorsAndLinks([operator.operatorID]); + expect(component.availableOperators(schemas).map(op => op.operatorType)).toEqual(types); + }); + + describe("Macros library", () => { + let panel: MacroPanelComponent; + let panelElement: HTMLElement; + let actions: WorkflowActionService; + const createButton = () => panelElement.querySelector('[data-action="create"]')!; + + beforeEach(() => { + const child = fixture.debugElement.query(By.directive(MacroPanelComponent)); + panel = child.componentInstance; + panelElement = child.nativeElement; + actions = TestBed.inject(WorkflowActionService); + vi.spyOn(actions.getJointGraphWrapper(), "getCurrentHighlightedOperatorIDs").mockReturnValue(["leaf"]); + }); + + it("labels library tooltips with the workflow revision rather than its storage id", () => { + const definition = { ...testMacro("macro", [], { version: 64, name: "Macro" }), revision: 2 }; + TestBed.inject(OperatorMetadataService).registerMacros([definition]); + TestBed.inject(MacroService).library$.next([definition]); + fixture.detectChanges(); + expect(panelElement.querySelector(".operator-label[title]")?.getAttribute("title")).toBe("Macro · Revision 2"); + }); + + it("explains empty selection and read-only state without hiding the library", () => { + vi.mocked(actions.getJointGraphWrapper().getCurrentHighlightedOperatorIDs).mockReturnValue([]); + fixture.detectChanges(); + expect(panelElement.textContent).toContain("No saved macros yet"); + expect(panelElement.textContent).toContain("Select operators on the canvas"); + expect(createButton().disabled).toBe(true); + actions.disableWorkflowModification(); + fixture.detectChanges(); + expect(panelElement.textContent).toContain("Read-only workflow"); + expect(panelElement.querySelector(".ant-collapse-header")?.textContent).toContain("Macros"); + }); + + it("submits a named selection from the keyboard and clears the name only after success", async () => { + const create = vi + .spyOn(TestBed.inject(MacroService), "createFromSelection") + .mockResolvedValue(testMacro("macro", [])); + const notifications = TestBed.inject(NotificationService); + const success = vi.spyOn(notifications, "success").mockImplementation(() => {}); + const error = vi.spyOn(notifications, "error").mockImplementation(() => {}); + panel.name = "清理 · café"; + actions.disableWorkflowModification(); + fixture.detectChanges(); + expect(createButton().disabled).toBe(true); + actions.enableWorkflowModification(); + fixture.detectChanges(); + expect(panelElement.textContent).toContain("1 operator selected"); + panelElement.querySelector("form")!.dispatchEvent(new Event("submit", { cancelable: true })); + await vi.waitFor(() => expect(success).toHaveBeenCalledWith("Macro created")); + expect(create).toHaveBeenCalledWith("清理 · café"); + expect(panel.name).toBe(""); + create.mockRejectedValue(new Error("save failed")); + panel.name = "Try again"; + await panel.create(); + expect(panel.name).toBe("Try again"); + expect(panel.busy).toBe(false); + expect(error).toHaveBeenCalledWith("save failed"); + }); + + it("keeps interface and usage controls out of the operator library", () => { + expect(panelElement.textContent).not.toContain("Used by"); + expect(panelElement.querySelector('[data-action="add-input"]')).toBeNull(); + expect(panelElement.querySelector('[aria-label="Property target"]')).toBeNull(); + }); + }); + it("should create", () => { expect(component).toBeTruthy(); }); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts index ed172ae5d13..d537c1358dd 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts @@ -18,6 +18,9 @@ */ import { Component } from "@angular/core"; +import { combineLatest } from "rxjs"; +import { MacroService } from "../../../service/macro/macro.service"; +import { macroType, isMacroTerminal } from "../../../../common/util/workflow-macro"; import Fuse from "fuse.js"; import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; import { GroupInfo, OperatorSchema } from "../../../types/operator-schema.interface"; @@ -36,6 +39,7 @@ import { FormsModule } from "@angular/forms"; import { NgFor, NgTemplateOutlet } from "@angular/common"; import { OperatorLabelComponent } from "./operator-label/operator-label.component"; import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/collapse"; +import { MacroPanelComponent } from "./macro-panel.component"; @UntilDestroy() @Component({ @@ -43,6 +47,7 @@ import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/col templateUrl: "operator-menu.component.html", styleUrls: ["operator-menu.component.scss"], imports: [ + MacroPanelComponent, NzSpaceCompactItemDirective, NzInputDirective, FormsModule, @@ -81,7 +86,8 @@ export class OperatorMenuComponent { private operatorMetadataService: OperatorMetadataService, private workflowActionService: WorkflowActionService, private workflowUtilService: WorkflowUtilService, - private dragDropService: DragDropService + private dragDropService: DragDropService, + private macros: MacroService ) { // clear the search box if an operator is dropped from operator search box this.dragDropService.operatorDropStream.pipe(untilDestroyed(this)).subscribe(() => { @@ -92,14 +98,14 @@ export class OperatorMenuComponent { .getWorkflowModificationEnabledStream() .pipe(untilDestroyed(this)) .subscribe(canModify => (this.canModify = canModify)); - this.operatorMetadataService - .getOperatorMetadata() + combineLatest([this.operatorMetadataService.getOperatorMetadata(), this.macros.library$]) .pipe(untilDestroyed(this)) - .subscribe(operatorMetadata => { + .subscribe(([operatorMetadata, definitions]) => { const ops = operatorMetadata.operators.filter( - operatorSchema => operatorSchema.operatorType !== "PythonUDF" && operatorSchema.operatorType !== "Dummy" + operatorSchema => !["PythonUDF", "Dummy"].includes(operatorSchema.operatorType) ); this.groupNames = operatorMetadata.groups; + this.opList.clear(); ops.forEach(x => { if (x.operatorType !== "Sleep") { const group = x.additionalMetadata.operatorGroupName; @@ -111,22 +117,38 @@ export class OperatorMenuComponent { this.opList.forEach(value => { value.sort((a, b) => a.operatorType.localeCompare(b.operatorType)); }); - this.fuse.setCollection(ops); + this.fuse.setCollection([ + ...ops, + ...definitions.map(d => + this.operatorMetadataService.getOperatorSchema(macroType(d), [d, ...(d.content.macroDependencies ?? [])]) + ), + ]); + this.search(this.searchInputValue); }); } + public availableOperators(operators: readonly OperatorSchema[] = []): readonly OperatorSchema[] { + return operators.filter( + op => + !isMacroTerminal(op) || + (!!this.workflowActionService.getMacroInterface() && + !this.workflowActionService + .getTexeraGraph() + .getAllOperators() + .some(node => node.operatorType === op.operatorType)) + ); + } + /** * create the search results observable * whenever the search box text is changed, perform the search using fuse.js */ onInput(e: Event): void { - const v = (e.target as HTMLInputElement).value; - if (v === null || v.trim().length === 0) { - this.autocompleteOptions = []; - } - this.autocompleteOptions = this.fuse.search(v).map(item => { - return item.item; - }); + this.search((e.target as HTMLInputElement).value); + } + + private search(value: string): void { + this.autocompleteOptions = this.fuse.search(value).map(item => item.item); } /** diff --git a/frontend/src/app/workspace/component/left-panel/time-travel/time-travel.component.spec.ts b/frontend/src/app/workspace/component/left-panel/time-travel/time-travel.component.spec.ts index ba44232002f..982c5b668a1 100644 --- a/frontend/src/app/workspace/component/left-panel/time-travel/time-travel.component.spec.ts +++ b/frontend/src/app/workspace/component/left-panel/time-travel/time-travel.component.spec.ts @@ -197,7 +197,7 @@ describe("TimeTravelComponent", () => { vi.spyOn(versionService, "retrieveWorkflowByVersion").mockReturnValue(of(versionedWorkflow)); const displaySpy = vi.spyOn(versionService, "displayReadonlyWorkflow").mockImplementation(() => {}); const infoSpy = vi.spyOn(notificationService, "info").mockImplementation(() => undefined as any); - const replaySpy = vi.spyOn(executeService, "executeWorkflowWithReplay").mockImplementation(() => {}); + const replaySpy = vi.spyOn(executeService, "executeWorkflowWithReplay").mockReturnValue(true); // This test sets revertedToInteraction, so ngOnDestroy (via fixture.destroy in // afterEach) would run the real replay-cleanup; keep those calls inert. vi.spyOn(versionService, "closeReadonlyWorkflowDisplay").mockImplementation(() => {}); diff --git a/frontend/src/app/workspace/component/left-panel/versions-list/versions-list.component.html b/frontend/src/app/workspace/component/left-panel/versions-list/versions-list.component.html index bed3bd1567a..4df8f0ee1c3 100644 --- a/frontend/src/app/workspace/component/left-panel/versions-list/versions-list.component.html +++ b/frontend/src/app/workspace/component/left-panel/versions-list/versions-list.component.html @@ -51,6 +51,7 @@ - + + + + diff --git a/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.spec.ts b/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.spec.ts index 3d751dffec7..7884096e0c2 100644 --- a/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.spec.ts +++ b/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.spec.ts @@ -17,6 +17,9 @@ * under the License. */ +import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; +import { StubOperatorMetadataService } from "../../../service/operator-metadata/stub-operator-metadata.service"; +import { setMacroGraph } from "../../../../common/testing/macro-graph"; import { SimpleChange } from "@angular/core"; import { ComponentFixture, fakeAsync, TestBed, tick } from "@angular/core/testing"; import { By } from "@angular/platform-browser"; @@ -48,7 +51,11 @@ describe("PortPropertyEditFrameComponent", () => { beforeEach(async () => { await TestBed.configureTestingModule({ - providers: [WorkflowActionService, ...commonTestProviders], + providers: [ + WorkflowActionService, + ...commonTestProviders, + { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, + ], imports: [ PortPropertyEditFrameComponent, HttpClientTestingModule, @@ -72,6 +79,23 @@ describe("PortPropertyEditFrameComponent", () => { fixture.destroy(); }); + it("offers explicit input replacement through the shared exposure action and respects viewer mode", () => { + const target = { operatorID: "op-1", portID: "input-0" }; + component.currentPortID = target; + setMacroGraph(workflowActionService, { id: "macro", inputs: [], outputs: [] }); + vi.spyOn(workflowActionService.macroTerminals, "direction").mockReturnValue("inputs"); + vi.spyOn(workflowActionService.macroTerminals, "needsReplacement").mockReturnValue(true); + const expose = vi.spyOn(workflowActionService.macroTerminals, "expose").mockReturnValue(undefined); + fixture.detectChanges(); + const button = fixture.nativeElement.querySelector('[data-action="expose-port"]'); + expect(button.textContent).toContain("Replace with macro input"); + button.click(); + expect(expose).toHaveBeenCalledWith(target, true); + (component as any).actsAsEditor = false; + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector('[data-action="expose-port"]').disabled).toBe(true); + }); + it("should create", () => { expect(component).toBeTruthy(); }); @@ -428,7 +452,7 @@ describe("PortPropertyEditFrameComponent", () => { it("should blur, drop the binding and exit editing on disconnect", () => { const blur = vi.fn(); component.quill = { blur } as any; - component.quillBinding = {} as any; + component.quillBinding = { destroy: vi.fn() } as any; component.editingTitle = true; component.disconnectQuillFromText(); diff --git a/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.ts b/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.ts index e8808c2279b..1ac894adb4e 100644 --- a/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.ts +++ b/frontend/src/app/workspace/component/property-editor/port-property-edit-frame/port-property-edit-frame.component.ts @@ -63,6 +63,7 @@ Quill.register("modules/cursors", QuillCursors); ], }) export class PortPropertyEditFrameComponent implements OnInit, OnChanges { + @Input() actsAsEditor = true; @Input() currentPortID: LogicalPort | undefined; // whether the editor can be edited @@ -91,10 +92,24 @@ export class PortPropertyEditFrameComponent implements OnInit, OnChanges { constructor( private formlyJsonschema: FormlyJsonschema, - private workflowActionService: WorkflowActionService, + public workflowActionService: WorkflowActionService, private dynamicSchemaService: DynamicSchemaService ) {} + public get exposureDirection() { + return this.currentPortID && this.workflowActionService.macroTerminals.direction(this.currentPortID); + } + public get needsReplacement(): boolean { + return !!this.currentPortID && this.workflowActionService.macroTerminals.needsReplacement(this.currentPortID); + } + public get canExpose(): boolean { + return this.actsAsEditor && this.interactive && this.workflowActionService.checkWorkflowModificationEnabled(); + } + public expose(): void { + if (this.canExpose && this.currentPortID) + this.workflowActionService.macroTerminals.expose(this.currentPortID, this.needsReplacement); + } + ngOnInit(): void { this.registerPortPropertyChangeHandler(); this.registerPortDisplayNameChangeHandler(); @@ -102,6 +117,7 @@ export class PortPropertyEditFrameComponent implements OnInit, OnChanges { } ngOnChanges(changes: SimpleChanges): void { + if (!changes.currentPortID) return; this.currentPortID = changes.currentPortID.currentValue; if (this.currentPortID) this.showPortPropertyEditor(this.currentPortID); } @@ -120,6 +136,7 @@ export class PortPropertyEditFrameComponent implements OnInit, OnChanges { * Connects the actual y-text structure of this operator's name to the editor's awareness manager. */ connectQuillToText() { + if (!this.interactive || !this.actsAsEditor) return; this.registerQuillBinding(); if (!this.currentPortID) return; const currentPortDescriptorSharedType = this.workflowActionService @@ -142,6 +159,7 @@ export class PortPropertyEditFrameComponent implements OnInit, OnChanges { */ disconnectQuillFromText() { this.quill.blur(); + this.quillBinding?.destroy(); this.quillBinding = undefined; this.editingTitle = false; } diff --git a/frontend/src/app/workspace/component/property-editor/property-editor.component.spec.ts b/frontend/src/app/workspace/component/property-editor/property-editor.component.spec.ts index a0fba29f5ed..2ca5aa0d932 100644 --- a/frontend/src/app/workspace/component/property-editor/property-editor.component.spec.ts +++ b/frontend/src/app/workspace/component/property-editor/property-editor.component.spec.ts @@ -37,6 +37,9 @@ import { StubOperatorMetadataService } from "../../service/operator-metadata/stu import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; import { MockComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; import { commonTestProviders } from "../../../common/testing/test-utils"; +import { FormlyModule } from "@ngx-formly/core"; +import { FormlyNgZorroAntdModule } from "@ngx-formly/ng-zorro-antd"; +import { TEXERA_FORMLY_CONFIG } from "../../../common/formly/formly-config"; describe("PropertyEditorComponent", () => { let component: PropertyEditorComponent; @@ -46,7 +49,13 @@ describe("PropertyEditorComponent", () => { beforeEach(async () => { await TestBed.configureTestingModule({ - imports: [PropertyEditorComponent, CommonModule, HttpClientTestingModule], + imports: [ + PropertyEditorComponent, + CommonModule, + HttpClientTestingModule, + FormlyModule.forRoot(TEXERA_FORMLY_CONFIG), + FormlyNgZorroAntdModule, + ], providers: [ { provide: OperatorMetadataService, @@ -338,7 +347,7 @@ describe("PropertyEditorComponent", () => { expect(jointGraphWrapper.getCurrentHighlightedPortIDs()).toEqual([port]); expect(component.currentComponent).toBe(PortPropertyEditFrameComponent); - expect(component.componentInputs).toEqual({ currentPortID: port }); + expect(component.componentInputs).toEqual({ currentPortID: port, actsAsEditor: true }); }); it("should switch from the operator frame to the port frame when the highlight moves from an operator to a port", () => { @@ -359,7 +368,7 @@ describe("PropertyEditorComponent", () => { expect(jointGraphWrapper.getCurrentHighlightedOperatorIDs()).toEqual([]); expect(component.currentComponent).toBe(PortPropertyEditFrameComponent); - expect(component.componentInputs).toEqual({ currentPortID: port }); + expect(component.componentInputs).toEqual({ currentPortID: port, actsAsEditor: true }); }); it("should clear the panel when a port and a link are highlighted at the same time", () => { diff --git a/frontend/src/app/workspace/component/property-editor/property-editor.component.ts b/frontend/src/app/workspace/component/property-editor/property-editor.component.ts index 39dacaf9c59..deed3839f94 100644 --- a/frontend/src/app/workspace/component/property-editor/property-editor.component.ts +++ b/frontend/src/app/workspace/component/property-editor/property-editor.component.ts @@ -154,6 +154,8 @@ export class PropertyEditorComponent implements OnInit, OnDestroy, OnChanges { const remountOn = ["exposeChoosing", "actsAsEditor"]; if (remountOn.some(input => changes[input] !== undefined && !changes[input].firstChange)) { this.remountOperatorFrame(); + if (this.currentComponent === PortPropertyEditFrameComponent) + this.componentInputs = { ...this.componentInputs, actsAsEditor: this.actsAsEditor }; } } @@ -313,7 +315,7 @@ export class PropertyEditorComponent implements OnInit, OnDestroy, OnChanges { }; } else if (highlightedPorts.length === 1 && highlightedLinks.length === 0) { this.currentComponent = PortPropertyEditFrameComponent; - this.componentInputs = { currentPortID: highlightedPorts[0] }; + this.componentInputs = { currentPortID: highlightedPorts[0], actsAsEditor: this.actsAsEditor }; } else { this.currentComponent = null; this.componentInputs = {}; diff --git a/frontend/src/app/workspace/component/property-editor/typecasting-display/type-casting-display.component.spec.ts b/frontend/src/app/workspace/component/property-editor/typecasting-display/type-casting-display.component.spec.ts index 68a2c4e935d..87bf596f4f5 100644 --- a/frontend/src/app/workspace/component/property-editor/typecasting-display/type-casting-display.component.spec.ts +++ b/frontend/src/app/workspace/component/property-editor/typecasting-display/type-casting-display.component.spec.ts @@ -17,6 +17,8 @@ * under the License. */ +import { ComputingUnitStatusService } from "../../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; +import { MockComputingUnitStatusService } from "../../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; import { ComponentFixture, TestBed } from "@angular/core/testing"; import { By } from "@angular/platform-browser"; import { HttpClientTestingModule } from "@angular/common/http/testing"; @@ -60,6 +62,7 @@ describe("TypecastingDisplayComponent", () => { WorkflowActionService, WorkflowCompilingService, ...commonTestProviders, + { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, ], }).compileComponents(); }); diff --git a/frontend/src/app/workspace/component/result-panel/ai-fix/ai-workflow-fixer.service.ts b/frontend/src/app/workspace/component/result-panel/ai-fix/ai-workflow-fixer.service.ts index 306ba4df35a..9a8fce888c7 100644 --- a/frontend/src/app/workspace/component/result-panel/ai-fix/ai-workflow-fixer.service.ts +++ b/frontend/src/app/workspace/component/result-panel/ai-fix/ai-workflow-fixer.service.ts @@ -339,7 +339,7 @@ export class AiWorkflowFixerService { // here means every one it gains has to be repeated, and missing one reports // "re-running" over a toast saying the opposite. The empty execution name is the one // the operator menu uses for an ad-hoc run. - const started = this.executeWorkflowService.executeWorkflow(""); + const started = await this.executeWorkflowService.executeWorkflow(""); if (seq !== this.analysisSeq) { return; } diff --git a/frontend/src/app/workspace/component/result-panel/console-frame/console-frame.component.spec.ts b/frontend/src/app/workspace/component/result-panel/console-frame/console-frame.component.spec.ts index 83e4018f9a9..a98ab8f8423 100644 --- a/frontend/src/app/workspace/component/result-panel/console-frame/console-frame.component.spec.ts +++ b/frontend/src/app/workspace/component/result-panel/console-frame/console-frame.component.spec.ts @@ -19,7 +19,7 @@ import { ComponentFixture, TestBed, fakeAsync, flush, tick } from "@angular/core/testing"; import { By } from "@angular/platform-browser"; -import { Subject } from "rxjs"; +import { EMPTY, Subject } from "rxjs"; import { ConsoleFrameComponent } from "./console-frame.component"; import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; import { StubOperatorMetadataService } from "../../../service/operator-metadata/stub-operator-metadata.service"; @@ -52,6 +52,11 @@ function consoleMessage(name: string): ConsoleMessage { type StateEvent = { previous: { state: ExecutionState }; current: { state: ExecutionState } }; +const compilingProvider = { + provide: WorkflowCompilingService, + useValue: { getWorkflowCompilationErrors: () => ({}), getCompilationStateInfoChangedStream: () => EMPTY }, +}; + describe("ConsoleFrameComponent", () => { let component: ConsoleFrameComponent; let fixture: ComponentFixture; @@ -106,6 +111,7 @@ describe("ConsoleFrameComponent", () => { getAllConsoleMessages, }, }, + compilingProvider, { provide: WorkflowWebsocketService, useValue: { send } }, { provide: NotificationService, useValue: { error: notifyError } }, { provide: UdfDebugService, useValue: { doStep, doContinue } }, @@ -824,6 +830,7 @@ describe("ConsoleFrameComponent settings dropdown", () => { getAllConsoleMessages: () => [], }, }, + compilingProvider, { provide: WorkflowWebsocketService, useValue: { send: () => {} } }, { provide: NotificationService, useValue: { error: () => {} } }, { provide: UdfDebugService, useValue: { doStep: () => {}, doContinue: () => {} } }, diff --git a/frontend/src/app/workspace/component/result-panel/result-table-frame/result-table-frame.component.ts b/frontend/src/app/workspace/component/result-panel/result-table-frame/result-table-frame.component.ts index 75ff2d6acfb..bc8580875a3 100644 --- a/frontend/src/app/workspace/component/result-panel/result-table-frame/result-table-frame.component.ts +++ b/frontend/src/app/workspace/component/result-panel/result-table-frame/result-table-frame.component.ts @@ -263,12 +263,6 @@ export class ResultTableFrameComponent implements OnInit, OnChanges { if (isDefined(this.containerHeight)) return; this.fitPageSize(size.height); }); - - if (this.operatorId) { - const paginatedResultService = this.workflowResultService.getPaginatedResultService(this.operatorId); - if (paginatedResultService) { - } - } } checkKeys( diff --git a/frontend/src/app/workspace/component/workflow-editor/context-menu/context-menu/context-menu.component.spec.ts b/frontend/src/app/workspace/component/workflow-editor/context-menu/context-menu/context-menu.component.spec.ts index a651a08f7ec..c3adafebaf0 100644 --- a/frontend/src/app/workspace/component/workflow-editor/context-menu/context-menu/context-menu.component.spec.ts +++ b/frontend/src/app/workspace/component/workflow-editor/context-menu/context-menu/context-menu.component.spec.ts @@ -17,6 +17,8 @@ * under the License. */ +import { ComputingUnitStatusService } from "../../../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; +import { MockComputingUnitStatusService } from "../../../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; import { ComponentFixture, TestBed } from "@angular/core/testing"; import { By } from "@angular/platform-browser"; import { OperatorMetadataService } from "src/app/workspace/service/operator-metadata/operator-metadata.service"; @@ -144,6 +146,7 @@ describe("ContextMenuComponent", () => { { provide: WorkflowCompilingService, useValue: workflowCompilingServiceSpy }, NzModalService, // Provide NzModalService ...commonTestProviders, + { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, ], imports: [ ContextMenuComponent, @@ -858,6 +861,7 @@ describe("ContextMenuComponent onDelete with real WorkflowActionService", () => { provide: ValidationWorkflowService, useValue: { validateOperator: vi.fn() } }, NzModalService, ...commonTestProviders, + { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, ], imports: [ ContextMenuComponent, diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.browser.spec.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.browser.spec.ts index 99fcf59ebfc..9b4f8bb74ba 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.browser.spec.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.browser.spec.ts @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { testMacro, testPort, setMacroGraph } from "../../../common/testing/macro-graph"; /** * Mouse / pointer event tests for WorkflowEditorComponent. @@ -33,13 +34,16 @@ * runner. Restored here per issue #5318. */ +import { macroInstance } from "../../../common/util/workflow-macro"; +import { readMacroInterface } from "../../../../../../common/workflow-core/src/main/typescript/workflow-macros"; import { ComponentFixture, TestBed } from "@angular/core/testing"; import { NzModalRef, NzModalService } from "ng-zorro-antd/modal"; import * as jQuery from "jquery"; +import * as joint from "jointjs"; import { WorkflowEditorComponent } from "./workflow-editor.component"; import { NzModalCommentBoxComponent } from "./comment-box-modal/nz-modal-comment-box.component"; -import { workflowEditorTestImports, workflowEditorTestProviders } from "./workflow-editor.test-utils"; +import { configureWorkflowEditor } from "./workflow-editor.test-utils"; import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; import { @@ -47,8 +51,14 @@ import { mockPoint, mockResultPredicate, mockScanPredicate, + mockSentimentPredicate, } from "../../service/workflow-graph/model/mock-workflow-data"; import { createYTypeFromObject } from "../../types/shared-editing.interface"; +import { page, userEvent } from "vitest/browser"; + +import { UndoRedoService } from "../../service/undo-redo/undo-redo.service"; +import { exportedWorkflow } from "../../../common/type/workflow"; +import { PortPropertyEditFrameComponent } from "../property-editor/port-property-edit-frame/port-property-edit-frame.component"; const createJQueryEvent = (event: string, properties?: object): JQuery.Event => (jQuery as unknown as JQueryStatic).Event(event, properties); @@ -59,12 +69,7 @@ describe("WorkflowEditorComponent - mouse and pointer event integration", () => let workflowActionService: WorkflowActionService; let nzModalService: NzModalService; - beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: workflowEditorTestImports, - providers: workflowEditorTestProviders, - }).compileComponents(); - }); + beforeEach(configureWorkflowEditor); beforeEach(() => { fixture = TestBed.createComponent(WorkflowEditorComponent); @@ -75,6 +80,181 @@ describe("WorkflowEditorComponent - mouse and pointer event integration", () => fixture.detectChanges(); }); + const prepareMacroCanvas = async (x = 300, y = 70) => { + await page.viewport(1200, 600); + workflowActionService.suspendWorkflow().shared.wsProvider.disconnect(); + fixture.nativeElement.style.cssText = "display:block;width:1200px;height:500px"; + component.paper.setDimensions(1200, 500); + component.paper.translate(x, y); + }; + const macroInterface = () => workflowActionService.getMacroInterface()!; + const terminalID = (direction: "inputs" | "outputs") => + workflowActionService + .getTexeraGraph() + .getAllOperators() + .find(op => op.operatorType === (direction === "inputs" ? "MacroInput" : "MacroOutput"))!.operatorID; + const terminalSVG = (direction: "inputs" | "outputs", selector: string) => + component.paper.findViewByModel(terminalID(direction)).el.querySelector(selector)!; + const magnet = (operatorID: string, portID: string) => { + // JointJS recreates port SVG nodes during a render. Track the graph port through those updates. + const id = `port:${operatorID}:${portID}`; + const cell = component.paper.getModelById(operatorID) as joint.dia.Element; + cell.portProp(portID, "attrs/.port-body/data-testid", id); + return page.getByTestId(id); + }; + + it("connects each saved macro output to an ordinary operator by dragging its magnet", async () => { + await prepareMacroCanvas(100, 50); + const definition = testMacro("macro", [mockScanPredicate], { + outputs: ["stream", "sorted"].map(portID => testPort(portID, mockScanPredicate.operatorID, "output-0")), + name: "Macro", + }); + workflowActionService.setMacroDependencies([definition]); + workflowActionService.addOperator(macroInstance(definition, "macro"), { x: 100, y: 120 }); + for (const [index, operatorID] of ["first", "second"].entries()) + workflowActionService.addOperator({ ...mockSentimentPredicate, operatorID }, { x: 350, y: 60 + index * 160 }); + const graph = workflowActionService.getTexeraGraph(); + for (const [port, target] of [ + ["stream", "first"], + ["sorted", "second"], + ]) { + await userEvent.dragAndDrop(magnet("macro", port), magnet(target, "input-0")); + expect(graph.getAllLinks()).toContainEqual( + expect.objectContaining({ + source: { operatorID: "macro", portID: port }, + target: { operatorID: target, portID: "input-0" }, + }) + ); + } + const connected = graph.getAllLinks(); + TestBed.inject(UndoRedoService).undoAction(); + expect(graph.getAllLinks().length).toBeLessThan(connected.length); + TestBed.inject(UndoRedoService).redoAction(); + expect(graph.getAllLinks()).toEqual(connected); + }); + + it("adds, wires, selects and removes grouped macro ports through real SVG controls", async () => { + await prepareMacroCanvas(); + for (const [index, operatorID] of ["a", "b"].entries()) + workflowActionService.addOperator({ ...mockSentimentPredicate, operatorID }, { x: 100 + index * 200, y: 140 }); + setMacroGraph(workflowActionService, { id: "macro", inputs: [], outputs: [] }); + const graph = workflowActionService.getJointGraph(); + const operator = component.paper.findViewByModel("a").el; + expect(terminalSVG("inputs", ".body").getBoundingClientRect().width).toBe( + operator.querySelector(".body")!.getBoundingClientRect().width + ); + expect(getComputedStyle(terminalSVG("inputs", ".add-output-port-button")).visibility).toBe("hidden"); + await userEvent.click(terminalSVG("inputs", ".boundary")); + expect(getComputedStyle(terminalSVG("inputs", ".add-output-port-button")).visibility).toBe("visible"); + await userEvent.click(terminalSVG("inputs", ".add-output-port-button rect")); + await userEvent.click(terminalSVG("inputs", ".add-output-port-button rect")); + await userEvent.click(terminalSVG("outputs", ".boundary")); + await userEvent.click(terminalSVG("outputs", ".add-input-port-button rect")); + const { + inputs: [first, second], + outputs: [output], + } = macroInterface(); + expect(graph.getElements()).toHaveLength(4); + const boundaryMagnet = (direction: "inputs" | "outputs", id: string) => magnet(terminalID(direction), id); + await userEvent.dragAndDrop(boundaryMagnet("inputs", first.portID), magnet("a", "input-0")); + await userEvent.dragAndDrop(magnet("a", "output-0"), boundaryMagnet("outputs", output.portID)); + const firstInput = { operatorID: "a", portID: "input-0" }; + const firstOutput = { operatorID: "a", portID: "output-0" }; + expect(macroInterface().inputs[0].targets).toEqual([firstInput]); + expect(macroInterface().outputs[0].targets).toEqual([firstOutput]); + await userEvent.dragAndDrop(magnet("b", "output-0"), boundaryMagnet("outputs", output.portID)); + expect(macroInterface().outputs[0].targets).toEqual([firstOutput]); + await userEvent.click(terminalSVG("outputs", ".boundary")); + await userEvent.click(terminalSVG("outputs", ".add-input-port-button rect")); + const secondOutput = macroInterface().outputs[1]; + await userEvent.dragAndDrop(magnet("b", "output-0"), boundaryMagnet("outputs", secondOutput.portID)); + expect(macroInterface().outputs.map(p => p.targets)).toEqual([ + [firstOutput], + [{ operatorID: "b", portID: "output-0" }], + ]); + await userEvent.click(terminalSVG("inputs", ".boundary")); + await userEvent.click(terminalSVG("inputs", ".joint-port-label")); + expect(workflowActionService.getJointGraphWrapper().getCurrentHighlightedPortIDs()).toContainEqual({ + operatorID: terminalID("inputs"), + portID: first.portID, + }); + await userEvent.click(terminalSVG("inputs", ".remove-output-port-button rect")); + expect(macroInterface().inputs.map(p => p.portID)).toEqual([second.portID]); + TestBed.inject(UndoRedoService).undoAction(); + expect(macroInterface().inputs[0].targets).toEqual([firstInput]); + await userEvent.click(boundaryMagnet("inputs", second.portID)); + await userEvent.click(terminalSVG("inputs", ".remove-output-port-button rect")); + const content = workflowActionService.getWorkflowContent(); + expect(JSON.parse(JSON.stringify(exportedWorkflow(content, undefined)))).toEqual( + JSON.parse(JSON.stringify(content)) + ); + expect(content.operators).toHaveLength(4); + expect(content.links).toHaveLength(3); + component.structureLocked = true; + fixture.detectChanges(); + expect(terminalSVG("inputs", ".add-output-port-button").getBoundingClientRect().width).toBe(0); + }); + + it.each(["inputs", "outputs"] as const)( + "removes a %s port with minus without first selecting a port", + async direction => { + await prepareMacroCanvas(); + const ports = [0, 1].map(i => ({ portID: `port-${i}`, targets: [] })); + setMacroGraph(workflowActionService, { id: "macro", inputs: ports, outputs: ports }); + const clickBoundary = () => userEvent.click(terminalSVG(direction, ".boundary")); + const side = direction === "inputs" ? "output" : "input"; + const minus = () => userEvent.click(terminalSVG(direction, `.remove-${side}-port-button rect`)); + await userEvent.click(terminalSVG(direction === "inputs" ? "outputs" : "inputs", ".port-body")); + await clickBoundary(); + await minus(); + expect(macroInterface()[direction].map(p => p.portID)).toEqual(["port-0"]); + TestBed.inject(UndoRedoService).undoAction(); + expect(macroInterface()[direction]).toEqual(ports); + await clickBoundary(); + await minus(); + expect(macroInterface()[direction].map(p => p.portID)).toEqual(["port-0"]); + await minus(); + await minus(); + expect(macroInterface()[direction]).toEqual([]); + } + ); + + it("renames a saved terminal through the native port name editor, including clearing and Undo/Redo", async () => { + workflowActionService.suspendWorkflow().shared.wsProvider.disconnect(); + workflowActionService.addOperator(mockSentimentPredicate, mockPoint); + setMacroGraph(workflowActionService, { + id: "macro", + inputs: [testPort("rows", mockSentimentPredicate.operatorID, "input-0", "Rows")], + outputs: [], + }); + const before = workflowActionService.getWorkflowContent(); + const snapshot = JSON.stringify(before); + const properties = TestBed.createComponent(PortPropertyEditFrameComponent); + properties.componentRef.setInput("currentPortID", { operatorID: terminalID("inputs"), portID: "rows" }); + properties.detectChanges(); + await userEvent.click(properties.nativeElement.querySelector("button")); + properties.detectChanges(); + const editor = properties.nativeElement.querySelector(".ql-editor") as HTMLElement; + await userEvent.clear(editor); + expect(macroInterface().inputs[0].displayName).toBe(""); + await userEvent.type(editor, "Rows 東京"); + await userEvent.keyboard("{Home}{ArrowRight}{ArrowRight}{ArrowRight}{ArrowRight} café"); + expect(macroInterface().inputs[0].displayName).toBe("Rows café 東京"); + expect(document.activeElement).toBe(editor); + expect(workflowActionService.getWorkflowContent().links).toEqual(before.links); + expect(JSON.stringify(before)).toBe(snapshot); + const copied = JSON.parse(JSON.stringify(exportedWorkflow(workflowActionService.getWorkflowContent(), undefined))); + expect(readMacroInterface(copied)!.inputs[0].displayName).toBe("Rows café 東京"); + properties.componentInstance.disconnectQuillFromText(); + TestBed.inject(UndoRedoService).undoAction(); + TestBed.inject(UndoRedoService).redoAction(); + expect(macroInterface().inputs[0].displayName).toBe("Rows café 東京"); + properties.componentRef.setInput("actsAsEditor", false); + properties.detectChanges(); + expect(properties.nativeElement.querySelector("button").disabled).toBe(true); + properties.destroy(); + }); + it("should try to highlight the operator when user mouse clicks on an operator", () => { const jointGraphWrapper = workflowActionService.getJointGraphWrapper(); // install a spy on the highlight operator function and pass the call through diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.html b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.html index 3fa5ea5a671..e2d2f5868c4 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.html +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.html @@ -17,7 +17,9 @@ under the License. --> -
+
diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.scss b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.scss index 9074d8e0360..b2c1d4e09f3 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.scss +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.scss @@ -27,6 +27,10 @@ height: 100%; } +::ng-deep .structure-locked [class$="-port-button"] { + display: none; +} + // Smoothly animate operator body fill changes for the heat-map recolor (view switch / toggle). // Scoped to the .heatmap-active class the overlay toggles, so it never animates the // disable/enable or validation fill paths when the overlay is off. diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts index d314de8cb27..66ed8f91a52 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ +import { testPort, testMacro, setMacroGraph } from "../../../common/testing/macro-graph"; import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; import { HeatmapView } from "../../service/heatmap/heatmap-scoring"; import { HEATMAP_NO_DATA_COLOR, scoreToColor } from "../../service/heatmap/heatmap-color"; import { UndoRedoService } from "../../service/undo-redo/undo-redo.service"; import { DragDropService } from "../../service/drag-drop/drag-drop.service"; -import { WorkflowUtilService } from "../../service/workflow-graph/util/workflow-util.service"; import { ComponentFixture, fakeAsync, TestBed, tick } from "@angular/core/testing"; import { ValidationWorkflowService } from "../../service/validation/validation-workflow.service"; import { @@ -32,7 +32,7 @@ import { WorkflowEditorComponent, } from "./workflow-editor.component"; import { PinnedResultWindowComponent } from "./pinned-result-window/pinned-result-window.component"; -import { workflowEditorTestImports, workflowEditorTestProviders } from "./workflow-editor.test-utils"; +import { configureWorkflowEditor } from "./workflow-editor.test-utils"; import { OperatorMetadataService } from "../../service/operator-metadata/operator-metadata.service"; import { StubOperatorMetadataService } from "../../service/operator-metadata/stub-operator-metadata.service"; import { @@ -59,22 +59,20 @@ import { import { WorkflowStatusService } from "../../service/workflow-status/workflow-status.service"; import { ExecutionState, OperatorState } from "../../types/execute-workflow.interface"; import { ExecuteWorkflowService } from "../../service/execute-workflow/execute-workflow.service"; -import { HttpClientTestingModule } from "@angular/common/http/testing"; import { OperatorLink, OperatorPredicate } from "../../types/workflow-common.interface"; import { tap } from "rxjs/operators"; import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; import { config as rxjsConfig, of, Subject } from "rxjs"; -import { NzContextMenuService, NzDropDownModule, NzDropdownMenuComponent } from "ng-zorro-antd/dropdown"; +import { NzDropdownMenuComponent } from "ng-zorro-antd/dropdown"; import { By } from "@angular/platform-browser"; import { ActivatedRoute, Router } from "@angular/router"; -import { RouterTestingModule } from "@angular/router/testing"; import { ContextMenuComponent } from "./context-menu/context-menu/context-menu.component"; -import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; -import { MockComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; -import { commonTestProviders } from "../../../common/testing/test-utils"; import { OperatorMenuService } from "../../service/operator-menu/operator-menu.service"; import { GuiConfigService } from "src/app/common/service/gui-config.service"; import { MockGuiConfigService } from "src/app/common/service/gui-config.service.mock"; +import { macroInstance } from "../../../common/util/workflow-macro"; +import { MacroService } from "../../service/macro/macro.service"; +import { WorkflowWebsocketService } from "../../service/workflow-websocket/workflow-websocket.service"; describe("WorkflowEditorComponent", () => { /** @@ -87,36 +85,7 @@ describe("WorkflowEditorComponent", () => { let fixture: ComponentFixture; let jointGraph: joint.dia.Graph; - beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: [ - RouterTestingModule, - HttpClientTestingModule, - NzModalModule, - NzDropDownModule, - WorkflowEditorComponent, - ContextMenuComponent, - ], - providers: [ - JointUIService, - WorkflowUtilService, - UndoRedoService, - DragDropService, - ValidationWorkflowService, - WorkflowActionService, - NzContextMenuService, - Overlay, - { - provide: OperatorMetadataService, - useClass: StubOperatorMetadataService, - }, - { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, - WorkflowStatusService, - ExecuteWorkflowService, - ...commonTestProviders, - ], - }).compileComponents(); - }); + beforeEach(configureWorkflowEditor); beforeEach(() => { fixture = TestBed.createComponent(WorkflowEditorComponent); @@ -752,12 +721,7 @@ describe("WorkflowEditorComponent", () => { let undoRedoService: UndoRedoService; let workflowVersionService: WorkflowVersionService; - beforeEach(async () => { - await TestBed.configureTestingModule({ - imports: workflowEditorTestImports, - providers: workflowEditorTestProviders, - }).compileComponents(); - }); + beforeEach(configureWorkflowEditor); beforeEach(() => { fixture = TestBed.createComponent(WorkflowEditorComponent); @@ -773,6 +737,28 @@ describe("WorkflowEditorComponent", () => { fixture.detectChanges(); }); + it("requests canvas navigation for the selected pinned instance without an Operators panel", async () => { + const actions = TestBed.inject(WorkflowActionService); + const definition = testMacro("macro", [mockSentimentPredicate], { + inputs: [testPort("input-0", mockSentimentPredicate.operatorID)], + outputs: [testPort("output-0", mockSentimentPredicate.operatorID)], + name: "Macro", + }); + actions.setMacroDependencies([definition]); + actions.addOperator(macroInstance(definition, "instance"), mockPoint); + const inspect = vi.spyOn(TestBed.inject(MacroService).inspect$, "next"); + const create = vi.spyOn((component as any).nzModalService, "create"); + component.paper.trigger( + "cell:pointerdblclick", + { model: component.paper.getModelById("instance") }, + { shiftKey: false } + ); + await Promise.resolve(); + await Promise.resolve(); + expect(inspect).toHaveBeenCalledWith("instance"); + expect(create).not.toHaveBeenCalled(); + }); + it("should react to operator highlight event and change the appearance of the operator to be highlighted", () => { const jointGraphWrapper = workflowActionService.getJointGraphWrapper(); workflowActionService.addOperator(mockScanPredicate, mockPoint); @@ -1116,6 +1102,18 @@ describe("WorkflowEditorComponent", () => { expect(jointGraphWrapper.getCurrentHighlightedOperatorIDs()).toContain(mockResultPredicate.operatorID); }); + it("does not replay a completed keyboard action when the history context changes", () => { + const modes = new Subject(); + vi.spyOn(workflowVersionService, "getDisplayParticularVersionStream").mockReturnValue(modes); + vi.spyOn(undoRedoService, "canUndo").mockReturnValue(true); + const undo = vi.spyOn(undoRedoService, "undoAction").mockImplementation(() => {}); + (component as any)._handleKeyboardAction({ ctrlKey: true, key: "z" }); + modes.next(false); + modes.next(true); + modes.next(false); + expect(undo).toHaveBeenCalledOnce(); + }); + //undo it("should undo action when user presses command + Z or control + Z", () => { vi.spyOn(workflowVersionService, "getDisplayParticularVersionStream").mockReturnValue(of(false)); @@ -1519,6 +1517,25 @@ describe("WorkflowEditorComponent", () => { expect(navigateSpy).toHaveBeenLastCalledWith([], expect.objectContaining({ fragment: undefined })); }); + it("does not cancel an outgoing route when restoring a macro changes the selection", () => { + const router = TestBed.inject(Router); + const navigate = vi.spyOn(router, "navigate").mockResolvedValue(true); + workflowActionService.addOperator(mockScanPredicate, mockPoint); + const wrapper = workflowActionService.getJointGraphWrapper(); + wrapper.unhighlightOperators(...wrapper.getCurrentHighlightedOperatorIDs()); + navigate.mockClear(); + const pending = vi.spyOn(router, "getCurrentNavigation").mockReturnValue({} as any); + wrapper.highlightOperators(mockScanPredicate.operatorID); + wrapper.unhighlightOperators(mockScanPredicate.operatorID); + expect(navigate).not.toHaveBeenCalled(); + pending.mockReturnValue(null); + wrapper.highlightOperators(mockScanPredicate.operatorID); + expect(navigate).toHaveBeenLastCalledWith( + [], + expect.objectContaining({ fragment: mockScanPredicate.operatorID }) + ); + }); + it("translates the paper toward the computed center on a center event", () => { workflowActionService.addOperator(mockScanPredicate, mockPoint); const translateSpy = vi.spyOn(component.paper, "translate"); @@ -1869,32 +1886,7 @@ describe("WorkflowEditorComponent link breakpoints", () => { let workflowActionService: WorkflowActionService; beforeEach(async () => { - TestBed.resetTestingModule(); - await TestBed.configureTestingModule({ - imports: [ - RouterTestingModule, - HttpClientTestingModule, - NzModalModule, - NzDropDownModule, - WorkflowEditorComponent, - ContextMenuComponent, - ], - providers: [ - JointUIService, - WorkflowUtilService, - UndoRedoService, - DragDropService, - ValidationWorkflowService, - WorkflowActionService, - NzContextMenuService, - Overlay, - { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, - { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, - WorkflowStatusService, - ExecuteWorkflowService, - ...commonTestProviders, - ], - }).compileComponents(); + await configureWorkflowEditor(); // Both halves of the guard at workflow-editor.component.ts:202 must be satisfied, and both must // be set before the first detectChanges: ngAfterViewInit reads them once, when it decides @@ -1941,6 +1933,21 @@ describe("WorkflowEditorComponent link breakpoints", () => { expect(view._toolsView.tools[0].isVisible()).toBe(false); }); + it("offers only removal on interface wires, which are not executable links", () => { + workflowActionService.addOperator(mockSentimentPredicate, mockPoint); + setMacroGraph(workflowActionService, { + id: "macro", + inputs: [testPort("rows", mockSentimentPredicate.operatorID, "input-0")], + outputs: [], + }); + const link = component.paper.model.getLinks()[0], + view = link.findView(component.paper) as any; + expect(view.hasTools()).toBe(false); + component.paper.trigger("link:mouseenter", view, new MouseEvent("mouseenter")); + expect(view._toolsView.tools).toHaveLength(1); + expect(view._toolsView.tools[0].el.getAttribute("data-tool-name")).toBe("remove-button"); + }); + it("highlights the link whose breakpoint button was clicked", () => { const { linkID, view } = withLink(); @@ -1998,11 +2005,7 @@ describe("WorkflowEditorComponent editor wiring", () => { let agentService: AgentService; beforeEach(async () => { - TestBed.resetTestingModule(); - await TestBed.configureTestingModule({ - imports: workflowEditorTestImports, - providers: workflowEditorTestProviders, - }).compileComponents(); + await configureWorkflowEditor(); // ngAfterViewInit reads both of these once, so they have to be set before the first // detectChanges: the flag decides whether the breakpoint tool is ever attached, and @@ -2141,14 +2144,17 @@ describe("WorkflowEditorComponent editor wiring", () => { workflowActionService.addOperator(mockScanPredicate, mockPoint); workflowActionService.addOperator(mockResultPredicate, mockPoint); const changeStatistics = vi.spyOn(jointUIService, "changeOperatorStatistics"); + const resetStatistics = vi.spyOn(jointUIService, "resetOperatorStatistics"); (workflowStatusService as any).statisticsSubject.next({ [mockScanPredicate.operatorID]: emptyStatistics, }); expect(changeStatistics).toHaveBeenCalledWith(component.paper, mockScanPredicate.operatorID, emptyStatistics); - // an operator missing from the payload is forwarded as undefined (a no-op render) - expect(changeStatistics).toHaveBeenCalledWith(component.paper, mockResultPredicate.operatorID, undefined); + expect(resetStatistics).toHaveBeenCalledWith( + component.paper, + expect.objectContaining({ operatorID: mockResultPredicate.operatorID }) + ); }); it("paints each operator's reported state, defaulting missing ones to Uninitialized", () => { @@ -2256,6 +2262,88 @@ describe("WorkflowEditorComponent editor wiring", () => { }); describe("region overlays", () => { + it.each(["inputs", "outputs"] as const)( + "extends a region to %s only when all its ports continue that region", + direction => { + addRegions([[1, ["visible"]]]); + (component.paper.getModelById("visible") as joint.dia.Element).position(direction === "inputs" ? 300 : 0, 100); + setMacroGraph(workflowActionService, { + id: "boundary", + inputs: [], + outputs: [], + [direction]: [ + { portID: "one", targets: [] }, + { portID: "two", targets: [] }, + ], + }); + const operator = workflowActionService + .getTexeraGraph() + .getAllOperators() + .find(op => op.operatorType === (direction === "inputs" ? "MacroInput" : "MacroOutput"))!; + const terminal = component.paper.getModelById(operator.operatorID) as joint.dia.Element; + terminal.position(direction === "inputs" ? 0 : 300, 0); + const emit = (ports: string[]) => + (executeWorkflowService as any).regionUpdateStream.next({ + regions: [[1, ["visible"]]], + terminals: { 1: ports.map(portID => ({ direction, portID })) }, + }); + const bounds = () => new joint.g.Path(component.paper.getModelById("region-1").attr("body/d")).bbox()!; + const outerEdge = (rect: joint.g.Rect) => (direction === "inputs" ? -rect.x : rect.x + rect.width); + emit(["one", "two"]); + expect(outerEdge(bounds())).toBeGreaterThan(outerEdge(terminal.getBBox())); + emit(["one"]); + expect(outerEdge(bounds())).toBeLessThan(outerEdge(terminal.getBBox())); + const before = outerEdge(bounds()); + terminal.translate(direction === "inputs" ? -100 : 100, 0); + expect(outerEdge(bounds())).toBeGreaterThan(before); + emit([]); + expect(outerEdge(bounds())).toBeLessThan(outerEdge(terminal.getBBox()) - 100); + } + ); + + it("ignores a late phase update after execution regions were cleared", fakeAsync(() => { + addRegions([[1, ["visible"]]]); + executeWorkflowService.resetExecutionState(); + (TestBed.inject(WorkflowWebsocketService).websocketEvent() as Subject).next({ + type: "RegionStateEvent", + id: 1, + state: "Completed", + }); + tick(); + expect(component.paper.getModelById("region-1")).toBeUndefined(); + })); + + it("restores region geometry and phase colors when Canvas remounts after Form View", () => { + workflowActionService.getJointGraphWrapper().setRegionsDisplayed(true); + addRegions([[1, ["visible"]]]); + (TestBed.inject(WorkflowWebsocketService).websocketEvent() as Subject).next({ + type: "RegionStateEvent", + id: 1, + state: "Completed", + }); + fixture.destroy(); + fixture = TestBed.createComponent(WorkflowEditorComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + const region = component.paper.getModelById("region-1"); + expect(region.attr("body/d")).toBeTruthy(); + expect(region.attr("body/fill")).toBe("rgba(76,175,80,0.2)"); + expect(region.attr("body/visibility")).toBe("visible"); + }); + + it("handles generated operators absent from the canvas and empty regions", fakeAsync(() => { + addRegions([[1, ["visible"]]]); + (executeWorkflowService as any).regionUpdateStream.next({ + regions: [ + [1, ["visible", "generated"]], + [2, []], + ], + }); + tick(); + expect(component.paper.getModelById("region-1").attr("body/d")).toBeTruthy(); + expect(component.paper.getModelById("region-2").attr("body/d")).toBeFalsy(); + })); + it("reshapes the region drawn around an operator that moved", () => { // Two regions so the handler's filter is exercised on both sides. Nothing can be asserted // about the untouched one: its outline is recomputed from its own, unmoved operator, so a diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts index a743fe9d73a..ca516531558 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts @@ -37,7 +37,7 @@ import { } from "../../service/heatmap/heatmap-scoring"; import { ExecutionState, OperatorState } from "../../types/execute-workflow.interface"; import { LogicalPort, OperatorLink, OperatorPredicate } from "../../types/workflow-common.interface"; -import { auditTime, filter, map, takeUntil, withLatestFrom } from "rxjs/operators"; +import { auditTime, filter, map, takeUntil, withLatestFrom, take } from "rxjs/operators"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { UndoRedoService } from "../../service/undo-redo/undo-redo.service"; import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; @@ -60,6 +60,9 @@ import { OperatorResultPreviewComponent } from "./operator-result-preview/operat import { PinnedResultWindowComponent } from "./pinned-result-window/pinned-result-window.component"; import { JupyterPanelService } from "../../service/jupyter-panel/jupyter-panel.service"; +import { MacroService } from "../../service/macro/macro.service"; +import { isMacro, isMacroTerminal } from "../../../common/util/workflow-macro"; + // jointjs interactive options for enabling and disabling interactivity // https://resources.jointjs.com/docs/jointjs/v3.2/joint.html#dia.Paper.prototype.options.interactive const defaultInteractiveOption = { vertexAdd: false, labelMove: false }; @@ -166,7 +169,6 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy private get interactive(): boolean { return this.paperInteractive && !this.structureLocked; } - private _onProcessKeyboardActionObservable: Subject = new Subject(); private wrapper; private currentOpenedOperatorID: string | null = null; private removeButton!: new () => joint.linkTools.Button; @@ -200,7 +202,8 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy private elementRef: ElementRef, private config: GuiConfigService, private agentService: AgentService, - private jupyterPanelService: JupyterPanelService + private jupyterPanelService: JupyterPanelService, + private macroService: MacroService ) { this.wrapper = this.workflowActionService.getJointGraphWrapper(); } @@ -266,8 +269,8 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy this.handleReuseCacheOperator(); this.registerOperatorDisplayNameChangeHandler(); this.handleViewDeleteLink(); - this.handleViewAddPort(); - this.handleViewRemovePort(); + this.handleViewPortChange(true); + this.handleViewPortChange(false); this.handlePortClick(); this.handlePaperPan(); this.handleOperatorSelectionEvents(); @@ -323,10 +326,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy } private _handleKeyboardAction(event: any) { - this._onProcessKeyboardActionObservable = new Subject(); + if (document.activeElement !== document.body) return; this.workflowVersionService .getDisplayParticularVersionStream() - .pipe(takeUntil(this._onProcessKeyboardActionObservable)) + .pipe(take(1), untilDestroyed(this)) .subscribe(displayParticularWorkflowVersion => { if (!displayParticularWorkflowVersion) { // cmd/ctrl+z undo ; ctrl+y or cmd/ctrl + shift+z for redo @@ -346,7 +349,6 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy } // below for future hotkeys } - this._onProcessKeyboardActionObservable.complete(); }); } @@ -432,7 +434,13 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy .getTexeraGraph() .getAllOperatorIDs() .forEach(operatorID => { - this.jointUIService.changeOperatorStatistics(this.paper, operatorID, statistics[operatorID]); + if (statistics[operatorID]) + this.jointUIService.changeOperatorStatistics(this.paper, operatorID, statistics[operatorID]); + else + this.jointUIService.resetOperatorStatistics( + this.paper, + this.workflowActionService.getTexeraGraph().getOperator(operatorID) + ); }); }); @@ -870,7 +878,7 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy } ); - let regionMap: { regionElement: joint.dia.Element; operators: joint.dia.Cell[] }[] = []; + let regionMap: { regionElement: joint.dia.Element; operators: joint.dia.Cell[]; ports: LogicalPort[] }[] = []; // update region elements on execution this.executeWorkflowService .getRegionUpdateStream() @@ -878,15 +886,28 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy .subscribe(event => { this.paper.model .getCells() - .filter(element => element instanceof Region) + .filter(element => element.get("type") === "region") .forEach(element => element.remove()); regionMap = event.regions.map(([id, region]) => { const element = new Region({ id: "region-" + id }); - const ops = region.map(id => this.paper.getModelById(id)); + const ops = region.map(id => this.paper.getModelById(id)).filter(Boolean); + const ports = (event.terminals?.[id] ?? []).flatMap(port => { + const node = this.paper.model.getElements().find( + e => + this.workflowActionService + .getTexeraGraph() + .getAllOperators() + .find(op => op.operatorID === e.id)?.operatorType === + (port.direction === "inputs" ? "MacroInput" : "MacroOutput") && + port.portID && + e.hasPort(port.portID) + ); + return node && port.portID ? [{ operatorID: String(node.id), portID: port.portID }] : []; + }); this.paper.model.addCell(element); - this.updateRegionElement(element, ops); - return { regionElement: element, operators: ops }; + this.updateRegionElement(element, ops, ports); + return { regionElement: element, operators: ops, ports }; }); // regions are recreated on every update, so reapply the current toggle state to the new elements this.setRegionsVisibility(this.wrapper.getRegionsDisplayed()); @@ -900,8 +921,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy this.paper.model.on("change:position", operator => { regionMap - .filter(region => region.operators.includes(operator)) - .forEach(region => this.updateRegionElement(region.regionElement, region.operators)); + .filter( + region => region.operators.includes(operator) || region.ports.some(p => p.operatorID === String(operator.id)) + ) + .forEach(region => this.updateRegionElement(region.regionElement, region.operators, region.ports)); }); // update region element colors on execution @@ -914,7 +937,7 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy ExecutingNonDependeePortsPhase: "rgba(255,213,79,0.2)", Completed: "rgba(76,175,80,0.2)", }; - this.paper.getModelById("region-" + region.id).attr("body/fill", colorMap[region.state]); + this.paper.getModelById("region-" + region.id)?.attr("body/fill", colorMap[region.state]); }); } @@ -925,8 +948,12 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy .forEach(element => element.attr("body/visibility", displayed ? "visible" : "hidden")); } - private updateRegionElement(regionElement: joint.dia.Element, operators: joint.dia.Cell[]) { - const points = operators.flatMap(op => { + private updateRegionElement( + regionElement: joint.dia.Element, + operators: joint.dia.Cell[], + ports: LogicalPort[] = [] + ) { + const corners = (op: joint.dia.Cell) => { const { x, y, width, height } = op.getBBox(), padding = 15; return [ @@ -935,8 +962,25 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy [x - padding, y + height + padding + 10], [x + width + padding, y + height + padding + 10], ]; - }); - regionElement.attr("body/d", line().curve(curveCatmullRomClosed)(concaveman(points, 2, 0) as [number, number][])); + }; + const points = operators.flatMap(corners); + for (const [id, selected] of Object.entries(_.groupBy(ports, p => p.operatorID))) { + const node = this.paper.getModelById(id) as joint.dia.Element | undefined; + if (!node) continue; + if (selected.length === node.getPorts().length) points.push(...corners(node)); + else + for (const port of selected) { + const offset = node.getPortsPositions(node.getPort(port.portID)?.group ?? "")[port.portID]; + if (!offset) continue; + const x = node.position().x + offset.x, + y = node.position().y + offset.y; + points.push([x - 6, y - 6], [x + 6, y - 6], [x - 6, y + 6], [x + 6, y + 6]); + } + } + regionElement.attr( + "body/d", + points.length ? line().curve(curveCatmullRomClosed)(concaveman(points, 2, 0) as [number, number][]) : null + ); } /** @@ -1119,9 +1163,13 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy if (this.workflowActionService.getTexeraGraph().hasCommentBox(elementID)) { this.openCommentBox(elementID); } else if (this.workflowActionService.getTexeraGraph().hasOperator(elementID)) { - this.pinResult(elementID); - // JointJS paper events fire outside Angular's zone (see handleHeatmapHover). - this.changeDetectorRef.detectChanges(); + if (isMacro(this.workflowActionService.getTexeraGraph().getOperator(elementID))) { + this.macroService.inspect$.next(elementID); + } else { + this.pinResult(elementID); + // JointJS paper events fire outside Angular's zone (see handleHeatmapHover). + this.changeDetectorRef.detectChanges(); + } } } }); @@ -1356,67 +1404,36 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy }); } - private handleViewAddPort(): void { - fromJointPaperEvent(this.paper, "element:add-input-port") - .pipe( - filter(() => this.interactive), - map(value => value[0]) - ) - .pipe(untilDestroyed(this)) - .subscribe(elementView => { - if (this.workflowActionService.getTexeraGraph().hasOperator(elementView.model.id.toString())) { - this.workflowActionService.addPort(elementView.model.id.toString(), true, false); - } - }); - fromJointPaperEvent(this.paper, "element:add-output-port") - .pipe( - filter(() => this.interactive), - map(value => value[0]) - ) - .pipe(untilDestroyed(this)) - .subscribe(elementView => { - if (this.workflowActionService.getTexeraGraph().hasOperator(elementView.model.id.toString())) { - this.workflowActionService.addPort(elementView.model.id.toString(), false); - } - }); - } - - private handleViewRemovePort(): void { - fromJointPaperEvent(this.paper, "element:remove-input-port") - .pipe( - filter(() => this.interactive), - map(value => value[0]) - ) - .pipe(untilDestroyed(this)) - .subscribe(elementView => { - if (this.workflowActionService.getTexeraGraph().hasOperator(elementView.model.id.toString())) { - this.workflowActionService.removePort(elementView.model.id.toString(), true); - } - }); - fromJointPaperEvent(this.paper, "element:remove-output-port") - .pipe( - filter(() => this.interactive), - map(value => value[0]) - ) - .pipe(untilDestroyed(this)) - .subscribe(elementView => { - if (this.workflowActionService.getTexeraGraph().hasOperator(elementView.model.id.toString())) { - this.workflowActionService.removePort(elementView.model.id.toString(), false); - } - }); + private handleViewPortChange(add: boolean): void { + for (const input of [true, false]) { + fromJointPaperEvent(this.paper, `element:${add ? "add" : "remove"}-${input ? "input" : "output"}-port`) + .pipe( + filter(() => this.interactive), + untilDestroyed(this) + ) + .subscribe(([view]) => { + if (this.workflowActionService.getTexeraGraph().hasOperator(String(view.model.id))) { + if (add) this.workflowActionService.addPort(String(view.model.id), input, input ? false : undefined); + else this.workflowActionService.removePort(String(view.model.id), input); + } + }); + } } private handlePortClick(): void { - fromJointPaperEvent(this.paper, "element:magnet:pointerclick") + merge( + fromJointPaperEvent(this.paper, "element:magnet:pointerclick"), + fromJointPaperEvent(this.paper, "input-port-label:pointerdown"), + fromJointPaperEvent(this.paper, "output-port-label:pointerdown") + ) .pipe(untilDestroyed(this)) .subscribe(event => { // set the multi-select mode this.wrapper.setMultiSelectMode(event[1].shiftKey); - const clickedPortID: LogicalPort = { - operatorID: event[0].model.id as string, - portID: event[2].getAttribute("port") as string, - }; + const portID = event[2]?.getAttribute?.("port") ?? event[1].target?.closest?.("[port]")?.getAttribute("port"); + if (!portID) return; + const clickedPortID: LogicalPort = { operatorID: String(event[0].model.id), portID }; if (event[1].shiftKey) { if (_.find(this.wrapper.getCurrentHighlightedPortIDs(), clickedPortID) !== undefined) { @@ -1437,7 +1454,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy private handleOperatorSelectionEvents(): void { fromJointPaperEvent(this.paper, "element:pointerdown") - .pipe(untilDestroyed(this)) + .pipe( + filter(([view]) => this.workflowActionService.getTexeraGraph().hasOperator(String(view.model.id))), + untilDestroyed(this) + ) .subscribe(event => { const operatorID = event[0].model.id.toString(); @@ -1452,7 +1472,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy }); fromJointPaperEvent(this.paper, "element:contextmenu") - .pipe(untilDestroyed(this)) + .pipe( + filter(([view]) => this.workflowActionService.getTexeraGraph().hasOperator(String(view.model.id))), + untilDestroyed(this) + ) .subscribe(event => { const operatorID = event[0].model.id.toString(); @@ -1470,7 +1493,8 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy .subscribe(event => { const linkID = event[0].model.id.toString(); // Highlight the link when right-clicked - this.workflowActionService.highlightLinks(false, linkID); + if (this.workflowActionService.getTexeraGraph().hasLinkWithID(linkID)) + this.workflowActionService.highlightLinks(false, linkID); }); fromJointPaperEvent(this.paper, "blank:pointerdown") @@ -1571,14 +1595,15 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy const targetCellID = targetView.model.id.toString(); const targetPortID = targetMagnet?.getAttribute("port"); - return this.validateOperatorConnection(sourceCellID, sourcePortID, targetCellID, targetPortID); + return this.validateOperatorConnection(sourceCellID, sourcePortID, targetCellID, targetPortID, linkView?.model); } private validateOperatorConnection( sourceCellID: string, sourcePortID: string | null | undefined, targetCellID: string, - targetPortID: string | null | undefined + targetPortID: string | null | undefined, + link?: joint.dia.Link ): boolean { // cannot connect to itself if (sourceCellID === targetCellID) { @@ -1602,7 +1627,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy const connectedLinksToTargetPort = this.workflowActionService .getTexeraGraph() .getAllLinks() - .filter(link => link.target.operatorID === targetCellID && link.target.portID === targetPortID); + .filter( + edge => + edge.linkID !== link?.id && edge.target.operatorID === targetCellID && edge.target.portID === targetPortID + ); // check if this link already exists, duplicate links are not allowed const isDuplicateLink = @@ -1622,7 +1650,11 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy if (portIndex >= 0) { const portInfo = this.dynamicSchemaService.getDynamicSchema(targetCellID).additionalMetadata.inputPorts[portIndex]; - disallowMultiInput = portInfo?.disallowMultiLinks ?? false; + disallowMultiInput = + portInfo?.disallowMultiLinks ?? + this.workflowActionService.getTexeraGraph().getOperator(targetCellID).inputPorts[portIndex] + .disallowMultiInputs ?? + false; } } return !(connectedLinksToTargetPort.length > 0 && disallowMultiInput); @@ -1782,7 +1814,7 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy const tools: joint.dia.ToolView[] = [new this.removeButton()]; // If breakpoints are enabled, also add the breakpoint button - if (this.config.env.linkBreakpointEnabled) { + if (this.config.env.linkBreakpointEnabled && !this.isMacroBoundaryLink(linkView.model)) { tools.push(new this.breakpointButton()); } @@ -1810,6 +1842,13 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy /** * handles events/observables related to the breakpoint */ + private isMacroBoundaryLink(link: joint.dia.Link): boolean { + const graph = this.workflowActionService.getTexeraGraph(); + return [link.source().id, link.target().id].some( + id => id && graph.hasOperator(String(id)) && isMacroTerminal(graph.getOperator(String(id))) + ); + } + private handleLinkBreakpoint(): void { this.handleLinkBreakpointToolAttachment(); this.handleLinkBreakpointButtonClick(); @@ -1822,8 +1861,8 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy .getJointLinkCellAddStream() .pipe(this.wrapper.jointGraphContext.bufferWhileAsync, untilDestroyed(this)) .subscribe(link => { - // No breakpoint tool on a structure-locked preview either (see handleLinkCursorHover). - if (this.structureLocked) { + // Only runtime links offer breakpoints. + if (this.structureLocked || this.isMacroBoundaryLink(link)) { return; } const linkView = link.findView(this.paper); @@ -1933,8 +1972,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy } private setURLFragment(fragment: string | null): void { + if (this.router.getCurrentNavigation()) return; this.router.navigate([], { relativeTo: this.route, + queryParamsHandling: "preserve", fragment: fragment !== null ? fragment : undefined, preserveFragment: false, }); diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.test-utils.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.test-utils.ts index 62f79627025..888d6f92393 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.test-utils.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.test-utils.ts @@ -31,6 +31,9 @@ * the two TestBed setups don't drift over time. */ +import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; +import { MockComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; +import { TestBed } from "@angular/core/testing"; import { Provider } from "@angular/core"; import { HttpClientTestingModule } from "@angular/common/http/testing"; import { NoopAnimationsModule } from "@angular/platform-browser/animations"; @@ -41,17 +44,8 @@ import { NzContextMenuService, NzDropDownModule } from "ng-zorro-antd/dropdown"; import { WorkflowEditorComponent } from "./workflow-editor.component"; import { NzModalCommentBoxComponent } from "./comment-box-modal/nz-modal-comment-box.component"; -import { WorkflowActionService } from "../../service/workflow-graph/model/workflow-action.service"; -import { WorkflowUtilService } from "../../service/workflow-graph/util/workflow-util.service"; -import { UndoRedoService } from "../../service/undo-redo/undo-redo.service"; -import { DragDropService } from "../../service/drag-drop/drag-drop.service"; -import { ValidationWorkflowService } from "../../service/validation/validation-workflow.service"; import { OperatorMetadataService } from "../../service/operator-metadata/operator-metadata.service"; import { StubOperatorMetadataService } from "../../service/operator-metadata/stub-operator-metadata.service"; -import { JointUIService } from "../../service/joint-ui/joint-ui.service"; -import { WorkflowStatusService } from "../../service/workflow-status/workflow-status.service"; -import { ExecuteWorkflowService } from "../../service/execute-workflow/execute-workflow.service"; -import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; import { UserService } from "src/app/common/service/user/user.service"; import { StubUserService } from "src/app/common/service/user/stub-user.service"; import { commonTestProviders } from "../../../common/testing/test-utils"; @@ -67,18 +61,19 @@ export const workflowEditorTestImports = [ ]; export const workflowEditorTestProviders: Provider[] = [ - JointUIService, - WorkflowUtilService, - WorkflowActionService, - UndoRedoService, - ValidationWorkflowService, - DragDropService, NzModalService, NzContextMenuService, { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, { provide: UserService, useClass: StubUserService }, - WorkflowStatusService, - ExecuteWorkflowService, - WorkflowVersionService, ...commonTestProviders, ]; + +export function configureWorkflowEditor() { + return TestBed.configureTestingModule({ + imports: workflowEditorTestImports, + providers: [ + ...workflowEditorTestProviders, + { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, + ], + }).compileComponents(); +} diff --git a/frontend/src/app/workspace/component/workflow-form/workflow-form.component.html b/frontend/src/app/workspace/component/workflow-form/workflow-form.component.html index a142107331f..979074c4170 100644 --- a/frontend/src/app/workspace/component/workflow-form/workflow-form.component.html +++ b/frontend/src/app/workspace/component/workflow-form/workflow-form.component.html @@ -219,8 +219,15 @@

{{ instructionTitle || "How to use this" }}

- {{ authoring ? "No inputs yet. Open the workflow below and click a step to expose its settings." : "This workflow - has no inputs to fill in." }} + {{ authoring ? "No inputs yet. Click an operator or macro in the workflow below and select the properties to show + here." : "This workflow has no inputs to fill in." }} +
{ h.validationWorkflowService as any, h.config as any, h.warehouseService as any, + h.macroNavigation as any, h.uiUdfParametersSyncService as any ); return component; @@ -1367,6 +1369,33 @@ describe("WorkflowFormComponent", () => { }); }); + describe("nested macro fields with real Formly schema conversion", () => { + nestedMacroFieldTests( + "form-input", + (schema, properties, propertyKey) => { + build(formViewWorkflow).ngOnInit(); + (component as any).formlyJsonschema = new FormlyJsonschema(); + vi.spyOn(h.dynamicSchemaService, "getDynamicSchema").mockReturnValue(schema); + h.hasOperatorIds.add("op-1"); + h.graphOperators.push({ + operatorID: "op-1", + operatorType: schema.operatorType, + operatorProperties: properties, + }); + formBindingService.resolveFields.mockReturnValue([ + resolved(propertyKey, "Macro parameter", { + binding: { ...binding("form-input", "Macro parameter"), propertyKey }, + value: properties[propertyKey], + }), + ]); + (component as any).readConfig(); + const { fields, model } = component.rendered[0]; + return { field: fields[0], model }; + }, + () => h.workflowCompilingService + ); + }); + // A nested (object) or repeated (array) property carries sub-fields; the author can rename and // hide each one, and the schema's own per-field notes are dropped so only the author's help text // guides a reader. Overrides are keyed by field path, array indices dropped. @@ -1872,6 +1901,11 @@ describe("WorkflowFormComponent", () => { expect(rebuild).not.toHaveBeenCalled(); expect(component.rendered[0]).toBe(cards[0]); expect(component.rendered[1]).toBe(cards[1]); + expect(cards[1].model["attributes"]).toEqual(["a", "b"]); + expect(component.canEdit).toBe(true); + cards[1].model[cards[1].resolved.binding.id] = true; + cards[1].flushWrite!(); + expect(formBindingService.writeValue).toHaveBeenCalledWith(cards[1].resolved.binding, true); }); it("leaves the cards alone when the operator's properties only came back in another order", async () => { @@ -2240,6 +2274,30 @@ describe("WorkflowFormComponent", () => { h.validationStream.next({ errors: {}, workflowEmpty: false }); }; + it.each([true, false])("disables macro definition execution regardless of validity (%s)", valid => { + build(formViewWorkflow).ngOnInit(); + makeReady(); + workflowActionService.getMacroInterface.mockReturnValue({ id: "macro", inputs: [], outputs: [] }); + component.isWorkflowValid = valid; + expect(component.runButtonState).toEqual({ label: "Run in workflow", icon: "caret-right", disabled: true }); + component.onRun(); + expect(h.executeWorkflowService.executeWorkflow).not.toHaveBeenCalled(); + workflowActionService.getMacroInterface.mockReturnValue(undefined); + component.isWorkflowValid = true; + expect(component.runButtonState).toEqual({ label: "Run", icon: "caret-right", disabled: false }); + }); + + it("blocks execution after a compiler error and releases Run after a successful compile", () => { + build(formViewWorkflow).ngOnInit(); + makeReady(); + h.workflowCompilingService.getWorkflowCompilationState.mockReturnValue("Failed"); + component.onRun(); + expect(component.runButtonState.disabled).toBe(true); + expect(h.executeWorkflowService.executeWorkflow).not.toHaveBeenCalled(); + h.workflowCompilingService.getWorkflowCompilationState.mockReturnValue("Succeeded"); + expect(component.runButtonState.disabled).toBe(false); + }); + it("stores the selected unit from the status service so Run can gate on write access", () => { build(formViewWorkflow).ngOnInit(); @@ -2471,6 +2529,35 @@ describe("WorkflowFormComponent", () => { h.validationStream.next({ errors: {}, workflowEmpty: false }); }; + it.each([false, true])( + "flushes a pending form value before Run and respects its validation (invalid=%s)", + invalid => { + build(formViewWorkflow).ngOnInit(); + makeReady(); + h.hasOperatorIds.add("op-1"); + formBindingService.resolveFields.mockReturnValue([resolved("n_hvg", "n_hvg", { value: "before" })]); + formBindingService.readValue.mockReturnValue("before"); + (component as any).readConfig(); + const card = component.rendered[0]; + card.model["n_hvg"] = invalid ? "bad value" : "東京"; + card.form.addControl("n_hvg", new FormControl(card.model["n_hvg"])); + card.form.markAsDirty(); + formBindingService.writeValue.mockImplementation(() => { + if (invalid) + h.validationStream.next({ + errors: { "op-1": { isValid: false, messages: { type: "must be integer" } } }, + workflowEmpty: false, + }); + }); + h.executeWorkflowService.executeWorkflow.mockImplementation(() => { + expect(formBindingService.writeValue).toHaveBeenCalledWith(card.resolved.binding, "東京"); + }); + component.onRun(); + expect(card.pendingWrite).toBe(false); + expect(h.executeWorkflowService.executeWorkflow).toHaveBeenCalledTimes(invalid ? 0 : 1); + } + ); + it("runs the workflow with its name and clears any prior error", () => { build(formViewWorkflow).ngOnInit(); makeReady(); diff --git a/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts b/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts index d964284d0a0..cf423d7dbe9 100644 --- a/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts +++ b/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts @@ -17,6 +17,9 @@ * under the License. */ +import { isMacroTerminal } from "../../../common/util/workflow-macro"; +import { setFieldBehavior, setFieldDependencies } from "../../../common/formly/formly-utils"; +import { CustomJSONSchema7 } from "../../types/custom-json-schema.interface"; import { ChangeDetectorRef, Component, ElementRef, HostListener, OnDestroy, OnInit } from "@angular/core"; import { CommonModule, DatePipe } from "@angular/common"; import { AbstractControl, FormArray, FormGroup, FormsModule, ReactiveFormsModule } from "@angular/forms"; @@ -47,10 +50,17 @@ import { WorkflowPersistService } from "../../../common/service/workflow-persist import { NotificationService } from "../../../common/service/notification/notification.service"; import { UserService } from "../../../common/service/user/user.service"; import { DynamicSchemaService } from "../../service/dynamic-schema/dynamic-schema.service"; -import { customFormlyFieldType, CANVAS_ONLY_FORMLY_TYPES, FIELD_OPERATOR_ID_PROP } from "../../util/custom-formly-type"; +import { + conditionalRequiredRules, + customFormlyFieldType, + CANVAS_ONLY_FORMLY_TYPES, + FIELD_OPERATOR_ID_PROP, +} from "../../util/custom-formly-type"; import { UiUdfParametersSyncService } from "../../service/code-editor/ui-udf-parameters-sync.service"; import type { UiUdfParameter } from "../../service/code-editor/ui-udf-parameters-parser.service"; import { WorkflowCompilingService } from "../../service/compile-workflow/workflow-compiling.service"; +import { MacroNavigationService } from "../../service/macro/macro-navigation.service"; +import { CompilationState } from "../../types/workflow-compiling.interface"; import { ExecuteWorkflowService, FORM_DEBOUNCE_TIME_MS } from "../../service/execute-workflow/execute-workflow.service"; import { OperatorMetadataService } from "../../service/operator-metadata/operator-metadata.service"; import { FormBindingService, ResolvedField } from "../../service/form-binding/form-binding.service"; @@ -327,10 +337,12 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { private validationWorkflowService: ValidationWorkflowService, private config: GuiConfigService, private warehouseService: WarehouseService, + private macroNavigation: MacroNavigationService, private uiUdfParametersSyncService: UiUdfParametersSyncService ) {} ngOnInit(): void { + this.workflowResultService.setMacroAliases?.(port => this.macroNavigation.resultAliases(port)); const wid = Number(this.route.snapshot.params.id); if (!Number.isFinite(wid)) { void this.router.navigate([USER_WORKFLOW]); @@ -871,7 +883,7 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { const hasEnabledDownstream = new Set(graph.getAllEnabledLinks().map(link => link.source.operatorID)); return graph .getAllOperators() - .filter(op => !(op.isDisabled ?? false) && !hasEnabledDownstream.has(op.operatorID)) + .filter(op => !isMacroTerminal(op) && !(op.isDisabled ?? false) && !hasEnabledDownstream.has(op.operatorID)) .map(op => op.operatorID); } @@ -906,6 +918,17 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { this.rebuildFormOrDefer(false); return; } + // Refresh sibling context without replacing controls or an edit still waiting to be written. + const graph = this.workflowActionService.getTexeraGraph(); + this.rendered.forEach(card => { + const { binding, brokenReason } = card.resolved; + if (brokenReason || !graph.hasOperator(binding.operatorID)) return; + const model = { + ...cloneDeep(graph.getOperator(binding.operatorID)?.operatorProperties ?? {}), + [binding.id]: card.model[binding.id], + }; + if (!sameValue(model, card.model)) card.model = model; + }); this.refreshShownResults(); this.rebuildResultChoices(); this.cdr.markForCheck(); @@ -1039,8 +1062,10 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { const schemaKey = JSON.stringify(schema); const operator = this.workflowActionService.getTexeraGraph().getOperator(binding.operatorID); const operatorType = operator?.operatorType; + const conditionalRules = conditionalRequiredRules(schema); const full = this.formlyJsonschema.toFieldConfig(cloneDeep(schema) as never, { map: (mapped, source) => { + setFieldBehavior(mapped, source as CustomJSONSchema7, conditionalRules.get(String(mapped.key))); // Render the exact custom widget the operator property panel would (file/model/dataset // pickers, image/audio uploaders, ...), shared via customFormlyFieldType so an exposed // property shows its real control instead of degrading to a text box. @@ -1059,6 +1084,11 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { return mapped; }, }); + setFieldDependencies( + schema as CustomJSONSchema7, + full.fieldGroup ?? [], + this.workflowCompilingService.getOperatorInputSchemaMap(binding.operatorID) + ); const source = (full.fieldGroup ?? []).find(child => child.key === binding.propertyKey); if (!source) { return undefined; @@ -1106,7 +1136,7 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { // writing that back silently wiped the operator's real value (both views edit one // workflow). So only accept a dirtied form, or a value that differs from the operator's // without being emptier (some controls set values without marking dirty). - const next = model[binding.id]; + const next = card.model[binding.id]; const current = this.formBindingService.readValue(binding.operatorID, binding.propertyKey); const isEmpty = (v: unknown) => v === undefined || v === null || v === ""; const unchanged = JSON.stringify(next ?? null) === JSON.stringify(current ?? null); @@ -1909,6 +1939,9 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { * button's only run control. */ public get runButtonState(): { label: string; icon: string; disabled: boolean } { + if (this.workflowActionService.getMacroInterface()) { + return { label: "Run in workflow", icon: "caret-right", disabled: true }; + } // Connecting is checked before Stop on purpose: if the socket drops mid-run, a "Stop" would // send killWorkflow() through a dead socket and do nothing, so a disconnected unit disables the // button (as the canvas does) rather than offering a kill that cannot be delivered. @@ -1920,7 +1953,10 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { if (this.isRunning && this.workflowWebsocketService.isConnected) { return { label: "Stop", icon: "stop", disabled: false }; } - if (!this.isWorkflowValid) { + if ( + !this.isWorkflowValid || + this.workflowCompilingService.getWorkflowCompilationState() === CompilationState.Failed + ) { return { label: "Invalid", icon: "warning", disabled: true }; } if (this.isWorkflowEmpty) { @@ -1956,6 +1992,7 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { this.executeWorkflowService.killWorkflow(); return; } + this.rendered.forEach(card => card.pendingWrite && card.flushWrite?.()); // The button is disabled in exactly the states a run cannot start from (invalid/empty workflow, // connecting, or no unit), so a stray call here would be a silent no-op. if (this.runButtonState.disabled) { diff --git a/frontend/src/app/workspace/component/workflow-form/workflow-form.rendered.spec.ts b/frontend/src/app/workspace/component/workflow-form/workflow-form.rendered.spec.ts index 2bd7b9880b6..b13040b6272 100644 --- a/frontend/src/app/workspace/component/workflow-form/workflow-form.rendered.spec.ts +++ b/frontend/src/app/workspace/component/workflow-form/workflow-form.rendered.spec.ts @@ -17,6 +17,7 @@ * under the License. */ +import { MacroNavigationService } from "../../service/macro/macro-navigation.service"; import { DatePipe } from "@angular/common"; import { CdkDropList } from "@angular/cdk/drag-drop"; import { FormGroup } from "@angular/forms"; @@ -40,7 +41,7 @@ import { PlusOutline, UpOutline, } from "@ant-design/icons-angular/icons"; -import { EMPTY, of, Subject } from "rxjs"; +import { EMPTY, Subject } from "rxjs"; import { WorkflowFormComponent } from "./workflow-form.component"; import { UserIconComponent } from "../../../dashboard/component/user/user-icon/user-icon.component"; @@ -74,6 +75,8 @@ import { WorkflowPveService } from "../../service/virtual-environment/virtual-en import { NzModalService } from "ng-zorro-antd/modal"; import { ExecutionState } from "../../types/execute-workflow.interface"; import { GuiConfigService } from "../../../common/service/gui-config.service"; +import { FormBindingConfig } from "../../../common/type/workflow"; +import { setupHarness } from "./workflow-form.spec-harness"; /** * The direct-construction spec exercises the component's logic without a DOM; this one stands @@ -84,10 +87,19 @@ import { GuiConfigService } from "../../../common/service/gui-config.service"; describe("WorkflowFormComponent (rendered template)", () => { let fixture: ComponentFixture; let workflow$: Subject; - const navigate = vi.fn(); const configure = async () => { workflow$ = new Subject(); + const h = setupHarness(); + h.workflowPersistService.retrieveWorkflow.mockReturnValue(workflow$); + h.formBindingService.getConfig.mockReturnValue({ + instruction: { title: "How to use this", body: "Fill in the inputs." }, + fields: [], + }); + h.formBindingService.updateConfig.mockImplementation((patch?: Partial) => { + h.formBindingService.getConfig.mockReturnValue({ ...h.formBindingService.getConfig(), ...patch }); + h.formBindingChanged.next(undefined); + }); // Blank out ONLY the two child icons: their ng-zorro dropdown/menu needs a host context this // page does not set up. The override is on the children, not the page, so the page's own // .component.html renders as shipped and stays covered -- which is the point of this spec, and @@ -132,146 +144,40 @@ describe("WorkflowFormComponent (rendered template)", () => { // formly-form still builds instead of throwing "missing forRoot()". imports: [WorkflowFormComponent, FormlyModule.forRoot()], providers: [ - // One co-editor so the collaborator row (the *ngFor) renders and is covered. + { provide: MacroNavigationService, useValue: h.macroNavigation }, { provide: CoeditorPresenceService, useValue: { coeditors: [{ clientId: "c1", userName: "co", color: "#888" }] }, }, - { provide: ActivatedRoute, useValue: { snapshot: { params: { id: "7" } } } }, - { - provide: Router, - useValue: { navigate, navigateByUrl: vi.fn(), getCurrentNavigation: () => null, serializeUrl: String }, - }, - { - provide: WorkflowActionService, - useValue: { - hasWorkflowOpen: () => false, - // The room the shared document is in: what the page keys the hand-over on, on its way out. - getOpenWorkflowId: () => 7, - resetAsNewWorkflow: vi.fn(), - setNewSharedModel: vi.fn(), - reloadWorkflow: vi.fn(), - disableWorkflowModification: vi.fn(), - getWorkflowModificationEnabledStream: () => EMPTY, - clearWorkflow: vi.fn(), - getWorkflowMetadata: () => ({ name: "scGPT", lastModifiedTime: undefined }), - getWorkflow: () => ({ wid: 7, content: { operators: [], operatorPositions: {} } }), - setWorkflowName: vi.fn(), - workflowChanged: () => EMPTY, - workflowMetaDataChanged: () => EMPTY, - formBindingChanged$: EMPTY, - setHighlightingEnabled: vi.fn(), - unhighlightOperators: vi.fn(), - getTexeraGraph: () => ({ - triggerCenterEvent: vi.fn(), - hasOperator: () => false, - getOperator: () => undefined, - getAllOperators: () => [], - getAllEnabledLinks: () => [], - getOperatorsToViewResult: () => new Set(), - getViewResultOperatorsChangedStream: () => EMPTY, - getOperatorAddStream: () => EMPTY, - getOperatorDeleteStream: () => EMPTY, - getLinkAddStream: () => EMPTY, - getLinkDeleteStream: () => EMPTY, - getDisabledOperatorsChangedStream: () => EMPTY, - getOperatorDisplayNameChangedStream: () => EMPTY, - updateSharedModelAwareness: vi.fn(), - }), - getJointGraphWrapper: () => ({ - // Reset by the page on leaving the workspace; the rendered tests leave, so it must exist. - setHeatmapView: () => {}, - getJointOperatorHighlightStream: () => EMPTY, - getJointOperatorUnhighlightStream: () => EMPTY, - getCurrentHighlightedOperatorIDs: () => [], - unhighlightOperators: vi.fn(), - }), - }, - }, - { - provide: WorkflowPersistService, - useValue: { - retrieveWorkflow: () => workflow$, - isWorkflowPersistEnabled: () => false, - persistWorkflow: () => of({}), - }, - }, - { provide: OperatorMetadataService, useValue: { getOperatorMetadata: () => of({}) } }, - { - provide: FormBindingService, - useValue: { - // An instruction so the instruction card renders and is covered. - getConfig: () => ({ - instruction: { title: "How to use this", body: "Fill in the inputs." }, - fields: [], - }), - resolveFields: () => [], - readValue: () => undefined, - writeValue: vi.fn(), - // Author-mode writes the rendered controls reach. - updateConfig: vi.fn(), - toggleShownResult: vi.fn(), - removeBinding: vi.fn(), - }, - }, - { provide: FormlyJsonschema, useValue: { toFieldConfig: () => ({ fieldGroup: [] }) } }, - { provide: DynamicSchemaService, useValue: { getDynamicSchema: () => ({ jsonSchema: {} }) } }, - { - provide: WorkflowCompilingService, - useValue: { getCompilationStateInfoChangedStream: () => EMPTY }, - }, - { provide: UiUdfParametersSyncService, useValue: { uiParametersChanged$: EMPTY } }, - { - provide: ExecuteWorkflowService, - useValue: { - getExecutionStateStream: () => EMPTY, - // The stream carries no current value, so the page reads the state it arrived on top - // of from here. Nothing is in flight in these tests. - getExecutionState: () => ({ state: ExecutionState.Uninitialized }), - getExecutionDurationStream: () => EMPTY, - executeWorkflow: vi.fn(), - killWorkflow: vi.fn(), - resetExecutionAndWorkers: vi.fn(), - }, - }, - { - provide: WorkflowResultService, - useValue: { - clearResults: vi.fn(), - getResultUpdateStream: () => EMPTY, - hasNonEmptyResult: () => false, - hasPaginatedResult: () => false, - getResultService: () => undefined, - }, - }, - { provide: PanelResizeService, useValue: { changePanelSize: vi.fn() } }, - { provide: NotificationService, useValue: { error: vi.fn() } }, - { provide: UserService, useValue: { getCurrentUser: () => undefined, isLogin: () => false } }, - { provide: MarkdownService, useValue: { parse: (s: string) => s } }, + { provide: ActivatedRoute, useValue: h.route }, + { provide: Router, useValue: h.router }, + { provide: WorkflowActionService, useValue: h.workflowActionService }, + { provide: WorkflowPersistService, useValue: h.workflowPersistService }, + { provide: OperatorMetadataService, useValue: h.operatorMetadataService }, + { provide: FormBindingService, useValue: h.formBindingService }, + { provide: FormlyJsonschema, useValue: h.formlyJsonschema }, + { provide: DynamicSchemaService, useValue: h.dynamicSchemaService }, + { provide: WorkflowCompilingService, useValue: h.workflowCompilingService }, + { provide: ExecuteWorkflowService, useValue: h.executeWorkflowService }, + { provide: WorkflowResultService, useValue: h.workflowResultService }, + { provide: PanelResizeService, useValue: h.panelResizeService }, + { provide: NotificationService, useValue: h.notificationService }, + { provide: UserService, useValue: h.userService }, + { provide: MarkdownService, useValue: h.markdownService }, + { provide: WorkflowConsoleService, useValue: h.workflowConsoleService }, + { provide: WorkflowWebsocketService, useValue: h.workflowWebsocketService }, + { provide: ValidationWorkflowService, useValue: h.validationWorkflowService }, + { provide: GuiConfigService, useValue: h.config }, + // The blanked selector still initializes and subscribes to the computing-unit list. { provide: ComputingUnitStatusService, - useValue: { - disconnect: vi.fn(), - getSelectedComputingUnit: () => EMPTY, - getStatus: () => EMPTY, - // Read by the (blanked) computing-unit selector's own ngOnInit. - getAllComputingUnits: () => EMPTY, - }, + useValue: { ...h.computingUnitStatusService, getAllComputingUnits: () => EMPTY }, }, - // The blanked computing-unit selector still constructs and runs ngOnInit; give it the few - // services it reads so it does not throw. It renders nothing (its template is blanked). { provide: WorkflowComputingUnitManagingService, useValue: { getComputingUnitLimitOptions: () => EMPTY } }, { provide: WorkflowExecutionsService, useValue: {} }, { provide: ComputingUnitActionsService, useValue: {} }, { provide: WorkflowPveService, useValue: {} }, { provide: NzModalService, useValue: {} }, - { provide: WorkflowConsoleService, useValue: { clearConsoleMessages: vi.fn() } }, - { - provide: WorkflowWebsocketService, - useValue: { subscribeToEvent: () => EMPTY, isConnected: true, getConnectionStatusStream: () => EMPTY }, - }, - { provide: ValidationWorkflowService, useValue: { getWorkflowValidationErrorStream: () => EMPTY } }, - { provide: GuiConfigService, useValue: { env: { formViewEnabled: true } } }, // Register the icons the run bar and instruction use, so nz-icon renders them inline instead // of fetching each SVG over HTTP (an unresolved fetch that would hang fixture.whenStable). { @@ -371,6 +277,15 @@ describe("WorkflowFormComponent (rendered template)", () => { expect(el(".pc-section-head .label")?.textContent?.trim()).toBe("Inputs"); expect(el(".empty")).not.toBeNull(); expect(el(".params .param")).toBeNull(); + const c = fixture.componentInstance; + c.canEdit = true; + fixture.detectChanges(); + const choose = vi.spyOn(c, "toggleAuthoring").mockImplementation(() => {}); + el(".empty button")!.click(); + expect(choose).toHaveBeenCalledOnce(); + c.canEdit = false; + fixture.detectChanges(); + expect(el(".empty button")).toBeNull(); }); it("renders an exposed input as a card holding its formly field", () => { @@ -414,7 +329,7 @@ describe("WorkflowFormComponent (rendered template)", () => { // Pointer blocking is hit testing only; the keyboard half is the directive on the same card. Here // it is wired through the real template, with a control of the kind a widget draws of its own. - it("keeps a read-only viewer's card out of the tab order, without hiding what it shows", () => { + it("keeps a read-only viewer's card out of the tab order, without hiding what it shows", async () => { fixture.detectChanges(); finishLoad(); const c = fixture.componentInstance; @@ -429,13 +344,11 @@ describe("WorkflowFormComponent (rendered template)", () => { const widgetButton = document.createElement("button"); widgetButton.className = "browse"; card.querySelector(".param-form")!.appendChild(widgetButton); - // The directive watches the card for what a widget adds; MutationObserver delivers on a microtask. - return Promise.resolve().then(() => { - expect(card.querySelector(".browse")!.getAttribute("tabindex")).toBe("-1"); - expect(card.querySelector(".browse")!.getAttribute("aria-disabled")).toBe("true"); - // Not inert: a reader still has the card's content in the accessibility tree. - expect(card.hasAttribute("inert")).toBe(false); - }); + // Wait for MutationObserver delivery in both jsdom and a real browser. + await expect.poll(() => widgetButton.getAttribute("tabindex")).toBe("-1"); + expect(widgetButton.getAttribute("aria-disabled")).toBe("true"); + // Not inert: a reader still has the card's content in the accessibility tree. + expect(card.hasAttribute("inert")).toBe(false); }); it("renders the author's instruction card and toggles it", async () => { @@ -726,6 +639,7 @@ describe("WorkflowFormComponent (rendered template)", () => { expect(el(".param .grip")).not.toBeNull(); expect(el(".param .field-help")?.textContent?.trim()).toBe("From Scan"); + expect(el(".param .field-help")?.getAttribute("title")).toBeNull(); // While authoring, the help text is edited in its own box rather than shown as the reader's line. expect(el(".param .param-help-text")).toBeNull(); const box = el(".param .edit input") as HTMLInputElement; diff --git a/frontend/src/app/workspace/component/workflow-form/workflow-form.spec-harness.ts b/frontend/src/app/workspace/component/workflow-form/workflow-form.spec-harness.ts index 87676269627..03282ea5a4e 100644 --- a/frontend/src/app/workspace/component/workflow-form/workflow-form.spec-harness.ts +++ b/frontend/src/app/workspace/component/workflow-form/workflow-form.spec-harness.ts @@ -45,11 +45,8 @@ export const resolved = (id: string, displayName: string, extra: Partial(); const uiUdfParametersSyncService = { uiParametersChanged$: uiParametersChanged.asObservable() }; const workflowActionService = { + getMacroInterface: vi.fn().mockReturnValue(undefined), setOperatorProperty: vi.fn(), resetAsNewWorkflow: vi.fn(), setNewSharedModel: vi.fn(), @@ -301,7 +299,9 @@ export function setupHarness() { }; const dynamicSchemaService = { getDynamicSchema: () => ({ jsonSchema: {} }) }; const workflowCompilingService = { + getWorkflowCompilationState: vi.fn().mockReturnValue("Uninitialized"), getCompilationStateInfoChangedStream: () => compilationChanged.asObservable(), + getOperatorInputSchemaMap: () => undefined, }; const workflowPersistService = { retrieveWorkflow: vi.fn().mockReturnValue(of(formViewWorkflow)), @@ -386,6 +386,7 @@ export function setupHarness() { return { useWorkflow, + macroNavigation: { resultAliases: () => ({}) }, router, coeditorPresenceService, route, diff --git a/frontend/src/app/workspace/component/workspace.component.spec.ts b/frontend/src/app/workspace/component/workspace.component.spec.ts index 8a7a33faef7..2572d1d41ae 100644 --- a/frontend/src/app/workspace/component/workspace.component.spec.ts +++ b/frontend/src/app/workspace/component/workspace.component.spec.ts @@ -27,22 +27,24 @@ import { Location } from "@angular/common"; import { CUSTOM_ELEMENTS_SCHEMA, NO_ERRORS_SCHEMA } from "@angular/core"; import { ComponentFixture, TestBed } from "@angular/core/testing"; import { HttpClientTestingModule } from "@angular/common/http/testing"; -import { ActivatedRoute, Router } from "@angular/router"; +import { ActivatedRoute, Router, convertToParamMap } from "@angular/router"; import { NzMessageService } from "ng-zorro-antd/message"; import { EMPTY, of, Subject, throwError } from "rxjs"; import { NotificationService } from "../../common/service/notification/notification.service"; import { UserService } from "../../common/service/user/user.service"; import { WorkflowPersistService } from "../../common/service/workflow-persist/workflow-persist.service"; +import { testWorkflow } from "../../common/testing/workflow-graph"; import { Workflow } from "../../common/type/workflow"; import { CodeEditorService } from "../service/code-editor/code-editor.service"; +import { MacroNavigationService } from "../service/macro/macro-navigation.service"; import { WorkflowCompilingService } from "../service/compile-workflow/workflow-compiling.service"; import { OperatorMetadataService } from "../service/operator-metadata/operator-metadata.service"; import { UndoRedoService } from "../service/undo-redo/undo-redo.service"; import { WorkflowConsoleService } from "../service/workflow-console/workflow-console.service"; import { ExecuteWorkflowService } from "../service/execute-workflow/execute-workflow.service"; import { WorkflowResultService } from "../service/workflow-result/workflow-result.service"; -import { WorkflowActionService } from "../service/workflow-graph/model/workflow-action.service"; +import { DEFAULT_WORKFLOW, WorkflowActionService } from "../service/workflow-graph/model/workflow-action.service"; import { OperatorReuseCacheStatusService } from "../service/workflow-status/operator-reuse-cache-status.service"; import { ComputingUnitStatusService } from "../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; import { EntityType, HubService } from "../../hub/service/hub.service"; @@ -73,29 +75,44 @@ describe("WorkspaceComponent", () => { let workflowVersionService: any; let connectionResetSubject: Subject; let metadataChangedSubject: Subject; + let navigation: ReturnType; let stubGraph: { triggerCenterEvent: ReturnType; hasElementWithID: ReturnType }; - const stubWorkflow: Workflow = { - wid: 42, - name: "test", - creationTime: 0, - lastModifiedTime: 0, - content: { - operators: [], - operatorPositions: {}, - links: [], - commentBoxes: [], - settings: { dataTransferBatchSize: 100 }, - }, - } as unknown as Workflow; + const stubWorkflow = { ...DEFAULT_WORKFLOW, wid: 42, name: "test", content: testWorkflow() }; + + function routeNavigation() { + const parents: { operatorID: string }[] = []; + const changed$ = new Subject(); + return { + parents, + changed$, + changing$: EMPTY, + get rootWorkflow() { + return workflowActionService.getWorkflow(); + }, + open: async (operatorID: string) => { + await Promise.resolve(); + parents.push({ operatorID }); + changed$.next(); + }, + back: vi.fn(async (index: number) => { + parents.splice(index); + changed$.next(); + }), + save: async () => workflowActionService.getWorkflow(), + clear: () => parents.splice(0), + }; + } function configureRoute(params: Record = {}, queryParams: Record = {}) { return { snapshot: { params, queryParams, fragment: null as string | null }, + queryParamMap: of(convertToParamMap(queryParams)), }; } async function createFixture(routeOverride: any = configureRoute()) { + navigation = routeNavigation(); metadataChangedSubject = new Subject(); stubGraph = { triggerCenterEvent: vi.fn(), @@ -115,13 +132,14 @@ describe("WorkspaceComponent", () => { highlightElements: vi.fn(), getTexeraGraph: vi.fn().mockReturnValue(stubGraph), getWorkflow: vi.fn().mockReturnValue(stubWorkflow), + getTempWorkflow: vi.fn().mockReturnValue(undefined), getWorkflowMetadata: vi.fn().mockReturnValue({ wid: 42, readonly: false }), // Off by default: most specs open a workflow that is not already live, and so load it. hasWorkflowOpen: vi.fn().mockReturnValue(false), // The room the shared document is in; the hand-over on the way out is keyed on this. getOpenWorkflowId: vi.fn().mockReturnValue(42), // One stub object, so the spy on it is the same one the assertions read. - getJointGraphWrapper: vi.fn().mockReturnValue({ setHeatmapView: vi.fn() }), + getJointGraphWrapper: vi.fn().mockReturnValue({ setHeatmapView: vi.fn(), getReloadingWorkflow: () => false }), workflowChanged: vi.fn().mockReturnValue(EMPTY), workflowMetaDataChanged: vi.fn().mockReturnValue(metadataChangedSubject.asObservable()), // As the real one does: the metadata it already holds, re-announced on the same stream. @@ -192,6 +210,7 @@ describe("WorkspaceComponent", () => { await TestBed.configureTestingModule({ imports: [WorkspaceComponent, HttpClientTestingModule], providers: [ + { provide: MacroNavigationService, useValue: navigation }, { provide: WorkflowActionService, useValue: workflowActionService }, { provide: WorkflowPersistService, useValue: workflowPersistService }, { provide: OperatorMetadataService, useValue: operatorMetadataService }, @@ -228,6 +247,28 @@ describe("WorkspaceComponent", () => { component.codeEditorViewRef = { clear: vi.fn() } as any; } + async function renderWorkspace(route = configureRoute()) { + await createFixture(route); + fixture.detectChanges(); + } + + describe("leaving a nested canvas", () => { + it.each([true, false])("restores the parent only if the draft saves (%s)", async saved => { + const route = configureRoute(); + route.queryParamMap = EMPTY; + await renderWorkspace(route); + navigation.parents.push({ operatorID: "inner" }); + if (!saved) navigation.back.mockRejectedValueOnce(new Error("save failed")); + expect(await component.canLeave()).toBe(saved); + expect(navigation.back).toHaveBeenCalledWith(0); + expect(navigation.parents.map(p => p.operatorID)).toEqual(saved ? [] : ["inner"]); + if (!saved) expect(notificationService.error).toHaveBeenCalledWith("save failed"); + expect(routerMock.navigate).not.toHaveBeenCalled(); + navigation.changed$.next(); + expect(routerMock.navigate).toHaveBeenCalledOnce(); + }); + }); + describe("ngOnInit", () => { it("enables highlighting on the workflow action service", async () => { await createFixture(); @@ -238,8 +279,7 @@ describe("WorkspaceComponent", () => { describe("ngAfterViewInit", () => { it("cold start (no wid in route): does not flip isLoading and registers metadata listener", async () => { - await createFixture(configureRoute({})); - fixture.detectChanges(); // triggers ngOnInit + ngAfterViewInit + await renderWorkspace(configureRoute({})); expect(component.isLoading).toBe(false); expect(workflowActionService.disableWorkflowModification).not.toHaveBeenCalled(); expect(operatorMetadataService.getOperatorMetadata).toHaveBeenCalled(); @@ -310,8 +350,7 @@ describe("WorkspaceComponent", () => { describe("loadWorkflowWithId", () => { it("on success: hands the workflow to the action service, clears undo/redo, and turns off loading", async () => { - await createFixture(configureRoute({ id: "42" })); - fixture.detectChanges(); + await renderWorkspace(configureRoute({ id: "42" })); expect(workflowActionService.setNewSharedModel).toHaveBeenCalledWith(42, { uid: 7 }); expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow, undefined); expect(undoRedoService.clearUndoStack).toHaveBeenCalled(); @@ -361,32 +400,85 @@ describe("WorkspaceComponent", () => { }); it("without autolayout: uses the default rendering and does not lay out", async () => { - await createFixture(configureRoute({ id: "42" })); - fixture.detectChanges(); + await renderWorkspace(configureRoute({ id: "42" })); expect(workflowActionService.reloadWorkflow).toHaveBeenCalledWith(stubWorkflow, undefined); expect(workflowActionService.autoLayoutWorkflow).not.toHaveBeenCalled(); }); - it("when URL fragment matches an element in the graph, highlights it", async () => { + const nestedPath = ["outer", "child"]; + it.each([ + { path: [], resumed: false, valid: true }, + { path: [], resumed: false, valid: false }, + { path: nestedPath, resumed: false, valid: true }, + { path: nestedPath, resumed: true, valid: true }, + { path: nestedPath, resumed: false, valid: false }, + ])( + "resolves a fragment after restoring path $path (resumed=$resumed, valid=$valid)", + async ({ path, resumed, valid }) => { + const query = path.length ? { macro: path, ...(valid ? {} : { autolayout: "1" }) } : {}; + const route = configureRoute({ id: "42" }, query); + const fragment = valid ? "operator-1" : "stale-id"; + route.snapshot.fragment = fragment; + await createFixture(route); + workflowActionService.hasWorkflowOpen.mockReturnValue(resumed); + stubGraph.hasElementWithID.mockImplementation( + id => valid && id === fragment && navigation.parents.length === path.length + ); + + fixture.detectChanges(); + await vi.waitFor(() => expect(stubGraph.hasElementWithID).toHaveBeenCalledWith(fragment)); + expect(navigation.parents.map(parent => parent.operatorID)).toEqual(path); + if (valid) { + expect(workflowActionService.highlightElements).toHaveBeenCalledWith(false, fragment); + expect(notificationService.error).not.toHaveBeenCalled(); + } else { + expect(workflowActionService.highlightElements).not.toHaveBeenCalled(); + expect(notificationService.error).toHaveBeenCalledExactlyOnceWith(`Element ${fragment} doesn't exist`); + } + expect(stubGraph.hasElementWithID).toHaveBeenCalledOnce(); + expect(routerMock.navigate).toHaveBeenLastCalledWith([], { + relativeTo: route, + ...(valid ? { fragment, preserveFragment: false } : {}), + queryParamsHandling: "preserve", + }); + } + ); + + it("keeps a resumed historical canvas and its Undo stack when selecting a root fragment", async () => { const route = configureRoute({ id: "42" }); - route.snapshot.fragment = "operator-1"; + route.snapshot.fragment = "historical-operator"; await createFixture(route); + workflowActionService.hasWorkflowOpen.mockReturnValue(true); + workflowActionService.getTempWorkflow.mockReturnValue(stubWorkflow); stubGraph.hasElementWithID.mockReturnValue(true); + fixture.detectChanges(); - expect(stubGraph.hasElementWithID).toHaveBeenCalledWith("operator-1"); - expect(workflowActionService.highlightElements).toHaveBeenCalledWith(false, "operator-1"); + + expect(workflowActionService.highlightElements).toHaveBeenCalledWith(false, "historical-operator"); + expect(workflowActionService.reloadWorkflow).not.toHaveBeenCalled(); + expect(undoRedoService.clearUndoStack).not.toHaveBeenCalled(); + expect(undoRedoService.clearRedoStack).not.toHaveBeenCalled(); + expect(workflowActionService.getTempWorkflow()).toBe(stubWorkflow); }); - it("when URL fragment does not match any element, surfaces an error and clears the fragment", async () => { - const route = configureRoute({ id: "42" }); - route.snapshot.fragment = "stale-id"; + it("preserves a nested fragment when correcting a macro definition's canvas route", async () => { + const route = configureRoute({ id: "42" }, { macro: ["outer", "child"] }); + route.snapshot.fragment = "nested-operator"; + (route.snapshot as any).routeConfig = { path: "workflow/:id" }; await createFixture(route); - // Default mock already returns false, but state explicitly for clarity. - stubGraph.hasElementWithID.mockReturnValue(false); + routerMock.url = "/user/workflow/42?macro=outer¯o=child#nested-operator"; + workflowActionService.getWorkflow.mockReturnValue({ + ...stubWorkflow, + content: { ...stubWorkflow.content, macroInterface: { id: "definition" } }, + }); + fixture.detectChanges(); - expect(notificationService.error).toHaveBeenCalledWith(expect.stringContaining("stale-id")); - // Two router.navigate calls: one preserving fragment, one clearing it. - expect(routerMock.navigate).toHaveBeenLastCalledWith([], { relativeTo: route }); + + expect(routerMock.navigate).toHaveBeenLastCalledWith(["/user/macros/42"], { + replaceUrl: true, + queryParams: route.snapshot.queryParams, + fragment: "nested-operator", + }); }); }); @@ -498,8 +590,7 @@ describe("WorkspaceComponent", () => { describe("updateViewCount", () => { it("posts a view event with the route's wid and the current user's uid", async () => { const route = configureRoute({ id: "42" }); - await createFixture(route); - fixture.detectChanges(); + await renderWorkspace(route); expect(hubService.postView).toHaveBeenCalledWith("42", 7, EntityType.Workflow); }); @@ -515,8 +606,7 @@ describe("WorkspaceComponent", () => { describe("onWIDChange", () => { it("syncs writeAccess from metadata.readonly each time the metadata changes", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); expect(component.writeAccess).toBe(false); // default before any emission workflowActionService.getWorkflowMetadata.mockReturnValue({ wid: 42, readonly: false }); @@ -529,8 +619,7 @@ describe("WorkspaceComponent", () => { }); it("ignores metadata emissions that have no wid yet", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); workflowActionService.getWorkflowMetadata.mockReturnValue({ wid: undefined, readonly: false }); metadataChangedSubject.next(); // writeAccess stays at its initial false — no metadata.wid means we don't know @@ -541,16 +630,14 @@ describe("WorkspaceComponent", () => { describe("ngOnDestroy", () => { it("persists the workflow on destroy when the user is signed in and persist is enabled", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); component.ngOnDestroy(); expect(workflowPersistService.persistWorkflow).toHaveBeenCalledWith(stubWorkflow); expect(workflowActionService.clearWorkflow).toHaveBeenCalled(); }); it("skips the persist call when the user is not signed in", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); userService.isLogin.mockReturnValue(false); component.ngOnDestroy(); expect(workflowPersistService.persistWorkflow).not.toHaveBeenCalled(); @@ -559,8 +646,7 @@ describe("WorkspaceComponent", () => { }); it("tears down every piece of websocket-derived state when leaving the workspace (issue #3120)", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); component.ngOnDestroy(); expect(computingUnitStatusService.disconnect).toHaveBeenCalled(); expect(executeWorkflowService.resetExecutionAndWorkers).toHaveBeenCalled(); @@ -590,8 +676,7 @@ describe("WorkspaceComponent", () => { // Dispatching the DOM event, rather than calling the handler, is what would catch the host // binding being removed or miswired. it("saves on beforeunload and tears nothing down, so a page restored from the cache still works", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); window.dispatchEvent(new Event("beforeunload")); @@ -607,8 +692,7 @@ describe("WorkspaceComponent", () => { // views -- the shared document and its room, the computing unit, the running execution -- // is the same one, and dropping it here would cost the Form View a reconnect for nothing. it("keeps the session when this workflow's Form View takes over", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); routerMock.getCurrentNavigation.mockReturnValue({ finalUrl: workspaceFormUrl(42) }); component.ngOnDestroy(); @@ -622,8 +706,7 @@ describe("WorkspaceComponent", () => { }); it("skips even the save on beforeunload when the user is not signed in", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); userService.isLogin.mockReturnValue(false); component.onBeforeUnload(); @@ -636,8 +719,7 @@ describe("WorkspaceComponent", () => { // this side handed such a workflow over while the arriving side, keyed on the room, declined // it and reloaded -- so both key on the room, and this one is rebuilt on its first switch. it("tears it down when the document is in no workflow's room, even bound for this one's form", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); workflowActionService.getOpenWorkflowId.mockReturnValue(undefined); routerMock.getCurrentNavigation.mockReturnValue({ finalUrl: workspaceFormUrl(42) }); @@ -651,8 +733,7 @@ describe("WorkspaceComponent", () => { // arriving menu had just restored it (#8552). It goes with the metrics now: reset on leaving, // kept on a hand-over. it("resets the heat-map view on leaving and keeps it on a hand-over", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); const setHeatmapView = workflowActionService.getJointGraphWrapper().setHeatmapView; routerMock.getCurrentNavigation.mockReturnValue({ finalUrl: workspaceFormUrl(42) }); @@ -665,8 +746,7 @@ describe("WorkspaceComponent", () => { }); it("tears it down when the destination is another workflow's Form View", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); routerMock.getCurrentNavigation.mockReturnValue({ finalUrl: workspaceFormUrl(43) }); component.ngOnDestroy(); @@ -676,8 +756,7 @@ describe("WorkspaceComponent", () => { }); it("clears the workflow session state when the computing unit is switched in-canvas (issue #3120)", async () => { - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); // Switching to a different unit emits on the connection-reset stream. connectionResetSubject.next(); expect(executeWorkflowService.resetExecutionAndWorkers).toHaveBeenCalled(); @@ -700,8 +779,7 @@ describe("WorkspaceComponent", () => { describe("child rendering side effects", () => { it("publishes the resolved ViewContainerRef to CodeEditorService.vc on view init", async () => { codeEditorService.vc = undefined; - await createFixture(); - fixture.detectChanges(); + await renderWorkspace(); // createEmbeddedView is present on a real ViewContainerRef but not on the // pre-fixture stub, so checking it distinguishes the resolved query from // the placeholder. diff --git a/frontend/src/app/workspace/component/workspace.component.ts b/frontend/src/app/workspace/component/workspace.component.ts index 3727c4b8c20..d753f79a2cb 100644 --- a/frontend/src/app/workspace/component/workspace.component.ts +++ b/frontend/src/app/workspace/component/workspace.component.ts @@ -17,6 +17,7 @@ * under the License. */ +import { isMacro } from "../../common/util/workflow-macro"; import { Location, NgIf } from "@angular/common"; import { AfterViewInit, @@ -37,9 +38,9 @@ import { OperatorMetadataService } from "../service/operator-metadata/operator-m import { UndoRedoService } from "../service/undo-redo/undo-redo.service"; import { WorkflowActionService } from "../service/workflow-graph/model/workflow-action.service"; import { NzMessageService } from "ng-zorro-antd/message"; -import { debounceTime, distinctUntilChanged, filter, switchMap, throttleTime } from "rxjs/operators"; +import { debounceTime, distinctUntilChanged, filter, switchMap, throttleTime, concatMap, map } from "rxjs/operators"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; -import { forkJoin, of } from "rxjs"; +import { forkJoin, of, from, firstValueFrom } from "rxjs"; import { isDefined } from "../../common/util/predicate"; import { NotificationService } from "src/app/common/service/notification/notification.service"; import { WorkflowConsoleService } from "../service/workflow-console/workflow-console.service"; @@ -49,7 +50,7 @@ import { WorkflowMetadata } from "src/app/dashboard/type/workflow-metadata.inter import { EntityType, HubService } from "../../hub/service/hub.service"; import { THROTTLE_TIME_MS } from "../../hub/component/workflow/detail/hub-workflow-detail.component"; import { WorkflowCompilingService } from "../service/compile-workflow/workflow-compiling.service"; -import { isLeavingWorkspace, USER_WORKSPACE } from "../../app-routing.constant"; +import { isLeavingWorkspace, USER_WORKSPACE, macroCanvasUrl, workspaceCanvasUrl } from "../../app-routing.constant"; import { GuiConfigService } from "../../common/service/gui-config.service"; import { ComputingUnitStatusService } from "../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; import { ExecuteWorkflowService } from "../service/execute-workflow/execute-workflow.service"; @@ -65,6 +66,8 @@ import { MiniMapComponent } from "./workflow-editor/mini-map/mini-map.component" import { LeftPanelComponent } from "./left-panel/left-panel.component"; import { AgentPanelComponent } from "./agent/agent-panel/agent-panel.component"; import { PropertyEditorComponent } from "./property-editor/property-editor.component"; +import { MacroNavigationService } from "../service/macro/macro-navigation.service"; +import { MacroService } from "../service/macro/macro.service"; import { JupyterNotebookPanelComponent } from "./jupyter-notebook-panel/jupyter-notebook-panel.component"; export const SAVE_DEBOUNCE_TIME_IN_MS = 5000; @@ -116,6 +119,8 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { * what it is read from. */ private resumedSession = false; + private macroNavigationRegistered = false; + private restoringMacroPath = false; constructor( private userService: UserService, @@ -142,10 +147,17 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { private executeWorkflowService: ExecuteWorkflowService, private workflowResultService: WorkflowResultService, private heatmapStatsRestoreService: HeatmapStatsRestoreService, + public macroNavigation: MacroNavigationService, + private macros: MacroService, private workflowVersionService: WorkflowVersionService ) {} ngOnInit() { + this.macroNavigation.changing$.pipe(untilDestroyed(this)).subscribe(() => this.codeEditorViewRef.clear()); + this.macros.inspect$.pipe(untilDestroyed(this)).subscribe(id => { + this.macroNavigation.open(id).catch(error => this.notificationService.error(error.message)); + }); + this.workflowResultService.setMacroAliases?.(port => this.macroNavigation.resultAliases(port)); this.workflowActionService.setHighlightingEnabled(true); // Clear session state when the user switches computing units in-canvas, so // the previous unit's status/console/results don't linger. @@ -188,6 +200,7 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { this.updateViewCount(); this.registerLoadOperatorMetadata(); this.codeEditorService.vc = this.codeEditorViewRef; + this.changeDetectorRef.detectChanges(); } /** @@ -212,6 +225,7 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { ngOnDestroy() { this.persistBeforeLeaving(); + this.macroNavigation.clear(); this.codeEditorViewRef.clear(); // Tear down the connection and all websocket-derived session state so a // re-entered workflow starts clean instead of reusing the previous one -- unless the Form @@ -229,10 +243,108 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { } } + private registerMacroNavigation(initialFragment = this.route.snapshot.fragment): void { + if (this.macroNavigationRegistered) return; + this.macroNavigationRegistered = true; + const route = this.route.snapshot.routeConfig?.path; + if (route === "workflow/:id" || route === "macros/:id") { + const workflow = this.macroNavigation.rootWorkflow; + const target = workflow.content.macroInterface + ? macroCanvasUrl(workflow.wid!) + : workspaceCanvasUrl(workflow.wid!); + if (this.router.url.split(/[?#]/)[0] !== target) { + this.router.navigate([target], { + replaceUrl: true, + queryParams: this.route.snapshot.queryParams, + fragment: initialFragment ?? undefined, + }); + return; + } + } + const syncURL = (replaceUrl = false) => + this.router.navigate([], { + relativeTo: this.route, + queryParamsHandling: "merge", + replaceUrl, + queryParams: { macro: this.macroNavigation.parents.map(p => p.operatorID) }, + fragment: undefined, + }); + this.macroNavigation.changed$.pipe(untilDestroyed(this)).subscribe(() => { + if (!this.macroNavigation.parents.length && !this.workflowActionService.getTempWorkflow()) + this.executeWorkflowService.reapplyExecutionLock(); + if (!this.restoringMacroPath) syncURL(); + }); + this.route.queryParamMap + .pipe( + map((params, index) => ({ + path: params.getAll("macro"), + fragment: index === 0 ? initialFragment : this.route.snapshot.fragment, + })), + distinctUntilChanged((a, b) => a.path.length === b.path.length && a.path.every((id, i) => id === b.path[i])), + concatMap(async ({ path, fragment }) => { + const current = this.macroNavigation.parents.map(p => p.operatorID); + let common = 0; + while (common < path.length && path[common] === current[common]) common++; + this.restoringMacroPath = true; + try { + if (common < current.length) await this.macroNavigation.back(common); + for (const id of path.slice(common)) await this.macroNavigation.open(id); + } catch (error) { + this.notificationService.error(error instanceof Error ? error.message : String(error)); + syncURL(true); + return; + } finally { + this.restoringMacroPath = false; + } + this.selectURLFragment(fragment); + }), + untilDestroyed(this) + ) + .subscribe(); + } + + private selectURLFragment(fragment: string | null): void { + // Loading a graph changes its highlights, so restore the URL after the breadcrumb document is ready. + this.router.navigate([], { + relativeTo: this.route, + fragment: fragment ?? undefined, + preserveFragment: false, + queryParamsHandling: "preserve", + }); + if (!fragment) return; + if (this.workflowActionService.getTexeraGraph().hasElementWithID(fragment)) { + this.workflowActionService.highlightElements(false, fragment); + } else { + this.notificationService.error(`Element ${fragment} doesn't exist`); + this.router.navigate([], { relativeTo: this.route, queryParamsHandling: "preserve" }); + } + } + + public async canLeave(): Promise { + // Restoring the root for an outgoing route must not start another breadcrumb navigation. + this.restoringMacroPath = true; + try { + if (this.macroNavigation.parents.length) await this.macroNavigation.back(0); + if (this.workflowActionService.getTempWorkflow()) this.workflowVersionService.closeParticularVersionDisplay(); + return true; + } catch (error) { + this.notificationService.error(error instanceof Error ? error.message : String(error)); + return false; + } finally { + this.restoringMacroPath = false; + } + } + private persistBeforeLeaving(): void { if (this.userService.isLogin() && this.workflowPersistService.isWorkflowPersistEnabled()) { const workflow = this.workflowActionService.getWorkflow(); - this.workflowPersistService.persistWorkflow(workflow).pipe(untilDestroyed(this)).subscribe(); + void firstValueFrom( + this.macroNavigation.parents.length + ? from(this.macroNavigation.save()) + : this.workflowPersistService.persistWorkflow(workflow) + ).catch((error: unknown) => + this.notificationService.error(error instanceof Error ? error.message : String(error)) + ); } } @@ -255,18 +367,29 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { this.workflowActionService .workflowChanged() - .pipe(debounceTime(SAVE_DEBOUNCE_TIME_IN_MS)) + .pipe( + filter(() => !this.workflowActionService.getJointGraphWrapper().getReloadingWorkflow()), + debounceTime(SAVE_DEBOUNCE_TIME_IN_MS) + ) .pipe(untilDestroyed(this)) .subscribe(() => { if (this.userService.isLogin() && this.workflowPersistService.isWorkflowPersistEnabled()) { - this.workflowPersistService - .persistWorkflow(this.workflowActionService.getWorkflow()) + const wid = this.workflowActionService.getWorkflowMetadata().wid; + (this.macroNavigation.parents.length + ? from(this.macroNavigation.save()) + : this.workflowPersistService.persistWorkflow(this.workflowActionService.getWorkflow()) + ) .pipe(untilDestroyed(this)) - .subscribe((updatedWorkflow: Workflow) => { - if (this.workflowActionService.getWorkflowMetadata().wid !== updatedWorkflow.wid) { - this.location.go(`${USER_WORKSPACE}/${updatedWorkflow.wid}`); - } - this.workflowActionService.setWorkflowMetadata(updatedWorkflow); + .subscribe({ + next: (updatedWorkflow: Workflow) => { + if (this.workflowActionService.getWorkflowMetadata().wid !== wid) return; + if (this.workflowActionService.getWorkflowMetadata().wid !== updatedWorkflow.wid) { + this.location.go(`${USER_WORKSPACE}/${updatedWorkflow.wid}`); + } + this.workflowActionService.setWorkflowMetadata(updatedWorkflow); + }, + error: (error: unknown) => + this.notificationService.error(error instanceof Error ? error.message : String(error)), }); // to sync up with the updated information, such as workflow.wid } @@ -304,40 +427,18 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { if (shouldAutoLayout) { this.workflowActionService.autoLayoutWorkflow(); } - // Best-effort: repaint the heat-map from the last run's persisted - // statistics. The service itself gates on the overlay being - // persisted-on (and skips during a live execution), so this is a - // no-op for everyone else. Fetch failures are swallowed there; an - // error arriving here is a mapping or ingestion bug, so it is logged. this.heatmapStatsRestoreService - .restoreLatestRunStatistics() + .restoreLatestRunStatistics(workflow.content.operators.some(isMacro)) .pipe(untilDestroyed(this)) .subscribe({ error: (err: unknown) => console.error("Failed to restore heat-map statistics:", err), }); - // set the URL fragment to previous value - // because reloadWorkflow will highlight/unhighlight all elements - // which will change the URL fragment - this.router.navigate([], { - relativeTo: this.route, - fragment: fragment !== null ? fragment : undefined, - preserveFragment: false, - }); - // highlight the operator, comment box, or link in the URL fragment - if (fragment) { - if (this.workflowActionService.getTexeraGraph().hasElementWithID(fragment)) { - this.workflowActionService.highlightElements(false, fragment); - } else { - this.notificationService.error(`Element ${fragment} doesn't exist`); - // remove the fragment from the URL - this.router.navigate([], { relativeTo: this.route }); - } - } // clear stack this.undoRedoService.clearUndoStack(); this.undoRedoService.clearRedoStack(); this.setLoadingState(false); this.triggerCenter(); + this.registerMacroNavigation(fragment); }, () => { this.workflowActionService.resetAsNewWorkflow(); @@ -374,6 +475,7 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { // and no id, the computing unit picker does not restore the unit this workflow last ran // on, and this page believes the user cannot write to the workflow. this.workflowActionService.republishWorkflowMetadata(); + this.registerMacroNavigation(); return; } // load workflow with wid if presented in the URL @@ -392,6 +494,7 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { .subscribe(() => { // no workflow to load; directly register auto persist for brand-new workflow this.registerAutoPersistWorkflow(); + this.registerMacroNavigation(); }); } diff --git a/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.spec.ts b/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.spec.ts index 28f975540e1..d3ae3fd7e5a 100644 --- a/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.spec.ts +++ b/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.spec.ts @@ -16,7 +16,11 @@ * specific language governing permissions and limitations * under the License. */ +import { testPort, testMacro, expansionFixture } from "../../../common/testing/macro-graph"; +import { testField, testLink } from "../../../common/testing/workflow-graph"; +import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; +import { MockComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; import { JSONSchema7Definition } from "json-schema"; import { fakeAsync, TestBed, tick } from "@angular/core/testing"; import { HttpTestingController, TestRequest } from "@angular/common/http/testing"; @@ -46,8 +50,13 @@ import { AppSettings } from "../../../common/app-setting"; import { serializePortIdentity } from "../../../common/util/port-identity-serde"; import { commonTestImports, commonTestProviders } from "../../../common/testing/test-utils"; import { firstValueFrom } from "rxjs"; -import { CompilationState } from "../../types/workflow-compiling.interface"; +import { CompilationState, OperatorPortSchemaMap } from "../../types/workflow-compiling.interface"; +import { CustomJSONSchema7 } from "../../types/custom-json-schema.interface"; +import { mockNlpSentimentSchema } from "../operator-metadata/mock-operator-metadata.data"; import { OperatorSchema } from "../../types/operator-schema.interface"; +import { macroInstance, macroRuntimeID } from "../../../common/util/workflow-macro"; + +const port = (id: number): string => serializePortIdentity({ id, internal: false }); describe("WorkflowCompilingService.dropInvalidAttributeValues", () => { // A schema shaped like the Aggregate operator after schema propagation has filled in the @@ -172,20 +181,7 @@ describe("WorkflowCompilingService schema propagation property cleanup", () => { let dynamicSchemaService: DynamicSchemaService; beforeEach(() => { - TestBed.configureTestingModule({ - imports: [...commonTestImports], - providers: [ - { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, - JointUIService, - WorkflowActionService, - WorkflowUtilService, - UndoRedoService, - DynamicSchemaService, - ValidationWorkflowService, - WorkflowCompilingService, - ...commonTestProviders, - ], - }); + configureCompilingTestBed(); service = TestBed.inject(WorkflowCompilingService); workflowActionService = TestBed.inject(WorkflowActionService); dynamicSchemaService = TestBed.inject(DynamicSchemaService); @@ -262,20 +258,7 @@ describe("WorkflowCompilingService public getters", () => { let service: WorkflowCompilingService; beforeEach(() => { - TestBed.configureTestingModule({ - imports: [...commonTestImports], - providers: [ - { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, - JointUIService, - WorkflowActionService, - WorkflowUtilService, - UndoRedoService, - DynamicSchemaService, - ValidationWorkflowService, - WorkflowCompilingService, - ...commonTestProviders, - ], - }); + configureCompilingTestBed(); service = TestBed.inject(WorkflowCompilingService); }); @@ -378,226 +361,76 @@ describe("WorkflowCompilingService public getters", () => { }); }); -describe("WorkflowCompilingService.setOperatorInputAttrs / restoreOperatorInputAttrs", () => { - const port0 = serializePortIdentity({ id: 0, internal: false }); - - // Builds an OperatorSchema wrapping the given json schema. Only jsonSchema matters for these pure transforms; - // additionalMetadata is included only to satisfy the OperatorSchema type. - const makeOperatorSchema = (jsonSchema: any): OperatorSchema => - ({ - operatorType: "TestOp", - operatorVersion: "1", - jsonSchema, - additionalMetadata: { - userFriendlyName: "Test Op", - operatorGroupName: "Test", - inputPorts: [{ displayName: "input" }], - outputPorts: [{ displayName: "output" }], - }, - }) as unknown as OperatorSchema; - - // input port 0 exposes two attributes. - const inputPortSchemaMap = { - [port0]: [ +describe("WorkflowCompilingService column schemas", () => { + const columns: OperatorPortSchemaMap = { + [port(0)]: [ { attributeName: "col_a", attributeType: "string" }, { attributeName: "col_b", attributeType: "integer" }, ], - } as any; - - describe("setOperatorInputAttrs", () => { - it("returns the original operator schema unchanged when the input schema map is undefined", () => { - const schema = makeOperatorSchema({ type: "object", properties: {} }); - expect(WorkflowCompilingService.setOperatorInputAttrs(schema, undefined)).toBe(schema); - }); - - it("returns the original operator schema unchanged when the input schema map is empty", () => { - const schema = makeOperatorSchema({ type: "object", properties: {} }); - expect(WorkflowCompilingService.setOperatorInputAttrs(schema, {})).toBe(schema); - }); - - it("injects the input attribute names as an enum on an optional attributeName property", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 0 }, - }, - }); - - const result = WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap); - const prop = (result.jsonSchema.properties as any).attribute; - // optional (not in `required`) properties append "" so the empty selection passes validation. - expect(prop.enum).toEqual(["col_a", "col_b", ""]); - expect(prop.type).toBe("string"); - expect(prop.uniqueItems).toBe(true); - }); - - it('appends the property\'s string default instead of "" for optional properties', () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 0, default: "col_a" }, - }, - }); - - const result = WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap); - expect((result.jsonSchema.properties as any).attribute.enum).toEqual(["col_a", "col_b", "col_a"]); - }); - - it("throws when the property's default value is not a string", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 0, default: 42 }, - }, - }); - - expect(() => WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap)).toThrow( - "default value must be a string" - ); - }); - - it("does not append an empty option for required properties", () => { - const schema = makeOperatorSchema({ - type: "object", - required: ["attribute"], - properties: { - attribute: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 0 }, - }, - }); - - expect( - (WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap).jsonSchema.properties as any) - .attribute.enum - ).toEqual(["col_a", "col_b"]); - }); - - it("includes additionalEnumValue before the optional empty option", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { - type: "string", - autofill: "attributeName", - autofillAttributeOnPort: 0, - additionalEnumValue: "*", - }, - }, - }); - - expect( - (WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap).jsonSchema.properties as any) - .attribute.enum - ).toEqual(["col_a", "col_b", "*", ""]); - }); - - it("yields an undefined enum when autofillAttributeOnPort is missing", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { type: "string", autofill: "attributeName" }, - }, - }); - - expect( - (WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap).jsonSchema.properties as any) - .attribute.enum - ).toBeUndefined(); - }); - - it("yields an undefined enum when the referenced input port has no schema", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - // port 5 is not present in inputPortSchemaMap. - attribute: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 5 }, - }, - }); - - expect( - (WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap).jsonSchema.properties as any) - .attribute.enum - ).toBeUndefined(); - }); - - it("injects the enum into the items of an attributeNameList property", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attributes: { - type: "array", - autofill: "attributeNameList", - autofillAttributeOnPort: 0, - items: { type: "string" }, - }, - }, - }); - - const prop = ( - WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap).jsonSchema.properties as any - ).attributes; - expect(prop.type).toBe("array"); - expect(prop.uniqueItems).toBe(true); - expect(prop.items.type).toBe("string"); - expect(prop.items.enum).toEqual(["col_a", "col_b", ""]); + }; + const scalar: CustomJSONSchema7 = { type: "string", autofill: "attributeName", autofillAttributeOnPort: 0 }; + const list: CustomJSONSchema7 = { + type: "array", + autofill: "attributeNameList", + autofillAttributeOnPort: 0, + items: { type: "string" }, + }; + const schemaWith = (attribute: CustomJSONSchema7, required = false): OperatorSchema => ({ + ...mockNlpSentimentSchema, + jsonSchema: { type: "object", properties: { attribute }, required: required ? ["attribute"] : [] }, + }); + const attributeOf = (schema: OperatorSchema) => schema.jsonSchema.properties!.attribute as CustomJSONSchema7; + const fill = (attribute: CustomJSONSchema7, required = false) => + attributeOf(WorkflowCompilingService.setOperatorInputAttrs(schemaWith(attribute, required), columns)); + + it.each([undefined, {}])("leaves the schema unchanged with no input schema (%j)", inputs => { + const schema = schemaWith(scalar); + expect(WorkflowCompilingService.setOperatorInputAttrs(schema, inputs)).toBe(schema); + }); + + it.each([ + { label: "optional", field: scalar, expected: ["col_a", "col_b", ""] }, + { label: "with a default", field: { ...scalar, default: "col_a" }, expected: ["col_a", "col_b", "col_a"] }, + { label: "required", field: scalar, required: true, expected: ["col_a", "col_b"] }, + { + label: "with an extra value", + field: { ...scalar, additionalEnumValue: "*" }, + expected: ["col_a", "col_b", "*", ""], + }, + { label: "with no port", field: { ...scalar, autofillAttributeOnPort: undefined }, expected: undefined }, + { label: "with no port schema", field: { ...scalar, autofillAttributeOnPort: 5 }, expected: undefined }, + ])("fills a scalar attribute $label", ({ field, required, expected }) => { + expect(fill(field, required)).toMatchObject({ type: "string", uniqueItems: true, enum: expected }); + }); + + it("rejects a non-string default for a scalar attribute", () => { + expect(() => fill({ ...scalar, default: 42 })).toThrow("default value must be a string"); + }); + + it("fills the items of an attribute list", () => { + expect(fill(list)).toMatchObject({ + type: "array", + uniqueItems: true, + items: { type: "string", enum: ["col_a", "col_b", ""] }, }); }); - describe("restoreOperatorInputAttrs", () => { - it("clears the injected enum/uniqueItems from an attributeName property", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { - type: "string", - autofill: "attributeName", - autofillAttributeOnPort: 0, - enum: ["col_a", "col_b", ""], - uniqueItems: true, - }, - }, - }); - - const prop = (WorkflowCompilingService.restoreOperatorInputAttrs(schema).jsonSchema.properties as any).attribute; - expect(prop.enum).toBeUndefined(); - expect(prop.uniqueItems).toBeUndefined(); - expect(prop.type).toBe("string"); - }); - - it("clears the injected enum from the items of an attributeNameList property", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attributes: { - type: "array", - autofill: "attributeNameList", - autofillAttributeOnPort: 0, - uniqueItems: true, - items: { type: "string", enum: ["col_a", "col_b", ""] }, - }, - }, - }); - - const prop = (WorkflowCompilingService.restoreOperatorInputAttrs(schema).jsonSchema.properties as any).attributes; - expect(prop.type).toBe("array"); - expect(prop.uniqueItems).toBeUndefined(); - expect(prop.items.enum).toBeUndefined(); - expect(prop.items.type).toBe("string"); - }); - - it("round-trips: restore reverses the enums that setOperatorInputAttrs injected", () => { - const schema = makeOperatorSchema({ - type: "object", - properties: { - attribute: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 0 }, - }, - }); - - const propagated = WorkflowCompilingService.setOperatorInputAttrs(schema, inputPortSchemaMap); - expect((propagated.jsonSchema.properties as any).attribute.enum).toEqual(["col_a", "col_b", ""]); + it.each([ + { label: "scalar", filled: { ...scalar, enum: ["col_a", "col_b", ""] }, original: scalar }, + { + label: "list", + filled: { ...list, items: { type: "string" as const, enum: ["col_a", "col_b", ""] } }, + original: list, + }, + ])("clears the propagated $label choices and uniqueness", ({ filled, original }) => { + const restored = WorkflowCompilingService.restoreOperatorInputAttrs(schemaWith({ ...filled, uniqueItems: true })); + expect(attributeOf(restored)).toEqual(original); + }); - const restored = WorkflowCompilingService.restoreOperatorInputAttrs(propagated); - expect((restored.jsonSchema.properties as any).attribute.enum).toBeUndefined(); - }); + it("restores the scalar schema after propagation", () => { + const propagated = fill(scalar); + expect(propagated.enum).toEqual(["col_a", "col_b", ""]); + expect(attributeOf(WorkflowCompilingService.restoreOperatorInputAttrs(schemaWith(propagated)))).toEqual(scalar); }); }); @@ -615,12 +448,11 @@ const configureCompilingTestBed = (): void => { ValidationWorkflowService, WorkflowCompilingService, ...commonTestProviders, + { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, ], }); }; -const port = (id: number): string => serializePortIdentity({ id, internal: false }); - describe("WorkflowCompilingService compile pipeline", () => { let service: WorkflowCompilingService; let workflowActionService: WorkflowActionService; @@ -661,7 +493,7 @@ describe("WorkflowCompilingService compile pipeline", () => { expect(request.request.method).toBe("POST"); expect(request.request.headers.get("Content-Type")).toBe("application/json"); - const body = JSON.parse(request.request.body); + const body = request.request.body; expect(body.operators.map((operator: any) => operator.operatorID).sort()).toEqual( [mockScanPredicate.operatorID, mockResultPredicate.operatorID].sort() ); @@ -672,6 +504,118 @@ describe("WorkflowCompilingService compile pipeline", () => { request.flush({ physicalPlan: { operators: [], links: [] }, operatorOutputSchemas: {}, operatorErrors: {} }); })); + it("sends pinned macro graphs and projects backend schemas and errors without changing snapshots", fakeAsync(() => { + const definition = testMacro("backend", [{ ...mockScanPredicate, operatorProperties: { tableName: "twitter" } }], { + outputs: [testPort("out", mockScanPredicate.operatorID, "output-0")], + name: "Nested source", + }); + workflowActionService.setMacroDependencies([definition]); + workflowActionService.addOperator(macroInstance(definition, "instance"), mockPoint); + const content = workflowActionService.getWorkflowContent(), + before = JSON.stringify(content); + tick(WORKFLOW_COMPILATION_DEBOUNCE_TIME_MS); + const request = httpTestingController.expectOne(compileUrl); + const plan = request.request.body; + expect(plan.operators).toEqual([]); + expect(plan.macroContent.macroDependencies).toEqual([definition]); + const expanded = expansionFixture(content, [ + [["instance", mockScanPredicate.operatorID], definition.content.operators[0]], + ]), + leaf = expanded.content.operators[0].operatorID; + request.flush({ + macroExpansion: expanded, + // A scoped passthrough's native source can be outside the visible expansion. + operatorOutputSchemas: { instance: { [port(0)]: scanOutputSchema } }, + operatorErrors: { + [leaf]: { operatorId: leaf, message: "invalid column" }, + instance: { operatorId: "instance", message: "Nested source / Scan: invalid column" }, + }, + }); + expect(service.getOperatorOutputSchemaMap("instance")).toEqual({ [port(0)]: scanOutputSchema }); + expect(service.getWorkflowCompilationErrors()["instance"].message).toBe("Nested source / Scan: invalid column"); + expect(workflowActionService.getMacroExpansion()).toEqual(expanded); + expect(JSON.stringify(workflowActionService.getWorkflowContent())).toBe(before); + })); + + it("cancels obsolete compilation responses when another graph edit starts compilation", fakeAsync(() => { + const previous = buildWorkflowAndCompile(); + workflowActionService.setOperatorProperty(mockScanPredicate.operatorID, { tableName: "new" }); + tick(WORKFLOW_COMPILATION_DEBOUNCE_TIME_MS); + const current = httpTestingController.expectOne(compileUrl); + expect(previous.cancelled).toBe(true); + current.flush({ physicalPlan: { operators: [], links: [] }, operatorOutputSchemas: {}, operatorErrors: {} }); + expect(service.getWorkflowCompilationState()).toBe(CompilationState.Succeeded); + })); + + it("propagates per-instance internal column choices through exposed nested fields and clears unavailable schemas", fakeAsync(() => { + const scan = { ...mockScanPredicate, operatorProperties: { tableName: "twitter" } }; + const secondScan = { + ...scan, + operatorID: "second-scan", + outputPorts: [{ portID: "output-0" }, { portID: "output-2" }], + }; + const secondSentiment = { ...mockSentimentPredicate, operatorID: "second-sentiment" }; + const operators = [scan, mockSentimentPredicate, secondScan, secondSentiment]; + const links = [ + testLink(scan.operatorID, mockSentimentPredicate.operatorID), + testLink(secondScan.operatorID, secondSentiment.operatorID, "output-2"), + ]; + const leaf = testMacro("leaf", operators, { + links, + formBinding: { + fields: [mockSentimentPredicate, secondSentiment].map((op, i) => + testField(op.operatorID, "attribute", `column-${i}`, `Column ${i}`) + ), + }, + name: "Internal columns", + }); + const outer = testMacro("outer", [macroInstance(leaf, "child")], { + formBinding: { + fields: leaf.content.formBinding.fields.map(f => ({ ...f, operatorID: "child", propertyKey: f.id })), + }, + name: "Nested columns", + }); + workflowActionService.setMacroDependencies([leaf, outer]); + workflowActionService.addOperator(macroInstance(outer, "outer-instance"), mockPoint); + const content = workflowActionService.getWorkflowContent(), + before = JSON.stringify(content); + const path = ["outer-instance", "child"]; + const runtime = (id: string) => macroRuntimeID(path, id); + const expanded = expansionFixture( + content, + operators.map(op => [[...path, op.operatorID], op]), + links.map(({ linkID, source, target }) => + testLink(runtime(source.operatorID), runtime(target.operatorID), source.portID, target.portID, linkID) + ) + ); + expanded.inspectors[JSON.stringify(path)].inputs = { + [mockSentimentPredicate.operatorID]: [{ link: 0, input: 0, output: 0 }], + [secondSentiment.operatorID]: [{ link: 1, input: 0, output: 1 }], + }; + const columns = () => + TestBed.inject(DynamicSchemaService).getDynamicSchema("outer-instance").jsonSchema.properties!; + tick(WORKFLOW_COMPILATION_DEBOUNCE_TIME_MS); + httpTestingController.expectOne(compileUrl).flush({ + macroExpansion: expanded, + operatorOutputSchemas: { + [runtime(scan.operatorID)]: { [port(0)]: scanOutputSchema }, + [runtime(secondScan.operatorID)]: { [port(1)]: [{ attributeName: "col_b", attributeType: "integer" }] }, + }, + operatorErrors: {}, + }); + expect((columns()["column-0"] as any).enum).toEqual(["col_a"]); + expect((columns()["column-1"] as any).enum).toEqual(["col_b"]); + expect(JSON.stringify(workflowActionService.getWorkflowContent())).toBe(before); + + workflowActionService.setOperatorProperty("outer-instance", { "column-0": "col_a" }); + tick(WORKFLOW_COMPILATION_DEBOUNCE_TIME_MS); + httpTestingController + .expectOne(compileUrl) + .flush({ macroExpansion: expanded, operatorOutputSchemas: {}, operatorErrors: {} }); + expect((columns()["column-0"] as any).enum).toBeUndefined(); + expect((columns()["column-1"] as any).enum).toBeUndefined(); + })); + it("records the physical plan and output schemas when the response carries a physical plan", fakeAsync(() => { const request = buildWorkflowAndCompile(); const physicalPlan = { operators: [{ id: "physical-1" }], links: [] }; @@ -839,34 +783,94 @@ describe("WorkflowCompilingService input port schema resolution", () => { predicates.forEach(predicate => workflowActionService.addOperator(predicate, mockPoint)); const link = (sourceID: string, sourcePort: string, targetID: string, targetPort: string, linkID: string) => - workflowActionService.addLink({ - linkID, - source: { operatorID: sourceID, portID: sourcePort }, - target: { operatorID: targetID, portID: targetPort }, + workflowActionService.addLink(testLink(sourceID, targetID, sourcePort, targetPort, linkID)); + + const addMacro = () => { + const definition = testMacro("schema-boundary", [], { + inputs: ["input-first", "input-second"].map(portID => ({ portID, targets: [] })), + outputs: ["output-first", "output-second"].map(portID => ({ portID, targets: [] })), + name: "Two ports", }); + workflowActionService.setMacroDependencies([definition]); + const instance = macroInstance(definition, "two-ports"); + addOperators(instance); + return instance; + }; - it("ignores an input link whose target port ID is not in the input- form", () => { - const target: OperatorPredicate = { ...mockSentimentPredicate, inputPorts: [{ portID: "the-input" }] }; - addOperators(mockScanPredicate, target); - link(mockScanPredicate.operatorID, "output-0", target.operatorID, "the-input", "link-bad-target"); + it.each([false, true])("uses the macro output ordinal even if its schema is absent (%s)", absent => { + const source = addMacro(); + addOperators(mockSentimentPredicate); + link(source.operatorID, "output-second", mockSentimentPredicate.operatorID, "input-0", "macro-output"); + setOutputSchemas({ + [source.operatorID]: absent ? { [port(0)]: schemaA } : { [port(0)]: schemaA, [port(1)]: schemaB }, + }); + expect(service.getPortInputSchema(mockSentimentPredicate.operatorID, 0)).toEqual(absent ? undefined : schemaB); + }); + + it("resolves a macro's stable input ID without assigning the schema to another port", () => { + const target = addMacro(); + addOperators(mockScanPredicate); + link(mockScanPredicate.operatorID, "output-0", target.operatorID, "input-second", "macro-input"); setOutputSchemas({ [mockScanPredicate.operatorID]: { [port(0)]: schemaA } }); - const inputSchemaMap = service.getOperatorInputSchemaMap(target.operatorID); + expect(service.getPortInputSchema(target.operatorID, 0)).toBeUndefined(); + expect(service.getPortInputSchema(target.operatorID, 1)).toEqual(schemaA); + }); - expect(Object.keys(inputSchemaMap!)).toEqual([port(0)]); - expect(inputSchemaMap![port(0)]).toBeUndefined(); + it("keeps native port schema ordinals after removing a middle input and output", () => { + const source = { + ...mockMultiInputOutputPredicate, + operatorID: "native-source", + outputPorts: [{ portID: "output-0" }, { portID: "output-2" }], + }; + const target = { + ...mockMultiInputOutputPredicate, + inputPorts: [{ portID: "input-0" }, { portID: "input-2" }], + }; + addOperators(source, target); + const base = dynamicSchemaService.getDynamicSchema(target.operatorID); + dynamicSchemaService.setDynamicSchema(target.operatorID, { + ...base, + additionalMetadata: { ...base.additionalMetadata, inputPorts: [{}, {}] }, + }); + link(source.operatorID, "output-2", target.operatorID, "input-2", "native-gaps"); + setOutputSchemas({ [source.operatorID]: { [port(0)]: schemaA, [port(1)]: schemaB } }); + + expect(service.getPortInputSchema(target.operatorID, 0)).toBeUndefined(); + expect(service.getPortInputSchema(target.operatorID, 1)).toEqual(schemaB); }); - it("ignores an input link whose source port ID is not in the output- form", () => { - const source: OperatorPredicate = { ...mockScanPredicate, outputPorts: [{ portID: "the-output" }] }; - addOperators(source, mockSentimentPredicate); - link(source.operatorID, "the-output", mockSentimentPredicate.operatorID, "input-0", "link-bad-source"); + it.each(["input", "output"] as const)("resolves a native %s independently of its ID format", direction => { + const sourcePort = direction === "output" ? "the-output" : "output-0"; + const targetPort = direction === "input" ? "the-input" : "input-0"; + const source = { ...mockScanPredicate, outputPorts: [{ portID: sourcePort }] }; + const target = { ...mockSentimentPredicate, inputPorts: [{ portID: targetPort }] }; + addOperators(source, target); + link(source.operatorID, sourcePort, target.operatorID, targetPort, "custom-port"); setOutputSchemas({ [source.operatorID]: { [port(0)]: schemaA } }); + expect(service.getOperatorInputSchemaMap(target.operatorID)).toEqual({ [port(0)]: schemaA }); + }); - const inputSchemaMap = service.getOperatorInputSchemaMap(mockSentimentPredicate.operatorID); + it("uses numbered projected ports only for sources outside the visible graph", () => { + addOperators(mockSentimentPredicate); + setOutputSchemas({}); + const input = { + source: { operatorID: "external", portID: "output-2" }, + target: { operatorID: mockSentimentPredicate.operatorID, portID: "input-0" }, + }; + (service as any).macroInputs = { + doc: workflowActionService.suspendWorkflow().shared, + inputLinks: () => [input], + schemas: { external: { [port(2)]: schemaB } }, + }; + expect(service.getPortInputSchema(mockSentimentPredicate.operatorID, 0)).toEqual(schemaB); - expect(Object.keys(inputSchemaMap!)).toEqual([port(0)]); - expect(inputSchemaMap![port(0)]).toBeUndefined(); + input.source.portID = "missing"; + expect(service.getPortInputSchema(mockSentimentPredicate.operatorID, 0)).toBeUndefined(); + + addOperators({ ...mockScanPredicate, operatorID: "external" }); + input.source.portID = "output-2"; + expect(service.getPortInputSchema(mockSentimentPredicate.operatorID, 0)).toBeUndefined(); }); it("resolves no schema when the upstream operator is missing from the output schema map", () => { @@ -901,41 +905,26 @@ describe("WorkflowCompilingService input port schema resolution", () => { expect(inputSchemaMap![port(2)]).toBeUndefined(); }); - it("accepts two links into the same input port when they agree on the schema", () => { - const secondScan: OperatorPredicate = { ...mockScanPredicate, operatorID: "scan-2" }; - addOperators(mockScanPredicate, secondScan, mockSentimentPredicate); - link(mockScanPredicate.operatorID, "output-0", mockSentimentPredicate.operatorID, "input-0", "link-scan-1"); - link(secondScan.operatorID, "output-0", mockSentimentPredicate.operatorID, "input-0", "link-scan-2"); - setOutputSchemas({ - [mockScanPredicate.operatorID]: { [port(0)]: schemaA }, - [secondScan.operatorID]: { [port(0)]: [...schemaA] }, - }); - - const inputSchemaMap = service.getOperatorInputSchemaMap(mockSentimentPredicate.operatorID); - - expect(inputSchemaMap![port(0)]).toEqual(schemaA); - expect(service.getWorkflowCompilationState()).toBe(CompilationState.Succeeded); - }); - - it("fails compilation when two links into the same input port disagree on the schema", () => { + it.each([true, false])("resolves two links to one port based on whether their schemas agree (%s)", agree => { const secondScan: OperatorPredicate = { ...mockScanPredicate, operatorID: "scan-2" }; addOperators(mockScanPredicate, secondScan, mockSentimentPredicate); link(mockScanPredicate.operatorID, "output-0", mockSentimentPredicate.operatorID, "input-0", "link-scan-1"); link(secondScan.operatorID, "output-0", mockSentimentPredicate.operatorID, "input-0", "link-scan-2"); setOutputSchemas({ [mockScanPredicate.operatorID]: { [port(0)]: schemaA }, - [secondScan.operatorID]: { [port(0)]: schemaB }, + [secondScan.operatorID]: { [port(0)]: agree ? [...schemaA] : schemaB }, }); const inputSchemaMap = service.getOperatorInputSchemaMap(mockSentimentPredicate.operatorID); - // the conflicting port is left unresolved and the compilation is marked failed - expect(inputSchemaMap![port(0)]).toBeUndefined(); - expect(service.getWorkflowCompilationState()).toBe(CompilationState.Failed); - const error = service.getWorkflowCompilationErrors()[mockSentimentPredicate.operatorID]; - expect(error.message).toBe("Multiple links with different schemas connected to the same input port 0"); - expect(error.details).toBe("Port 0 received 2 different schemas (some may be undefined)"); - expect(error.operatorId).toBe(mockSentimentPredicate.operatorID); + expect(inputSchemaMap![port(0)]).toEqual(agree ? schemaA : undefined); + expect(service.getWorkflowCompilationState()).toBe(agree ? CompilationState.Succeeded : CompilationState.Failed); + if (!agree) + expect(service.getWorkflowCompilationErrors()[mockSentimentPredicate.operatorID]).toMatchObject({ + message: "Multiple links with different schemas connected to the same input port 0", + details: "Port 0 received 2 different schemas (some may be undefined)", + operatorId: mockSentimentPredicate.operatorID, + }); }); it("resolves nothing for an operator whose dynamic schema declares no input ports", () => { diff --git a/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts b/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts index dae47a0088b..a54df3f5a09 100644 --- a/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts +++ b/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts @@ -17,16 +17,16 @@ * under the License. */ -import { HttpClient, HttpHeaders } from "@angular/common/http"; +import { HttpClient } from "@angular/common/http"; import { Injectable } from "@angular/core"; import { JSONSchema7Definition } from "json-schema"; -import { EMPTY, merge, Observable, ReplaySubject } from "rxjs"; +import { defer, EMPTY, merge, Observable, of, ReplaySubject } from "rxjs"; import { CustomJSONSchema7 } from "src/app/workspace/types/custom-json-schema.interface"; import { AppSettings } from "../../../common/app-setting"; import { areOperatorSchemasEqual, OperatorSchema } from "../../types/operator-schema.interface"; import { ExecuteWorkflowService } from "../execute-workflow/execute-workflow.service"; import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; -import { catchError, debounceTime, mergeMap } from "rxjs/operators"; +import { catchError, debounceTime, map, switchMap, filter } from "rxjs/operators"; import { DynamicSchemaService } from "../dynamic-schema/dynamic-schema.service"; import { AttributeType, @@ -43,6 +43,14 @@ import { WorkflowGraphReadonly } from "../workflow-graph/model/workflow-graph"; import { serializePortIdentity } from "../../../common/util/port-identity-serde"; import { addCompilationError, areAllPortSchemasEqual } from "../../../common/util/workflow-compilation-utils"; import { parseLogicalOperatorPortID } from "../../../common/util/logical-operator-port-serde"; +import { isMacro, compiledInputLinks, macroSchema } from "../../../common/util/workflow-macro"; +import { portOrdinal } from "@texera/workflow-macros"; + +import { MacroDraftRequest, MacroNavigationService } from "../macro/macro-navigation.service"; +import { LogicalPort, OperatorLink, OperatorPredicate } from "../../types/workflow-common.interface"; +import { MacroExpansionService } from "../macro/macro-expansion.service"; +import { WorkflowContent } from "../../../common/type/workflow"; +import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; // endpoint for workflow compile export const WORKFLOW_COMPILATION_ENDPOINT = "compile"; @@ -68,13 +76,22 @@ export class WorkflowCompilingService { private currentCompilationStateInfo: CompilationStateInfo = { state: CompilationState.Uninitialized, }; + private macroInputs?: { + doc: object; + inputLinks: (id: string) => OperatorLink[]; + schemas: Record; + }; private compilationStateInfoChangedStream = new ReplaySubject(1); + private macroSchemas = new Map(); constructor( private httpClient: HttpClient, private workflowActionService: WorkflowActionService, private dynamicSchemaService: DynamicSchemaService, - private validationWorkflowService: ValidationWorkflowService + private validationWorkflowService: ValidationWorkflowService, + private macros: MacroNavigationService, + private expansion: MacroExpansionService, + private metadata: OperatorMetadataService ) { // Subscribe to compilation state changes to apply schema propagation this.compilationStateInfoChangedStream.subscribe(() => { @@ -90,17 +107,84 @@ export class WorkflowCompilingService { this.workflowActionService.getTexeraGraph().getOperatorAddStream(), this.workflowActionService.getTexeraGraph().getOperatorDeleteStream(), this.workflowActionService.getTexeraGraph().getOperatorPropertyChangeStream(), - this.workflowActionService.getTexeraGraph().getDisabledOperatorsChangedStream() + this.workflowActionService.getTexeraGraph().getDisabledOperatorsChangedStream(), + this.workflowActionService.macroChanged$ ?? EMPTY, + this.macros.changed$ ) .pipe(debounceTime(WORKFLOW_COMPILATION_DEBOUNCE_TIME_MS)) .pipe( - mergeMap(() => { - const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest( - this.validationWorkflowService.getValidTexeraGraph(), - undefined - ); - return this.compile(logicalPlan); - }) + switchMap(() => + defer(() => { + const content = this.workflowActionService.getWorkflowContent(); + const doc = this.workflowActionService.editorDocument; + const draft = this.macros.compilationDraft(); + const overrides = this.macros.parentOverrides(); + const logicalPlan = + draft ?? + (content.macroInterface + ? { operators: [], links: [], macroContent: content, opsToViewResult: [], opsToReuseResult: [] } + : ExecuteWorkflowService.getLogicalPlanRequest( + this.validationWorkflowService.getValidTexeraGraph(), + undefined, + this.workflowActionService.getMacroDependencies?.() ?? [] + )); + return this.compile(logicalPlan).pipe( + filter(() => this.workflowActionService.editorDocument === doc), + map(response => { + if (response.macroExpansion && !draft) this.expansion.remember(content, response.macroExpansion); + const context = + (draft || content.macroInterface) && response.macroExpansion + ? this.macros.compilationContext(response.macroExpansion, content, overrides) + : undefined; + this.macroInputs = context + ? { doc, inputLinks: context.inputLinks, schemas: response.operatorOutputSchemas } + : undefined; + const remap = (values: Record): Record => + Object.fromEntries( + Object.entries(context?.ids ?? {}).flatMap(([local, runtime]) => + values[runtime] === undefined ? [] : [[local, values[runtime]]] + ) + ); + const projected = context + ? { + ...response, + operatorOutputSchemas: remap(response.operatorOutputSchemas), + operatorErrors: Object.fromEntries( + Object.entries(remap(response.operatorErrors ?? {})).map(([id, error]) => [ + id, + { ...error, operatorId: id }, + ]) + ), + } + : response; + this.macroSchemas = this.compiledMacroSchemas(content, response, context?.path); + return projected; + }) + ); + }).pipe( + catchError((error: unknown) => { + this.macroSchemas.clear(); + const graph = this.workflowActionService.getTexeraGraph(); + const operatorID = + graph.getAllOperators().find(isMacro)?.operatorID ?? + graph.getAllOperators()[0]?.operatorID ?? + "workflow"; + return of({ + operatorOutputSchemas: {}, + operatorErrors: { + [operatorID]: { + message: error instanceof Error ? error.message : String(error), + details: "", + operatorId: operatorID, + workerId: "", + type: { name: "MacroValidation" }, + timestamp: { nanos: 0, seconds: 0 }, + }, + }, + } as WorkflowCompilationResponse); + }) + ) + ) ) .subscribe(response => { if (response.physicalPlan) { @@ -142,6 +226,10 @@ export class WorkflowCompilingService { return undefined; } + if (this.macroInputs && this.macroInputs.doc === this.workflowActionService.editorDocument) + return this.extractOperatorInputPortSchemaMap(operatorID, this.macroInputs.schemas, { + getInputLinksByOperatorId: this.macroInputs.inputLinks, + }); return this.extractOperatorInputPortSchemaMap( operatorID, this.currentCompilationStateInfo.operatorOutputPortSchemaMap, @@ -189,8 +277,10 @@ export class WorkflowCompilingService { const inputSchema = this.getOperatorInputSchemaMap(operatorID); let newDynamicSchema: OperatorSchema; - if (inputSchema) { - newDynamicSchema = WorkflowCompilingService.setOperatorInputAttrs(currentDynamicSchema, inputSchema); + if (inputSchema || this.macroSchemas.has(operatorID)) { + newDynamicSchema = + this.macroSchemas.get(operatorID) ?? + WorkflowCompilingService.setOperatorInputAttrs(currentDynamicSchema, inputSchema); // Now that the list of input attributes is known, drop any operator property // values that reference attributes which no longer exist in the input schema (e.g. a copy-pasted @@ -220,6 +310,34 @@ export class WorkflowCompilingService { }); } + /** Resolve exposed properties against their actual compiled inputs, including internal transformations. */ + private compiledMacroSchemas( + content: WorkflowContent, + response: WorkflowCompilationResponse, + prefix: string[] = [] + ): Map { + const expanded = response.macroExpansion; + if (!expanded) return new Map(); + const getSchema = (type: string) => this.metadata.getOperatorSchema(type); + const schemaFor = (op: OperatorPredicate, path: string[] = prefix): OperatorSchema => { + if (isMacro(op)) + return macroSchema(this.metadata.getMacroDefinition(op.operatorType)!, getSchema, child => + schemaFor(child, [...path, op.operatorID]) + ); + const schema = getSchema(op.operatorType); + return WorkflowCompilingService.setOperatorInputAttrs( + schema, + this.extractOperatorInputPortSchemaMap( + op.operatorID, + response.operatorOutputSchemas, + { getInputLinksByOperatorId: () => compiledInputLinks(expanded, path, op) }, + schema + ) + ); + }; + return new Map(content.operators.filter(isMacro).map(op => [op.operatorID, schemaFor(op)])); + } + /** * Extracts input schema per port for an operator by looking at the output schemas of operators that are connecting to it. * @@ -231,15 +349,23 @@ export class WorkflowCompilingService { private extractOperatorInputPortSchemaMap( operatorID: string, outputSchemas: Record, - workflowGraph: WorkflowGraphReadonly + workflowGraph: Pick, + operatorSchema?: OperatorSchema ): OperatorPortSchemaMap | undefined { const inputLinks = workflowGraph.getInputLinksByOperatorId(operatorID); if (!inputLinks.length) return undefined; // Get the operator's dynamic schema to know what input ports it has - const dynamicSchema = this.dynamicSchemaService.getDynamicSchema(operatorID); + const dynamicSchema = operatorSchema ?? this.dynamicSchemaService.getDynamicSchema(operatorID); if (!dynamicSchema) return undefined; + const graph = this.workflowActionService.getTexeraGraph(); + const ordinal = (end: LogicalPort & { portIndex?: number }, direction: "inputPorts" | "outputPorts") => { + if (end.portIndex !== undefined) return end.portIndex; + const op = graph.hasOperator(end.operatorID) ? graph.getOperator(end.operatorID) : undefined; + return op ? portOrdinal(op, direction, end.portID) : parseLogicalOperatorPortID(end.portID)?.portNumber; + }; + const inputPortSchemaMap = new Map(); dynamicSchema.additionalMetadata.inputPorts.forEach((inputPort, portIndex) => { @@ -247,11 +373,7 @@ export class WorkflowCompilingService { inputPortSchemaMap.set(portId, undefined); // Find all links that connect to this input port - const linksToThisPort = inputLinks.filter(link => { - const inputPort = parseLogicalOperatorPortID(link.target.portID); - if (!inputPort) return false; - return inputPort.portNumber === portIndex; - }); + const linksToThisPort = inputLinks.filter(link => ordinal(link.target, "inputPorts") === portIndex); if (linksToThisPort.length > 0) { // Check if multiple links have different schemas @@ -261,12 +383,12 @@ export class WorkflowCompilingService { return undefined; } - const outputPort = parseLogicalOperatorPortID(link.source.portID); - if (!outputPort) { + const outputPort = ordinal(link.source, "outputPorts"); + if (outputPort === undefined) { return undefined; } - return sourcePortSchemaMap[serializePortIdentity({ id: outputPort.portNumber, internal: false })]; + return sourcePortSchemaMap[serializePortIdentity({ id: outputPort, internal: false })]; }); // Check if all schemas are the same using utility function @@ -299,32 +421,19 @@ export class WorkflowCompilingService { * that users can easily set the properties of the next operator. For eg: If there are two operators Source:Scan and KeywordSearch and * a link is created between them, the attributed of the table selected in Source can be propagated to the KeywordSearch operator. */ - private compile(logicalPlan: LogicalPlan): Observable { - // create a Logical Plan based on the workflow graph - // remove unnecessary information for schema propagation. - const body = { - operators: logicalPlan.operators, - links: logicalPlan.links, - opsToReuseResult: [], - opsToViewResult: [], - }; - // make a http post request to the API endpoint with the logical plan object - return this.httpClient - .post( - `${AppSettings.getApiEndpoint()}/${WORKFLOW_COMPILATION_ENDPOINT}`, - JSON.stringify(body), - { - headers: new HttpHeaders({ - "Content-Type": "application/json", - }), - } - ) - .pipe( - catchError((err: unknown) => { - console.warn("compile workflow API returns error", err); - return EMPTY; - }) - ); + private compile(plan: LogicalPlan | MacroDraftRequest): Observable { + const url = `${AppSettings.getApiEndpoint()}/${WORKFLOW_COMPILATION_ENDPOINT}`; + const draft = "drafts" in plan; + const body = draft ? plan : { ...plan, opsToReuseResult: [], opsToViewResult: [] }; + const request = this.httpClient.post(draft ? `${url}/draft` : url, body, { + headers: { "Content-Type": "application/json" }, + }); + return request.pipe( + catchError((error: unknown) => { + console.warn("compile workflow API returns error", error); + return EMPTY; + }) + ); } public static setOperatorInputAttrs( diff --git a/frontend/src/app/workspace/service/drag-drop/drag-drop.service.spec.ts b/frontend/src/app/workspace/service/drag-drop/drag-drop.service.spec.ts index 7d186914064..9539de6da27 100644 --- a/frontend/src/app/workspace/service/drag-drop/drag-drop.service.spec.ts +++ b/frontend/src/app/workspace/service/drag-drop/drag-drop.service.spec.ts @@ -16,7 +16,11 @@ * specific language governing permissions and limitations * under the License. */ +import { testMacro } from "../../../common/testing/macro-graph"; +import { MacroService } from "../macro/macro.service"; +import { MacroDefinition, macroInstance, macroType } from "../../../common/util/workflow-macro"; +import { NotificationService } from "../../../common/service/notification/notification.service"; import { JointUIService } from "../joint-ui/joint-ui.service"; import { inject, TestBed } from "@angular/core/testing"; import { DragDropService } from "./drag-drop.service"; @@ -66,6 +70,77 @@ describe("DragDropService", () => { ); }); + describe("asynchronous macro drops", () => { + let actions: WorkflowActionService, + resolve: (d: MacroDefinition) => void, + latest: ReturnType, + definition: MacroDefinition, + notification: ReturnType, + complete: ReturnType; + beforeEach(() => { + actions = TestBed.inject(WorkflowActionService); + actions.suspendWorkflow().shared.wsProvider.disconnect(); + definition = testMacro("macro", actions.getWorkflowContent().operators, { + ...actions.getWorkflowContent(), + name: "Macro1", + version: 2, + }); + (dragDropService as any).op = macroInstance({ ...definition, version: 1 }, "pending-drop"); + latest = vi.spyOn(TestBed.inject(MacroService), "latestFor").mockImplementation( + () => + new Promise(r => { + resolve = r; + }) + ); + notification = vi.spyOn(TestBed.inject(NotificationService), "error").mockImplementation(() => {}); + complete = vi.spyOn(dragDropService as any, "completeDrop").mockImplementation(() => {}); + }); + it("accepts autosave metadata changes while preserving the requested drop and latest pin", async () => { + dragDropService.dragDropped({ x: 100, y: 200 }); + actions.setWorkflowMetadata({ ...actions.getWorkflowMetadata(), lastModifiedTime: 123 }); + resolve(definition); + await vi.waitFor(() => expect(complete).toHaveBeenCalledWith({ x: 100, y: 200 })); + expect((dragDropService as any).op.operatorType).toBe(macroType(definition)); + expect(notification).not.toHaveBeenCalled(); + }); + it("ignores a drop on a read-only canvas without fetching a macro or showing a false change error", async () => { + actions.disableWorkflowModification(); + dragDropService.dragDropped({ x: 100, y: 200 }); + await Promise.resolve(); + expect(latest).not.toHaveBeenCalled(); + expect(complete).not.toHaveBeenCalled(); + expect(notification).not.toHaveBeenCalled(); + }); + it("cancels a pending drop when editing is disabled", async () => { + dragDropService.dragDropped({ x: 100, y: 200 }); + actions.disableWorkflowModification(); + resolve(definition); + await Promise.resolve(); + await Promise.resolve(); + expect(complete).not.toHaveBeenCalled(); + expect(notification).not.toHaveBeenCalled(); + }); + it("rejects real graph changes while a macro loads", async () => { + dragDropService.dragDropped({ x: 100, y: 200 }); + actions.addOperator(mockScanPredicate, { x: 10, y: 20 }); + resolve(definition); + await vi.waitFor(() => + expect(notification).toHaveBeenCalledWith("The workflow changed while loading the macro. Drag it again.") + ); + expect(complete).not.toHaveBeenCalled(); + expect(actions.getTexeraGraph().hasOperator(mockScanPredicate.operatorID)).toBe(true); + }); + it("does not insert into another editor document with identical workflow content", async () => { + const workflow = actions.getWorkflow(); + dragDropService.dragDropped({ x: 100, y: 200 }); + actions.openWorkflowDraft(workflow, false); + resolve(definition); + await Promise.resolve(); + await Promise.resolve(); + expect(complete).not.toHaveBeenCalled(); + }); + }); + it("should be created", inject([DragDropService], (injectedService: DragDropService) => { expect(injectedService).toBeTruthy(); })); diff --git a/frontend/src/app/workspace/service/drag-drop/drag-drop.service.ts b/frontend/src/app/workspace/service/drag-drop/drag-drop.service.ts index 358f60664eb..99cead521ed 100644 --- a/frontend/src/app/workspace/service/drag-drop/drag-drop.service.ts +++ b/frontend/src/app/workspace/service/drag-drop/drag-drop.service.ts @@ -24,9 +24,14 @@ import { WorkflowUtilService } from "../workflow-graph/util/workflow-util.servic import { JointUIService } from "../joint-ui/joint-ui.service"; import { Injectable } from "@angular/core"; import { filter, first, map } from "rxjs/operators"; +import { isEqual } from "lodash-es"; import TinyQueue from "tinyqueue"; import * as joint from "jointjs"; +import { MacroService } from "../macro/macro.service"; +import { isMacro, macroInstance } from "../../../common/util/workflow-macro"; +import { NotificationService } from "../../../common/service/notification/notification.service"; + @Injectable({ providedIn: "root", }) @@ -42,7 +47,9 @@ export class DragDropService { constructor( private jointUIService: JointUIService, private workflowUtilService: WorkflowUtilService, - private workflowActionService: WorkflowActionService + private workflowActionService: WorkflowActionService, + private macros: MacroService, + private notification: NotificationService ) {} public dragStarted(operatorType: string): void { @@ -58,6 +65,41 @@ export class DragDropService { } public dragDropped(dropPoint: Point): void { + if (!this.op || !this.workflowActionService.checkWorkflowModificationEnabled()) { + this.resetSuggestions(); + return; + } + if (!isMacro(this.op)) { + this.completeDrop(dropPoint); + return; + } + const op = this.op; + const before = this.workflowActionService.getWorkflow(); + const document = this.workflowActionService.editorDocument; + this.macros + .latestFor(op.operatorType) + .then(definition => { + if (this.op !== op) return; + if ( + !this.workflowActionService.checkWorkflowModificationEnabled() || + this.workflowActionService.editorDocument !== document || + this.workflowActionService.getWorkflowMetadata().wid !== before.wid + ) { + this.resetSuggestions(); + return; + } + if (!isEqual(this.workflowActionService.getWorkflowContent(), before.content)) + throw new Error("The workflow changed while loading the macro. Drag it again."); + this.op = macroInstance(definition, op.operatorID); + this.completeDrop(dropPoint); + }) + .catch(error => { + this.resetSuggestions(); + this.notification.error(error instanceof Error ? error.message : String(error)); + }); + } + + private completeDrop(dropPoint: Point): void { const coordinates = this.workflowActionService .getJointGraphWrapper() .getMainJointPaper() diff --git a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts index 8f71b167415..1c028600294 100644 --- a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts +++ b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts @@ -16,7 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +import { testPort, testMacro, expansionFixture, setMacroGraph } from "../../../common/testing/macro-graph"; +import { testBinding, testLink } from "../../../common/testing/workflow-graph"; +import { MacroExpansionService } from "../macro/macro-expansion.service"; import "zone.js/testing"; import { DOCUMENT } from "@angular/core"; @@ -30,7 +33,7 @@ import { UndoRedoService } from "../undo-redo/undo-redo.service"; import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; import { StubOperatorMetadataService } from "../operator-metadata/stub-operator-metadata.service"; import { JointUIService } from "../joint-ui/joint-ui.service"; -import { BehaviorSubject, of, Subject } from "rxjs"; +import { BehaviorSubject, of, ObservedValueOf, Subject } from "rxjs"; import { WorkflowWebsocketService } from "../workflow-websocket/workflow-websocket.service"; import { WorkflowStatusService } from "../workflow-status/workflow-status.service"; import { NotificationService } from "../../../common/service/notification/notification.service"; @@ -38,7 +41,6 @@ import { GuiConfigService } from "../../../common/service/gui-config.service"; import { mockLogicalPlan_scan_result, mockWorkflowPlan_scan_result } from "./mock-workflow-plan"; import { HttpClientTestingModule } from "@angular/common/http/testing"; -import { WorkflowUtilService } from "../workflow-graph/util/workflow-util.service"; import { WorkflowSettings } from "src/app/common/type/workflow"; import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; @@ -49,14 +51,17 @@ import { StubAuthService } from "src/app/common/service/user/stub-auth.service"; import { UserService } from "src/app/common/service/user/user.service"; import { StubUserService } from "src/app/common/service/user/stub-user.service"; import { MockComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; -import { commonTestProviders } from "../../../common/testing/test-utils"; import { RegionStateEvent, RegionUpdateEvent, ReplayExecutionInfo, TexeraWebsocketEvent, } from "../../types/workflow-websocket.interface"; -import { mockScanPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { mockScanPredicate, mockSentimentPredicate } from "../workflow-graph/model/mock-workflow-data"; + +import { WorkflowGraph } from "../workflow-graph/model/workflow-graph"; +import { macroWorkspaceTestProviders } from "../macro/macro-workspace.spec-harness"; +import { macroInstance } from "../../../common/util/workflow-macro"; import { PAGINATION_INFO_STORAGE_KEY, ResultPaginationInfo } from "../../types/result-table.interface"; import { sessionGetObject, sessionSetObject } from "../../../common/util/storage"; @@ -67,29 +72,14 @@ describe("ExecuteWorkflowService", () => { let connected$: BehaviorSubject; beforeEach(() => { - mockDocument = { - location: { - origin: "https://texera.example.com", - }, - } as Document; - TestBed.configureTestingModule({ imports: [HttpClientTestingModule], providers: [ - ExecuteWorkflowService, - WorkflowActionService, - WorkflowUtilService, + ...macroWorkspaceTestProviders, { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, - UndoRedoService, - JointUIService, - { - provide: OperatorMetadataService, - useClass: StubOperatorMetadataService, - }, - { provide: DOCUMENT, useValue: mockDocument }, + { provide: DOCUMENT, useValue: { location: { origin: "https://texera.example.com" } } }, { provide: AuthService, useClass: StubAuthService }, { provide: UserService, useClass: StubUserService }, - ...commonTestProviders, ], }); @@ -118,11 +108,182 @@ describe("ExecuteWorkflowService", () => { const emitWsEvent = (event: TexeraWebsocketEvent): void => { (TestBed.inject(WorkflowWebsocketService).websocketEvent() as Subject).next(event); }; + const emitRegions = (regions: RegionUpdateEvent["regions"]) => emitWsEvent({ type: "RegionUpdateEvent", regions }); + const observeRegions = () => { + const updates: ObservedValueOf>[] = []; + service.getRegionUpdateStream().subscribe(event => updates.push(event)); + return updates; + }; + const observeRegionStates = () => { + const states: RegionStateEvent[] = []; + service.getRegionStateStream().subscribe(event => states.push(event)); + return states; + }; + const watchRun = () => ({ + send: vi.spyOn(TestBed.inject(WorkflowWebsocketService), "send"), + reset: vi.spyOn(service, "resetExecutionState"), + resetStatus: vi.spyOn(TestBed.inject(WorkflowStatusService), "resetStatus"), + error: vi.spyOn(TestBed.inject(NotificationService), "error").mockImplementation(() => {}), + }); it("should be created", inject([ExecuteWorkflowService], (injectedService: ExecuteWorkflowService) => { expect(injectedService).toBeTruthy(); })); + it("sends pinned snapshots and canvas result IDs for backend expansion", () => { + const d = testMacro("analysis", [mockSentimentPredicate], { + inputs: [testPort("input-0", mockSentimentPredicate.operatorID)], + outputs: [testPort("output-0", mockSentimentPredicate.operatorID)], + version: 8, + name: "Analysis", + }); + const macro = { ...macroInstance(d, "m"), viewResult: true, markedForReuse: true }; + const graph = new WorkflowGraph( + [mockScanPredicate, macro], + [testLink(mockScanPredicate.operatorID, "m", "output-0", "input-0", "input")] + ); + try { + const plan = ExecuteWorkflowService.getLogicalPlanRequest(graph, "m", [d]); + expect(plan.operators).toEqual([]); + expect(plan.links).toEqual([]); + expect(plan.macroContent).toEqual({ + operators: graph.getAllOperators(), + links: graph.getAllLinks(), + macroDependencies: [d], + }); + expect(plan.executeOperatorIds).toEqual(expect.arrayContaining(["m", mockScanPredicate.operatorID])); + expect(plan.opsToViewResult).toEqual(["m"]); + expect(plan.opsToReuseResult).toEqual(["m"]); + expect(ExecuteWorkflowService.getLogicalPlanRequest(graph).macroContent?.macroDependencies).toEqual([]); + // Structural validation belongs to the backend, including for whole-workflow requests. + const cycle = testLink("m", "m", "output-0", "input-0", "cycle"); + graph.addLink(cycle); + expect(ExecuteWorkflowService.getLogicalPlanRequest(graph, undefined, [d]).macroContent!.links).toContainEqual( + cycle + ); + } finally { + graph.sharedModel.destroy(); + } + }); + + it("preserves ordinary multi-port plans, authoritative operator IDs, and empty graphs", () => { + const source = { + ...mockScanPredicate, + operatorID: "café", + operatorProperties: { operatorID: "user value", value: "東京" }, + outputPorts: [{ portID: "left" }, { portID: "right" }], + }; + const target = { + ...mockSentimentPredicate, + operatorID: "target", + inputPorts: [{ portID: "first" }, { portID: "second" }], + }; + const graph = new WorkflowGraph([source, target], [testLink("café", "target", "right", "second", "connection")]); + const empty = new WorkflowGraph(); + try { + const plan = ExecuteWorkflowService.getLogicalPlanRequest(graph, "target"); + expect(plan.operators.find(op => op.operatorID === "café")).toMatchObject({ + value: "東京", + outputPorts: source.outputPorts, + }); + expect(plan.links).toEqual([ + { + fromOpId: "café", + fromPortId: { id: 1, internal: false }, + toOpId: "target", + toPortId: { id: 1, internal: false }, + }, + ]); + expect(graph.getOperator("café")).toEqual(source); + expect(ExecuteWorkflowService.getLogicalPlanRequest(empty)).toEqual({ + operators: [], + links: [], + opsToViewResult: [], + opsToReuseResult: [], + }); + } finally { + graph.sharedModel.destroy(); + empty.sharedModel.destroy(); + } + }); + + it.each([false, true])( + "refuses macro definition execution with external inputs %s, preserving previous results", + external => { + const actions = TestBed.inject(WorkflowActionService); + actions.addOperator(mockScanPredicate, { x: 0, y: 0 }); + setMacroGraph(actions, { + id: "macro", + inputs: external ? [{ portID: "rows", targets: [] }] : [], + outputs: [testPort("result", mockScanPredicate.operatorID, "output-0")], + }); + const before = actions.getWorkflowContent(); + const run = watchRun(); + service.executeWorkflow("macro"); + service.executeWorkflow("macro", mockScanPredicate.operatorID); + service.executeWorkflowWithReplay({ eid: 42, interaction: "step-3" }); + expect(run.error).toHaveBeenCalledTimes(3); + expect(run.error).toHaveBeenCalledWith("A macro runs through a workflow. Add it to a workflow to execute it."); + expect(run.send).not.toHaveBeenCalled(); + expect(run.reset).not.toHaveBeenCalled(); + expect(run.resetStatus).not.toHaveBeenCalled(); + expect(actions.getWorkflowContent()).toEqual(before); + } + ); + + it("reports a missing macro bundle without throwing or resetting the execution", async () => { + const actions = TestBed.inject(WorkflowActionService); + const d = testMacro("missing", [mockScanPredicate], { + outputs: [testPort("output-0", mockScanPredicate.operatorID)], + name: "Missing", + }); + actions.setMacroDependencies([d]); + actions.addOperator(macroInstance(d, "macro"), { x: 0, y: 0 }); + actions.setMacroDependencies([]); + const { error, reset } = watchRun(); + vi.spyOn(TestBed.inject(MacroExpansionService), "expand").mockRejectedValue(new Error("Missing macro version")); + await service.executeWorkflow("Broken"); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Missing macro version")); + expect(reset).not.toHaveBeenCalled(); + }); + + it("reconfigures a paused macro using backend leaves and rejects missing versions before sending", async () => { + const actions = TestBed.inject(WorkflowActionService); + const definition = testMacro("live", [mockSentimentPredicate], { + formBinding: testBinding(mockSentimentPredicate.operatorID, "attribute", "attribute", "Attribute"), + inputs: [testPort("input-0", mockSentimentPredicate.operatorID)], + outputs: [testPort("output-0", mockSentimentPredicate.operatorID)], + name: "Live", + }); + actions.setMacroDependencies([definition]); + actions.addOperator( + { ...macroInstance(definition, "m"), operatorProperties: { attribute: "new column" } }, + { x: 0, y: 0 } + ); + (service as any).currentState = { state: ExecutionState.Paused }; + const send = vi.spyOn(TestBed.inject(WorkflowWebsocketService), "send"); + const compile = vi + .spyOn(TestBed.inject(MacroExpansionService), "expand") + .mockResolvedValue( + expansionFixture(actions.getWorkflowContent(), [ + [ + ["m", mockSentimentPredicate.operatorID], + { ...mockSentimentPredicate, operatorProperties: { attribute: "new column" } }, + ], + ]) + ); + await service.modifyOperatorLogic("m"); + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith("ModifyLogicRequest", { + operator: expect.objectContaining({ operatorType: "NlpSentiment", attribute: "new column" }), + }); + send.mockClear(); + actions.setMacroDependencies([]); + compile.mockRejectedValue(new Error("Missing macro version")); + await expect(service.modifyOperatorLogic("m")).rejects.toThrow(/macro version/i); + expect(send).not.toHaveBeenCalled(); + }); + // A view handed a session mid-run subscribes after the last state change, and the stream carries // no current value, so it heard nothing about the run: it showed Run for a workflow that was // running. The lock is reapplied only when the state changes too, so a canvas that unlocked the @@ -468,7 +629,11 @@ describe("ExecuteWorkflowService", () => { service.executeWorkflowWithEmailNotification(executionName, emailNotificationEnabled, targetOperatorId); - expect(logicalPlanSpy).toHaveBeenCalledWith(service["workflowActionService"].getTexeraGraph(), targetOperatorId); + expect(logicalPlanSpy).toHaveBeenCalledWith( + service["workflowActionService"].getTexeraGraph(), + targetOperatorId, + [] + ); expect(settingsSpy).toHaveBeenCalled(); expect(resetExecutionStateSpy).toHaveBeenCalled(); expect(resetStatusSpy).toHaveBeenCalled(); @@ -476,7 +641,8 @@ describe("ExecuteWorkflowService", () => { executionName, expect.any(Object), expect.any(Object), - emailNotificationEnabled + emailNotificationEnabled, + undefined ); }); @@ -496,7 +662,11 @@ describe("ExecuteWorkflowService", () => { service.executeWorkflowWithEmailNotification(executionName, emailNotificationEnabled, targetOperatorId); }).toThrowError("Logical plan error"); - expect(logicalPlanSpy).toHaveBeenCalledWith(service["workflowActionService"].getTexeraGraph(), targetOperatorId); + expect(logicalPlanSpy).toHaveBeenCalledWith( + service["workflowActionService"].getTexeraGraph(), + targetOperatorId, + [] + ); expect(resetExecutionStateSpy).not.toHaveBeenCalled(); expect(resetStatusSpy).not.toHaveBeenCalled(); expect(sendExecutionRequestSpy).not.toHaveBeenCalled(); @@ -505,12 +675,10 @@ describe("ExecuteWorkflowService", () => { // ---- constructor: websocket-event routing -------------------------------------------------- it("routes region and worker-assignment websocket events to their dedicated streams", () => { - const regionUpdates: RegionUpdateEvent[] = []; - const regionStates: RegionStateEvent[] = []; - service.getRegionUpdateStream().subscribe(event => regionUpdates.push(event)); - service.getRegionStateStream().subscribe(event => regionStates.push(event)); + const regionUpdates = observeRegions(); + const regionStates = observeRegionStates(); - emitWsEvent({ type: "RegionUpdateEvent", regions: [] }); + emitRegions([]); emitWsEvent({ type: "RegionStateEvent", id: 7, state: "COMPLETED" }); emitWsEvent({ type: "WorkerAssignmentUpdateEvent", operatorId: "opX", workerIds: ["w1", "w2"] }); @@ -519,6 +687,118 @@ describe("ExecuteWorkflowService", () => { expect(service.getWorkerIds("opX")).toEqual(["w1", "w2"]); }); + it("reprojects the last regions through nested runtime IDs when the inspector changes", () => { + const actions = TestBed.inject(WorkflowActionService); + const rootPaths = { leafA: ["macro", "nested", "a"], leafB: ["macro", "nested", "b"] }; + let paths: Record = rootPaths; + let ids: Record | undefined; + actions.runtimeOperatorIDs = () => ids; + vi.spyOn(actions, "getMacroExpansion").mockImplementation(() => ({ + content: actions.getWorkflowContent(), + paths, + ports: {}, + })); + const updates = observeRegions(); + emitRegions([ + [1, ["source", "leafA", "leafB"]], + [2, []], + ]); + const expectMembers = (members: string[]) => + expect(updates.at(-1)?.regions).toEqual([ + [1, members], + [2, []], + ]); + const inspect = (mapping: typeof ids, projection: typeof paths) => { + ids = mapping; + paths = projection; + TestBed.inject(WorkflowStatusService).refresh(); + }; + expectMembers(["source", "macro"]); + inspect(undefined, rootPaths); + expect(updates).toHaveLength(1); + inspect({ localA: "leafA", localB: "leafB" }, { localA: ["nested", "a"], localB: ["nested", "b"] }); + expectMembers(["nested"]); + inspect({ a: "leafA", b: "leafB" }, {}); + expectMembers(["a", "b"]); + // An unverified revision must not inherit an old run's regions. + inspect({}, {}); + expectMembers([]); + inspect(undefined, rootPaths); + expectMembers(["source", "macro"]); + }); + + it("redraws equal region projections when nested documents reuse local operator IDs", () => { + const actions = TestBed.inject(WorkflowActionService); + const status = TestBed.inject(WorkflowStatusService); + const updates = observeRegions(); + emitRegions([[1, ["child"]]]); + status.refresh(); + expect(updates).toHaveLength(1); + + actions.openWorkflowDraft(actions.getWorkflow()); + status.refresh(); + expect(updates).toHaveLength(2); + expect(updates[1]).toEqual(updates[0]); + status.refresh(); + expect(updates).toHaveLength(2); + }); + + it("replays the latest topology and each region's latest phase to a remounted canvas", () => { + emitRegions([ + [1, ["a"]], + [2, ["b"]], + ]); + emitWsEvent({ type: "RegionStateEvent", id: 1, state: "Running" }); + emitWsEvent({ type: "RegionStateEvent", id: 1, state: "Completed" }); + emitWsEvent({ type: "RegionStateEvent", id: 2, state: "ExecutingDependeePortsPhase" }); + expect(observeRegions()).toHaveLength(1); + expect(observeRegionStates()).toEqual([ + { type: "RegionStateEvent", id: 1, state: "Completed" }, + { type: "RegionStateEvent", id: 2, state: "ExecutingDependeePortsPhase" }, + ]); + }); + + it("continues a terminal only when both real endpoints of its boundary link share the region", () => { + const actions = TestBed.inject(WorkflowActionService); + actions.runtimeBoundaryLinks = () => [ + { + terminal: { direction: "inputs", portID: "rows" }, + links: [testLink("outside", "inside", "output-0", "input-0", "in")], + }, + { + terminal: { direction: "outputs", portID: "result" }, + links: [testLink("inside", "next", "output-0", "input-0", "out")], + }, + { terminal: { direction: "inputs", portID: "unconnected" }, links: [] }, + ]; + const updates = observeRegions(); + emitRegions([ + [1, ["outside", "inside"]], + [2, ["inside", "next"]], + [3, ["inside"]], + [4, ["outside"]], + [5, []], + ]); + expect(updates.at(-1)?.terminals).toEqual({ + 1: [{ direction: "inputs", portID: "rows" }], + 2: [{ direction: "outputs", portID: "result" }], + 3: [], + 4: [], + 5: [], + }); + }); + + it.each(["resetExecutionState", "resetExecutionAndWorkers"] as const)( + "clears region topology and phases on %s", + reset => { + emitRegions([[1, ["old-operator"]]]); + emitWsEvent({ type: "RegionStateEvent", id: 1, state: "Completed" }); + service[reset](); + expect(observeRegions().at(-1)?.regions).toEqual([]); + expect(observeRegionStates()).toEqual([]); + } + ); + it("routes execution-status websocket events into the execution-state stream", () => { const states: ExecutionState[] = []; service.getExecutionStateStream().subscribe(event => states.push(event.current.state)); @@ -665,7 +945,7 @@ describe("ExecuteWorkflowService", () => { service.executeWorkflowWithReplay(replayInfo); - expect(logicalPlanSpy).toHaveBeenCalledWith(service["workflowActionService"].getTexeraGraph()); + expect(logicalPlanSpy).toHaveBeenCalledWith(service["workflowActionService"].getTexeraGraph(), undefined, []); expect(resetExecutionStateSpy).toHaveBeenCalled(); expect(resetStatusSpy).toHaveBeenCalled(); expect(sendExecutionRequestSpy).toHaveBeenCalledWith( @@ -696,38 +976,20 @@ describe("ExecuteWorkflowService", () => { ); })); - it("a refused run leaves the previous execution's state untouched (#7817)", () => { + it("preserves the previous run when a required warehouse is missing (#7817)", fakeAsync(() => { TestBed.inject(GuiConfigService).env.warehouseEnabled = true; try { TestBed.inject(WarehouseService).selectWarehouse(undefined); - const resetSpy = vi.spyOn(service, "resetExecutionState"); - const statusResetSpy = vi.spyOn(TestBed.inject(WorkflowStatusService), "resetStatus"); - vi.spyOn(TestBed.inject(NotificationService), "error").mockReturnValue(undefined as never); - - service.executeWorkflowWithEmailNotification("exec", false); - - expect(resetSpy).not.toHaveBeenCalled(); - expect(statusResetSpy).not.toHaveBeenCalled(); - } finally { - TestBed.inject(GuiConfigService).env.warehouseEnabled = false; - } - }); - - it("refuses to run without a warehouse while the deployment requires one (#7817)", fakeAsync(() => { - // Paths that bypass the menu gate (form view, run-up-to, replay) all funnel - // through sendExecutionRequest; the shared storage must not catch them. - TestBed.inject(GuiConfigService).env.warehouseEnabled = true; - try { - TestBed.inject(WarehouseService).selectWarehouse(undefined); - const wsSendSpy = vi.spyOn(service["workflowWebsocketService"], "send"); - const errorSpy = vi.spyOn(TestBed.inject(NotificationService), "error").mockReturnValue(undefined as never); + const run = watchRun(); service.executeWorkflowWithEmailNotification("exec", false); tick(FORM_DEBOUNCE_TIME_MS + 1); flush(); - expect(wsSendSpy).not.toHaveBeenCalledWith("WorkflowExecuteRequest", expect.anything()); - expect(errorSpy).toHaveBeenCalledWith("Create or select a warehouse before running."); + expect(run.send).not.toHaveBeenCalledWith("WorkflowExecuteRequest", expect.anything()); + expect(run.reset).not.toHaveBeenCalled(); + expect(run.resetStatus).not.toHaveBeenCalled(); + expect(run.error).toHaveBeenCalledWith("Create or select a warehouse before running."); } finally { TestBed.inject(GuiConfigService).env.warehouseEnabled = false; } @@ -746,6 +1008,7 @@ describe("ExecuteWorkflowService", () => { }; const replayInfo: ReplayExecutionInfo = { eid: 42, interaction: "step-3" }; + const unavailableMessage = "The selected computing unit is unavailable. Select a running unit or create a new one."; const entryPoints: { name: string; run: () => void }[] = [ { @@ -756,44 +1019,29 @@ describe("ExecuteWorkflowService", () => { ]; for (const entryPoint of entryPoints) { - it(`${entryPoint.name} refuses a terminating unit and keeps the previous results on screen`, fakeAsync(() => { - selectUnitWithStatus("Terminating"); - const wsSendSpy = vi.spyOn(service["workflowWebsocketService"], "send"); - const resetSpy = vi.spyOn(service, "resetExecutionState"); - const statusResetSpy = vi.spyOn(TestBed.inject(WorkflowStatusService), "resetStatus"); - const errorSpy = vi.spyOn(TestBed.inject(NotificationService), "error").mockReturnValue(undefined as never); - - entryPoint.run(); - tick(FORM_DEBOUNCE_TIME_MS + 1); - flush(); - - expect(wsSendSpy).not.toHaveBeenCalledWith("WorkflowExecuteRequest", expect.anything()); - expect(resetSpy).not.toHaveBeenCalled(); - expect(statusResetSpy).not.toHaveBeenCalled(); - expect(errorSpy).toHaveBeenCalledWith( - "The selected computing unit is shutting down. Wait for it to finish, then select or create another one." - ); - })); - - it.each(["Failed", "Unknown"] as const)( - `${entryPoint.name} refuses a %s unit and keeps the previous results on screen`, - status => { + const blocked = [ + [ + "Terminating", + "The selected computing unit is shutting down. Wait for it to finish, then select or create another one.", + ], + ["Failed", unavailableMessage], + ["Unknown", unavailableMessage], + ] as const; + for (const [status, message] of blocked) { + it(`${entryPoint.name} refuses a ${status} unit and preserves the previous run`, fakeAsync(() => { selectUnitWithStatus(status); - const wsSendSpy = vi.spyOn(service["workflowWebsocketService"], "send"); - const resetSpy = vi.spyOn(service, "resetExecutionState"); - const statusResetSpy = vi.spyOn(TestBed.inject(WorkflowStatusService), "resetStatus"); - const errorSpy = vi.spyOn(TestBed.inject(NotificationService), "error").mockReturnValue(undefined as never); + const run = watchRun(); entryPoint.run(); - - expect(wsSendSpy).not.toHaveBeenCalledWith("WorkflowExecuteRequest", expect.anything()); - expect(resetSpy).not.toHaveBeenCalled(); - expect(statusResetSpy).not.toHaveBeenCalled(); - expect(errorSpy).toHaveBeenCalledWith( - "The selected computing unit is unavailable. Select a running unit or create a new one." - ); - } - ); + tick(FORM_DEBOUNCE_TIME_MS + 1); + flush(); + + expect(run.send).not.toHaveBeenCalledWith("WorkflowExecuteRequest", expect.anything()); + expect(run.reset).not.toHaveBeenCalled(); + expect(run.resetStatus).not.toHaveBeenCalled(); + expect(run.error).toHaveBeenCalledWith(message); + })); + } it.each(["Running", "Pending"] as const)(`${entryPoint.name} still runs on a %s unit`, status => { // Pending is still starting up, so it must not be blocked. @@ -818,9 +1066,7 @@ describe("ExecuteWorkflowService", () => { service.executeWorkflowWithEmailNotification("e", false); expect(errorSpy).toHaveBeenCalledTimes(1); - expect(errorSpy).toHaveBeenCalledWith( - "The selected computing unit is unavailable. Select a running unit or create a new one." - ); + expect(errorSpy).toHaveBeenCalledWith(unavailableMessage); } finally { TestBed.inject(GuiConfigService).env.warehouseEnabled = false; } @@ -931,9 +1177,9 @@ describe("ExecuteWorkflowService", () => { expect(wsSendSpy).toHaveBeenCalledWith("RetryRequest", { workers: ["w9"] }); }); - it("modifyOperatorLogic requires a paused state and sends the operator logic", () => { + it("modifyOperatorLogic requires a paused state and sends the operator logic", async () => { (service as any).currentState = { state: ExecutionState.Running }; - expect(() => service.modifyOperatorLogic("1")).toThrowError( + await expect(service.modifyOperatorLogic("1")).rejects.toThrowError( "cannot modify logic, the current execution state is Running" ); diff --git a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts index 8ead716b691..573a1b0eb17 100644 --- a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts +++ b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts @@ -18,13 +18,25 @@ */ import { Inject, Injectable, DOCUMENT } from "@angular/core"; -import { BehaviorSubject, combineLatest, interval, Observable, Subject, Subscription } from "rxjs"; +import { + BehaviorSubject, + combineLatest, + distinctUntilChanged, + interval, + map, + merge, + mergeMap, + Observable, + ReplaySubject, + startWith, + Subject, + Subscription, +} from "rxjs"; import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; import { WorkflowGraphReadonly } from "../workflow-graph/model/workflow-graph"; import { ExecutionState, ExecutionStateInfo, - LogicalLink, LogicalOperator, LogicalPlan, } from "../../types/execute-workflow.interface"; @@ -51,6 +63,9 @@ import { ComputingUnitStatusService } from "../../../common/service/computing-un import { unavailableComputingUnitReason } from "../../../common/util/computing-unit.util"; import { WarehouseService } from "../../../common/service/warehouse/warehouse.service"; import { GuiConfigService } from "../../../common/service/gui-config.service"; +import { isMacro, MacroDefinition } from "../../../common/util/workflow-macro"; +import { toLogicalGraph } from "@texera/workflow-macros"; +import { MacroTerminal } from "../macro/macro-terminals"; // TODO: change this declaration export const FORM_DEBOUNCE_TIME_MS = 150; @@ -85,8 +100,9 @@ export class ExecuteWorkflowService { current: ExecutionStateInfo; }>(); - private regionUpdateStream = new Subject(); + private regionUpdateStream = new ReplaySubject(1); private regionStateStream = new Subject(); + private regionStates = new Map(); // TODO: move this to another service, or redesign how this // information is stored on the frontend. @@ -120,9 +136,11 @@ export class ExecuteWorkflowService { workflowWebsocketService.websocketEvent().subscribe(event => { switch (event.type) { case "RegionUpdateEvent": + this.regionStates.clear(); this.regionUpdateStream.next(event); break; case "RegionStateEvent": + this.regionStates.set(event.id, event); this.regionStateStream.next(event); break; case "WorkerAssignmentUpdateEvent": @@ -207,7 +225,12 @@ export class ExecuteWorkflowService { return { state: ExecutionState.Failed, errorMessages: event.fatalErrors.map(err => { - return { ...err, message: err.message.replace("\\n", "
") }; + const path = this.workflowActionService.getMacroExpansion?.()?.paths[err.operatorId]; + return { + ...err, + operatorId: path && path.length > 1 ? path[0] : err.operatorId, + message: err.message.replace("\\n", "
"), + }; }), }; default: @@ -280,47 +303,51 @@ export class ExecuteWorkflowService { executionName: string, emailNotificationEnabled: boolean, targetOperatorId?: string - ): boolean { - const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest( - this.workflowActionService.getTexeraGraph(), - targetOperatorId - ); - const settings = this.workflowActionService.getWorkflowSettings(); - if (this.refuseToRunOnUnavailableUnit()) { - return false; - } - if (this.refuseToRunWithoutWarehouse()) { - return false; - } - this.resetExecutionState(); - this.workflowStatusService.resetStatus(); - this.sendExecutionRequest(executionName, logicalPlan, settings, emailNotificationEnabled); - return true; + ): boolean | Promise { + return this.startExecution(executionName, emailNotificationEnabled, targetOperatorId); } - /** Returns whether the request was sent; see executeWorkflowWithEmailNotification. */ - public executeWorkflow(executionName: string, targetOperatorId?: string): boolean { + public executeWorkflow(executionName: string, targetOperatorId?: string): boolean | Promise { return this.executeWorkflowWithEmailNotification(executionName, false, targetOperatorId); } - public executeWorkflowWithReplay(replayExecutionInfo: ReplayExecutionInfo): void { - const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest(this.workflowActionService.getTexeraGraph()); - const settings = this.workflowActionService.getWorkflowSettings(); - if (this.refuseToRunOnUnavailableUnit()) { - return; - } - if (this.refuseToRunWithoutWarehouse()) { - return; - } - this.resetExecutionState(); - this.workflowStatusService.resetStatus(); - this.sendExecutionRequest( - `Replay run of ${replayExecutionInfo.eid} to ${replayExecutionInfo.interaction}`, - logicalPlan, - settings, - false, - replayExecutionInfo - ); + public executeWorkflowWithReplay(replayExecutionInfo: ReplayExecutionInfo): boolean | Promise { + const name = `Replay run of ${replayExecutionInfo.eid} to ${replayExecutionInfo.interaction}`; + return this.startExecution(name, false, undefined, replayExecutionInfo); + } + + private startExecution( + name: string, + notify: boolean, + target?: string, + replay?: ReplayExecutionInfo + ): boolean | Promise { + const plan = this.executionPlan(target); + if (!plan) return false; + const execute = () => { + if (this.refuseToRunOnUnavailableUnit() || this.refuseToRunWithoutWarehouse()) return false; + this.resetExecutionState(); + this.workflowStatusService.resetStatus(); + this.sendExecutionRequest(name, plan, this.workflowActionService.getWorkflowSettings(), notify, replay); + return true; + }; + if (!plan.macroContent) return execute(); + const content = this.workflowActionService.getWorkflowContent(); + const doc = this.workflowActionService.editorDocument; + return this.workflowActionService.macroExpansion + .expand(content) + .then(() => { + if ( + this.workflowActionService.editorDocument !== doc || + !isEqual(this.workflowActionService.getWorkflowContent(), content) + ) + throw new Error("The workflow changed while preparing execution. Try again."); + return execute(); + }) + .catch(error => { + this.notificationService.error(error instanceof Error ? error.message : String(error)); + return false; + }); } /** @@ -346,6 +373,24 @@ export class ExecuteWorkflowService { return true; } + private executionPlan(targetOperatorId?: string): LogicalPlan | undefined { + if (this.workflowActionService.getMacroInterface?.()) { + this.notificationService.error("A macro runs through a workflow. Add it to a workflow to execute it."); + return; + } + try { + return ExecuteWorkflowService.getLogicalPlanRequest( + this.workflowActionService.getTexeraGraph(), + targetOperatorId, + this.workflowActionService.getMacroDependencies?.() ?? [] + ); + } catch (error) { + if (!this.workflowActionService.getTexeraGraph().getAllOperators().some(isMacro)) throw error; + this.notificationService.error(error instanceof Error ? error.message : String(error)); + return; + } + } + /** * While the deployment requires a warehouse (#7817) and none is picked, * refuses with a toast and returns true. Checked at every public entry @@ -457,17 +502,26 @@ export class ExecuteWorkflowService { this.workflowWebsocketService.send("RetryRequest", { workers }); } - public modifyOperatorLogic(operatorID: string): void { + public async modifyOperatorLogic(operatorID: string): Promise { if (this.currentState.state !== ExecutionState.Paused) { throw new Error("cannot modify logic, the current execution state is " + this.currentState.state); } const op = this.workflowActionService.getTexeraGraph().getOperator(operatorID); - const operator: LogicalOperator = { - ...op.operatorProperties, - operatorID: op.operatorID, - operatorType: op.operatorType, - }; - this.workflowWebsocketService.send("ModifyLogicRequest", { operator }); + const expanded = isMacro(op) + ? await this.workflowActionService.macroExpansion.expand(this.workflowActionService.getWorkflowContent()) + : undefined; + if (this.currentState.state !== ExecutionState.Paused) throw new Error("Execution is no longer paused"); + const operators = expanded + ? expanded.content.operators.filter(o => expanded.paths[o.operatorID][0] === operatorID) + : [op]; + for (const leaf of operators) { + const operator: LogicalOperator = { + ...leaf.operatorProperties, + operatorID: leaf.operatorID, + operatorType: leaf.operatorType, + }; + this.workflowWebsocketService.send("ModifyLogicRequest", { operator }); + } } public getExecutionStateStream(): Observable<{ @@ -477,15 +531,74 @@ export class ExecuteWorkflowService { return this.executionStateStream.asObservable(); } - public getRegionUpdateStream(): Observable { - return this.regionUpdateStream.asObservable(); + public getRegionUpdateStream(): Observable }> { + return combineLatest([ + this.regionUpdateStream, + this.workflowStatusService.getStatisticsUpdateStream().pipe(startWith(undefined)), + ]).pipe( + map(([event]) => { + const paths = this.workflowActionService.getMacroExpansion?.()?.paths ?? {}; + const ids = this.workflowActionService.runtimeOperatorIDs?.(); + const localIDs = ids && Object.fromEntries(Object.entries(ids).map(([local, runtime]) => [runtime, local])); + const boundaries = this.workflowActionService.runtimeBoundaryLinks?.() ?? []; + return { + ...event, + ...(boundaries.length + ? { + terminals: Object.fromEntries( + event.regions.map(([id, ops]) => [ + id, + boundaries + .filter(b => + b.links.some( + link => ops.includes(link.source.operatorID) && ops.includes(link.target.operatorID) + ) + ) + .map(b => b.terminal), + ]) + ), + } + : {}), + regions: event.regions.map(([id, ops]): [number, string[]] => [ + id, + [ + ...new Set( + ops.flatMap(op => { + const local = localIDs ? localIDs[op] : op; + return local === undefined ? [] : [paths[local]?.[0] ?? local]; + }) + ), + ], + ]), + }; + }), + map(event => ({ event, document: this.workflowActionService.editorDocument })), + distinctUntilChanged( + (previous, current) => previous.document === current.document && isEqual(previous.event, current.event) + ), + map(({ event }) => event) + ); } public getRegionStateStream(): Observable { - return this.regionStateStream.asObservable(); + // Reapply the latest phases after topology subscribers redraw the visible canvas. + return merge( + this.regionStateStream, + this.getRegionUpdateStream().pipe( + mergeMap(event => + event.regions.flatMap(([id]) => (this.regionStates.has(id) ? [this.regionStates.get(id)!] : [])) + ) + ) + ); + } + + private resetRegions(): void { + this.regionStates.clear(); + this.regionUpdateStream.next({ regions: [] }); } public resetExecutionState(): void { + this.resetRegions(); this.currentState = { state: ExecutionState.Uninitialized, }; @@ -499,6 +612,8 @@ export class ExecuteWorkflowService { * executionStateStream so subscribers drop the previous unit's status. */ public resetExecutionAndWorkers(): void { + this.resetRegions(); + this.workflowStatusService.clearStatus(); this.updateExecutionState({ state: ExecutionState.Uninitialized }); this.assignedWorkerIds.clear(); // Leaving the workspace takes this path, not resetExecutionState, so the clock has to be @@ -572,47 +687,52 @@ export class ExecuteWorkflowService { * @param workflowGraph * @param targetOperatorId */ - public static getLogicalPlanRequest(workflowGraph: WorkflowGraphReadonly, targetOperatorId?: string): LogicalPlan { - const getInputPortOrdinal = (operatorID: string, inputPortID: string): number => { - return workflowGraph.getOperator(operatorID).inputPorts.findIndex(port => port.portID === inputPortID); - }; - - const getOutputPortOrdinal = (operatorID: string, outputPortID: string): number => { - return workflowGraph.getOperator(operatorID).outputPorts.findIndex(port => port.portID === outputPortID); - }; - const subDAG = workflowGraph.getSubDAG(targetOperatorId); - - const operators: LogicalOperator[] = subDAG.operators.map(op => ({ - ...op.operatorProperties, - operatorID: op.operatorID, - operatorType: op.operatorType, - inputPorts: op.inputPorts, - outputPorts: op.outputPorts, - })); - - const links: LogicalLink[] = subDAG.links.map(link => { - const outputPortIdx = getOutputPortOrdinal(link.source.operatorID, link.source.portID); - const inputPortIdx = getInputPortOrdinal(link.target.operatorID, link.target.portID); + public static getLogicalPlanRequest( + workflowGraph: WorkflowGraphReadonly, + targetOperatorId?: string, + dependencies: MacroDefinition[] = [] + ): LogicalPlan { + let subDAG = workflowGraph.getSubDAG(targetOperatorId); + let resultIDs = [...workflowGraph.getOperatorsToViewResult()]; + let reuseIDs = [...workflowGraph.getOperatorsMarkedForReuseResult()]; + if (workflowGraph.getAllOperators().some(isMacro)) { return { - fromOpId: link.source.operatorID, - fromPortId: { id: outputPortIdx, internal: false }, - toOpId: link.target.operatorID, - toPortId: { id: inputPortIdx, internal: false }, + operators: [], + links: [], + opsToViewResult: resultIDs, + opsToReuseResult: reuseIDs, + macroContent: { + operators: workflowGraph.getAllOperators(), + links: workflowGraph.getAllLinks(), + macroDependencies: dependencies, + }, + executeOperatorIds: subDAG.operators.map(op => op.operatorID), }; - }); + } const operatorIds = new Set(subDAG.operators.map(op => op.operatorID)); - const opsToViewResult: string[] = Array.from(intersection(operatorIds, workflowGraph.getOperatorsToViewResult())); + const opsToViewResult: string[] = Array.from(intersection(operatorIds, new Set(resultIDs))); - const opsToReuseResult: string[] = Array.from( - intersection(operatorIds, workflowGraph.getOperatorsMarkedForReuseResult()) - ); + const opsToReuseResult: string[] = Array.from(intersection(operatorIds, new Set(reuseIDs))); - return { operators, links, opsToViewResult, opsToReuseResult }; + return { ...toLogicalGraph(subDAG), opsToViewResult, opsToReuseResult }; } public getWorkerIds(operatorId: string): ReadonlyArray { - return this.assignedWorkerIds.get(operatorId) || []; + if (this.workflowActionService.getTempWorkflow()) return []; + const ids = this.workflowActionService.runtimeOperatorIDs?.(); + if (ids && !Object.keys(ids).length) return []; + const expanded = this.workflowActionService.getMacroExpansion?.(); + if ( + expanded && + this.workflowActionService.getTexeraGraph().hasOperator(operatorId) && + isMacro(this.workflowActionService.getTexeraGraph().getOperator(operatorId)) + ) { + return Object.entries(expanded.paths) + .filter(([, path]) => path[0] === operatorId) + .flatMap(([id]) => this.assignedWorkerIds.get(ids?.[id] ?? id) ?? []); + } + return this.assignedWorkerIds.get(ids?.[operatorId] ?? operatorId) || []; } } diff --git a/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts b/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts index cd32c277957..1ceab33315e 100644 --- a/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts +++ b/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts @@ -16,6 +16,8 @@ * specific language governing permissions and limitations * under the License. */ +import { testPort, testMacro, setMacroGraph } from "../../../common/testing/macro-graph"; +import { testBinding } from "../../../common/testing/workflow-graph"; import { TestBed } from "@angular/core/testing"; import { FormBindingService } from "./form-binding.service"; @@ -27,6 +29,8 @@ import { OperatorMetadataService } from "../operator-metadata/operator-metadata. import { StubOperatorMetadataService } from "../operator-metadata/stub-operator-metadata.service"; import { commonTestProviders } from "../../../common/testing/test-utils"; import { mockPoint, mockScanPredicate, mockSentimentPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { MacroDefinition, macroInstance } from "../../../common/util/workflow-macro"; +import { exportedWorkflow } from "../../../common/type/workflow"; describe("FormBindingService", () => { let service: FormBindingService; @@ -51,6 +55,72 @@ describe("FormBindingService", () => { const scanId = mockScanPredicate.operatorID; + it("undoes macro parameter exposure, renaming and removal through the existing form bindings", () => { + setMacroGraph(workflowActionService, { id: "macro", inputs: [], outputs: [] }); + const undo = TestBed.inject(UndoRedoService); + workflowActionService.bundleMacroActions(() => service.addBinding(scanId, "tableName")); + const original = service.getConfig().fields[0]; + workflowActionService.bundleMacroActions(() => service.updateBinding(original.id, { displayName: "名前 café" })); + workflowActionService.bundleMacroActions(() => service.removeBinding(original.id)); + expect(service.getConfig().fields).toEqual([]); + undo.undoAction(); + expect(service.getConfig().fields[0].displayName).toBe("名前 café"); + undo.undoAction(); + expect(service.getConfig().fields).toEqual([original]); + undo.undoAction(); + expect(service.getConfig().fields).toEqual([]); + undo.redoAction(); + expect(service.getConfig().fields).toEqual([original]); + }); + + it("resolves and edits three nested macro levels without changing another instance or the pinned definitions", () => { + const definitions: MacroDefinition[] = []; + let inner = { ...mockScanPredicate, operatorID: "leaf", operatorProperties: { tableName: "initial" } }; + for (let level = 0; level < 3; level++) { + const id = `form-level-${level}`; + const definition = testMacro(id, [inner], { + ...workflowActionService.getWorkflowContent(), + links: [], + name: `Level ${level}`, + outputs: [testPort("output-0", inner.operatorID)], + formBinding: testBinding(inner.operatorID, level ? "exposed" : "tableName", "exposed", "Table"), + }); + definitions.push(definition); + inner = macroInstance(definition, `nested-${level}`) as typeof inner; + } + workflowActionService.setMacroDependencies(definitions); + for (const id of ["first", "second"]) { + workflowActionService.addOperator( + { ...inner, operatorID: id, operatorProperties: { exposed: "initial" } }, + mockPoint + ); + service.addBinding(id, "exposed"); + } + const before = JSON.stringify(definitions); + const [first, second] = service.resolveFields(); + expect(first.schema?.title).toBe("Table"); + expect(first.brokenReason).toBeUndefined(); + const portable = JSON.parse( + JSON.stringify(exportedWorkflow(workflowActionService.getWorkflowContent(), undefined)) + ); + workflowActionService.reloadWorkflow({ ...workflowActionService.getWorkflow(), content: portable }, false); + const undo = TestBed.inject(UndoRedoService); + for (const value of ["東京 café", ""]) { + const beforeEdit = workflowActionService.getWorkflowContent(); + workflowActionService.bundleMacroActions(() => service.writeValue(first.binding, value)); + const afterEdit = workflowActionService.getWorkflowContent(); + undo.undoAction(); + expect(workflowActionService.getWorkflowContent()).toEqual(beforeEdit); + undo.redoAction(); + expect(workflowActionService.getWorkflowContent()).toEqual(afterEdit); + expect(service.resolveFields().map(f => f.value)).toEqual([value, "initial"]); + } + expect(JSON.stringify(definitions)).toBe(before); + service.updateBinding(first.binding.id, { propertyKey: "missing" }); + expect(service.resolveFields()[0].brokenReason).toContain("not a setting"); + expect(service.readValue(second.binding.operatorID, second.binding.propertyKey)).toBe("initial"); + }); + it("starts with nothing exposed and no result list, so the final steps show", () => { expect(service.getConfig().fields).toEqual([]); expect(service.getConfig().shownResultIds).toBeUndefined(); diff --git a/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.spec.ts b/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.spec.ts index f017a21c0ec..451eb86949a 100644 --- a/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.spec.ts +++ b/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.spec.ts @@ -82,6 +82,7 @@ describe("HeatmapStatsRestoreService", () => { savePersistedHeatmapView(HeatmapView.Runtime); executionsService = { + retrieveMacroStatistics: vi.fn(), retrieveWorkflowExecutions: vi.fn(() => of([makeExecution({})])), retrieveWorkflowRuntimeStatistics: vi.fn(() => of([makeStatsRow({})])), } as unknown as Mocked; @@ -118,15 +119,38 @@ describe("HeatmapStatsRestoreService", () => { expect(executionsService.retrieveWorkflowExecutions).toHaveBeenCalledWith(42); expect(executionsService.retrieveWorkflowRuntimeStatistics).toHaveBeenCalledWith(42, 1, 7); - expect(statusService.setExternalStatus).toHaveBeenCalledWith({ - op1: expect.objectContaining({ + expect(statusService.setExternalStatus).toHaveBeenCalledWith( + { + op1: expect.objectContaining({ + operatorState: OperatorState.Completed, + aggregatedInputRowCount: 10, + aggregatedOutputRowCount: 5, + aggregatedDataProcessingTime: 1_000_000, + numWorkers: 2, + }), + }, + 1 + ); + }); + + it("reopens macro port counts with the heatmap off and falls back only for aggregate-only history", () => { + savePersistedHeatmapView(null); + const stats = { + inner: { operatorState: OperatorState.Completed, - aggregatedInputRowCount: 10, - aggregatedOutputRowCount: 5, - aggregatedDataProcessingTime: 1_000_000, - numWorkers: 2, - }), - }); + aggregatedInputRowCount: 3, + aggregatedOutputRowCount: 3, + outputPortMetrics: { "0": 1, "1": 2 }, + outputPortSizes: { "0": 4, "1": 9 }, + }, + }; + executionsService.retrieveMacroStatistics.mockReturnValueOnce(of(stats)).mockReturnValueOnce(of(null)); + service.restoreLatestRunStatistics(true).subscribe(); + expect(executionsService.retrieveMacroStatistics).toHaveBeenCalledWith(42, 1, 7); + expect(statusService.setExternalStatus).toHaveBeenCalledWith(stats, 1); + expect(executionsService.retrieveWorkflowRuntimeStatistics).not.toHaveBeenCalled(); + service.restoreLatestRunStatistics(true).subscribe(); + expect(executionsService.retrieveWorkflowRuntimeStatistics).toHaveBeenCalledWith(42, 1, 7); }); it("does nothing when the overlay is not persisted on", () => { diff --git a/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.ts b/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.ts index b541e131e56..e8da06351e5 100644 --- a/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.ts +++ b/frontend/src/app/workspace/service/heatmap/heatmap-stats-restore.service.ts @@ -18,7 +18,7 @@ */ import { Injectable } from "@angular/core"; -import { EMPTY, Observable, defer } from "rxjs"; +import { of, EMPTY, Observable, defer } from "rxjs"; import { catchError, map, switchMap, takeUntil, tap } from "rxjs/operators"; import { WorkflowExecutionsService } from "../../../dashboard/service/user/workflow-executions/workflow-executions.service"; import { EXECUTION_STATUS_CODE, WorkflowExecutionsEntry } from "../../../dashboard/type/workflow-executions-entry"; @@ -29,14 +29,7 @@ import { ExecutionState, isNotInExecution } from "../../types/execute-workflow.i import { loadPersistedHeatmapView } from "./heatmap-overlay-persistence"; import { toOperatorRuntimeStatusMap } from "./runtime-statistics-mapper"; -/** - * Restores the last execution's per-operator statistics after a page refresh, - * so the performance heat-map can render a finished run without re-executing. - * - * All gating lives here rather than in the caller: the persisted-overlay check - * is a synchronous localStorage read, so a user who never enabled the overlay - * incurs no fetch at all. - */ +/** Restores the last run for the heatmap and nested macro inspectors. */ @Injectable({ providedIn: "root", }) @@ -52,16 +45,16 @@ export class HeatmapStatsRestoreService { * Fetches the latest run's statistics and feeds them into * WorkflowStatusService. Cold: nothing happens until subscribed. Skips * silently (including on HTTP errors — restoring is best-effort) when: - * - the overlay is not persisted on, + * - neither a macro inspector nor the heatmap needs statistics, * - the workflow has never been saved (no wid), * - an execution is in progress, on entry or by the time the fetches return * (the live stream wins), * - the workflow has no executions or the run left no statistics, * - another producer writes statistics first (a new run clears the canvas). */ - public restoreLatestRunStatistics(): Observable { + public restoreLatestRunStatistics(hasMacros = false): Observable { return defer(() => { - if (loadPersistedHeatmapView() === null) { + if (!hasMacros && loadPersistedHeatmapView() === null) { return EMPTY; } const wid = this.workflowActionService.getWorkflowMetadata()?.wid; @@ -78,19 +71,25 @@ export class HeatmapStatsRestoreService { if (run === undefined) { return EMPTY; } - return this.workflowExecutionsService.retrieveWorkflowRuntimeStatistics(wid, run.eId, run.cuId); + const history = () => + this.workflowExecutionsService + .retrieveWorkflowRuntimeStatistics(wid, run.eId, run.cuId) + .pipe(map(toOperatorRuntimeStatusMap)); + const stats = hasMacros + ? this.workflowExecutionsService + .retrieveMacroStatistics(wid, run.eId, run.cuId) + .pipe(switchMap(snapshot => (snapshot === null ? history() : of(snapshot)))) + : history(); + return stats.pipe(map(runtimeStatus => ({ runtimeStatus, source: run.eId }))); }), - // Only the fetches are best-effort. Placed above the map so a mapping or ingestion - // failure still surfaces instead of looking like a run with nothing to restore. catchError(() => EMPTY), - tap(rows => { - const runtimeStatus = toOperatorRuntimeStatusMap(rows); + tap(({ runtimeStatus, source }) => { // Re-checked, not redundant: the entry guard ran two round trips ago, and the // websocket can connect and start streaming a live run inside that window. if (this.isExecuting() || Object.keys(runtimeStatus).length === 0) { return; } - this.workflowStatusService.setExternalStatus(runtimeStatus); + this.workflowStatusService.setExternalStatus(runtimeStatus, source); }), map(() => undefined), // Any other producer writing statistics means the canvas is no longer ours to restore: diff --git a/frontend/src/app/workspace/service/heatmap/runtime-statistics-mapper.ts b/frontend/src/app/workspace/service/heatmap/runtime-statistics-mapper.ts index 980c6bd6da9..8f6f8362fbc 100644 --- a/frontend/src/app/workspace/service/heatmap/runtime-statistics-mapper.ts +++ b/frontend/src/app/workspace/service/heatmap/runtime-statistics-mapper.ts @@ -48,8 +48,8 @@ export function operatorStateFromStatusCode(code: number): OperatorState { * per operator (by timestamp; the later row wins a tie, matching write * order), mapped to the wire shape WorkflowStatusService ingests. * - * The engine sums the ports away before persisting, so the port maps are left - * absent rather than empty — an empty map would zero the port labels. + * Aggregate history rows carry no port counters. Omitting the maps lets the + * canvas fall back to the aggregate where appropriate. */ export function toOperatorRuntimeStatusMap(rows: WorkflowRuntimeStatistics[]): Record { const latestByOperator: Record = {}; diff --git a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.spec.ts b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.spec.ts index eeeab17390c..c56ca7fb6e9 100644 --- a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.spec.ts +++ b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.spec.ts @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { testOperator } from "../../../common/testing/workflow-graph"; import { of } from "rxjs"; import * as joint from "jointjs"; @@ -502,6 +503,20 @@ describe("JointUIService", () => { }); describe("getJointOperatorElement", () => { + it.each([ + ["MacroInput", "MacroInput.svg"], + ["MacroOutput", "MacroOutput.svg"], + ["WorkflowMacro:example:1", "SubWorkflow.svg"], + ["TextInput", "TextInput.png"], + ])("uses the appropriate icon for %s", (operatorType, icon) => { + const element = JointUIService.createJointOperatorElement( + testOperator("icon", { operatorType }), + { x: 0, y: 0 }, + "Icon" + ); + expect(element.attr(".texera-operator-icon/xlink:href")).toBe(`assets/operator_images/${icon}`); + }); + function buildMetadataWithSchemas(schemas: object[]) { return { getOperatorMetadata: () => @@ -734,6 +749,18 @@ describe("JointUIService", () => { }); describe("changeOperatorStatistics", () => { + it("restores port names and removes worker counts when a view loses its execution context", () => { + const { paper, attrSpy, portPropSpy } = makeStatsPaper(() => []); + const service = new JointUIService(emptyMetadataStub as never); + service.resetOperatorStatistics(paper, { + operatorID: "op-1", + inputPorts: [{ portID: "in", displayName: "Rows" }], + outputPorts: [{ portID: "out" }], + } as OperatorPredicate); + expect(attrSpy).toHaveBeenCalledWith(".operator-worker-count/text", ""); + expect(portPropSpy).toHaveBeenCalledWith("in", "attrs/.port-label/text", "Rows"); + expect(portPropSpy).toHaveBeenCalledWith("out", "attrs/.port-label/text", ""); + }); function makeStatsPaper(getPortsImpl: () => Array<{ id?: string; group?: string; attrs?: unknown }>) { const attrSpy = vi.fn(); const portPropSpy = vi.fn(); @@ -802,6 +829,25 @@ describe("JointUIService", () => { expect(valuesWritten).toContain("#workers: 8"); }); + it("uses boundary ordinals when named ports have been removed or reordered, including zero counts", () => { + const { paper, portPropSpy } = makeStatsPaper(() => [ + { id: "input-1", group: "in" }, + { id: "input-0", group: "in" }, + { id: "output-1", group: "out" }, + { id: "output-0", group: "out" }, + ]); + new JointUIService(emptyMetadataStub as never).changeOperatorStatistics(paper, "macro", { + aggregatedInputRowCount: 21, + aggregatedOutputRowCount: 5, + inputPortMetrics: { "0": 0, "1": 21 }, + outputPortMetrics: { "0": 5, "1": 0 }, + }); + expect(portPropSpy).toHaveBeenCalledWith("input-1", "attrs/.port-label/text", "0"); + expect(portPropSpy).toHaveBeenCalledWith("input-0", "attrs/.port-label/text", "21"); + expect(portPropSpy).toHaveBeenCalledWith("output-1", "attrs/.port-label/text", "5"); + expect(portPropSpy).toHaveBeenCalledWith("output-0", "attrs/.port-label/text", "0"); + }); + it("defaults the worker count to 1 when numWorkers is unspecified", () => { const { paper, attrSpy } = makeStatsPaper(() => []); const service = new JointUIService(emptyMetadataStub as never); diff --git a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts index 7d51e3a6199..767418b75de 100644 --- a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts +++ b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts @@ -27,6 +27,7 @@ import { fromEventPattern, Observable } from "rxjs"; import { Coeditor } from "../../../common/type/user"; import { OperatorResultCacheStatus } from "../../types/workflow-websocket.interface"; import { HEATMAP_NO_DATA_COLOR, scoreToColor } from "../heatmap/heatmap-color"; +import { isMacro, isMacroTerminal } from "../../../common/util/workflow-macro"; /** * Defines the SVG path for the delete button @@ -289,11 +290,26 @@ export class JointUIService { public getJointOperatorElement(operator: OperatorPredicate, point: Point): joint.dia.Element { // check if the operatorType exists in the operator metadata - const operatorSchema = this.operatorSchemas.find(op => op.operatorType === operator.operatorType); + const operatorSchema = isMacro(operator) + ? this.operatorMetadataService.getOperatorSchema(operator.operatorType) + : this.operatorSchemas.find(op => op.operatorType === operator.operatorType); if (operatorSchema === undefined) { throw new Error(`operator type ${operator.operatorType} doesn't exist`); } + return JointUIService.createJointOperatorElement( + operator, + point, + operatorSchema.additionalMetadata.userFriendlyName + ); + } + + /** Shared canvas rendering for executable operators and macro interface terminals. */ + public static createJointOperatorElement( + operator: OperatorPredicate, + point: Point, + friendlyName: string + ): joint.dia.Element { // construct a custom Texera JointJS operator element // and customize the styles of the operator box and ports const operatorElement = new TexeraCustomJointElement({ @@ -304,11 +320,9 @@ export class JointUIService { }, attrs: JointUIService.getCustomOperatorStyleAttrs( operator, - JointUIService.truncateOperatorDisplayName( - operator.customDisplayName ?? operatorSchema.additionalMetadata.userFriendlyName - ), - operatorSchema.operatorType, - operatorSchema.additionalMetadata.userFriendlyName + JointUIService.truncateOperatorDisplayName(operator.customDisplayName ?? friendlyName), + operator.operatorType, + friendlyName ), ports: { groups: { @@ -326,45 +340,40 @@ export class JointUIService { operatorElement.set("id", operator.operatorID); operatorElement.set("z", 1); - // set the input ports and output ports based on operator predicate - operator.inputPorts.forEach(port => - operatorElement.addPort({ - group: "in", - id: port.portID, - attrs: { - ".port-label": { - text: port.displayName ?? "", - event: "input-port-label:pointerdown", - }, - }, - label: { - position: { - name: "left", - args: { x: -5, y: 10 }, - }, - }, - }) - ); - operator.outputPorts.forEach(port => - operatorElement.addPort({ - group: "out", - id: port.portID, - attrs: { - ".port-label": { - text: port.displayName ?? "", - event: "output-port-label:pointerdown", + JointUIService.updateOperatorPorts(operatorElement, operator); + return operatorElement; + } + + public static updateOperatorPorts( + operatorElement: joint.dia.Element, + operator: Pick + ): void { + const current = new Map(operatorElement.getPorts().map(port => [port.id, port])); + const ports = (declared: OperatorPredicate["inputPorts"], input: boolean) => + declared.map(port => { + const previous = current.get(port.portID); + return { + ...previous, + id: port.portID, + group: input ? "in" : "out", + attrs: { + ...previous?.attrs, + ".port-label": { + ...previous?.attrs?.[".port-label"], + text: port.displayName ?? "", + port: port.portID, + event: input ? "input-port-label:pointerdown" : "output-port-label:pointerdown", + }, }, - }, - label: { - position: { - name: "right", - args: { x: 5, y: -10 }, + label: previous?.label ?? { + position: { name: input ? "left" : "right", args: { x: input ? -5 : 5, y: input ? 10 : -10 } }, }, - }, - }) - ); - - return operatorElement; + }; + }); + // Reconcile by identity and preserve graph order, including a middle port restored by Undo. + operatorElement.prop("ports/items", [...ports(operator.inputPorts, true), ...ports(operator.outputPorts, false)], { + rewrite: true, + }); } /** @@ -395,29 +404,37 @@ export class JointUIService { // Absent map: no per-port information in this snapshot, so leave the labels alone. // Empty map: every port measured zero, so write the zeros. if (inputMetrics !== undefined) { - inPorts.forEach(portDef => { + inPorts.forEach((portDef, index) => { const portId = portDef.id; if (portId != null) { const parts = portId.split("-"); const numericSuffix = parts.length > 1 ? parts[1] : portId; - const count: number = inputMetrics[numericSuffix] ?? 0; + const count: number = inputMetrics[String(index)] ?? inputMetrics[numericSuffix] ?? 0; element.portProp(portId, "attrs/.port-label/text", count.toLocaleString()); } }); } if (outputMetrics !== undefined) { - outPorts.forEach(portDef => { + outPorts.forEach((portDef, index) => { const portId = portDef.id; if (portId != null) { const parts = portId.split("-"); const numericSuffix = parts.length > 1 ? parts[1] : portId; - const count: number = outputMetrics[numericSuffix] ?? 0; + const count: number = outputMetrics[String(index)] ?? outputMetrics[numericSuffix] ?? 0; element.portProp(portId, "attrs/.port-label/text", count.toLocaleString()); } }); } } + public resetOperatorStatistics(jointPaper: joint.dia.Paper, operator: OperatorPredicate): void { + const element = jointPaper.getModelById(operator.operatorID) as joint.shapes.devs.Model; + if (!element) return; + element.attr(`.${operatorWorkerCountClass}/text`, ""); + [...operator.inputPorts, ...operator.outputPorts].forEach(port => + element.portProp(port.portID, "attrs/.port-label/text", port.displayName ?? "") + ); + } public foldOperatorDetails(jointPaper: joint.dia.Paper, operatorID: string): void { jointPaper.getModelById(operatorID).attr({ [`.${operatorStateClass}`]: { visibility: "hidden" }, @@ -934,7 +951,9 @@ export class JointUIService { visibility: "hidden", }, ".texera-operator-icon": { - "xlink:href": "assets/operator_images/" + operatorType + ".png", + "xlink:href": isMacro(operator) + ? "assets/operator_images/SubWorkflow.svg" + : `assets/operator_images/${operatorType}.${isMacroTerminal(operator) ? "svg" : "png"}`, width: 35, height: 35, "ref-x": 0.5, diff --git a/frontend/src/app/workspace/service/macro/macro-expansion.service.spec.ts b/frontend/src/app/workspace/service/macro/macro-expansion.service.spec.ts new file mode 100644 index 00000000000..a00a9c35c5d --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-expansion.service.spec.ts @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { macroGraph } from "../../../../../../common/workflow-core/src/test/typescript/macro-test-graph"; +import { TestBed } from "@angular/core/testing"; +import { provideHttpClient } from "@angular/common/http"; +import { HttpTestingController, provideHttpClientTesting } from "@angular/common/http/testing"; +import { MacroExpansionService } from "./macro-expansion.service"; +import { testWorkflow } from "../../../common/testing/workflow-graph"; + +describe("backend macro expansion", () => { + let service: MacroExpansionService; + let http: HttpTestingController; + const graph = testWorkflow(); + const expanded = { content: graph, paths: {}, ports: {} }; + const expansionRequest = () => http.expectOne(request => request.url.endsWith("/compile/expand")); + beforeEach(() => { + TestBed.configureTestingModule({ providers: [provideHttpClient(), provideHttpClientTesting()] }); + service = TestBed.inject(MacroExpansionService); + http = TestBed.inject(HttpTestingController); + }); + afterEach(() => http.verify()); + + it("shares an in-flight request and uses the backend response without changing the snapshot", async () => { + const original = JSON.stringify(graph); + const first = service.expand(graph), + second = service.expand(graph); + const request = expansionRequest(); + expect(request.request.body).toEqual(graph); + request.flush(expanded); + expect(await first).toEqual(expanded); + expect(await second).toEqual(expanded); + expect(service.get(graph)).toEqual(expanded); + expect(JSON.stringify(graph)).toBe(original); + expect(await service.expand(graph)).toEqual(expanded); + http.expectNone(request => request.url.endsWith("/compile/expand")); + }); + + it("does not cache a failure and retries the same snapshot", async () => { + const failed = expect(service.expand(graph)).rejects.toThrow("Recursive macro reference"); + expansionRequest().flush({ message: "Recursive macro reference" }, { status: 400, statusText: "Bad Request" }); + await failed; + expect(service.get(graph)).toBeUndefined(); + const retry = service.expand(graph); + expansionRequest().flush(expanded); + await retry; + }); + + it("invalidates exposed-property and interface changes while reusing presentation-only changes", () => { + service.remember(graph, expanded); + expect(service.get({ ...graph, operatorPositions: { moved: { x: 30, y: 20 } } })).toBe(expanded); + expect(service.get({ ...graph, formBinding: { fields: [] } })).toBeUndefined(); + expect( + service.get(macroGraph({ ...graph, macroInterface: { id: "new", inputs: [], outputs: [] } })) + ).toBeUndefined(); + }); + it("retains an actively inspected execution while evicting unused previews", () => { + service.remember(graph, expanded); + const previews = Array.from({ length: 25 }, (_, i) => ({ + ...graph, + formBinding: { fields: [], instruction: { body: String(i) } }, + })); + for (const preview of previews) { + expect(service.get(graph)).toBe(expanded); + service.remember(preview, { ...expanded, content: preview }); + } + expect(service.get(graph)).toBe(expanded); + expect(service.get(previews[0])).toBeUndefined(); + expect(service.get(previews.at(-1)!)).toBeDefined(); + }); +}); diff --git a/frontend/src/app/workspace/service/macro/macro-expansion.service.ts b/frontend/src/app/workspace/service/macro/macro-expansion.service.ts new file mode 100644 index 00000000000..0d6e891749a --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-expansion.service.ts @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Injectable } from "@angular/core"; +import { HttpClient } from "@angular/common/http"; +import { firstValueFrom, Subject } from "rxjs"; +import { AppSettings } from "../../../common/app-setting"; +import { WorkflowContent } from "../../../common/type/workflow"; +import { MacroExpansion } from "../../../common/util/workflow-macro"; + +/** Runtime projections are cached by graph semantics; compilation is the authority. */ +@Injectable({ providedIn: "root" }) +export class MacroExpansionService { + public readonly changed$ = new Subject(); + private readonly cache = new Map(); + private readonly pending = new Map>(); + constructor(private http: HttpClient) {} + + private key(content: WorkflowContent): string { + return JSON.stringify([ + content.operators, + content.links, + content.macroDependencies, + content.macroInterface, + content.formBinding, + ]); + } + public get(content: WorkflowContent): MacroExpansion | undefined { + const key = this.key(content), + cached = this.cache.get(key); + if (cached) { + this.cache.delete(key); + this.cache.set(key, cached); + } + return cached; + } + public remember(content: WorkflowContent, expanded: MacroExpansion): void { + const key = this.key(content); + this.cache.delete(key); + this.cache.set(key, expanded); + if (this.cache.size > 20) this.cache.delete(this.cache.keys().next().value!); + this.changed$.next(); + } + public expand(content: WorkflowContent): Promise { + const cached = this.get(content); + if (cached) return Promise.resolve(cached); + const key = this.key(content); + if (!this.pending.has(key)) { + const snapshot = structuredClone(content); + const request = firstValueFrom( + this.http.post(`${AppSettings.getApiEndpoint()}/compile/expand`, snapshot) + ) + .then(expanded => { + this.remember(snapshot, expanded); + return expanded; + }) + .catch(error => { + throw new Error(error.error?.message ?? error.message ?? "Could not validate macro graph"); + }) + .finally(() => this.pending.delete(key)); + this.pending.set(key, request); + } + return this.pending.get(key)!; + } +} diff --git a/frontend/src/app/workspace/service/macro/macro-navigation.service.spec.ts b/frontend/src/app/workspace/service/macro/macro-navigation.service.spec.ts new file mode 100644 index 00000000000..9e7885d45f2 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-navigation.service.spec.ts @@ -0,0 +1,1056 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Text } from "yjs"; +import type { MockInstance } from "vitest"; +import { updateYTypeFromObject } from "../../types/shared-editing.interface"; +import { testPort, testMacro, setMacroGraph, expansionFixture } from "../../../common/testing/macro-graph"; +import { testBinding, testWorkflow, testLink } from "../../../common/testing/workflow-graph"; +import { + isMacroTerminal, + readMacroInterface, +} from "../../../../../../common/workflow-core/src/main/typescript/workflow-macros"; + +import { macroWorkspaceTestProviders } from "./macro-workspace.spec-harness"; +import { MacroExpansionService } from "./macro-expansion.service"; +import { TestBed } from "@angular/core/testing"; +import { of, throwError, Subject, switchMap, take } from "rxjs"; +import { WorkflowExecutionsService } from "../../../dashboard/service/user/workflow-executions/workflow-executions.service"; +import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; +import { TexeraWebsocketEvent } from "../../types/workflow-websocket.interface"; +import { WorkflowWebsocketService } from "../workflow-websocket/workflow-websocket.service"; +import { OperatorState, OperatorRuntimeStatus } from "../../types/execute-workflow.interface"; +import { OperatorPredicate } from "../../types/workflow-common.interface"; +import { MacroNavigationService } from "./macro-navigation.service"; +import { MacroService } from "./macro.service"; +import { WorkflowActionService, DEFAULT_WORKFLOW } from "../workflow-graph/model/workflow-action.service"; +import { + MacroSaveStep, + WorkflowPersistService, +} from "../../../common/service/workflow-persist/workflow-persist.service"; +import { WorkflowStatusService } from "../workflow-status/workflow-status.service"; +import { + mockScanPredicate, + mockSentimentPredicate, + mockResultPredicate, +} from "../workflow-graph/model/mock-workflow-data"; +import { Workflow, WorkflowContent } from "../../../common/type/workflow"; +import { + MacroDefinition, + macroInstance, + macroType, + replaceMacroVersion, + macroRuntimeID, + MacroExpansion, +} from "../../../common/util/workflow-macro"; + +import { UndoRedoService } from "../undo-redo/undo-redo.service"; +import { ValidationWorkflowService } from "../validation/validation-workflow.service"; +import { FormBindingService } from "../form-binding/form-binding.service"; +import { DynamicSchemaService } from "../dynamic-schema/dynamic-schema.service"; + +describe("MacroNavigationService", () => { + let actions: WorkflowActionService; + let service: MacroNavigationService; + let sourceFor: MockInstance; + let persist: ReturnType; + let status: WorkflowStatusService; + let undo: UndoRedoService; + let events: Subject; + let saved: Map; + let revisions: Map; + let leaf: MacroDefinition; + let parent: MacroDefinition; + let root: Workflow; + let originalRoot: Workflow; + const sourceMacro = ( + id: "leaf" | "parent", + operators: OperatorPredicate[], + options: Parameters[2] = {} + ) => testMacro(id, operators, { name: id === "leaf" ? "Leaf" : "Parent", wid: id === "leaf" ? 20 : 10, ...options }); + const siblings = (definition: MacroDefinition, first: string, second: string, properties = {}) => [ + { ...macroInstance(definition, first), operatorProperties: properties }, + macroInstance(definition, second), + ]; + const expectEditablePin = (definition: MacroDefinition) => { + expect(actions.checkWorkflowModificationEnabled()).toBe(true); + expect(actions.getWorkflowContent().operators).toEqual(definition.content.operators); + expect(actions.openWorkflowDraft).toHaveBeenLastCalledWith(expect.anything(), false); + expect(service.readOnlyReason).toBe(""); + }; + const siblingRoot = (...dependencies: MacroDefinition[]) => + testWorkflow(siblings(parent, "selected", "other"), [], { + macroDependencies: [parent, ...dependencies], + }); + const pinned = (bundle: WorkflowContent, op: OperatorPredicate) => + bundle.macroDependencies!.find(d => macroType(d) === op.operatorType)!; + const unavailableRevision = (head?: MacroDefinition) => + sourceFor.mockResolvedValue({ + metadata: saved.get(10)!, + head, + revisions: head ? [head.version] : [], + }); + const expectNoSave = () => { + expect(persist.persistWorkflow).not.toHaveBeenCalled(); + expect(persist.persistMacroPath).not.toHaveBeenCalled(); + }; + const editScan = (tableName: string) => actions.setOperatorProperty(mockScanPredicate.operatorID, { tableName }); + const scanLeaves = (tableName?: string): [string[], OperatorPredicate][] => { + const scan = leaf.content.operators[0]; + const edited = tableName === undefined ? scan : { ...scan, operatorProperties: { tableName } }; + return ["selected", "other"].flatMap(instance => + ["child", "sibling"].map((child): [string[], OperatorPredicate] => [ + [instance, child, scan.operatorID], + instance === "selected" && child === "child" ? edited : scan, + ]) + ); + }; + const cacheExpansion = (content: WorkflowContent, leaves = scanLeaves(), links: WorkflowContent["links"] = []) => { + const response = expansionFixture(content, leaves, links); + TestBed.inject(MacroExpansionService).remember(content, response); + return response; + }; + const completedScan: OperatorRuntimeStatus = { + operatorState: OperatorState.Completed, + aggregatedInputRowCount: 0, + aggregatedOutputRowCount: 4, + numWorkers: 1, + }; + const publishStatistics = ( + expanded: MacroExpansion, + sample = (_path: string[]): OperatorRuntimeStatus => completedScan + ) => { + const leaves = Object.fromEntries(Object.entries(expanded.paths).map(([id, path]) => [id, sample(path)])); + status.setExternalStatus( + { ...leaves, selected: { ...completedScan, numWorkers: 2 }, other: completedScan }, + root.content + ); + }; + const recordedExecution = async (samples: Record, eid: number) => { + vi.mocked(TestBed.inject(WorkflowExecutionsService).retrieveMacroContent).mockReturnValue(of(root.content)); + status.setExternalStatus(samples, eid); + await vi.waitFor(() => expect(status.getRuntimeContent()).toEqual(root.content)); + }; + const openPath = async (...ids: string[]) => { + for (const id of ids) await service.open(id); + }; + const bodyTypes = (content: WorkflowContent) => + content.operators.filter(op => !isMacroTerminal(op)).map(op => op.operatorType); + const editNestedScan = async (tableName: string) => { + await openPath("selected", "child"); + editScan(tableName); + }; + const expectRejectedSave = async (reason: string, head: number) => { + const before = structuredClone(saved); + await editNestedScan("draft"); + persist.persistMacroPath.mockReturnValueOnce(throwError(() => new Error(reason))); + await expect(service.save()).rejects.toThrow(reason); + const [steps] = persist.persistMacroPath.mock.lastCall!; + expect(steps[0].head?.version).toBe(head); + expect(steps.map(step => step.workflow.wid)).toEqual([20, 10, 1]); + expect(saved).toEqual(before); + expect(actions.getWorkflowContent().operators[0].operatorProperties.tableName).toBe("draft"); + }; + const startPendingSave = async () => { + const release = new Subject(); + const write = persist.persistMacroPath.getMockImplementation()!; + persist.persistMacroPath.mockImplementationOnce((steps, rootBefore) => + release.pipe( + take(1), + switchMap(() => write(steps, rootBefore)) + ) + ); + const nextWrite = persist.persistMacroPath.mock.calls.length + 1; + const saving = service.save(); + await vi.waitFor(() => expect(persist.persistMacroPath).toHaveBeenCalledTimes(nextWrite)); + return { saving, release: () => release.next() }; + }; + const install = (content = root.content) => { + root = { ...root, content }; + originalRoot = structuredClone(root); + for (const { wid, name, content } of [leaf, parent]) saved.set(wid!, { ...DEFAULT_WORKFLOW, wid, name, content }); + saved.set(1, root); + actions.reloadWorkflow(structuredClone(root), false); + }; + const forwardedSettings = (output = false) => { + leaf = sourceMacro("leaf", [mockScanPredicate], { + formBinding: testBinding(mockScanPredicate.operatorID, "tableName", "table", "Table"), + outputs: output ? [testPort("output-0", mockScanPredicate.operatorID)] : [], + }); + parent = sourceMacro("parent", siblings(leaf, "child", "sibling", { table: "child override" }), { + macroDependencies: [leaf], + formBinding: testBinding("child", "table", "forward", "Forward"), + outputs: output ? [testPort("output-0", "child")] : [], + }); + const usages = siblings(parent, "selected", "other", { forward: "root override" }); + if (output) usages.push(mockResultPredicate); + const links = output + ? [testLink("selected", mockResultPredicate.operatorID, "output-0", "input-0", "connection")] + : []; + install( + testWorkflow(usages, links, { + macroDependencies: [parent, leaf], + formBinding: testBinding("selected", "forward", "form", "Table"), + }) + ); + }; + const saveRevision = (workflow: Workflow) => { + const wid = workflow.wid!; + const version = (revisions.get(wid) ?? 1) + 1; + revisions.set(wid, version); + saved.set(wid, structuredClone(workflow)); + return { version, revision: version, content: workflow.content }; + }; + const persistence = () => ({ + retrieveWorkflow: vi.fn((id: number) => of(structuredClone(saved.get(id)))), + persistWorkflow: vi.fn((w: Workflow) => { + saveRevision(w); + return of(w); + }), + persistMacroPath: vi.fn((steps: MacroSaveStep[], _rootBefore: Workflow) => { + let child: MacroDefinition | undefined; + return of( + steps.map(({ workflow, operatorID }) => { + const content = child ? replaceMacroVersion(workflow.content, operatorID!, child) : workflow.content; + const receipt = saveRevision({ ...workflow, content }); + if (content.macroInterface) + child = { ...receipt, id: content.macroInterface.id, wid: workflow.wid, name: workflow.name }; + return receipt; + }) + ); + }), + }); + beforeEach(() => { + events = new Subject(); + saved = new Map(); + revisions = new Map(); + persist = persistence(); + TestBed.configureTestingModule({ + providers: [ + ...macroWorkspaceTestProviders, + { provide: WorkflowPersistService, useValue: persist }, + { provide: WorkflowExecutionsService, useValue: { retrieveMacroContent: vi.fn() } }, + { provide: ComputingUnitStatusService, useValue: { getSelectedComputingUnitValue: () => undefined } }, + WorkflowStatusService, + { provide: WorkflowWebsocketService, useValue: { websocketEvent: () => events } }, + ], + }); + actions = TestBed.inject(WorkflowActionService); + actions.suspendWorkflow().shared.wsProvider.disconnect(); + sourceFor = vi.spyOn(TestBed.inject(MacroService), "sourceFor").mockImplementation(async definition => { + const { content, ...metadata } = saved.get(definition.wid!)!; + const version = revisions.get(definition.wid!) ?? 1; + return { + metadata, + revisions: [version], + head: { ...definition, name: metadata.name, version, revision: version, content: structuredClone(content) }, + }; + }); + TestBed.inject(DynamicSchemaService); + const open = actions.openWorkflowDraft.bind(actions); + vi.spyOn(actions, "openWorkflowDraft").mockImplementation(workflow => open(workflow, false)); + leaf = sourceMacro("leaf", [mockScanPredicate]); + parent = sourceMacro("parent", siblings(leaf, "child", "sibling"), { + macroDependencies: [leaf], + }); + root = { ...DEFAULT_WORKFLOW, wid: 1, name: "Root", content: siblingRoot(leaf) }; + install(); + service = TestBed.inject(MacroNavigationService); + status = TestBed.inject(WorkflowStatusService); + undo = TestBed.inject(UndoRedoService); + }); + afterEach(() => { + service.clear(); + actions.destroySharedModel(); + expect(root).toEqual(originalRoot); // Every edit must preserve the submitted graph and all its pins. + }); + + it.each(["property", "interface"] as const)("saves a nested %s edit along only the selected path", async edit => { + const rootDoc = actions.suspendWorkflow().shared; + await openPath("selected", "child"); + expect(service.parents.map(p => p.definition.name)).toEqual(["Parent", "Leaf"]); + expectNoSave(); + if (edit === "property") { + editScan("changed"); + await service.save(); + expect(saved.get(20)!.content.operators[0].operatorProperties.tableName).toBe("changed"); + } else { + setMacroGraph(actions, { + ...actions.getMacroInterface()!, + outputs: [testPort("new-output", mockScanPredicate.operatorID, "output-0")], + }); + actions.setFormBinding(testBinding(mockScanPredicate.operatorID, "tableName", "table", "Table")); + } + await service.back(0); + const latest = saved.get(1)!.content; + expect(actions.suspendWorkflow().shared).toBe(rootDoc); + expect(actions.getWorkflowContent()).toEqual(latest); + expect(service.parents).toHaveLength(0); + expect(actions.getWorkflowMetadata().wid).toBe(1); + expect(bodyTypes(latest)).toEqual(["WorkflowMacro:parent:2", macroType(parent)]); + const nextParent = pinned(latest, latest.operators[0]); + expect(bodyTypes(nextParent.content)).toEqual(["WorkflowMacro:leaf:2", macroType(leaf)]); + if (edit === "interface") { + const nextLeaf = pinned(latest, nextParent.content.operators[0]); + expect(readMacroInterface(nextLeaf.content)!.outputs[0].portID).toBe("new-output"); + expect(nextParent.content.operators[0].outputPorts[0].portID).toBe("new-output"); + expect(nextLeaf.content.formBinding?.fields[0].id).toBe("table"); + } + }); + + it("propagates removed nested ports and parameters, validates the parent, and undoes the entire adoption", async () => { + forwardedSettings(true); + const validation = TestBed.inject(ValidationWorkflowService); + const forms = TestBed.inject(FormBindingService); + const before = structuredClone(actions.getWorkflowContent()); + + expect(validation.validateOperator(mockResultPredicate.operatorID).isValid).toBe(true); + expect(forms.resolveFields()[0].brokenReason).toBeUndefined(); + await openPath("selected", "child"); + setMacroGraph(actions, { ...actions.getMacroInterface()!, outputs: [] }); + actions.setFormBinding({ fields: [] }); + await service.back(1); + expect(actions.getMacroInterface()!.outputs).toEqual([{ portID: "output-0", targets: [] }]); + expect(actions.getFormBinding()).toEqual({ fields: [] }); + expect(actions.getWorkflowContent().operators[0].operatorProperties).toEqual({}); + expect(actions.getWorkflowContent().operators[1].operatorType).toBe(macroType(leaf)); + await service.back(0); + const changed = actions.getWorkflowContent(); + expect(changed.links).toEqual(before.links); + + expect(changed.operators[0].operatorProperties).toEqual({}); + expect(changed.operators[1].operatorType).toBe(macroType(parent)); + expect(validation.validateOperator(mockResultPredicate.operatorID)).toEqual({ isValid: true }); + expect(forms.resolveFields()).toEqual([]); + + undo.undoAction(); + expect(actions.getWorkflowContent()).toEqual(before); + expect(validation.validateOperator(mockResultPredicate.operatorID).isValid).toBe(true); + expect(forms.resolveFields()[0].brokenReason).toBeUndefined(); + undo.redoAction(); + expect(actions.getWorkflowContent()).toEqual(changed); + + expect(readMacroInterface(saved.get(20)!.content)!.outputs).toEqual([]); + }); + + it.each([false, true])( + "restores forwarding after nested autosave and Undo/Redo (awaiting sync: %s)", + async pendingSync => { + forwardedSettings(); + + await openPath("selected", "child"); + if (pendingSync) + for (const { session } of service.parents) { + session.shared.wsProvider.shouldConnect = true; + session.shared.wsProvider.synced = false; + } + const forms = TestBed.inject(FormBindingService); + actions.bundleMacroActions(() => forms.removeBinding("table")); + await service.save(); + expect(saved.get(10)!.content.formBinding).toEqual({ fields: [] }); + expect(saved.get(1)!.content.formBinding).toEqual({ fields: [] }); + + editScan("edited after removal"); + await service.save(); + undo.undoAction(); + await service.save(); + expect(forms.getConfig().fields).toEqual([]); + undo.undoAction(); + expect(forms.getConfig()).toEqual(leaf.content.formBinding); + await service.save(); + expect(saved.get(10)!.content.formBinding).toEqual(parent.content.formBinding); + expect(saved.get(1)!.content.formBinding).toEqual(root.content.formBinding); + expect(saved.get(1)!.content.operators[0].operatorProperties).toEqual({ forward: "root override" }); + undo.redoAction(); + await service.save(); + expect(saved.get(1)!.content.formBinding).toEqual({ fields: [] }); + undo.undoAction(); + await service.save(); + await service.back(0); + expect(forms.resolveFields()[0]).toMatchObject({ value: "root override" }); + expect(forms.resolveFields()[0].brokenReason).toBeUndefined(); + } + ); + + it("preserves forwarding when a field removal is undone while its save is in flight", async () => { + forwardedSettings(); + + await openPath("selected", "child"); + actions.bundleMacroActions(() => TestBed.inject(FormBindingService).removeBinding("table")); + const { saving, release } = await startPendingSave(); + undo.undoAction(); + release(); + await saving; + await service.save(); + await service.back(0); + expect(actions.getFormBinding()).toEqual(root.content.formBinding); + expect(TestBed.inject(FormBindingService).resolveFields()[0]).toMatchObject({ value: "root override" }); + expect(actions.getWorkflowContent().operators[1]).toEqual(root.content.operators[1]); + }); + + it("keeps incompatible retained values visible to ordinary validation and Form View", async () => { + leaf = sourceMacro("leaf", [mockScanPredicate, mockResultPredicate], { + links: [testLink("1", "3", "output-0", "input-0", "internal")], + formBinding: testBinding("1", "tableName", "parameter", "Parameter"), + }); + install( + testWorkflow([{ ...macroInstance(leaf, "selected"), operatorProperties: { parameter: "保留" } }], [], { + macroDependencies: [leaf], + formBinding: testBinding("selected", "parameter", "form", "Parameter"), + }) + ); + const validation = TestBed.inject(ValidationWorkflowService); + const forms = TestBed.inject(FormBindingService); + expect(validation.validateOperator("selected").isValid).toBe(true); + await service.open("selected"); + actions.setFormBinding(testBinding("3", "limit", "parameter", "Limit")); + await service.back(0); + expect(actions.getWorkflowContent().operators[0].operatorProperties).toEqual({ parameter: "保留" }); + expect(validation.validateOperator("selected")).toEqual({ isValid: false, messages: { type: "must be integer" } }); + expect(forms.resolveFields()[0]).toMatchObject({ value: "保留", schema: { type: "integer" } }); + forms.writeValue(forms.getConfig().fields[0], 7); + expect(validation.validateOperator("selected").isValid).toBe(true); + }); + + it("joins the source document with its own dependency bundle and current name", async () => { + saved.set(10, { ...saved.get(10)!, name: "Renamed definition" }); + await service.open("selected"); + expect(actions.getMacroDependencies().map(d => d.id)).toEqual(["leaf"]); + expect(actions.getWorkflow().name).toBe("Renamed definition"); + expect(service.hasUnsavedChanges).toBe(false); + await service.back(0); + expectNoSave(); + expect(actions.getMacroDependencies()).toHaveLength(2); + }); + + it("repeatedly saves and undoes an earlier self-revision without changing other pins", async () => { + const earlier = sourceMacro("parent", [mockScanPredicate], { name: "Leaf" }); + parent = sourceMacro( + "parent", + [{ ...macroInstance(earlier, "inner"), customDisplayName: "Macro3" }, macroInstance(earlier, "sibling")], + { version: 4, macroDependencies: [earlier] } + ); + revisions.set(10, 4); + install(siblingRoot(earlier)); + + const before = structuredClone(actions.getWorkflowContent()); + + await service.open("selected"); + expect(actions.checkWorkflowModificationEnabled()).toBe(true); + await service.open("inner"); + expect(service.current!.instanceName).toBe("Macro3"); + expectEditablePin(earlier); + await service.save(); + expectNoSave(); + + editScan("旧版 café 東京"); + undo.undoAction(); + expect(actions.getWorkflowContent().operators[0].operatorProperties).toEqual(mockScanPredicate.operatorProperties); + undo.redoAction(); + await service.save(); + expect(service.current!.definition.version).toBe(5); + expect(revisions.get(10)).toBe(6); + expect(bodyTypes(saved.get(10)!.content)).toEqual(["WorkflowMacro:parent:5", "WorkflowMacro:parent:1"]); + expect(bodyTypes(saved.get(1)!.content)).toEqual(["WorkflowMacro:parent:6", "WorkflowMacro:parent:4"]); + editScan("second edit"); + await service.save(); + expect(service.current!.definition.version).toBe(7); + expect(revisions.get(10)).toBe(8); + await service.back(1); + const outer = structuredClone(actions.getWorkflowContent()); + expect(bodyTypes(outer)).toEqual(["WorkflowMacro:parent:7", "WorkflowMacro:parent:1"]); + undo.undoAction(); + expect(actions.getWorkflowContent().operators[0].operatorType).toBe("WorkflowMacro:parent:5"); + undo.redoAction(); + expect(actions.getWorkflowContent()).toEqual(outer); + await service.back(0); + const after = structuredClone(actions.getWorkflowContent()); + expect(bodyTypes(after)).toEqual(["WorkflowMacro:parent:8", "WorkflowMacro:parent:4"]); + undo.undoAction(); + undo.undoAction(); + expect(actions.getWorkflowContent()).toEqual(before); + undo.redoAction(); + undo.redoAction(); + expect(actions.getWorkflowContent()).toEqual(after); + }); + + it("opens the exact older pin without saving, then forks it in the source history and preserves sibling usages", async () => { + root.content.operators[0] = { ...root.content.operators[0], customDisplayName: "Outer alias" }; + parent.content.operators[0] = { ...parent.content.operators[0], customDisplayName: "Macro3" }; + leaf.revision = 1; + install(); + revisions.set(20, 9); + const latest = structuredClone(saved.get(20)!); + latest.content.operators[0] = { ...mockScanPredicate, operatorProperties: { tableName: "latest source" } }; + saved.set(20, latest); + + await openPath("selected", "child"); + expect(service.breadcrumb(0)).toBe("Root"); + expect(service.breadcrumb(1)).toBe("Outer alias · Revision 1"); + expect(service.current!.instanceName).toBe("Macro3"); + expect(service.hint).toContain("Revision 1"); + expectEditablePin(leaf); + await service.save(); + expectNoSave(); + + await service.back(0); + expect(service.readOnlyReason).toBe(""); + await openPath("selected", "child"); + editScan("from old revision"); + await service.back(0); + const updated = saved.get(1)!; + const latestParent = pinned(updated.content, updated.content.operators[0]); + expect(bodyTypes(latestParent.content)).toEqual(["WorkflowMacro:leaf:10", "WorkflowMacro:leaf:1"]); + expect(updated.content.operators[1].operatorType).toBe(macroType(parent)); + expect(saved.get(20)!.content.operators[0].operatorProperties.tableName).toBe("from old revision"); + }); + + it.each([false, true])( + "edits a self-pin from its directly opened source, recovering a failed parent save: %s", + async failParent => { + const source: Workflow = { + ...saved.get(20)!, + content: sourceMacro("leaf", [macroInstance(leaf, "inner")], { macroDependencies: [leaf] }).content, + }; + saved.set(20, source); + revisions.set(20, 4); + actions.reloadWorkflow(source, false); + const before = structuredClone(actions.getWorkflowContent()); + + await service.open("inner"); + expectEditablePin(leaf); + editScan("edited self"); + if (failParent) { + persist.persistMacroPath.mockReturnValueOnce(throwError(() => new Error("parent save failed"))); + await expect(service.save()).rejects.toThrow("parent save failed"); + } + await service.back(0); + const after = structuredClone(actions.getWorkflowContent()); + expect(after).toEqual(saved.get(20)!.content); + expect(after.operators[0].operatorType).not.toBe(macroType(leaf)); + + undo.undoAction(); + expect(actions.getWorkflowContent()).toEqual(before); + undo.redoAction(); + expect(actions.getWorkflowContent()).toEqual(after); + } + ); + + it("creates new revisions through an indirect self-path A4 → B3 → A1 without changing other usages", async () => { + const earlier = sourceMacro("parent", [mockScanPredicate], { name: "Leaf" }); + leaf = sourceMacro("leaf", [macroInstance(earlier, "inner")], { version: 3, macroDependencies: [earlier] }); + parent = sourceMacro("parent", [macroInstance(leaf, "child")], { + version: 4, + macroDependencies: [leaf, earlier], + }); + revisions.set(10, 4); + revisions.set(20, 3); + install(siblingRoot(leaf, earlier)); + + await openPath("selected", "child", "inner"); + expectNoSave(); + editScan("indirect edit"); + await service.back(0); + expect(revisions.get(10)).toBe(6); + expect(revisions.get(20)).toBe(4); + expect(saved.get(10)!.content.operators[0].operatorType).toBe("WorkflowMacro:leaf:4"); + expect(saved.get(20)!.content.operators[0].operatorType).toBe("WorkflowMacro:parent:5"); + expect(bodyTypes(actions.getWorkflowContent())).toEqual(["WorkflowMacro:parent:6", "WorkflowMacro:parent:4"]); + }); + + it.each([ + ["read-only source", /read-only access/i, () => saved.set(10, { ...saved.get(10)!, readonly: true })], + ["missing source", /source unavailable/i, () => saved.delete(10)], + [ + "foreign source", + /source unavailable/i, + () => sourceFor.mockRejectedValue(new Error("Macro source identity differs")), + ], + ["read-only parent", /parent workflow/i, () => actions.disableWorkflowModification()], + ["missing history", /could not be verified/i, () => unavailableRevision()], + ["foreign revision", /could not be verified/i, () => unavailableRevision(testMacro("foreign", [], { version: 2 }))], + ] as const)("opens the pinned graph for a %s and leaves it unchanged", async (_reason, message, changeAccess) => { + const before = structuredClone(actions.getWorkflowContent()); + changeAccess(); + await service.open("selected"); + expect(actions.checkWorkflowModificationEnabled()).toBe(false); + expect(service.readOnlyReason).toMatch(message); + expect(actions.getWorkflowContent().macroInterface!.id).toBe("parent"); + await service.save(); + expectNoSave(); + await service.back(0); + expect(actions.getWorkflowContent()).toEqual(before); + }); + + it.each(["inputs", "outputs"] as const)( + "saves unfinished %s and leaves nested drafts without changing other pins", + async direction => { + await openPath("selected", "child"); + setMacroGraph(actions, { + ...actions.getMacroInterface()!, + [direction]: [{ portID: "pending", displayName: "Pending", targets: [] }], + }); + await service.back(0); + const updated = actions.getWorkflowContent(); + expect(updated.operators[1].operatorType).toBe(macroType(parent)); + expect(saved.get(10)!.content.operators[1].operatorType).toBe(macroType(leaf)); + expect(readMacroInterface(saved.get(20)!.content)![direction][0].targets).toEqual([]); + + await openPath("selected", "child"); + expect(actions.getMacroInterface()![direction][0].displayName).toBe("Pending"); + await service.back(0); + + undo.undoAction(); + expect(actions.getWorkflowContent().operators[0].operatorType).toBe(macroType(parent)); + } + ); + + it.each(["parent", "child"])( + "lets users edit a disabled %s path without inheriting sibling statistics", + async disabled => { + const body = disabled === "parent" ? root.content : parent.content; + body.operators[0] = { ...body.operators[0], isDisabled: true }; + install(); + const run = cacheExpansion( + root.content, + scanLeaves().filter(([path]) => (disabled === "parent" ? path[0] !== "selected" : path[1] !== "child")) + ); + publishStatistics(run); + await openPath("other", "sibling"); + const active = service.compilationContext(run); + expect(status.getCurrentStatistics()[mockScanPredicate.operatorID]?.aggregatedOutputRowCount).toBe(4); + await service.back(0); + await service.open("selected"); + if (disabled === "parent") expect(service.compilationContext(run).ids).toEqual({}); + await service.open("child"); + const context = service.compilationContext(run); + expect(context.local.content.operators).toEqual([]); + expect(context.inputLinks(mockScanPredicate.operatorID)).toEqual([]); + expect(TestBed.inject(MacroExpansionService).get(context.content)).toBe(active.local); + expect(status.getCurrentStatistics()).toEqual({}); + expect(service.resultAliases()[mockScanPredicate.operatorID]).toBeNull(); + expect(actions.checkWorkflowModificationEnabled()).toBe(true); + editScan("edited while disabled"); + expect(actions.getWorkflowContent().operators[0].operatorProperties.tableName).toBe("edited while disabled"); + } + ); + + it("does not roll old worker counts onto a root usage after its input changes", () => { + forwardedSettings(true); + const run = cacheExpansion(actions.getWorkflowContent()); + publishStatistics(run); + + expect(status.getCurrentStatistics()["selected"]?.numWorkers).toBe(2); + actions.setOperatorProperty("selected", { forward: "another table" }); + cacheExpansion(actions.getWorkflowContent(), scanLeaves("another table")); + publishStatistics(run); + expect(status.getCurrentStatistics()["selected"]).toBeUndefined(); + }); + + it("hides macro results during an incomplete root graph edit", () => { + status.setExternalStatus({}, root.content); + vi.spyOn(actions, "getWorkflowContent").mockReturnValue({ + ...root.content, + links: [testLink("removed", "selected")], + }); + expect(service.resultAliases()).toEqual({ selected: null, other: null }); + }); + + it("resolves each pinned output to its occurrence before and after reopening", async () => { + const scan = mockScanPredicate; + leaf = sourceMacro("leaf", [scan], { outputs: [testPort("rows", scan.operatorID, "output-0")] }); + parent = sourceMacro("parent", siblings(leaf, "child", "sibling"), { + macroDependencies: [leaf], + outputs: [testPort("left", "child", "rows"), testPort("right", "sibling", "rows")], + }); + install(siblingRoot(leaf)); + const compiled = cacheExpansion(root.content); + const targets = (instance: string) => + ["child", "sibling"].map(child => ({ + operatorID: macroRuntimeID([instance, child], scan.operatorID), + portIndex: 0, + })); + compiled.resultPorts = { selected: targets("selected"), other: targets("other") }; + compiled.resultTargets = Object.fromEntries( + Object.entries(compiled.resultPorts).map(([id, ports]) => [id, ports[0]!]) + ); + const check = () => { + expect(service.resultAliases().selected).toEqual(targets("selected")[0]); + expect(service.resultAliases(1)).toEqual({ selected: targets("selected")[1], other: targets("other")[1] }); + expect(service.resultAliases(2).selected).toBeNull(); + }; + status.setRuntimeContent(root.content); + check(); + status.clearStatus(); + await recordedExecution({}, 42); + check(); + }); + + it.each([false, true])( + "reopens the exact execution and ignores slower responses (invalid draft: %s)", + async invalidDraft => { + await openPath("selected", "child"); + const content = root.content; + const pending = new Subject(); + const api = vi.mocked(TestBed.inject(WorkflowExecutionsService).retrieveMacroContent); + api.mockReturnValueOnce(pending).mockReturnValueOnce(of(content)); + cacheExpansion(content); + if (invalidDraft) { + const compiler = TestBed.inject(MacroExpansionService); + const expand = compiler.expand.bind(compiler); + vi.spyOn(compiler, "expand").mockImplementation(content => + content.macroInterface ? Promise.reject(new Error("Invalid draft")) : expand(content) + ); + } else cacheExpansion(actions.getWorkflowContent()); + + status.setExternalStatus({}, 18); + status.setExternalStatus({}, 19); + await vi.waitFor(() => expect(status.getRuntimeContent()).toEqual(content)); + pending.next({ ...content, operators: [] }); + pending.complete(); + await Promise.resolve(); + expect(status.getRuntimeContent()).toEqual(content); + expect(api.mock.calls).toEqual([ + [1, 18], + [1, 19], + ]); + expect(status.runtimeExecutionId).toBe(19); + } + ); + + it("projects live statistics onto the selected nested instance and clears them as its draft changes", async () => { + const expanded = cacheExpansion(root.content); + publishStatistics(expanded, path => ({ + ...completedScan, + operatorState: OperatorState.Running, + aggregatedOutputRowCount: path[0] === "other" ? 100 : path[1] === "sibling" ? 7 : 4, + })); + await openPath("selected", "child"); + service.compilationContext(expanded); + expect(status.getCurrentStatistics()[mockScanPredicate.operatorID].aggregatedOutputRowCount).toBe(4); + expect(Object.keys(status.getCurrentStatistics())).toEqual([mockScanPredicate.operatorID]); + editScan("draft"); + await vi.waitFor(() => expect(status.getCurrentStatistics()).toEqual({})); + await service.back(0, true); + service.compilationContext(expanded); + expect(status.getCurrentStatistics()).toHaveProperty("selected"); + publishStatistics(expanded, () => ({ ...completedScan, aggregatedInputRowCount: 1 })); + await editNestedScan("new revision"); + await service.save(); + await service.back(0); + await openPath("selected", "child"); + expect(status.getCurrentStatistics()).toEqual({}); + await service.back(0); + status.clearStatus(); + await service.open("selected"); + expect(status.getCurrentStatistics()).toEqual({}); + }); + + it("keeps live statistics for an explicitly stored default but clears a real ancestor override", async () => { + leaf = sourceMacro("leaf", [{ ...mockScanPredicate, operatorProperties: { tableName: "default" } }], { + formBinding: testBinding(mockScanPredicate.operatorID, "tableName", "table", "Table"), + }); + parent.content = { + ...parent.content, + macroDependencies: [leaf], + formBinding: testBinding("child", "table", "table", "Table"), + }; + install(siblingRoot(leaf)); + const expanded = cacheExpansion(root.content); + publishStatistics(expanded); + await openPath("selected", "child"); + service.compilationContext(expanded); + const usage = service.parents[0].session.shared.operatorIDMap.get("selected")!; + const override = (value: string) => { + updateYTypeFromObject(usage, { ...usage.toJSON(), operatorProperties: { table: value } }); + service.compilationContext(cacheExpansion(service.compilationDraft()!.content, scanLeaves(value))); + status.refresh(); + }; + override("default"); + expect(status.getCurrentStatistics()[mockScanPredicate.operatorID]?.aggregatedOutputRowCount).toBe(4); + usage.set("isDisabled", true); + status.refresh(); + expect(status.getCurrentStatistics()).toEqual({}); + usage.set("isDisabled", false); + status.refresh(); + expect(status.getCurrentStatistics()[mockScanPredicate.operatorID]?.aggregatedOutputRowCount).toBe(4); + override("changed"); + expect(status.getCurrentStatistics()).toEqual({}); + }); + + it.each([ + [1, "A newer saved revision exists"], + [9, "A newer saved revision exists"], + [9, "The parent is read-only"], + ] as const)("keeps the draft from revision %s after the server rejects it: %s", async (head, reason) => { + revisions.set(20, head); + const published = actions.getWorkflowContent(); + await expectRejectedSave(reason, head); + await service.back(0, true); + expect(actions.getWorkflowContent()).toEqual(published); + }); + + it("retains a failed propagation for retry without rewriting earlier snapshots", async () => { + await expectRejectedSave("parent save failed", 1); + expect(saved.get(1)).toEqual(root); + expect(revisions.has(10)).toBe(false); + expect(revisions.has(20)).toBe(false); + await service.save(); + await service.back(0); + expect(actions.getWorkflowContent().operators[0].operatorType).not.toBe(macroType(parent)); + expect(actions.getWorkflowContent().operators[1].operatorType).toBe(macroType(parent)); + const updated = actions.getWorkflowContent(); + expect(pinned(updated, updated.operators[1]).content).toEqual({ ...parent.content, macroDependencies: undefined }); + }); + + it("waits for an in-flight save before discarding a draft and restoring its parent", async () => { + await editNestedScan("saving"); + const { saving, release } = await startPendingSave(); + const leaving = service.back(0, true); + await Promise.resolve(); + expect(service.parents).toHaveLength(2); + release(); + await saving; + await leaving; + expect(service.parents).toHaveLength(0); + expect(actions.getWorkflowMetadata().wid).toBe(1); + expect(actions.getWorkflowContent()).toEqual(saved.get(1)!.content); + }); + + it("uses the committed root for a subsequent save and preserves a rename queued during it", async () => { + await openPath("selected", "child"); + const commit = persist.persistMacroPath.getMockImplementation()!; + // A server-side layout change is part of the saved baseline even while its canvas is suspended. + persist.persistMacroPath.mockImplementationOnce((steps, before) => { + const committed = structuredClone(steps); + committed.at(-1)!.workflow.content.operatorPositions.selected = { x: 123, y: 456 }; + return commit(committed, before); + }); + editScan("first"); + await service.save(); + const committed = saved.get(root.wid!)!; + editScan("second"); + const { saving, release } = await startPendingSave(); + expect(persist.persistMacroPath.mock.lastCall![1]).toEqual(committed); + actions.setWorkflowName("Renamed while saving"); + const queued = service.save(); + release(); + await saving; + await queued; + expect(saved.get(20)!.name).toBe("Renamed while saving"); + }); + + it("keeps a collaborator's rename and deletion at different levels during a nested save", async () => { + await editNestedScan("changed"); + const { saving, release } = await startPendingSave(); + const session = service.parents[0].session; + const intermediate = service.parents[1].session; + intermediate.shared.operatorIDMap.get("child")!.set("customDisplayName", new Text("Renamed while saving")); + session.shared.operatorIDMap.delete("selected"); + release(); + await saving; + expect(actions.getSessionWorkflow(session).content.operators).toEqual([root.content.operators[1]]); + expect(actions.getSessionWorkflow(intermediate).content.operators[0]).toMatchObject({ + operatorType: "WorkflowMacro:leaf:2", + customDisplayName: "Renamed while saving", + }); + }); + + it("compiles an edited body in its parent context and maps results and boundary inputs to that instance", async () => { + const pins = actions.getMacroDependencies(); + await openPath("selected", "child"); + expect(service.compilationDraft()!.drafts).toEqual([]); + const operator = { ...mockSentimentPredicate, inputPorts: [{ portID: "custom-input" }] }; + actions.addOperatorsAndLinks( + [{ op: operator, pos: { x: 50, y: 30 } }], + [testLink(mockScanPredicate.operatorID, operator.operatorID, "output-0", "custom-input", "inside")] + ); + const leaves = [ + ...scanLeaves(), + [["selected", "child", operator.operatorID], operator] as [string[], OperatorPredicate], + ]; + const preview = service.compilationDraft()!; + expect(preview.drafts).toEqual([{ path: ["selected", "child"], content: actions.getWorkflowContent() }]); + expect(preview.content.macroDependencies).toEqual(pins); + const compiled = cacheExpansion(preview.content, leaves, [ + testLink( + macroRuntimeID(["selected", "child"], mockScanPredicate.operatorID), + macroRuntimeID(["selected", "child"], operator.operatorID), + "output-0", + "custom-input", + "inside" + ), + ]); + compiled.inspectors[JSON.stringify(["selected", "child"])].inputs[operator.operatorID] = [ + { link: 0, input: 0, output: 0 }, + ]; + const context = service.compilationContext(compiled); + expect(context.root.content.operators).toHaveLength(5); + expect(context.inputLinks(operator.operatorID)).toMatchObject([ + { + source: { operatorID: context.ids[mockScanPredicate.operatorID], portID: "output-0" }, + target: { operatorID: operator.operatorID, portID: "custom-input", portIndex: 0 }, + }, + ]); + expect(service.resultAliases()[operator.operatorID]).toBeNull(); + + expectNoSave(); + await service.save(); + TestBed.inject(MacroExpansionService).remember(service.rootWorkflow.content, compiled); + status.setRuntimeContent(service.rootWorkflow.content); + expect(service.resultAliases()[operator.operatorID]).toEqual({ + operatorID: context.ids[operator.operatorID], + portIndex: 0, + }); + }); + it("preserves visible native port IDs in standalone macro compilation contexts", () => { + const source = { ...mockScanPredicate, outputPorts: [{ portID: "output-0" }, { portID: "output-2" }] }; + const target = { ...mockSentimentPredicate, inputPorts: [{ portID: "input-0" }, { portID: "input-2" }] }; + const content = { + ...leaf.content, + operators: [...leaf.content.operators.filter(isMacroTerminal), source, target], + operatorPositions: { ...leaf.content.operatorPositions, [target.operatorID]: { x: 200, y: 30 } }, + links: [testLink(source.operatorID, target.operatorID, "output-2", "input-2", "native-gap")], + }; + actions.reloadWorkflow({ ...root, content }, false); + const before = JSON.stringify(actions.getWorkflowContent()); + const compiled = cacheExpansion( + content, + [ + [[source.operatorID], source], + [[target.operatorID], target], + ], + content.links + ); + compiled.inspectors["[]"].inputs[target.operatorID] = [{ link: 0, input: 1, output: 1 }]; + const context = service.compilationContext(compiled); + expect(context.inputLinks(target.operatorID)).toMatchObject([ + { + ...content.links[0], + source: { ...content.links[0].source, portIndex: 1 }, + target: { ...content.links[0].target, portIndex: 1 }, + }, + ]); + expect(JSON.stringify(actions.getWorkflowContent())).toBe(before); + }); + it("keeps nested terminal regions and live statistics through layout, Undo/Redo, save and reopening", async () => { + const sentiment = { ...mockSentimentPredicate, operatorVersion: "Nlp1" }; + leaf = sourceMacro("leaf", [sentiment], { + inputs: [testPort("rows", sentiment.operatorID, "input-0")], + outputs: [testPort("result", sentiment.operatorID, "output-0")], + }); + parent = sourceMacro("parent", [macroInstance(leaf, "child")], { + macroDependencies: [leaf], + inputs: [testPort("rows", "child")], + outputs: [testPort("result", "child")], + }); + install( + testWorkflow( + [mockScanPredicate, macroInstance(parent, "selected"), mockResultPredicate], + [ + testLink(mockScanPredicate.operatorID, "selected", "output-0", "rows", "in"), + testLink("selected", mockResultPredicate.operatorID, "result", "input-0", "out"), + ], + { macroDependencies: [parent, leaf] } + ) + ); + const scope = ["selected", "child"]; + const nativeID = macroRuntimeID(scope, sentiment.operatorID); + const compiled = cacheExpansion( + root.content, + [ + [[mockScanPredicate.operatorID], mockScanPredicate], + [["selected", "child", sentiment.operatorID], sentiment], + [[mockResultPredicate.operatorID], mockResultPredicate], + ], + [testLink(mockScanPredicate.operatorID, nativeID), testLink(nativeID, mockResultPredicate.operatorID)] + ); + const terminalLinks = [ + { direction: "inputs" as const, portID: "rows", links: [0] }, + { direction: "outputs" as const, portID: "result", links: [1] }, + ]; + const innerView = compiled.inspectors[JSON.stringify(scope)]; + const [input, output] = leaf.content.operators.filter(isMacroTerminal); + for (const terminal of [input, output]) + innerView.ids[terminal.operatorID] = macroRuntimeID(scope, terminal.operatorID); + innerView.terminals = terminalLinks; + compiled.inspectors[JSON.stringify(["selected"])].terminals = terminalLinks; + const inspect = async (id: string) => { + await service.open(id); + service.compilationContext(compiled); + return actions.runtimeBoundaryLinks!(); + }; + status.setRuntimeContent(root.content); + + const outer = await inspect("selected"); + expect(outer).toEqual( + terminalLinks.map(({ direction, portID, links }) => ({ + terminal: { direction, portID }, + links: links.map(index => compiled.content.links[index]), + })) + ); + expect(await inspect("child")).toEqual(outer); + expect(actions.getWorkflowContent()).toEqual(leaf.content); + const checkEditedRegion = () => { + actions.setOperatorProperty(sentiment.operatorID, { attribute: "edited" }); + expect(actions.runtimeBoundaryLinks!()).toEqual([]); + undo.undoAction(); + expect(actions.runtimeBoundaryLinks!()).toEqual(outer); + }; + checkEditedRegion(); + const usage = { ...completedScan, aggregatedOutputRowCount: 3, outputPortMetrics: { "0": 3 } }; + const samples = { + [nativeID]: usage, + [innerView.ids[input.operatorID]]: { ...usage, numWorkers: 0 }, + [innerView.ids[output.operatorID]]: { + ...usage, + numWorkers: 0, + inputPortMetrics: { "0": 3 }, + }, + [macroRuntimeID(["other", "child"], output.operatorID)]: { ...usage, inputPortMetrics: { "0": 99 } }, + }; + events.next({ type: "OperatorStatisticsUpdateEvent", operatorStatistics: samples }); + const statistics = status.getCurrentStatistics(); + expect(statistics[mockSentimentPredicate.operatorID].aggregatedOutputRowCount).toBe(3); + expect(statistics[input.operatorID].outputPortMetrics).toEqual({ "0": 3 }); + expect(statistics[output.operatorID].inputPortMetrics).toEqual({ "0": 3 }); + const aliases = service.resultAliases(); + const expectRuntime = () => { + expect(actions.runtimeBoundaryLinks!()).toEqual(outer); + expect(status.getCurrentStatistics()).toEqual(statistics); + expect(service.resultAliases()).toEqual(aliases); + }; + actions.autoLayoutWorkflow(); + status.refresh(); + expectRuntime(); + + undo.undoAction(); + expectRuntime(); + undo.redoAction(); + expectRuntime(); + await service.save(); + expectRuntime(); + await service.back(0); + status.clearStatus(); + await recordedExecution(samples, 42); + await inspect("selected"); + await inspect("child"); + expectRuntime(); + checkEditedRegion(); + }); +}); diff --git a/frontend/src/app/workspace/service/macro/macro-navigation.service.ts b/frontend/src/app/workspace/service/macro/macro-navigation.service.ts new file mode 100644 index 00000000000..9065aa131e6 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-navigation.service.ts @@ -0,0 +1,498 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Injectable } from "@angular/core"; +import { firstValueFrom, Subject, merge } from "rxjs"; +import { auditTime } from "rxjs/operators"; +import { isEqual } from "lodash-es"; +import { ResultTarget, toLogicalGraph } from "@texera/workflow-macros"; +import { WorkflowMetadata } from "../../../dashboard/type/workflow-metadata.interface"; +import { Workflow, WorkflowContent } from "../../../common/type/workflow"; +import { + MacroSaveStep, + WorkflowPersistService, +} from "../../../common/service/workflow-persist/workflow-persist.service"; +import { + MacroDefinition, + MacroExpansion, + isMacro, + readMacroInterface, + retargetMacro, + projectMacroExpansion, + compiledInputLinks, + macroType, + replaceMacroVersion, + macroRevisionLabel, + mergeMacroDefinitions, +} from "../../../common/util/workflow-macro"; +import { + WorkflowActionService, + WorkflowEditorSession, + DEFAULT_WORKFLOW, +} from "../workflow-graph/model/workflow-action.service"; +import { MacroService } from "./macro.service"; +import { WorkflowStatusService } from "../workflow-status/workflow-status.service"; +import { OperatorRuntimeStatus } from "../../types/execute-workflow.interface"; +import { OperatorPredicate } from "../../types/workflow-common.interface"; + +import { WorkflowExecutionsService } from "../../../dashboard/service/user/workflow-executions/workflow-executions.service"; + +import { MacroExpansionService } from "./macro-expansion.service"; + +function runtimePlan(graph: WorkflowContent) { + return { + ...toLogicalGraph(graph), + disabled: graph.operators.filter(op => op.isDisabled).map(op => op.operatorID), + }; +} + +function macroHead({ name, description }: WorkflowMetadata, version: number) { + return { name, description, version }; +} + +export interface MacroDraftRequest { + content: WorkflowContent; + drafts: { path: string[]; content: WorkflowContent }[]; +} + +interface MacroNavigationEntry { + session: WorkflowEditorSession; + operatorID: string; + instanceName: string; + head?: MacroSaveStep["head"]; + definition: MacroDefinition; + source?: Pick; + saved: Workflow; +} + +/** A breadcrumb is an editor document, not another canvas or another kind of version history. */ +@Injectable({ providedIn: "root" }) +export class MacroNavigationService { + public readonly parents: MacroNavigationEntry[] = []; + public busy = false; + public dirty = false; + public readonly changed$ = new Subject(); + public get changing$(): Subject { + return this.actions.editorChanging$; + } + private generation = 0; + private rootBefore?: Workflow; + private readonly compiledOverrides = new WeakMap(); + private attemptedRuntimeEpoch = -1; + private saving: Promise = Promise.resolve(); + + constructor( + public actions: WorkflowActionService, + private macros: MacroService, + private persist: WorkflowPersistService, + private status: WorkflowStatusService, + private executions: WorkflowExecutionsService, + private expansion: MacroExpansionService + ) { + actions.executionWorkflow = () => this.rootWorkflow; + actions.runtimeOperatorIDs = () => (this.parents.length ? this.runtimeContext()?.ids ?? {} : undefined); + actions.runtimeBoundaryLinks = () => this.boundaryLinks(); + status.setProjection( + raw => this.projectStatistics(raw), + () => this.rootWorkflow.content + ); + status.getStatisticsUpdateStream().subscribe(() => this.loadRuntimeSnapshot()); + expansion.changed$.subscribe(() => status.refresh()); + merge(actions.workflowChanged(), actions.workflowMetaDataChanged()) + .pipe(auditTime(0)) + .subscribe(() => { + if (!actions.getJointGraphWrapper().getReloadingWorkflow()) { + this.dirty = this.hasUnsavedChanges; + if (this.parents.length || this.rootWorkflow.content.operators.some(isMacro)) status.refresh(); + } + }); + } + + private get path(): string[] { + return this.parents.map(parent => parent.operatorID); + } + public get rootWorkflow(): Workflow { + return this.parents.length ? this.actions.getSessionWorkflow(this.parents[0].session) : this.actions.getWorkflow(); + } + public get hasUnsavedChanges(): boolean { + return !!this.current && this.draftChanged(this.actions.getWorkflow(), this.current.saved); + } + private draftChanged(current: Workflow, saved: Workflow): boolean { + return ( + !current.readonly && + (!isEqual(current.content, saved.content) || + current.name !== saved.name || + current.description !== saved.description) + ); + } + public get current(): MacroNavigationEntry | undefined { + return this.parents.at(-1); + } + public breadcrumb(index: number): string { + const entered = this.parents[index - 1]; + return entered + ? `${entered.instanceName} · ${macroRevisionLabel(entered.definition)}` + : this.parents[0]?.session.metadata.name ?? ""; + } + public get readOnlyReason(): string { + const entry = this.current; + if (!entry?.saved.readonly) return ""; + if (!entry.source) return "Source unavailable. This saved snapshot still runs and exports with its workflow."; + if (entry.source.readonly) return "You have read-only access to this macro definition."; + if (!entry.session.modifiable) return "The parent workflow is read-only or currently locked."; + return "The latest source revision could not be verified. This saved snapshot remains read-only."; + } + public get hint(): string { + if (!this.parents.length) return ""; + if (this.readOnlyReason) return this.readOnlyReason; + if (!this.actions.checkWorkflowModificationEnabled()) return "Editing is currently locked."; + return `Editing ${macroRevisionLabel(this.current!.definition)}. Saving creates new revisions along this parent path; other usages keep their pins.`; + } + + public async open(operatorID: string): Promise { + if (this.busy) return; + this.busy = true; + const generation = this.generation; + try { + this.changing$.next(); + if (this.parents.length) await this.save(); + else if (this.actions.checkWorkflowModificationEnabled()) { + const current = this.actions.getWorkflow(); + const stored = current.wid ? await firstValueFrom(this.persist.retrieveWorkflow(current.wid)) : undefined; + this.rootBefore = stored; + if (!stored) { + this.rootBefore = await firstValueFrom(this.persist.persistWorkflow(current)); + this.actions.setWorkflowMetadata(this.rootBefore); + } + } + const workflow = this.actions.getWorkflow(); + const shared = this.actions.editorDocument; + const op = workflow.content.operators.find(o => o.operatorID === operatorID); + const definition = workflow.content.macroDependencies?.find(d => macroType(d) === op?.operatorType); + if (!definition) throw new Error("Missing pinned macro definition"); + if (this.parents.length >= 32) throw new Error("Macro nesting exceeds 32 levels"); + let source: WorkflowMetadata | undefined, latest: MacroDefinition | undefined; + try { + const details = await this.macros.sourceFor(definition, true); + source = details.metadata; + latest = details.head && this.macros.checkPin(details.head); + } catch { + // The pinned snapshot can still open when its source or revision is unavailable. + } + const session = this.actions.suspendWorkflow(); + if ( + generation !== this.generation || + session.shared !== shared || + !isEqual(this.actions.getWorkflowContent(), workflow.content) + ) + throw new Error("The workflow changed while opening the macro. Open it again."); + const atLatest = latest?.version === definition.version; + const editable = session.modifiable && !source?.readonly && latest?.id === definition.id; + const { macroDependencies: dependencies, ...content } = + editable && atLatest + ? latest!.content + : { ...definition.content, macroDependencies: this.actions.getMacroDependencies() }; + const draft: Workflow = { + ...DEFAULT_WORKFLOW, + ...source, + wid: source?.wid, + name: editable ? source!.name : definition.name, + readonly: !editable, + content: { + ...content, + ...(dependencies?.length ? { macroDependencies: mergeMacroDefinitions(dependencies) } : {}), + }, + }; + this.parents.push({ + session, + operatorID, + instanceName: op!.customDisplayName?.trim() || definition.name, + head: source && latest ? macroHead(source, latest.version) : undefined, + definition: atLatest ? { ...definition, revision: latest!.revision } : definition, + source: source && { wid: source.wid, readonly: source.readonly }, + saved: draft, + }); + this.dirty = false; + try { + this.actions.openWorkflowDraft(draft, editable && atLatest); + } catch (error) { + this.parents.pop(); + this.actions.resumeWorkflow(session); + throw error; + } + this.actions.disableWorkflowModification(); + if (editable) this.actions.enableWorkflowModification(); + this.status.refresh(); + this.actions.getTexeraGraph().triggerCenterEvent(); + this.changed$.next(); + } finally { + this.busy = false; + } + } + + /** The existing autosave/manual-save callers share one serialized chain, including parent propagation. */ + public save(): Promise { + const save = this.saving.catch(() => this.actions.getWorkflow()).then(() => this.saveCurrent()); + this.saving = save; + return save; + } + + private async saveCurrent(): Promise { + const workflow = this.actions.getWorkflow(); + if (!this.actions.checkWorkflowModificationEnabled() || workflow.readonly) return workflow; + if (!this.parents.length) return firstValueFrom(this.persist.persistWorkflow(workflow)); + const entry = this.parents.at(-1)!; + if (!this.draftChanged(workflow, entry.saved)) return workflow; + const undo = this.actions.captureMacroUndo(); + const generation = this.generation; + const path = [...this.parents].reverse(); + const steps: MacroSaveStep[] = [{ workflow, head: entry.head }]; + const originals: { content: WorkflowContent; type: string }[] = []; + for (const [i, parent] of path.entries()) { + const document = this.actions.getSessionWorkflow(parent.session); + const type = document.content.operators.find(op => op.operatorID === parent.operatorID)?.operatorType; + const pin = document.content.macroDependencies?.find(d => macroType(d) === type); + if (!pin || pin.id !== parent.definition.id) + throw new Error("The parent macro usage changed. Your draft is still open."); + const content = retargetMacro(document.content, parent.operatorID, { + ...pin, + content: steps[i].workflow.content, + }); + steps.push({ workflow: { ...document, content }, operatorID: parent.operatorID, head: path[i + 1]?.head }); + originals.push({ content: document.content, type: macroType(pin) }); + } + const receipts = await firstValueFrom(this.persist.persistMacroPath(steps, this.rootBefore!)); + if (generation !== this.generation) return workflow; + const committed = steps.map(({ workflow }, i) => ({ ...workflow, content: receipts[i].content })); + const heads = new Map(committed.map((w, i) => [w.wid, macroHead(w, receipts[i].version)])); + for (const [i, parent] of path.entries()) { + const { operatorID, session } = parent; + const original = originals[i]; + const saved = committed[i]; + parent.definition = { ...parent.definition, ...receipts[i], name: saved.name }; + parent.saved = saved; + parent.head = heads.get(saved.wid); + const live = this.actions.getWorkflowContent(session.shared); + if (live.operators.find(op => op.operatorID === operatorID)?.operatorType === original.type) { + const content = isEqual(live, original.content) + ? committed[i + 1].content + : replaceMacroVersion(live, operatorID, parent.definition); + this.actions.updateMacroUsage(operatorID, content, session, undo); + } + } + this.rootBefore = committed.at(-1)!; + this.status.refresh(); + return workflow; + } + + public compilationDraft(): MacroDraftRequest | undefined { + if (!this.parents.length) return; + const drafts: MacroDraftRequest["drafts"] = []; + let content = this.actions.getWorkflowContent(); + const path = this.path; + for (let i = this.parents.length - 1; i >= 0; i--) { + const parent = this.parents[i]; + const document = this.actions.getSessionWorkflow(parent.session).content; + if (isEqual(content, parent.saved.content)) { + content = document; + } else { + drafts.unshift({ path: path.slice(0, i + 1), content }); + content = retargetMacro(document, parent.operatorID, { ...parent.definition, content }); + } + } + return { content, drafts }; + } + + public resultAliases(portIndex = 0): Record { + const nested = this.parents.length > 0; + const current = this.actions.getWorkflowContent(); + const operators = nested ? current.operators : current.operators.filter(isMacro); + if (!operators.length) return {}; + const context = nested ? this.runtimeContext() : undefined; + const executed = !nested && this.matchingExecution(current); + const root = executed ? this.expansion.get(executed) : context?.root; + return Object.fromEntries( + operators.map(op => { + const id = nested ? context?.ids[op.operatorID] : op.operatorID; + if (!id || !root) return [op.operatorID, null]; + if (!isMacro(op)) return [op.operatorID, { operatorID: id, portIndex }]; + const target = portIndex === 0 ? root.resultTargets?.[id] : root.resultPorts?.[id]?.[portIndex]; + return [op.operatorID, target ?? null]; + }) + ); + } + + private boundaryLinks() { + const context = this.runtimeContext(); + if (!context) return []; + return context.inspector.terminals.map(({ direction, portID, links }) => ({ + terminal: { direction, portID }, + links: links.map(i => context.root.content.links[i]), + })); + } + + private parentOperators() { + return this.parents.map(({ session, operatorID }) => + this.actions.getSessionWorkflow(session).content.operators.find(op => op.operatorID === operatorID) + ); + } + + public parentOverrides(): unknown[] { + return this.parentOperators().map(op => op?.operatorProperties); + } + + public compilationContext( + root: MacroExpansion, + content = this.actions.getWorkflowContent(), + overrides = this.parentOverrides() + ) { + this.compiledOverrides.set(this.actions.editorDocument, overrides); + const path = this.path; + const { local, ids, inspector } = projectMacroExpansion(content, root, path); + if (inspector) this.expansion.remember(content, local); + const inputLinks = (id: string) => { + const op = content.operators.find(op => op.operatorID === id); + return op ? compiledInputLinks(root, path, op) : []; + }; + return { content, local, root, path, ids, inputLinks, doc: this.actions.editorDocument }; + } + + public async back(index: number, discard = false): Promise { + if (this.busy || index < 0 || index >= this.parents.length) return; + this.busy = true; + try { + this.changing$.next(); + if (discard) await this.saving.catch(() => undefined); + if (!discard) { + await this.save(); + // Keep edits made during a save; leave only after the visible draft has been saved too. + for (let retry = 0; this.hasUnsavedChanges; retry++) { + if (retry === 3) throw new Error("The draft is still changing. Save it before leaving."); + await this.save(); + } + } + const operatorID = this.parents[index].operatorID; + while (this.parents.length > index) this.actions.resumeWorkflow(this.parents.pop()!.session); + this.actions.highlightElements(false, operatorID); + this.status.refresh(); + this.changed$.next(); + } finally { + this.busy = false; + } + } + + public clear(): void { + this.generation++; + this.rootBefore = undefined; + this.attemptedRuntimeEpoch = -1; + this.dirty = false; + this.parents.splice(0).forEach(p => p.session.shared.destroy()); + this.status.refresh(); + } + + /** Resolve live and restored statistics against the graph submitted for that execution. */ + private async loadRuntimeSnapshot(): Promise { + const { runtimeEpoch: epoch, runtimeExecutionId: eid } = this.status; + if (!eid || epoch === this.attemptedRuntimeEpoch || this.status.getRuntimeContent()) return; + const root = this.rootWorkflow; + if (!root.wid || (!this.parents.length && !root.content.operators.some(isMacro))) return; + this.attemptedRuntimeEpoch = epoch; + try { + const content = await firstValueFrom(this.executions.retrieveMacroContent(root.wid, eid)); + if (epoch !== this.status.runtimeEpoch || this.rootWorkflow.wid !== root.wid) return; + this.status.setRuntimeContent(content); + this.status.refresh(); + await Promise.all([this.expansion.expand(content), this.expansion.expand(this.actions.getWorkflowContent())]); + } catch { + // Statistics need both the recorded execution and a valid projection of the editor graph. + } + } + + private runtimeContext() { + let content = this.status.getRuntimeContent(); + if (!this.current || !content) return; + try { + const runtime = this.expansion.get(content), + current = this.actions.getWorkflowContent(); + const expanded = this.expansion.get(current); + if (!runtime || !expanded) return; + const definitions = new Map(mergeMacroDefinitions(content.macroDependencies ?? []).map(d => [macroType(d), d])); + const ancestors = this.parentOperators(); + if (ancestors.some(op => !op || op.isDisabled)) return; + const overrides = ancestors.map(op => op?.operatorProperties); + let overridesChanged = false; + for (const [index, parent] of this.parents.entries()) { + const op: OperatorPredicate | undefined = content.operators.find(op => op.operatorID === parent.operatorID); + const definition: MacroDefinition | undefined = op && definitions.get(op.operatorType); + if (!definition || definition.id !== parent.definition.id) return; + overridesChanged ||= !isEqual(op?.operatorProperties, overrides[index]); + content = definition.content; + } + if (overridesChanged && !isEqual(overrides, this.compiledOverrides.get(this.actions.editorDocument))) return; + const projection = projectMacroExpansion(current, runtime, this.path); + if (!projection.inspector) return; + const usages = new Set(current.operators.filter(isMacro).map(op => op.operatorID)); + const recipe = (graph: MacroExpansion) => ({ + ...runtimePlan(graph.content), + ports: Object.fromEntries(Object.entries(graph.ports).filter(([key]) => usages.has(JSON.parse(key)[0]))), + }); + if ( + isEqual(readMacroInterface(current), readMacroInterface(content)) && + isEqual(recipe(expanded), recipe(projection.local)) + ) + return { ...projection, root: runtime }; + } catch { + return; + } + } + + private matchingExecution(current: WorkflowContent): WorkflowContent | undefined { + const executed = this.status.getRuntimeContent(); + try { + if (executed && isEqual(runtimePlan(current), runtimePlan(executed))) return executed; + } catch { + // A link can briefly outlive an endpoint during a collaborative edit. + } + } + + private projectStatistics(raw: Record): Record { + if (!this.parents.length) { + const current = this.actions.getWorkflowContent(); + if (!current.operators.some(isMacro)) return raw; + if (this.matchingExecution(current)) return raw; + return Object.fromEntries( + current.operators + .filter(op => !isMacro(op) && raw[op.operatorID]) + .map(op => [op.operatorID, raw[op.operatorID]]) + ); + } + const context = this.runtimeContext(); + if (this.actions.getTempWorkflow() || !context) return {}; + if ( + this.parents.some( + p => + this.actions.getSessionWorkflow(p.session).content.operators.find(op => op.operatorID === p.operatorID) + ?.operatorType !== macroType(p.definition) + ) + ) + return {}; + return Object.fromEntries( + Object.entries(context.ids).flatMap(([local, id]) => (raw[id] ? [[local, raw[id]]] : [])) + ); + } +} diff --git a/frontend/src/app/workspace/service/macro/macro-terminals.spec.ts b/frontend/src/app/workspace/service/macro/macro-terminals.spec.ts new file mode 100644 index 00000000000..c818b1553d3 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-terminals.spec.ts @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as Y from "yjs"; +import * as joint from "jointjs"; +import { TestBed } from "@angular/core/testing"; +import { WorkflowActionService, DEFAULT_WORKFLOW } from "../workflow-graph/model/workflow-action.service"; +import { UndoRedoService } from "../undo-redo/undo-redo.service"; +import { macroWorkspaceTestProviders } from "./macro-workspace.spec-harness"; +import { mockSentimentPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { WorkflowContent, exportedWorkflow } from "../../../common/type/workflow"; +import { extractMacro, readMacroInterface } from "../../../common/util/workflow-macro"; + +describe("Saved macro terminals", () => { + let actions: WorkflowActionService, undo: UndoRedoService, content: WorkflowContent; + const node = (input: boolean) => + actions + .getTexeraGraph() + .getAllOperators() + .find(op => op.operatorType === (input ? "MacroInput" : "MacroOutput"))!; + beforeEach(() => { + TestBed.configureTestingModule({ + providers: macroWorkspaceTestProviders, + }); + actions = TestBed.inject(WorkflowActionService); + actions.suspendWorkflow().shared.wsProvider.disconnect(); + undo = TestBed.inject(UndoRedoService); + content = extractMacro( + { + ...actions.getWorkflowContent(), + operators: ["a", "b"].map(operatorID => ({ ...mockSentimentPredicate, operatorID, operatorVersion: "Nlp1" })), + operatorPositions: { a: { x: 100, y: 100 }, b: { x: 300, y: 250 } }, + }, + ["a", "b"], + "macro" + ); + actions.reloadWorkflow({ ...DEFAULT_WORKFLOW, content }, false); + }); + afterEach(() => actions.destroySharedModel()); + + it("persists normal operators, links and positions through export and reopening", () => { + const saved = actions.getWorkflowContent(); + expect(saved.operators.filter(op => op.operatorType.startsWith("Macro")).length).toBe(2); + expect(actions.getTexeraGraph().hasOperator(node(true).operatorID)).toBe(true); + expect(saved.operatorPositions[node(false).operatorID]).toBeDefined(); + const exported = JSON.parse(JSON.stringify(exportedWorkflow(saved, undefined))); + actions.openWorkflowDraft({ ...DEFAULT_WORKFLOW, content: exported }); + expect(actions.getWorkflowContent()).toEqual(saved); + expect(readMacroInterface(exported)).toEqual(actions.getMacroInterface()); + }); + + for (const input of [true, false]) { + it(`removes the selected ${input ? "input" : "output"} port and its links, restoring both with Undo/Redo`, () => { + const terminal = node(input), + direction = input ? "outputPorts" : "inputPorts"; + const first = terminal[direction][0].portID; + actions.highlightPorts(false, { operatorID: terminal.operatorID, portID: first }); + const before = actions.getWorkflowContent(); + actions.removePort(terminal.operatorID, !input); + expect(node(input)[direction].some(p => p.portID === first)).toBe(false); + expect( + (actions.getJointGraph().getCell(terminal.operatorID) as joint.dia.Element).getPorts().some(p => p.id === first) + ).toBe(false); + expect(actions.getWorkflowContent().links.length).toBe(before.links.length - 1); + undo.undoAction(); + expect(actions.getWorkflowContent()).toEqual(before); + expect( + (actions.getJointGraph().getCell(terminal.operatorID) as joint.dia.Element) + .getPorts() + .filter(port => port.group === (input ? "out" : "in")) + .map(port => port.id) + ).toEqual(node(input)[direction].map(port => port.portID)); + undo.redoAction(); + expect(node(input)[direction].some(p => p.portID === first)).toBe(false); + undo.undoAction(); + actions.addPort(terminal.operatorID, !input); + const added = node(input)[direction].at(-1)!; + expect(added.portID).not.toBe(first); + expect(added.isDynamicPort).toBe(true); + }); + } + + it("uses the native shared port name editor and preserves wiring on rename", () => { + const terminal = node(true), + port = terminal.outputPorts[0]; + const before = actions.getWorkflowContent().links; + const record = actions + .getTexeraGraph() + .getSharedPortDescriptionType({ operatorID: terminal.operatorID, portID: port.portID })!; + if (!record.has("displayName")) record.set("displayName", new Y.Text()); + const name = record.get("displayName")!; + name.delete(0, name.length); + name.insert(0, "Rows 東京"); + expect(actions.getMacroInterface()?.inputs[0].displayName).toBe("Rows 東京"); + expect(actions.getWorkflowContent().links).toEqual(before); + }); + + it("exposes a chosen native port once through normal graph commands", () => { + const target = { operatorID: "a", portID: "input-0" }; + actions.macroTerminals.expose(target); + expect(actions.getMacroInterface()?.inputs.length).toBe(2); + actions.removePort(node(true).operatorID, false); + const count = actions.getMacroInterface()!.inputs.length; + actions.macroTerminals.expose(target, true); + expect(actions.getMacroInterface()?.inputs.length).toBe(count + 1); + undo.undoAction(); + expect(actions.getMacroInterface()?.inputs.length).toBe(count); + }); + + it("stores terminal layout in the normal undoable position map", () => { + const before = actions.getWorkflowContent().operatorPositions; + actions.autoLayoutWorkflow(); + const after = actions.getWorkflowContent().operatorPositions; + expect(after).not.toEqual(before); + expect(after[node(true).operatorID]).toEqual( + actions.getJointGraphWrapper().getElementPosition(node(true).operatorID) + ); + undo.undoAction(); + expect(actions.getWorkflowContent().operatorPositions).toEqual(before); + undo.redoAction(); + expect(actions.getWorkflowContent().operatorPositions).toEqual(after); + }); +}); diff --git a/frontend/src/app/workspace/service/macro/macro-terminals.ts b/frontend/src/app/workspace/service/macro/macro-terminals.ts new file mode 100644 index 00000000000..e6e8b374958 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-terminals.ts @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; +import { LogicalPort } from "../../types/workflow-common.interface"; +import { isMacroTerminal } from "../../../common/util/workflow-macro"; +import { v4 as uuid } from "uuid"; + +export interface MacroTerminal { + direction: "inputs" | "outputs"; + portID?: string; +} + +/** Expose a normal operator port using the same graph actions as dragging a connection. */ +export class MacroTerminals { + constructor(private readonly actions: WorkflowActionService) {} + + public direction(target: LogicalPort): MacroTerminal["direction"] | undefined { + if (!this.actions.getMacroInterface()) return undefined; + const op = this.actions.getTexeraGraph().getOperator(target.operatorID); + if (isMacroTerminal(op)) return undefined; + return op.inputPorts.some(port => port.portID === target.portID) ? "inputs" : "outputs"; + } + + public needsReplacement(target: LogicalPort): boolean { + return ( + this.direction(target) === "inputs" && + this.actions + .getTexeraGraph() + .getInputLinksByOperatorId(target.operatorID) + .some(link => link.target.portID === target.portID) + ); + } + + public expose(target: LogicalPort, replace = false): void { + const direction = this.direction(target); + if (!direction || !this.actions.checkWorkflowModificationEnabled()) return; + const input = direction === "inputs"; + const graph = this.actions.getTexeraGraph(); + const node = graph.getAllOperators().find(op => op.operatorType === (input ? "MacroInput" : "MacroOutput")); + if (!node) return; + const links = graph.getAllLinks(); + const existing = links.find(link => { + const end = input ? link.target : link.source; + const boundary = input ? link.source : link.target; + return ( + end.operatorID === target.operatorID && end.portID === target.portID && boundary.operatorID === node.operatorID + ); + }); + if (existing) { + this.actions.highlightPorts(false, input ? existing.source : existing.target); + return; + } + if (this.needsReplacement(target) && !replace) return; + this.actions.bundleMacroActions(() => { + if (input && replace) + links + .filter(link => link.target.operatorID === target.operatorID && link.target.portID === target.portID) + .forEach(link => this.actions.deleteLinkWithID(link.linkID)); + this.actions.addPort(node.operatorID, !input); + const portID = ( + input ? graph.getOperator(node.operatorID).outputPorts : graph.getOperator(node.operatorID).inputPorts + ).at(-1)!.portID; + const boundary = { operatorID: node.operatorID, portID }; + this.actions.addLink({ linkID: uuid(), source: input ? boundary : target, target: input ? target : boundary }); + this.actions.highlightPorts(false, boundary); + }); + } +} diff --git a/frontend/src/app/workspace/service/macro/macro-workspace.spec-harness.ts b/frontend/src/app/workspace/service/macro/macro-workspace.spec-harness.ts new file mode 100644 index 00000000000..658dbe3bb48 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro-workspace.spec-harness.ts @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { commonTestProviders } from "../../../common/testing/test-utils"; +import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; +import { StubOperatorMetadataService } from "../operator-metadata/stub-operator-metadata.service"; +import { HttpClient } from "@angular/common/http"; +import { throwError } from "rxjs"; +import { MacroExpansionService } from "./macro-expansion.service"; + +/** Unconfigured previews fail promptly while other services keep their own HTTP test backend. */ +export const macroWorkspaceTestProviders = [ + ...commonTestProviders, + { + provide: MacroExpansionService, + useFactory: () => + new MacroExpansionService( + new HttpClient({ + handle: () => throwError(() => new Error("No backend expansion response configured")), + }) + ), + }, + { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, +]; diff --git a/frontend/src/app/workspace/service/macro/macro.service.spec.ts b/frontend/src/app/workspace/service/macro/macro.service.spec.ts new file mode 100644 index 00000000000..39f1ddc9b66 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro.service.spec.ts @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { setMacroGraph, testMacro, testPort } from "../../../common/testing/macro-graph"; +import { testBinding, testWorkflow } from "../../../common/testing/workflow-graph"; + +import { editMacroGraph } from "../../../../../../common/workflow-core/src/test/typescript/macro-test-graph"; + +import { of, Subject, throwError } from "rxjs"; +import { MacroService } from "./macro.service"; +import { TestBed } from "@angular/core/testing"; +import { HttpTestingController } from "@angular/common/http/testing"; +import { commonTestImports } from "../../../common/testing/test-utils"; +import { AppSettings } from "../../../common/app-setting"; +import { macroWorkspaceTestProviders } from "./macro-workspace.spec-harness"; +import { WorkflowActionService, DEFAULT_WORKFLOW } from "../workflow-graph/model/workflow-action.service"; +import { WorkflowPersistService } from "../../../common/service/workflow-persist/workflow-persist.service"; +import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; +import { Workflow, WorkflowContent } from "../../../common/type/workflow"; +import { OperatorPredicate } from "../../types/workflow-common.interface"; +import { mockSentimentPredicate, mockResultPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { macroInstance, macroType, readMacroInterface } from "../../../common/util/workflow-macro"; +import { UndoRedoService } from "../undo-redo/undo-redo.service"; + +describe("MacroService", () => { + const op: OperatorPredicate = { + ...mockSentimentPredicate, + operatorID: "leaf", + operatorVersion: "Nlp1", + operatorProperties: { attribute: "original" }, + }; + let actions: WorkflowActionService, persist: any, versions: any, service: MacroService; + const content = () => actions.getWorkflowContent(); + const catalogRequest = () => + TestBed.inject(HttpTestingController).expectOne(`${AppSettings.getApiEndpoint()}/workflow/macros`); + const load = (content: WorkflowContent, wid = 1) => + actions.reloadWorkflow({ ...DEFAULT_WORKFLOW, wid, name: "Workflow", content }, false); + beforeEach(() => { + versions = { + pinMacro: vi.fn((wid: number, id: string) => { + const [content, name] = persist.createWorkflow.mock.lastCall; + return of({ wid, id, version: 5, revision: 1, content, name }); + }), + }; + TestBed.configureTestingModule({ + imports: commonTestImports, + providers: [...macroWorkspaceTestProviders, { provide: WorkflowVersionService, useValue: versions }], + }); + persist = TestBed.inject(WorkflowPersistService); + vi.spyOn(persist, "createWorkflow").mockImplementation((content, name) => + of({ workflow: { wid: 12, name, content } }) + ); + actions = TestBed.inject(WorkflowActionService); + actions.suspendWorkflow().shared.wsProvider.disconnect(); + service = TestBed.inject(MacroService); + load( + testWorkflow([op], [], { + operatorPositions: { leaf: { x: 25, y: 40 } }, + formBinding: testBinding("leaf", "attribute", "value", "Value"), + }) + ); + actions.getJointGraphWrapper().highlightOperators("leaf"); + vi.spyOn(actions, "deleteOperatorsAndLinks"); + }); + afterEach(() => actions.destroySharedModel()); + + it("creates an independent version before collapsing, preserving property bindings", async () => { + const d = await service.createFromSelection("Clean"); + expect(d.version).toBe(5); + expect(d.wid).toBe(12); + expect(persist.createWorkflow).toHaveBeenCalledOnce(); + expect(content().operators).toHaveLength(1); + expect(content().operators[0].operatorType).toBe(macroType(d)); + expect(content().formBinding!.fields[0].operatorID).toBe(content().operators[0].operatorID); + expect(content().formBinding!.fields[0].propertyKey).toBe("value"); + }); + + it("resolves the latest source on insertion while keeping existing usages and history ordinals", async () => { + const previous = await service.createFromSelection("Clean"); + versions.pinMacro.mockReturnValueOnce(of({ ...previous, version: 64, revision: 3 })); + const before = content(); + const latest = await service.latestFor(macroType(previous)); + expect(latest.version).toBe(64); + expect(latest.revision).toBe(3); + expect(versions.pinMacro).toHaveBeenCalledWith(12, previous.id, undefined); + versions.pinMacro.mockReturnValueOnce(of({ ...previous, revision: 2 })); + expect((await service.pin(previous, 5)).revision).toBe(2); + expect(versions.pinMacro).toHaveBeenCalledWith(12, previous.id, 5); + versions.pinMacro.mockReturnValueOnce(throwError(() => new Error("Macro source identity differs"))); + await expect(service.latestFor(macroType(previous))).rejects.toThrow(/source/i); + versions.pinMacro.mockReturnValueOnce(throwError(() => new Error("Macro revision is not in its source history"))); + await expect(service.pin(previous, 64)).rejects.toThrow("Macro revision is not in its source history"); + expect(content()).toEqual(before); + }); + + it("leaves the selection intact on save failure or while locked", async () => { + persist.createWorkflow.mockReturnValueOnce(throwError(() => new Error("save failed"))); + await expect(service.createFromSelection("Clean")).rejects.toThrow("save failed"); + versions.pinMacro.mockReturnValueOnce(throwError(() => new Error("Saved version is not a macro definition"))); + await expect(service.createFromSelection("Clean")).rejects.toThrow("Saved version is not a macro definition"); + expect(actions.deleteOperatorsAndLinks).not.toHaveBeenCalled(); + await expect(service.createFromSelection(" ")).rejects.toThrow(/name/); + actions.disableWorkflowModification(); + await expect(service.createFromSelection("Clean")).rejects.toThrow(/read.only/i); + }); + + it("remaps a parent macro's exposed ports when its operators are collapsed into a nested macro", async () => { + setMacroGraph(actions, { + id: "parent", + inputs: [testPort("in", "leaf", "input-0")], + outputs: [testPort("out", "leaf", "output-0")], + }); + actions.getJointGraphWrapper().highlightOperators("leaf"); + const nested = await service.createFromSelection("Nested"); + const instance = content().operators.find(op => op.operatorType === macroType(nested))!; + expect(actions.getMacroInterface()).toMatchObject({ + id: "parent", + inputs: [testPort("in", instance.operatorID, instance.inputPorts[0].portID)], + outputs: [testPort("out", instance.operatorID, instance.outputPorts[0].portID)], + }); + }); + + it.each(["unavailable", "ordinary workflow"])("discovers valid macros alongside a %s", async other => { + const created = await service.createFromSelection("Clean"); + const usages = [{ id: created.id, wid: 42, label: "Workflow / Clean", path: ["selected"] }]; + const errors = other === "unavailable" ? ["Other: not available"] : []; + const refreshed = service.refresh(); + expect(service.refresh()).toBe(refreshed); + catalogRequest().flush({ definitions: [created], usages, errors }); + if (other === "unavailable") await expect(refreshed).rejects.toThrow(/Other/); + else await expect(refreshed).resolves.toEqual([created]); + expect(service.library$.value).toEqual([created]); + expect(service.usages$.value).toEqual(usages); + TestBed.inject(HttpTestingController).expectNone(req => req.url.includes("/workflow/")); + }); + + it.each(["edit", "workflow"])("does not collapse after a concurrent %s during creation", async change => { + const response = new Subject<{ workflow: Workflow }>(); + persist.createWorkflow.mockReturnValue(response); + const creating = service.createFromSelection("Clean"); + await vi.waitFor(() => expect(persist.createWorkflow).toHaveBeenCalledOnce()); + const saved = persist.createWorkflow.mock.calls[0][0]; + if (change === "edit") actions.setOperatorProperty("leaf", { attribute: "new edit" }); + else load(content(), 2); + vi.mocked(actions.deleteOperatorsAndLinks).mockClear(); // Ignore the explicit reload's normal cleanup. + response.next({ workflow: { wid: 12, name: "Clean", content: saved } as Workflow }); + response.complete(); + await expect(creating).rejects.toThrow(/changed/i); + expect(actions.deleteOperatorsAndLinks).not.toHaveBeenCalled(); + if (change === "edit") expect(content().operators[0].operatorProperties.attribute).toBe("new edit"); + else expect(content().operators).toEqual([op]); + }); + + it("forks a portable definition into a newly versioned editable copy", async () => { + const original = await service.createFromSelection("Clean"); + const copy = await service.copyDefinition(original, []); + expect(copy.id).not.toBe(original.id); + expect(copy.content.operators).toEqual(original.content.operators); + expect(copy.content.macroInterface?.id).toBe(copy.id); + expect(service.library$.value).toContain(copy); + expect(original.content.macroInterface?.id).toBe(original.id); + }); + + it.each(["inputs", "outputs"] as const)( + "pins, adopts and copies unfinished %s without making them runnable", + async direction => { + const original = await service.createFromSelection("Clean"); + const draft = structuredClone(original.content); + editMacroGraph(draft, boundary => { + boundary[direction] = [{ portID: "pending", targets: [] }]; + }); + versions.pinMacro.mockReturnValueOnce(of({ ...original, version: 6, revision: 2, content: draft })); + const latest = await service.latestFor(macroType(original)); + expect(latest.version).toBe(6); + await service.adoptVersion(content().operators[0].operatorID, latest); + expect(content().operators[0].operatorType).toBe(macroType(latest)); + + const copy = await service.copyDefinition(latest, []); + expect(readMacroInterface(copy.content)![direction][0].targets).toEqual([]); + expect(copy.id).not.toBe(original.id); + expect(readMacroInterface(original.content)![direction].every(p => p.targets.length > 0)).toBe(true); + } + ); + + it("adopts changed schemas without discarding retained overrides, leaving validation to Texera", async () => { + const previous = await service.createFromSelection("Clean"); + const latest = { ...previous, version: 6 }; + const id = content().operators[0].operatorID; + const editor = actions.getJointGraphWrapper(); + editor.highlightOperators(id); + actions.setOperatorProperty(id, { value: "mine" }); + const before = content(); + const moved = vi.fn(); + actions.suspendWorkflow().shared.elementPositionMap.observe(moved); + service.adoptVersion(id, latest); + expect(moved).not.toHaveBeenCalled(); + expect(editor.getCurrentHighlightedOperatorIDs()).toContain(id); + const adopted = content(); + const undo = TestBed.inject(UndoRedoService); + undo.undoAction(); + expect(content()).toEqual(before); + undo.redoAction(); + expect(content()).toEqual(adopted); + expect(content().operators[0].operatorType).toBe(macroType(latest)); + expect(content().operators[0].operatorProperties).toEqual({ value: "mine" }); + const next = structuredClone(latest); + next.version = 7; + next.content.operators[0] = { ...mockResultPredicate, operatorID: "leaf" }; + next.content = { ...next.content, formBinding: testBinding("leaf", "limit", "value", "Value") }; + service.adoptVersion(content().operators[0].operatorID, next); + expect(content().operators[0].operatorType).toBe(macroType(next)); + expect(content().operators[0].operatorProperties).toEqual({ value: "mine" }); + }); + + it("adopts a version with its bundled nested definitions while preserving overrides", async () => { + const previous = await service.createFromSelection("Clean"); + actions.setOperatorProperty(content().operators[0].operatorID, { value: "mine" }); + const { inputs, outputs } = readMacroInterface(previous.content)!; + const child = testMacro("child", [op], { + inputs, + outputs, + formBinding: testBinding("leaf", "attribute", "forward", "Value"), + }); + const latest = testMacro(previous.id, [macroInstance(child, "leaf")], { + version: 6, + inputs, + outputs, + macroDependencies: [child], + formBinding: testBinding("leaf", "forward", "value", "Value"), + }); + service.adoptVersion(content().operators[0].operatorID, latest); + expect(content().operators[0].operatorType).toBe(macroType(latest)); + expect(content().operators[0].operatorProperties).toEqual({ value: "mine" }); + expect(content().macroDependencies!.find(d => d.id === child.id)!.content.formBinding).toEqual( + child.content.formBinding + ); + }); + + it("rejects a conflicting local revision on adoption and on library refresh", async () => { + const previous = await service.createFromSelection("Clean"); + const conflict = structuredClone(previous); + conflict.content.operators[0] = { ...op, operatorProperties: { attribute: "different deployment" } }; + const before = JSON.stringify(content()); + expect(() => service.adoptVersion(content().operators[0].operatorID, conflict)).toThrow( + /Conflicting macro version/ + ); + expect(JSON.stringify(content())).toBe(before); + service.library$.next([]); + const refreshed = service.refresh(); + catalogRequest().flush({ definitions: [conflict], usages: [], errors: [] }); + await expect(refreshed).rejects.toThrow(/Conflicting macro version/); + expect(service.library$.value).toEqual([]); + expect(content().macroDependencies).toEqual([previous]); + }); + it("stores repeated creation as a flat bundle while preserving the saved bodies", async () => { + const savedBodies = new Map(); + const bundleSizes: number[] = []; + for (let depth = 1; depth <= 12; depth++) { + actions.getJointGraphWrapper().highlightOperators(...content().operators.map(op => op.operatorID)); + const definition = await service.createFromSelection(`Nested ${depth}`); + const submitted: WorkflowContent = persist.createWorkflow.mock.lastCall[0]; + const { macroDependencies = [], ...body } = structuredClone(submitted); + expect(macroDependencies.every(d => !d.content.macroDependencies)).toBe(true); + savedBodies.set(definition.id, body); + const dependencies = content().macroDependencies!; + expect(dependencies.every(d => !d.content.macroDependencies)).toBe(true); + expect(new Map(dependencies.map(d => [d.id, d.content]))).toEqual(savedBodies); + bundleSizes.push(JSON.stringify(content()).length); + } + expect(bundleSizes[7]).toBeLessThan(20000); + expect(bundleSizes[11]).toBeLessThan(bundleSizes[7] * 1.6); + }); +}); diff --git a/frontend/src/app/workspace/service/macro/macro.service.ts b/frontend/src/app/workspace/service/macro/macro.service.ts new file mode 100644 index 00000000000..1afed84679b --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro.service.ts @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Injectable } from "@angular/core"; +import { BehaviorSubject, firstValueFrom, Subject } from "rxjs"; +import { v4 as uuid } from "uuid"; +import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; +import { + MacroCatalog, + WorkflowPersistService, +} from "../../../common/service/workflow-persist/workflow-persist.service"; +import { + MacroSource, + WorkflowVersionService, +} from "../../../dashboard/service/user/workflow-version/workflow-version.service"; +import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; +import { WorkflowContent } from "../../../common/type/workflow"; +import { + extractMacro, + readMacroInterface, + isMacroTerminal, + MacroDefinition, + macroInstance, + macroType, + mergeMacroDefinitions, + replaceMacroVersion, +} from "../../../common/util/workflow-macro"; +import { LogicalPort } from "../../types/workflow-common.interface"; + +@Injectable({ providedIn: "root" }) +export class MacroService { + public readonly library$ = new BehaviorSubject([]); + public readonly usages$ = new BehaviorSubject([]); + public readonly inspect$ = new Subject(); + private refreshing?: Promise; + + constructor( + private actions: WorkflowActionService, + private persist: WorkflowPersistService, + private versions: WorkflowVersionService, + private metadata: OperatorMetadataService + ) {} + + public refresh(): Promise { + return (this.refreshing ??= this.loadLibrary().finally(() => { + this.refreshing = undefined; + })); + } + + private async loadLibrary(): Promise { + const catalog = await firstValueFrom(this.persist.retrieveMacroCatalog()); + const errors = [...catalog.errors]; + const definitions = catalog.definitions.filter(definition => { + try { + this.checkPin(definition); + return true; + } catch (error) { + errors.push(`${definition.name}: ${error instanceof Error ? error.message : String(error)}`); + return false; + } + }); + this.metadata.registerMacros(definitions.flatMap(d => [d, ...(d.content.macroDependencies ?? [])])); + this.library$.next(definitions); + this.usages$.next(catalog.usages); + if (errors.length) + throw new Error(`Could not load library entries: ${errors.join(", ")}. Fix them or refresh to retry.`); + return definitions; + } + + public checkPin(definition: MacroDefinition): MacroDefinition { + mergeMacroDefinitions(this.actions.getMacroDependencies(), [definition]); + return definition; + } + + public async pin(source?: Pick, version?: number): Promise { + if (!source?.wid) throw new Error("Macro source unavailable; refresh the library before inserting it"); + return this.checkPin(await firstValueFrom(this.versions.pinMacro(source.wid, source.id, version))); + } + + public async copyDefinition( + definition: Pick, + dependencies: MacroDefinition[] + ): Promise { + return this.saveDefinition( + { + ...definition.content, + macroDependencies: mergeMacroDefinitions(dependencies), + macroInterface: { ...definition.content.macroInterface!, id: uuid() }, + }, + definition.name + " copy" + ); + } + + public async sourceFor(definition: MacroDefinition, head = false): Promise { + if (!definition.wid) throw new Error("Macro source unavailable"); + return firstValueFrom(this.versions.macroSource(definition.wid, definition.id, head)); + } + + public async latestFor(type: string): Promise { + const latest = await this.pin(this.metadata.getMacroDefinition(type)); + this.metadata.registerMacros([latest, ...(latest.content.macroDependencies ?? [])]); + return latest; + } + + private async saveDefinition(content: WorkflowContent, name: string): Promise { + const created = await firstValueFrom(this.persist.createWorkflow(content, name)); + const definition = await this.pin({ id: content.macroInterface!.id, wid: created.workflow.wid }); + this.metadata.registerMacros([definition, ...(definition.content.macroDependencies ?? [])]); + this.library$.next([...this.library$.value, definition]); + return definition; + } + + public async createFromSelection(name: string): Promise { + if (!this.actions.checkWorkflowModificationEnabled()) throw new Error("Workflow is read-only"); + if (!name.trim()) throw new Error("Enter a macro name"); + const content = this.actions.getWorkflowContent(); + const before = this.snapshot(content); + const selected = this.actions + .getJointGraphWrapper() + .getCurrentHighlightedOperatorIDs() + .filter(id => !isMacroTerminal(this.actions.getTexeraGraph().getOperator(id))); + const body = extractMacro(content, selected, uuid()); + const d = await this.saveDefinition(body, name.trim()); + if (!this.unchanged(before)) { + throw new Error("The workflow changed while saving. The macro was saved; add it from the Macros panel."); + } + const instance = macroInstance(d, `WorkflowMacro-${uuid()}`); + const ids = new Set(selected); + const exposed = readMacroInterface(body)!; + const boundary = (p: LogicalPort, direction: "inputs" | "outputs"): LogicalPort => { + if (!ids.has(p.operatorID)) return p; + const port = exposed[direction].find(b => + b.targets.some(t => t.operatorID === p.operatorID && t.portID === p.portID) + ); + if (!port) throw new Error("Missing boundary mapping"); + return { operatorID: instance.operatorID, portID: port.portID }; + }; + const links = content.links + .filter(l => ids.has(l.source.operatorID) !== ids.has(l.target.operatorID)) + .map(l => ({ ...l, source: boundary(l.source, "outputs"), target: boundary(l.target, "inputs") })); + this.actions.bundleMacroActions(() => { + this.actions.setMacroDependencies([...this.actions.getMacroDependencies(), d]); + this.actions.deleteOperatorsAndLinks(selected); + this.actions.addOperatorsAndLinks([{ op: instance, pos: content.operatorPositions[selected[0]] }], links); + if (content.formBinding) + this.actions.setFormBinding({ + ...content.formBinding, + fields: content.formBinding.fields.map(f => + ids.has(f.operatorID) ? { ...f, operatorID: instance.operatorID, propertyKey: f.id } : f + ), + ...(content.formBinding.shownResultIds === undefined + ? {} + : { + shownResultIds: [ + ...new Set(content.formBinding.shownResultIds.map(id => (ids.has(id) ? instance.operatorID : id))), + ], + }), + }); + }); + return d; + } + + public adoptVersion(operatorID: string, latest: MacroDefinition): void { + if (!this.actions.checkWorkflowModificationEnabled()) throw new Error("Workflow is read-only"); + const content = this.actions.getWorkflowContent(); + const result = replaceMacroVersion(content, operatorID, latest); + if (this.actions.getTexeraGraph().getOperator(operatorID).operatorType !== macroType(latest)) { + this.actions.updateMacroUsage(operatorID, result); + } + } + + private snapshot(content = this.actions.getWorkflowContent()): string { + return JSON.stringify([this.actions.getWorkflowMetadata().wid, content]); + } + + private unchanged(snapshot: string): boolean { + return this.actions.checkWorkflowModificationEnabled() && this.snapshot() === snapshot; + } +} diff --git a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts index 987ddf47e69..209e5a35381 100644 --- a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts +++ b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts @@ -16,16 +16,16 @@ * specific language governing permissions and limitations * under the License. */ +import { testMacro, expansionFixture } from "../../../common/testing/macro-graph"; +import { testWorkflow } from "../../../common/testing/workflow-graph"; +import { macroWorkspaceTestProviders } from "../macro/macro-workspace.spec-harness"; import { TestBed } from "@angular/core/testing"; -import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; -import { StubOperatorMetadataService } from "../operator-metadata/stub-operator-metadata.service"; import { OperatorMenuService } from "./operator-menu.service"; import { HttpClientTestingModule } from "@angular/common/http/testing"; import { ComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/computing-unit-status.service"; import { MockComputingUnitStatusService } from "../../../common/service/computing-unit/computing-unit-status/mock-computing-unit-status.service"; -import { commonTestProviders } from "../../../common/testing/test-utils"; import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; import { mockCommentBox, @@ -39,6 +39,9 @@ import { import { CommentBox } from "../../types/workflow-common.interface"; import { NotificationService } from "../../../common/service/notification/notification.service"; import { ExecuteWorkflowService } from "../execute-workflow/execute-workflow.service"; +import { macroInstance } from "../../../common/util/workflow-macro"; + +import { MacroExpansionService } from "../macro/macro-expansion.service"; import { Subscription } from "rxjs"; describe("OperatorMenuService", () => { @@ -51,7 +54,6 @@ describe("OperatorMenuService", () => { beforeEach(() => { TestBed.configureTestingModule({ providers: [ - { provide: OperatorMetadataService, useClass: StubOperatorMetadataService }, { provide: ComputingUnitStatusService, useClass: MockComputingUnitStatusService }, // Stub the two collaborators OperatorMenuService only calls into (executeWorkflow, // error/info toasts) so the spec doesn't boot the real execution/notification stacks @@ -61,7 +63,7 @@ describe("OperatorMenuService", () => { provide: NotificationService, useValue: { error: vi.fn(), info: vi.fn(), success: vi.fn(), warning: vi.fn() }, }, - ...commonTestProviders, + ...macroWorkspaceTestProviders, ], imports: [HttpClientTestingModule], }); @@ -360,6 +362,33 @@ describe("OperatorMenuService", () => { expect(errorSpy).toHaveBeenCalledWith("Copy failed. You don't have the permission to write to the clipboard."); }); + + it("bundles pinned definitions when copying a macro to another workflow", async () => { + vi.spyOn(TestBed.inject(MacroExpansionService), "expand").mockResolvedValue(expansionFixture(testWorkflow(), [])); + const d = testMacro("clipboard-macro", [mockScanPredicate], { + operatorPositions: { [mockScanPredicate.operatorID]: mockPoint }, + version: 4, + name: "Scan", + }); + workflowActionService.setMacroDependencies([d]); + const instance = macroInstance(d, "macro-instance"); + workflowActionService.addOperator(instance, mockPoint); + workflowActionService.getJointGraphWrapper().highlightOperators(instance.operatorID); + service.saveHighlightedElements(); + const saved = writeText.mock.calls[0][0]; + expect(JSON.parse(saved).macroDependencies).toEqual([d]); + workflowActionService.clearWorkflow(); + Object.defineProperty(navigator, "clipboard", { + value: { readText: vi.fn().mockResolvedValue(saved) }, + configurable: true, + }); + service.performPasteOperation(); + await flushAsync(); + const content = workflowActionService.getWorkflowContent(); + expect(content.operators).toHaveLength(1); + expect(content.operators[0].operatorID).not.toBe(instance.operatorID); + expect(content.macroDependencies![0].content.operators[0].operatorType).toBe(mockScanPredicate.operatorType); + }); }); describe("performPasteOperation", () => { @@ -426,6 +455,24 @@ describe("OperatorMenuService", () => { expect(workflowActionService.getTexeraGraph().getAllOperators().length).toBe(0); }); + it("refuses a clipboard macro with a missing definition before changing the graph", async () => { + vi.spyOn(TestBed.inject(MacroExpansionService), "expand").mockRejectedValue(new Error("Missing macro version")); + const invalid = { ...mockScanPredicate, operatorType: "WorkflowMacro:missing:1" }; + readText.mockResolvedValue( + JSON.stringify({ + operators: [invalid], + links: [], + commentBoxes: [], + operatorPositions: { [invalid.operatorID]: mockPoint }, + }) + ); + const error = vi.spyOn(TestBed.inject(NotificationService), "error").mockImplementation(() => {}); + service.performPasteOperation(); + await flushAsync(); + expect(error).toHaveBeenCalledWith(expect.stringMatching(/macro version/i)); + expect(workflowActionService.getTexeraGraph().getAllOperators()).toEqual([]); + }); + it("notifies the user when reading the clipboard is rejected", async () => { readText.mockRejectedValue(new Error("blocked")); const notificationService = TestBed.inject(NotificationService); diff --git a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts index a5f9224a522..72bbbdfe314 100644 --- a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts +++ b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts @@ -26,6 +26,7 @@ import { WorkflowUtilService } from "../workflow-graph/util/workflow-util.servic import { NotificationService } from "src/app/common/service/notification/notification.service"; import { ExecuteWorkflowService } from "../execute-workflow/execute-workflow.service"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; +import { isMacroTerminal, isMacro, MacroDefinition, mergeMacroDefinitions } from "../../../common/util/workflow-macro"; type OperatorPositions = { [key: string]: Point; @@ -42,6 +43,7 @@ type SerializedString = { operatorPositions: OperatorPositions; links: OperatorLink[]; commentBoxes: CommentBox[]; + macroDependencies?: MacroDefinition[]; }; /** @@ -51,6 +53,8 @@ type SerializedString = { * - right-click menu * - keyboard shortcuts */ +import { MacroExpansionService } from "../macro/macro-expansion.service"; + @UntilDestroy() @Injectable({ providedIn: "root", @@ -76,6 +80,7 @@ export class OperatorMenuService { public readonly COPY_OFFSET = 20; constructor( + private macroExpansion: MacroExpansionService, private workflowActionService: WorkflowActionService, private workflowUtilService: WorkflowUtilService, private notificationService: NotificationService, @@ -154,7 +159,10 @@ export class OperatorMenuService { const allDisabled = highlighted.every(op => texeraGraph.isOperatorDisabled(op)); this.isDisableOperator = !allDisabled; - this.isDisableOperatorClickable = highlighted.length !== 0 && modificationEnabled; + this.isDisableOperatorClickable = + highlighted.length !== 0 && + modificationEnabled && + highlighted.every(id => !isMacroTerminal(texeraGraph.getOperator(id))); const allViewing = highlightedExcludingSinks.every(op => texeraGraph.isViewingResult(op)); this.isToViewResult = !allViewing; @@ -167,7 +175,9 @@ export class OperatorMenuService { private highlightedOperatorIdsExcludingSinks(): string[] { const texeraGraph = this.workflowActionService.getTexeraGraph(); - return this._highlightedOperators$.value.filter(op => !isSink(texeraGraph.getOperator(op))); + return this._highlightedOperators$.value.filter( + op => !isSink(texeraGraph.getOperator(op)) && !isMacroTerminal(texeraGraph.getOperator(op)) + ); } /** @@ -210,6 +220,8 @@ export class OperatorMenuService { }); serializedString.operators = operatorsCopy; + if (operatorsCopy.some(isMacro)) + serializedString.macroDependencies = this.workflowActionService.getMacroDependencies(); serializedString.operatorPositions = operatorPositionsCopy; // get all the highlighted links, and sort them by their layers @@ -263,7 +275,7 @@ export class OperatorMenuService { // by reading from the clipboard, permission needs to be granted // a permission prompt automatically shows up by calling readText() navigator.clipboard.readText().then( - text => { + async text => { try { // convert the JSON string in the system clipboard to a JS Map var elementsInClipboard: Map = new Map(Object.entries(JSON.parse(text))); @@ -296,7 +308,32 @@ export class OperatorMenuService { ) as OperatorPositions; // get all the operators from the clipboard, which are already sorted by their layers let copiedOps: OperatorPredicate[] = elementsInClipboard.get("operators") as OperatorPredicate[]; + let dependencies: MacroDefinition[] | undefined; + const session = this.workflowActionService.editorDocument; + if (copiedOps.some(isMacro)) { + try { + dependencies = mergeMacroDefinitions( + this.workflowActionService.getMacroDependencies(), + elementsInClipboard.get("macroDependencies") ?? [] + ); + await this.macroExpansion.expand({ + ...this.workflowActionService.getWorkflowContent(), + operators: copiedOps, + links: linksInClipboard, + macroInterface: undefined, + macroDependencies: dependencies, + }); + } catch (error) { + this.notificationService.error(error instanceof Error ? error.message : String(error)); + return; + } + } + if ( + this.workflowActionService.editorDocument !== session || + !this.workflowActionService.checkWorkflowModificationEnabled() + ) + return; let linksCopy: LinkWithID = {}; copiedOps.forEach(copiedOperator => { // copyOperator assigns a new randomly generated operator ID to the new operator @@ -352,7 +389,12 @@ export class OperatorMenuService { // actually add all operators and links to the workflow try { - this.workflowActionService.addOperatorsAndLinks(operatorsAndPositions, links); + const add = () => { + if (dependencies) this.workflowActionService.setMacroDependencies(dependencies); + this.workflowActionService.addOperatorsAndLinks(operatorsAndPositions, links); + }; + if (dependencies) this.workflowActionService.bundleMacroActions(add); + else add(); } catch (e) { this.notificationService.info( "Some of the links that you selected don't have operators attached to both ends of them. These links won't be pasted, since links can't exist without operators." diff --git a/frontend/src/app/workspace/service/operator-metadata/mock-operator-metadata.data.ts b/frontend/src/app/workspace/service/operator-metadata/mock-operator-metadata.data.ts index 54e4d194836..95bea8d0c66 100644 --- a/frontend/src/app/workspace/service/operator-metadata/mock-operator-metadata.data.ts +++ b/frontend/src/app/workspace/service/operator-metadata/mock-operator-metadata.data.ts @@ -403,6 +403,19 @@ export const mockLoopEndSchema: OperatorSchema = { }; export const mockOperatorSchemaList: ReadonlyArray = [ + ...[true, false].map(input => ({ + operatorType: input ? "MacroInput" : "MacroOutput", + operatorVersion: "1", + jsonSchema: { type: "object" as const, properties: {} }, + additionalMetadata: { + userFriendlyName: input ? "Inputs" : "Outputs", + operatorGroupName: "Utilities", + inputPorts: [], + outputPorts: [], + dynamicInputPorts: !input, + dynamicOutputPorts: input, + }, + })), mockScanSourceSchema, mockFileSourceSchema, mockKeywordSourceSchema, diff --git a/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.spec.ts b/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.spec.ts index 08e040bb5c7..184666cc969 100644 --- a/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.spec.ts +++ b/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.spec.ts @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ +import { testBinding } from "../../../common/testing/workflow-graph"; +import { macroGraph } from "../../../../../../common/workflow-core/src/test/typescript/macro-test-graph"; import { TestBed } from "@angular/core/testing"; import { HttpClient } from "@angular/common/http"; import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; @@ -25,6 +27,9 @@ import { mockOperatorMetaData } from "./mock-operator-metadata.data"; import { commonTestProviders } from "../../../common/testing/test-utils"; import { AppSettings } from "../../../common/app-setting"; import type { OperatorMetadata } from "../../types/operator-schema.interface"; +import { MacroDefinition, macroInstance, macroType } from "../../../common/util/workflow-macro"; +import { WorkflowContent } from "../../../common/type/workflow"; +import { mockScanPredicate } from "../workflow-graph/model/mock-workflow-data"; describe("OperatorMetadataService", () => { let service: OperatorMetadataService; @@ -80,6 +85,49 @@ describe("OperatorMetadataService", () => { expect(schema).toEqual(mockOperatorMetaData.operators.find(op => op.operatorType === "ScanSource")); }); + it("resolves nested defaults from each definition's own bundle instead of another library entry", () => { + flushMetadata(); + const child = (value: string): MacroDefinition => ({ + id: "child", + version: 5, + name: "Child", + content: macroGraph({ + operators: [{ ...mockScanPredicate, operatorID: "leaf", operatorProperties: { tableName: value } }], + links: [], + macroInterface: { id: "child", inputs: [], outputs: [] }, + formBinding: testBinding("leaf", "tableName", "table", "Table"), + }) as unknown as WorkflowContent, + }); + const parent = (id: string, dependency: MacroDefinition): MacroDefinition => ({ + id, + version: 8, + name: id, + content: macroGraph({ + operators: [macroInstance(dependency, "inner")], + links: [], + macroDependencies: [dependency], + macroInterface: { id, inputs: [], outputs: [] }, + formBinding: testBinding("inner", "table", "table", "Table"), + }) as unknown as WorkflowContent, + }); + const a = child("café"), + b = child("東京"); + const first = parent("first", a), + second = parent("second", b); + service.registerMacros([a, first, b, second]); + expect((service.getOperatorSchema(macroType(first)).jsonSchema.properties!.table as any).default).toBe("café"); + expect((service.getOperatorSchema(macroType(second)).jsonSchema.properties!.table as any).default).toBe("東京"); + const broken = parent("broken", { + ...a, + content: { + ...a.content, + formBinding: testBinding("leaf", "missing", "table", "Missing"), + }, + }); + service.registerMacros([broken]); + expect(service.getOperatorSchema(macroType(broken)).jsonSchema.properties).toEqual({}); + }); + it("getOperatorSchema throws for an unknown operator type", () => { flushMetadata(); expect(() => service.getOperatorSchema("NoSuchOperator")).toThrow( diff --git a/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts b/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts index ce564a302ce..1006391a67d 100644 --- a/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts +++ b/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts @@ -23,6 +23,7 @@ import { Observable } from "rxjs"; import { AppSettings } from "../../../common/app-setting"; import { OperatorMetadata, OperatorSchema } from "../../types/operator-schema.interface"; import { shareReplay } from "rxjs/operators"; +import { MacroDefinition, macroSchema, macroType } from "../../../common/util/workflow-macro"; export const OPERATOR_METADATA_ENDPOINT = "resources/operator-metadata"; @@ -46,6 +47,19 @@ export type IOperatorMetadataService = Pick(); + private schemaStack = new Set(); + private macroProvider: () => MacroDefinition[] = () => []; + + public setMacroProvider(provider: () => MacroDefinition[]): void { + this.macroProvider = provider; + } + public registerMacros(definitions: MacroDefinition[]): void { + definitions.forEach(d => this.macros.set(macroType(d), d)); + } + public getMacroDefinition(type: string): MacroDefinition | undefined { + return this.macroProvider().find(d => macroType(d) === type) ?? this.macros.get(type); + } // holds the current version of operator metadata private currentOperatorMetadata: OperatorMetadata | undefined; @@ -54,7 +68,7 @@ export class OperatorMetadataService { .pipe(shareReplay(1)); constructor(private httpClient: HttpClient) { - this.getOperatorMetadata().subscribe(data => { + this.operatorMetadataObservable.subscribe(data => { this.currentOperatorMetadata = data; }); } @@ -69,12 +83,24 @@ export class OperatorMetadataService { return this.operatorMetadataObservable; } - public getOperatorSchema(operatorType: string): OperatorSchema { + public getOperatorSchema(operatorType: string, definitions = this.macroProvider()): OperatorSchema { if (!this.currentOperatorMetadata) { throw new Error("operator metadata is undefined"); } const operatorSchema = this.currentOperatorMetadata.operators.find(schema => schema.operatorType === operatorType); if (!operatorSchema) { + const definition = definitions.find(d => macroType(d) === operatorType) ?? this.getMacroDefinition(operatorType); + if (definition) { + if (this.schemaStack.has(operatorType)) throw new Error("Recursive macro property schema"); + this.schemaStack.add(operatorType); + try { + return macroSchema(definition, type => + this.getOperatorSchema(type, [...definitions, ...(definition.content.macroDependencies ?? [])]) + ); + } finally { + this.schemaStack.delete(operatorType); + } + } throw new Error(`can\'t find operator schema of type ${operatorType}`); } return operatorSchema; @@ -94,6 +120,7 @@ export class OperatorMetadataService { userFriendlyNameFilter: boolean = false, caseInsensitive: boolean = false ): boolean { + if (this.getMacroDefinition(operatorType)) return true; if (!this.currentOperatorMetadata) { return false; } diff --git a/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts b/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts index 382d1f87db8..275199f9acb 100644 --- a/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts +++ b/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts @@ -18,33 +18,14 @@ */ import { Injectable } from "@angular/core"; -import { Observable, of } from "rxjs"; +import { of } from "rxjs"; import { mockOperatorMetaData } from "./mock-operator-metadata.data"; -import { OperatorMetadata, OperatorSchema } from "../../types/operator-schema.interface"; -import { IOperatorMetadataService } from "./operator-metadata.service"; -import { shareReplay } from "rxjs/operators"; +import { OperatorMetadataService } from "./operator-metadata.service"; +import { HttpClient } from "@angular/common/http"; @Injectable() -export class StubOperatorMetadataService implements IOperatorMetadataService { - private operatorMetadataObservable = of(mockOperatorMetaData).pipe(shareReplay(1)); - - public getOperatorSchema(operatorType: string): OperatorSchema { - const operatorSchema = mockOperatorMetaData.operators.find(schema => schema.operatorType === operatorType); - if (!operatorSchema) { - throw new Error(`can\'t find operator schema of type ${operatorType}`); - } - return operatorSchema; - } - - public getOperatorMetadata(): Observable { - return this.operatorMetadataObservable; - } - - public operatorTypeExists(operatorType: string): boolean { - const operator = mockOperatorMetaData.operators.filter(op => op.operatorType === operatorType); - if (operator.length === 0) { - return false; - } - return true; +export class StubOperatorMetadataService extends OperatorMetadataService { + constructor() { + super({ get: () => of(mockOperatorMetaData) } as unknown as HttpClient); } } diff --git a/frontend/src/app/workspace/service/validation/validation-workflow.service.spec.ts b/frontend/src/app/workspace/service/validation/validation-workflow.service.spec.ts index f0bee8f1c4e..d2df864e5f8 100644 --- a/frontend/src/app/workspace/service/validation/validation-workflow.service.spec.ts +++ b/frontend/src/app/workspace/service/validation/validation-workflow.service.spec.ts @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { testPort, setMacroGraph } from "../../../common/testing/macro-graph"; import { inject, TestBed } from "@angular/core/testing"; import { Validation, ValidationError, ValidationWorkflowService } from "./validation-workflow.service"; @@ -45,6 +46,18 @@ import { commonTestProviders } from "../../../common/testing/test-utils"; describe("ValidationWorkflowService", () => { let validationWorkflowService: ValidationWorkflowService; let workflowActionservice: WorkflowActionService; + it("accepts declared macro inputs as external connections without accepting undeclared ports", () => { + workflowActionservice.addOperator(mockSentimentPredicate, mockPoint); + const inputs = [testPort("external", mockSentimentPredicate.operatorID, "input-0")]; + setMacroGraph(workflowActionservice, { id: "macro", inputs, outputs: [] }); + expect((validationWorkflowService as any).validateOperatorConnection(mockSentimentPredicate.operatorID)).toEqual({ + isValid: true, + }); + setMacroGraph(workflowActionservice, { id: "macro", inputs: [], outputs: [] }); + expect( + (validationWorkflowService as any).validateOperatorConnection(mockSentimentPredicate.operatorID).isValid + ).toBe(false); + }); beforeEach(() => { TestBed.configureTestingModule({ providers: [ diff --git a/frontend/src/app/workspace/service/validation/validation-workflow.service.ts b/frontend/src/app/workspace/service/validation/validation-workflow.service.ts index fb586513ba1..4bab32c5fd2 100644 --- a/frontend/src/app/workspace/service/validation/validation-workflow.service.ts +++ b/frontend/src/app/workspace/service/validation/validation-workflow.service.ts @@ -351,7 +351,9 @@ export class ValidationWorkflowService { throw new Error(`operator with ID ${operatorID} doesn't exist`); } - const operatorSchema = this.operatorSchemaList.find(schema => schema.operatorType === operator.operatorType); + const operatorSchema = + this.operatorSchemaList.find(schema => schema.operatorType === operator.operatorType) ?? + this.operatorMetadataService.getOperatorSchema(operator.operatorType); if (operatorSchema === undefined) { throw new Error("operatorSchema doesn't exist"); } diff --git a/frontend/src/app/workspace/service/workflow-graph/model/mock-workflow-data.ts b/frontend/src/app/workspace/service/workflow-graph/model/mock-workflow-data.ts index 4ae85b76962..19316db5193 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/mock-workflow-data.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/mock-workflow-data.ts @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { testOperator } from "../../../../common/testing/workflow-graph"; import { CommentBox, OperatorLink, OperatorPredicate, Point } from "../../../types/workflow-common.interface"; import { VIEW_RESULT_OP_TYPE } from "./workflow-graph"; @@ -46,92 +47,65 @@ export const mockPoint: Point = { y: 100, }; -export const mockScanPredicate: OperatorPredicate = { - operatorID: "1", +export const mockScanPredicate = testOperator("1", { operatorType: "ScanSource", operatorVersion: "scan", - operatorProperties: {}, inputPorts: [], - outputPorts: [{ portID: "output-0" }], showAdvanced: true, isDisabled: false, -}; +}); -export const mockSentimentPredicate: OperatorPredicate = { - operatorID: "2", +export const mockSentimentPredicate = testOperator("2", { operatorType: "NlpSentiment", operatorVersion: "nlp1", - operatorProperties: {}, - inputPorts: [{ portID: "input-0" }], - outputPorts: [{ portID: "output-0" }], showAdvanced: true, isDisabled: false, -}; +}); -export const mockResultPredicate: OperatorPredicate = { - operatorID: "3", +export const mockResultPredicate = testOperator("3", { operatorType: VIEW_RESULT_OP_TYPE, operatorVersion: "view1", - operatorProperties: {}, - inputPorts: [{ portID: "input-0" }], outputPorts: [], showAdvanced: true, isDisabled: false, -}; +}); -export const mockMultiInputOutputPredicate: OperatorPredicate = { - operatorID: "4", +export const mockMultiInputOutputPredicate = testOperator("4", { operatorType: "MultiInputOutput", operatorVersion: "m1", - operatorProperties: {}, inputPorts: [{ portID: "input-0" }, { portID: "input-1" }, { portID: "input-2" }], outputPorts: [{ portID: "output-0" }, { portID: "output-1" }, { portID: "output-2" }], showAdvanced: true, isDisabled: false, -}; +}); -export const mockPresetEnabledPredicate: OperatorPredicate = { - operatorID: "5", +export const mockPresetEnabledPredicate = testOperator("5", { operatorType: "PresetEnabledOp", operatorVersion: "p1", - operatorProperties: {}, inputPorts: [], outputPorts: [], showAdvanced: true, -}; +}); -export const mockJavaUDFPredicate: OperatorPredicate = { - operatorID: "6", +export const mockJavaUDFPredicate = testOperator("6", { operatorType: "JavaUDF", operatorVersion: "p1", - operatorProperties: {}, - inputPorts: [{ portID: "input-0" }], - outputPorts: [{ portID: "output-0" }], - showAdvanced: false, isDisabled: false, -}; +}); -export const mockPythonUDFPredicate: OperatorPredicate = { - operatorID: "7", +export const mockPythonUDFPredicate = testOperator("7", { operatorType: "PythonUDF", operatorVersion: "p1", - operatorProperties: {}, - inputPorts: [{ portID: "input-0" }], - outputPorts: [{ portID: "output-0" }], - showAdvanced: false, isDisabled: false, -}; +}); -export const mockHuggingFacePredicate: OperatorPredicate = { - operatorID: "8", +export const mockHuggingFacePredicate = testOperator("8", { operatorType: "HuggingFace", operatorVersion: "hf1", operatorProperties: { task: "text-generation", modelId: "" }, - inputPorts: [{ portID: "input-0" }], - outputPorts: [{ portID: "output-0" }], showAdvanced: true, isDisabled: false, -}; +}); /** An operator backed by a Scala executor, where a loop-variable reference binds; see mockScalaExecutorSchema. */ export const mockScalaExecutorPredicate: OperatorPredicate = { @@ -168,43 +142,6 @@ export const mockLoopEndPredicate: OperatorPredicate = { isDisabled: false, }; -export const mockScanResultLink: OperatorLink = { - linkID: "link-1", - source: { - operatorID: mockScanPredicate.operatorID, - portID: mockScanPredicate.outputPorts[0].portID, - }, - target: { - operatorID: mockResultPredicate.operatorID, - portID: mockResultPredicate.inputPorts[0].portID, - }, -}; - -export const mockScanSentimentLink: OperatorLink = { - linkID: "link-2", - source: { - operatorID: mockScanPredicate.operatorID, - portID: mockScanPredicate.outputPorts[0].portID, - }, - target: { - operatorID: mockSentimentPredicate.operatorID, - portID: mockSentimentPredicate.inputPorts[0].portID, - }, -}; - -export const mockSentimentResultLink: OperatorLink = { - linkID: "link-3", - source: { - operatorID: mockSentimentPredicate.operatorID, - portID: mockSentimentPredicate.outputPorts[0].portID, - }, - target: { - operatorID: mockResultPredicate.operatorID, - portID: mockResultPredicate.inputPorts[0].portID, - }, -}; - -/** Loop Start -> Scala executor operator: puts the operator inside the block. */ export const mockLoopStartScalaExecutorLink: OperatorLink = { linkID: "link-loop-1", source: { @@ -216,8 +153,6 @@ export const mockLoopStartScalaExecutorLink: OperatorLink = { portID: mockScalaExecutorPredicate.inputPorts[0].portID, }, }; - -/** Scala executor operator -> Loop End: closes the block around the operator. */ export const mockScalaExecutorLoopEndLink: OperatorLink = { linkID: "link-loop-2", source: { @@ -229,33 +164,23 @@ export const mockScalaExecutorLoopEndLink: OperatorLink = { portID: mockLoopEndPredicate.inputPorts[0].portID, }, }; - -export const mockFalseResultSentimentLink: OperatorLink = { - linkID: "link-4", - source: { - operatorID: mockResultPredicate.operatorID, - portID: undefined as any, - }, - target: { - operatorID: mockSentimentPredicate.operatorID, - portID: mockSentimentPredicate.inputPorts[0].portID, - }, -}; - -export const mockFalseSentimentScanLink: OperatorLink = { - linkID: "link-5", - source: { - operatorID: mockSentimentPredicate.operatorID, - portID: mockSentimentPredicate.outputPorts[0].portID, - }, - target: { - operatorID: mockScanPredicate.operatorID, - portID: undefined as any, - }, -}; +// Leave missing ports undefined so the invalid-link fixtures exercise graph validation. +function firstPortLink(linkID: string, source: OperatorPredicate, target: OperatorPredicate): OperatorLink { + return { + linkID, + source: { operatorID: source.operatorID, portID: source.outputPorts[0]?.portID }, + target: { operatorID: target.operatorID, portID: target.inputPorts[0]?.portID }, + }; +} + +export const mockScanResultLink = firstPortLink("link-1", mockScanPredicate, mockResultPredicate); +export const mockScanSentimentLink = firstPortLink("link-2", mockScanPredicate, mockSentimentPredicate); +export const mockSentimentResultLink = firstPortLink("link-3", mockSentimentPredicate, mockResultPredicate); +export const mockFalseResultSentimentLink = firstPortLink("link-4", mockResultPredicate, mockSentimentPredicate); +export const mockFalseSentimentScanLink = firstPortLink("link-5", mockSentimentPredicate, mockScanPredicate); export const mockCommentBox: CommentBox = { - commentBoxID: "1", + commentBoxID: "comment-box-1", comments: [], commentBoxPosition: mockPoint, }; diff --git a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts index 72e1902acf5..4c8798d6bc4 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts @@ -25,7 +25,7 @@ import { StubOperatorMetadataService } from "../../operator-metadata/stub-operat import { JointUIService } from "../../joint-ui/joint-ui.service"; import { UndoRedoService } from "../../undo-redo/undo-redo.service"; import { WorkflowUtilService } from "../util/workflow-util.service"; -import { WorkflowActionService } from "./workflow-action.service"; +import { DEFAULT_WORKFLOW, WorkflowActionService } from "./workflow-action.service"; import { WorkflowGraph } from "./workflow-graph"; import { JointGraphWrapper } from "./joint-graph-wrapper"; import { commonTestProviders } from "../../../../common/testing/test-utils"; @@ -98,6 +98,44 @@ describe("SharedModelChangeHandler", () => { } describe("operator add and delete", () => { + for (const withComment of [false, true]) { + it(`reloads a shared document whose visual cells are already cleared (comments: ${withComment})`, () => { + addOperatorWithPosition(mockScanPredicate); + addOperatorWithPosition(mockResultPredicate); + texeraGraph.addLink(mockScanResultLink); + if (withComment) texeraGraph.addCommentBox(mockCommentBox); + const content = service.getWorkflowContent(); + jointGraph.clear({ silent: true }); + expect(texeraGraph.getAllOperators()).toHaveLength(2); + expect(() => service.reloadWorkflow({ ...DEFAULT_WORKFLOW, content }, false, false)).not.toThrow(); + expect(service.getWorkflowContent().operators).toEqual(content.operators); + expect(service.getWorkflowContent().links).toEqual(content.links); + expect(service.getWorkflowContent().commentBoxes).toEqual(content.commentBoxes); + expect(jointGraph.getCell(mockScanPredicate.operatorID)).toBeDefined(); + expect(jointGraph.getCell(mockScanResultLink.linkID)).toBeDefined(); + expect(texeraGraph.getSyncTexeraGraph()).toBe(true); + }); + } + + it("updates a replaced operator in one transaction without losing its links", () => { + addOperatorWithPosition(mockScanPredicate); + addOperatorWithPosition(mockResultPredicate); + texeraGraph.addLink(mockScanResultLink); + const link = jointGraph.getCell(mockScanResultLink.linkID); + const updated = { ...mockSentimentPredicate, operatorID: mockScanPredicate.operatorID }; + const added: string[] = []; + const subscription = texeraGraph.getOperatorAddStream().subscribe(op => added.push(op.operatorID)); + texeraGraph.bundleActions(() => { + texeraGraph.deleteOperator(updated.operatorID); + texeraGraph.addOperator(updated); + }); + expect((jointGraph.getCell(updated.operatorID) as joint.dia.Element).getPorts().length).toBe( + updated.inputPorts.length + updated.outputPorts.length + ); + expect(jointGraph.getCell(mockScanResultLink.linkID)).toBe(link); + expect(added).toEqual([updated.operatorID]); + subscription.unsubscribe(); + }); it("should emit operatorAddSubject and add a joint element when an operator is added", () => { let added: OperatorPredicate | undefined; const sub = texeraGraph.getOperatorAddStream().subscribe(op => (added = op)); @@ -518,7 +556,11 @@ describe("SharedModelChangeHandler", () => { const element = jointGraph.getCell(mockScanPredicate.operatorID) as joint.dia.Element; const portsBefore = element.getPorts().length; - (handler as any).onPortRemoved(mockScanPredicate.operatorID, true); + (handler as any).handlePortEvent( + { path: [mockScanPredicate.operatorID, "inputPorts"] }, + mockScanPredicate.operatorID, + true + ); expect(element.getPorts().length).toEqual(portsBefore); }); diff --git a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts index e5eab7a812e..1e115bb63f0 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts @@ -43,6 +43,7 @@ import { GuiConfigService } from "../../../../common/service/gui-config.service" */ export class SharedModelChangeHandler { + private observed = new WeakSet(); private config: GuiConfigService | null = null; constructor( @@ -51,20 +52,35 @@ export class SharedModelChangeHandler { private jointGraphWrapper: JointGraphWrapper, private jointUIService: JointUIService ) { + this.attach(); + this.texeraGraph.newYDocLoadedSubject.subscribe(() => this.attach()); + } + + private attach(): void { + const shared = this.texeraGraph.sharedModel; + if (this.observed.has(shared)) return; + this.observed.add(shared); this.handleOperatorAddAndDelete(); this.handleLinkAddAndDelete(); this.handleElementPositionChange(); this.handleCommentBoxAddAndDelete(); this.handleOperatorDeep(); this.handleCommentBoxDeep(); - this.texeraGraph.newYDocLoadedSubject.subscribe(_ => { - this.handleOperatorAddAndDelete(); - this.handleLinkAddAndDelete(); - this.handleElementPositionChange(); - this.handleCommentBoxAddAndDelete(); - this.handleOperatorDeep(); - this.handleCommentBoxDeep(); - }); + } + + public render(): void { + const graph = this.texeraGraph; + graph + .getAllOperators() + .forEach(op => + this.jointGraph.addCell( + this.jointUIService.getJointOperatorElement(op, graph.sharedModel.elementPositionMap.get(op.operatorID)!) + ) + ); + graph.getAllLinks().forEach(link => this.jointGraph.addCell(JointUIService.getJointLinkCell(link))); + graph.getAllCommentBoxes().forEach(box => this.jointGraph.addCell(this.jointUIService.getCommentElement(box))); + graph.getAllOperators().forEach(op => graph.operatorAddSubject.next(op)); + graph.getAllLinks().forEach(link => graph.linkAddSubject.next(link)); } setConfigService(config: GuiConfigService): void { @@ -77,18 +93,29 @@ export class SharedModelChangeHandler { */ private handleOperatorAddAndDelete(): void { // A new key in the map means a new operator + const owner = this.texeraGraph.sharedModel; this.texeraGraph.sharedModel.operatorIDMap.observe((event: Y.YMapEvent>) => { + if (owner !== this.texeraGraph.sharedModel) return; const jointElementsToAdd: joint.dia.Element[] = []; const newOpIDs: string[] = []; event.changes.keys.forEach((change, key) => { - if (change.action === "add") { + if (change.action === "add" || change.action === "update") { const newOperator = this.texeraGraph.sharedModel.operatorIDMap.get(key) as YType; // Also find its position if (this.texeraGraph.sharedModel.elementPositionMap?.has(key)) { const newPos = this.texeraGraph.sharedModel.elementPositionMap?.get(key) as Point; // Add the operator into joint graph const jointOperator = this.jointUIService.getJointOperatorElement(newOperator.toJSON(), newPos); - jointElementsToAdd.push(jointOperator); + if (change.action === "update") { + this.texeraGraph.setSyncTexeraGraph(false); + try { + this.jointGraph.getCell(key).set(jointOperator.attributes); + } finally { + this.texeraGraph.setSyncTexeraGraph(true); + } + } else { + jointElementsToAdd.push(jointOperator); + } newOpIDs.push(key); } else { throw new Error(`operator with key ${key} does not exist in position map`); @@ -106,7 +133,7 @@ export class SharedModelChangeHandler { ports: this.jointGraphWrapper.getCurrentHighlightedPortIDs(), }); } - this.jointGraph.getCell(key).remove(); + this.jointGraph.getCell(key)?.remove(); // Emit the event streams here, after joint graph is synced. this.texeraGraph.setSyncTexeraGraph(true); this.texeraGraph.operatorDeleteSubject.next({ deletedOperatorID: key }); @@ -145,7 +172,9 @@ export class SharedModelChangeHandler { * @private */ private handleLinkAddAndDelete(): void { + const owner = this.texeraGraph.sharedModel; this.texeraGraph.sharedModel.operatorLinkMap.observe((event: Y.YMapEvent) => { + if (owner !== this.texeraGraph.sharedModel) return; const jointElementsToAdd: joint.dia.Link[] = []; const linksToAdd: OperatorLink[] = []; const keysToDelete: string[] = []; @@ -235,7 +264,9 @@ export class SharedModelChangeHandler { * @private */ private handleElementPositionChange(): void { + const owner = this.texeraGraph.sharedModel; this.texeraGraph.sharedModel.elementPositionMap?.observe((event: Y.YMapEvent) => { + if (owner !== this.texeraGraph.sharedModel) return; event.changes.keys.forEach((change, key) => { if (change.action === "update") { this.texeraGraph.setSyncTexeraGraph(false); @@ -256,7 +287,9 @@ export class SharedModelChangeHandler { * @private */ private handleCommentBoxAddAndDelete(): void { + const owner = this.texeraGraph.sharedModel; this.texeraGraph.sharedModel.commentBoxMap.observe((event: Y.YMapEvent>) => { + if (owner !== this.texeraGraph.sharedModel) return; event.changes.keys.forEach((change, key) => { if (change.action === "add") { const commentBox = this.texeraGraph.sharedModel.commentBoxMap.get(key) as YType; @@ -265,7 +298,7 @@ export class SharedModelChangeHandler { this.texeraGraph.commentBoxAddSubject.next(commentBox.toJSON()); } if (change.action === "delete") { - this.jointGraph.getCell(key).remove(); + this.jointGraph.getCell(key)?.remove(); } }); }); @@ -281,7 +314,9 @@ export class SharedModelChangeHandler { * @private */ private handleOperatorDeep(): void { + const owner = this.texeraGraph.sharedModel; this.texeraGraph.sharedModel.operatorIDMap.observeDeep((events: Y.YEvent>[]) => { + if (owner !== this.texeraGraph.sharedModel) return; events.forEach(event => { if (event.target !== this.texeraGraph.sharedModel.operatorIDMap) { const operatorID = event.path[0] as string; @@ -369,13 +404,9 @@ export class SharedModelChangeHandler { */ private handlePortEvent(event: Y.YEvent>, operatorID: string, isInput: boolean) { if (event.path.length === 2) { - // Port added or deleted inferred by event.delta - const addedPort = event.delta[1]?.insert; - if (isDefined(addedPort)) { - this.onPortAdded(operatorID, isInput, (addedPort as Y.Map[])[0].toJSON() as PortDescription); - } else if (isDefined(event.delta[0]?.delete) || isDefined(event.delta[1]?.delete)) { - this.onPortRemoved(operatorID, isInput); - } + const operator = this.texeraGraph.getOperator(operatorID); + JointUIService.updateOperatorPorts(this.jointGraph.getCell(operatorID) as joint.dia.Element, operator); + this.texeraGraph.portAddedOrDeletedSubject.next({ newOperator: operator }); } else { const changedOperator = this.texeraGraph.getOperator(operatorID); if (event.path.includes("displayName")) { @@ -429,40 +460,6 @@ export class SharedModelChangeHandler { } } - private onPortAdded(operatorID: string, isInput: boolean, port: PortDescription) { - const operatorJointElement = this.jointGraph.getCell(operatorID); - const portGroup = isInput ? "in" : "out"; - operatorJointElement.addPort({ - group: portGroup, - id: port.portID, - attrs: { - ".port-label": { - text: port.displayName ?? "", - }, - }, - }); - - const operator = this.texeraGraph.getOperator(operatorID); - this.texeraGraph.portAddedOrDeletedSubject.next({ newOperator: operator }); - } - - private onPortRemoved(operatorID: string, isInput: boolean) { - const operatorJointElement = this.jointGraph.getCell(operatorID); - const portGroup = isInput ? "in" : "out"; - let lastPort; - for (let p of operatorJointElement.getPorts()) { - if (p.group === portGroup) { - lastPort = p; - } - } - if (lastPort) { - operatorJointElement.removePort(lastPort); - } - - const operator = this.texeraGraph.getOperator(operatorID); - this.texeraGraph.portAddedOrDeletedSubject.next({ newOperator: operator }); - } - /** * Syncs changes that are on nested-structures of comment boxes, including changes of: * - adding comments @@ -471,7 +468,9 @@ export class SharedModelChangeHandler { * @private */ private handleCommentBoxDeep(): void { + const owner = this.texeraGraph.sharedModel; this.texeraGraph.sharedModel.commentBoxMap.observeDeep((events: Y.YEvent[]) => { + if (owner !== this.texeraGraph.sharedModel) return; events.forEach(event => { if (event.target !== this.texeraGraph.sharedModel.commentBoxMap) { const commentBox: CommentBox = this.texeraGraph.getCommentBox(event.path[0] as string); diff --git a/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts b/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts index c3179d6d5ae..5289f5817f8 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts @@ -29,6 +29,7 @@ import { } from "../../../types/workflow-common.interface"; import { CoeditorState, User } from "../../../../common/type/user"; import { FormBindingConfig, WorkflowSettings } from "../../../../common/type/workflow"; +import { MacroDefinition } from "../../../../common/util/workflow-macro"; import { getWebsocketUrl } from "../../../../common/util/url"; import { v4 as uuid } from "uuid"; import { YType } from "../../../types/shared-editing.interface"; @@ -38,10 +39,10 @@ import { YType } from "../../../types/shared-editing.interface"; * the shared doc. Keyed "settings" (WorkflowSettings) and "formBinding" (the Form View * definition); the union keeps the map off `unknown` and rejects unrelated values. */ -export type ContentMetaValue = WorkflowSettings | FormBindingConfig; +export type ContentMetaValue = WorkflowSettings | FormBindingConfig | { id: string } | MacroDefinition[]; /** The keys `contentMetaMap` holds. */ -export type ContentMetaKey = "settings" | "formBinding"; +export type ContentMetaKey = "settings" | "formBinding" | "macroInterface" | "macroDependencies"; /** * SharedModel encapsulates everything related to real-time shared editing for the current workflow. @@ -91,7 +92,7 @@ export class SharedModel { // Initialize Y-undo manager by aggregating intended Y-structures. Only structures included here will be undoable. this.undoManager = new Y.UndoManager( - [this.operatorIDMap, this.elementPositionMap, this.operatorLinkMap, this.commentBoxMap], + [this.operatorIDMap, this.elementPositionMap, this.operatorLinkMap, this.commentBoxMap, this.contentMetaMap], { captureTimeout: 100, } diff --git a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts index 33c64d010fa..ad460b0286e 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts @@ -16,9 +16,12 @@ * specific language governing permissions and limitations * under the License. */ +import { testMacro, expansionFixture, setMacroGraph } from "../../../../common/testing/macro-graph"; +import { testBinding } from "../../../../common/testing/workflow-graph"; + +import { MacroExpansionService } from "../../macro/macro-expansion.service"; +import { macroWorkspaceTestProviders } from "../../macro/macro-workspace.spec-harness"; -import { StubOperatorMetadataService } from "./../../operator-metadata/stub-operator-metadata.service"; -import { OperatorMetadataService } from "./../../operator-metadata/operator-metadata.service"; import { JointUIService } from "./../../joint-ui/joint-ui.service"; import { WorkflowGraph } from "./workflow-graph"; import { UndoRedoService } from "./../../undo-redo/undo-redo.service"; @@ -41,7 +44,6 @@ import * as Y from "yjs"; import { DEFAULT_WORKFLOW, DEFAULT_WORKFLOW_NAME, WorkflowActionService } from "./workflow-action.service"; import { LogicalPort, OperatorPredicate } from "../../../types/workflow-common.interface"; import { WorkflowUtilService } from "../util/workflow-util.service"; -import { commonTestProviders } from "../../../../common/testing/test-utils"; import { ExecutionMode, FormBindingConfig, @@ -51,6 +53,8 @@ import { } from "../../../../common/type/workflow"; import { WorkflowMetadata } from "../../../../dashboard/type/workflow-metadata.interface"; +import { macroInstance } from "../../../../common/util/workflow-macro"; + describe("WorkflowActionService", () => { let service: WorkflowActionService; let undoRedo: UndoRedoService; @@ -64,11 +68,7 @@ describe("WorkflowActionService", () => { WorkflowUtilService, JointUIService, UndoRedoService, - { - provide: OperatorMetadataService, - useClass: StubOperatorMetadataService, - }, - ...commonTestProviders, + ...macroWorkspaceTestProviders, ], imports: [], }); @@ -78,6 +78,28 @@ describe("WorkflowActionService", () => { jointGraph = (service as any).jointGraph; }); + it("rejects colliding bulk edits and loads atomically, preserving Undo/Redo", () => { + service.addOperator(mockScanPredicate, mockPoint); + undoRedo.clearUndoStack(); + const before = service.getWorkflowContent(); + const addition = { op: mockSentimentPredicate, pos: mockPoint }; + const bad = { ...mockScanSentimentLink, linkID: mockScanPredicate.operatorID }; + expect(() => service.addOperatorsAndLinks([addition], [bad])).toThrow(/Duplicate graph element ID/); + expect(service.getWorkflowContent()).toEqual(before); + expect(undoRedo.canUndo()).toBe(false); + expect(() => service.reloadWorkflow({ ...DEFAULT_WORKFLOW, content: { ...before, links: [bad] } })).toThrow( + /Duplicate graph element ID/ + ); + expect(service.getWorkflowContent()).toEqual(before); + expect(service.getJointGraphWrapper().getReloadingWorkflow()).toBe(false); + service.addOperatorsAndLinks([addition], [mockScanSentimentLink]); + const connected = service.getWorkflowContent(); + undoRedo.undoAction(); + expect(service.getWorkflowContent()).toEqual(before); + undoRedo.redoAction(); + expect(service.getWorkflowContent()).toEqual(connected); + }); + /** * The shared document has finished its first sync with the room. A seed of the workflow's * settings or form definition waits for this, so that a co-editor's newer value is already in @@ -90,10 +112,133 @@ describe("WorkflowActionService", () => { texeraGraph.sharedModel.wsProvider.synced = true; } + it("preserves links added while a parent save is pending and undoes removed-port connections together", () => { + service.addOperator(mockScanPredicate, mockPoint); + service.addOperator(mockSentimentPredicate, mockPoint); + service.addOperator(mockResultPredicate, mockPoint); + service.addLink(mockScanSentimentLink); + const session = service.suspendWorkflow(), + content = service.getWorkflowContent(); + service.addLink(mockSentimentResultLink); + service.addLink(mockScanResultLink); + service.updateMacroUsage( + mockScanPredicate.operatorID, + { + ...content, + operators: content.operators.map(o => + o.operatorID === mockScanPredicate.operatorID ? { ...o, outputPorts: [] } : o + ), + links: [], + }, + session + ); + expect(service.getWorkflowContent().links).toEqual([mockSentimentResultLink]); + undoRedo.undoAction(); + expect(service.getWorkflowContent().links).toEqual( + expect.arrayContaining([mockScanSentimentLink, mockSentimentResultLink, mockScanResultLink]) + ); + }); + it("should be created", inject([WorkflowActionService], (injectedService: WorkflowActionService) => { expect(injectedService).toBeTruthy(); })); + it("returns from a nested canvas with the same shared document, pending metadata and undo history", () => { + service.setWorkflowMetadata({ ...DEFAULT_WORKFLOW, wid: 10, name: "Parent" }); + service.addOperator(mockScanPredicate, mockPoint); + service.setFormBinding({ fields: [], instruction: { body: "Parent draft 日本語" } }); + const parent = service.suspendWorkflow(); + const model = texeraGraph.sharedModel; + const depth = model.undoManager.undoStack.length; + service.openWorkflowDraft({ + ...DEFAULT_WORKFLOW, + wid: 20, + name: "Child", + content: { + ...service.getWorkflowContent(), + operators: [], + links: [], + operatorPositions: {}, + formBinding: undefined, + }, + }); + expect(service.getWorkflowMetadata().wid).toBe(20); + expect(jointGraph.getCell(mockScanPredicate.operatorID)).toBeUndefined(); + // A delayed parent sync must land in its own document, never in the child. + model.wsProvider.synced = true; + expect(service.getFormBinding().instruction).toBeUndefined(); + service.resumeWorkflow(parent); + expect(texeraGraph.sharedModel).toBe(model); + expect(service.getFormBinding().instruction?.body).toBe("Parent draft 日本語"); + expect(model.undoManager.undoStack).toHaveLength(depth); + expect(jointGraph.getCell(mockScanPredicate.operatorID)).toBeDefined(); + const events: unknown[] = []; + service + .getTexeraGraph() + .getOperatorPropertyChangeStream() + .subscribe(e => events.push(e)); + service.setOperatorProperty(mockScanPredicate.operatorID, { tableName: "restored" }); + expect(events).toHaveLength(1); + expect(service.getWorkflowMetadata().wid).toBe(10); + }); + + it("reads nested legacy bundles from shared metadata without rewriting their snapshots", () => { + const inner = testMacro("inner", [mockScanPredicate], { name: "Inner" }); + const outer = testMacro("outer", [macroInstance(inner, "child")], { macroDependencies: [inner], name: "Outer" }); + const legacy = [outer], + before = JSON.stringify(legacy); + const shared = texeraGraph.sharedModel.contentMetaMap; + shared.set("macroDependencies", legacy); + expect(service.getMacroDependencies()).toHaveLength(2); + expect(service.getMacroDependencies().every(d => !d.content.macroDependencies)).toBe(true); + expect(JSON.stringify(legacy)).toBe(before); + service.addOperator(macroInstance(outer, "root"), { x: 0, y: 0 }); + const content = service.getWorkflowContent(); + TestBed.inject(MacroExpansionService).remember( + content, + expansionFixture(content, [[["root", "child", mockScanPredicate.operatorID], mockScanPredicate]]) + ); + expect(service.getMacroExpansion()!.content.operators).toHaveLength(1); + shared.set("macroDependencies", [{ ...outer, version: 2 }]); + expect(service.getMacroDependencies().find(d => d.id === "outer")!.version).toBe(2); + expect(service.getMacroExpansion()).toBeUndefined(); + shared.set("macroDependencies", [{ ...outer, version: 0 }]); + expect(() => service.getMacroExpansion()).not.toThrow(); + expect(service.getMacroExpansion()).toBeUndefined(); + }); + + it("undoes macro metadata together with its graph transaction", () => { + syncSharedDoc(); + const boundary = { id: "macro", inputs: [], outputs: [] }; + texeraGraph.bundleActions(() => { + setMacroGraph(service, boundary); + service.addOperator(mockScanPredicate, mockPoint); + }); + expect(service.getWorkflowContent().macroInterface).toEqual({ id: boundary.id }); + undoRedo.undoAction(); + expect(texeraGraph.getAllOperators()).toEqual([]); + expect(service.getMacroInterface()).toBeUndefined(); + undoRedo.redoAction(); + expect(service.getMacroInterface()).toEqual(boundary); + expect(texeraGraph.getAllOperators()).toHaveLength(3); + }); + + it("undoes and redoes macro bindings while the shared document is waiting to sync", () => { + const original = testBinding("old", "value", "field", "Value"); + service.setFormBinding(original); + service.bundleMacroActions(() => { + service.setFormBinding({ fields: [{ ...original.fields[0], operatorID: "new" }] }); + service.addOperator(mockScanPredicate, mockPoint); + }); + undoRedo.undoAction(); + expect(service.getFormBinding()).toEqual(original); + undoRedo.redoAction(); + expect(service.getFormBinding().fields[0].operatorID).toBe("new"); + syncSharedDoc(); + undoRedo.undoAction(); + expect(service.getFormBinding()).toEqual(original); + }); + // The operator canvas and the Form View hand one open workflow between them; the arriving view // asks this before loading, because seeding a second document for the same workflow would leave // the co-editing room and rejoin it. diff --git a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts index 4c00b0cf080..1479bfc0789 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts @@ -21,7 +21,8 @@ import { Injectable } from "@angular/core"; import * as joint from "jointjs"; import * as Y from "yjs"; -import { isEqual } from "lodash-es"; +import { v4 as uuid } from "uuid"; +import { cloneDeep, isEqual } from "lodash-es"; import { BehaviorSubject, merge, Observable, Subject } from "rxjs"; import { ExecutionMode, @@ -52,8 +53,22 @@ import { filter } from "rxjs/operators"; import { isDefined } from "../../../../common/util/predicate"; import { User } from "../../../../common/type/user"; import { SharedModelChangeHandler } from "./shared-model-change-handler"; -import { ContentMetaKey, ContentMetaValue } from "./shared-model"; +import { ContentMetaKey, ContentMetaValue, SharedModel } from "./shared-model"; +import { MacroTerminal, MacroTerminals } from "../../macro/macro-terminals"; +import { createYTypeFromObject } from "../../../types/shared-editing.interface"; import { GuiConfigService } from "../../../../common/service/gui-config.service"; +import { + isMacro, + assertWorkflowIDs, + macroType, + readMacroInterface, + linkFitsMacroPorts, + isMacroTerminal, + mergeMacroDefinitions, + MacroDefinition, + MacroExpansion, + MacroInterface, +} from "../../../../common/util/workflow-macro"; /** A content-meta seed waiting to land (see seedContentMeta): its identity, and how to stop * listening for the sync it waits on once it is superseded or cancelled. */ @@ -61,6 +76,25 @@ interface ContentMetaSeed { detach?: () => void; } +interface MacroUndoGroup { + undone: boolean; + items: [Y.UndoManager, Y.UndoManager["undoStack"][number]][]; +} + +interface ContentMetaState { + pendingContentMeta: Map; + contentMetaSeeds: Map; + seedingContentMetaKeys: Set; + roomOwnedContentMeta: Set; + seededContentMetaKeys: Set; +} +export interface WorkflowEditorSession { + shared: SharedModel; + metadata: WorkflowMetadata; + modifiable: boolean; + viewport?: { zoom: number; offset: { tx: number; ty: number } }; +} + export const DEFAULT_WORKFLOW_NAME = "Untitled Workflow"; export const DEFAULT_WORKFLOW = { name: DEFAULT_WORKFLOW_NAME, @@ -90,6 +124,8 @@ export const DEFAULT_WORKFLOW = { * */ +import { MacroExpansionService } from "../../macro/macro-expansion.service"; + @Injectable({ providedIn: "root", }) @@ -97,6 +133,7 @@ export class WorkflowActionService { private readonly texeraGraph: WorkflowGraph; private readonly jointGraph: joint.dia.Graph; private readonly jointGraphWrapper: JointGraphWrapper; + public readonly macroTerminals: MacroTerminals; private readonly syncTexeraModel: SyncTexeraModel; private readonly sharedModelChangeHandler: SharedModelChangeHandler; // variable to temporarily hold the current workflow to switch view to a particular version @@ -110,6 +147,15 @@ export class WorkflowActionService { private workflowMetadataChangeSubject: Subject = new Subject(); private workflowResetSubject = new Subject(); + public readonly macroChanged$ = new Subject(); + public readonly editorChanging$ = new Subject(); + private macroDefinitionsCache?: { source: MacroDefinition[]; definitions: MacroDefinition[] }; + public executionWorkflow?: () => Workflow; + public runtimeOperatorIDs?: () => Record | undefined; + public runtimeBoundaryLinks?: () => { terminal: MacroTerminal; links: OperatorLink[] }[]; + public getExecutionWorkflowMetadata(): WorkflowMetadata { + return this.executionWorkflow?.() ?? this.getWorkflowMetadata(); + } // The Form View definition. Presentation, not structure, but it still lives in the shared // doc (shared-model contentMetaMap, key "formBinding") so a co-editor sees it live and no @@ -122,30 +168,36 @@ export class WorkflowActionService { private workflowSettingsChangeSubject = new Subject(); public readonly workflowSettingsChanged$: Observable = this.workflowSettingsChangeSubject.asObservable(); - // The shared content-map observer, tracked so re-attaching on a new doc detaches the old one. - private contentMetaObserver?: (event: Y.YMapEvent) => void; - private observedContentMetaMap?: Y.Map; - // Per document, all reset when a new one is loaded (see observeContentMeta): database copies, - // and edits, waiting for the document's first sync (see seedContentMeta), read from meanwhile; the seed - // each key is waiting on, which a later reload replaces and a local edit of the key cancels; - // the keys a seed is writing right now, which is not an edit and so is not announced (an edit - // held for the sync is announced when it is made, see writeContentMeta); the keys - // whose entry came from the room, which the first seed of the key yields to; and the keys seeded - // on this document already, whose next seed is a reload that replaces the value. - private pendingContentMeta = new Map(); - private contentMetaSeeds = new Map(); - private seedingContentMetaKeys = new Set(); - private roomOwnedContentMeta = new Set(); - private seededContentMetaKeys = new Set(); + private readonly contentMetaStates = new WeakMap(); + private readonly observedModels = new WeakSet(); + private get contentMetaState(): ContentMetaState { + return this.stateFor(this.texeraGraph.sharedModel); + } + private stateFor(shared: SharedModel): ContentMetaState { + let state = this.contentMetaStates.get(shared); + if (!state) { + state = { + pendingContentMeta: new Map(), + contentMetaSeeds: new Map(), + seedingContentMetaKeys: new Set(), + roomOwnedContentMeta: new Set(), + seededContentMetaKeys: new Set(), + }; + this.contentMetaStates.set(shared, state); + } + return state; + } constructor( private operatorMetadataService: OperatorMetadataService, private jointUIService: JointUIService, private undoRedoService: UndoRedoService, private workflowUtilService: WorkflowUtilService, - private config: GuiConfigService + private config: GuiConfigService, + public readonly macroExpansion: MacroExpansionService ) { this.texeraGraph = new WorkflowGraph(); + this.operatorMetadataService.setMacroProvider(() => this.getMacroDependencies()); this.jointGraph = new joint.dia.Graph(); this.jointGraphWrapper = new JointGraphWrapper(this.jointGraph); @@ -170,38 +222,38 @@ export class WorkflowActionService { this.texeraGraph.newYDocLoadedSubject.subscribe(() => this.observeContentMeta()); this.handleJointElementDrag(); + this.macroTerminals = new MacroTerminals(this); } private observeContentMeta(): void { - // Detach the observer from the previous shared model before attaching to the new one, so - // re-attaching on each opened workflow does not stack listeners on the same map. - this.observedContentMetaMap?.unobserve(this.contentMetaObserver!); - // A new document starts with no seed state: nothing in it came from the previous room, no key - // has been seeded on it yet, and whatever copy the previously open workflow left waiting is - // not this one's -- dropped here, so it cannot leak into the next save of a workflow opened - // (or a blank one started) on this document. - for (const key of [...this.contentMetaSeeds.keys()]) { - this.cancelContentMetaSeed(key); // drops the copy it holds, and stops its listening - } - this.roomOwnedContentMeta.clear(); - this.seededContentMetaKeys.clear(); - const contentMetaMap = this.texeraGraph.sharedModel.contentMetaMap; - this.contentMetaObserver = event => { + const shared = this.texeraGraph.sharedModel; + if (this.observedModels.has(shared)) return; + this.observedModels.add(shared); + const state = this.stateFor(shared); + const contentMetaMap = shared.contentMetaMap; + contentMetaMap.observe(event => { if (!event.transaction.local) { // An entry that arrived from the room belongs to the room, a co-editor's value or its // cleared mark alike: the seed a workflow is opened with yields to it (see seedContentMeta). // The first sync delivers the room's state as a remote transaction, which is how an // opening client learns what the room already holds. for (const key of event.changes.keys.keys()) { - this.roomOwnedContentMeta.add(key as ContentMetaKey); + state.roomOwnedContentMeta.add(key as ContentMetaKey); } } - if (this.jointGraphWrapper.getReloadingWorkflow()) { + if (shared !== this.texeraGraph.sharedModel || this.jointGraphWrapper.getReloadingWorkflow()) { return; } if (event.changes.keys.has("formBinding") && this.announces("formBinding")) { this.formBindingChangeSubject.next(this.getFormBinding()); } + if ( + ["macroInterface", "macroDependencies"].some( + key => event.changes.keys.has(key) && this.announces(key as ContentMetaKey) + ) + ) { + this.macroChanged$.next(); + } // Settings are announced the same way, so a co-editor's change reaches the settings panel, // which refreshes from this stream, instead of sitting stale until an unrelated edit. Not // through workflowChanged, though: the panel persists a settings change itself, and the @@ -209,16 +261,46 @@ export class WorkflowActionService { if (event.changes.keys.has("settings") && this.announces("settings")) { this.workflowSettingsChangeSubject.next(this.getWorkflowSettings()); } - }; - contentMetaMap.observe(this.contentMetaObserver); - this.observedContentMetaMap = contentMetaMap; + }); + shared.yDoc.on("destroy", () => { + for (const key of [...state.contentMetaSeeds.keys()]) this.cancelContentMetaSeed(key, state); + }); + shared.undoManager.on("stack-item-popped", ({ stackItem, type }) => { + const destination = type === "undo" ? shared.undoManager.redoStack : shared.undoManager.undoStack; + const parents: MacroUndoGroup | undefined = stackItem.meta.get("macroParents"); + if (parents) { + parents.undone = type === "undo"; + const inverse: MacroUndoGroup["items"] = []; + for (const [manager, item] of parents.items.slice().reverse()) { + // Do not undo a later, unrelated edit in a suspended parent document. + if ((parents.undone ? manager.undoStack : manager.redoStack).at(-1) !== item) continue; + manager[type](); + inverse.push([manager, (parents.undone ? manager.redoStack : manager.undoStack).at(-1)!]); + } + parents.items = inverse; + destination.at(-1)?.meta.set("macroParents", parents); + } + if (stackItem.meta.has("macroContent")) + destination.at(-1)?.meta.set("macroContent", stackItem.meta.get("macroContent")); + const snapshots = stackItem.meta.get("macroContent"); + if (!snapshots) return; + shared.yDoc.transact(() => { + for (const [key, value] of snapshots[type === "undo" ? "before" : "after"]) { + this.seedContentMeta(key, value, true, shared); + } + }, "macro-content-undo"); + this.formBindingChangeSubject.next(this.getFormBinding()); + this.macroChanged$.next(); + }); } /** Whether a change to a key in the shared map is announced: not a seed's own write, which is an * open and not an edit; and not while an edit held for the first sync owns the key -- that edit * was announced when it was made, reads answer it, and its landing is a seed's write. */ private announces(key: ContentMetaKey): boolean { - return !this.seedingContentMetaKeys.has(key) && !this.pendingContentMeta.get(key)?.edit; + return ( + !this.contentMetaState.seedingContentMetaKeys.has(key) && !this.contentMetaState.pendingContentMeta.get(key)?.edit + ); } private getDefaultSettings(): WorkflowSettings { @@ -298,16 +380,22 @@ export class WorkflowActionService { * @param point */ public addOperator(operator: OperatorPredicate, point: Point): void { + this.texeraGraph.assertOperatorNotExists(operator.operatorID); + let dependencies: MacroDefinition[] | undefined; + if (isMacro(operator) && !this.getMacroDependencies().some(d => macroType(d) === operator.operatorType)) { + const definition = this.operatorMetadataService.getMacroDefinition(operator.operatorType); + if (!definition) throw new Error("Missing pinned macro version"); + dependencies = [...this.getMacroDependencies(), definition, ...(definition.content.macroDependencies ?? [])]; + } // turn off multiselect since there's only one operator added this.jointGraphWrapper.setMultiSelectMode(false); - // check that the operator doesn't exist - this.texeraGraph.assertOperatorNotExists(operator.operatorID); // check that the operator type exists if (!this.operatorMetadataService.operatorTypeExists(operator.operatorType)) { throw new Error(`operator type ${operator.operatorType} is invalid`); } this.texeraGraph.bundleActions(() => { + if (dependencies) this.setMacroDependencies(dependencies); // add operator to texera graph this.texeraGraph.addOperator(operator); this.texeraGraph.sharedModel.elementPositionMap?.set(operator.operatorID, point); @@ -340,15 +428,16 @@ export class WorkflowActionService { let suffix = isInput ? operator.inputPorts.length : operator.outputPorts.length; let portID = prefix + suffix; // make sure portID has no conflict - while (operator.inputPorts.find(p => p.portID === portID) !== undefined) { + while ((isInput ? operator.inputPorts : operator.outputPorts).some(p => p.portID === portID)) { suffix += 1; portID = prefix + suffix; } + const terminal = isMacroTerminal(operator); const port: PortDescription = { - portID, - displayName: "", - disallowMultiInputs, + portID: terminal ? prefix + uuid() : portID, + displayName: terminal ? (isInput ? "Output" : "Input") : "", + disallowMultiInputs: terminal && isInput ? true : disallowMultiInputs, isDynamicPort: true, dependencies: [], }; @@ -373,7 +462,8 @@ export class WorkflowActionService { public removePort(operatorID: string, isInput: boolean): void { this.texeraGraph.bundleActions(() => { this.texeraGraph.assertOperatorExists(operatorID); - this.texeraGraph.removePort(operatorID, isInput); + const selected = this.jointGraphWrapper.getCurrentHighlightedPortIDs().find(p => p.operatorID === operatorID); + this.texeraGraph.removePort(operatorID, isInput, selected?.portID); }); } @@ -404,6 +494,13 @@ export class WorkflowActionService { links?: readonly OperatorLink[], commentBoxes?: ReadonlyArray ): void { + const current = this.getWorkflowContent(); + assertWorkflowIDs({ + ...current, + operators: [...current.operators, ...operatorsAndPositions.map(item => item.op)], + links: [...current.links, ...(links ?? [])], + commentBoxes: [...current.commentBoxes, ...(commentBoxes ?? [])], + }); // remember currently highlighted operators and groups const currentHighlights = this.jointGraphWrapper.getCurrentHighlights(); // unhighlight previous highlights @@ -485,30 +582,15 @@ export class WorkflowActionService { * Calculating the top-left (minimum x and y) position of all operators */ public calculateTopLeftOperatorPosition(): void { - this.texeraGraph.bundleActions(() => { - this.undoRedoService.setListenJointCommand(false); - const allOperators = this.getTexeraGraph().getAllOperators(); - if (allOperators.length === 0) return; - - let minX = Infinity; - let minY = Infinity; - - for (const operator of allOperators) { - const operatorID = operator.operatorID; - const position = this.jointGraphWrapper.getElementPosition(operatorID); - - if (position.x < minX) { - minX = position.x; - } - if (position.y < minY) { - minY = position.y; - } - } - - this.centerPoint = { x: minX, y: minY }; - - this.undoRedoService.setListenJointCommand(true); - }); + const positions = this.jointGraph + .getElements() + .filter(e => this.texeraGraph.hasOperator(String(e.id))) + .map(e => e.position()); + if (positions.length) + this.centerPoint = { + x: Math.min(...positions.map(p => p.x)), + y: Math.min(...positions.map(p => p.y)), + }; } /** @@ -681,6 +763,105 @@ export class WorkflowActionService { // Below are workflow-level and metadata-related methods. // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /** Retain the document and its undo history while another workflow occupies the canvas. */ + public get editorDocument(): object { + return this.texeraGraph.sharedModel; + } + + public suspendWorkflow(): WorkflowEditorSession { + const paper = this.jointGraphWrapper.getMainJointPaper(); + return { + shared: this.texeraGraph.sharedModel, + metadata: this.workflowMetadata, + modifiable: this.checkWorkflowModificationEnabled(), + viewport: paper ? { zoom: paper.scale().sx, offset: paper.translate() } : undefined, + }; + } + + public getSessionWorkflow(session: WorkflowEditorSession): Workflow { + return { ...session.metadata, content: this.getWorkflowContent(session.shared) }; + } + + /** Keep asynchronous parent propagation on the initiating edit's existing Yjs undo item. */ + public captureMacroUndo(): MacroUndoGroup | undefined { + const item = this.texeraGraph.sharedModel.undoManager.undoStack.at(-1); + if (!item) return; + const group: MacroUndoGroup = item.meta.get("macroParents") ?? { undone: false, items: [] }; + item.meta.set("macroParents", group); + return group; + } + + public updateMacroUsage( + operatorID: string, + content: WorkflowContent, + session?: WorkflowEditorSession, + origin?: MacroUndoGroup + ): void { + const shared = session?.shared ?? this.texeraGraph.sharedModel; + const updated = content.operators.find(o => o.operatorID === operatorID)!; + this.bundleMacroActions(() => { + if (content.macroDependencies) this.setMacroDependencies(content.macroDependencies, shared); + if (content.formBinding) this.setFormBinding(content.formBinding, shared); + for (const [id, link] of shared.operatorLinkMap) { + if (!linkFitsMacroPorts(link, updated)) shared.operatorLinkMap.delete(id); + } + shared.operatorIDMap.set(operatorID, createYTypeFromObject(updated)); + }, shared); + if (origin) { + const manager = shared.undoManager, + item = manager.undoStack.at(-1); + if (item) { + if (origin.undone) manager.undo(); + origin.items.push([manager, origin.undone ? manager.redoStack.at(-1)! : item]); + } + } + } + + public openWorkflowDraft(workflow: Workflow, sharedEditing = false): void { + this.jointGraphWrapper.setReloadingWorkflow(true); + this.jointGraphWrapper.unhighlightElements(this.jointGraphWrapper.getCurrentHighlights()); + const previous = this.texeraGraph.getAllOperators(); + this.texeraGraph.sharedModel = new SharedModel( + sharedEditing ? workflow.wid : undefined, + this.texeraGraph.sharedModel.user, + this.config.env.productionSharedEditingServer + ); + if (!sharedEditing) this.texeraGraph.sharedModel.wsProvider.disconnect(); + this.texeraGraph.newYDocLoadedSubject.next(undefined); + previous.forEach(op => this.texeraGraph.operatorDeleteSubject.next({ deletedOperatorID: op.operatorID })); + this.undoRedoService.setUndoManager(this.texeraGraph.sharedModel.undoManager); + this.reloadWorkflow(workflow, false); + } + + public resumeWorkflow(session: WorkflowEditorSession): void { + this.texeraGraph.sharedModel.destroy(); + const previous = this.texeraGraph.getAllOperators(); + this.jointGraphWrapper.setReloadingWorkflow(true); + this.texeraGraph.setSyncTexeraGraph(false); + try { + this.jointGraphWrapper.unhighlightElements(this.jointGraphWrapper.getCurrentHighlights()); + this.jointGraph.clear(); + previous.forEach(op => this.texeraGraph.operatorDeleteSubject.next({ deletedOperatorID: op.operatorID })); + this.texeraGraph.sharedModel = session.shared; + this.texeraGraph.newYDocLoadedSubject.next(undefined); + this.undoRedoService.setUndoManager(session.shared.undoManager); + this.operatorMetadataService.registerMacros(this.getMacroDependencies()); + this.sharedModelChangeHandler.render(); + this.setWorkflowMetadata(session.metadata); + if (session.modifiable) this.enableWorkflowModification(); + else this.disableWorkflowModification(); + } finally { + this.texeraGraph.setSyncTexeraGraph(true); + this.jointGraphWrapper.setReloadingWorkflow(false); + } + this.formBindingChangeSubject.next(this.getFormBinding()); + this.macroChanged$.next(); + if (session.viewport) { + this.jointGraphWrapper.setZoomProperty(session.viewport.zoom); + this.jointGraphWrapper.getMainJointPaper().translate(session.viewport.offset.tx, session.viewport.offset.ty); + } else this.texeraGraph.triggerCenterEvent(); + } + /** * Refreshes the internal shared model and joins a new shared-editing room. * @@ -763,6 +944,7 @@ export class WorkflowActionService { asyncRendering = this.config.env.asyncRenderingEnabled, restoreViewport = true ): void { + if (workflow) assertWorkflowIDs(workflow.content); this.jointGraphWrapper.setReloadingWorkflow(true); this.jointGraphWrapper.jointGraphContext.withContext({ async: asyncRendering }, () => { this.setWorkflowMetadata(workflow); @@ -791,6 +973,8 @@ export class WorkflowActionService { const workflowContent: WorkflowContent = workflow.content; this.hydrateSettings(workflowContent.settings); this.hydrateFormBinding(workflowContent.formBinding); + this.seedContentMeta("macroInterface", workflowContent.macroInterface); + this.seedContentMeta("macroDependencies", workflowContent.macroDependencies); let operatorsAndPositions: { op: OperatorPredicate; pos: Point }[] = []; workflowContent.operators.forEach(op => { @@ -845,6 +1029,7 @@ export class WorkflowActionService { this.getTexeraGraph().getPortDisplayNameChangedSubject(), this.getTexeraGraph().getPortPropertyChangedStream(), this.formBindingChanged$, + this.macroChanged$, this.workflowResetSubject.asObservable() ); } @@ -939,37 +1124,42 @@ export class WorkflowActionService { * once (see writeContentMeta), and goes in after the sync as a write that follows the room's * state and replaces it -- which is what an edit means -- so it yields to nothing. */ - private seedContentMeta(key: ContentMetaKey, value: ContentMetaValue | undefined, edit = false): void { + private seedContentMeta( + key: ContentMetaKey, + value: ContentMetaValue | undefined, + edit = false, + shared = this.texeraGraph.sharedModel + ): void { + const state = this.stateFor(shared); // This seed replaces any the key was still waiting on (and stops its listening); it has an // identity of its own, since a cleared key (undefined) has no value to be told apart by. - this.cancelContentMetaSeed(key); + this.cancelContentMetaSeed(key, state); const seed: ContentMetaSeed = {}; - this.contentMetaSeeds.set(key, seed); + state.contentMetaSeeds.set(key, seed); if (value !== undefined) { - this.pendingContentMeta.set(key, { value, edit }); + state.pendingContentMeta.set(key, { value, edit }); } - const shared = this.texeraGraph.sharedModel; // Runs at once or when the sync comes, and then for this seed only: a later seed of the key, a // local edit of it and the loading of another document all detach the listening before they // would make this seed stale (see cancelContentMetaSeed, observeContentMeta). const land = () => { - this.cancelContentMetaSeed(key); - const first = !this.seededContentMetaKeys.has(key); - this.seededContentMetaKeys.add(key); + this.cancelContentMetaSeed(key, state); + const first = !state.seededContentMetaKeys.has(key); + state.seededContentMetaKeys.add(key); // Over the first seed the room's word wins, a value or the cleared mark, whether a // co-editor's or this client's from another tab; a later reload, or an edit, replaces it. - if (first && !edit && this.roomOwnedContentMeta.has(key)) { + if (first && !edit && state.roomOwnedContentMeta.has(key)) { return; } const stored = value ?? null; // Opening a workflow is not an edit, so the seed is not announced (see observeContentMeta). - this.seedingContentMetaKeys.add(key); + state.seedingContentMetaKeys.add(key); try { if (!isEqual(shared.contentMetaMap.get(key), stored)) { - shared.contentMetaMap.set(key, stored); + shared.yDoc.transact(() => shared.contentMetaMap.set(key, stored), "content-meta-seed"); } } finally { - this.seedingContentMetaKeys.delete(key); + state.seedingContentMetaKeys.delete(key); } }; if (!shared.wsProvider.shouldConnect || shared.wsProvider.synced) { @@ -985,21 +1175,21 @@ export class WorkflowActionService { /** Forget the seed a key is waiting on, stop its listening, and drop the copy it holds; the * document's value stands. */ - private cancelContentMetaSeed(key: ContentMetaKey): void { - this.contentMetaSeeds.get(key)?.detach?.(); - this.contentMetaSeeds.delete(key); - this.pendingContentMeta.delete(key); + private cancelContentMetaSeed(key: ContentMetaKey, state = this.contentMetaState): void { + state.contentMetaSeeds.get(key)?.detach?.(); + state.contentMetaSeeds.delete(key); + state.pendingContentMeta.delete(key); } /** An edit still waiting for the first sync, which is going to replace whatever the document * holds; else the shared document's value for a key -- none, if the document marks it cleared * -- or, for a key the document says nothing about yet, the database copy still waiting to go in. */ - private readContentMeta(key: ContentMetaKey): ContentMetaValue | undefined { - const pending = this.pendingContentMeta.get(key); + private readContentMeta(key: ContentMetaKey, shared = this.texeraGraph.sharedModel): ContentMetaValue | undefined { + const pending = this.stateFor(shared).pendingContentMeta.get(key); if (pending?.edit) { return pending.value; } - const held = this.texeraGraph.sharedModel.contentMetaMap.get(key); + const held = shared.contentMetaMap.get(key); if (held !== undefined) { return held ?? undefined; } @@ -1011,89 +1201,121 @@ export class WorkflowActionService { * unless the document has not had its first sync yet, when the edit is held and announced here * instead, and goes in after the sync (see seedContentMeta). */ - private writeContentMeta(key: ContentMetaKey, value: ContentMetaValue, announce: () => void): void { - const shared = this.texeraGraph.sharedModel; + private writeContentMeta( + key: ContentMetaKey, + value: ContentMetaValue, + announce: () => void, + shared = this.texeraGraph.sharedModel + ): void { if (shared.wsProvider.shouldConnect && !shared.wsProvider.synced) { - this.seedContentMeta(key, value, true); - announce(); + this.seedContentMeta(key, value, true, shared); + if (shared === this.texeraGraph.sharedModel) announce(); return; } shared.contentMetaMap.set(key, value); } - /** Whether the workflow has a value for a key: in the document, or still waiting to go in. */ - private hasContentMeta(key: ContentMetaKey): boolean { - return this.readContentMeta(key) !== undefined; - } - - /** A form binding worth persisting: an author populated it (fields, a chosen result list -- an - * empty one included, it means "none" -- or an instruction), as opposed to the empty default a - * plain workflow carries. */ - private isFormBindingNonEmpty(fb: FormBindingConfig): boolean { - return fb.fields.length > 0 || fb.shownResultIds !== undefined || fb.instruction !== undefined; - } - /** * Replace the definition as an edit. The shared map's observer republishes it on * `formBindingChanged$`, which feeds workflowChanged() and so reaches the existing autosave. */ - public setFormBinding(formBinding: FormBindingConfig): void { + public setFormBinding(formBinding: FormBindingConfig, shared = this.texeraGraph.sharedModel): void { // Skip a redundant write (as setWorkflowSettings does): an unchanged value would still cut a // Yjs update and re-fire the observer for every collaborator. - if (isEqual(this.getFormBinding(), formBinding)) { + if (isEqual(this.getFormBinding(shared), formBinding)) { return; } - this.writeContentMeta("formBinding", formBinding, () => this.formBindingChangeSubject.next(formBinding)); + this.writeContentMeta("formBinding", formBinding, () => this.formBindingChangeSubject.next(formBinding), shared); } - public getFormBinding(): FormBindingConfig { - return (this.readContentMeta("formBinding") as FormBindingConfig) ?? getDefaultFormBinding(); + public getFormBinding(shared = this.texeraGraph.sharedModel): FormBindingConfig { + return (this.readContentMeta("formBinding", shared) as FormBindingConfig) ?? getDefaultFormBinding(); } public getWorkflowMetadata(): WorkflowMetadata { return this.workflowMetadata; } - public getWorkflowContent(): WorkflowContent { - // collect workflow content - const texeraGraph = this.getTexeraGraph(); - const operators = texeraGraph.getAllOperators(); - const links = texeraGraph.getAllLinks(); - const operatorPositions: { [key: string]: Point } = {}; - const commentBoxes = texeraGraph.getAllCommentBoxes(); - const settings = this.getWorkflowSettings(); - // Read the binding once so the has-check and the value are the same snapshot. - const formBinding = this.getFormBinding(); - - texeraGraph - .getAllOperators() - .forEach( - op => - (operatorPositions[op.operatorID] = this.texeraGraph.sharedModel.elementPositionMap?.get( - op.operatorID - ) as Point) - ); + public getMacroDependencies(): MacroDefinition[] { + const source = this.readContentMeta("macroDependencies") as MacroDefinition[] | undefined; + if (source === undefined) return []; + if (this.macroDefinitionsCache?.source !== source) { + this.macroDefinitionsCache = { source, definitions: mergeMacroDefinitions(source) }; + } + return this.macroDefinitionsCache.definitions; + } + + public getMacroExpansion(): MacroExpansion | undefined { + try { + const content = this.getWorkflowContent(); + return content.macroInterface || content.operators.some(isMacro) ? this.macroExpansion.get(content) : undefined; + } catch { + return undefined; // Compilation reports malformed edits; runtime streams stay alive. + } + } + + public setMacroDependencies(definitions: MacroDefinition[], shared = this.texeraGraph.sharedModel): void { + definitions = mergeMacroDefinitions(definitions); + assertWorkflowIDs({ macroDependencies: definitions }); + if (shared === this.texeraGraph.sharedModel) this.operatorMetadataService.registerMacros(definitions); + if (isEqual(this.readContentMeta("macroDependencies", shared), definitions)) return; + this.writeContentMeta("macroDependencies", definitions, () => this.macroChanged$.next(), shared); + } + + /** Pending metadata has not entered Yjs yet; attach it to the same undo item as the collapse. */ + public bundleMacroActions(action: () => void, shared = this.texeraGraph.sharedModel): void { + const keys: ContentMetaKey[] = ["formBinding", "macroDependencies", "macroInterface"]; + const pending = shared.wsProvider.shouldConnect && !shared.wsProvider.synced; + const before = keys.map(key => [key, cloneDeep(this.readContentMeta(key, shared))]); + shared.undoManager.stopCapturing(); + shared.transact(() => { + action(); + // An interface-only edit still needs an Undo item while its authoritative write waits for sync. + if (pending) + keys.forEach((key, i) => { + const value = this.readContentMeta(key, shared); + if (!isEqual(before[i][1], value)) shared.contentMetaMap.set(key, cloneDeep(value) ?? null); + }); + }); + if (pending) + shared.undoManager.undoStack.at(-1)?.meta.set("macroContent", { + before, + after: keys.map(key => [key, cloneDeep(this.readContentMeta(key, shared))]), + }); + shared.undoManager.stopCapturing(); + } + + public getMacroInterface(): MacroInterface | undefined { + return readMacroInterface({ + operators: this.texeraGraph.getAllOperators(), + links: this.texeraGraph.getAllLinks(), + macroInterface: this.readContentMeta("macroInterface") as { id: string } | undefined, + }); + } + + public setMacroInterface(boundary: { id: string }): void { + if (!this.checkWorkflowModificationEnabled()) return; + this.writeContentMeta("macroInterface", { id: boundary.id }, () => this.macroChanged$.next()); + } + + public getWorkflowContent(shared = this.texeraGraph.sharedModel): WorkflowContent { + const read = (key: ContentMetaKey) => this.readContentMeta(key, shared); + const formBinding = read("formBinding") as FormBindingConfig | undefined; + const dependencies = read("macroDependencies") as MacroDefinition[] | undefined; return { - operators, - operatorPositions, - links, - commentBoxes, - settings, - // Carry formBinding only when the workflow has one (opened with it, or an author - // populated it since), so a plain workflow's content is unchanged and its save cuts no - // needless version. hasContentMeta stands in for the old "loaded" flag: hydrate sets the - // key (or holds the copy until the sync) for a workflow opened with a binding and deletes - // it for one without. Presence and value come from the same place, so a binding that is - // present but empty is carried while the seed waits, not dropped. - ...(this.hasContentMeta("formBinding") || this.isFormBindingNonEmpty(formBinding) ? { formBinding } : {}), + operators: [...shared.operatorIDMap.values()].map(op => op.toJSON()), + links: [...shared.operatorLinkMap.values()], + operatorPositions: Object.fromEntries(shared.elementPositionMap.entries()), + commentBoxes: [...shared.commentBoxMap.values()].map(box => box.toJSON()), + settings: (read("settings") as WorkflowSettings) ?? this.getDefaultSettings(), + ...(read("macroInterface") ? { macroInterface: read("macroInterface") as { id: string } } : {}), + ...(dependencies?.length ? { macroDependencies: mergeMacroDefinitions(dependencies) } : {}), + ...(formBinding ? { formBinding } : {}), }; } public getWorkflow(): Workflow { - return { - ...this.workflowMetadata, - ...{ content: this.getWorkflowContent() }, - }; + return { ...this.workflowMetadata, content: this.getWorkflowContent() }; } /** diff --git a/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.spec.ts b/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.spec.ts index 48007ae9498..9fc940628a5 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.spec.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.spec.ts @@ -56,6 +56,25 @@ describe("WorkflowGraph", () => { expect(workflowGraph.getAllLinks().length).toEqual(0); }); + it("rejects graph ID collisions before adding an operator, link or comment box", () => { + workflowGraph.addOperator(mockScanPredicate); + workflowGraph.addOperator(mockResultPredicate); + const before = workflowGraph.getAllOperators(); + expect(() => workflowGraph.addLink({ ...mockScanResultLink, linkID: mockScanPredicate.operatorID })).toThrow( + /already exists/ + ); + expect(() => + workflowGraph.addCommentBox({ ...mockCommentBox, commentBoxID: mockScanPredicate.operatorID }) + ).toThrow(/already exists/); + workflowGraph.addLink(mockScanResultLink); + expect(() => + workflowGraph.addOperator({ ...mockSentimentPredicate, operatorID: mockScanResultLink.linkID }) + ).toThrow(/already exists/); + expect(workflowGraph.getAllOperators()).toEqual(before); + expect(workflowGraph.getAllLinks()).toEqual([mockScanResultLink]); + expect(workflowGraph.getAllCommentBoxes()).toEqual([]); + }); + it("should load an existing graph properly", () => { workflowGraph = new WorkflowGraph( [mockScanPredicate, mockSentimentPredicate, mockResultPredicate], @@ -499,6 +518,24 @@ describe("WorkflowGraph", () => { expect(() => workflowGraph.removePort("nonexist", false)).toThrowError(new RegExp("doesn't exist")); }); + it("resolves and edits imported macro ports by membership rather than their names", () => { + workflowGraph.addOperator({ + ...mockSentimentPredicate, + inputPorts: [{ portID: "入口", displayName: "Rows" }], + outputPorts: [{ portID: "result", displayName: "Result" }], + }); + const input = { operatorID: "2", portID: "入口" }, + output = { operatorID: "2", portID: "result" }; + expect(workflowGraph.hasPort(input)).toBe(true); + expect(workflowGraph.getPortDescription(input)?.displayName).toBe("Rows"); + expect(workflowGraph.getPortDescription(output)?.displayName).toBe("Result"); + expect(workflowGraph.getSharedPortDescriptionType(input)?.get("portID").toString()).toBe("入口"); + workflowGraph.setPortProperty(input, { partitionInfo: { type: "none" }, dependencies: [] }); + expect(workflowGraph.getPortDescription(input)?.partitionRequirement).toEqual({ type: "none" }); + expect(workflowGraph.hasPort({ ...input, portID: "missing" })).toBe(false); + expect(workflowGraph.getSharedPortDescriptionType({ ...input, portID: "missing" })).toBeUndefined(); + }); + it("should report port existence via hasPort", () => { workflowGraph.addOperator(mockSentimentPredicate); diff --git a/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.ts b/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.ts index 195c050139c..44f40d465ae 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/workflow-graph.ts @@ -210,7 +210,9 @@ export class WorkflowGraph { } public getSharedPortDescriptionType(operatorPortID: LogicalPort): YType | undefined { - const isInput = operatorPortID?.portID.includes("input"); + const isInput = this.getOperator(operatorPortID.operatorID).inputPorts.some( + p => p.portID === operatorPortID.portID + ); const portListObject = isInput ? this.getOperator(operatorPortID.operatorID).inputPorts : this.getOperator(operatorPortID.operatorID).outputPorts; @@ -679,44 +681,33 @@ export class WorkflowGraph { } } - public removePort(operatorID: string, isInput: boolean): void { + public removePort(operatorID: string, isInput: boolean, portID?: string): void { this.assertOperatorExists(operatorID); - if (isInput) { - const inputPorts = this.sharedModel.operatorIDMap.get(operatorID)?.get("inputPorts") as Y.Array< - YType - >; - inputPorts.delete(inputPorts.length - 1, 1); - } else { - const outputPorts = this.sharedModel.operatorIDMap.get(operatorID)?.get("outputPorts") as Y.Array< - YType - >; - outputPorts.delete(outputPorts.length - 1, 1); + const ports = this.sharedModel.operatorIDMap + .get(operatorID)! + .get(isInput ? "inputPorts" : "outputPorts") as Y.Array>; + const selected = portID ? ports.toArray().findIndex(port => port.get("portID").toJSON() === portID) : -1; + const index = selected >= 0 ? selected : ports.length - 1; + if (index < 0) return; + const removed = ports.get(index).get("portID").toJSON(); + for (const link of this.getAllLinks()) { + const end = isInput ? link.target : link.source; + if (end.operatorID === operatorID && end.portID === removed) this.deleteLinkWithID(link.linkID); } + ports.delete(index, 1); } public hasPort(operatorPortID: LogicalPort): boolean { if (!this.hasOperator(operatorPortID.operatorID)) return false; const operator = this.getOperator(operatorPortID.operatorID); - if (operatorPortID.portID.includes("input")) { - return ( - operator.inputPorts.find(portDescription => portDescription.portID === operatorPortID.portID) !== undefined - ); - } else if (operatorPortID.portID.includes("output")) { - return ( - operator.outputPorts.find(portDescription => portDescription.portID === operatorPortID.portID) !== undefined - ); - } else return false; + return [...operator.inputPorts, ...operator.outputPorts].some(p => p.portID === operatorPortID.portID); } public getPortDescription(operatorPortID: LogicalPort): PortDescription | undefined { if (!this.hasPort(operatorPortID)) throw new Error(`operator port ${(operatorPortID.operatorID, operatorPortID.portID)} does not exist`); const operator = this.getOperator(operatorPortID.operatorID); - if (operatorPortID.portID.includes("input")) { - return operator.inputPorts.find(portDescription => portDescription.portID === operatorPortID.portID); - } else if (operatorPortID.portID.includes("output")) { - return operator.outputPorts.find(portDescription => portDescription.portID === operatorPortID.portID); - } else return undefined; + return [...operator.inputPorts, ...operator.outputPorts].find(p => p.portID === operatorPortID.portID); } /** @@ -1041,15 +1032,17 @@ export class WorkflowGraph { * @param operator */ public assertOperatorNotExists(operatorID: string): void { - if (this.hasOperator(operatorID)) { - throw new Error(`operator with ID ${operatorID} already exists`); - } + this.assertCellIDUnused(operatorID, "operator"); } public assertCommentBoxNotExists(commentBoxID: string): void { - if (this.hasCommentBox(commentBoxID)) { - throw new Error(`commentBox with ID ${commentBoxID} already exists`); - } + this.assertCellIDUnused(commentBoxID, "commentBox"); + } + + private assertCellIDUnused(id: string, kind: string): void { + if (typeof id !== "string" || !id.trim()) throw new Error("Invalid graph element ID"); + if (this.hasOperator(id) || this.hasLinkWithID(id) || this.hasCommentBox(id)) + throw new Error(`${kind} with ID ${id} already exists`); } /** @@ -1061,9 +1054,7 @@ export class WorkflowGraph { * @param link */ public assertLinkNotExists(link: OperatorLink): void { - if (this.hasLinkWithID(link.linkID)) { - throw new Error(`link with ID ${link.linkID} already exists`); - } + this.assertCellIDUnused(link.linkID, "link"); if (this.hasLink(link.source, link.target)) { throw new Error(`link from ${link.source.operatorID}.${link.source.portID} to ${link.target.operatorID}.${link.target.portID} already exists`); diff --git a/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts b/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts index 8b0c54fd593..45034da2fb3 100644 --- a/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts +++ b/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts @@ -27,6 +27,7 @@ import { import { OperatorMetadataService } from "../../operator-metadata/operator-metadata.service"; import { InputPortInfo, OperatorSchema, OutputPortInfo } from "../../../types/operator-schema.interface"; import { Injectable } from "@angular/core"; +import { isMacro, macroInstance } from "../../../../common/util/workflow-macro"; import { v4 as uuid } from "uuid"; import Ajv from "ajv"; @@ -85,6 +86,12 @@ export class WorkflowUtilService { * @returns a new OperatorPredicate of the operatorType */ public getNewOperatorPredicate(operatorType: string, customDisplayName?: string): OperatorPredicate { + const macro = this.operatorMetadataService.getMacroDefinition?.(operatorType); + if (macro) + return { + ...macroInstance(macro, `WorkflowMacro-${this.getOperatorRandomUUID()}`), + customDisplayName: customDisplayName ?? macro.name, + }; const operatorSchema = this.operatorSchemaList.find(schema => schema.operatorType === operatorType); if (operatorSchema === undefined) { throw new Error(`operatorType ${operatorType} doesn't exist in operator metadata`); @@ -184,6 +191,7 @@ export class WorkflowUtilService { } public updateOperatorVersion(op: OperatorPredicate) { + if (isMacro(op)) return op; const operatorType = op.operatorType; const operatorSchema = this.operatorSchemaList.find(schema => schema.operatorType === operatorType); if (operatorSchema === undefined) { diff --git a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts index 963d4ae93b3..a080bde992d 100644 --- a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts +++ b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts @@ -27,7 +27,12 @@ import { import { WorkflowWebsocketService } from "../workflow-websocket/workflow-websocket.service"; import { firstValueFrom, of, Subject } from "rxjs"; import { SchemaAttribute } from "../../types/workflow-compiling.interface"; -import { WebDataUpdate, WebPaginationUpdate } from "../../types/execute-workflow.interface"; +import { + WebDataUpdate, + WebPaginationUpdate, + WebResultUpdate, + WorkflowResultTableStats, +} from "../../types/execute-workflow.interface"; import { PaginatedResultEvent } from "../../types/workflow-websocket.interface"; import { commonTestProviders } from "../../../common/testing/test-utils"; import type { Mocked } from "vitest"; @@ -49,13 +54,19 @@ function snapshotUpdate(table: ReadonlyArray): WebDataUpdate { } describe("WorkflowResultService", () => { - let service: WorkflowResultService; + let service: WorkflowResultService, ws: WorkflowWebsocketService; + + const sendResults = ( + updates: Record, + tableStats: WorkflowResultTableStats = {} + ) => pushWsEvent(ws, { type: "WebResultUpdateEvent", updates, tableStats }); beforeEach(() => { TestBed.configureTestingModule({ providers: [WorkflowResultService, ...commonTestProviders], }); service = TestBed.inject(WorkflowResultService); + ws = TestBed.inject(WorkflowWebsocketService); }); it("should be created", () => { @@ -63,8 +74,7 @@ describe("WorkflowResultService", () => { }); it("clearResults() drops cached operator results", () => { - (service as any).operatorResultServices.set("op1", {}); - (service as any).paginatedResultServices.set("op2", {}); + sendResults({ op1: snapshotUpdate([]), op2: paginationUpdate(0) }); expect(service.hasAnyResult("op1")).toBe(true); expect(service.hasAnyResult("op2")).toBe(true); @@ -90,7 +100,6 @@ describe("WorkflowResultService", () => { }); it("routes pagination updates to a paginated service and data updates to a result service", () => { - const ws = TestBed.inject(WorkflowWebsocketService); const updateEvents: Record[] = []; // Sampled from INSIDE the subscriber so the emit point is pinned, not just the payload: // consumers (visualization-frame-content) read the operator's result service back out of this @@ -102,7 +111,7 @@ describe("WorkflowResultService", () => { }); const updates = { pagOp: paginationUpdate(42, [2]), dataOp: snapshotUpdate([{ a: 1 }]) }; - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates, tableStats: {} }); + sendResults(updates); expect(routedAtEmit).toEqual([true, true]); expect(service.hasPaginatedResult("pagOp")).toBe(true); @@ -144,16 +153,11 @@ describe("WorkflowResultService", () => { }); it("hasNonEmptyResult tells an empty result apart from one with rows", () => { - const ws = TestBed.inject(WorkflowWebsocketService); - pushWsEvent(ws, { - type: "WebResultUpdateEvent", - updates: { - emptyPag: paginationUpdate(0), - fullPag: paginationUpdate(5), - emptySnap: snapshotUpdate([]), - fullSnap: snapshotUpdate([{ a: 1 }]), - }, - tableStats: {}, + sendResults({ + emptyPag: paginationUpdate(0), + fullPag: paginationUpdate(5), + emptySnap: snapshotUpdate([]), + fullSnap: snapshotUpdate([{ a: 1 }]), }); // A service exists for the empty step (hasAnyResult), but it holds nothing. @@ -166,21 +170,87 @@ describe("WorkflowResultService", () => { }); it("announces newly-created operators on the result-initiate stream", () => { - const ws = TestBed.inject(WorkflowWebsocketService); const initiated: string[] = []; service.getResultInitiateStream().subscribe(op => initiated.push(op)); - pushWsEvent(ws, { - type: "WebResultUpdateEvent", - updates: { pagOp: paginationUpdate(1), dataOp: snapshotUpdate([]) }, - tableStats: {}, - }); + sendResults({ pagOp: paginationUpdate(1), dataOp: snapshotUpdate([]) }); expect(initiated).toEqual(["pagOp", "dataOp"]); }); + it("resolves a macro result to the existing leaf service and forwards boundary updates", () => { + service.setMacroAliases(() => ({ macro: { operatorID: "leaf", portIndex: 0 } })); + const updates: Record[] = []; + service.getResultUpdateStream().subscribe(update => updates.push(update)); + sendResults({ leaf: snapshotUpdate([{ a: 1 }]) }); + expect(service.getResultService("macro")).toBe(service.getResultService("leaf")); + expect(service.hasNonEmptyResult("macro")).toBe(true); + expect(updates[0]["macro"]).toBe(updates[0]["leaf"]); + service.clearResults(); + expect(service.hasAnyResult("macro")).toBe(false); + }); + + it("keeps replayed results when the execution identity arrives after them", () => { + const send = vi.spyOn(ws, "send").mockImplementation(() => {}); + sendResults({ op: paginationUpdate(3) }); + pushWsEvent(ws, { type: "WorkflowStateEvent", state: "Completed", executionId: 23 }); + expect(service.hasNonEmptyResult("op")).toBe(true); + service.getPaginatedResultService("op")!.selectPage(1, 10); + expect(send.mock.calls.at(-1)?.[1]).toMatchObject({ operatorID: "op", executionId: 23 }); + }); + + it("pages the exact output of the reported execution and discards its cache on the next run", () => { + const send = vi.spyOn(ws, "send").mockImplementation(() => {}); + service.setMacroAliases((port = 0) => ({ + macro: port === 0 ? { operatorID: "split", portIndex: 1 } : { operatorID: "leaf", portIndex: 0 }, + })); + pushWsEvent(ws, { type: "WorkflowStateEvent", state: "Running", executionId: 23 }); + const updates: Record[] = []; + service.getResultUpdateStream().subscribe(update => updates.push(update)); + const stats: Record[] = []; + service.getResultTableStats().subscribe(([, current]) => stats.push(current)); + sendResults({ split: paginationUpdate(0) }, { split: { firstColumn: { min: 1 } } }); + expect(updates.at(-1)).not.toHaveProperty("macro"); + expect(stats.at(-1)).not.toHaveProperty("macro"); + const selected = service.getPaginatedResultService("macro")!; + const request = send.mock.calls.at(-1)![1] as { requestID: string }; + expect(request).toMatchObject({ operatorID: "split", portIndex: 1, executionId: 23 }); + pushWsEvent(ws, { + type: "PaginatedResultEvent", + requestID: request.requestID, + operatorID: "split", + portIndex: 1, + pageIndex: 1, + totalTuples: 7, + table: [], + schema: [], + }); + expect(service.hasNonEmptyResult("macro")).toBe(true); + expect(service.hasNonEmptyResult("split")).toBe(false); + expect(updates.at(-1)).toEqual({ macro: paginationUpdate(7) }); + sendResults({ leaf: paginationUpdate(11) }); + const second = service.getPortPaginatedResultService("macro", 1)!; + expect(second).toBe(service.getPaginatedResultService("leaf")); + second.peekFirstRows(5); + expect(send.mock.calls.at(-1)![1]).toMatchObject({ operatorID: "leaf", executionId: 23 }); + pushWsEvent(ws, { type: "WorkflowStateEvent", state: "Running", executionId: 24 }); + expect(service.getPaginatedResultService("macro")).toBeUndefined(); + expect(selected.getCurrentTotalNumTuples()).toBe(7); + }); + + it("hides results and updates when the visible macro draft does not match the executed revision", () => { + sendResults({ leaf: snapshotUpdate([{ a: 1 }]) }); + service.setMacroAliases(() => ({ leaf: null })); + expect(service.hasAnyResult("leaf")).toBe(false); + const updates: Record[] = []; + service.getResultUpdateStream().subscribe(update => updates.push(update)); + sendResults({ leaf: snapshotUpdate([{ a: 2 }]) }); + expect(updates.at(-1)).toEqual({}); + service.setMacroAliases(() => ({})); + expect(service.getResultService("leaf")?.getCurrentResultSnapshot()).toEqual([{ a: 2 }]); + }); + it("creates no result service for a cleared operator entry but still forwards the update record", () => { - const ws = TestBed.inject(WorkflowWebsocketService); const updateEvents: Record[] = []; service.getResultUpdateStream().subscribe(u => updateEvents.push(u)); const initiated: string[] = []; @@ -189,7 +259,7 @@ describe("WorkflowResultService", () => { // An undefined entry means the operator's result was cleared: it is neither a pagination // nor a data update, so no service of either kind may be created for it. const updates = { clearedOp: undefined }; - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates, tableStats: {} }); + sendResults(updates); expect(service.hasResult("clearedOp")).toBe(false); expect(service.hasPaginatedResult("clearedOp")).toBe(false); @@ -201,15 +271,14 @@ describe("WorkflowResultService", () => { }); it("reuses the existing result service for a second data update and announces the operator only once", () => { - const ws = TestBed.inject(WorkflowWebsocketService); const initiated: string[] = []; service.getResultInitiateStream().subscribe(op => initiated.push(op)); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { op: snapshotUpdate([{ a: 1 }]) }, tableStats: {} }); + sendResults({ op: snapshotUpdate([{ a: 1 }]) }); const firstService = service.getResultService("op"); expect(firstService).toBeDefined(); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { op: snapshotUpdate([{ b: 2 }]) }, tableStats: {} }); + sendResults({ op: snapshotUpdate([{ b: 2 }]) }); // the SAME instance is reused for the second update -- this identity check carries the whole // reuse claim on its own @@ -222,21 +291,16 @@ describe("WorkflowResultService", () => { }); it("reuses the existing paginated service for a second pagination update and announces it only once", () => { - const ws = TestBed.inject(WorkflowWebsocketService); const initiated: string[] = []; service.getResultInitiateStream().subscribe(op => initiated.push(op)); // tableStats also names pagOp, so handleTableStatsUpdate reaches the same get-or-init path in // the same frame; neither that nor the second frame may re-announce the operator. - pushWsEvent(ws, { - type: "WebResultUpdateEvent", - updates: { pagOp: paginationUpdate(3) }, - tableStats: { pagOp: { colA: { count: 1 } } }, - }); + sendResults({ pagOp: paginationUpdate(3) }, { pagOp: { colA: { count: 1 } } }); const firstService = service.getPaginatedResultService("pagOp"); expect(firstService).toBeDefined(); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { pagOp: paginationUpdate(9) }, tableStats: {} }); + sendResults({ pagOp: paginationUpdate(9) }); expect(service.getPaginatedResultService("pagOp")).toBe(firstService); expect(service.getPaginatedResultService("pagOp")!.getCurrentTotalNumTuples()).toBe(9); @@ -244,39 +308,32 @@ describe("WorkflowResultService", () => { }); it("feeds table stats to the matching paginated service and republishes the snapshot", () => { - const ws = TestBed.inject(WorkflowWebsocketService); const statsPairs: [unknown, unknown][] = []; service.getResultTableStats().subscribe(p => statsPairs.push(p)); const tableStats = { pagOp: { colA: { count: 5 } } }; - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { pagOp: paginationUpdate(3) }, tableStats }); + sendResults({ pagOp: paginationUpdate(3) }, tableStats); expect(service.getPaginatedResultService("pagOp")!.getStats()).toEqual({ colA: { count: 5 } }); expect(statsPairs[statsPairs.length - 1][1]).toEqual(tableStats); }); it("switches an operator from data output to paginated output and clears the stale service", () => { - const ws = TestBed.inject(WorkflowWebsocketService); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { op: snapshotUpdate([{ a: 1 }]) }, tableStats: {} }); + sendResults({ op: snapshotUpdate([{ a: 1 }]) }); expect(service.hasResult("op")).toBe(true); expect(service.hasPaginatedResult("op")).toBe(false); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { op: paginationUpdate(9) }, tableStats: {} }); + sendResults({ op: paginationUpdate(9) }); expect(service.hasPaginatedResult("op")).toBe(true); expect(service.hasResult("op")).toBe(false); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { op: snapshotUpdate([{ b: 2 }]) }, tableStats: {} }); + sendResults({ op: snapshotUpdate([{ b: 2 }]) }); expect(service.hasResult("op")).toBe(true); expect(service.hasPaginatedResult("op")).toBe(false); }); it("determineOutputTypes distinguishes binary from non-binary paginated output", () => { - const ws = TestBed.inject(WorkflowWebsocketService); - pushWsEvent(ws, { - type: "WebResultUpdateEvent", - updates: { binOp: paginationUpdate(1), tblOp: paginationUpdate(1) }, - tableStats: {}, - }); + sendResults({ binOp: paginationUpdate(1), tblOp: paginationUpdate(1) }); (service.getPaginatedResultService("binOp") as any).schema = [{ attributeName: "b", attributeType: "binary" }]; (service.getPaginatedResultService("tblOp") as any).schema = [{ attributeName: "s", attributeType: "string" }]; @@ -295,8 +352,7 @@ describe("WorkflowResultService", () => { }); it("determineOutputTypes marks a data-only operator as visualization output", () => { - const ws = TestBed.inject(WorkflowWebsocketService); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { vizOp: snapshotUpdate([{ a: 1 }]) }, tableStats: {} }); + sendResults({ vizOp: snapshotUpdate([{ a: 1 }]) }); expect(service.determineOutputTypes("vizOp")).toEqual({ hasAnyResult: true, @@ -320,14 +376,12 @@ describe("WorkflowResultService", () => { }); it("determineOutputExtension returns html for visualization output", () => { - const ws = TestBed.inject(WorkflowWebsocketService); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { vizOp: snapshotUpdate([{ a: 1 }]) }, tableStats: {} }); + sendResults({ vizOp: snapshotUpdate([{ a: 1 }]) }); expect(service.determineOutputExtension("vizOp", "csv")).toBe("html"); }); it("determineOutputExtension keeps csv for table output and passes other extensions through", () => { - const ws = TestBed.inject(WorkflowWebsocketService); - pushWsEvent(ws, { type: "WebResultUpdateEvent", updates: { tblOp: paginationUpdate(1) }, tableStats: {} }); + sendResults({ tblOp: paginationUpdate(1) }); expect(service.determineOutputExtension("tblOp", "csv")).toBe("csv"); expect(service.determineOutputExtension("tblOp", "json")).toBe("json"); @@ -339,13 +393,14 @@ describe("WorkflowResultService", () => { describe("OperatorPaginationResultService", () => { let service: OperatorPaginationResultService; let mockWorkflowWebsocketService: Mocked; + let eventSubject: Subject; beforeEach(() => { + eventSubject = new Subject(); mockWorkflowWebsocketService = { - subscribeToEvent: vi.fn(), + subscribeToEvent: vi.fn().mockReturnValue(eventSubject.asObservable()), send: vi.fn(), } as unknown as Mocked; - mockWorkflowWebsocketService.subscribeToEvent.mockReturnValue(new Subject()); service = new OperatorPaginationResultService("testOperator", mockWorkflowWebsocketService); }); @@ -363,67 +418,49 @@ describe("OperatorPaginationResultService", () => { }); describe("selectTuple", () => { - it("should return the correct tuple and schema", async () => { - const testSchema: SchemaAttribute[] = [ + it.each([ + { index: 1, rows: 3, expected: { id: 2, name: "Bob" } }, + { index: 2, rows: 2, expected: undefined }, + ])("reads tuple $index from a page with $rows rows", async ({ index, rows, expected }) => { + const schema: SchemaAttribute[] = [ { attributeName: "id", attributeType: "integer" }, { attributeName: "name", attributeType: "string" }, ]; - service["schema"] = testSchema; - - const testTable = [ - { id: 1, name: "Alice" }, - { id: 2, name: "Bob" }, - { id: 3, name: "Charlie" }, - ]; + const table = ["Alice", "Bob", "Charlie"].slice(0, rows).map((name, i) => ({ id: i + 1, name })); + const result = firstValueFrom(service.selectTuple(index, 3)); + const request = mockWorkflowWebsocketService.send.mock.calls[0][1] as { requestID: string }; + eventSubject.next({ + requestID: request.requestID, + operatorID: "testOperator", + portIndex: 0, + pageIndex: 1, + totalTuples: rows, + table, + schema, + }); + expect(await result).toEqual({ tuple: expected, schema }); + }); + }); - vi.spyOn(service, "selectPage").mockReturnValue( - of({ - requestID: "test", - operatorID: "testOperator", - pageIndex: 1, - table: testTable, - schema: testSchema, - portIndex: 0, - totalTuples: testTable.length, - }) - ); + describe("selectPage", () => { + it("omits a column limit larger than the server can represent", () => { + const columnLimit = Number.MAX_SAFE_INTEGER; + service.selectPage(1, 10, 0, columnLimit).subscribe(); - const result = await firstValueFrom(service.selectTuple(1, 3)); - expect(result.tuple).toEqual({ id: 2, name: "Bob" }); - expect(result.schema).toEqual(testSchema); + const payload = mockWorkflowWebsocketService.send.mock.calls[0][1]; + expect(JSON.parse(JSON.stringify(payload))).not.toHaveProperty("columnLimit"); }); - it("should handle out-of-bounds tuple index", async () => { - const testSchema: SchemaAttribute[] = [ - { attributeName: "id", attributeType: "integer" }, - { attributeName: "name", attributeType: "string" }, - ]; - service["schema"] = testSchema; - - const testTable = [ - { id: 1, name: "Alice" }, - { id: 2, name: "Bob" }, - ]; + it.each([0, 25])("preserves explicit column limit %s and bypasses the full-page cache", columnLimit => { + (service as any).resultCache.set(1, [{ id: 1 }]); + service.selectPage(1, 10, 0, columnLimit).subscribe(); - vi.spyOn(service, "selectPage").mockReturnValue( - of({ - requestID: "test", - operatorID: "testOperator", - pageIndex: 1, - table: testTable, - schema: testSchema, - portIndex: 0, - totalTuples: testTable.length, - }) + expect(mockWorkflowWebsocketService.send).toHaveBeenCalledExactlyOnceWith( + "ResultPaginationRequest", + expect.objectContaining({ pageIndex: 1, pageSize: 10, columnOffset: 0, columnLimit }) ); - - const result = await firstValueFrom(service.selectTuple(2, 3)); - expect(result.tuple).toBeUndefined(); - expect(result.schema).toEqual(testSchema); }); - }); - describe("selectPage", () => { it("returns the cached page without contacting the server", () => { const cachedPage = [{ id: 1 }, { id: 2 }]; (service as any).resultCache.set(2, cachedPage); @@ -446,8 +483,6 @@ describe("OperatorPaginationResultService", () => { }); it("fetches from the server and resolves once the matching page event arrives", () => { - const eventSubject = mockWorkflowWebsocketService.subscribeToEvent.mock.results[0] - .value as unknown as Subject; const received: PaginatedResultEvent[] = []; service.selectPage(3, 10).subscribe(e => received.push(e)); @@ -499,8 +534,6 @@ describe("OperatorPaginationResultService", () => { // A hover preview asks for the first rows while the result table may be on any page; the // table's page and cache must come through untouched. it("fetches the first rows without moving the table's page or reading its cache", () => { - const eventSubject = mockWorkflowWebsocketService.subscribeToEvent.mock.results[0] - .value as unknown as Subject; (service as any).currentPageIndex = 4; (service as any).resultCache.set(1, [{ id: "cached" }]); const received: PaginatedResultEvent[] = []; @@ -582,7 +615,6 @@ describe("OperatorPaginationResultService", () => { describe("for an output port past the first", () => { let secondPort: OperatorPaginationResultService; - let eventSubject: Subject; const reply = (requestID: string, totalTuples: number): PaginatedResultEvent => ({ requestID, operatorID: "testOperator", @@ -595,8 +627,6 @@ describe("OperatorPaginationResultService", () => { beforeEach(() => { secondPort = new OperatorPaginationResultService("testOperator", mockWorkflowWebsocketService, 1); - eventSubject = mockWorkflowWebsocketService.subscribeToEvent.mock.results[1] - .value as unknown as Subject; }); it("names the port in every request, and the first port's service leaves it out", () => { @@ -633,21 +663,17 @@ describe("OperatorPaginationResultService", () => { }); describe("incoming page events", () => { - // Every operator's and every port's replies arrive on one stream, so a reply to another - // service's request must not hand this one its columns. - it("ignores an event whose request id is not its own, schema included", () => { - const eventSubject = mockWorkflowWebsocketService.subscribeToEvent.mock.results[0] - .value as unknown as Subject; + it("keeps its schema when another port or operator receives a reply", () => { const schema: SchemaAttribute[] = [{ attributeName: "name", attributeType: "string" }]; eventSubject.next({ + portIndex: 0, + totalTuples: 1, requestID: "no-such-request", operatorID: "testOperator", pageIndex: 1, table: [], schema, - portIndex: 0, - totalTuples: 0, }); expect(service.getSchema()).toEqual([]); diff --git a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts index e6c76c1eda5..b63c2206809 100644 --- a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts +++ b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts @@ -18,6 +18,8 @@ */ import { Injectable } from "@angular/core"; +import type { ResultTarget } from "@texera/workflow-macros"; +import { WorkflowStatusService } from "../workflow-status/workflow-status.service"; import { isWebDataUpdate, isWebPaginationUpdate, @@ -42,19 +44,49 @@ import { SchemaAttribute } from "../../types/workflow-compiling.interface"; providedIn: "root", }) export class WorkflowResultService { - private paginatedResultServices = new Map(); - // the services for an operator's output ports past the first, keyed "operatorID:portIndex" - private otherPortResultServices = new Map(); + private macroAliases: (portIndex?: number) => Record = () => ({}); + public setMacroAliases(provider: (portIndex?: number) => Record): void { + this.macroAliases = provider; + } + private withMacroAliases(event: Record): Record { + const aliases = this.macroAliases(); + if (!Object.keys(aliases).length) return event; + const result = { ...event }; + for (const [alias, target] of Object.entries(aliases)) { + if (!target || target.portIndex !== 0) delete result[alias]; + else if (event[target.operatorID] !== undefined) result[alias] = event[target.operatorID]; + } + return result; + } + private activeExecution?: number; + private readonly tables = new Map>(); + private get tableServices() { + return [...this.tables.values()].flatMap(ports => [...ports.values()]); + } private operatorResultServices = new Map(); // event stream of operator result update, undefined indicates the operator result is cleared private resultUpdateStream = new Subject>(); - private resultTableStats = new ReplaySubject>>>(1); + private resultTableStats = new ReplaySubject(1); private resultInitiateStream = new Subject(); // emits when clearResults() drops cached results, so the UI can drop stale frames private resultClearedStream = new Subject(); - constructor(private wsService: WorkflowWebsocketService) { + constructor( + private wsService: WorkflowWebsocketService, + private status: WorkflowStatusService + ) { + this.wsService.subscribeToEvent("WorkflowStateEvent").subscribe(({ executionId }) => { + if (!executionId || executionId === this.activeExecution) return; + const tables = this.tableServices; + if (this.activeExecution || tables.some(table => table.executionId && table.executionId !== executionId)) { + this.clearResults(); + } else { + // Reconnecting replays the workflow's result store before its execution metadata. + tables.forEach(table => (table.executionId = executionId)); + } + this.activeExecution = executionId; + }); this.wsService.subscribeToEvent("WebResultUpdateEvent").subscribe(event => { this.handleResultUpdate(event.updates); this.handleTableStatsUpdate(event.tableStats); @@ -92,9 +124,7 @@ export class WorkflowResultService { return this.resultUpdateStream; } - public getResultTableStats(): Observable< - [Record>>, Record>>] - > { + public getResultTableStats(): Observable<[WorkflowResultTableStats, WorkflowResultTableStats]> { return this.resultTableStats.pipe(pairwise()); } @@ -111,7 +141,8 @@ export class WorkflowResultService { } public getPaginatedResultService(operatorID: string): OperatorPaginationResultService | undefined { - return this.paginatedResultServices.get(operatorID); + const id = this.macroAliases()[operatorID]; + return id === null ? undefined : this.tableForPort(id?.operatorID ?? operatorID, id?.portIndex ?? 0); } /** @@ -123,19 +154,37 @@ export class WorkflowResultService { operatorID: string, portIndex: number ): OperatorPaginationResultService | undefined { - const firstPort = this.getPaginatedResultService(operatorID); - if (portIndex === 0 || !firstPort) return firstPort; - const key = `${operatorID}:${portIndex}`; - let service = this.otherPortResultServices.get(key); - if (!service) { - service = new OperatorPaginationResultService(operatorID, this.wsService, portIndex); - this.otherPortResultServices.set(key, service); - } - return service; + if (portIndex === 0) return this.getPaginatedResultService(operatorID); + const target = this.macroAliases(portIndex)[operatorID]; + return target === null + ? undefined + : this.tableForPort(target?.operatorID ?? operatorID, target?.portIndex ?? portIndex); + } + + private tableForPort(operatorID: string, portIndex: number): OperatorPaginationResultService | undefined { + const firstPort = this.tables.get(operatorID)?.get(0); + if (portIndex === 0 || (!firstPort && !this.operatorResultServices.has(operatorID))) return firstPort; + return this.getOrInitPaginatedResultService(operatorID, portIndex); + } + + private loadPortCount(service: OperatorPaginationResultService): void { + if (service.hasPendingPage) return; + service.selectPage(service.getCurrentPageIndex(), 1).subscribe(({ totalTuples }) => { + const update: WebPaginationUpdate = { + mode: { type: "PaginationMode" }, + totalNumTuples: totalTuples, + dirtyPageIndices: [], + }; + const aliases = Object.entries(this.macroAliases()).filter( + ([, target]) => target?.operatorID === service.operatorID && target.portIndex === service.portIndex + ); + if (aliases.length) this.resultUpdateStream.next(Object.fromEntries(aliases.map(([alias]) => [alias, update]))); + }); } public getResultService(operatorID: string): OperatorResultService | undefined { - return this.operatorResultServices.get(operatorID); + const id = this.macroAliases()[operatorID]; + return id === null || id?.portIndex ? undefined : this.operatorResultServices.get(id?.operatorID ?? operatorID); } /** @@ -144,9 +193,10 @@ export class WorkflowResultService { * Emits resultClearedStream so subscribers tear down already-displayed frames. */ public clearResults(): void { + this.activeExecution = undefined; this.operatorResultServices.clear(); - this.paginatedResultServices.clear(); - this.otherPortResultServices.clear(); + this.tableServices.forEach(service => service.dispose()); + this.tables.clear(); this.resultTableStats.next({}); this.resultClearedStream.next(); } @@ -163,10 +213,16 @@ export class WorkflowResultService { const resultService = this.getOrInitResultService(operatorID); resultService.handleResultUpdate(update); // clear previously saved paginated result service - this.paginatedResultServices.delete(operatorID); + const ports = this.tables.get(operatorID); + ports?.get(0)?.dispose(); + ports?.delete(0); + if (ports?.size === 0) this.tables.delete(operatorID); } + this.tables.get(operatorID)?.forEach(port => { + if (update && port.portIndex !== 0) this.loadPortCount(port); + }); }); - this.resultUpdateStream.next(event); + this.resultUpdateStream.next(this.withMacroAliases(event)); } private handleTableStatsUpdate(event: WorkflowResultTableStats): void { @@ -174,21 +230,28 @@ export class WorkflowResultService { const paginatedResultService = this.getOrInitPaginatedResultService(operatorID); paginatedResultService.handleStatsUpdate(event[operatorID]); }); - this.resultTableStats.next(event); - } - - private getOrInitPaginatedResultService(operatorID: string): OperatorPaginationResultService { - let service = this.getPaginatedResultService(operatorID); - if (!service) { - service = new OperatorPaginationResultService(operatorID, this.wsService); - this.paginatedResultServices.set(operatorID, service); - this.resultInitiateStream.next(operatorID); - } + this.resultTableStats.next(this.withMacroAliases(event)); + } + + private getOrInitPaginatedResultService(operatorID: string, portIndex = 0): OperatorPaginationResultService { + const ports = this.tables.get(operatorID) ?? new Map(); + const cached = ports.get(portIndex); + if (cached) return cached; + const service = new OperatorPaginationResultService( + operatorID, + this.wsService, + portIndex, + this.status.runtimeExecutionId + ); + ports.set(portIndex, service); + this.tables.set(operatorID, ports); + if (portIndex === 0) this.resultInitiateStream.next(operatorID); + else this.loadPortCount(service); return service; } private getOrInitResultService(operatorID: string): OperatorResultService { - let service = this.getResultService(operatorID); + let service = this.operatorResultServices.get(operatorID); if (!service) { service = new OperatorResultService(operatorID); this.operatorResultServices.set(operatorID, service); @@ -203,14 +266,13 @@ export class WorkflowResultService { isVisualizationOutput: boolean; containsBinaryData: boolean; } { - const resultService = this.getResultService(operatorId); - const paginatedResultService = this.getPaginatedResultService(operatorId); - + const snapshot = this.getResultService(operatorId); + const table = this.getPaginatedResultService(operatorId); return { - hasAnyResult: this.hasAnyResult(operatorId), - isTableOutput: this.hasTableOutput(paginatedResultService), - containsBinaryData: this.hasBinaryData(paginatedResultService), - isVisualizationOutput: this.hasVisualizationOutput(resultService, paginatedResultService), + hasAnyResult: !!(snapshot || table), + isTableOutput: !!table, + containsBinaryData: table?.getSchema().some(attribute => attribute.attributeType === "binary") ?? false, + isVisualizationOutput: !!snapshot && !table, }; } @@ -222,21 +284,6 @@ export class WorkflowResultService { if (outputType.isTableOutput && defaultExtension === "csv") return "csv"; return defaultExtension; } - - private hasTableOutput(paginatedResultService?: OperatorPaginationResultService): boolean { - return paginatedResultService !== undefined; - } - - private hasBinaryData(paginatedResultService?: OperatorPaginationResultService): boolean { - return paginatedResultService?.getSchema().some(attribute => attribute.attributeType === "binary") ?? false; - } - - private hasVisualizationOutput( - resultService?: OperatorResultService, - paginatedResultService?: OperatorPaginationResultService - ): boolean { - return resultService !== undefined && paginatedResultService === undefined; - } } export class OperatorResultService { @@ -269,6 +316,10 @@ export class OperatorResultService { export const ALL_COLUMNS = 2147483647; export class OperatorPaginationResultService { + private readonly subscription; + public get hasPendingPage(): boolean { + return this.pendingRequests.size > 0; + } private pendingRequests: Map> = new Map(); private resultCache: Map> = new Map(); private statsCache: Record> = {}; @@ -281,13 +332,20 @@ export class OperatorPaginationResultService { private workflowWebsocketService: WorkflowWebsocketService, // which output port it pages. The running execution reports only on the first, so any other // port keeps no page cache and takes its row count from each reply. - public readonly portIndex: number = 0 + public readonly portIndex: number = 0, + public executionId?: number ) { - this.workflowWebsocketService.subscribeToEvent("PaginatedResultEvent").subscribe(event => { + this.subscription = this.workflowWebsocketService.subscribeToEvent("PaginatedResultEvent").subscribe(event => { this.handlePaginationResult(event); }); } + public dispose(): void { + this.subscription.unsubscribe(); + this.pendingRequests.forEach(request => request.complete()); + this.pendingRequests.clear(); + } + public getStats(): Record> { return this.statsCache; } @@ -341,24 +399,8 @@ export class OperatorPaginationResultService { portIndex: this.portIndex, totalTuples: this.currentTotalNumTuples, }); - } else { - // fetch result data from server - const requestID = uuid(); - const operatorID = this.operatorID; - this.workflowWebsocketService.send("ResultPaginationRequest", { - requestID, - operatorID, - pageIndex, - pageSize, - columnOffset, - columnLimit, - columnSearch, - ...this.portField(), - }); - const pendingRequestSubject = new Subject(); - this.pendingRequests.set(requestID, pendingRequestSubject); - return pendingRequestSubject; } + return this.requestPage(pageIndex, pageSize, columnOffset, columnLimit, columnSearch); } /** @@ -368,20 +410,31 @@ export class OperatorPaginationResultService { * page's would be. */ public peekFirstRows(rowCount: number): Observable { + return this.requestPage(1, rowCount, 0, ALL_COLUMNS, ""); + } + + private requestPage( + pageIndex: number, + pageSize: number, + columnOffset: number, + columnLimit: number, + columnSearch: string + ) { const requestID = uuid(); + const response = new ReplaySubject(1); + this.pendingRequests.set(requestID, response); this.workflowWebsocketService.send("ResultPaginationRequest", { requestID, operatorID: this.operatorID, - pageIndex: 1, - pageSize: rowCount, - columnOffset: 0, - columnLimit: ALL_COLUMNS, - columnSearch: "", - ...this.portField(), + pageIndex, + pageSize, + columnOffset, + columnLimit: columnLimit === Number.MAX_SAFE_INTEGER ? undefined : columnLimit, + columnSearch, + ...(this.portIndex !== 0 && { portIndex: this.portIndex }), + ...(this.executionId !== undefined && { executionId: this.executionId }), }); - const pendingRequestSubject = new Subject(); - this.pendingRequests.set(requestID, pendingRequestSubject); - return pendingRequestSubject; + return response.asObservable(); } public reset(): void { @@ -402,11 +455,6 @@ export class OperatorPaginationResultService { this.statsCache = statsUpdate; } - // the first port is the server's default, so a request for it leaves the field out - private portField(): { portIndex?: number } { - return this.portIndex === 0 ? {} : { portIndex: this.portIndex }; - } - private handlePaginationResult(res: PaginatedResultEvent): void { const pendingRequestSubject = this.pendingRequests.get(res.requestID); if (!pendingRequestSubject) { diff --git a/frontend/src/app/workspace/service/workflow-status/workflow-status.service.spec.ts b/frontend/src/app/workspace/service/workflow-status/workflow-status.service.spec.ts index 66c4a4241f5..aa749f91dae 100644 --- a/frontend/src/app/workspace/service/workflow-status/workflow-status.service.spec.ts +++ b/frontend/src/app/workspace/service/workflow-status/workflow-status.service.spec.ts @@ -25,6 +25,8 @@ import { OperatorPerformanceMetrics } from "./performance-metrics"; import { OperatorRuntimeStatus, OperatorState, OperatorStatistics } from "../../types/execute-workflow.interface"; import { TexeraWebsocketEvent } from "../../types/workflow-websocket.interface"; +import { testWorkflow } from "../../../common/testing/workflow-graph"; + const sampleStatistics: OperatorStatistics = { aggregatedInputRowCount: 1_000, aggregatedInputSize: 8_000, @@ -38,6 +40,17 @@ const sampleStatistics: OperatorStatistics = { aggregatedIdleTime: 700_000, }; +const expectedMetrics: OperatorPerformanceMetrics = { + dataProcessingTimeNs: 5_000_000, + controlProcessingTimeNs: 1_000_000, + idleTimeNs: 700_000, + inputRows: 1_000, + outputRows: 250, + inputSize: 8_000, + outputSize: 2_000, + numWorkers: 2, +}; + // The wire object the engine streams: state and statistics bundled together. const sampleRuntimeStatus: OperatorRuntimeStatus = { operatorState: OperatorState.Running, @@ -63,6 +76,18 @@ describe("WorkflowStatusService", () => { service = TestBed.inject(WorkflowStatusService); }); + it("invalidates a restored run recipe when live statistics arrive", () => { + const content = testWorkflow(); + service.setExternalStatus({ op: sampleRuntimeStatus }, 23); + service.setRuntimeContent(content); + const epoch = service.runtimeEpoch; + websocketEventSubject.next(statsEvent({ op: sampleRuntimeStatus })); + expect(service.runtimeEpoch).toBeGreaterThan(epoch); + expect(service.getRuntimeContent()).toBeUndefined(); + expect(service.runtimeExecutionId).toBeUndefined(); + expect(service.getCurrentStatistics().op).toEqual(sampleStatistics); + }); + it("emits nothing on subscribe, before any producer has written", () => { // Load-bearing for HeatmapStatsRestoreService, which uses this stream as a takeUntil // notifier: were it a BehaviorSubject, the restore would be cancelled on every page load. @@ -155,16 +180,7 @@ describe("WorkflowStatusService", () => { // BehaviorSubject seeds {} then emits the derived map. const latest = emissions[emissions.length - 1]; - expect(latest["op1"]).toEqual({ - dataProcessingTimeNs: 5_000_000, - controlProcessingTimeNs: 1_000_000, - idleTimeNs: 700_000, - inputRows: 1_000, - outputRows: 250, - inputSize: 8_000, - outputSize: 2_000, - numWorkers: 2, - }); + expect(latest["op1"]).toEqual(expectedMetrics); expect(service.getCurrentPerformanceMetrics()).toEqual(latest); }); @@ -256,4 +272,22 @@ describe("WorkflowStatusService", () => { expect(service.getCurrentState()).toEqual({ op1: OperatorState.Paused }); }); + + it("displays backend macro metrics identically live and restored and clears them between runs", () => { + const snapshot = { macro: sampleRuntimeStatus, terminal: { ...sampleRuntimeStatus, numWorkers: 0 } }; + websocketEventSubject.next(statsEvent(snapshot)); + const live = service.getCurrentPerformanceMetrics(); + expect(live).toEqual({ macro: expectedMetrics, terminal: { ...expectedMetrics, numWorkers: 0 } }); + expect(service.getCurrentState().macro).toBe(OperatorState.Running); + service.resetStatus(); + expect(service.getCurrentPerformanceMetrics().macro.inputRows).toBe(0); + expect(service.getCurrentPerformanceMetrics().macro.dataProcessingTimeNs).toBe(0); + service.setExternalStatus(snapshot); + expect(service.getCurrentPerformanceMetrics()).toEqual(live); + expect(service.getCurrentStatistics().terminal.outputPortMetrics).toEqual({ "0": 250 }); + service.clearStatus(); + expect(service.getCurrentPerformanceMetrics()).toEqual({}); + service.setExternalStatus({}); + expect(service.getCurrentState()).toEqual({}); + }); }); diff --git a/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts b/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts index 0e20939d1b7..7106c2851c9 100644 --- a/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts +++ b/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts @@ -19,14 +19,43 @@ import { Injectable } from "@angular/core"; import { BehaviorSubject, Observable, Subject } from "rxjs"; -import { OperatorRuntimeStatus, OperatorState, OperatorStatistics } from "../../types/execute-workflow.interface"; +import { + OperatorRuntimeStatus, + OperatorState, + OperatorStatistics, + ExecutionState, + isNotInExecution, +} from "../../types/execute-workflow.interface"; import { WorkflowWebsocketService } from "../workflow-websocket/workflow-websocket.service"; import { OperatorPerformanceMetrics, extractPerformanceMetrics } from "./performance-metrics"; +import { WorkflowContent } from "../../../common/type/workflow"; @Injectable({ providedIn: "root", }) export class WorkflowStatusService { + public runtimeEpoch = 0; + public runtimeExecutionId?: number; + private restored = false; + private runtimeContent?: WorkflowContent; + private graphProvider?: () => WorkflowContent; + private preparedExecution = false; + private wireState = ExecutionState.Uninitialized; + public getRuntimeContent(): WorkflowContent | undefined { + return this.runtimeContent; + } + public setRuntimeContent(content: WorkflowContent): void { + this.runtimeContent = structuredClone(content); + } + private rawStatus: Record = {}; + private projection?: (raw: Record) => Record; + public setProjection(projection: typeof this.projection, graph?: () => WorkflowContent): void { + this.projection = projection; + this.graphProvider = graph; + } + public refresh(): void { + this.ingestRuntimeStatus(this.rawStatus); + } // The engine streams operator state and operator statistics bundled in one // wire object (OperatorRuntimeStatus); this service splits them into two // separate sub-concepts, each with its own stream and snapshot. Derived @@ -61,15 +90,41 @@ export class WorkflowStatusService { // subscriber reading getCurrentStatistics() sees the previous emission. // Pinned by the "emits state before statistics" spec. this.workflowWebsocketService.websocketEvent().subscribe(event => { + if (event.type === "WorkflowStateEvent") { + const state = ExecutionState[event.state]; + if (event.executionId && event.executionId !== this.runtimeExecutionId) { + this.runtimeEpoch++; + this.runtimeContent = undefined; + this.runtimeExecutionId = event.executionId; + this.ingestRuntimeStatus({}); + } else if (!event.executionId && isNotInExecution(this.wireState) && !isNotInExecution(state)) { + this.runtimeEpoch++; + if (!this.preparedExecution) this.runtimeContent = undefined; + this.runtimeExecutionId = undefined; + } + if (event.executionId || !isNotInExecution(state)) { + this.restored = false; + this.preparedExecution = false; + } + this.wireState = state; + } if (event.type !== "OperatorStatisticsUpdateEvent") { return; } + if (this.restored) { + this.runtimeEpoch++; + this.runtimeExecutionId = undefined; + this.restored = false; + this.runtimeContent = undefined; + } this.ingestRuntimeStatus(event.operatorStatistics); }); } /** Splits a bundled runtime-status map into the two sub-concept emissions. */ private ingestRuntimeStatus(runtimeStatus: Record): void { + this.rawStatus = runtimeStatus; + runtimeStatus = this.projection?.(runtimeStatus) ?? runtimeStatus; const state: Record = {}; const statistics: Record = {}; for (const [operatorId, update] of Object.entries(runtimeStatus)) { @@ -87,7 +142,14 @@ export class WorkflowStatusService { * same split-and-emit path as live websocket updates — the emission order * and derived performance metrics behave identically. */ - public setExternalStatus(runtimeStatus: Record): void { + public setExternalStatus( + runtimeStatus: Record, + source?: WorkflowContent | number + ): void { + this.runtimeEpoch++; + this.runtimeContent = typeof source === "object" ? structuredClone(source) : undefined; + this.runtimeExecutionId = typeof source === "number" ? source : undefined; + this.restored = typeof source === "number"; this.ingestRuntimeStatus(runtimeStatus); } @@ -132,6 +194,12 @@ export class WorkflowStatusService { } public resetStatus(): void { + this.runtimeEpoch++; + this.runtimeContent = this.graphProvider ? structuredClone(this.graphProvider()) : undefined; + this.runtimeExecutionId = undefined; + this.restored = false; + this.preparedExecution = true; + this.rawStatus = {}; const initState: Record = {}; for (const operatorId of Object.keys(this.currentState)) { initState[operatorId] = OperatorState.Uninitialized; @@ -150,6 +218,13 @@ export class WorkflowStatusService { } public clearStatus(): void { + this.runtimeEpoch++; + this.runtimeContent = undefined; + this.runtimeExecutionId = undefined; + this.restored = false; + this.preparedExecution = false; + this.wireState = ExecutionState.Uninitialized; + this.rawStatus = {}; this.stateSubject.next({}); this.statisticsSubject.next({}); } diff --git a/frontend/src/app/workspace/types/custom-json-schema.interface.ts b/frontend/src/app/workspace/types/custom-json-schema.interface.ts index 14e81f481f6..b088fbf2535 100644 --- a/frontend/src/app/workspace/types/custom-json-schema.interface.ts +++ b/frontend/src/app/workspace/types/custom-json-schema.interface.ts @@ -18,6 +18,17 @@ */ import { JSONSchema7, JSONSchema7Definition } from "json-schema"; +import { FormlyFieldConfig } from "@ngx-formly/core"; + +/** Native field context, projected for presentation only; saved bindings retain their public IDs. */ +export interface PropertyContext { + operatorType: string; + propertyKey: string; + properties: Record; + bindings: Record; + required?: import("../util/custom-formly-type").ConditionalRequiredRule; + hiddenBy?: string; +} export const hideTypes = ["regex", "equals"] as const; export type HideType = (typeof hideTypes)[number]; @@ -78,6 +89,7 @@ export type ValueRuleSet = Readonly<{ }>; export interface CustomJSONSchema7 extends JSONSchema7 { + widget?: { formlyConfig: FormlyFieldConfig }; propertyOrder?: number; properties?: { [key: string]: CustomJSONSchema7 | boolean; diff --git a/frontend/src/app/workspace/types/execute-workflow.interface.ts b/frontend/src/app/workspace/types/execute-workflow.interface.ts index 2b5a4569404..b3939a4e4cb 100644 --- a/frontend/src/app/workspace/types/execute-workflow.interface.ts +++ b/frontend/src/app/workspace/types/execute-workflow.interface.ts @@ -53,6 +53,8 @@ export interface LogicalOperator */ export interface LogicalPlan extends Readonly<{ + macroContent?: import("@texera/workflow-macros").MacroContent; + executeOperatorIds?: string[]; operators: LogicalOperator[]; links: LogicalLink[]; opsToViewResult?: string[]; @@ -78,9 +80,11 @@ export interface OperatorStatistics aggregatedInputSize?: number; /** Absent when the snapshot has no per-port information at all; `{}` means every port was zero. */ inputPortMetrics?: Record; + inputPortSizes?: Record; aggregatedOutputRowCount: number; aggregatedOutputSize?: number; outputPortMetrics?: Record; + outputPortSizes?: Record; numWorkers?: number; aggregatedDataProcessingTime?: number; aggregatedControlProcessingTime?: number; diff --git a/frontend/src/app/workspace/types/workflow-common.interface.ts b/frontend/src/app/workspace/types/workflow-common.interface.ts index 3fb3aaa3d4b..57998292768 100644 --- a/frontend/src/app/workspace/types/workflow-common.interface.ts +++ b/frontend/src/app/workspace/types/workflow-common.interface.ts @@ -18,6 +18,14 @@ */ import { JSONSchema7 } from "json-schema"; +import type { PartitionInfo } from "@texera/workflow-macros"; +export type { + LogicalPort, + OperatorLink, + OperatorPredicate, + PartitionInfo, + PortDescription, +} from "@texera/workflow-macros"; /** * This file contains multiple type declarations related to workflow-graph. @@ -30,19 +38,6 @@ export interface Point y: number; }> {} -export interface LogicalPort - extends Readonly<{ - operatorID: string; - portID: string; - }> {} - -export type PartitionInfo = - | Readonly<{ type: "hash"; hashAttributeNames: string[] }> - | Readonly<{ type: "range"; rangeAttributeNames: string[]; rangeMin: number; rangeMax: number }> - | Readonly<{ type: "single" }> - | Readonly<{ type: "broadcast" }> - | Readonly<{ type: "none" }>; - export interface PortSchema extends Readonly<{ jsonSchema: Readonly; @@ -51,33 +46,6 @@ export interface PortSchema export interface PortProperty extends Readonly<{ partitionInfo: PartitionInfo; dependencies: { id: number; internal: boolean }[] }> {} -export interface PortDescription - extends Readonly<{ - portID: string; - displayName?: string; - disallowMultiInputs?: boolean; - isDynamicPort?: boolean; - partitionRequirement?: PartitionInfo; - dependencies?: { id: number; internal: boolean }[]; - }> {} - -export interface OperatorPredicate - extends Readonly<{ - operatorID: string; - operatorType: string; - operatorVersion: string; - operatorProperties: Readonly<{ [key: string]: any }>; - inputPorts: PortDescription[]; - outputPorts: PortDescription[]; - dynamicInputPorts?: boolean; - dynamicOutputPorts?: boolean; - showAdvanced: boolean; - isDisabled?: boolean; - viewResult?: boolean; - markedForReuse?: boolean; - customDisplayName?: string; - }> {} - export interface Comment extends Readonly<{ content: string; @@ -92,13 +60,6 @@ export interface CommentBox { commentBoxPosition: Point; } -export interface OperatorLink - extends Readonly<{ - linkID: string; - source: LogicalPort; - target: LogicalPort; - }> {} - /** * refer to src/main/scalapb/org/apache/texera/web/workflowruntimestate/ConsoleMessage.scala */ diff --git a/frontend/src/app/workspace/types/workflow-compiling.interface.ts b/frontend/src/app/workspace/types/workflow-compiling.interface.ts index 8c4499d3d4c..da95388e3ca 100644 --- a/frontend/src/app/workspace/types/workflow-compiling.interface.ts +++ b/frontend/src/app/workspace/types/workflow-compiling.interface.ts @@ -37,6 +37,7 @@ import { WorkflowFatalError } from "./workflow-websocket.interface"; */ export interface WorkflowCompilationResponse extends Readonly<{ + macroExpansion?: import("../../common/util/workflow-macro").MacroExpansion; physicalPlan?: PhysicalPlan; operatorOutputSchemas: { [key: string]: OperatorPortSchemaMap; diff --git a/frontend/src/app/workspace/types/workflow-websocket.interface.ts b/frontend/src/app/workspace/types/workflow-websocket.interface.ts index 6d0af704193..5e590c5a303 100644 --- a/frontend/src/app/workspace/types/workflow-websocket.interface.ts +++ b/frontend/src/app/workspace/types/workflow-websocket.interface.ts @@ -98,6 +98,7 @@ export type PaginationRequest = Readonly<{ columnSearch?: string; // which of the operator's output ports to read; left out, the first portIndex?: number; + executionId?: number; }>; export type PaginatedResultEvent = Readonly<{ @@ -175,6 +176,7 @@ export type DebugCommandRequest = Readonly<{ export type WorkflowStateInfo = Readonly<{ state: ExecutionState; + executionId?: number; }>; export type TexeraWebsocketRequestTypeMap = { diff --git a/frontend/src/app/workspace/util/custom-formly-type.ts b/frontend/src/app/workspace/util/custom-formly-type.ts index 33d40fe7b50..759798a27ca 100644 --- a/frontend/src/app/workspace/util/custom-formly-type.ts +++ b/frontend/src/app/workspace/util/custom-formly-type.ts @@ -17,6 +17,27 @@ * under the License. */ +import type { FormlyFieldConfig } from "@ngx-formly/core"; +import type { PropertyContext } from "../types/custom-json-schema.interface"; + +/** Read siblings in their native namespace, including unexposed defaults and a renamed Form View field. */ +export function propertyModel(field?: FormlyFieldConfig): any { + const model = field?.parent?.model; + const context = field?.props?.["propertyContext"] as PropertyContext | undefined; + if (!context || !model) return model; + return { + ...context.properties, + ...Object.fromEntries( + Object.entries(context.bindings) + .filter(([, key]) => Object.hasOwn(model, key)) + .map(([key, id]) => [key, model[id]]) + ), + ...(field?.key !== undefined && Object.hasOwn(model, String(field.key)) + ? { [context.propertyKey]: model[String(field.key)] } + : {}), + }; +} + /** * Widget types that cannot be a form field at all, so the property is not offered for exposure on * the Form View. Only the code editor: editing code is not "filling in a value", and a form reader @@ -116,3 +137,55 @@ export function customFormlyFieldType(input: { } return undefined; } + +/** A field the schema requires only while a sibling holds a particular value. */ +export interface ConditionalRequiredRule { + sibling: string; + value: unknown; + requiredOnMatch: boolean; +} + +/** + * The conditional `required` rules a schema declares, keyed by the field each + * governs. A schema states one as + * + * allOf: [{ if: { properties: { sibling: { const: v } } }, then: { required: [field] } }] + * + * with `else` for the inverted form. Validation already honours these, but a + * field's own config never learns of them, so the required marker would not + * appear. Reading them out lets the marker follow the condition the validator + * applies, and lets an operator declare it in one place rather than here. + * + * The walk covers nested schemas because a rule may govern a field inside an + * array item, where it sits under `definitions`. Keying by field name is enough: + * the marker resolves the sibling against the field's own parent model, which is + * the row for an array item and the operator for a top-level field. + */ +export function conditionalRequiredRules(schema: unknown): Map { + const rules = new Map(); + const visit = (node: any): void => { + if (node === null || typeof node !== "object") { + return; + } + for (const branch of Array.isArray(node.allOf) ? node.allOf : []) { + // `if.properties` distinguishes a real condition from the `attributeTypeRules` + // blocks, which name the sibling directly and require nothing. + const condition = branch?.if?.properties; + const sibling = condition === undefined ? undefined : Object.keys(condition)[0]; + if (sibling === undefined || !("const" in (condition[sibling] ?? {}))) { + continue; + } + for (const [outcome, requiredOnMatch] of [ + ["then", true], + ["else", false], + ] as const) { + for (const field of branch?.[outcome]?.required ?? []) { + rules.set(field, { sibling, value: condition[sibling].const, requiredOnMatch }); + } + } + } + Object.values(node).forEach(visit); + }; + visit(schema); + return rules; +} diff --git a/frontend/src/app/workspace/util/hugging-face-field-bindings.spec.ts b/frontend/src/app/workspace/util/hugging-face-field-bindings.spec.ts index 05de5e0266e..d0cc2075d73 100644 --- a/frontend/src/app/workspace/util/hugging-face-field-bindings.spec.ts +++ b/frontend/src/app/workspace/util/hugging-face-field-bindings.spec.ts @@ -49,123 +49,103 @@ function validatorPasses( } describe("applyHuggingFaceFieldBindings", () => { + it("keeps task visibility and validators on projected macro field aliases", () => { + const model = { "public-task": "image-classification", "public-image": "", "public-column": "" }; + const field: FormlyFieldConfig = { + key: "public-image", + model, + parent: { model }, + props: { + propertyContext: { + operatorType: "HuggingFace", + propertyKey: "imageInput", + properties: {}, + bindings: { task: "public-task", imageInput: "public-image", inputImageColumn: "public-column" }, + }, + }, + formControl: { value: "" } as AbstractControl, + }; + applyHuggingFaceFieldBindings(field); + const hide = field.expressions?.hide as (f: FormlyFieldConfig) => boolean; + const valid = (field.validators as any)?.requiredImageInput?.expression; + expect(hide?.(field)).toBe(false); + expect(valid?.({} as AbstractControl, field)).toBe(false); + model["public-column"] = "image"; + expect(valid({} as AbstractControl, field)).toBe(true); + model["public-task"] = "text-generation"; + expect(hide(field)).toBe(true); + }); it("hides the task field itself", () => { expect(bind("task").hide).toBe(true); }); - it("does nothing for a non-string key", () => { - const field = bind(undefined); - expect(field.expressions).toBeUndefined(); - expect(field.validators).toBeUndefined(); - }); - - it("does nothing for a key the operator does not drive", () => { - const field = bind("modelId"); + it.each([undefined, "modelId"])("leaves an unmanaged field %s alone", key => { + const field = bind(key); expect(field.expressions).toBeUndefined(); expect(field.validators).toBeUndefined(); }); - describe("imageInput", () => { - it("shows for an image task and hides otherwise", () => { - const field = bind("imageInput"); - expect(hiddenForTask(field, "image-classification")).toBe(false); - expect(hiddenForTask(field, "visual-question-answering")).toBe(false); - expect(hiddenForTask(field, "text-generation")).toBe(true); - expect(hiddenForTask(field, undefined)).toBe(true); - }); - - it("requires an upload or an Input Image Column only for an image task", () => { - const field = bind("imageInput"); - // non-image task: always valid - expect(validatorPasses(field, "requiredImageInput", { task: "text-generation" })).toBe(true); - // image task, nothing provided: invalid - expect(validatorPasses(field, "requiredImageInput", { task: "image-classification" })).toBe(false); - // satisfied by an uploaded value - expect(validatorPasses(field, "requiredImageInput", { task: "image-classification", value: "pic.png" })).toBe( - true - ); - // satisfied by an Input Image Column instead - expect( - validatorPasses(field, "requiredImageInput", { - task: "image-classification", - model: { inputImageColumn: "col" }, - }) - ).toBe(true); - }); - }); - - describe("audioInput", () => { - it("shows for an audio task and hides otherwise", () => { - const field = bind("audioInput"); - expect(hiddenForTask(field, "automatic-speech-recognition")).toBe(false); - expect(hiddenForTask(field, "image-classification")).toBe(true); - }); - - it("requires an upload or an Input Audio Column only for an audio task", () => { - const field = bind("audioInput"); - expect(validatorPasses(field, "requiredAudioInput", { task: "text-generation" })).toBe(true); - expect(validatorPasses(field, "requiredAudioInput", { task: "audio-classification" })).toBe(false); - expect(validatorPasses(field, "requiredAudioInput", { task: "audio-classification", value: "a.wav" })).toBe(true); - expect( - validatorPasses(field, "requiredAudioInput", { - task: "audio-classification", - model: { inputAudioColumn: "col" }, - }) - ).toBe(true); - }); - }); - - it("hides inputImageColumn / inputAudioColumn by their task families", () => { - expect(hiddenForTask(bind("inputImageColumn"), "object-detection")).toBe(false); - expect(hiddenForTask(bind("inputImageColumn"), "audio-classification")).toBe(true); - expect(hiddenForTask(bind("inputAudioColumn"), "audio-classification")).toBe(false); - expect(hiddenForTask(bind("inputAudioColumn"), "object-detection")).toBe(true); - }); - - describe("promptColumn", () => { - it("hides only for image-only and audio tasks", () => { - const field = bind("promptColumn"); - expect(hiddenForTask(field, "image-classification")).toBe(true); - expect(hiddenForTask(field, "audio-classification")).toBe(true); - expect(hiddenForTask(field, "text-generation")).toBe(false); - expect(hiddenForTask(field, "visual-question-answering")).toBe(false); - expect(hiddenForTask(field, undefined)).toBe(false); - }); - - it("requires a value only for a prompt-required task", () => { - const field = bind("promptColumn"); - expect(validatorPasses(field, "requiredPromptColumn", { task: "image-classification" })).toBe(true); - expect(validatorPasses(field, "requiredPromptColumn", { task: "summarization" })).toBe(false); - expect(validatorPasses(field, "requiredPromptColumn", { task: "summarization", value: "col" })).toBe(true); - }); - }); - - it("shows text-generation-only controls solely for text-generation", () => { - for (const key of ["systemPrompt", "maxNewTokens", "temperature"]) { - const field = bind(key); - expect(hiddenForTask(field, "text-generation")).toBe(false); - expect(hiddenForTask(field, "summarization")).toBe(true); - } - }); - - it("shows contextColumn only for question-answering", () => { - const field = bind("contextColumn"); - expect(hiddenForTask(field, "question-answering")).toBe(false); - expect(hiddenForTask(field, "text-generation")).toBe(true); + it.each([ + { + key: "imageInput", + visible: ["image-classification", "visual-question-answering"], + hidden: ["text-generation", undefined], + }, + { key: "audioInput", visible: ["automatic-speech-recognition"], hidden: ["image-classification"] }, + { key: "inputImageColumn", visible: ["object-detection"], hidden: ["audio-classification"] }, + { key: "inputAudioColumn", visible: ["audio-classification"], hidden: ["object-detection"] }, + { + key: "promptColumn", + visible: ["text-generation", "visual-question-answering", undefined], + hidden: ["image-classification", "audio-classification"], + }, + ...["systemPrompt", "maxNewTokens", "temperature"].map(key => ({ + key, + visible: ["text-generation"], + hidden: ["summarization"], + })), + { key: "contextColumn", visible: ["question-answering"], hidden: ["text-generation"] }, + { + key: "candidateLabels", + visible: ["zero-shot-classification", "zero-shot-image-classification"], + hidden: ["text-classification"], + }, + { key: "sentencesColumn", visible: ["sentence-similarity", "text-ranking"], hidden: ["translation"] }, + ])("shows $key for its supported task families", ({ key, visible, hidden }) => { + const field = bind(key); + for (const task of visible) expect(hiddenForTask(field, task), String(task)).toBe(false); + for (const task of hidden) expect(hiddenForTask(field, task), String(task)).toBe(true); }); - it("shows candidateLabels only for the zero-shot tasks", () => { - const field = bind("candidateLabels"); - expect(hiddenForTask(field, "zero-shot-classification")).toBe(false); - expect(hiddenForTask(field, "zero-shot-image-classification")).toBe(false); - expect(hiddenForTask(field, "text-classification")).toBe(true); + it.each([ + { + key: "imageInput", + validator: "requiredImageInput", + task: "image-classification", + column: "inputImageColumn", + upload: "pic.png", + }, + { + key: "audioInput", + validator: "requiredAudioInput", + task: "audio-classification", + column: "inputAudioColumn", + upload: "a.wav", + }, + ])("requires an upload or a column for $key when applicable", ({ key, validator, task, column, upload }) => { + const field = bind(key); + expect(validatorPasses(field, validator, { task: "text-generation" })).toBe(true); + expect(validatorPasses(field, validator, { task })).toBe(false); + expect(validatorPasses(field, validator, { task, value: upload })).toBe(true); + expect(validatorPasses(field, validator, { task, model: { [column]: "col" } })).toBe(true); + expect(validatorPasses(field, validator, { task, value: " ", model: { [column]: " " } })).toBe(false); }); - it("shows sentencesColumn only for similarity and ranking tasks", () => { - const field = bind("sentencesColumn"); - expect(hiddenForTask(field, "sentence-similarity")).toBe(false); - expect(hiddenForTask(field, "text-ranking")).toBe(false); - expect(hiddenForTask(field, "translation")).toBe(true); + it("requires a prompt column only for a prompt-required task", () => { + const field = bind("promptColumn"); + expect(validatorPasses(field, "requiredPromptColumn", { task: "image-classification" })).toBe(true); + expect(validatorPasses(field, "requiredPromptColumn", { task: "summarization" })).toBe(false); + expect(validatorPasses(field, "requiredPromptColumn", { task: "summarization", value: "col" })).toBe(true); }); it("preserves expressions already present on the field", () => { diff --git a/frontend/src/app/workspace/util/hugging-face-field-bindings.ts b/frontend/src/app/workspace/util/hugging-face-field-bindings.ts index d9724a912d7..03b61e9c574 100644 --- a/frontend/src/app/workspace/util/hugging-face-field-bindings.ts +++ b/frontend/src/app/workspace/util/hugging-face-field-bindings.ts @@ -19,6 +19,17 @@ import { AbstractControl } from "@angular/forms"; import { FormlyFieldConfig } from "@ngx-formly/core"; +import { PropertyContext } from "../types/custom-json-schema.interface"; +import { propertyModel } from "./custom-formly-type"; + +/** Only exposed siblings are writable; a Form View card also renames its own control. */ +export function huggingFacePropertyKey(field: FormlyFieldConfig, key: string): string | undefined { + const context = field.props?.["propertyContext"] as PropertyContext | undefined; + if (!context) return key; + return key === context.propertyKey ? String(field.key) : context.bindings[key]; +} + +const nativeModel = (field: FormlyFieldConfig) => propertyModel(field) ?? field.model; const imageOnlyTasks = ["image-classification", "object-detection", "image-segmentation", "image-to-text"]; const imageInputTasks = [ @@ -49,16 +60,14 @@ const promptRequiredTasks = [ "zero-shot-image-classification", ]; +const hasText = (value: unknown): value is string => typeof value === "string" && value.trim().length > 0; + const getSelectedTask = (field: FormlyFieldConfig): string | undefined => { - const fromForm = field.form?.get("task")?.value ?? field.formControl?.parent?.get("task")?.value; - if (typeof fromForm === "string" && fromForm.trim().length > 0) { - return fromForm; - } - const fromModel = field.model?.task; - if (typeof fromModel === "string" && fromModel.trim().length > 0) { - return fromModel; - } - return undefined; + const key = huggingFacePropertyKey(field, "task"); + const fromForm = key ? field.form?.get(key)?.value ?? field.formControl?.parent?.get(key)?.value : undefined; + if (hasText(fromForm)) return fromForm; + const fromModel = nativeModel(field)?.task; + return hasText(fromModel) ? fromModel : undefined; }; function hideUnlessTask(mappedField: FormlyFieldConfig, predicate: (task: string | undefined) => boolean): void { @@ -89,11 +98,11 @@ function bindMediaInput( if (t === undefined || !tasks.includes(t)) { return true; } - const inputCol = field.model?.[columnKey]; + const inputCol = nativeModel(field)?.[columnKey]; if (typeof inputCol === "string" && inputCol.trim().length > 0) { return true; } - const value = field.formControl?.value ?? field.model?.[inputKey]; + const value = field.formControl?.value ?? nativeModel(field)?.[inputKey]; return typeof value === "string" && value.trim().length > 0; }, message: () => message, @@ -114,14 +123,11 @@ function bindMediaInput( * separately in {@link customFormlyFieldType}. */ export function applyHuggingFaceFieldBindings(mappedField: FormlyFieldConfig): void { - if (mappedField.key === "task") { - mappedField.hide = true; - } + const context = mappedField.props?.["propertyContext"] as PropertyContext | undefined; + const hfKey = context?.propertyKey ?? mappedField.key; + if (typeof hfKey !== "string") return; - if (typeof mappedField.key !== "string") { - return; - } - const hfKey = mappedField.key; + if (hfKey === "task") mappedField.hide = true; if (hfKey === "imageInput") { // type ("huggingface-image-upload") is set by customFormlyFieldType bindMediaInput( @@ -160,7 +166,7 @@ export function applyHuggingFaceFieldBindings(mappedField: FormlyFieldConfig): v if (t === undefined || !promptRequiredTasks.includes(t)) { return true; } - const value = field.formControl?.value ?? field.model?.promptColumn; + const value = field.formControl?.value ?? nativeModel(field)?.promptColumn; return typeof value === "string" && value.trim().length > 0; }, message: () => "Select a prompt column for this task.", diff --git a/frontend/src/assets/operator_images/MacroInput.svg b/frontend/src/assets/operator_images/MacroInput.svg new file mode 100644 index 00000000000..a3675e6b04f --- /dev/null +++ b/frontend/src/assets/operator_images/MacroInput.svg @@ -0,0 +1,14 @@ + + + + + diff --git a/frontend/src/assets/operator_images/MacroOutput.svg b/frontend/src/assets/operator_images/MacroOutput.svg new file mode 100644 index 00000000000..143cdb1bc08 --- /dev/null +++ b/frontend/src/assets/operator_images/MacroOutput.svg @@ -0,0 +1,14 @@ + + + + + diff --git a/frontend/src/assets/operator_images/SubWorkflow.svg b/frontend/src/assets/operator_images/SubWorkflow.svg new file mode 100644 index 00000000000..9501a671af3 --- /dev/null +++ b/frontend/src/assets/operator_images/SubWorkflow.svg @@ -0,0 +1,16 @@ + + + + + + + diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 5407b53efc0..3e96f2c9558 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -6,6 +6,9 @@ "paths": { "path": [ "./node_modules/path-browserify" + ], + "@texera/workflow-macros": [ + "../common/workflow-core/src/main/typescript/workflow-macros/index.ts" ] }, "strict": true, diff --git a/frontend/yarn.lock b/frontend/yarn.lock index def213ca703..57db772158e 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -5815,6 +5815,13 @@ __metadata: languageName: node linkType: hard +"@texera/workflow-macros@file:../common/workflow-core/src/main/typescript/workflow-macros::locator=gui%40workspace%3A.": + version: 0.0.0 + resolution: "@texera/workflow-macros@file:../common/workflow-core/src/main/typescript/workflow-macros#../common/workflow-core/src/main/typescript/workflow-macros::hash=45fee0&locator=gui%40workspace%3A." + checksum: 10c0/dc3ab1a46ae5ecb850e80d50d5e2c1c9a3c45bde8e3bdbe9feb5ef8efd305262ac223348fcaafdce9cd5bf551a7744ce4ff1ce504af452d72fa37b5694b1825e + languageName: node + linkType: hard + "@tsconfig/node10@npm:^1.0.7": version: 1.0.12 resolution: "@tsconfig/node10@npm:1.0.12" @@ -10703,6 +10710,7 @@ __metadata: "@ngx-formly/ng-zorro-antd": "npm:7.1.0" "@nx/angular": "npm:23.0.2" "@schematics/angular": "npm:21.2.30" + "@texera/workflow-macros": "file:../common/workflow-core/src/main/typescript/workflow-macros" "@types/backbone": "npm:1.4.23" "@types/concaveman": "npm:1.1.6" "@types/d3-shape": "npm:2.1.7" diff --git a/sql/changelog.xml b/sql/changelog.xml index 62dfe42d522..549263c345a 100644 --- a/sql/changelog.xml +++ b/sql/changelog.xml @@ -174,6 +174,10 @@ + + + +