Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht
- Semver tag helper script (`scripts/release-version.sh`) with `patch`, `minor`, `major` bump modes.
- Workflow file export schema metadata (`schema`, `version`) with backward-compatible import checks.
- Workflow builder controls for undo/redo and explicit edge disconnect.
- Web recorder stop endpoint (`/api/recorders/web/stop`) and recorder navigation event capture.
- Recorder draft review panel with reorder/edit/skip controls before inserting recorded steps.

### Changed
- CI now includes browser smoke validation (`Web E2E Smoke`).
- Web editor keyboard shortcuts now include undo/redo and selection-aware delete behavior.
- Web recorder now follows capture -> review -> insert flow instead of immediate node injection.

## [1.0.7] - 2026-02-13

Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ It combines a drag-and-drop workflow studio, resilient execution, AI-assisted au
- Visual workflow builder (React Flow)
- Core editor UX: undo/redo, duplicate, edge disconnect, auto-layout, and JSON import/export
- Web automation (Playwright) and desktop automation (agent service)
- Recorder flows for web and desktop action capture
- Recorder flows for web and desktop action capture with review-before-insert draft editing
- Autopilot workflow generation from natural-language prompts
- AI nodes: `transform_llm`, `document_understanding`, `clipboard_ai_transfer`
- Integrations (`http_api`, `postgresql`, `mysql`, `mongodb`, `google_sheets`, `airtable`, `s3`)
Expand Down Expand Up @@ -53,6 +53,7 @@ Use these guided demos to evaluate the platform quickly:
- `docs/DEMOS.md#demo-2-orchestrator-unattended-queue`
- `docs/DEMOS.md#demo-3-document-understanding-and-clipboard-ai`
- `docs/DEMOS.md#demo-4-workflow-builder-mvp-controls`
- `docs/DEMOS.md#demo-5-recorder-draft-review-and-insert`

## Contributor Onboarding
New contributors should start here:
Expand Down
17 changes: 16 additions & 1 deletion apps/server/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { WebSocketServer } from "ws";
import { PrismaClient } from "@prisma/client";
import { z } from "zod";
import { authMiddleware, getAuthContext, requirePermission, signToken, verifyLogin } from "./lib/auth.js";
import { startWebRecorder, attachRecorderWs } from "./lib/recorder.js";
import { startWebRecorder, stopWebRecorder, attachRecorderWs } from "./lib/recorder.js";
import { getActiveRunCount, startRun, waitForActiveRuns } from "./lib/runner.js";
import { startDesktopRecording, stopDesktopRecording } from "./lib/agent.js";
import {
Expand Down Expand Up @@ -2176,6 +2176,21 @@ app.post("/api/recorders/web/start", canWriteWorkflows, async (req, res) => {
res.json(session);
});

app.post("/api/recorders/web/stop", canWriteWorkflows, async (req, res) => {
const schema = z.object({ sessionId: z.string().min(1) });
const parsed = schema.safeParse(req.body || {});
if (!parsed.success) {
res.status(400).json({ error: "Invalid payload" });
return;
}
const result = await stopWebRecorder(parsed.data.sessionId);
if (!result) {
res.status(404).json({ error: "Recorder session not found" });
return;
}
res.json(result);
});

app.post("/api/recorders/desktop/start", canWriteWorkflows, async (req, res) => {
const schema = z.object({ label: z.string().optional() });
const parsed = schema.safeParse(req.body || {});
Expand Down
21 changes: 20 additions & 1 deletion apps/server/src/lib/recorder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,23 @@ import { attachRecorderWs, startWebRecorder, stopWebRecorder } from "./recorder.
class FakePage {
binding: ((source: unknown, payload: any) => void) | null = null;
gotoUrl: string | null = null;
listeners = new Map<string, Array<(payload: any) => void>>();
async addInitScript(_fn: unknown) {}
async exposeBinding(_name: string, fn: (source: unknown, payload: any) => void) {
this.binding = fn;
}
on(event: string, handler: (payload: any) => void) {
const list = this.listeners.get(event) || [];
list.push(handler);
this.listeners.set(event, list);
}
async goto(url: string) {
this.gotoUrl = url;
}
emit(event: string, payload: any) {
const list = this.listeners.get(event) || [];
list.forEach((handler) => handler(payload));
}
}

class FakeContext {
Expand Down Expand Up @@ -69,12 +79,21 @@ test("web recorder broadcasts captured events to websocket clients", async () =>
assert.equal(JSON.parse(sent[0]).type, "recorder:ready");

browser.context.page.binding?.({}, { type: "click", selector: "#submit" });
browser.context.page.emit("framenavigated", {
parentFrame: () => null,
url: () => "https://example.com/next"
});
assert.equal(sent.length >= 2, true);
assert.equal(JSON.parse(sent[1]).type, "recorder:event");
assert.equal(sent.length >= 3, true);
const navEvent = JSON.parse(sent[2]);
assert.equal(navEvent.type, "recorder:event");
assert.equal(navEvent.payload.type, "navigate");

const stopped = await stopWebRecorder("session-1");
assert.equal(stopped?.events?.length, 1);
assert.equal(stopped?.events?.length, 2);
assert.equal((stopped?.events?.[0] as any).selector, "#submit");
assert.equal((stopped?.events?.[1] as any).type, "navigate");
assert.equal(browser.closed, true);
});

Expand Down
181 changes: 106 additions & 75 deletions apps/server/src/lib/recorder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ type RecorderDeps = {
const sessions = new Map<string, RecorderSession>();
const sessionsByPage = new Map<Page, RecorderSession>();

function emitSessionEvent(session: RecorderSession, payload: Record<string, unknown>) {
session.events.push(payload);
session.clients.forEach((client) => {
if (client.readyState === 1) {
client.send(JSON.stringify({ type: "recorder:event", payload }));
}
});
}

export async function startWebRecorder(
{ startUrl }: { startUrl?: string },
deps: RecorderDeps = {}
Expand All @@ -39,80 +48,6 @@ export async function startWebRecorder(
});
const context = await browser.newContext();
const page = await context.newPage();

await page.addInitScript(() => {
const buildSelector = (el: Element | null): string => {
if (!el) return "";
if ((el as HTMLElement).id) return `#${(el as HTMLElement).id}`;
const parts: string[] = [];
let current: Element | null = el;
while (current && current.nodeType === 1 && parts.length < 5) {
const tag = current.tagName.toLowerCase();
const className = (current as HTMLElement).className
.split(" ")
.filter(Boolean)
.slice(0, 2)
.join(".");
const index = current.parentElement
? Array.from(current.parentElement.children).indexOf(current) + 1
: 1;
const selector = className ? `${tag}.${className}:nth-child(${index})` : `${tag}:nth-child(${index})`;
parts.unshift(selector);
current = current.parentElement;
}
return parts.join(" > ");
};

const send = (payload: any) => {
// @ts-expect-error injected binding
window.rpaRecordEvent(payload);
};

document.addEventListener("click", (event) => {
const target = event.target as Element;
send({
type: "click",
selector: buildSelector(target),
text: (target as HTMLElement)?.innerText?.slice(0, 120)
});
}, true);

document.addEventListener("input", (event) => {
const target = event.target as HTMLInputElement;
if (!target) return;
send({
type: "fill",
selector: buildSelector(target),
value: target.value
});
}, true);

document.addEventListener("change", (event) => {
const target = event.target as HTMLInputElement;
if (!target) return;
send({
type: "change",
selector: buildSelector(target),
value: target.value
});
}, true);
});

await page.exposeBinding("rpaRecordEvent", (_source, payload) => {
const session = sessionsByPage.get(page);
if (!session) return;
session.events.push(payload);
session.clients.forEach((client) => {
if (client.readyState === 1) {
client.send(JSON.stringify({ type: "recorder:event", payload }));
}
});
});

if (startUrl) {
await page.goto(startUrl);
}

const id = makeId();
const session: RecorderSession = {
id,
Expand All @@ -124,7 +59,103 @@ export async function startWebRecorder(
sessions.set(id, session);
sessionsByPage.set(page, session);

return { sessionId: id, wsUrl: `/ws?type=recorder&sessionId=${id}` };
try {
await page.addInitScript(() => {
const buildSelector = (el: Element | null): string => {
if (!el) return "";
if ((el as HTMLElement).id) return `#${(el as HTMLElement).id}`;
const parts: string[] = [];
let current: Element | null = el;
while (current && current.nodeType === 1 && parts.length < 5) {
const tag = current.tagName.toLowerCase();
const className = (current as HTMLElement).className
.split(" ")
.filter(Boolean)
.slice(0, 2)
.join(".");
const index = current.parentElement
? Array.from(current.parentElement.children).indexOf(current) + 1
: 1;
const selector = className ? `${tag}.${className}:nth-child(${index})` : `${tag}:nth-child(${index})`;
parts.unshift(selector);
current = current.parentElement;
}
return parts.join(" > ");
};

const send = (payload: any) => {
// @ts-expect-error injected binding
window.rpaRecordEvent(payload);
};

document.addEventListener(
"click",
(event) => {
const target = event.target as Element;
send({
type: "click",
selector: buildSelector(target),
text: (target as HTMLElement)?.innerText?.slice(0, 120)
});
},
true
);

document.addEventListener(
"input",
(event) => {
const target = event.target as HTMLInputElement;
if (!target) return;
send({
type: "fill",
selector: buildSelector(target),
value: target.value
});
},
true
);

document.addEventListener(
"change",
(event) => {
const target = event.target as HTMLInputElement;
if (!target) return;
send({
type: "change",
selector: buildSelector(target),
value: target.value
});
},
true
);
});

await page.exposeBinding("rpaRecordEvent", (_source, payload) => {
const currentSession = sessionsByPage.get(page);
if (!currentSession) return;
emitSessionEvent(currentSession, payload);
});

page.on("framenavigated", (frame: any) => {
const currentSession = sessionsByPage.get(page);
if (!currentSession) return;
if (typeof frame?.parentFrame === "function" && frame.parentFrame()) return;
const url = typeof frame?.url === "function" ? String(frame.url() || "") : "";
if (!url) return;
emitSessionEvent(currentSession, { type: "navigate", url });
});

if (startUrl) {
await page.goto(startUrl);
}

return { sessionId: id, wsUrl: `/ws?type=recorder&sessionId=${id}` };
} catch (error) {
sessions.delete(id);
sessionsByPage.delete(page);
await browser.close();
throw error;
}
}

export function attachRecorderWs(wss: WebSocketServer) {
Expand Down
Loading
Loading