diff --git a/agent-server/nodejs/src/api-server.js b/agent-server/nodejs/src/api-server.js index c7150bc6bd..f8ca689e38 100644 --- a/agent-server/nodejs/src/api-server.js +++ b/agent-server/nodejs/src/api-server.js @@ -228,6 +228,31 @@ class APIServer { result = await this.getAccessibilityTree(method === 'POST' ? JSON.parse(body) : parsedUrl.query); break; + // Recording control endpoints + case '/recording/start': + if (method !== 'POST') { + this.sendError(res, 405, 'Method not allowed'); + return; + } + result = await this.startRecording(JSON.parse(body)); + break; + + case '/recording/stop': + if (method !== 'POST') { + this.sendError(res, 405, 'Method not allowed'); + return; + } + result = await this.stopRecording(JSON.parse(body)); + break; + + case '/recording/status': + if (method !== 'GET') { + this.sendError(res, 405, 'Method not allowed'); + return; + } + result = await this.getRecordingStatus(parsedUrl.query); + break; + default: this.sendError(res, 404, 'Not found'); return; @@ -1414,6 +1439,274 @@ class APIServer { return response; } + // ============================================================================ + // Recording Control Methods + // ============================================================================ + + /** + * Start recording user interactions on a browser tab. + * @param {Object} payload - Request payload + * @param {string} payload.clientId - The client ID (DevTools connection) + * @param {string} payload.tabId - The tab ID to record + * @param {string} [payload.title] - Optional title for the recording + * @param {string[]} [payload.selectorTypes] - Selector types to record (aria, css, xpath, text) + * @returns {Object} Recording start result + */ + async startRecording(payload) { + const { clientId, tabId, title, selectorTypes, selectorAttribute } = payload; + + // Validate required params + if (!clientId) { + throw new Error('Client ID is required'); + } + if (!tabId) { + throw new Error('Tab ID is required'); + } + + // Get the base client ID (without tab suffix) + const baseClientId = clientId.split(':')[0]; + const compositeClientId = `${baseClientId}:${tabId}`; + + // Find the connected client + const connection = this.browserAgentServer.connectedClients.get(compositeClientId); + if (!connection) { + logger.warn('Recording requires DevTools connection', { + compositeClientId, + availableClients: Array.from(this.browserAgentServer.connectedClients.keys()) + }); + return { + success: false, + message: `Tab ${tabId} not connected. DevTools may still be initializing - try again in a moment.`, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } + + logger.info('Starting recording', { + clientId: baseClientId, + tabId, + title, + selectorTypes + }); + + // Send RPC request to DevTools to start recording + const rpcId = uuidv4(); + const rpcRequest = { + jsonrpc: '2.0', + method: 'recording_control', + params: { + action: 'start', + title: title || `Recording ${Date.now()}`, + selectorTypes: selectorTypes || ['aria', 'css', 'xpath', 'text'], + selectorAttribute + }, + id: rpcId + }; + + try { + const response = await connection.rpcClient.callMethod( + connection.ws, + 'recording_control', + rpcRequest.params, + 30000 // 30 second timeout + ); + + return { + success: response.success, + recordingId: response.recordingId, + message: response.message, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } catch (error) { + logger.error('Failed to start recording:', error); + return { + success: false, + message: error.message, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } + } + + /** + * Stop recording and return the captured data. + * @param {Object} payload - Request payload + * @param {string} payload.clientId - The client ID + * @param {string} payload.tabId - The tab ID + * @param {string} [payload.format] - Output format: 'userflow' or 'replay' + * @returns {Object} Recording result with userFlow or replayTranscript + */ + async stopRecording(payload) { + const { clientId, tabId, format = 'userflow' } = payload; + + // Validate required params + if (!clientId) { + throw new Error('Client ID is required'); + } + if (!tabId) { + throw new Error('Tab ID is required'); + } + + // Get the base client ID (without tab suffix) + const baseClientId = clientId.split(':')[0]; + const compositeClientId = `${baseClientId}:${tabId}`; + + // Find the connected client + const connection = this.browserAgentServer.connectedClients.get(compositeClientId); + if (!connection) { + logger.warn('Stop recording requires DevTools connection', { + compositeClientId, + availableClients: Array.from(this.browserAgentServer.connectedClients.keys()) + }); + return { + success: false, + message: `Tab ${tabId} not connected. DevTools may still be initializing - try again in a moment.`, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } + + logger.info('Stopping recording', { + clientId: baseClientId, + tabId, + format + }); + + // Send RPC request to DevTools to stop recording + const rpcId = uuidv4(); + const rpcRequest = { + jsonrpc: '2.0', + method: 'recording_control', + params: { + action: 'stop', + format + }, + id: rpcId + }; + + try { + const response = await connection.rpcClient.callMethod( + connection.ws, + 'recording_control', + rpcRequest.params, + 20000 // DevTools-side stop has 10s internal timeout; allow extra for serialization + transport + ); + + return { + success: response.success, + recordingId: response.recordingId, + message: response.message, + userFlow: response.userFlow, + replayTranscript: response.replayTranscript, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } catch (error) { + logger.error('Failed to stop recording:', error); + return { + success: false, + message: error.message, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } + } + + /** + * Get the current recording status. + * @param {Object} query - Query parameters + * @param {string} query.clientId - The client ID + * @param {string} query.tabId - The tab ID + * @returns {Object} Recording status + */ + async getRecordingStatus(query) { + const { clientId, tabId } = query; + + // Validate required params + if (!clientId) { + throw new Error('Client ID is required'); + } + if (!tabId) { + throw new Error('Tab ID is required'); + } + + // Get the base client ID (without tab suffix) + const baseClientId = clientId.split(':')[0]; + const compositeClientId = `${baseClientId}:${tabId}`; + + // Find the connected client + const connection = this.browserAgentServer.connectedClients.get(compositeClientId); + if (!connection) { + logger.warn('Recording status requires DevTools connection', { + compositeClientId, + availableClients: Array.from(this.browserAgentServer.connectedClients.keys()) + }); + return { + success: false, + message: `Tab ${tabId} not connected. DevTools may still be initializing - try again in a moment.`, + isRecording: false, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } + + logger.info('Getting recording status', { + clientId: baseClientId, + tabId + }); + + // Send RPC request to DevTools to get status + const rpcId = uuidv4(); + const rpcRequest = { + jsonrpc: '2.0', + method: 'recording_control', + params: { + action: 'status' + }, + id: rpcId + }; + + try { + const response = await connection.rpcClient.callMethod( + connection.ws, + 'recording_control', + rpcRequest.params, + 10000 // 10 second timeout + ); + + return { + success: response.success, + recordingId: response.recordingId, + isRecording: response.status?.isRecording || false, + isPaused: response.status?.isPaused || false, + stepCount: response.status?.stepCount || 0, + duration_ms: response.status?.duration_ms || 0, + title: response.status?.title, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } catch (error) { + logger.error('Failed to get recording status:', error); + return { + success: false, + message: error.message, + isRecording: false, + clientId: baseClientId, + tabId, + timestamp: Date.now() + }; + } + } + sendResponse(res, statusCode, data) { res.writeHead(statusCode, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(data, null, 2)); diff --git a/config/gni/devtools_grd_files.gni b/config/gni/devtools_grd_files.gni index a7186ea4c4..1beeb977eb 100644 --- a/config/gni/devtools_grd_files.gni +++ b/config/gni/devtools_grd_files.gni @@ -839,6 +839,8 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/common/EvaluationConfig.js", "front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.js", "front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.js", + "front_end/panels/ai_chat/replay/ReplayTranscript.js", + "front_end/panels/ai_chat/replay/UserFlowConverter.js", "front_end/panels/ai_chat/tracing/TracingProvider.js", "front_end/panels/ai_chat/tracing/LangfuseProvider.js", "front_end/panels/ai_chat/tracing/TracingConfig.js", @@ -854,6 +856,8 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/agent_framework/RuntimeContext.js", "front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgent.js", + "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV1.js", + "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV2.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionVerificationAgent.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/AgentVersion.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ClickActionAgent.js", @@ -865,10 +869,10 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/agent_framework/implementation/agents/KeyboardInputActionAgent.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ScrollActionAgent.js", - "front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV1.js", "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV2.js", + "front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.js", "front_end/panels/ai_chat/common/MarkdownViewerUtil.js", "front_end/panels/ai_chat/evaluation/runner/VisionAgentEvaluationRunner.js", "front_end/panels/ai_chat/evaluation/runner/EvaluationRunner.js", diff --git a/docker/Dockerfile b/docker/Dockerfile index 8ba4f9b930..18ba0cfaa2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,5 @@ -# Multi-stage build for Chrome DevTools Frontend +# Multi-stage build for Browser Operator DevTools Frontend +# This builds directly from the Browser Operator codebase without mixing with upstream. FROM --platform=linux/amd64 ubuntu:22.04 AS builder # Install required packages @@ -27,20 +28,31 @@ RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ENV PATH="/workspace/depot_tools:${PATH}" ENV DEPOT_TOOLS_UPDATE=0 -# Follow README instructions exactly: -# fetching code -RUN mkdir devtools -WORKDIR /workspace/devtools -RUN fetch devtools-frontend +# Create workspace for Browser Operator +RUN mkdir browser-operator +WORKDIR /workspace/browser-operator -# Build steps -WORKDIR /workspace/devtools/devtools-frontend +# Copy entire Browser Operator repository +COPY . /workspace/browser-operator/src -RUN gclient sync +# Fix submodule .git reference - initialize as a fresh git repo +WORKDIR /workspace/browser-operator/src +RUN rm -f .git && git init && \ + git config user.email "docker@build" && \ + git config user.name "Docker Build" && \ + git add -A && git commit -m "Docker build" + +# Create .gclient file pointing to local source +WORKDIR /workspace/browser-operator +RUN echo 'solutions = [{"name": "src", "url": None, "managed": False}]' > .gclient + +WORKDIR /workspace/browser-operator/src + +# Ensure bootstrap is available RUN /workspace/depot_tools/ensure_bootstrap -# Build standard DevTools first -RUN npm run build +# Sync dependencies using Browser Operator's DEPS file +RUN gclient sync --nohooks --no-history # Add Browser Operator fork and switch to it # Branch is configurable via build arg (default: main) @@ -90,7 +102,7 @@ FROM --platform=linux/amd64 nginx:alpine RUN apk add --no-cache nodejs npm bash # Copy the built DevTools frontend -COPY --from=builder /workspace/devtools/devtools-frontend/out/Default/gen/front_end /usr/share/nginx/html +COPY --from=builder /workspace/browser-operator/src/out/Default/gen/front_end /usr/share/nginx/html # Copy agent-server from builder stage COPY --from=agent-server-builder /workspace/agent-server /opt/agent-server @@ -143,4 +155,4 @@ EXPOSE 8000 8080 8082 # Override the nginx entrypoint ENTRYPOINT [] -CMD ["/bin/bash", "/usr/local/bin/start-services.sh"] \ No newline at end of file +CMD ["/bin/bash", "/usr/local/bin/start-services.sh"] diff --git a/front_end/core/i18n/locales.js b/front_end/core/i18n/locales.js new file mode 100644 index 0000000000..d1f99fe8ee --- /dev/null +++ b/front_end/core/i18n/locales.js @@ -0,0 +1,8 @@ +// Generated stub for eval-runner CLI +// This file is normally generated by the build process + +export const LOCALES = ['en-US']; +export const BUNDLED_LOCALES = ['en-US']; +export const DEFAULT_LOCALE = 'en-US'; +export const REMOTE_FETCH_PATTERN = ''; +export const LOCAL_FETCH_PATTERN = './locales/@LOCALE@.json'; diff --git a/front_end/panels/ai_chat/BUILD.gn b/front_end/panels/ai_chat/BUILD.gn index 2b994b4212..4d4afd5d26 100644 --- a/front_end/panels/ai_chat/BUILD.gn +++ b/front_end/panels/ai_chat/BUILD.gn @@ -206,21 +206,23 @@ devtools_module("ai_chat") { "agent_framework/AgentSessionTypes.ts", "agent_framework/RuntimeContext.ts", "agent_framework/implementation/agents/AgentVersion.ts", - "agent_framework/implementation/agents/DirectURLNavigatorAgent.ts", - "agent_framework/implementation/agents/ResearchAgent.ts", - "agent_framework/implementation/agents/ContentWriterAgent.ts", "agent_framework/implementation/agents/ActionAgent.ts", "agent_framework/implementation/agents/ActionAgentV1.ts", "agent_framework/implementation/agents/ActionAgentV2.ts", "agent_framework/implementation/agents/ActionVerificationAgent.ts", "agent_framework/implementation/agents/ClickActionAgent.ts", + "agent_framework/implementation/agents/ContentWriterAgent.ts", + "agent_framework/implementation/agents/DirectURLNavigatorAgent.ts", + "agent_framework/implementation/agents/EcommerceProductInfoAgent.ts", "agent_framework/implementation/agents/FormFillActionAgent.ts", - "agent_framework/implementation/agents/KeyboardInputActionAgent.ts", "agent_framework/implementation/agents/HoverActionAgent.ts", + "agent_framework/implementation/agents/KeyboardInputActionAgent.ts", + "agent_framework/implementation/agents/MemoryAgent.ts", + "agent_framework/implementation/agents/ResearchAgent.ts", "agent_framework/implementation/agents/ScrollActionAgent.ts", - "agent_framework/implementation/agents/WebTaskAgent.ts", - "agent_framework/implementation/agents/EcommerceProductInfoAgent.ts", "agent_framework/implementation/agents/SearchAgent.ts", + "agent_framework/implementation/agents/SearchMemoryAgent.ts", + "agent_framework/implementation/agents/WebTaskAgent.ts", "agent_framework/implementation/ConfiguredAgents.ts", "evaluation/framework/types.ts", "evaluation/framework/judges/LLMEvaluator.ts", @@ -261,6 +263,8 @@ devtools_module("ai_chat") { "vendor/readability-source.ts", "evaluation/remote/EvaluationProtocol.ts", "evaluation/remote/EvaluationAgent.ts", + "replay/ReplayTranscript.ts", + "replay/UserFlowConverter.ts", "tracing/TracingProvider.ts", "tracing/LangfuseProvider.ts", "tracing/TracingConfig.ts", @@ -286,6 +290,7 @@ devtools_module("ai_chat") { "../../core/sdk:bundle", "../../generated:protocol", "../../models/logs:bundle", + "../recorder/models:bundle", "../../third_party/mcp-sdk:bundle", "../../third_party/mcp-sdk:bundle_v2", "../../ui/components/helpers:bundle", @@ -552,6 +557,8 @@ _ai_chat_sources = [ "vendor/readability-source.ts", "evaluation/remote/EvaluationProtocol.ts", "evaluation/remote/EvaluationAgent.ts", + "replay/ReplayTranscript.ts", + "replay/UserFlowConverter.ts", "tracing/TracingProvider.ts", "tracing/LangfuseProvider.ts", "tracing/TracingConfig.ts", diff --git a/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts b/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts index 1aff675e57..df222c3193 100644 --- a/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts +++ b/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts @@ -30,12 +30,17 @@ import { LLMConfigurationRequest, LLMConfigurationResponse, ToolExecutionRequest, + RecordingControlRequest, + RecordingControlResponse, + RecordingControlResult, + RecordingUpdateMessage, ErrorCodes, isWelcomeMessage, isRegistrationAckMessage, isEvaluationRequest, isLLMConfigurationRequest, isToolExecutionRequest, + isRecordingControlRequest, isPongMessage, createRegisterMessage, createReadyMessage, @@ -45,8 +50,16 @@ import { createErrorResponse, createLLMConfigurationResponse, createToolExecutionSuccessResponse, - createToolExecutionErrorResponse + createToolExecutionErrorResponse, + createRecordingControlResponse, + createRecordingUpdateMessage, + type RecordingSelectorType, } from './EvaluationProtocol.js'; +import * as SDK from '../../../../core/sdk/sdk.js'; +import { RecordingSession, Events as RecordingEvents } from '../../../recorder/models/RecordingSession.js'; +import type { UserFlow, SelectorType } from '../../../recorder/models/Schema.js'; +import { getUserFlowConverter } from '../../replay/UserFlowConverter.js'; +import type { ReplayTranscript } from '../../replay/ReplayTranscript.js'; const logger = createLogger('EvaluationAgent'); @@ -79,6 +92,12 @@ export class EvaluationAgent { private nanoModel: string; private orchestratorDescriptorPromise: Promise; + // Recording state + private recordingSession: RecordingSession | null = null; + private currentRecordingId: string | null = null; + private recordingStartTime: number | null = null; + private recordingUpdateListener: ((event: any) => void) | null = null; + constructor(options: EvaluationAgentOptions) { this.clientId = options.clientId; this.endpoint = options.endpoint; @@ -205,6 +224,9 @@ export class EvaluationAgent { else if (isToolExecutionRequest(message)) { await this.handleToolExecutionRequest(message); } + else if (isRecordingControlRequest(message)) { + await this.handleRecordingControlRequest(message); + } else if (isPongMessage(message)) { logger.debug('Received pong'); } @@ -1440,4 +1462,322 @@ export class EvaluationAgent { } } } + + /** + * Handle recording control requests (start, stop, status). + * Integrates with DevTools Recorder to capture user interactions. + */ + private async handleRecordingControlRequest(request: RecordingControlRequest): Promise { + const { params, id } = request; + + logger.info('Received recording control request', { + action: params.action, + hasTitle: !!params.title, + selectorTypes: params.selectorTypes, + format: params.format + }); + + try { + let result: RecordingControlResult; + + switch (params.action) { + case 'start': + result = await this.startRecording(params); + break; + case 'stop': + result = await this.stopRecording(params.format || 'userflow'); + break; + case 'status': + result = this.getRecordingStatus(); + break; + case 'pause': + result = this.pauseRecording(); + break; + case 'resume': + result = await this.resumeRecording(); + break; + default: + result = { + success: false, + message: `Unknown action: ${params.action}` + }; + } + + // Send response + const response = createRecordingControlResponse(id, result); + if (this.client) { + this.client.send(response); + } + + } catch (error) { + logger.error('Recording control failed:', error); + + const errorResponse = createErrorResponse( + id, + ErrorCodes.INTERNAL_ERROR, + 'Recording control failed', + { + action: params.action, + error: error instanceof Error ? error.message : String(error), + timestamp: new Date().toISOString() + } + ); + + if (this.client) { + this.client.send(errorResponse); + } + } + } + + /** + * Start a new recording session. + */ + private async startRecording(params: RecordingControlRequest['params']): Promise { + // Check if already recording + if (this.recordingSession) { + return { + success: false, + recordingId: this.currentRecordingId || undefined, + message: 'Recording already in progress' + }; + } + + // Get the primary page target + const targetManager = SDK.TargetManager.TargetManager.instance(); + const target = targetManager.primaryPageTarget(); + + if (!target) { + return { + success: false, + message: 'No primary page target available. Is a page loaded?' + }; + } + + // Generate recording ID + this.currentRecordingId = `rec-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + this.recordingStartTime = Date.now(); + + // Map selector types from protocol to recorder format + const selectorTypes: SelectorType[] = (params.selectorTypes || ['aria', 'css', 'xpath', 'text']) + .map(type => type as SelectorType); + + // Create recording session + this.recordingSession = new RecordingSession(target, { + title: params.title || `Recording ${this.currentRecordingId}`, + selectorTypesToRecord: selectorTypes, + selectorAttribute: params.selectorAttribute + }); + + // Set up update listener to send real-time updates + this.recordingUpdateListener = (event: any) => { + const userFlow = event.data as UserFlow; + if (this.client && this.currentRecordingId) { + const latestStep = userFlow.steps[userFlow.steps.length - 1]; + // Normalize selectors: Selector is (string | string[]), but we need string[][] + const normalizeSelectors = (selectors: any[] | undefined): string[][] | undefined => { + if (!selectors) return undefined; + return selectors.map(s => Array.isArray(s) ? s : [s]); + }; + const updateMessage = createRecordingUpdateMessage( + this.currentRecordingId, + userFlow.steps.length, + latestStep ? { + type: latestStep.type, + selectors: 'selectors' in latestStep ? normalizeSelectors((latestStep as any).selectors) : undefined, + url: 'url' in latestStep ? (latestStep as any).url : undefined, + value: 'value' in latestStep ? (latestStep as any).value : undefined + } : undefined + ); + this.client.send(updateMessage); + } + }; + + this.recordingSession.addEventListener( + RecordingEvents.RECORDING_UPDATED, + this.recordingUpdateListener as any + ); + + // Start recording + await this.recordingSession.start(); + + logger.info('Recording started', { + recordingId: this.currentRecordingId, + title: params.title, + selectorTypes + }); + + return { + success: true, + recordingId: this.currentRecordingId, + message: 'Recording started successfully' + }; + } + + /** + * Stop the current recording and return the captured data. + * + * IMPORTANT ordering: stop() must be called BEFORE cloneUserFlow() because + * RecordingClient defers all user interaction steps (clicks, keyboard, input) + * to #pendingSteps. These are only flushed to the UserFlow when stop() + * triggers DevToolsRecorder.stopRecording() → RecordingClient.stop() → + * #processPendingSteps() → window.addStep(). Cloning before stop() would + * return a UserFlow missing all user interaction steps. + */ + private async stopRecording(format: 'userflow' | 'replay'): Promise { + if (!this.recordingSession) { + return { + success: false, + message: 'No recording in progress' + }; + } + + // Remove event listener before stopping + if (this.recordingUpdateListener) { + this.recordingSession.removeEventListener( + RecordingEvents.RECORDING_UPDATED, + this.recordingUpdateListener as any + ); + this.recordingUpdateListener = null; + } + + // Stop the recording first to flush pending steps. RecordingSession.stop() + // tears down targets and injects cleanup scripts, which can hang on complex + // pages with unresponsive frames/service workers. Use a timeout so we don't + // block indefinitely. + const session = this.recordingSession; + const STOP_TIMEOUT_MS = 10000; + let stopTimedOut = false; + try { + await Promise.race([ + session.stop(), + new Promise((_, reject) => + setTimeout(() => reject(new Error('RecordingSession.stop() timed out')), STOP_TIMEOUT_MS) + ), + ]); + } catch (e) { + stopTimedOut = true; + logger.warn('Recording stop did not complete cleanly', { + error: e instanceof Error ? e.message : String(e), + }); + } + + // Brief grace period for async binding events (window.addStep calls from + // the flushed pending steps) to be processed by #handleAddStepBinding. + if (!stopTimedOut) { + await new Promise(resolve => setTimeout(resolve, 200)); + } + + // Clone AFTER stop so the UserFlow includes all flushed interaction steps. + const userFlow = session.cloneUserFlow(); + + const recordingId = this.currentRecordingId; + const duration = this.recordingStartTime ? Date.now() - this.recordingStartTime : 0; + + // Clean up state + this.recordingSession = null; + this.currentRecordingId = null; + this.recordingStartTime = null; + + logger.info('Recording stopped', { + recordingId, + stepCount: userFlow.steps.length, + duration, + format, + stopTimedOut, + }); + + // Prepare result based on format + const result: RecordingControlResult = { + success: true, + recordingId: recordingId || undefined, + message: `Recording stopped. Captured ${userFlow.steps.length} steps.` + }; + + if (format === 'userflow') { + result.userFlow = userFlow; + } else if (format === 'replay') { + // Convert to ReplayTranscript format + const converter = getUserFlowConverter(); + const transcript = converter.convert(userFlow, { + sessionId: recordingId || undefined + }); + result.replayTranscript = transcript; + } + + return result; + } + + /** + * Get the current recording status. + */ + private getRecordingStatus(): RecordingControlResult { + if (!this.recordingSession) { + return { + success: true, + message: 'No recording in progress', + status: { + isRecording: false, + isPaused: false, + stepCount: 0, + duration_ms: 0 + } + }; + } + + const userFlow = this.recordingSession.cloneUserFlow(); + const duration = this.recordingStartTime ? Date.now() - this.recordingStartTime : 0; + + return { + success: true, + recordingId: this.currentRecordingId || undefined, + message: 'Recording in progress', + status: { + isRecording: true, + isPaused: false, // RecordingSession doesn't support pause natively + stepCount: userFlow.steps.length, + duration_ms: duration, + title: userFlow.title + } + }; + } + + /** + * Pause recording (not fully supported by RecordingSession). + */ + private pauseRecording(): RecordingControlResult { + // RecordingSession doesn't have native pause support + // This is a placeholder for future implementation + if (!this.recordingSession) { + return { + success: false, + message: 'No recording in progress' + }; + } + + return { + success: false, + recordingId: this.currentRecordingId || undefined, + message: 'Pause is not supported by the current recording implementation' + }; + } + + /** + * Resume recording (not fully supported by RecordingSession). + */ + private async resumeRecording(): Promise { + // RecordingSession doesn't have native pause/resume support + // This is a placeholder for future implementation + if (!this.recordingSession) { + return { + success: false, + message: 'No recording in progress' + }; + } + + return { + success: false, + recordingId: this.currentRecordingId || undefined, + message: 'Resume is not supported by the current recording implementation' + }; + } } diff --git a/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts b/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts index 196545ddd8..0c05dd62af 100644 --- a/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts +++ b/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts @@ -436,4 +436,166 @@ export function createToolExecutionErrorResponse( }, id }; +} + +// ============================================================================ +// Recording Control Messages +// ============================================================================ + +/** + * Selector types for recording configuration. + */ +export type RecordingSelectorType = 'aria' | 'css' | 'xpath' | 'pierce' | 'text'; + +/** + * Recording control actions. + */ +export type RecordingAction = 'start' | 'stop' | 'status' | 'pause' | 'resume'; + +/** + * Request to control recording (start, stop, status). + */ +export interface RecordingControlRequest { + jsonrpc: '2.0'; + method: 'recording_control'; + params: RecordingControlParams; + id: string; +} + +export interface RecordingControlParams { + /** + * The action to perform. + */ + action: RecordingAction; + + /** + * Recording title (for 'start' action). + */ + title?: string; + + /** + * Selector types to record (for 'start' action). + * Defaults to ['aria', 'css', 'xpath', 'text']. + */ + selectorTypes?: RecordingSelectorType[]; + + /** + * Custom selector attribute (for 'start' action). + */ + selectorAttribute?: string; + + /** + * Output format when stopping (for 'stop' action). + * 'userflow' returns Puppeteer UserFlow JSON. + * 'replay' returns unified ReplayTranscript format. + */ + format?: 'userflow' | 'replay'; +} + +/** + * Response for recording control operations. + */ +export interface RecordingControlResponse { + jsonrpc: '2.0'; + result: RecordingControlResult; + id: string; +} + +export interface RecordingControlResult { + success: boolean; + + /** + * Recording ID (present after 'start' or for 'status'). + */ + recordingId?: string; + + /** + * Human-readable message. + */ + message: string; + + /** + * Current recording status (for 'status' action). + */ + status?: { + isRecording: boolean; + isPaused: boolean; + stepCount: number; + duration_ms: number; + title?: string; + }; + + /** + * UserFlow data (when format is 'userflow' and action is 'stop'). + */ + userFlow?: any; + + /** + * ReplayTranscript data (when format is 'replay' and action is 'stop'). + */ + replayTranscript?: any; +} + +/** + * Real-time recording update message (pushed from DevTools to server). + */ +export interface RecordingUpdateMessage { + type: 'recording_update'; + recordingId: string; + stepCount: number; + latestStep?: { + type: string; + selectors?: string[][]; + url?: string; + value?: string; + }; +} + +// Type guard for recording control request +export function isRecordingControlRequest(msg: any): msg is RecordingControlRequest { + return msg?.jsonrpc === '2.0' && msg?.method === 'recording_control'; +} + +// Type guard for recording update message +export function isRecordingUpdateMessage(msg: any): msg is RecordingUpdateMessage { + return msg?.type === 'recording_update'; +} + +// Helper function to create recording control request +export function createRecordingControlRequest( + id: string, + params: RecordingControlParams +): RecordingControlRequest { + return { + jsonrpc: '2.0', + method: 'recording_control', + params, + id + }; +} + +// Helper function to create recording control response +export function createRecordingControlResponse( + id: string, + result: RecordingControlResult +): RecordingControlResponse { + return { + jsonrpc: '2.0', + result, + id + }; +} + +// Helper function to create recording update message +export function createRecordingUpdateMessage( + recordingId: string, + stepCount: number, + latestStep?: RecordingUpdateMessage['latestStep'] +): RecordingUpdateMessage { + return { + type: 'recording_update', + recordingId, + stepCount, + latestStep + }; } \ No newline at end of file diff --git a/front_end/panels/ai_chat/replay/ReplayTranscript.ts b/front_end/panels/ai_chat/replay/ReplayTranscript.ts new file mode 100644 index 0000000000..56b89d09c6 --- /dev/null +++ b/front_end/panels/ai_chat/replay/ReplayTranscript.ts @@ -0,0 +1,663 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/** + * Unified Replay Transcript Format + * + * This format combines fields from multiple sources to enable robust replay: + * - AI Agent tool calls (via Langfuse traces) + * - User manual demonstrations (via DevTools Recorder UserFlow) + * + * The format is designed to support: + * - Replay via stable selectors (CSS, XPath, ARIA) + * - Debugging via session-specific nodeIds + * - Understanding via reasoning and page changes + */ + +// ============================================================================ +// Selector Types +// ============================================================================ + +/** + * Selector types supported for element resolution during replay. + * These match the DevTools Recorder selector types. + */ +export type SelectorType = 'aria' | 'css' | 'xpath' | 'pierce' | 'text'; + +/** + * A selector with its type and value. + * During replay, selectors are tried in priority order until one resolves. + */ +export interface TypedSelector { + type: SelectorType; + value: string; +} + +/** + * Element resolution data captured during recording. + * Contains multiple strategies for finding the element during replay. + */ +export interface ElementResolution { + /** + * Session-specific node ID from accessibility tree. + * Only valid during the recording session - use selectors for replay. + */ + nodeId?: string; + + /** + * Backend DOM node ID (CDP backendNodeId). + * More stable than accessibility nodeId but still session-specific. + */ + backendDOMNodeId?: number; + + /** + * Multiple selector strategies, tried in order during replay. + * Each inner array contains equivalent selectors (any can match). + */ + selectors: string[][]; + + /** + * Typed selectors with explicit type information. + * Alternative to the string[][] selectors format. + */ + typedSelectors?: TypedSelector[]; + + /** + * ARIA role of the element. + */ + role?: string; + + /** + * ARIA label or accessible name. + */ + ariaLabel?: string; + + /** + * HTML tag name (lowercase). + */ + tag?: string; + + /** + * Element attributes relevant for matching. + */ + attributes?: Record; + + /** + * Text content of the element (for text-based matching). + */ + textContent?: string; + + /** + * XPath computed from the element. + */ + xpath?: string; +} + +// ============================================================================ +// Action Types +// ============================================================================ + +/** + * Normalized action types that map to both AI agent tools and UserFlow steps. + */ +export type ActionType = + | 'navigate' // Navigate to URL + | 'click' // Click element + | 'doubleClick' // Double click element + | 'fill' // Fill input field + | 'type' // Type text (keystroke by keystroke) + | 'press' // Press keyboard key + | 'scroll' // Scroll page or element + | 'hover' // Hover over element + | 'select' // Select dropdown option + | 'wait' // Wait for condition + | 'screenshot' // Take screenshot + | 'assert' // Assert condition + | 'custom'; // Custom action + +/** + * Arguments specific to each action type. + */ +export interface ActionArgs { + // Navigate + url?: string; + + // Click/DoubleClick + offsetX?: number; + offsetY?: number; + button?: 'left' | 'middle' | 'right'; + duration?: number; // Long press duration in ms + + // Fill/Type + text?: string; + clearFirst?: boolean; + + // Press + key?: string; + modifiers?: Array<'Alt' | 'Control' | 'Meta' | 'Shift'>; + + // Scroll + direction?: 'up' | 'down' | 'left' | 'right'; + amount?: number; + x?: number; + y?: number; + + // Select + value?: string; + index?: number; + label?: string; + + // Wait + condition?: 'navigation' | 'networkIdle' | 'element' | 'timeout'; + timeout?: number; + selector?: string; + + // Assert + expectation?: string; + actual?: string; + + // Custom + customData?: any; +} + +// ============================================================================ +// Page Change Types +// ============================================================================ + +/** + * Information about page state changes after an action. + * Useful for validation during replay. + */ +export interface PageChange { + hasChanges: boolean; + summary?: string; + + /** + * Elements added to the page (accessibility tree format). + */ + added?: string[]; + + /** + * Elements removed from the page. + */ + removed?: string[]; + + /** + * Elements that were modified. + */ + modified?: string[]; + + /** + * URL change if navigation occurred. + */ + urlChange?: { + from: string; + to: string; + }; +} + +/** + * State verification performed after an action. + */ +export interface StateVerification { + verified: boolean; + summary?: string; + checks?: Array<{ + type: 'element_exists' | 'element_visible' | 'text_content' | 'url' | 'custom'; + expected: any; + actual?: any; + passed: boolean; + }>; +} + +// ============================================================================ +// Replay Step +// ============================================================================ + +/** + * A single step in the replay transcript. + * Represents one browser action (click, type, navigate, etc.). + */ +export interface ReplayStep { + /** + * Unique identifier for this step. + */ + id: string; + + /** + * Timestamp when the action was recorded. + */ + timestamp: string; + + /** + * Tool name from AI agent (e.g., 'navigate_url', 'perform_action'). + * Null for user demonstrations. + */ + tool?: string; + + /** + * Normalized action type. + */ + action: ActionType; + + /** + * Element targeting information (for element-based actions). + */ + element?: ElementResolution; + + /** + * Action-specific arguments. + */ + args: ActionArgs; + + /** + * AI reasoning for this action (from agent). + * Useful for understanding the agent's decision. + */ + reasoning?: string; + + /** + * Result of executing the action. + */ + result?: { + success: boolean; + output?: any; + error?: string; + duration_ms?: number; + }; + + /** + * Page changes detected after the action. + */ + pageChange?: PageChange; + + /** + * State verification after the action. + */ + stateVerification?: StateVerification; + + /** + * Screenshot path (relative to transcript location). + */ + screenshot?: string; + + /** + * For iframe/popup targeting. + */ + frame?: { + frameId?: string; + url?: string; + name?: string; + selectors?: string[][]; + frameIndices?: number[]; + }; + + /** + * For multi-tab targeting. + */ + target?: { + tabId?: string; + url?: string; + }; +} + +// ============================================================================ +// Metadata +// ============================================================================ + +/** + * Source of the recording. + */ +export type RecordingSource = + | 'langfuse' // From Langfuse AI agent trace + | 'user_demo' // From DevTools Recorder (user demonstration) + | 'manual' // Manually created + | 'hybrid'; // Combined from multiple sources + +/** + * Metadata about the recording. + */ +export interface ReplayMetadata { + /** + * When the recording was created. + */ + recordedAt: string; + + /** + * Source of the recording. + */ + source: RecordingSource; + + /** + * Session ID (for grouping related recordings). + */ + sessionId?: string; + + /** + * Langfuse trace ID (if source is 'langfuse'). + */ + traceId?: string; + + /** + * Starting URL for the recording. + */ + startUrl: string; + + /** + * The objective/task the agent was trying to accomplish. + */ + objective?: string; + + /** + * Agent name (if from AI agent). + */ + agent?: string; + + /** + * Model used (if from AI agent). + */ + model?: string; + + /** + * Provider used (if from AI agent). + */ + provider?: string; + + /** + * Viewport size at recording time. + */ + viewport?: { + width: number; + height: number; + deviceScaleFactor?: number; + }; + + /** + * Browser user agent at recording time. + */ + userAgent?: string; + + /** + * Custom tags for categorization. + */ + tags?: string[]; + + /** + * Recording title/name. + */ + title?: string; + + /** + * Recording description. + */ + description?: string; +} + +// ============================================================================ +// Final State +// ============================================================================ + +/** + * Expected final state after replay completion. + * Used for validation. + */ +export interface FinalState { + /** + * Expected final URL. + */ + url?: string; + + /** + * Screenshot of final state. + */ + screenshot?: string; + + /** + * Elements that should exist after replay. + */ + expectedElements?: Array<{ + selector: string; + minCount?: number; + maxCount?: number; + visible?: boolean; + }>; + + /** + * Text that should be present on the page. + */ + expectedText?: string[]; + + /** + * Custom validation criteria. + */ + customValidation?: any; +} + +// ============================================================================ +// Replay Transcript +// ============================================================================ + +/** + * The complete replay transcript format. + * This is the unified format for storing replayable browser recordings. + */ +export interface ReplayTranscript { + /** + * Format version for future compatibility. + */ + version: '1.0'; + + /** + * Recording metadata. + */ + metadata: ReplayMetadata; + + /** + * The recorded steps. + */ + steps: ReplayStep[]; + + /** + * Expected final state (optional, for validation). + */ + finalState?: FinalState; +} + +// ============================================================================ +// Replay Options +// ============================================================================ + +/** + * Options for controlling replay execution. + */ +export interface ReplayOptions { + /** + * How to handle divergence from recorded state. + */ + divergenceMode: 'strict' | 'lenient' | 'interactive' | 'adaptive'; + + /** + * Whether to stop on the first divergence. + */ + stopOnDivergence: boolean; + + /** + * Whether to take screenshots at each step. + */ + screenshotEachStep: boolean; + + /** + * Default timeout for element resolution (ms). + */ + elementTimeout: number; + + /** + * Default timeout for actions (ms). + */ + actionTimeout: number; + + /** + * Delay between steps (ms). + */ + stepDelay: number; + + /** + * Whether to validate final state. + */ + validateFinalState: boolean; + + /** + * Custom element resolution callback. + */ + onElementResolution?: (element: ElementResolution) => Promise; + + /** + * Custom divergence handler. + */ + onDivergence?: (step: ReplayStep, expected: any, actual: any) => Promise<'continue' | 'skip' | 'fail' | 'retry'>; +} + +/** + * Default replay options. + */ +export const DEFAULT_REPLAY_OPTIONS: ReplayOptions = { + divergenceMode: 'lenient', + stopOnDivergence: false, + screenshotEachStep: false, + elementTimeout: 10000, + actionTimeout: 30000, + stepDelay: 100, + validateFinalState: true, +}; + +// ============================================================================ +// Replay Result +// ============================================================================ + +/** + * Result of a single step execution during replay. + */ +export interface StepResult { + stepId: string; + success: boolean; + duration_ms: number; + divergence?: { + type: 'element_not_found' | 'result_mismatch' | 'timeout' | 'error'; + expected?: any; + actual?: any; + message: string; + }; + screenshot?: string; +} + +/** + * Result of executing a complete replay. + */ +export interface ReplayResult { + success: boolean; + totalSteps: number; + completedSteps: number; + failedSteps: number; + skippedSteps: number; + totalDuration_ms: number; + stepResults: StepResult[]; + finalStateValidation?: { + passed: boolean; + checks: Array<{ + type: string; + passed: boolean; + message?: string; + }>; + }; +} + +// ============================================================================ +// Utility Functions +// ============================================================================ + +/** + * Generate a unique step ID. + */ +export function generateStepId(): string { + return `step-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; +} + +/** + * Create an empty replay transcript. + */ +export function createEmptyTranscript( + startUrl: string, + source: RecordingSource = 'manual', + title?: string +): ReplayTranscript { + return { + version: '1.0', + metadata: { + recordedAt: new Date().toISOString(), + source, + startUrl, + title, + }, + steps: [], + }; +} + +/** + * Validate a replay transcript structure. + */ +export function validateTranscript(transcript: any): { valid: boolean; errors: string[] } { + const errors: string[] = []; + + if (!transcript) { + errors.push('Transcript is null or undefined'); + return { valid: false, errors }; + } + + if (transcript.version !== '1.0') { + errors.push(`Unsupported version: ${transcript.version}`); + } + + if (!transcript.metadata) { + errors.push('Missing metadata'); + } else { + if (!transcript.metadata.recordedAt) { + errors.push('Missing metadata.recordedAt'); + } + if (!transcript.metadata.source) { + errors.push('Missing metadata.source'); + } + if (!transcript.metadata.startUrl) { + errors.push('Missing metadata.startUrl'); + } + } + + if (!Array.isArray(transcript.steps)) { + errors.push('Steps must be an array'); + } else { + transcript.steps.forEach((step: any, index: number) => { + if (!step.id) { + errors.push(`Step ${index}: missing id`); + } + if (!step.action) { + errors.push(`Step ${index}: missing action`); + } + if (!step.timestamp) { + errors.push(`Step ${index}: missing timestamp`); + } + }); + } + + return { + valid: errors.length === 0, + errors, + }; +} + +/** + * Serialize transcript to YAML-friendly format. + * This converts dates to ISO strings and removes undefined values. + */ +export function serializeForYAML(transcript: ReplayTranscript): any { + return JSON.parse(JSON.stringify(transcript)); +} diff --git a/front_end/panels/ai_chat/replay/UserFlowConverter.ts b/front_end/panels/ai_chat/replay/UserFlowConverter.ts new file mode 100644 index 0000000000..7a07517194 --- /dev/null +++ b/front_end/panels/ai_chat/replay/UserFlowConverter.ts @@ -0,0 +1,463 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/** + * Converts DevTools Recorder UserFlow (Puppeteer format) to unified ReplayTranscript. + * + * The DevTools Recorder captures user interactions and exports them as UserFlow JSON. + * This converter transforms that format into ReplayTranscript for unified replay. + */ + +import type { + ReplayTranscript, + ReplayStep, + SelectorType, + TypedSelector, + ElementResolution, +} from './ReplayTranscript.js'; +import { generateStepId } from './ReplayTranscript.js'; +import type { + UserFlow, + Step, + Selector, + FrameSelector, + StepWithSelectors, +} from '../../recorder/models/Schema.js'; + +// Re-export UserFlow for convenience +export type { UserFlow } from '../../recorder/models/Schema.js'; + +// ============================================================================ +// Converter Options +// ============================================================================ + +export interface ConverterOptions { + /** + * Session ID for grouping. + */ + sessionId?: string; + + /** + * Override the title from UserFlow. + */ + title?: string; + + /** + * Add custom tags. + */ + tags?: string[]; + + /** + * User agent string (if known). + */ + userAgent?: string; +} + +// ============================================================================ +// Converter Implementation +// ============================================================================ + +/** + * Parse selector string to extract type and value. + * Selectors can be in format: + * - "aria/Label Text" -> { type: 'aria', value: 'Label Text' } + * - "xpath//div[@id='foo']" -> { type: 'xpath', value: '//div[@id="foo"]' } + * - "text/Submit" -> { type: 'text', value: 'Submit' } + * - "pierce/#shadow-element" -> { type: 'pierce', value: '#shadow-element' } + * - "#css-selector" -> { type: 'css', value: '#css-selector' } + */ +function parseSelector(selector: string): TypedSelector { + // Check for prefixed selectors + if (selector.startsWith('aria/')) { + return { type: 'aria', value: selector.substring(5) }; + } + if (selector.startsWith('xpath/')) { + return { type: 'xpath', value: selector.substring(6) }; + } + if (selector.startsWith('text/')) { + return { type: 'text', value: selector.substring(5) }; + } + if (selector.startsWith('pierce/')) { + return { type: 'pierce', value: selector.substring(7) }; + } + // Default to CSS + return { type: 'css', value: selector }; +} + +/** + * Normalize a Selector (string | string[]) to string[]. + */ +function normalizeSelector(selector: Selector): string[] { + return Array.isArray(selector) ? selector : [selector]; +} + +/** + * Convert UserFlow selectors to ElementResolution. + * Selectors in puppeteer-replay are Selector[] where Selector = string | string[] + */ +function convertSelectors(selectors?: Selector[]): ElementResolution | undefined { + if (!selectors || selectors.length === 0) { + return undefined; + } + + // Normalize all selectors to string[][] format for our internal use + const normalizedSelectors: string[][] = selectors.map(normalizeSelector); + + // Parse selectors to extract type information + const typedSelectors: TypedSelector[] = []; + const selectorsByType: Map = new Map(); + + for (const selectorGroup of normalizedSelectors) { + for (const selector of selectorGroup) { + const parsed = parseSelector(selector); + typedSelectors.push(parsed); + + const existing = selectorsByType.get(parsed.type) || []; + existing.push(parsed.value); + selectorsByType.set(parsed.type, existing); + } + } + + // Extract specific values for ElementResolution + const ariaSelectors = selectorsByType.get('aria') || []; + const xpathSelectors = selectorsByType.get('xpath') || []; + + return { + selectors: normalizedSelectors, + typedSelectors, + ariaLabel: ariaSelectors[0], // First ARIA selector as label + xpath: xpathSelectors[0], // First XPath as primary + }; +} + +/** + * Convert UserFlow button to standard button name. + */ +function convertButton(button?: string): 'left' | 'middle' | 'right' | undefined { + switch (button) { + case 'primary': + return 'left'; + case 'auxiliary': + return 'middle'; + case 'secondary': + return 'right'; + default: + return undefined; + } +} + +/** + * Convert frame selectors to ReplayStep frame format. + * FrameSelector in puppeteer-replay is number[] (frame indices). + */ +function convertFrame(frame?: FrameSelector): ReplayStep['frame'] | undefined { + if (!frame || frame.length === 0) { + return undefined; + } + + // Convert frame indices to selectors format + return { + selectors: [frame.map(index => `[data-frame-index="${index}"]`)], + frameIndices: frame, + }; +} + +/** + * Type guard to check if step has selectors. + */ +function hasSelectors(step: Step): step is StepWithSelectors & Step { + return 'selectors' in step; +} + +/** + * Get selectors from step if available. + */ +function getSelectors(step: Step): Selector[] | undefined { + if (hasSelectors(step)) { + return (step as StepWithSelectors).selectors; + } + return undefined; +} + +/** + * Get frame from step if available. + */ +function getFrame(step: Step): FrameSelector | undefined { + if ('frame' in step) { + return (step as { frame?: FrameSelector }).frame; + } + return undefined; +} + +/** + * Get target from step if available. + */ +function getTarget(step: Step): string | undefined { + if ('target' in step) { + return (step as { target?: string }).target; + } + return undefined; +} + +/** + * Convert a single UserFlow step to ReplayStep. + */ +function convertStep(step: Step, index: number, baseTimestamp: Date): ReplayStep { + const timestamp = new Date(baseTimestamp.getTime() + index * 1000).toISOString(); + const id = generateStepId(); + + switch (step.type) { + case 'setViewport': + return { + id, + timestamp, + action: 'custom', + args: { + customData: { + type: 'setViewport', + width: step.width, + height: step.height, + deviceScaleFactor: step.deviceScaleFactor, + isMobile: step.isMobile, + hasTouch: step.hasTouch, + isLandscape: step.isLandscape, + }, + }, + }; + + case 'navigate': + return { + id, + timestamp, + tool: 'navigate_url', + action: 'navigate', + args: { + url: step.url, + }, + }; + + case 'click': + return { + id, + timestamp, + tool: 'perform_action', + action: 'click', + element: convertSelectors(getSelectors(step)), + args: { + offsetX: step.offsetX, + offsetY: step.offsetY, + button: convertButton(step.button), + duration: step.duration, + }, + frame: convertFrame(getFrame(step)), + target: getTarget(step) ? { tabId: getTarget(step)! } : undefined, + }; + + case 'doubleClick': + return { + id, + timestamp, + tool: 'perform_action', + action: 'doubleClick', + element: convertSelectors(getSelectors(step)), + args: { + offsetX: step.offsetX, + offsetY: step.offsetY, + button: convertButton(step.button), + }, + frame: convertFrame(getFrame(step)), + target: getTarget(step) ? { tabId: getTarget(step)! } : undefined, + }; + + case 'hover': + return { + id, + timestamp, + tool: 'perform_action', + action: 'hover', + element: convertSelectors(getSelectors(step)), + args: {}, + frame: convertFrame(getFrame(step)), + target: getTarget(step) ? { tabId: getTarget(step)! } : undefined, + }; + + case 'change': + return { + id, + timestamp, + tool: 'perform_action', + action: 'fill', + element: convertSelectors(getSelectors(step)), + args: { + text: step.value, + }, + frame: convertFrame(getFrame(step)), + target: getTarget(step) ? { tabId: getTarget(step)! } : undefined, + }; + + case 'keyDown': + return { + id, + timestamp, + tool: 'perform_action', + action: 'press', + args: { + key: step.key, + }, + }; + + case 'keyUp': + // Key up is often paired with key down, skip or mark as custom + return { + id, + timestamp, + action: 'custom', + args: { + customData: { + type: 'keyUp', + key: step.key, + }, + }, + }; + + case 'scroll': + return { + id, + timestamp, + tool: 'perform_action', + action: 'scroll', + element: convertSelectors(getSelectors(step)), + args: { + x: step.x, + y: step.y, + // Infer direction from x/y values + direction: step.y && step.y > 0 ? 'down' : step.y && step.y < 0 ? 'up' : + step.x && step.x > 0 ? 'right' : step.x && step.x < 0 ? 'left' : undefined, + amount: step.y ? Math.abs(step.y) : step.x ? Math.abs(step.x) : undefined, + }, + frame: convertFrame(getFrame(step)), + target: getTarget(step) ? { tabId: getTarget(step)! } : undefined, + }; + + case 'waitForElement': + return { + id, + timestamp, + action: 'wait', + element: convertSelectors(getSelectors(step)), + args: { + condition: 'element', + timeout: step.timeout, + }, + frame: convertFrame(getFrame(step)), + target: getTarget(step) ? { tabId: getTarget(step)! } : undefined, + }; + + case 'waitForExpression': + return { + id, + timestamp, + action: 'wait', + args: { + condition: 'element', // Maps to custom expression wait + timeout: step.timeout, + customData: { + type: 'waitForExpression', + expression: step.expression, + }, + }, + }; + + default: + // Unknown step type + return { + id, + timestamp, + action: 'custom', + args: { + customData: step, + }, + }; + } +} + +/** + * Extract the starting URL from UserFlow steps. + */ +function extractStartUrl(steps: Step[]): string { + const navigateStep = steps.find(s => s.type === 'navigate'); + if (navigateStep && navigateStep.type === 'navigate') { + return navigateStep.url; + } + return 'about:blank'; +} + +/** + * Extract viewport from UserFlow steps. + */ +function extractViewport(steps: Step[]): { width: number; height: number; deviceScaleFactor?: number } | undefined { + const viewportStep = steps.find(s => s.type === 'setViewport'); + if (!viewportStep || viewportStep.type !== 'setViewport') { + return undefined; + } + return { + width: viewportStep.width, + height: viewportStep.height, + deviceScaleFactor: viewportStep.deviceScaleFactor, + }; +} + +/** + * Main converter class. + */ +export class UserFlowConverter { + /** + * Convert a UserFlow to ReplayTranscript. + */ + convert(userFlow: UserFlow, options: ConverterOptions = {}): ReplayTranscript { + const now = new Date(); + const startUrl = extractStartUrl(userFlow.steps); + const viewport = extractViewport(userFlow.steps); + + // Convert steps, filtering out setViewport (captured in metadata) + const steps: ReplayStep[] = userFlow.steps + .filter(step => step.type !== 'setViewport') + .map((step, index) => convertStep(step, index, now)); + + return { + version: '1.0', + metadata: { + recordedAt: now.toISOString(), + source: 'user_demo', + sessionId: options.sessionId, + startUrl, + title: options.title || userFlow.title, + viewport, + userAgent: options.userAgent, + tags: options.tags, + }, + steps, + }; + } + + /** + * Convert a UserFlow JSON string to ReplayTranscript. + */ + convertFromJSON(json: string, options: ConverterOptions = {}): ReplayTranscript { + const userFlow = JSON.parse(json) as UserFlow; + return this.convert(userFlow, options); + } +} + +/** + * Singleton instance for convenience. + */ +let converterInstance: UserFlowConverter | null = null; + +export function getUserFlowConverter(): UserFlowConverter { + if (!converterInstance) { + converterInstance = new UserFlowConverter(); + } + return converterInstance; +} diff --git a/front_end/panels/recorder/models/BUILD.gn b/front_end/panels/recorder/models/BUILD.gn index 129c7e5477..37f07c6a86 100644 --- a/front_end/panels/recorder/models/BUILD.gn +++ b/front_end/panels/recorder/models/BUILD.gn @@ -59,6 +59,7 @@ devtools_entrypoint("bundle") { "../../recorder/controllers/*", "../../recorder/converters/*", "../components/*", + "../../ai_chat/*", ] }