From fb76cbc509a627690202456e8fefd2541e34b5f6 Mon Sep 17 00:00:00 2001 From: Decvil <2534450611@qq.com> Date: Sat, 22 Aug 2026 10:38:05 +0800 Subject: [PATCH 1/4] feat(scene): add scene-get-screenshot tool for headless offscreen scene rendering with image and metadata --- package-lock.json | 265 +++++++ package.json | 9 +- .../__snapshots__/dts-snapshot.test.ts.snap | 62 ++ src/api/scene/scene.ts | 4 + src/api/scene/screenshot-schema.ts | 80 ++ src/api/scene/screenshot.ts | 107 +++ src/core/engine/@types/config.d.ts | 5 + src/core/engine/index.ts | 259 +++++- src/core/scene/browser-scene-state.ts | 45 ++ src/core/scene/common/editor/index.ts | 8 + src/core/scene/common/index.ts | 1 + src/core/scene/common/screenshot.ts | 82 ++ src/core/scene/main-process/index.ts | 5 + .../main-process/proxy/screenshot-proxy.ts | 8 + src/core/scene/main-process/rpc.ts | 2 + src/core/scene/process-rpc/process-rpc.ts | 4 + src/core/scene/scene-process/main.ts | 5 + src/core/scene/scene-process/rpc.ts | 4 + .../scene/scene-process/service/editor.ts | 117 +++ .../scene/scene-process/service/interfaces.ts | 4 + .../service/preview/scene-preview.ts | 1 + .../scene/scene-process/service/screenshot.ts | 736 ++++++++++++++++++ .../service/screenshot/screenshot-buffer.ts | 266 +++++++ .../scene/test/browser-scene-state.test.ts | 34 + .../scene/test/editor-close-options.test.ts | 97 +++ .../test/screenshot-camera-params.test.ts | 394 ++++++++++ src/mcp/image-result.ts | 19 + src/mcp/mcp.middleware.ts | 58 +- tests/mcp-image-result.test.ts | 34 + tests/scene-screenshot-api.test.ts | 132 ++++ 30 files changed, 2802 insertions(+), 45 deletions(-) create mode 100644 src/api/scene/screenshot-schema.ts create mode 100644 src/api/scene/screenshot.ts create mode 100644 src/core/scene/browser-scene-state.ts create mode 100644 src/core/scene/common/screenshot.ts create mode 100644 src/core/scene/main-process/proxy/screenshot-proxy.ts create mode 100644 src/core/scene/scene-process/service/screenshot.ts create mode 100644 src/core/scene/scene-process/service/screenshot/screenshot-buffer.ts create mode 100644 src/core/scene/test/browser-scene-state.test.ts create mode 100644 src/core/scene/test/screenshot-camera-params.test.ts create mode 100644 src/mcp/image-result.ts create mode 100644 tests/mcp-image-result.test.ts create mode 100644 tests/scene-screenshot-api.test.ts diff --git a/package-lock.json b/package-lock.json index 6e4304554..c44200f7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "@ffprobe-installer/ffprobe": "^1.4.1", "@microsoft/api-extractor": "^7.55.0", "@modelcontextprotocol/sdk": "^1.18.0", + "@napi-rs/canvas": "^1.0.7", "@sentry/node": "^10.21.0", "@types/semver": "^7.7.1", "@types/ws": "^8.18.1", @@ -6221,6 +6222,270 @@ "node": ">=18" } }, + "node_modules/@napi-rs/canvas": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas/-/canvas-1.0.7.tgz", + "integrity": "sha512-26wVFEgs6gbe7wmzCrud1pK8q6oOgcUu7OOF24BuazB8ZUCskU9ZSLrCjoWIFVxx09rjAxsXxPleaWowHvdPCA==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "1.0.7", + "@napi-rs/canvas-darwin-arm64": "1.0.7", + "@napi-rs/canvas-darwin-x64": "1.0.7", + "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.7", + "@napi-rs/canvas-linux-arm64-gnu": "1.0.7", + "@napi-rs/canvas-linux-arm64-musl": "1.0.7", + "@napi-rs/canvas-linux-riscv64-gnu": "1.0.7", + "@napi-rs/canvas-linux-x64-gnu": "1.0.7", + "@napi-rs/canvas-linux-x64-musl": "1.0.7", + "@napi-rs/canvas-win32-arm64-msvc": "1.0.7", + "@napi-rs/canvas-win32-x64-msvc": "1.0.7" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.7.tgz", + "integrity": "sha512-d5p4hHTykc/9PjiBXkvCH/IC5hT5jH7BjQ1u8ITq+G8x4VmvveOHdy/4BWYcC3ebWRmrG9zEc/oCTy+Yf3iZ4A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.7.tgz", + "integrity": "sha512-jKfZl2QDqBr6/Ap/8NKkX0Po9SFfVjUPBJbQQmYGVtQQIazWWIAO60riH3Mz2sOyysa2oJO39sLEfXerypu0vg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.7.tgz", + "integrity": "sha512-v1asrnKBu0tD+sdSD2qVIUfhoXSrr8LFTn7z76pN+8xsiOrmFcAVty57R/5DB8ZNqNLKsUBDXFSrzf4Wt9NaSg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.7.tgz", + "integrity": "sha512-c4Hcu4L5bXCgLY8jrZ2hy/Avl65MsbfH9DqNWrrd9InbpBZZF/rmrh7XkgmTUmOFcUrt/PaFSBinW2C0moDgcA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.7.tgz", + "integrity": "sha512-OVW6T1x65BTVfWb//xylCoWxbdII+nzHu3L5T035aerBAnZ5e0nmeTMkMEO9qQrVJq4WcWW8hp+T0dF+JvJPUQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.7.tgz", + "integrity": "sha512-KX/k1UO61XdKlXxhtIkq1ZUH8uP+NNK4vSrBcM2/4wWUUCZaG93BU5s0KuE9n+TFn0jEoqLSlXOuH1pYuV5dcQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.7.tgz", + "integrity": "sha512-r+0Bg+fK8r2AsrbeT7JwBUAERZSjlyhfAMQfZE/zxYGmbswS92hN0FPjynVWbeuz5fIrLfZ6JA5pH8V6drN6qw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.7.tgz", + "integrity": "sha512-0tT9KzEcTfb6MWdUWIDfy6uq6UNF691r/9NfXxxl8s9+G5QrD7VP1UC+PAwzsCzJTClIKyidNau/grYTL+QmHw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.7.tgz", + "integrity": "sha512-rvT0xo1xA+C7e+U/2ngsxWsl9gC5knHU+z8KTT5VK0Zos4AQbWZvtjvegrmzxm4o2yHE32Lexj6CDEJNvuTczA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.7.tgz", + "integrity": "sha512-N6s3yoFFPUDkfRpjUiKERVYkli7ex5Sf0Bu/My6D6gOGYxYolC1KvL6x8U0aN+ScCWAzbkIWcMYR9g0OGsFuVw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.7.tgz", + "integrity": "sha512-SKTNdBV/ljfhkPsLhXjm4x2PQ0ILpc0PhkBzRHuroJXidZUaF5yh0j3s3dIzB8PrRmW+nEASzyMTaWT3nH1ywQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmmirror.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/package.json b/package.json index 0be8fa16b..d2a2fce87 100644 --- a/package.json +++ b/package.json @@ -126,11 +126,12 @@ "@ffprobe-installer/ffprobe": "^1.4.1", "@microsoft/api-extractor": "^7.55.0", "@modelcontextprotocol/sdk": "^1.18.0", + "@napi-rs/canvas": "^1.0.7", "@sentry/node": "^10.21.0", "@types/semver": "^7.7.1", "@types/ws": "^8.18.1", - "@types/xxhashjs": "^0.2.4", "@types/xml2js": "^0.4.11", + "@types/xxhashjs": "^0.2.4", "cc": "file:./packages/cc-module", "chalk": "^2.4.2", "cli-progress": "^3.12.0", @@ -185,12 +186,12 @@ "uuid": "^8.3.2", "ws": "^8.18.3", "xhr2": "^0.2.1", + "xml2js": "^0.6.0", "xmldom": "^0.1.31", "xxhashjs": "^0.2.2", "xxtea-node": "^1.1.5", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6", - "zod-to-ts": "1.1.4", - "xml2js": "^0.6.0" + "zod-to-ts": "1.1.4" } -} \ No newline at end of file +} diff --git a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap index c6a1aba2f..62e9381da 100644 --- a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap +++ b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap @@ -6084,6 +6084,7 @@ export declare interface IBaseCreateNodeParams { keepWorldTransform?: boolean; canvasRequired?: boolean; prefabCanvasHandling?: PrefabCanvasHandling; + preflightToken?: string; unlinkPrefab?: boolean; } export declare interface IBaseIdentifier { @@ -6204,6 +6205,7 @@ export declare interface ICreateNodePreflightResult { canvasRequired: boolean; canvasPath: string | null; uiTransformPath: string | null; + preflightToken: string; } export declare interface ICreateOptions { type: ICreateType; @@ -6243,6 +6245,7 @@ export declare interface IEditorService extends IServiceEvents { create(params: ICreateOptions): Promise; hasOpen(): Promise; queryCurrent(): Promise; + withScreenshotScene(urlOrUUID: string | undefined, operation: () => Promise): Promise; getRootNode(): TEditorInstance | null; lock(): Promise; unlock(): void; @@ -6629,6 +6632,61 @@ export declare interface ISceneViewService { onComponentAdded(comp: Component): void; onComponentRemoved(comp: Component): void; } +export declare interface IScreenshotCameraInfo { + source: 'scene' | 'editor'; + nodeName?: string; + projection: 'perspective' | 'ortho'; + position: { + x: number; + y: number; + z: number; + }; + rotation: { + x: number; + y: number; + z: number; + w: number; + }; + fov?: number; + orthoHeight?: number; + priority: number; + clearFlags: number; + visibility: number; + viewport: { + x: number; + y: number; + width: number; + height: number; + }; +} +export declare interface IScreenshotNodeSummary { + name: string; + active: boolean; + components: string[]; + children?: IScreenshotNodeSummary[]; +} +export declare interface IScreenshotOptions { + sceneUrlOrUUID?: string; + width?: number; + height?: number; + camera?: string; + viewMode?: TScreenshotViewMode; +} +export declare interface IScreenshotResult { + filePath: string; + width: number; + height: number; + sceneUrl?: string; + sceneName?: string; + mtime?: number; + actualCamera: IScreenshotCameraInfo; + actualCameras: IScreenshotCameraInfo[]; + nodeSummary?: IScreenshotNodeSummary; + renderNote?: string; +} +export declare interface IScreenshotService { + capture(options: IScreenshotOptions): Promise; +} export declare interface IScriptService extends IServiceEvents { investigatePackerDriver(): Promise; loadScript(): Promise; @@ -6695,6 +6753,7 @@ export declare interface IServiceManager { SceneView: ISceneViewService; Preview: IPreviewService; UI: IUIService; + Screenshot: IScreenshotService; } export declare interface ISetParentParams { paths: string[]; @@ -7148,6 +7207,7 @@ export declare interface TextureCubeAssetUserData extends TextureBaseAssetUserDa bottom?: string; } export declare type TSceneTemplateType = typeof SCENE_TEMPLATE_TYPE[number]; +export declare type TScreenshotViewMode = 'auto' | '2d' | '3d'; export declare type UIAlignType = 'top' | 'v-center' | 'bottom' | 'left' | 'h-center' | 'right'; export declare interface Vec2 { x: number; @@ -7376,6 +7436,7 @@ export declare interface IInitEngineInfo { writablePath: string; serverURL?: string; enableCustomPipeline?: boolean; + enableHeadlessWebGL?: boolean; } export declare interface IJointTextureLayoutDeviceTip { level: JointTextureLayoutDeviceTipLevel; @@ -8378,6 +8439,7 @@ export declare interface IInitEngineInfo { writablePath: string; serverURL?: string; enableCustomPipeline?: boolean; + enableHeadlessWebGL?: boolean; } export declare interface IJointTextureLayoutDeviceTip { level: JointTextureLayoutDeviceTipLevel; diff --git a/src/api/scene/scene.ts b/src/api/scene/scene.ts index 9ee39f740..58e63560c 100644 --- a/src/api/scene/scene.ts +++ b/src/api/scene/scene.ts @@ -22,17 +22,21 @@ import { Scene, TSceneTemplateType } from '../../core/scene'; import { ComponentApi } from './component'; import { NodeApi } from './node'; import { PrefabApi } from './prefab'; +import { SceneScreenshotApi } from './screenshot'; import { options } from '../../core/builder/platforms/android/i18n/en'; export class SceneApi { public component: ComponentApi; public node: NodeApi; public prefab: PrefabApi; + public screenshot: SceneScreenshotApi; constructor() { this.component = new ComponentApi(); this.node = new NodeApi(); this.prefab = new PrefabApi(); + // 场景截图工具（实例化以触发 @tool 装饰器注册） + this.screenshot = new SceneScreenshotApi(); } @tool('scene-query-current') diff --git a/src/api/scene/screenshot-schema.ts b/src/api/scene/screenshot-schema.ts new file mode 100644 index 000000000..31dde0990 --- /dev/null +++ b/src/api/scene/screenshot-schema.ts @@ -0,0 +1,80 @@ +import { z } from 'zod'; + +/** + * scene-get-screenshot 工具的入参 / 返回 schema。 + */ +export const SchemaScreenshotOptions = z.object({ + sceneUrlOrUUID: z.string().optional().describe( + 'Optional scene asset url or uuid. If omitted, the scene currently opened in PinK is used. An explicit target is reopened from the latest imported asset and the previous scene is restored after capture. A switch that would discard unsaved scene-process edits is rejected.', + ), // 可选：先打开该场景再截图；不传则截当前打开的场景 + width: z.number().int().positive().max(4096).optional().describe( + 'Render width in pixels before compression (default: project design resolution width).', + ), + height: z.number().int().positive().max(4096).optional().describe( + 'Render height in pixels before compression (default: project design resolution height; one specified edge preserves its aspect ratio).', + ), + camera: z.string().optional().describe( + 'Optional camera node path or uuid to frame the shot. If omitted, all enabled cameras that render to the screen are composited in ascending priority order; the editor camera is used only when no scene camera is available.', + ), + viewMode: z.enum(['auto', '2d', '3d']).optional().describe( + 'View mode: auto follows the scene 2D/3D state.', + ), + format: z.enum(['jpeg', 'png']).default('jpeg').describe( + 'Output image format sent to the agent (default jpeg for smaller size).', + ), + quality: z.number().int().min(1).max(100).default(80).describe( + 'Compression quality 1-100 (jpeg / png), default 80.', + ), + maxSize: z.number().int().positive().max(4096).optional().describe( + 'Optional max long-edge size of the returned image in pixels. If omitted, the design-resolution render size is preserved.', + ), +}).describe('Scene screenshot options'); + +const SchemaScreenshotCameraInfo = z.object({ + source: z.enum(['scene', 'editor']), + nodeName: z.string().optional(), + projection: z.enum(['perspective', 'ortho']), + position: z.object({ x: z.number(), y: z.number(), z: z.number() }), + rotation: z.object({ x: z.number(), y: z.number(), z: z.number(), w: z.number() }), + fov: z.number().optional(), + orthoHeight: z.number().optional(), + priority: z.number(), + clearFlags: z.number(), + visibility: z.number(), + viewport: z.object({ + x: z.number(), + y: z.number(), + width: z.number(), + height: z.number(), + }), +}).describe('Actual camera parameters used for the shot'); + +const SchemaScreenshotNodeSummary: z.ZodType = z.lazy(() => z.object({ + name: z.string(), + active: z.boolean(), + components: z.array(z.string()), + children: z.array(SchemaScreenshotNodeSummary).optional(), +})); + +export const SchemaScreenshotResult = z.object({ + image: z.object({ + mimeType: z.string().describe('e.g. image/jpeg or image/png'), + attached: z.literal(true).describe('The rendered image is attached as an MCP image content block'), + }).describe('Metadata for the rendered image attached to the MCP response'), + meta: z.object({ + sceneUrl: z.string().optional(), + sceneName: z.string().optional(), + mtime: z.number().optional().describe('Scene file mtime in ms; use it to judge whether the shot matches the on-disk file'), + actualCamera: SchemaScreenshotCameraInfo, + actualCameras: z.array(SchemaScreenshotCameraInfo).describe('All rendered cameras in ascending priority order'), + nodeSummary: SchemaScreenshotNodeSummary.optional(), + renderNote: z.string().optional().describe('Notes about the render, e.g. editor-camera fallback'), + width: z.number().describe('Width of the returned image after maxSize processing'), + height: z.number().describe('Height of the returned image after maxSize processing'), + renderWidth: z.number().describe('Original offscreen render width before output resizing'), + renderHeight: z.number().describe('Original offscreen render height before output resizing'), + }).describe('Scene metadata bound to the screenshot'), +}).describe('Scene screenshot result'); + +export type TScreenshotOptions = z.infer; +export type TScreenshotResult = z.infer; diff --git a/src/api/scene/screenshot.ts b/src/api/scene/screenshot.ts new file mode 100644 index 000000000..808f7f321 --- /dev/null +++ b/src/api/scene/screenshot.ts @@ -0,0 +1,107 @@ +import sharp from 'sharp'; +import { readFile, unlink } from 'fs/promises'; +import { description, param, result, title, tool } from '../decorator/decorator.js'; +import { COMMON_STATUS, CommonResultType, getCommonErrorStatus } from '../base/schema-base'; +import { Scene } from '../../core/scene'; +import { + SchemaScreenshotOptions, + SchemaScreenshotResult, + TScreenshotOptions, + TScreenshotResult, +} from './screenshot-schema'; + +type TScreenshotResultWithPayload = TScreenshotResult & { + image: TScreenshotResult['image'] & { base64: string }; +}; + +/** + * 场景截图 MCP 工具。 + * + * 让具备视觉能力的 Agent 以 headless 离屏渲染的方式「看到」当前场景画面： + * 场景进程离屏渲染 → 临时 PNG → 这里用 sharp 缩放压缩 → base64 内联返回（MCP image 内容块）。 + */ +export class SceneScreenshotApi { + @tool('scene-get-screenshot') + @title('Capture scene screenshot') // 截取当前场景画面 + @description( + 'Headlessly render the currently open scene from project files and return the image (base64, for multimodal input) plus scene metadata ' + + '(scene name, file mtime, actual camera params, node-tree summary). ' + + 'Use it to visually reason about layout, spacing, occlusion and spatial relations. ' + + 'The image is scaled/compressed and is a tool-chain rendered preview, not pixel-accurate; confirm exact values with structured data.', + ) // 以 headless 离屏渲染当前场景，返回图片 + 场景元数据，供多模态视觉分析布局/遮挡/间距 + @result(SchemaScreenshotResult) + async getSceneScreenshot( + @param(SchemaScreenshotOptions) options: TScreenshotOptions, + ): Promise> { + let tempFilePath: string | undefined; + try { + // 场景进程在截图 RPC 内安全刷新目标，避免旧实例在重新打开前被保存。 + const shot = await Scene.Screenshot.capture({ + sceneUrlOrUUID: options.sceneUrlOrUUID, + width: options.width, + height: options.height, + camera: options.camera, + viewMode: options.viewMode, + }); + tempFilePath = shot.filePath; + + const format = options.format ?? 'jpeg'; + const quality = options.quality ?? 80; + const mimeType = format === 'png' ? 'image/png' : 'image/jpeg'; + + // 默认保持设计分辨率；调用方显式指定 maxSize 时才等比缩小。 + const raw = await readFile(shot.filePath); + let pipeline = sharp(raw); + if (options.maxSize != null) { + pipeline = pipeline.resize({ + width: options.maxSize, + height: options.maxSize, + fit: 'inside', + withoutEnlargement: true, + }); + } + pipeline = format === 'png' + ? pipeline.png({ quality, compressionLevel: 9 }) + : pipeline.jpeg({ quality }); + + const { data: out, info: outputInfo } = await pipeline.toBuffer({ resolveWithObject: true }); + const base64 = out.toString('base64'); + + return { + code: COMMON_STATUS.SUCCESS, + data: { + image: { base64, mimeType, attached: true }, + meta: { + sceneUrl: shot.sceneUrl, + sceneName: shot.sceneName, + mtime: shot.mtime, + actualCamera: shot.actualCamera, + actualCameras: shot.actualCameras, + nodeSummary: shot.nodeSummary, + renderNote: shot.renderNote, + width: outputInfo.width, + height: outputInfo.height, + renderWidth: shot.width, + renderHeight: shot.height, + }, + }, + }; + } catch (e) { + console.error(e); + return { + code: getCommonErrorStatus(e), + reason: e instanceof Error ? e.message : String(e), + }; + } finally { + if (tempFilePath) { + try { + await unlink(tempFilePath); + } catch (cleanupError: any) { + if (cleanupError?.code !== 'ENOENT') { + console.warn(`[Screenshot] Failed to remove temporary image ${tempFilePath}:`, cleanupError); + } + } + } + } + } +} diff --git a/src/core/engine/@types/config.d.ts b/src/core/engine/@types/config.d.ts index 6c7d63c9d..976cffbac 100644 --- a/src/core/engine/@types/config.d.ts +++ b/src/core/engine/@types/config.d.ts @@ -170,6 +170,11 @@ export interface IInitEngineInfo { writablePath: string; serverURL?: string; enableCustomPipeline?: boolean; + /** + * Use a real WebGL device backed by the Node headless-gl context. + * The default remains the empty headless device for non-rendering workers. + */ + enableHeadlessWebGL?: boolean; } interface CCEModuleConfig { diff --git a/src/core/engine/index.ts b/src/core/engine/index.ts index d8a6f2b6a..8dd1564c5 100644 --- a/src/core/engine/index.ts +++ b/src/core/engine/index.ts @@ -47,6 +47,222 @@ for (let i = 0; i <= 19; i++) { layerMask[i] = 1 << i; } +const headlessCanvasBacking = Symbol('headlessCanvasBacking'); + +/** + * The Node web adapter only implements a 2D canvas context. When a worker + * actually needs pixels (for example scene screenshots), bridge that canvas + * to headless-gl before cc.game initializes the gfx device. + */ +function installHeadlessWebGLCanvas(): void { + const globalWindow = (globalThis as any).window; + const canvas = globalWindow?.__canvas as HTMLCanvasElement | undefined; + if (!canvas) { + throw new Error('Unable to initialize headless WebGL: engine canvas is unavailable.'); + } + // findCanvas() looks up #GameCanvas. The Node adapter names its singleton + // canvas "glcanvas" by default, so without this the engine receives a plain + // HTMLElement whose getContext is missing and silently falls back to + // EmptyDevice. + canvas.id = 'GameCanvas'; + + // Load the native dependency only for rendering workers. Most CLI engine + // users intentionally keep using EmptyDevice and must not require a GL ABI. + const createContext = require('gl') as typeof import('gl'); + const context = createContext( + Math.max(1, Math.floor(canvas.width || 1)), + Math.max(1, Math.floor(canvas.height || 1)), + { + alpha: true, + antialias: true, + depth: true, + stencil: true, + premultipliedAlpha: false, + preserveDrawingBuffer: true, + }, + ); + if (!context) { + throw new Error('Unable to initialize headless WebGL: context creation failed.'); + } + + const originalGetContext = canvas.getContext.bind(canvas); + canvas.getContext = ((contextId: string, ...args: any[]) => { + if (contextId === 'webgl' || contextId === 'experimental-webgl') { + return context; + } + return originalGetContext(contextId as any, ...args); + }) as typeof canvas.getContext; + globalWindow.WebGLRenderingContext = createContext.WebGLRenderingContext; + (globalThis as any).WebGLRenderingContext = createContext.WebGLRenderingContext; + + installHeadlessCanvas2D(globalWindow); + installHeadlessImageDecoder(globalWindow); +} + +/** + * A real gfx device is optional for most scene-process APIs. Native GL/canvas + * initialization can fail on unsupported hosts, so keep the worker usable and + * let rendering-only features report that no gfx device is available. + */ +function tryInstallHeadlessWebGLCanvas(): boolean { + try { + installHeadlessWebGLCanvas(); + return true; + } catch (error) { + console.warn( + '[Engine] Headless WebGL initialization failed; falling back to EmptyDevice. ' + + 'Rendering features such as scene screenshots will be unavailable.', + error, + ); + return false; + } +} + +/** + * The Node web adapter ships a placeholder 2D context: its text methods do not + * rasterize anything. Keep the adapter's HTMLCanvasElement wrapper (the engine + * relies on its DOM identity), but back its 2D operations with Skia so labels + * produce real pixels that headless-gl can upload. + */ +function installHeadlessCanvas2D(globalWindow: any): void { + const canvasPrototype = globalWindow.HTMLCanvasElement?.prototype; + if (!canvasPrototype) { + throw new Error('Unable to initialize headless WebGL: HTMLCanvasElement is unavailable.'); + } + + const { createCanvas } = require('@napi-rs/canvas') as typeof import('@napi-rs/canvas'); + const originalGetContext = canvasPrototype.getContext; + + const getBackingCanvas = (canvas: any): any => { + const width = Math.max(1, Math.floor(canvas._width || 1)); + const height = Math.max(1, Math.floor(canvas._height || 1)); + let backing = canvas[headlessCanvasBacking]; + if (!backing) { + backing = createCanvas(width, height); + canvas[headlessCanvasBacking] = backing; + } else { + if (backing.width !== width) backing.width = width; + if (backing.height !== height) backing.height = height; + } + return backing; + }; + + Object.defineProperty(canvasPrototype, 'getContext', { + configurable: true, + value(this: any, contextId: string, ...args: any[]) { + if (contextId === '2d') { + return getBackingCanvas(this).getContext('2d', args[0]); + } + return originalGetContext.call(this, contextId, ...args); + }, + }); + + Object.defineProperty(canvasPrototype, 'width', { + configurable: true, + get(this: any) { + return this._width; + }, + set(this: any, value: number) { + this._width = Math.max(0, Math.ceil(Number(value) || 0)); + const backing = this[headlessCanvasBacking]; + if (backing) backing.width = Math.max(1, this._width); + }, + }); + + Object.defineProperty(canvasPrototype, 'height', { + configurable: true, + get(this: any) { + return this._height; + }, + set(this: any, value: number) { + this._height = Math.max(0, Math.ceil(Number(value) || 0)); + const backing = this[headlessCanvasBacking]; + if (backing) backing.height = Math.max(1, this._height); + }, + }); + + Object.defineProperty(canvasPrototype, 'data', { + configurable: true, + get(this: any) { + if (this._width <= 0 || this._height <= 0) { + return new Uint8ClampedArray(0); + } + return getBackingCanvas(this).getContext('2d').getImageData(0, 0, this._width, this._height).data; + }, + }); +} + +/** + * headless-gl accepts image-like objects only when they expose decoded pixel + * data. The Node web adapter's HTMLImageElement normally reads metadata only, + * which makes headless-gl fall back to CanvasRenderingContext2D.drawImage (not + * implemented by that adapter). Decode to RGBA before emitting the load event + * so every texture upload path can consume the image synchronously afterwards. + */ +function installHeadlessImageDecoder(globalWindow: any): void { + const imagePrototype = globalWindow.HTMLImageElement?.prototype; + const sharp = (globalThis as any).nodeEnv?.sharp; + if (!imagePrototype || !sharp) { + throw new Error('Unable to initialize headless WebGL: image decoder is unavailable.'); + } + + Object.defineProperty(imagePrototype, 'data', { + configurable: true, + get(this: any) { + return this._data; + }, + }); + + Object.defineProperty(imagePrototype, 'src', { + configurable: true, + get(this: any) { + return this._src; + }, + set(this: any, value: string | ArrayBufferView) { + this._src = value; + this._data = null; + this.complete = false; + if (value === '') { + return; + } + + let source: string | ArrayBufferView = value; + if (typeof value === 'string' && value.startsWith('data:')) { + const match = /^data:[^;,]+;base64,(.*)$/.exec(value); + if (match) { + source = Buffer.from(match[1], 'base64'); + } + } + + sharp(source) + .toColourspace('srgb') + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }) + .then(({ data, info }: { data: Buffer; info: { width: number; height: number } }) => { + // Ignore a decode that completed after the image was reused. + if (this._src !== value) { + return; + } + this.width = info.width; + this.height = info.height; + this._data = data; + this.complete = true; + setTimeout(() => this.dispatchEvent(new globalWindow.Event('load')), 0); + }) + .catch((error: unknown) => { + if (this._src !== value) { + return; + } + const label = typeof value === 'string' && !value.startsWith('data:') ? value : ''; + console.warn(`Failed to decode source image ${label}:`, error); + this._data = null; + setTimeout(() => this.dispatchEvent(new globalWindow.Event('error')), 0); + }); + }, + }); +} + const Backends = { 'physics-cannon': 'cannon.js', 'physics-ammo': 'bullet', @@ -473,27 +689,36 @@ class EngineManager implements IEngine { async initEngine(info: IInitEngineInfo, onBeforeGameInit?: () => Promise, onAfterGameInit?: () => Promise) { const { default: preload } = await import('cc/preload'); await this.importEditorExtensions(); + const requiredModules = [ + 'cc', + 'cc/editor/populate-internal-constants', + 'cc/editor/serialization', + 'cc/editor/new-gen-anim', + 'cc/editor/embedded-player', + 'cc/editor/reflection-probe', + 'cc/editor/lod-group-utils', + 'cc/editor/material', + 'cc/editor/2d-misc', + 'cc/editor/offline-mappings', + 'cc/editor/custom-pipeline', + 'cc/editor/animation-clip-migration', + 'cc/editor/exotic-animation', + 'cc/editor/color-utils', + ]; + if (info.enableHeadlessWebGL) { + // The default editor bundle only installs EmptyDevice. Importing + // this module registers WebGLDevice on cclegacy before game.init. + requiredModules.push('cce:/internal/x/cc-fu/gfx-webgl'); + } await preload({ engineRoot: this._info.typescript.path, engineDev: join(this._info.typescript.path, 'bin', '.cache', 'dev-cli'), writablePath: info.writablePath, - requiredModules: [ - 'cc', - 'cc/editor/populate-internal-constants', - 'cc/editor/serialization', - 'cc/editor/new-gen-anim', - 'cc/editor/embedded-player', - 'cc/editor/reflection-probe', - 'cc/editor/lod-group-utils', - 'cc/editor/material', - 'cc/editor/2d-misc', - 'cc/editor/offline-mappings', - 'cc/editor/custom-pipeline', - 'cc/editor/animation-clip-migration', - 'cc/editor/exotic-animation', - 'cc/editor/color-utils', - ] + requiredModules, }); + const headlessWebGLReady = info.enableHeadlessWebGL + ? tryInstallHeadlessWebGLCanvas() + : false; await this.initEditorExtensions(); const modules = this.getConfig().includeModules || []; @@ -525,7 +750,7 @@ class EngineManager implements IEngine { exactFitScreen: true, }, rendering: { - renderMode: 3, + renderMode: headlessWebGLReady ? 2 : 3, renderPipeline, customPipeline: enableCustomPipeline, highQualityMode: highQuality, diff --git a/src/core/scene/browser-scene-state.ts b/src/core/scene/browser-scene-state.ts new file mode 100644 index 000000000..af314bd24 --- /dev/null +++ b/src/core/scene/browser-scene-state.ts @@ -0,0 +1,45 @@ +export interface IBrowserSceneState { + uuid: string; + url: string; + type: string; + name: string; + updatedAt: number; +} + +/** + * Main-process bridge for the scene currently opened by PinK's browser editor. + * + * The browser editor and the headless screenshot worker are separate processes, + * so the worker cannot infer a scene switch from its own EditorService state. + */ +class BrowserSceneState { + private current: IBrowserSceneState | null = null; + + setCurrent(info: Omit): IBrowserSceneState { + if (!info?.uuid) { + throw new Error('Browser scene UUID is required.'); + } + this.current = { + uuid: info.uuid, + url: info.url ?? '', + type: info.type ?? '', + name: info.name ?? '', + updatedAt: Date.now(), + }; + return { ...this.current }; + } + + getCurrent(): IBrowserSceneState | null { + return this.current ? { ...this.current } : null; + } + + clearCurrent(expectedUuid?: string): boolean { + if (expectedUuid && this.current?.uuid !== expectedUuid) { + return false; + } + this.current = null; + return true; + } +} + +export const browserSceneState = new BrowserSceneState(); diff --git a/src/core/scene/common/editor/index.ts b/src/core/scene/common/editor/index.ts index 402ad24cf..0dbb572ea 100644 --- a/src/core/scene/common/editor/index.ts +++ b/src/core/scene/common/editor/index.ts @@ -36,6 +36,7 @@ export interface IEditorTarget { export interface IPublicEditorService extends Omit; + /** + * Run screenshot work while the selected scene is open. Explicit target + * switches are restored afterwards and never discard unsaved editor state. + * This callback-based API is scene-process internal only. + */ + withScreenshotScene(urlOrUUID: string | undefined, operation: () => Promise): Promise; + /** * */ diff --git a/src/core/scene/common/index.ts b/src/core/scene/common/index.ts index c37aabece..c4a407701 100644 --- a/src/core/scene/common/index.ts +++ b/src/core/scene/common/index.ts @@ -16,4 +16,5 @@ export * from './gizmo'; export * from './scene-view'; export * from './preview'; export * from './ui'; +export * from './screenshot'; export * from './message'; diff --git a/src/core/scene/common/screenshot.ts b/src/core/scene/common/screenshot.ts new file mode 100644 index 000000000..c8cc1bd18 --- /dev/null +++ b/src/core/scene/common/screenshot.ts @@ -0,0 +1,82 @@ +/** + * 场景截图服务的公共类型定义。 + * + * 截图由场景进程以 headless 方式离屏渲染当前打开的场景生成， + * 返回临时 PNG 文件路径 + 元数据（不跨进程传裸像素，见 screenshot.ts 说明）。 + */ + +/** 取景 / 视图模式 */ +export type TScreenshotViewMode = 'auto' | '2d' | '3d'; + +/** capture 入参（场景进程侧） */ +export interface IScreenshotOptions { + /** Optional target scene. It is refreshed for capture and the previous scene is restored afterwards. */ + sceneUrlOrUUID?: string; + /** 目标渲染宽度（像素），默认使用工程设计分辨率宽度 */ + width?: number; + /** 目标渲染高度（像素），默认使用工程设计分辨率高度；只指定一条边时按设计分辨率比例推导 */ + height?: number; + /** 指定相机：节点路径或 uuid；不传则自动选择场景相机，再退回编辑器相机 frame-all */ + camera?: string; + /** 视图模式：auto 跟随场景 2D/3D 判定 */ + viewMode?: TScreenshotViewMode; +} + +/** 实际使用的相机参数（回传给 Agent 便于复现取景） */ +export interface IScreenshotCameraInfo { + /** 取景来源：scene = 场景内 cc.Camera；editor = 编辑器相机 frame-all */ + source: 'scene' | 'editor'; + /** 相机节点名（scene 来源时有意义） */ + nodeName?: string; + projection: 'perspective' | 'ortho'; + position: { x: number; y: number; z: number }; + rotation: { x: number; y: number; z: number; w: number }; + /** 透视相机垂直 FOV（度） */ + fov?: number; + /** 正交相机高度 */ + orthoHeight?: number; + /** Camera render priority; lower values render first. */ + priority: number; + /** Original camera clear flags used for stack composition. */ + clearFlags: number; + /** Camera visibility mask. */ + visibility: number; + /** Normalized viewport rectangle. */ + viewport: { x: number; y: number; width: number; height: number }; +} + +/** 节点树概要中的单个节点 */ +export interface IScreenshotNodeSummary { + name: string; + active: boolean; + components: string[]; + children?: IScreenshotNodeSummary[]; +} + +/** 场景进程返回的原始截图结果（JSON-safe） */ +export interface IScreenshotResult { + /** 离屏渲染写出的临时 PNG 绝对路径 */ + filePath: string; + width: number; + height: number; + /** 当前场景在 asset-db 中的 url */ + sceneUrl?: string; + /** 场景文件名 */ + sceneName?: string; + /** 场景磁盘文件修改时间（ms），供 Agent 判断时效 */ + mtime?: number; + actualCamera: IScreenshotCameraInfo; + /** All cameras rendered into the offscreen target, ordered by priority. */ + actualCameras: IScreenshotCameraInfo[]; + nodeSummary?: IScreenshotNodeSummary; + /** 渲染备注（如无场景相机、退回编辑器相机等提示） */ + renderNote?: string; +} + +export interface IScreenshotEvents {} + +export interface IScreenshotService { + capture(options: IScreenshotOptions): Promise; +} + +export type IPublicScreenshotService = Pick; diff --git a/src/core/scene/main-process/index.ts b/src/core/scene/main-process/index.ts index c6d14b7dd..c5a62dfcd 100644 --- a/src/core/scene/main-process/index.ts +++ b/src/core/scene/main-process/index.ts @@ -6,17 +6,20 @@ import { ComponentProxy } from './proxy/component-proxy'; import { AssetProxy } from './proxy/asset-proxy'; import { EngineProxy } from './proxy/engine-proxy'; import { PrefabProxy } from './proxy/prefab-proxy'; +import { ScreenshotProxy } from './proxy/screenshot-proxy'; import { assetManager } from '../../assets'; import scriptManager from '../../scripting'; import { sceneConfigInstance } from '../scene-configs'; import i18n from '../../base/i18n'; +import { browserSceneState } from '../browser-scene-state'; export interface IMainModule { 'assetManager': typeof assetManager; 'programming': typeof scriptManager; 'sceneConfigInstance': typeof sceneConfigInstance; 'i18n': typeof i18n; + 'browserSceneState': typeof browserSceneState; } export const Scene = { @@ -29,6 +32,8 @@ export const Scene = { Node: NodeProxy, // 组件相关的接口 Component: ComponentProxy, + // 场景截图 + Screenshot: ScreenshotProxy, // 场景进程 worker: sceneWorker, }; diff --git a/src/core/scene/main-process/proxy/screenshot-proxy.ts b/src/core/scene/main-process/proxy/screenshot-proxy.ts new file mode 100644 index 000000000..6947da46e --- /dev/null +++ b/src/core/scene/main-process/proxy/screenshot-proxy.ts @@ -0,0 +1,8 @@ +import type { IPublicScreenshotService, IScreenshotOptions } from '../../common'; +import { Rpc } from '../rpc'; + +export const ScreenshotProxy: IPublicScreenshotService = { + capture(options: IScreenshotOptions) { + return Rpc.getInstance().request('Screenshot', 'capture', [options]); + }, +}; diff --git a/src/core/scene/main-process/rpc.ts b/src/core/scene/main-process/rpc.ts index a76bd0c0f..0c4ff2e70 100644 --- a/src/core/scene/main-process/rpc.ts +++ b/src/core/scene/main-process/rpc.ts @@ -4,6 +4,7 @@ import { assetManager } from '../../assets'; import scriptManager from '../../scripting'; import { sceneConfigInstance } from '../scene-configs'; import i18n from '../../base/i18n'; +import { browserSceneState } from '../browser-scene-state'; import type { IPublicServiceManager } from '../scene-process'; @@ -35,6 +36,7 @@ export class RpcProxy { programming: scriptManager, sceneConfigInstance: sceneConfigInstance, i18n: i18n, + browserSceneState, }); console.log(`[Node] Scene Process RPC ready ${prc ? '(Attached)' : '(Detached - Web Mode)'}`); } diff --git a/src/core/scene/process-rpc/process-rpc.ts b/src/core/scene/process-rpc/process-rpc.ts index 753c274ed..32d005f8d 100644 --- a/src/core/scene/process-rpc/process-rpc.ts +++ b/src/core/scene/process-rpc/process-rpc.ts @@ -134,6 +134,10 @@ export class ProcessRPC> { return this.process?.connected; } + public isWebTransport() { + return this.isWebMode; + } + /** * 监听 incoming 消息 */ diff --git a/src/core/scene/scene-process/main.ts b/src/core/scene/scene-process/main.ts index 47d7ba8e2..723646665 100644 --- a/src/core/scene/scene-process/main.ts +++ b/src/core/scene/scene-process/main.ts @@ -43,9 +43,14 @@ async function startup() { nativeBase: assetBase, writablePath: join(projectPath, 'temp'), enableCustomPipeline: false, + // scene-get-screenshot needs a real render target. EmptyDevice executes + // the pipeline but deliberately produces no pixels. + enableHeadlessWebGL: true, }, async () => { // 导入 service，处理装饰器，捕获开发的 api await import('./service'); + // Screenshot 依赖 pngjs/fs/os，仅在 Node 场景进程注册，避免进入浏览器 scene-bundle。 + await import('./service/screenshot'); console.log('[Scene] import service'); await Rpc.startup(); console.log('[Scene] startup Rpc'); diff --git a/src/core/scene/scene-process/rpc.ts b/src/core/scene/scene-process/rpc.ts index d3b8e7286..587448f33 100644 --- a/src/core/scene/scene-process/rpc.ts +++ b/src/core/scene/scene-process/rpc.ts @@ -26,6 +26,10 @@ export class RpcProxy { } } + isWebTransport(): boolean { + return this.rpcInstance?.isWebTransport() ?? false; + } + /** * 清理 RPC 实例 */ diff --git a/src/core/scene/scene-process/service/editor.ts b/src/core/scene/scene-process/service/editor.ts index 502d3071d..ecec6c735 100644 --- a/src/core/scene/scene-process/service/editor.ts +++ b/src/core/scene/scene-process/service/editor.ts @@ -128,6 +128,84 @@ export class EditorService extends BaseService implements IEditor return editor ? await editor.encode() : null; } + /** + * Run screenshot work against a safely prepared scene while holding the + * editor lifecycle queue. Explicit target switches are restored afterwards. + * A switch/reload that would discard unsaved scene-process edits is rejected. + */ + async withScreenshotScene(urlOrUUID: string | undefined, operation: () => Promise): Promise { + return this.runLifecycle(async () => { + const previousUuid = this.currentEditorUuid; + let targetRef = urlOrUUID; + if (!targetRef) { + try { + const browserCurrent = await Rpc.getInstance().request('browserSceneState', 'getCurrent', []); + targetRef = browserCurrent?.uuid; + } catch (error) { + // Older/standalone hosts may not provide the browser-state bridge. + console.warn('[Scene] Unable to query PinK current scene, using scene-process current scene.', error); + } + } + + const targetInfo = targetRef + ? await Rpc.getInstance().request('assetManager', 'queryAssetInfo', [targetRef]) + : null; + + if (targetRef && !targetInfo) { + throw new Error(`Unable to find screenshot scene asset: ${targetRef}`); + } + + const targetUuid = targetInfo?.uuid ?? previousUuid; + if (!targetUuid) { + return operation(); + } + + const targetChanged = Boolean(previousUuid && targetUuid !== previousUuid); + const isDirty = Boolean(previousUuid && this.isCurrentEditorDirty()); + if (isDirty && (Boolean(urlOrUUID) || targetChanged)) { + throw new Error( + `当前场景 ${previousUuid} 存在未保存修改，无法切换或重新加载截图目标 ${targetUuid}。请先保存或撤销修改。`, + ); + } + if (previousUuid && !urlOrUUID && !targetChanged && isDirty) { + // Scene-process edits and screenshots already share this instance, + // so keep unsaved edits visible in the captured image. + return operation(); + } + + const shouldRestore = Boolean(urlOrUUID && targetUuid !== previousUuid); + let primaryError: unknown; + try { + if (previousUuid) { + await this.closeUnlocked({ urlOrUUID: previousUuid, save: false }); + } + await this.openUnlocked({ urlOrUUID: targetUuid }); + return await operation(); + } catch (error) { + primaryError = error; + throw error; + } finally { + if (shouldRestore && this.currentEditorUuid !== previousUuid) { + try { + const captureUuid = this.currentEditorUuid; + if (captureUuid) { + await this.closeUnlocked({ urlOrUUID: captureUuid, save: false }); + } + if (previousUuid) { + await this.openUnlocked({ urlOrUUID: previousUuid }); + } + } catch (restoreError) { + if (primaryError) { + console.error('[Screenshot] Failed to restore the previous editor after capture failure.', restoreError); + } else { + throw restoreError; + } + } + } + } + }); + } + getRootNode(): cc.Scene | cc.Node | null { const editor = this.currentEditorUuid && this.editorMap.get(this.currentEditorUuid); return editor ? editor.getRootNode() : null; @@ -199,6 +277,7 @@ export class EditorService extends BaseService implements IEditor this.invalidateEditorSession(); this.emit('editor:open', cc.director.getScene()); this.isOpen = true; + await this.publishBrowserSceneState(assetInfo); console.log(`打开 ${assetInfo.url}`); return encode; } catch (err) { @@ -208,6 +287,7 @@ export class EditorService extends BaseService implements IEditor this.currentEditorUuid = null; this.isOpen = false; } + await this.clearBrowserSceneState(currentEditorUuid ?? uuid); console.error(err); throw err; } @@ -253,6 +333,7 @@ export class EditorService extends BaseService implements IEditor // 真正关闭编辑器时的会话清理边界；重载只复用内容卸载/挂载边界。 this.emitInternal(InternalServiceEvents.EditorDisposed); this.isOpen = false; + await this.clearBrowserSceneState(currentEditorUuid); console.log(`关闭 ${assetInfo?.url ?? urlOrUUID}`); return result; } catch (error) { @@ -485,6 +566,42 @@ export class EditorService extends BaseService implements IEditor } } + private isCurrentEditorDirty(): boolean { + try { + return Boolean(Service.Undo?.isDirty?.()); + } catch (_e) { + // UndoService may not be registered during early editor setup. + return false; + } + } + + private async publishBrowserSceneState(assetInfo: IAssetInfo): Promise { + if (!(Rpc as any).isWebTransport?.()) { + return; + } + try { + await Rpc.getInstance().request('browserSceneState', 'setCurrent', [{ + uuid: assetInfo.uuid, + url: assetInfo.url, + type: assetInfo.type, + name: assetInfo.name, + }]); + } catch (error) { + console.warn('[Scene] Failed to publish PinK current scene.', error); + } + } + + private async clearBrowserSceneState(expectedUuid: string): Promise { + if (!(Rpc as any).isWebTransport?.()) { + return; + } + try { + await Rpc.getInstance().request('browserSceneState', 'clearCurrent', [expectedUuid]); + } catch (error) { + console.warn('[Scene] Failed to clear PinK current scene.', error); + } + } + private _markUndoSaved(): void { try { Service.Undo?.markSaved(); diff --git a/src/core/scene/scene-process/service/interfaces.ts b/src/core/scene/scene-process/service/interfaces.ts index 30ddcf09f..bb0c2d14a 100644 --- a/src/core/scene/scene-process/service/interfaces.ts +++ b/src/core/scene/scene-process/service/interfaces.ts @@ -32,6 +32,8 @@ import { IPublicUIService, IUIService, IAnimationService, + IPublicScreenshotService, + IScreenshotService, } from '../../common'; /** @@ -54,6 +56,7 @@ export interface IPublicServiceManager { SceneView: IPublicSceneViewService, Preview: IPublicPreviewService, UI: IPublicUIService, + Screenshot: IPublicScreenshotService, } export interface IServiceManager { @@ -74,4 +77,5 @@ export interface IServiceManager { SceneView: ISceneViewService, Preview: IPreviewService, UI: IUIService, + Screenshot: IScreenshotService, } diff --git a/src/core/scene/scene-process/service/preview/scene-preview.ts b/src/core/scene/scene-process/service/preview/scene-preview.ts index b99650277..0eddf0cb5 100644 --- a/src/core/scene/scene-process/service/preview/scene-preview.ts +++ b/src/core/scene/scene-process/service/preview/scene-preview.ts @@ -21,6 +21,7 @@ export class ScenePreview extends PreviewBase { public onComponentAdded(comp: CameraComponent) { if (!comp) return; if (comp instanceof Camera) { + if (comp.node?.layer & editorMask) return; Promise.resolve().then(() => { if (comp.camera) comp.camera.detachCamera(); }); diff --git a/src/core/scene/scene-process/service/screenshot.ts b/src/core/scene/scene-process/service/screenshot.ts new file mode 100644 index 000000000..db02afcb9 --- /dev/null +++ b/src/core/scene/scene-process/service/screenshot.ts @@ -0,0 +1,736 @@ +import { Camera, Canvas, Color, Layers, Node as CCNode, Quat, Rect, UITransform, Vec3, renderer } from 'cc'; +import { PNG } from 'pngjs'; +import { writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { BaseService, register, Service } from './core'; +import { Rpc } from '../rpc'; +import { ScreenshotBuffer } from './screenshot/screenshot-buffer'; +import type { + IScreenshotCameraInfo, + IScreenshotEvents, + IScreenshotNodeSummary, + IScreenshotOptions, + IScreenshotResult, + IScreenshotService, +} from '../../common'; + +const DEFAULT_SIZE = 1024; +const MAX_SIZE = 4096; +const NODE_SUMMARY_MAX_DEPTH = 4; +const NODE_SUMMARY_MAX_CHILDREN = 40; + +/** 临时截图相机在场景树里的节点名，销毁后不残留 */ +const TEMP_CAMERA_NODE_NAME = '__cli_screenshot_camera__'; + +/** 从任意相机组件抽取出的、创建临时相机所需的取景参数（JSON 无关，含引擎对象） */ +interface ICameraParams { + position: Vec3; + rotation: Quat; + /** Camera.ProjectionType 数值 */ + projection: number; + priority: number; + fov: number; + fovAxis: number; + orthoHeight: number; + near: number; + far: number; + clearFlags: number; + clearColor?: Color; + clearDepth: number; + clearStencil: number; + visibility: number; + rect: Rect; + aperture?: number; + shutter?: number; + iso?: number; + screenScale?: number; + usePostProcess?: boolean; + postProcess?: any; +} + +interface IResolvedFraming { + info: IScreenshotCameraInfo; + source: 'scene' | 'editor'; + renderNote?: string; + params: ICameraParams; + /** 原始场景相机，仅用于判断它是否由 Canvas 自动适配。 */ + sourceCamera?: Camera; + /** Prefab 自动取景时直接记录 Canvas，避免依赖场景相机。 */ + canvas?: Canvas; +} + +/** + * 场景截图服务。 + * + * 以 headless 离屏方式渲染当前打开的场景一帧，读回像素编码为 PNG 写入临时文件， + * 连同实际相机参数、节点树概要、场景 mtime 一并返回。 + * + * 取景策略（PRD 4.2 / 已决策「自动 + 基础入参」）： + * 指定相机 > 场景内所有屏幕相机（按 priority 合成）> 编辑器相机当前视角兜底。 + * + * 关键实现（修复渲染崩溃）： + * 绝不改动运行中的编辑器相机 / 场景相机的 targetWindow 或 enabled 状态——那会在下一帧 + * 编辑态 tick（ForwardPipeline.getRenderPass）里读到 null 帧缓冲而崩溃 + * （见 InteractivePreview.initSceneCamera 的注释）。这里改为创建完全自持的临时相机栈， + * 把源相机的取景和合成参数拷贝过来渲染，渲染后立即禁用、移出 renderScene 并销毁其节点， + * 因此不会污染编辑态。 + * + * 为什么不直接把裸像素跨进程回传：RPC 走 JSON 序列化，Uint8Array 会被展开成巨型 + * `{"0":..}` 对象。因此在场景进程内就编码 PNG 落临时文件，只回传路径，由 API 层用 + * sharp 缩放压缩转 base64。 + */ +@register('Screenshot') +export class ScreenshotService extends BaseService implements IScreenshotService { + private _buffer: ScreenshotBuffer | null = null; + private _seq = 0; + + private get buffer(): ScreenshotBuffer { + if (!this._buffer) { + this._buffer = new ScreenshotBuffer('cli-screenshot'); + } + return this._buffer; + } + + async capture(options: IScreenshotOptions = {}): Promise { + // 截图进程可能长期驻留，工程设计分辨率变更后 cc.view 仍可能保留旧值。 + // 在重新打开场景前主动同步，让 Canvas/Widget 在实例化时就按最新设计分辨率布局。 + await (Service.Engine as any)?.syncDesignResolution?.(); + + // Keep prepare/render/restore in one serialized editor lifecycle scope. + // Explicit target captures restore the previous clean scene afterwards; + // a switch that would discard unsaved edits is rejected. + return Service.Editor.withScreenshotScene( + options.sceneUrlOrUUID, + () => this._capturePreparedScene(options), + ); + } + + private async _capturePreparedScene(options: IScreenshotOptions): Promise { + const scene = (cc as any).director?.getScene?.() as CCNode | null; + if (!scene) { + throw new Error('当前没有打开的场景，请先调用 scene-open 打开场景后再截图。'); + } + + if (!options.camera && this._findSceneCameras().length === 0) { + this._focusEditorCameraForCapture(options); + } + + const { width, height } = this._resolveCaptureSize(options, scene); + // 解析取景后，按最终离屏目标修正每台自动适配的 Canvas 相机。 + const framings = this._resolveFramings(options); + framings.forEach(framing => this._fitAlignedCanvasFraming(framing, width, height)); + + // 每个源相机对应一台完全自持的临时相机；所有临时相机共享同一个离屏窗口， + // 由 RenderWindow 按 priority 从小到大完成一帧合成。 + const temporaryCameras: Array<{ node: CCNode; comp: Camera; cam: any }> = []; + + let result: { width: number; height: number; buffer: Uint8Array }; + try { + for (let index = 0; index < framings.length; index++) { + temporaryCameras.push(this._createTemporaryCamera(scene, framings[index], index)); + } + + const renderScene = (scene as any).renderScene ?? (scene as any)._renderScene; + result = await this.buffer.render(renderScene, temporaryCameras, width, height); + } finally { + for (const { node, comp } of temporaryCameras.reverse()) { + this._teardownCamera(node, comp); + } + // 恢复编辑器正常渲染 + try { + const engine = Service.Engine as any; + if (typeof engine.forceRepaintInEditMode === 'function') { + engine.forceRepaintInEditMode(); + } else { + void engine.repaintInEditMode(); + } + } catch { + // engine 未就绪时忽略 + } + } + + const filePath = this._writePng(result.width, result.height, result.buffer); + const meta = await this._collectSceneMeta(); + const cameraInfos = framings.map(framing => framing.info); + const renderNotes = [...new Set(framings.map(framing => framing.renderNote).filter(Boolean))]; + + return { + filePath, + width: result.width, + height: result.height, + sceneUrl: meta.url, + sceneName: meta.name, + mtime: meta.mtime, + // Keep the historical singular field as the top-most/only camera. + actualCamera: cameraInfos[cameraInfos.length - 1], + actualCameras: cameraInfos, + nodeSummary: this._summarizeNode(scene, 0), + renderNote: renderNotes.length ? renderNotes.join(' ') : undefined, + }; + } + + private _createTemporaryCamera( + scene: CCNode, + framing: IResolvedFraming, + index: number, + ): { node: CCNode; comp: Camera; cam: any } { + const { params } = framing; + const node = new CCNode(`${TEMP_CAMERA_NODE_NAME}_${index}`); + let comp: Camera | null = null; + try { + // Must be on the EDITOR layer before addComponent(Camera), otherwise + // CameraService asynchronously detaches it as a normal game camera. + node.layer = Layers.Enum.EDITOR; + node.setParent(scene as any); + node.setWorldPosition(params.position); + node.setWorldRotation(params.rotation); + + comp = node.addComponent(Camera); + comp.enabled = false; + comp.priority = params.priority; + comp.projection = params.projection; + comp.fov = params.fov; + comp.fovAxis = params.fovAxis; + comp.orthoHeight = params.orthoHeight; + comp.near = params.near; + comp.far = params.far; + comp.clearFlags = params.clearFlags; + comp.clearDepth = params.clearDepth; + comp.clearStencil = params.clearStencil; + comp.visibility = params.visibility; + comp.rect = params.rect; + if (params.clearColor) comp.clearColor = params.clearColor; + if (params.aperture != null) comp.aperture = params.aperture; + if (params.shutter != null) comp.shutter = params.shutter; + if (params.iso != null) comp.iso = params.iso; + if (params.screenScale != null) comp.screenScale = params.screenScale; + if (params.postProcess != null) comp.postProcess = params.postProcess; + if (params.usePostProcess != null) comp.usePostProcess = params.usePostProcess; + + let cam = (comp as any).camera; + if (!cam) { + (comp as any)._createCamera?.(); + cam = (comp as any).camera; + } + if (!cam) { + throw new Error(`创建临时截图相机失败：${framing.info.nodeName ?? index}`); + } + cam.isWindowSize = false; + cam.enabled = false; + // Edit mode only renders EDITOR-purpose cameras into its pipeline. + cam.cameraUsage = renderer.scene.CameraUsage.EDITOR; + (node as any).updateWorldTransform?.(); + cam.update(true); + return { node, comp, cam }; + } catch (error) { + // The caller can only track a camera after this method returns. Clean + // up locally when construction or parameter assignment fails midway. + this._teardownCamera(node, comp); + throw error; + } + } + + // ---- 取景 ---- + + private _resolveFramings(options: IScreenshotOptions): IResolvedFraming[] { + // 1) 指定相机 + if (options.camera) { + const cam = this._findCameraByRef(options.camera); + if (!cam) { + throw new Error(`未找到指定相机：${options.camera}`); + } + const params = this._paramsFromComponent(cam); + return [{ + info: this._cameraInfoFromParams(params, 'scene', cam.node.name), + source: 'scene', + params, + sourceCamera: cam, + }]; + } + + // 2) 场景内所有输出到屏幕的启用相机，保持原 priority 合成顺序。 + const sceneCameras = this._findSceneCameras(); + if (sceneCameras.length) { + return sceneCameras.map((camera) => { + const params = this._paramsFromComponent(camera); + return { + info: this._cameraInfoFromParams(params, 'scene', camera.node.name), + source: 'scene', + params, + sourceCamera: camera, + }; + }); + } + + // UI Prefab 在 headless 场景进程中没有浏览器视口尺寸，编辑器 2D + // 相机可能尚未得到有效的 orthoHeight。直接按其预览 Canvas 的世界 + // 边界构造取景参数，避免只渲染出清屏颜色。 + const prefab2D = this._resolvePrefab2DFraming(options); + if (prefab2D) { + return [prefab2D]; + } + + // 3) 编辑器相机当前视角兜底（不做 frame-all，避免移动用户的编辑器视角） + const editorCam = (Service.Camera as any)?.getCamera?.(); + if (!editorCam || !editorCam.node) { + throw new Error('场景内没有可用相机，且编辑器相机不可用，无法截图。'); + } + const params = this._paramsFromComponent(editorCam); + // 编辑器相机通常没有面向游戏内容的 clearColor / visibility，这里给出安全默认： + // 纯色背景 + 排除性能面板与 Gizmo 图层，尽量呈现场景内容本身。 + params.clearFlags = Camera.ClearFlag.SOLID_COLOR; + params.clearColor = new Color(51, 51, 51, 255); + params.visibility = Layers.makeMaskExclude([ + Layers.BitMask.PROFILER, + Layers.Enum.GIZMOS, + Layers.Enum.SCENE_GIZMO, + ]); + return [{ + info: this._cameraInfoFromParams(params, 'editor', editorCam.node.name), + source: 'editor', + params, + renderNote: '场景内无相机，已使用编辑器相机当前视角取景（不含 Gizmo/网格）。', + }]; + } + + private get _editorMask(): number { + return Layers.makeMaskInclude([ + Layers.Enum.GIZMOS, + Layers.Enum.SCENE_GIZMO, + Layers.Enum.EDITOR, + ]); + } + + private _findSceneCameras(): Camera[] { + const root = Service.Editor?.getRootNode?.() as CCNode | null; + if (!root) return []; + const editorCamera = (Service.Camera as any)?.getCamera?.(); + const cams: Camera[] = (root as any).getComponentsInChildren?.(Camera) ?? []; + const candidates = cams.filter((cam) => { + if (!cam || cam === editorCamera) return false; + if (!cam.enabledInHierarchy) return false; + if (cam.targetTexture) return false; + if (cam.node && (cam.node.layer & this._editorMask)) return false; + return true; + }); + // RenderWindow uses ascending priority: lower cameras establish the + // background and higher cameras compose overlays such as UI afterwards. + candidates.sort((a, b) => (a.priority ?? 0) - (b.priority ?? 0)); + return candidates; + } + + private _findCameraByRef(ref: string): Camera | null { + const EditorExtends = (cc as any).EditorExtends || (globalThis as any).EditorExtends; + let node: CCNode | null = null; + // 先按路径，再按 uuid + node = EditorExtends?.Node?.getNodeByPath?.(ref) ?? EditorExtends?.Node?.getNode?.(ref) ?? null; + if (!node) return null; + return node.getComponent(Camera) ?? node.getComponentInChildren(Camera) ?? null; + } + + private _resolvePrefab2DFraming(options: IScreenshotOptions): { + info: IScreenshotCameraInfo; + source: 'editor'; + renderNote: string; + params: ICameraParams; + canvas: Canvas; + } | null { + if (options.viewMode === '3d') { + return null; + } + + const editor = Service.Editor as any; + if (editor?.getCurrentEditorType?.() !== 'prefab') { + return null; + } + const root = editor.getRootNode?.() as CCNode | null; + if (!root) { + return null; + } + + let canvas = root.getComponentInChildren?.(Canvas) as Canvas | null; + let ancestor: CCNode | null = root; + while (!canvas && ancestor) { + canvas = ancestor.getComponent?.(Canvas) as Canvas | null; + ancestor = ancestor.parent; + } + if (!canvas?.node) { + return null; + } + + const transform = canvas.node.getComponent(UITransform); + if (!transform) { + return null; + } + const bounds = transform.getBoundingBoxToWorld?.(); + const width = this._finitePositive(bounds?.width, transform.width); + const height = this._finitePositive(bounds?.height, transform.height); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null; + } + + const canvasPosition = canvas.node.getWorldPosition?.() ?? Vec3.ZERO; + const centerX = this._finiteNumber(bounds?.x, canvasPosition.x - width / 2) + width / 2; + const centerY = this._finiteNumber(bounds?.y, canvasPosition.y - height / 2) + height / 2; + const params: ICameraParams = { + position: new Vec3(centerX, centerY, this._finiteNumber(canvasPosition.z, 0) + 1000), + rotation: new Quat(0, 0, 0, 1), + projection: Camera.ProjectionType?.ORTHO ?? 0, + priority: 0, + fov: 45, + fovAxis: Camera.FOVAxis?.VERTICAL ?? 0, + orthoHeight: height / 2, + near: 0.01, + far: 10000, + clearFlags: Camera.ClearFlag.SOLID_COLOR, + clearColor: new Color(51, 51, 51, 255), + clearDepth: 1, + clearStencil: 0, + visibility: Layers.makeMaskExclude([ + Layers.BitMask.PROFILER, + Layers.Enum.GIZMOS, + Layers.Enum.SCENE_GIZMO, + ]), + rect: new Rect(0, 0, 1, 1), + }; + return { + info: this._cameraInfoFromParams(params, 'editor', 'Prefab 2D Camera'), + source: 'editor', + params, + canvas, + renderNote: 'UI Prefab 已按预览 Canvas 边界自动取景（不含 Gizmo/网格）。', + }; + } + + /** + * Canvas 的 alignCanvasWithScreen 会按当前预览窗口更新源相机 orthoHeight。 + * 截图改用设计分辨率离屏窗口后，不能继续复制该预览值，否则会把 960×640 + * 预览下的 SHOW_ALL 可视范围带到 1280×720 截图中并产生黑边。 + */ + private _fitAlignedCanvasFraming(framing: IResolvedFraming, targetWidth: number, targetHeight: number): void { + const canvas = framing.canvas + ?? (framing.sourceCamera ? this._findAlignedCanvasForCamera(framing.sourceCamera) : null); + const orthoEnum = Camera.ProjectionType?.ORTHO ?? 0; + if (!canvas?.node || !canvas.alignCanvasWithScreen || framing.params.projection !== orthoEnum) { + return; + } + + const designSize = (cc as any).view?.getDesignResolutionSize?.(); + const designWidth = this._finitePositive(designSize?.width, 0); + const designHeight = this._finitePositive(designSize?.height, 0); + const targetAspect = targetWidth / targetHeight; + if (designWidth <= 0 || designHeight <= 0 || !Number.isFinite(targetAspect) || targetAspect <= 0) { + return; + } + + // 场景进程使用 SHOW_ALL：目标较窄时扩大纵向范围，目标相同或更宽时保持设计高度。 + const visibleHeight = Math.max(designHeight, designWidth / targetAspect); + const canvasPosition = canvas.node.getWorldPosition?.() ?? Vec3.ZERO; + framing.params.position = new Vec3( + this._finiteNumber(canvasPosition.x, 0), + this._finiteNumber(canvasPosition.y, 0), + this._finiteNumber(canvasPosition.z, 0) + 1000, + ); + framing.params.orthoHeight = visibleHeight / 2; + framing.info = this._cameraInfoFromParams( + framing.params, + framing.source, + framing.info.nodeName, + ); + + const note = 'Canvas 相机已按工程设计分辨率重新取景。'; + framing.renderNote = framing.renderNote ? `${framing.renderNote} ${note}` : note; + } + + private _findAlignedCanvasForCamera(camera: Camera): Canvas | null { + const root = Service.Editor?.getRootNode?.() as CCNode | null; + if (!root) return null; + const canvases: Canvas[] = (root as any).getComponentsInChildren?.(Canvas) ?? []; + return canvases.find((canvas) => ( + Boolean(canvas?.enabledInHierarchy) + && canvas.alignCanvasWithScreen + && canvas.cameraComponent === camera + )) ?? null; + } + + /** + * A freshly opened prefab restores its editor camera asynchronously. Capture + * happens in the same RPC turn, so synchronously frame the prefab first and + * copy that stable view into the temporary screenshot camera. + */ + private _focusEditorCameraForCapture(options: IScreenshotOptions): void { + try { + const editor = Service.Editor as any; + const camera = Service.Camera as any; + const root = editor?.getRootNode?.() as CCNode | null; + if (!root || !camera?.getCamera?.()) { + return; + } + + const has2DContent = Boolean(root.getComponentInChildren?.(Canvas)) + || ((root.getComponentsInChildren?.(UITransform) ?? []).length > 0); + const use2D = options.viewMode === '2d' + || (options.viewMode !== '3d' && has2DContent); + camera.is2D = use2D; + + const uuid = editor.getCurrentEditorUuid?.(); + if (uuid && typeof camera.defaultFocus === 'function') { + camera.defaultFocus(uuid); + } else if (root.uuid && typeof camera.focus === 'function') { + camera.focus([root.uuid], undefined, true); + } + } catch (error) { + console.warn('[Screenshot] Failed to focus editor camera before capture.', error); + } + } + + // ---- 相机参数抽取 ---- + + /** 从任意相机组件（场景相机 / 编辑器相机）抽取创建临时相机所需参数 */ + private _paramsFromComponent(comp: any): ICameraParams { + const node = comp.node as CCNode; + const position = node.getWorldPosition(); + const rotation = node.getWorldRotation(); + const orthoEnum = Camera.ProjectionType?.ORTHO ?? 0; + const perspEnum = Camera.ProjectionType?.PERSPECTIVE ?? 1; + const isOrtho = (comp.projection as number) === orthoEnum; + return { + position, + rotation, + projection: isOrtho ? orthoEnum : perspEnum, + priority: typeof comp.priority === 'number' ? comp.priority : 0, + fov: typeof comp.fov === 'number' ? comp.fov : 45, + fovAxis: typeof comp.fovAxis === 'number' ? comp.fovAxis : (Camera.FOVAxis?.VERTICAL ?? 0), + orthoHeight: typeof comp.orthoHeight === 'number' ? comp.orthoHeight : 10, + near: typeof comp.near === 'number' ? comp.near : 0.01, + far: typeof comp.far === 'number' ? comp.far : 10000, + clearFlags: typeof comp.clearFlags === 'number' ? comp.clearFlags : Camera.ClearFlag.SOLID_COLOR, + clearColor: comp.clearColor?.clone ? comp.clearColor.clone() : undefined, + clearDepth: typeof comp.clearDepth === 'number' ? comp.clearDepth : 1, + clearStencil: typeof comp.clearStencil === 'number' ? comp.clearStencil : 0, + visibility: typeof comp.visibility === 'number' ? comp.visibility : 0xffffffff, + rect: comp.rect?.clone ? comp.rect.clone() : new Rect( + this._finiteNumber(comp.rect?.x, 0), + this._finiteNumber(comp.rect?.y, 0), + this._finitePositive(comp.rect?.width, 1), + this._finitePositive(comp.rect?.height, 1), + ), + aperture: typeof comp.aperture === 'number' ? comp.aperture : undefined, + shutter: typeof comp.shutter === 'number' ? comp.shutter : undefined, + iso: typeof comp.iso === 'number' ? comp.iso : undefined, + screenScale: typeof comp.screenScale === 'number' ? comp.screenScale : undefined, + usePostProcess: typeof comp.usePostProcess === 'boolean' ? comp.usePostProcess : undefined, + postProcess: comp.postProcess, + }; + } + + private _cameraInfoFromParams( + params: ICameraParams, + source: 'scene' | 'editor', + nodeName?: string, + ): IScreenshotCameraInfo { + const isOrtho = params.projection === (Camera.ProjectionType?.ORTHO ?? 0); + return { + source, + nodeName, + projection: isOrtho ? 'ortho' : 'perspective', + position: { + x: this._finiteNumber(params.position.x, 0), + y: this._finiteNumber(params.position.y, 0), + z: this._finiteNumber(params.position.z, 0), + }, + rotation: { + x: this._finiteNumber(params.rotation.x, 0), + y: this._finiteNumber(params.rotation.y, 0), + z: this._finiteNumber(params.rotation.z, 0), + w: this._finiteNumber(params.rotation.w, 1), + }, + fov: isOrtho ? undefined : this._finitePositive(params.fov, 45), + orthoHeight: isOrtho ? this._finitePositive(params.orthoHeight, 10) : undefined, + priority: this._finiteNumber(params.priority, 0), + clearFlags: this._finiteNumber(params.clearFlags, Camera.ClearFlag.SOLID_COLOR), + visibility: this._finiteNumber(params.visibility, 0xffffffff), + viewport: { + x: this._finiteNumber(params.rect.x, 0), + y: this._finiteNumber(params.rect.y, 0), + width: this._finitePositive(params.rect.width, 1), + height: this._finitePositive(params.rect.height, 1), + }, + }; + } + + // ---- 临时相机销毁 ---- + + /** 禁用 + 移出 renderScene + 销毁节点，保证下一帧编辑态 tick 不再渲染它 */ + private _teardownCamera(node: CCNode, comp: any) { + try { + const cam = comp?.camera; + if (cam) { + cam.enabled = false; + if (cam.scene?.removeCamera) { + cam.scene.removeCamera(cam); + } else { + cam.detachCamera?.(); + } + } + } catch (e) { + console.warn('[Screenshot] detach temp camera from render scene failed:', e); + } + try { + if (comp) comp.enabled = false; + } catch (e) { + console.warn('[Screenshot] disable temp camera component failed:', e); + } + try { + // Node.destroy() is deferred. Detach immediately so metadata collected + // in the same turn cannot include the temporary screenshot camera. + node.setParent(null); + } catch (e) { + console.warn('[Screenshot] detach temp camera node failed:', e); + } + try { + node.destroy(); + } catch (e) { + console.warn('[Screenshot] destroy temp camera node failed:', e); + } + } + + // ---- PNG 编码 ---- + + /** + * 默认使用工程设计分辨率，让离屏目标不受当前编辑器预览视口尺寸影响。 + * 只指定一条边时按设计分辨率比例推导另一条边；两条边都指定时保持调用方的精确尺寸。 + */ + private _resolveCaptureSize(options: IScreenshotOptions, scene: CCNode): { width: number; height: number } { + const canvasSize = this._getSceneCanvasSize(scene); + const requestedWidth = this._validSize(options.width); + const requestedHeight = this._validSize(options.height); + + if (requestedWidth != null && requestedHeight != null) { + return { + width: this._clampSize(requestedWidth, canvasSize.width), + height: this._clampSize(requestedHeight, canvasSize.height), + }; + } + + const aspect = canvasSize.width / canvasSize.height; + let width = requestedWidth ?? (requestedHeight != null ? requestedHeight * aspect : canvasSize.width); + let height = requestedHeight ?? (requestedWidth != null ? requestedWidth / aspect : canvasSize.height); + const scale = Math.min(1, MAX_SIZE / width, MAX_SIZE / height); + width *= scale; + height *= scale; + return { + width: Math.max(1, Math.floor(width)), + height: Math.max(1, Math.floor(height)), + }; + } + + private _getSceneCanvasSize(scene: CCNode): { width: number; height: number } { + // 场景进程启动及配置同步时会把工程设计分辨率写入 cc.view。 + // 这里优先使用它，避免 alignCanvasWithScreen / Widget 将 Canvas 的运行时尺寸 + // 改成当前预览视口大小，从而让截图尺寸随 PinK 窗口变化。 + const designSize = (cc as any).view?.getDesignResolutionSize?.(); + if (designSize?.width > 0 && designSize?.height > 0) { + return { width: designSize.width, height: designSize.height }; + } + + const root = (Service.Editor?.getRootNode?.() as CCNode | null) ?? scene; + const canvases: Canvas[] = (root as any).getComponentsInChildren?.(Canvas) ?? []; + const canvas = canvases.find(item => item.enabledInHierarchy) ?? canvases[0]; + const transform = canvas?.node?.getComponent(UITransform); + if (transform && transform.width > 0 && transform.height > 0) { + return { width: transform.width, height: transform.height }; + } + + // ScenePreview 的缓冲区尺寸来自编辑器画布最近一次 query-preview-data， + // 仅在设计分辨率和场景 Canvas 均不可用时作为兼容兜底。 + const previewBuffer = (Service.Preview as any)?.scenePreview?.previewBuffer; + if (previewBuffer?.width > 0 && previewBuffer?.height > 0) { + return { width: previewBuffer.width, height: previewBuffer.height }; + } + + const gameCanvas = (cc as any).game?.canvas; + if (gameCanvas?.width > 0 && gameCanvas?.height > 0) { + return { width: gameCanvas.width, height: gameCanvas.height }; + } + + return { width: DEFAULT_SIZE, height: DEFAULT_SIZE }; + } + + private _writePng(width: number, height: number, rgba: Uint8Array): string { + const png = new PNG({ width, height }); + png.data = Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength); + const buffer = PNG.sync.write(png); + const file = join(tmpdir(), `cocos-cli-screenshot-${Date.now()}-${this._seq++}.png`); + writeFileSync(file, buffer); + return file; + } + + // ---- 场景元数据 ---- + + private async _collectSceneMeta(): Promise<{ url?: string; name?: string; mtime?: number }> { + try { + const uuid = (Service.Editor as any)?.getCurrentEditorUuid?.(); + if (!uuid) return {}; + const info = await Rpc.getInstance().request('assetManager', 'queryAssetInfo', [uuid]); + if (!info) return {}; + let mtime: number | undefined; + if (info.file) { + try { + // 动态引入 fs，读取磁盘文件 mtime（PRD 4.3 / 风险 8） + const { statSync } = await import('fs'); + mtime = statSync(info.file).mtimeMs; + } catch { + // 文件不存在（如未保存的新场景）时无 mtime + } + } + return { url: info.url, name: info.name, mtime }; + } catch { + return {}; + } + } + + private _summarizeNode(node: CCNode, depth: number): IScreenshotNodeSummary { + const components = (node.components ?? []) + .map((c) => cc.js.getClassName(c)) + .filter(Boolean); + const summary: IScreenshotNodeSummary = { + name: node.name, + active: node.active, + components, + }; + if (depth < NODE_SUMMARY_MAX_DEPTH && node.children?.length) { + const kids = node.children.slice(0, NODE_SUMMARY_MAX_CHILDREN); + summary.children = kids.map((child) => this._summarizeNode(child, depth + 1)); + if (node.children.length > NODE_SUMMARY_MAX_CHILDREN) { + summary.children.push({ + name: `…(+${node.children.length - NODE_SUMMARY_MAX_CHILDREN} more)`, + active: true, + components: [], + }); + } + } + return summary; + } + + private _clampSize(value: number | undefined, fallback: number): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + return Math.min(MAX_SIZE, Math.max(1, Math.floor(fallback))); + } + return Math.min(MAX_SIZE, Math.max(1, Math.floor(value))); + } + + private _validSize(value: number | undefined): number | null { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : null; + } + + private _finiteNumber(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; + } + + private _finitePositive(value: unknown, fallback: number): number { + const finite = this._finiteNumber(value, fallback); + return finite > 0 ? finite : fallback; + } +} diff --git a/src/core/scene/scene-process/service/screenshot/screenshot-buffer.ts b/src/core/scene/scene-process/service/screenshot/screenshot-buffer.ts new file mode 100644 index 000000000..4412e5de3 --- /dev/null +++ b/src/core/scene/scene-process/service/screenshot/screenshot-buffer.ts @@ -0,0 +1,266 @@ +import { gfx } from 'cc'; + +const SCREENSHOT_RENDER_TIMEOUT_MS = 10_000; + +/** + * 截图专用离屏渲染缓冲。 + * + * 复用 service/preview/buffer.ts 中已验证的离屏窗口 + gl.readPixels 读回思路。 + * 关键点：本类只接收由上层 ScreenshotService 临时创建、完全自持的相机栈， + * 绝不触碰运行中的编辑器相机 / 场景相机。渲染完毕后由上层销毁这些临时相机， + * 因此不会像旧实现那样把 live 相机指向离屏窗口后残留在渲染管线里，导致下一帧 + * `ForwardPipeline.getRenderPass` 读到 null 帧缓冲而崩溃。 + * + * 本类只负责： + * 1. 创建/复用一个 isOffscreen 的渲染窗口； + * 2. 把传入的临时相机切到该窗口渲一帧（tempWindow + repaint）； + * 3. 读回帧缓冲像素（含垂直翻转 / BGRA 处理），返回 RGBA。 + */ +export interface IReadbackResult { + width: number; + height: number; + /** RGBA，长度 = width * height * 4 */ + buffer: Uint8Array; +} + +export interface IScreenshotRenderCamera { + comp: any; + cam: any; +} + +export class ScreenshotBuffer { + private _name: string; + private device = cc.director.root!.device; + private width = 0; + private height = 0; + private data: Uint8Array = new Uint8Array(0); + private window: any = null; + private regions = [new gfx.BufferTextureCopy()]; + + private readonly needInvertGFXApi = [ + gfx.API.GLES2, + gfx.API.GLES3, + gfx.API.WEBGL, + gfx.API.WEBGL2, + ]; + + private static readonly indexOfRGBA = [0, 1, 2, 3]; + private static readonly indexOfBGRA = [2, 1, 0, 3]; + + constructor(name = 'screenshot') { + this._name = name; + } + + /** 确保离屏窗口存在并匹配目标尺寸 */ + private ensureWindow(width: number, height: number) { + width = Math.max(1, Math.floor(width)); + height = Math.max(1, Math.floor(height)); + + if (!this.window) { + this.createWindow(width, height); + } else if (width !== this.width || height !== this.height) { + this.window.resize(width, height); + } + + this.width = width; + this.height = height; + this.data = new Uint8Array(this.width * this.height * 4); + this.regions[0].texExtent.width = this.width; + this.regions[0].texExtent.height = this.height; + } + + private createWindow(width: number, height: number) { + const root = cc.director.root!; + const swapchain = root.mainWindow?.swapchain; + const swapchainColorFormat = swapchain?.colorTexture?.format; + const swapchainDepthFormat = swapchain?.depthStencilTexture?.format; + // headless 场景进程的 swapchain 纹理可能存在，但 format 仍是 UNKNOWN。 + // RenderWindow 遇到 UNKNOWN 深度格式不会创建 depthStencilTexture， + // 而 legacy ForwardPipeline 会无条件读取该附件并在渲染时崩溃。 + const colorFormat = swapchainColorFormat != null && swapchainColorFormat !== gfx.Format.UNKNOWN + ? swapchainColorFormat + : gfx.Format.RGBA8; + const depthStencilFormat = swapchainDepthFormat != null && swapchainDepthFormat !== gfx.Format.UNKNOWN + ? swapchainDepthFormat + : gfx.Format.DEPTH_STENCIL; + const renderPassInfo = new gfx.RenderPassInfo( + [new gfx.ColorAttachment(colorFormat)], + new gfx.DepthStencilAttachment(depthStencilFormat), + ); + renderPassInfo.colorAttachments[0].barrier = root.device.getGeneralBarrier( + new gfx.GeneralBarrierInfo(0, gfx.AccessFlagBit.FRAGMENT_SHADER_READ_TEXTURE), + ); + this.window = root.createWindow({ + title: this._name, + width, + height, + renderPassInfo, + isOffscreen: true, + }); + if (!this.window?.framebuffer?.colorTextures?.[0] || !this.window.framebuffer.depthStencilTexture) { + const invalidWindow = this.window; + this.window = null; + if (invalidWindow) { + root.destroyWindow(invalidWindow); + } + throw new Error('创建截图离屏窗口失败：framebuffer 颜色或深度附件不可用。'); + } + } + + get renderWindow() { + return this.window; + } + + /** + * 把上层自持的临时相机栈渲染到同一个离屏窗口并读回合成像素。 + * + * @param renderScene 当前场景的 renderScene（cam 需加入其中才能看到场景内容） + * @param cameras 按 priority 升序排列的临时相机 + */ + async render( + renderScene: any, + cameras: IScreenshotRenderCamera[], + width: number, + height: number, + ): Promise { + if (this.device.gfxAPI === gfx.API.UNKNOWN) { + throw new Error('截图不可用：无头 WebGL 初始化失败，场景进程已降级为 EmptyDevice。请检查 gl/@napi-rs/canvas 原生依赖和图形运行环境。'); + } + if (!cameras.length) { + throw new Error('截图不可用：没有可用于离屏渲染的相机。'); + } + this.ensureWindow(width, height); + const currWindow = this.window; + + // 先把全部相机指向同一离屏窗口，再启用并触发一帧。RenderWindow 会按 + // priority 排序，因此 DEPTH_ONLY / DONT_CLEAR 的 UI 相机会叠加在主相机之上。 + // 若在指向离屏窗口前启用，_activate/onEnable 会把它挂到 mainWindow，触发 framebuffer 错误。 + for (const { comp, cam } of cameras) { + if (renderScene && !cam.scene) { + renderScene.addCamera(cam); + } + cam.isWindowSize = false; + cam.changeTargetWindow(currWindow); + if (cam.width !== this.width || cam.height !== this.height) { + cam.resize(this.width, this.height); + } + comp.enabled = true; + cam.enabled = true; + cam.update(true); + } + + const root = cc.director.root!; + const prevTempWindow = (root as any).tempWindow; + (root as any).tempWindow = currWindow; + + // 强制在编辑模式下渲染一帧 + const engine: any = (await import('../core/decorator')).Service.Engine; + return await new Promise((resolve, reject) => { + let settled = false; + let timeout: ReturnType | undefined; + + const cleanup = () => { + cc.director.off(cc.Director.EVENT_AFTER_DRAW, onAfterDraw); + if (timeout) { + clearTimeout(timeout); + } + (root as any).tempWindow = prevTempWindow; + }; + + const onAfterDraw = () => { + if (settled) return; + settled = true; + cleanup(); + try { + resolve(this.copyFrameBuffer()); + } catch (error) { + reject(error); + } + }; + cc.director.once(cc.Director.EVENT_AFTER_DRAW, onAfterDraw); + timeout = setTimeout(() => { + if (settled) return; + settled = true; + cleanup(); + reject(new Error( + `截图渲染超时（${SCREENSHOT_RENDER_TIMEOUT_MS}ms）：未收到 EVENT_AFTER_DRAW。`, + )); + }, SCREENSHOT_RENDER_TIMEOUT_MS); + try { + if (typeof engine.forceRepaintInEditMode === 'function') { + engine.forceRepaintInEditMode(); + } else { + void engine.repaintInEditMode?.(); + } + } catch (error) { + if (settled) return; + settled = true; + cleanup(); + reject(error); + } + }); + } + + private copyFrameBuffer(): IReadbackResult { + const result: IReadbackResult = { width: this.width, height: this.height, buffer: this.data }; + const window = this.window; + if (!window || !window.framebuffer) { + return result; + } + + const destBuffer = new Uint8Array(this.data.buffer); + const colorTex = window.framebuffer.colorTextures[0]; + if (colorTex) { + // Use the gfx API so its WebGL state cache stays synchronized. + // Binding a raw temporary FBO here leaves the cache pointing at a + // framebuffer that is no longer bound and breaks later frames. + this.device.copyTextureToBuffers(colorTex, [destBuffer], this.regions); + } + + this.formatBuffer( + this.data, + this.needInvertGFXApi.includes(this.device.gfxAPI), + this.device.gfxAPI === gfx.API.METAL, + ); + return result; + } + + /** 垂直翻转 +（Metal 下）BGRA→RGBA，逻辑与 PreviewBuffer.formatBuffer 一致 */ + private formatBuffer(buffer: Uint8Array, needInvert: boolean, conversionBGRA: boolean) { + if (!needInvert) { + return buffer; + } + const indexArr = conversionBGRA ? ScreenshotBuffer.indexOfBGRA : ScreenshotBuffer.indexOfRGBA; + const tmp = { r: 0, g: 0, b: 0, a: 0 }; + + for (let w = 0; w < this.width; w++) { + for (let h = 0; h < this.height / 2; h++) { + const startIndex = (h * this.width + w) * 4; + const invertIndex = ((this.height - 1 - h) * this.width + w) * 4; + + tmp.r = buffer[startIndex + indexArr[0]]; + tmp.g = buffer[startIndex + indexArr[1]]; + tmp.b = buffer[startIndex + indexArr[2]]; + tmp.a = buffer[startIndex + indexArr[3]]; + + buffer[startIndex + 0] = buffer[invertIndex + indexArr[0]]; + buffer[startIndex + 1] = buffer[invertIndex + indexArr[1]]; + buffer[startIndex + 2] = buffer[invertIndex + indexArr[2]]; + buffer[startIndex + 3] = buffer[invertIndex + indexArr[3]]; + + buffer[invertIndex + 0] = tmp.r; + buffer[invertIndex + 1] = tmp.g; + buffer[invertIndex + 2] = tmp.b; + buffer[invertIndex + 3] = tmp.a; + } + } + return buffer; + } + + destroy() { + if (this.window) { + cc.director.root!.destroyWindow(this.window); + this.window = null; + } + } +} diff --git a/src/core/scene/test/browser-scene-state.test.ts b/src/core/scene/test/browser-scene-state.test.ts new file mode 100644 index 000000000..ddea02380 --- /dev/null +++ b/src/core/scene/test/browser-scene-state.test.ts @@ -0,0 +1,34 @@ +import { browserSceneState } from '../browser-scene-state'; + +describe('BrowserSceneState', () => { + beforeEach(() => { + browserSceneState.clearCurrent(); + }); + + it('stores a copy of the current PinK scene', () => { + const stored = browserSceneState.setCurrent({ + uuid: 'scene-uuid', + url: 'db://assets/main.scene', + type: 'scene', + name: 'main.scene', + }); + + expect(stored).toMatchObject({ uuid: 'scene-uuid', url: 'db://assets/main.scene' }); + expect(browserSceneState.getCurrent()).toEqual(stored); + expect(stored.updatedAt).toEqual(expect.any(Number)); + }); + + it('does not let an older browser scene clear a newer selection', () => { + browserSceneState.setCurrent({ + uuid: 'new-scene-uuid', + url: 'db://assets/new.scene', + type: 'scene', + name: 'new.scene', + }); + + expect(browserSceneState.clearCurrent('old-scene-uuid')).toBe(false); + expect(browserSceneState.getCurrent()?.uuid).toBe('new-scene-uuid'); + expect(browserSceneState.clearCurrent('new-scene-uuid')).toBe(true); + expect(browserSceneState.getCurrent()).toBeNull(); + }); +}); diff --git a/src/core/scene/test/editor-close-options.test.ts b/src/core/scene/test/editor-close-options.test.ts index 019bd25ec..718312a15 100644 --- a/src/core/scene/test/editor-close-options.test.ts +++ b/src/core/scene/test/editor-close-options.test.ts @@ -49,6 +49,7 @@ jest.mock('../scene-process/rpc', () => ({ import { Prefab } from 'cc'; import { SceneEditor } from '../scene-process/service/editors/scene-editor'; import { PrefabEditor } from '../scene-process/service/editors/prefab-editor'; +import { EditorService } from '../scene-process/service/editor'; import { sceneUtils } from '../scene-process/service/scene/utils'; import { editorPrefabUtils } from '../scene-process/service/prefab/prefab-editor-utils'; @@ -167,3 +168,99 @@ describe('Editor close options', () => { }); }); + +describe('Screenshot scene preparation', () => { + function createService(currentUuid: string | null, dirty: boolean) { + const service = new EditorService(); + (service as any).currentEditorUuid = currentUuid; + jest.spyOn(service as any, 'isCurrentEditorDirty').mockReturnValue(dirty); + const close = jest.spyOn(service as any, 'closeUnlocked').mockImplementation(async () => { + (service as any).currentEditorUuid = null; + return true; + }); + const open = jest.spyOn(service as any, 'openUnlocked').mockImplementation(async (...args: unknown[]) => { + const { urlOrUUID } = args[0] as { urlOrUUID: string }; + (service as any).currentEditorUuid = urlOrUUID; + return {}; + }); + return { service, close, open }; + } + + beforeEach(() => { + mockRpcRequest.mockReset(); + }); + + it('reopens a clean current scene without saving before a screenshot', async () => { + const { service, close, open } = createService('scene-uuid', false); + const operation = jest.fn(async () => 'captured'); + + await expect(service.withScreenshotScene(undefined, operation)).resolves.toBe('captured'); + + expect(close).toHaveBeenCalledWith({ urlOrUUID: 'scene-uuid', save: false }); + expect(open).toHaveBeenCalledWith({ urlOrUUID: 'scene-uuid' }); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it('prefers the scene currently opened in PinK over the worker current scene', async () => { + mockRpcRequest + .mockResolvedValueOnce({ uuid: 'pink-scene-uuid' }) + .mockResolvedValueOnce({ uuid: 'pink-scene-uuid' }); + const { service, close, open } = createService('worker-scene-uuid', false); + + await service.withScreenshotScene(undefined, async () => undefined); + + expect(mockRpcRequest).toHaveBeenNthCalledWith(1, 'browserSceneState', 'getCurrent', []); + expect(mockRpcRequest).toHaveBeenNthCalledWith(2, 'assetManager', 'queryAssetInfo', ['pink-scene-uuid']); + expect(close).toHaveBeenCalledWith({ urlOrUUID: 'worker-scene-uuid', save: false }); + expect(open).toHaveBeenCalledWith({ urlOrUUID: 'pink-scene-uuid' }); + }); + + it('keeps unsaved scene-process changes when capturing the current scene', async () => { + const { service, close, open } = createService('scene-uuid', true); + const operation = jest.fn(async () => undefined); + + await service.withScreenshotScene(undefined, operation); + + expect(close).not.toHaveBeenCalled(); + expect(open).not.toHaveBeenCalled(); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it('restores the previous clean scene after capturing an explicit target', async () => { + mockRpcRequest.mockResolvedValue({ uuid: 'target-uuid' }); + const { service, close, open } = createService('current-uuid', false); + + await service.withScreenshotScene('db://assets/target.scene', async () => undefined); + + expect(close).toHaveBeenNthCalledWith(1, { urlOrUUID: 'current-uuid', save: false }); + expect(open).toHaveBeenNthCalledWith(1, { urlOrUUID: 'target-uuid' }); + expect(close).toHaveBeenNthCalledWith(2, { urlOrUUID: 'target-uuid', save: false }); + expect(open).toHaveBeenNthCalledWith(2, { urlOrUUID: 'current-uuid' }); + expect((service as any).currentEditorUuid).toBe('current-uuid'); + }); + + it('rejects an explicit target instead of discarding unsaved changes', async () => { + mockRpcRequest.mockResolvedValue({ uuid: 'target-uuid' }); + const { service, close, open } = createService('current-uuid', true); + + await expect(service.withScreenshotScene('target-uuid', async () => undefined)) + .rejects.toThrow('存在未保存修改'); + + expect(close).not.toHaveBeenCalled(); + expect(open).not.toHaveBeenCalled(); + expect((service as any).currentEditorUuid).toBe('current-uuid'); + }); + + it('restores the previous scene when capture fails', async () => { + mockRpcRequest.mockResolvedValue({ uuid: 'target-uuid' }); + const { service, close, open } = createService('current-uuid', false); + + await expect(service.withScreenshotScene('target-uuid', async () => { + throw new Error('capture failed'); + })).rejects.toThrow('capture failed'); + + expect(close).toHaveBeenCalledTimes(2); + expect(open).toHaveBeenCalledTimes(2); + expect((service as any).currentEditorUuid).toBe('current-uuid'); + }); +}); diff --git a/src/core/scene/test/screenshot-camera-params.test.ts b/src/core/scene/test/screenshot-camera-params.test.ts new file mode 100644 index 000000000..5096629ba --- /dev/null +++ b/src/core/scene/test/screenshot-camera-params.test.ts @@ -0,0 +1,394 @@ +jest.mock('cc', () => { + class Vec3 { + constructor(public x = 0, public y = 0, public z = 0) {} + } + class Quat { + constructor(public x = 0, public y = 0, public z = 0, public w = 1) {} + } + class Rect { + constructor( + public x = 0, + public y = 0, + public width = 1, + public height = 1, + ) {} + + clone() { + return new Rect(this.x, this.y, this.width, this.height); + } + } + class Camera { + static ProjectionType = { ORTHO: 0, PERSPECTIVE: 1 }; + static ClearFlag = { SOLID_COLOR: 1 }; + static FOVAxis = { VERTICAL: 0, HORIZONTAL: 1 }; + } + class Node { + public layer = 0; + public parent: unknown = null; + public destroyed = false; + + constructor(public name = '') {} + + setParent(parent: unknown) { + this.parent = parent; + } + + setWorldPosition() {} + + setWorldRotation() {} + + addComponent() { + return {}; + } + + destroy() { + this.destroyed = true; + } + } + return { + __esModule: true, + default: {}, + Camera, + Canvas: class {}, + Color: class {}, + Layers: { + BitMask: { PROFILER: 1 }, + Enum: { EDITOR: 2, GIZMOS: 4, SCENE_GIZMO: 8 }, + makeMaskExclude: jest.fn(() => 0), + makeMaskInclude: jest.fn((layers: number[]) => layers.reduce((mask, layer) => mask | layer, 0)), + }, + Node, + Quat, + Rect, + UITransform: class {}, + Vec3, + renderer: { scene: { CameraUsage: { EDITOR: 1 } } }, + }; +}); + +jest.mock('../scene-process/service/core', () => ({ + BaseService: class {}, + register: () => (target: unknown) => target, + Service: {}, +})); + +jest.mock('../scene-process/rpc', () => ({ + Rpc: { getInstance: jest.fn() }, +})); + +jest.mock('../scene-process/service/screenshot/screenshot-buffer', () => ({ + ScreenshotBuffer: class {}, +})); + +import { Canvas, Node, UITransform } from 'cc'; +import { ScreenshotService } from '../scene-process/service/screenshot'; + +describe('Screenshot camera parameter normalization', () => { + it('normalizes only returned metadata without changing render camera values', () => { + const service = new ScreenshotService(); + const params = (service as any)._paramsFromComponent({ + node: { + getWorldPosition: () => ({ x: Number.NaN, y: Number.POSITIVE_INFINITY, z: 3 }), + getWorldRotation: () => ({ x: Number.NaN, y: 0, z: 0, w: Number.NaN }), + }, + projection: 0, + fov: Number.NaN, + orthoHeight: Number.NaN, + near: Number.POSITIVE_INFINITY, + far: -1, + visibility: Number.NaN, + }); + const info = (service as any)._cameraInfoFromParams(params, 'editor', 'Editor Camera'); + + expect(Number.isNaN(params.orthoHeight)).toBe(true); + expect(info.position).toEqual({ x: 0, y: 0, z: 3 }); + expect(info.rotation).toEqual({ x: 0, y: 0, z: 0, w: 1 }); + expect(info.orthoHeight).toBe(10); + expect(JSON.parse(JSON.stringify(info)).orthoHeight).toBe(10); + }); + + it('focuses a 2D prefab before copying the editor camera', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const camera = { + is2D: false, + getCamera: jest.fn(() => ({ node: {} })), + defaultFocus: jest.fn(), + }; + Service.Editor = { + getRootNode: jest.fn(() => ({ + uuid: 'prefab-root', + getComponentInChildren: jest.fn(() => ({})), + getComponentsInChildren: jest.fn(() => []), + })), + getCurrentEditorUuid: jest.fn(() => 'prefab-uuid'), + }; + Service.Camera = camera; + + (service as any)._focusEditorCameraForCapture({ viewMode: '2d' }); + + expect(camera.is2D).toBe(true); + expect(camera.defaultFocus).toHaveBeenCalledWith('prefab-uuid'); + }); + + it('frames a UI prefab from its preview canvas instead of an invalid editor viewport', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const transform = { + width: 960, + height: 640, + getBoundingBoxToWorld: jest.fn(() => ({ x: -480, y: -320, width: 960, height: 640 })), + }; + const canvasNode: any = { + parent: null, + getComponent: jest.fn((type: unknown) => type === Canvas ? canvas : type === UITransform ? transform : null), + getWorldPosition: jest.fn(() => ({ x: 0, y: 0, z: 0 })), + }; + const canvas = { node: canvasNode }; + const prefabRoot = { + parent: canvasNode, + getComponentInChildren: jest.fn(() => null), + getComponent: jest.fn(() => null), + }; + Service.Editor = { + getCurrentEditorType: jest.fn(() => 'prefab'), + getRootNode: jest.fn(() => prefabRoot), + }; + + const framing = (service as any)._resolvePrefab2DFraming({ viewMode: '2d' }); + + expect(framing).not.toBeNull(); + expect(framing.params.position).toMatchObject({ x: 0, y: 0, z: 1000 }); + expect(framing.params.orthoHeight).toBe(320); + expect(framing.info).toMatchObject({ + source: 'editor', + projection: 'ortho', + orthoHeight: 320, + }); + }); + + it('reframes a screen-aligned Canvas camera for the design-resolution target', () => { + const service = new ScreenshotService(); + const previousCc = (globalThis as any).cc; + const camera = {}; + const canvas = { + alignCanvasWithScreen: true, + cameraComponent: camera, + node: { + getWorldPosition: jest.fn(() => ({ x: 640, y: 360, z: 0 })), + }, + }; + const params = { + position: { x: 480, y: 320, z: 1000 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + projection: 0, + priority: 0, + fov: 45, + fovAxis: 0, + orthoHeight: 426.67, + near: 0.01, + far: 10000, + clearFlags: 1, + clearDepth: 1, + clearStencil: 0, + visibility: 0xffffffff, + rect: { x: 0, y: 0, width: 1, height: 1 }, + }; + const framing: any = { + info: (service as any)._cameraInfoFromParams(params, 'scene', 'Canvas Camera'), + source: 'scene', + params, + sourceCamera: camera, + canvas, + }; + + try { + (globalThis as any).cc = { + view: { + getDesignResolutionSize: () => ({ width: 1280, height: 720 }), + }, + }; + + (service as any)._fitAlignedCanvasFraming(framing, 1280, 720); + + expect(framing.params.orthoHeight).toBe(360); + expect(framing.params.position).toMatchObject({ x: 640, y: 360, z: 1000 }); + expect(framing.info).toMatchObject({ + source: 'scene', + projection: 'ortho', + orthoHeight: 360, + }); + expect(framing.renderNote).toContain('工程设计分辨率'); + } finally { + (globalThis as any).cc = previousCc; + } + }); + + it('selects all screen cameras in ascending priority and excludes non-rendering cameras', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const makeCamera = (name: string, priority: number, overrides: Record = {}) => ({ + node: { name, layer: 1 }, + priority, + enabledInHierarchy: true, + targetTexture: null, + ...overrides, + }); + const ui = makeCamera('UI Camera', 100); + const main = makeCamera('Main Camera', 0); + const disabled = makeCamera('Disabled Camera', 10, { enabledInHierarchy: false }); + const editorLayer = makeCamera('Editor Camera', 20, { node: { name: 'Editor Camera', layer: 2 } }); + const renderTexture = makeCamera('Render Texture Camera', 30, { targetTexture: {} }); + Service.Editor = { + getRootNode: jest.fn(() => ({ + getComponentsInChildren: jest.fn(() => [ui, disabled, editorLayer, renderTexture, main]), + })), + }; + Service.Camera = { getCamera: jest.fn(() => null) }; + + expect((service as any)._findSceneCameras()).toEqual([main, ui]); + }); + + it('preserves camera composition parameters in render metadata', () => { + const service = new ScreenshotService(); + const params = (service as any)._paramsFromComponent({ + node: { + getWorldPosition: () => ({ x: 1, y: 2, z: 3 }), + getWorldRotation: () => ({ x: 0, y: 0, z: 0, w: 1 }), + }, + projection: 1, + priority: 50, + fov: 60, + fovAxis: 1, + orthoHeight: 10, + near: 0.1, + far: 2000, + clearFlags: 2, + clearDepth: 0.5, + clearStencil: 3, + visibility: 0x1234, + rect: { x: 0.1, y: 0.2, width: 0.7, height: 0.8 }, + }); + const info = (service as any)._cameraInfoFromParams(params, 'scene', 'UI Camera'); + + expect(params).toMatchObject({ + priority: 50, + fovAxis: 1, + clearFlags: 2, + clearDepth: 0.5, + clearStencil: 3, + visibility: 0x1234, + rect: { x: 0.1, y: 0.2, width: 0.7, height: 0.8 }, + }); + expect(info).toMatchObject({ + nodeName: 'UI Camera', + priority: 50, + clearFlags: 2, + visibility: 0x1234, + viewport: { x: 0.1, y: 0.2, width: 0.7, height: 0.8 }, + }); + }); + + it('uses only the requested camera when camera is explicitly specified', () => { + const service = new ScreenshotService(); + const requestedCamera = { + node: { + name: 'Requested Camera', + getWorldPosition: () => ({ x: 0, y: 0, z: 10 }), + getWorldRotation: () => ({ x: 0, y: 0, z: 0, w: 1 }), + }, + projection: 1, + priority: 25, + fov: 45, + near: 0.1, + far: 1000, + clearFlags: 1, + visibility: 0xffffffff, + rect: { x: 0, y: 0, width: 1, height: 1 }, + }; + jest.spyOn(service as any, '_findCameraByRef').mockReturnValue(requestedCamera); + const findSceneCameras = jest.spyOn(service as any, '_findSceneCameras'); + + const framings = (service as any)._resolveFramings({ camera: 'requested-camera' }); + + expect(framings).toHaveLength(1); + expect(framings[0]).toMatchObject({ + source: 'scene', + sourceCamera: requestedCamera, + info: { nodeName: 'Requested Camera', priority: 25 }, + }); + expect(findSceneCameras).not.toHaveBeenCalled(); + }); + + it('tears down a temporary camera when parameter assignment fails', () => { + const service = new ScreenshotService(); + const tempCamera: any = { enabled: true }; + Object.defineProperty(tempCamera, 'postProcess', { + set() { + throw new Error('post-process assignment failed'); + }, + }); + jest.spyOn(Node.prototype as any, 'addComponent').mockReturnValue(tempCamera); + const teardown = jest.spyOn(service as any, '_teardownCamera'); + const framing: any = { + info: { nodeName: 'Main Camera' }, + params: { + position: { x: 0, y: 0, z: 10 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + projection: 1, + priority: 0, + fov: 45, + fovAxis: 0, + orthoHeight: 10, + near: 0.1, + far: 1000, + clearFlags: 1, + clearDepth: 1, + clearStencil: 0, + visibility: 0xffffffff, + rect: { x: 0, y: 0, width: 1, height: 1 }, + postProcess: {}, + }, + }; + + expect(() => (service as any)._createTemporaryCamera({}, framing, 0)) + .toThrow('post-process assignment failed'); + expect(teardown).toHaveBeenCalledTimes(1); + const [node, component] = teardown.mock.calls[0]; + expect(component).toBe(tempCamera); + expect((node as any).parent).toBeNull(); + expect((node as any).destroyed).toBe(true); + }); + + it('still detaches and destroys a temporary node when render-scene cleanup fails', () => { + const service = new ScreenshotService(); + const node = new Node('Temporary Camera') as any; + node.setParent({}); + const component = { + enabled: true, + camera: { + enabled: true, + scene: { + removeCamera() { + throw new Error('remove camera failed'); + }, + }, + }, + }; + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + + try { + (service as any)._teardownCamera(node, component); + + expect(component.enabled).toBe(false); + expect(node.parent).toBeNull(); + expect(node.destroyed).toBe(true); + expect(warn).toHaveBeenCalledWith( + '[Screenshot] detach temp camera from render scene failed:', + expect.any(Error), + ); + } finally { + warn.mockRestore(); + } + }); +}); diff --git a/src/mcp/image-result.ts b/src/mcp/image-result.ts new file mode 100644 index 000000000..0bc6319db --- /dev/null +++ b/src/mcp/image-result.ts @@ -0,0 +1,19 @@ +/** + * Remove binary image data from JSON/text channels after the full tool result + * has passed schema validation. The original result remains untouched so its + * base64 payload can still be emitted as an MCP image content block. + */ +export function redactInlineImageData(result: any): any { + const image = result?.data?.image; + if (!image || typeof image.base64 !== 'string') { + return result; + } + const { base64: _base64, ...imageMetadata } = image; + return { + ...result, + data: { + ...result.data, + image: imageMetadata, + }, + }; +} diff --git a/src/mcp/mcp.middleware.ts b/src/mcp/mcp.middleware.ts index d06340ba7..edb54d7f4 100644 --- a/src/mcp/mcp.middleware.ts +++ b/src/mcp/mcp.middleware.ts @@ -14,6 +14,7 @@ import stripAnsi from 'strip-ansi'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; import { assetManager } from '../core/assets'; +import { redactInlineImageData } from './image-result'; export function isToolErrorCode(code: unknown): boolean { return typeof code === 'number' && code >= 500 && code < 600; @@ -154,25 +155,34 @@ export class McpMiddleware { // 注意：args 是对象，prepareMethodArguments 需要处理对象 const methodArgs = this.prepareMethodArguments(meta, args, toolName); const result = await this.callToolMethod(target, meta, methodArgs); - - const formattedResult = this.formatToolResult(meta, result); - - let structuredContent: any; - if (meta.returnSchema) { - try { - const validatedResult = meta.returnSchema.parse(result); - structuredContent = { result: validatedResult }; - } catch { - structuredContent = { result: result }; - } - } else { - structuredContent = { result: result }; - } + // The binary payload is intentionally outside the advertised + // structured output schema and is emitted only as image content. + const image = result?.data?.image; + const validatedResult = this.validateToolResult(meta, result); + const responseResult = image?.base64 && typeof image.base64 === 'string' + ? redactInlineImageData(validatedResult) + : validatedResult; + const formattedResult = this.formatToolResult(responseResult); + const structuredContent = { result: responseResult }; + console.debug(`call ${toolName} with args:${methodArgs.toString()} result: ${formattedResult}`); + + // 图片二进制只放进 MCP image 内容块。文本、structuredContent 和日志 + // 仅保留图片元数据，避免同一份 base64 被重复传输和写入日志。 + const content: any[] = [ + { type: 'text' as const, text: formattedResult }, + ]; + if (image?.base64 && typeof image.base64 === 'string') { + content.push({ + type: 'image' as const, + data: image.base64, + mimeType: image.mimeType || 'image/png', + }); + } return { - content: [{ type: 'text' as const, text: formattedResult }], + content, structuredContent: structuredContent, - isError: isToolErrorCode(result?.code) + isError: isToolErrorCode(validatedResult?.code) }; } catch (error) { @@ -317,21 +327,21 @@ export class McpMiddleware { /** * 格式化工具结果 */ - private formatToolResult(meta: any, result: any): string { - // 构建符合 schema 的结果结构，用 result 字段包装 + private validateToolResult(meta: any, result: any): any { + const normalizedResult = result?.reason + ? { ...result, reason: stripAnsi(result.reason) } + : result; if (meta.returnSchema) { - // 验证结果是否符合预期的 schema try { - if (result.reason) { - result.reason = stripAnsi(result.reason); - } - const validatedResult = meta.returnSchema.parse(result); - return JSON.stringify({ result: validatedResult }, null, 2); + return meta.returnSchema.parse(normalizedResult); } catch (error) { throw new Error(`Tool result validation failed: ${error instanceof Error ? error.message : String(error)}`); } } + return normalizedResult; + } + private formatToolResult(result: any): string { return JSON.stringify({ result: result }, null, 2); } diff --git a/tests/mcp-image-result.test.ts b/tests/mcp-image-result.test.ts new file mode 100644 index 000000000..8025291ad --- /dev/null +++ b/tests/mcp-image-result.test.ts @@ -0,0 +1,34 @@ +import { redactInlineImageData } from '../src/mcp/image-result'; + +describe('MCP inline image result', () => { + it('keeps base64 only in the original image payload', () => { + const result = { + code: 200, + data: { + image: { + base64: 'large-image-payload', + mimeType: 'image/jpeg', + attached: true, + }, + meta: { width: 1280, height: 720 }, + }, + }; + + const redacted = redactInlineImageData(result); + + expect(redacted).toEqual({ + code: 200, + data: { + image: { mimeType: 'image/jpeg', attached: true }, + meta: { width: 1280, height: 720 }, + }, + }); + expect(result.data.image.base64).toBe('large-image-payload'); + expect(JSON.stringify(redacted)).not.toContain('large-image-payload'); + }); + + it('does not clone ordinary tool results', () => { + const result = { code: 200, data: { value: 1 } }; + expect(redactInlineImageData(result)).toBe(result); + }); +}); diff --git a/tests/scene-screenshot-api.test.ts b/tests/scene-screenshot-api.test.ts new file mode 100644 index 000000000..b2e497846 --- /dev/null +++ b/tests/scene-screenshot-api.test.ts @@ -0,0 +1,132 @@ +const mockCapture = jest.fn(); +const mockReadFile = jest.fn(); +const mockUnlink = jest.fn(); +const mockResize = jest.fn(); +const mockPng = jest.fn(); +const mockJpeg = jest.fn(); +const mockToBuffer = jest.fn(); + +const mockPipeline = { + resize: mockResize, + png: mockPng, + jpeg: mockJpeg, + toBuffer: mockToBuffer, +}; + +mockResize.mockReturnValue(mockPipeline); +mockPng.mockReturnValue(mockPipeline); +mockJpeg.mockReturnValue(mockPipeline); + +jest.mock('../src/api/decorator/decorator.js', () => ({ + description: () => jest.fn(), + param: () => jest.fn(), + result: () => jest.fn(), + title: () => jest.fn(), + tool: () => jest.fn(), +}), { virtual: true }); + +jest.mock('../src/core/scene', () => ({ + Scene: { + Screenshot: { + capture: (...args: unknown[]) => mockCapture(...args), + }, + }, +})); + +jest.mock('fs/promises', () => ({ + readFile: (...args: unknown[]) => mockReadFile(...args), + unlink: (...args: unknown[]) => mockUnlink(...args), +})); + +jest.mock('sharp', () => ({ + __esModule: true, + default: jest.fn(() => mockPipeline), +})); + +import { SceneScreenshotApi } from '../src/api/scene/screenshot'; +import { SchemaScreenshotResult } from '../src/api/scene/screenshot-schema'; + +const shot = { + filePath: 'C:\\Temp\\cocos-cli-screenshot-test.png', + width: 1200, + height: 600, + sceneUrl: 'db://assets/main.scene', + sceneName: 'main.scene', + actualCamera: { + source: 'scene' as const, + projection: 'perspective' as const, + position: { x: 0, y: 0, z: 10 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + fov: 45, + priority: 0, + clearFlags: 1, + visibility: 0xffffffff, + viewport: { x: 0, y: 0, width: 1, height: 1 }, + }, + actualCameras: [{ + source: 'scene' as const, + projection: 'perspective' as const, + position: { x: 0, y: 0, z: 10 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + fov: 45, + priority: 0, + clearFlags: 1, + visibility: 0xffffffff, + viewport: { x: 0, y: 0, width: 1, height: 1 }, + }], +}; + +describe('SceneScreenshotApi image transport', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockResize.mockReturnValue(mockPipeline); + mockPng.mockReturnValue(mockPipeline); + mockJpeg.mockReturnValue(mockPipeline); + mockCapture.mockResolvedValue(shot); + mockReadFile.mockResolvedValue(Buffer.from('raw-png')); + mockUnlink.mockResolvedValue(undefined); + mockToBuffer.mockResolvedValue({ + data: Buffer.from('encoded-image'), + info: { width: 400, height: 200 }, + }); + }); + + it('returns output dimensions and removes the temporary PNG', async () => { + const result = await new SceneScreenshotApi().getSceneScreenshot({ + format: 'jpeg', + quality: 80, + maxSize: 400, + }); + + expect(result.code).toBe(200); + expect(result.data?.meta).toMatchObject({ + width: 400, + height: 200, + renderWidth: 1200, + renderHeight: 600, + }); + expect(result.data?.image).toMatchObject({ + base64: Buffer.from('encoded-image').toString('base64'), + mimeType: 'image/jpeg', + attached: true, + }); + expect(mockUnlink).toHaveBeenCalledWith(shot.filePath); + + const structured = SchemaScreenshotResult.parse(result.data); + expect(structured.image).toEqual({ mimeType: 'image/jpeg', attached: true }); + expect(structured).not.toHaveProperty('meta.filePath'); + }); + + it('removes the temporary PNG when image processing fails', async () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined); + mockToBuffer.mockRejectedValue(new Error('encode failed')); + + try { + const result = await new SceneScreenshotApi().getSceneScreenshot({ format: 'jpeg', quality: 80 }); + expect(result.code).toBeGreaterThanOrEqual(500); + expect(mockUnlink).toHaveBeenCalledWith(shot.filePath); + } finally { + consoleError.mockRestore(); + } + }); +}); From c39f4ab610c4495ee3786ae3bf8b696492e6ef8a Mon Sep 17 00:00:00 2001 From: Decvil <2534450611@qq.com> Date: Mon, 24 Aug 2026 18:02:33 +0800 Subject: [PATCH 2/4] feat(scene): add gizmos and live editor state to screenshots --- .../__snapshots__/dts-snapshot.test.ts.snap | 1 + src/api/scene/screenshot-schema.ts | 5 + src/api/scene/screenshot.ts | 1 + src/core/scene/browser-scene-state.ts | 133 +++++- src/core/scene/common/screenshot.ts | 6 + .../scene/scene-process/service/camera.ts | 113 ++++- .../scene/scene-process/service/editor.ts | 11 +- .../service/gizmo/utils/editor-node.ts | 119 +++++- .../scene/scene-process/service/screenshot.ts | 380 ++++++++++++++++- .../scene/scene-process/service/selection.ts | 134 +++++- .../scene/test/browser-scene-state.test.ts | 76 ++++ .../scene/test/editor-close-options.test.ts | 6 +- .../test/screenshot-camera-params.test.ts | 395 +++++++++++++++++- .../scene/test/selection-prefab-path.test.ts | 130 ++++++ tests/scene-screenshot-api.test.ts | 15 + 15 files changed, 1496 insertions(+), 29 deletions(-) diff --git a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap index 62e9381da..21fbdf05d 100644 --- a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap +++ b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap @@ -6671,6 +6671,7 @@ export declare interface IScreenshotOptions { height?: number; camera?: string; viewMode?: TScreenshotViewMode; + includeGizmos?: boolean; } export declare interface IScreenshotResult { filePath: string; diff --git a/src/api/scene/screenshot-schema.ts b/src/api/scene/screenshot-schema.ts index 31dde0990..417080fee 100644 --- a/src/api/scene/screenshot-schema.ts +++ b/src/api/scene/screenshot-schema.ts @@ -19,6 +19,11 @@ export const SchemaScreenshotOptions = z.object({ viewMode: z.enum(['auto', '2d', '3d']).optional().describe( 'View mode: auto follows the scene 2D/3D state.', ), + includeGizmos: z.boolean().optional().describe( + 'Overlay selected-node/component gizmos and the 3D scene-axis gizmo while preserving the normal scene-camera framing. ' + + 'This cannot be combined with camera. ' + + 'If omitted, it is enabled automatically when the current editor has a node selection.', + ), format: z.enum(['jpeg', 'png']).default('jpeg').describe( 'Output image format sent to the agent (default jpeg for smaller size).', ), diff --git a/src/api/scene/screenshot.ts b/src/api/scene/screenshot.ts index 808f7f321..cd0bfb94f 100644 --- a/src/api/scene/screenshot.ts +++ b/src/api/scene/screenshot.ts @@ -42,6 +42,7 @@ export class SceneScreenshotApi { height: options.height, camera: options.camera, viewMode: options.viewMode, + includeGizmos: options.includeGizmos, }); tempFilePath = shot.filePath; diff --git a/src/core/scene/browser-scene-state.ts b/src/core/scene/browser-scene-state.ts index af314bd24..8b435a04a 100644 --- a/src/core/scene/browser-scene-state.ts +++ b/src/core/scene/browser-scene-state.ts @@ -1,8 +1,42 @@ +export interface IBrowserEditorCameraState { + is2D: boolean; + position: { x: number; y: number; z: number }; + rotation: { x: number; y: number; z: number; w: number }; + projection: number; + fov: number; + orthoHeight: number; + near: number; + far: number; +} + +export interface IBrowserNodeTransformState { + uuid: string; + path: string; + revision: number; + position: { x: number; y: number; z: number }; + rotation: { x: number; y: number; z: number; w: number }; + scale: { x: number; y: number; z: number }; +} + +export interface IBrowserNodeSnapshotState { + uuid: string; + path: string; + revision: number; + /** JSON-safe inspector dump produced by dumpUtil.dumpNode(). */ + dump: unknown; +} + export interface IBrowserSceneState { uuid: string; url: string; type: string; name: string; + selection: string[]; + camera?: IBrowserEditorCameraState; + /** Unsaved transforms from the browser editor, merged by node UUID. */ + nodeTransforms: IBrowserNodeTransformState[]; + /** Latest unsaved node/component inspector values, merged by node UUID. */ + nodeSnapshots: IBrowserNodeSnapshotState[]; updatedAt: number; } @@ -15,7 +49,17 @@ export interface IBrowserSceneState { class BrowserSceneState { private current: IBrowserSceneState | null = null; - setCurrent(info: Omit): IBrowserSceneState { + setCurrent( + info: Omit< + IBrowserSceneState, + 'updatedAt' | 'selection' | 'camera' | 'nodeTransforms' | 'nodeSnapshots' + > & { + selection?: string[]; + camera?: IBrowserEditorCameraState; + nodeTransforms?: IBrowserNodeTransformState[]; + nodeSnapshots?: IBrowserNodeSnapshotState[]; + }, + ): IBrowserSceneState { if (!info?.uuid) { throw new Error('Browser scene UUID is required.'); } @@ -24,13 +68,59 @@ class BrowserSceneState { url: info.url ?? '', type: info.type ?? '', name: info.name ?? '', + selection: Array.isArray(info.selection) ? [...info.selection] : [], + camera: info.camera ? { ...info.camera } : undefined, + nodeTransforms: this.cloneNodeTransforms(info.nodeTransforms ?? []), + nodeSnapshots: this.cloneNodeSnapshots(info.nodeSnapshots ?? []), updatedAt: Date.now(), }; - return { ...this.current }; + return this.cloneCurrent(); + } + + setEditorState( + expectedUuid: string, + state: { + selection?: string[]; + camera?: IBrowserEditorCameraState; + nodeTransforms?: IBrowserNodeTransformState[]; + nodeSnapshots?: IBrowserNodeSnapshotState[]; + }, + ): boolean { + if (!this.current || this.current.uuid !== expectedUuid) { + return false; + } + if (Array.isArray(state.selection)) { + this.current.selection = [...state.selection]; + } + if (state.camera) { + this.current.camera = { ...state.camera }; + } + if (Array.isArray(state.nodeTransforms)) { + for (const transform of state.nodeTransforms) { + const index = this.current.nodeTransforms.findIndex(item => item.uuid === transform.uuid); + if (index < 0) { + this.current.nodeTransforms.push(this.cloneNodeTransform(transform)); + } else if (transform.revision >= this.current.nodeTransforms[index].revision) { + this.current.nodeTransforms[index] = this.cloneNodeTransform(transform); + } + } + } + if (Array.isArray(state.nodeSnapshots)) { + for (const snapshot of state.nodeSnapshots) { + const index = this.current.nodeSnapshots.findIndex(item => item.uuid === snapshot.uuid); + if (index < 0) { + this.current.nodeSnapshots.push(this.cloneNodeSnapshot(snapshot)); + } else if (snapshot.revision >= this.current.nodeSnapshots[index].revision) { + this.current.nodeSnapshots[index] = this.cloneNodeSnapshot(snapshot); + } + } + } + this.current.updatedAt = Date.now(); + return true; } getCurrent(): IBrowserSceneState | null { - return this.current ? { ...this.current } : null; + return this.current ? this.cloneCurrent() : null; } clearCurrent(expectedUuid?: string): boolean { @@ -40,6 +130,43 @@ class BrowserSceneState { this.current = null; return true; } + + private cloneCurrent(): IBrowserSceneState { + const current = this.current!; + return { + ...current, + selection: [...current.selection], + camera: current.camera ? { ...current.camera } : undefined, + nodeTransforms: this.cloneNodeTransforms(current.nodeTransforms), + nodeSnapshots: this.cloneNodeSnapshots(current.nodeSnapshots), + }; + } + + private cloneNodeTransforms(transforms: IBrowserNodeTransformState[]): IBrowserNodeTransformState[] { + return transforms.map(transform => this.cloneNodeTransform(transform)); + } + + private cloneNodeTransform(transform: IBrowserNodeTransformState): IBrowserNodeTransformState { + return { + ...transform, + position: { ...transform.position }, + rotation: { ...transform.rotation }, + scale: { ...transform.scale }, + }; + } + + private cloneNodeSnapshots(snapshots: IBrowserNodeSnapshotState[]): IBrowserNodeSnapshotState[] { + return snapshots.map(snapshot => this.cloneNodeSnapshot(snapshot)); + } + + private cloneNodeSnapshot(snapshot: IBrowserNodeSnapshotState): IBrowserNodeSnapshotState { + return { + ...snapshot, + // RPC transport is JSON-based; clone with the same semantics so callers + // cannot mutate the main-process cache after publishing. + dump: snapshot.dump == null ? snapshot.dump : JSON.parse(JSON.stringify(snapshot.dump)), + }; + } } export const browserSceneState = new BrowserSceneState(); diff --git a/src/core/scene/common/screenshot.ts b/src/core/scene/common/screenshot.ts index c8cc1bd18..365c36fd4 100644 --- a/src/core/scene/common/screenshot.ts +++ b/src/core/scene/common/screenshot.ts @@ -20,6 +20,12 @@ export interface IScreenshotOptions { camera?: string; /** 视图模式：auto 跟随场景 2D/3D 判定 */ viewMode?: TScreenshotViewMode; + /** + * 是否在正常场景相机取景上叠加选中节点/组件 Gizmo 和 3D 场景坐标轴。 + * Gizmo 会临时按内容相机计算大小和朝向，因此启用后不能同时指定 camera； + * 未传时，当前编辑器存在选中节点则自动启用。 + */ + includeGizmos?: boolean; } /** 实际使用的相机参数（回传给 Agent 便于复现取景） */ diff --git a/src/core/scene/scene-process/service/camera.ts b/src/core/scene/scene-process/service/camera.ts index 0f9f415a8..f34ef9c22 100644 --- a/src/core/scene/scene-process/service/camera.ts +++ b/src/core/scene/scene-process/service/camera.ts @@ -1,4 +1,4 @@ -import { Camera, Canvas, Color, Layers, Vec3, gfx } from 'cc'; +import { Camera, Canvas, Color, Layers, Quat, Vec3, gfx } from 'cc'; import { BaseService } from './core'; import { register, Service, queryRegisteredService } from './core/decorator'; import { CameraController2D } from './camera/camera-controller-2d'; @@ -24,6 +24,8 @@ export class CameraService extends BaseService implements ICamera private _currentUuid = ''; private _cameraInfos: Record = {}; private _cameraUuids: string[] = []; + private _restorePromise: Promise = Promise.resolve(); + private _browserCameraPublishTimer: ReturnType | null = null; get controller2D() { return this._controller2D; } get controller3D() { return this._controller3D; } @@ -47,6 +49,7 @@ export class CameraService extends BaseService implements ICamera this._controllerFirstChange = true; } Service.Engine.repaintInEditMode(); + this._scheduleBrowserCameraStatePublish(); } get is2D() { return this._controller === this._controller2D; } @@ -112,7 +115,8 @@ export class CameraService extends BaseService implements ICamera } this._detachSceneCameras(); - void this._restoreCameraView(initConfigTask); + this._restorePromise = this._restoreCameraView(initConfigTask); + void this._restorePromise; } catch (e) { console.warn('[Camera] onEditorOpened failed:', e); } @@ -130,6 +134,7 @@ export class CameraService extends BaseService implements ICamera this.defaultFocus(uuid); this._refreshSelectedGizmos(); Service.Engine.repaintInEditMode(); + this._scheduleBrowserCameraStatePublish(); } catch (e) { console.warn('[Camera] restore camera view failed:', e); } @@ -304,6 +309,7 @@ export class CameraService extends BaseService implements ICamera // --- 代理方法 --- focus(nodes?: string[] | null, editorCameraInfo?: any, immediate = false): void { this._controller?.focus(nodes as any, editorCameraInfo, immediate); + this._scheduleBrowserCameraStatePublish(immediate ? 50 : 350); } defaultFocus(uuid: string): void { @@ -332,6 +338,7 @@ export class CameraService extends BaseService implements ICamera changeProjection(): void { this._controller?.changeProjection(); + this._scheduleBrowserCameraStatePublish(); } setGridVisible(value: boolean, persist = true): void { @@ -383,6 +390,7 @@ export class CameraService extends BaseService implements ICamera if (persist) { void this._saveConfig(); } + this._scheduleBrowserCameraStatePublish(); } resetCameraProperty(): void { @@ -431,9 +439,9 @@ export class CameraService extends BaseService implements ICamera return this._camera?.fov ?? 45; } - zoomUp(): void { this._controller?.zoomUp(); } - zoomDown(): void { this._controller?.zoomDown(); } - zoomReset(): void { this._controller?.zoomReset(); } + zoomUp(): void { this._controller?.zoomUp(); this._scheduleBrowserCameraStatePublish(); } + zoomDown(): void { this._controller?.zoomDown(); this._scheduleBrowserCameraStatePublish(); } + zoomReset(): void { this._controller?.zoomReset(); this._scheduleBrowserCameraStatePublish(); } alignNodeToSceneView(nodes: string[]): void { this._controller?.alignNodeToSceneView(nodes); @@ -451,14 +459,23 @@ export class CameraService extends BaseService implements ICamera private onMouseDBlDown(event: any) { return this._controller?.onMouseDBlDown(event); } private onMouseDown(event: any) { return this._controller?.onMouseDown(event); } private onMouseMove(event: any) { return this._controller?.onMouseMove(event); } - private onMouseUp(event: any) { return this._controller?.onMouseUp(event); } - private onMouseWheel(event: any) { return this._controller?.onMouseWheel(event); } + private onMouseUp(event: any) { + const result = this._controller?.onMouseUp(event); + this._scheduleBrowserCameraStatePublish(); + return result; + } + private onMouseWheel(event: any) { + const result = this._controller?.onMouseWheel(event); + this._scheduleBrowserCameraStatePublish(); + return result; + } private onKeyDown(event: any) { return this._controller?.onKeyDown(event); } private onKeyUp(event: any) { return this._controller?.onKeyUp(event); } // --- 其他方法 --- onResize(size: any): void { this._controller?.onResize(size); + this._scheduleBrowserCameraStatePublish(); } refresh(): void { @@ -469,6 +486,88 @@ export class CameraService extends BaseService implements ICamera return this._camera; } + async waitForRestore(): Promise { + await this._restorePromise; + } + + getScreenshotState(): any { + const camera = this._camera; + if (!camera?.node) { + return undefined; + } + const position = camera.node.getWorldPosition(); + const rotation = camera.node.getWorldRotation(); + return { + is2D: this.is2D, + position: { x: position.x, y: position.y, z: position.z }, + rotation: { x: rotation.x, y: rotation.y, z: rotation.z, w: rotation.w }, + projection: camera.projection, + fov: camera.fov, + fovAxis: camera.fovAxis, + orthoHeight: camera.orthoHeight, + near: camera.near, + far: camera.far, + }; + } + + applyScreenshotState(state: any): void { + if (!state || !this._camera?.node) { + return; + } + if (typeof state.is2D === 'boolean') { + this.is2D = state.is2D; + } + if (state.position) { + this._camera.node.setWorldPosition(new Vec3( + state.position.x ?? 0, + state.position.y ?? 0, + state.position.z ?? 0, + )); + } + if (state.rotation) { + this._camera.node.setWorldRotation(new Quat( + state.rotation.x ?? 0, + state.rotation.y ?? 0, + state.rotation.z ?? 0, + state.rotation.w ?? 1, + )); + } + for (const key of ['projection', 'fov', 'fovAxis', 'orthoHeight', 'near', 'far'] as const) { + if (Number.isFinite(state[key])) { + (this._camera as any)[key] = state[key]; + } + } + this._controller?.updateGrid(); + this._refreshSelectedGizmos(); + Service.Engine?.repaintInEditMode?.(); + } + + private _scheduleBrowserCameraStatePublish(delay = 50): void { + if (!(Rpc as any).isWebTransport?.()) { + return; + } + if (this._browserCameraPublishTimer) { + clearTimeout(this._browserCameraPublishTimer); + } + this._browserCameraPublishTimer = setTimeout(() => { + this._browserCameraPublishTimer = null; + try { + const editorUuid = (Service.Editor as any)?.getCurrentEditorUuid?.(); + const camera = this.getScreenshotState(); + if (!editorUuid || !camera) { + return; + } + void Rpc.getInstance() + .request('browserSceneState', 'setEditorState', [editorUuid, { camera }]) + .catch((error: unknown) => { + console.warn('[Camera] Failed to publish PinK camera state.', error); + }); + } catch (error) { + console.warn('[Camera] Failed to publish PinK camera state.', error); + } + }, delay); + } + getCurCameraInfo(): any { const curCameraNode = this._controller3D.node; const curCameraPos = curCameraNode.getWorldPosition(); diff --git a/src/core/scene/scene-process/service/editor.ts b/src/core/scene/scene-process/service/editor.ts index ecec6c735..5f0bd5500 100644 --- a/src/core/scene/scene-process/service/editor.ts +++ b/src/core/scene/scene-process/service/editor.ts @@ -167,9 +167,12 @@ export class EditorService extends BaseService implements IEditor `当前场景 ${previousUuid} 存在未保存修改，无法切换或重新加载截图目标 ${targetUuid}。请先保存或撤销修改。`, ); } - if (previousUuid && !urlOrUUID && !targetChanged && isDirty) { - // Scene-process edits and screenshots already share this instance, - // so keep unsaved edits visible in the captured image. + if (previousUuid && !urlOrUUID && !targetChanged) { + // The browser and scene process already point at the same live editor + // instance. Reopening it would unnecessarily rebuild editor-only state + // (selection gizmos, grid/controller nodes, and the current camera view) + // immediately before capture. Explicit targets are still reopened below + // so callers can deliberately request the latest imported asset. return operation(); } @@ -585,6 +588,8 @@ export class EditorService extends BaseService implements IEditor url: assetInfo.url, type: assetInfo.type, name: assetInfo.name, + selection: Service.Selection?.query?.() ?? [], + camera: (Service.Camera as any)?.getScreenshotState?.(), }]); } catch (error) { console.warn('[Scene] Failed to publish PinK current scene.', error); diff --git a/src/core/scene/scene-process/service/gizmo/utils/editor-node.ts b/src/core/scene/scene-process/service/gizmo/utils/editor-node.ts index 53e7006e9..8384472fb 100644 --- a/src/core/scene/scene-process/service/gizmo/utils/editor-node.ts +++ b/src/core/scene/scene-process/service/gizmo/utils/editor-node.ts @@ -1,4 +1,5 @@ import cc, { type Node } from 'cc'; +import { formatUniqueName, normalizeNodePath, sanitizeNodeName } from '../../../../../engine/editor-extends/manager/path-utils'; function getEditorNodeApi(): any { const ccGlobal = (globalThis as any).cc; @@ -12,14 +13,35 @@ export function getEditorNodeByPath(path: string): Node | null { if (!path) { return null; } - return getEditorNodeApi()?.getNodeByPath?.(path) ?? getPrefabNodeByRelativePath(path); + const editorType = getCurrentEditorType(); + if (editorType !== 'prefab') { + const nodeInEditor = findSceneNodeByPath(getCurrentEditorRoot(), path); + if (nodeInEditor) { + return nodeInEditor; + } + } + + const node = getEditorNodeApi()?.getNodeByPath?.(path) ?? null; + if (node && !isEditorOnlyNode(node)) { + return node; + } + return getPrefabNodeByRelativePath(path); } export function getEditorNodeByUuid(uuid: string): Node | null { if (!uuid) { return null; } - return getEditorNodeApi()?.getNode?.(uuid) ?? null; + + // Runtime-created editor helpers (grid, gizmos, temporary cameras, etc.) + // use generated `Node.xxx` UUIDs. A deserialized scene node can carry the + // same UUID, while EditorExtends.Node.getNode() only returns one entry from + // the global lookup table. Prefer the node that belongs to the currently + // edited scene so selection/gizmo lookup cannot be redirected to an editor + // helper with a colliding UUID. + const editorRoot = getCurrentEditorRoot(); + const nodeInEditor = findNodeByUuid(editorRoot, uuid); + return nodeInEditor ?? getEditorNodeApi()?.getNode?.(uuid) ?? null; } export function getEditorNodeUuidByPath(path: string): string { @@ -50,6 +72,99 @@ function getPrefabNodeByRelativePath(path: string): Node | null { } } +function getCurrentEditorRoot(): Node | null { + try { + const { Service } = require('../../core/decorator'); + return Service.Editor?.getRootNode?.() as Node | null; + } catch { + return null; + } +} + +function getCurrentEditorType(): 'scene' | 'prefab' | 'unknown' { + try { + const { Service } = require('../../core/decorator'); + return Service.Editor?.getCurrentEditorType?.() ?? 'unknown'; + } catch { + return 'unknown'; + } +} + +function findNodeByUuid(root: Node | null, uuid: string): Node | null { + if (!root) { + return null; + } + if (root.uuid === uuid && !isEditorOnlyNode(root)) { + return root; + } + for (const child of root.children ?? []) { + const result = findNodeByUuid(child, uuid); + if (result) { + return result; + } + } + return null; +} + +/** + * Resolve a scene path from the actual hierarchy. NodeManager indexes by UUID, + * so a generated editor helper with a colliding UUID can otherwise make a + * perfectly valid path resolve to that helper. This mirrors NodePathManager's + * sibling suffix allocation while excluding editor-only branches. + */ +function findSceneNodeByPath(root: Node | null, path: string): Node | null { + if (!root) { + return null; + } + const normalized = normalizeNodePath(path.replace(/\\/g, '/')); + if (normalized === '/') { + return root; + } + const segments = normalized.split('/').filter(Boolean); + let current = root; + for (const segment of segments) { + const usedNames = new Set(); + const exactMatches: Node[] = []; + const insensitiveMatches: Node[] = []; + for (const child of current.children ?? []) { + if (isEditorOnlyNode(child)) { + continue; + } + const baseName = sanitizeNodeName(child.name); + let uniqueName = baseName; + let suffix = 1; + while (usedNames.has(uniqueName)) { + uniqueName = formatUniqueName(baseName, suffix++); + } + usedNames.add(uniqueName); + if (uniqueName === segment) { + exactMatches.push(child); + } else if (uniqueName.toLowerCase() === segment.toLowerCase()) { + insensitiveMatches.push(child); + } + } + const next = exactMatches[0] ?? (insensitiveMatches.length === 1 ? insensitiveMatches[0] : null); + if (!next) { + return null; + } + current = next; + } + return current; +} + +function isEditorOnlyNode(node: Node): boolean { + const layers = (cc as any)?.Layers?.Enum ?? (globalThis as any)?.cc?.Layers?.Enum; + const editorMask = (layers?.GIZMOS ?? 0) | (layers?.SCENE_GIZMO ?? 0) | (layers?.EDITOR ?? 0); + let current: Node | null = node; + while (current) { + if (editorMask && (current.layer & editorMask) !== 0) { + return true; + } + current = current.parent; + } + return false; +} + /** * Find child node within a prefab by traversing path segments level by level. * After name/path decoupling, prefers matching by system path last segment (uniquely determined); diff --git a/src/core/scene/scene-process/service/screenshot.ts b/src/core/scene/scene-process/service/screenshot.ts index db02afcb9..c795014e2 100644 --- a/src/core/scene/scene-process/service/screenshot.ts +++ b/src/core/scene/scene-process/service/screenshot.ts @@ -1,4 +1,4 @@ -import { Camera, Canvas, Color, Layers, Node as CCNode, Quat, Rect, UITransform, Vec3, renderer } from 'cc'; +import { Camera, Canvas, Color, gfx, Layers, Node as CCNode, Quat, Rect, UITransform, Vec3, renderer } from 'cc'; import { PNG } from 'pngjs'; import { writeFileSync } from 'fs'; import { tmpdir } from 'os'; @@ -6,6 +6,12 @@ import { join } from 'path'; import { BaseService, register, Service } from './core'; import { Rpc } from '../rpc'; import { ScreenshotBuffer } from './screenshot/screenshot-buffer'; +import { getEditorNodeByPath, getEditorNodeByUuid } from './gizmo/utils/editor-node'; +import dumpUtil from './dump'; +import type { + IBrowserNodeSnapshotState, + IBrowserNodeTransformState, +} from '../../browser-scene-state'; import type { IScreenshotCameraInfo, IScreenshotEvents, @@ -22,6 +28,10 @@ const NODE_SUMMARY_MAX_CHILDREN = 40; /** 临时截图相机在场景树里的节点名，销毁后不残留 */ const TEMP_CAMERA_NODE_NAME = '__cli_screenshot_camera__'; +// These names are engine contracts. PostProcessBuilder explicitly recognizes +// them when ordering editor cameras, so do not add CLI-specific prefixes. +const EDITOR_GIZMO_CAMERA_NAME = 'Editor UIGizmoCamera'; +const SCENE_GIZMO_CAMERA_NAME = 'Scene Gizmo Camera'; /** 从任意相机组件抽取出的、创建临时相机所需的取景参数（JSON 无关，含引擎对象） */ interface ICameraParams { @@ -54,12 +64,22 @@ interface IResolvedFraming { source: 'scene' | 'editor'; renderNote?: string; params: ICameraParams; + /** Engine-recognized camera name used by editor render-pipeline ordering. */ + runtimeCameraName?: string; /** 原始场景相机，仅用于判断它是否由 Canvas 自动适配。 */ sourceCamera?: Camera; /** Prefab 自动取景时直接记录 Canvas，避免依赖场景相机。 */ canvas?: Canvas; } +interface IBrowserSceneCaptureState { + uuid?: string; + selection?: string[]; + camera?: unknown; + nodeTransforms?: IBrowserNodeTransformState[]; + nodeSnapshots?: IBrowserNodeSnapshotState[]; +} + /** * 场景截图服务。 * @@ -97,36 +117,195 @@ export class ScreenshotService extends BaseService implements // 在重新打开场景前主动同步，让 Canvas/Widget 在实例化时就按最新设计分辨率布局。 await (Service.Engine as any)?.syncDesignResolution?.(); + const browserState = await this._queryBrowserSceneState(); + // Keep prepare/render/restore in one serialized editor lifecycle scope. // Explicit target captures restore the previous clean scene afterwards; // a switch that would discard unsaved edits is rejected. return Service.Editor.withScreenshotScene( options.sceneUrlOrUUID, - () => this._capturePreparedScene(options), + async () => { + await (Service.Camera as any)?.waitForRestore?.(); + await this._applyBrowserEditorStateForCapture(browserState); + return this._capturePreparedScene(options); + }, ); } + private async _queryBrowserSceneState(): Promise { + // A browser scene owns its local SelectionService and publishes changes; + // only the separate headless worker needs to consume that snapshot. + if ((Rpc as any).isWebTransport?.()) { + return null; + } + try { + return await Rpc.getInstance().request('browserSceneState', 'getCurrent', []); + } catch (error) { + console.warn('[Screenshot] Unable to query PinK selection state.', error); + return null; + } + } + + private async _applyBrowserEditorStateForCapture(browserState: IBrowserSceneCaptureState | null): Promise { + if (!browserState) { + return; + } + const currentUuid = (Service.Editor as any)?.getCurrentEditorUuid?.(); + if (!currentUuid || currentUuid !== browserState.uuid) { + return; + } + if (browserState.camera) { + (Service.Camera as any)?.applyScreenshotState?.(browserState.camera); + } + const snapshotRevisions = await this._applyBrowserNodeSnapshots(browserState.nodeSnapshots); + this._applyBrowserNodeTransforms(browserState.nodeTransforms, snapshotRevisions); + if (!Array.isArray(browserState.selection)) { + return; + } + const currentSelection: string[] = (Service.Selection as any)?.query?.() ?? []; + if (currentSelection.length === browserState.selection.length + && currentSelection.every((path, index) => path === browserState.selection![index])) { + return; + } + (Service.Selection as any)?.clear?.(); + for (const path of browserState.selection) { + if (typeof path === 'string' && path) { + (Service.Selection as any)?.select?.(path); + } + } + } + + private async _applyBrowserNodeSnapshots( + snapshots?: IBrowserNodeSnapshotState[], + ): Promise> { + const revisions = new Map(); + if (!Array.isArray(snapshots)) { + return revisions; + } + for (const snapshot of snapshots) { + const node = this._resolveBrowserStateNode(snapshot.uuid, snapshot.path); + if (!node || !snapshot.dump) { + continue; + } + await dumpUtil.restoreNodeSnapshotProperties(node, snapshot.dump); + const componentDumps = Array.isArray((snapshot.dump as any).__comps__) + ? (snapshot.dump as any).__comps__ + : []; + for (let index = 0; index < componentDumps.length; index++) { + const component = this._resolveSnapshotComponent(node, componentDumps[index], index); + if (component) { + await dumpUtil.restoreComponentSnapshotProperties(component, componentDumps[index]); + } + } + (node as any).updateWorldTransform?.(); + revisions.set(snapshot.uuid || snapshot.path, snapshot.revision); + } + return revisions; + } + + private _applyBrowserNodeTransforms( + transforms?: IBrowserNodeTransformState[], + snapshotRevisions = new Map(), + ): void { + if (!Array.isArray(transforms)) { + return; + } + for (const transform of transforms) { + const snapshotRevision = snapshotRevisions.get(transform.uuid || transform.path) ?? -1; + if (transform.revision <= snapshotRevision) { + continue; + } + const node = this._resolveBrowserStateNode(transform.uuid, transform.path); + if (!node) { + continue; + } + node.setPosition(new Vec3( + this._finiteNumber(transform.position?.x, node.position.x), + this._finiteNumber(transform.position?.y, node.position.y), + this._finiteNumber(transform.position?.z, node.position.z), + )); + node.setRotation(new Quat( + this._finiteNumber(transform.rotation?.x, node.rotation.x), + this._finiteNumber(transform.rotation?.y, node.rotation.y), + this._finiteNumber(transform.rotation?.z, node.rotation.z), + this._finiteNumber(transform.rotation?.w, node.rotation.w), + )); + node.setScale(new Vec3( + this._finiteNumber(transform.scale?.x, node.scale.x), + this._finiteNumber(transform.scale?.y, node.scale.y), + this._finiteNumber(transform.scale?.z, node.scale.z), + )); + (node as any).updateWorldTransform?.(); + } + } + + private _resolveBrowserStateNode(uuid: string, path: string): CCNode | null { + return getEditorNodeByUuid(uuid) ?? getEditorNodeByPath(path); + } + + private _resolveSnapshotComponent(node: CCNode, componentDump: any, index: number): any | null { + const components = (node as any).components ?? (node as any)._components ?? []; + const uuid = componentDump?.value?.uuid?.value; + if (uuid) { + const byUuid = components.find((component: any) => component?.uuid === uuid); + if (byUuid) { + return byUuid; + } + } + const candidate = components[index]; + if (!candidate) { + return null; + } + const className = (cc as any).js?.getClassName?.(candidate); + return !componentDump?.type || !className || className === componentDump.type + ? candidate + : null; + } + private async _capturePreparedScene(options: IScreenshotOptions): Promise { const scene = (cc as any).director?.getScene?.() as CCNode | null; if (!scene) { throw new Error('当前没有打开的场景，请先调用 scene-open 打开场景后再截图。'); } + const includeGizmos = this._shouldIncludeEditorGizmos(options); + const captureOptions = includeGizmos === options.includeGizmos + ? options + : { ...options, includeGizmos }; - if (!options.camera && this._findSceneCameras().length === 0) { - this._focusEditorCameraForCapture(options); + if (includeGizmos && captureOptions.camera) { + throw new Error('includeGizmos 不能与 camera 同时指定。'); } - const { width, height } = this._resolveCaptureSize(options, scene); + // Whether or not gizmos are requested, resolve the content exactly like a + // normal screenshot. Gizmos are appended later as a transparent overlay. + if (!captureOptions.camera && this._findSceneCameras().length === 0) { + this._focusEditorCameraForCapture(captureOptions); + } + + const { width, height } = this._resolveCaptureSize(captureOptions, scene); // 解析取景后，按最终离屏目标修正每台自动适配的 Canvas 相机。 - const framings = this._resolveFramings(options); + const framings = this._resolveFramings(captureOptions, width, height); framings.forEach(framing => this._fitAlignedCanvasFraming(framing, width, height)); // 每个源相机对应一台完全自持的临时相机；所有临时相机共享同一个离屏窗口， // 由 RenderWindow 按 priority 从小到大完成一帧合成。 const temporaryCameras: Array<{ node: CCNode; comp: Camera; cam: any }> = []; + let restoreEditorCamera: (() => void) | null = null; let result: { width: number; height: number; buffer: Uint8Array }; try { + if (includeGizmos) { + const reference = this._topmostFraming(framings); + restoreEditorCamera = this._alignEditorCameraForGizmoCapture(reference); + this._refreshEditorGizmosForCapture(); + framings.push(...this._resolveGizmoOverlayFramings( + reference, + captureOptions, + width, + height, + )); + } + for (let index = 0; index < framings.length; index++) { temporaryCameras.push(this._createTemporaryCamera(scene, framings[index], index)); } @@ -137,6 +316,16 @@ export class ScreenshotService extends BaseService implements for (const { node, comp } of temporaryCameras.reverse()) { this._teardownCamera(node, comp); } + if (restoreEditorCamera) { + try { + restoreEditorCamera(); + } catch (error) { + console.warn('[Screenshot] Failed to restore editor camera after capture.', error); + } + // The gizmo models were rebuilt against the temporary scene-camera + // view. Restore their normal editor-view geometry as well. + this._refreshEditorGizmosForCapture(); + } // 恢复编辑器正常渲染 try { const engine = Service.Engine as any; @@ -176,7 +365,7 @@ export class ScreenshotService extends BaseService implements index: number, ): { node: CCNode; comp: Camera; cam: any } { const { params } = framing; - const node = new CCNode(`${TEMP_CAMERA_NODE_NAME}_${index}`); + const node = new CCNode(framing.runtimeCameraName ?? `${TEMP_CAMERA_NODE_NAME}_${index}`); let comp: Camera | null = null; try { // Must be on the EDITOR layer before addComponent(Camera), otherwise @@ -233,7 +422,30 @@ export class ScreenshotService extends BaseService implements // ---- 取景 ---- - private _resolveFramings(options: IScreenshotOptions): IResolvedFraming[] { + private _shouldIncludeEditorGizmos(options: IScreenshotOptions): boolean { + if (typeof options.includeGizmos === 'boolean') { + return options.includeGizmos; + } + if (options.camera) { + return false; + } + try { + const selectedPaths = (Service.Selection as any)?.query?.(); + return Array.isArray(selectedPaths) && selectedPaths.length > 0; + } catch { + return false; + } + } + + private _resolveFramings( + options: IScreenshotOptions, + targetWidth = DEFAULT_SIZE, + targetHeight = DEFAULT_SIZE, + ): IResolvedFraming[] { + if (options.includeGizmos && options.camera) { + throw new Error('includeGizmos 不能与 camera 同时指定。'); + } + // 1) 指定相机 if (options.camera) { const cam = this._findCameraByRef(options.camera); @@ -294,6 +506,158 @@ export class ScreenshotService extends BaseService implements }]; } + /** + * 只在正常内容相机栈上追加 Gizmo 覆盖层，不改变场景内容的取景。 + */ + private _resolveGizmoOverlayFramings( + reference: IResolvedFraming, + options: IScreenshotOptions, + targetWidth: number, + targetHeight: number, + ): IResolvedFraming[] { + const gizmoParams = this._cloneCameraParams(reference.params); + gizmoParams.priority = reference.params.priority + 2; + gizmoParams.clearFlags = gfx.ClearFlagBit.NONE; + gizmoParams.visibility = Layers.Enum.GIZMOS | Layers.Enum.IGNORE_RAYCAST; + gizmoParams.usePostProcess = false; + gizmoParams.postProcess = undefined; + const framings: IResolvedFraming[] = [{ + info: this._cameraInfoFromParams(gizmoParams, 'editor', EDITOR_GIZMO_CAMERA_NAME), + source: 'editor', + params: gizmoParams, + runtimeCameraName: EDITOR_GIZMO_CAMERA_NAME, + renderNote: '已保持场景相机取景，并叠加选中节点/组件 Gizmo。', + }]; + + const gizmoService = Service.Gizmo as any; + const sceneGizmoCamera = gizmoService?.sceneGizmoCamera as Camera | null; + const is2DView = options.viewMode === '2d' + || (options.viewMode !== '3d' && gizmoService?.is2D === true); + const includeSceneGizmo = !is2DView + && Boolean(sceneGizmoCamera?.node) + && sceneGizmoCamera?.enabledInHierarchy !== false; + if (includeSceneGizmo && sceneGizmoCamera) { + const sceneGizmoParams = this._paramsFromComponent(sceneGizmoCamera); + sceneGizmoParams.priority = Math.max(sceneGizmoParams.priority, gizmoParams.priority + 1); + sceneGizmoParams.rect = this._sceneGizmoRect(targetWidth, targetHeight); + sceneGizmoParams.visibility = Layers.Enum.SCENE_GIZMO; + sceneGizmoParams.clearFlags = gfx.ClearFlagBit.DEPTH_STENCIL; + sceneGizmoParams.usePostProcess = false; + sceneGizmoParams.postProcess = undefined; + framings.push({ + info: this._cameraInfoFromParams(sceneGizmoParams, 'editor', SCENE_GIZMO_CAMERA_NAME), + source: 'editor', + params: sceneGizmoParams, + runtimeCameraName: SCENE_GIZMO_CAMERA_NAME, + renderNote: '已按截图尺寸叠加右上角 3D 场景坐标轴。', + }); + } + + return framings; + } + + private _topmostFraming(framings: IResolvedFraming[]): IResolvedFraming { + if (!framings.length) { + throw new Error('没有可用于 Gizmo 叠加的内容相机。'); + } + return framings.reduce((topmost, framing) => ( + framing.params.priority >= topmost.params.priority ? framing : topmost + )); + } + + /** + * Transform Gizmo 的世界几何由编辑器相机计算。截图期间让编辑器相机临时 + * 对齐内容栈最上层相机，使生成的 Gizmo 与离屏覆盖相机拥有相同投影；调用者 + * 必须在 finally 中执行返回的恢复函数。 + */ + private _alignEditorCameraForGizmoCapture(reference: IResolvedFraming): (() => void) | null { + const cameraService = Service.Camera as any; + if (typeof cameraService?.getScreenshotState !== 'function' + || typeof cameraService?.applyScreenshotState !== 'function') { + return null; + } + + const previousState = cameraService.getScreenshotState(); + if (!previousState) { + return null; + } + const { params } = reference; + const captureState = { + is2D: params.projection === (Camera.ProjectionType?.ORTHO ?? 0), + position: { x: params.position.x, y: params.position.y, z: params.position.z }, + rotation: { + x: params.rotation.x, + y: params.rotation.y, + z: params.rotation.z, + w: params.rotation.w, + }, + projection: params.projection, + fov: params.fov, + fovAxis: params.fovAxis, + orthoHeight: params.orthoHeight, + near: params.near, + far: params.far, + }; + + try { + cameraService.applyScreenshotState(captureState); + } catch (error) { + try { + cameraService.applyScreenshotState(previousState); + } catch { + // Preserve the original alignment failure. + } + throw error; + } + return () => cameraService.applyScreenshotState(previousState); + } + + private _cloneCameraParams(params: ICameraParams): ICameraParams { + return { + ...params, + position: params.position.clone(), + rotation: params.rotation.clone(), + clearColor: params.clearColor?.clone(), + rect: params.rect.clone(), + }; + } + + /** 与 GizmoService.setSceneGizmoCameraRect 相同，但按离屏截图尺寸计算。 */ + private _sceneGizmoRect(width: number, height: number): Rect { + const safeWidth = this._finitePositive(width, DEFAULT_SIZE); + const safeHeight = this._finitePositive(height, DEFAULT_SIZE); + const heightPercent = 1 / 6; + const delta = ((safeWidth - safeHeight) * heightPercent) / 2 / safeWidth; + const pixelRatio = typeof window !== 'undefined' ? window.devicePixelRatio || 1 : 1; + const padding = (30 * pixelRatio) / safeHeight; + return new Rect( + 1 - heightPercent + delta, + 1 - heightPercent - padding, + heightPercent, + heightPercent, + ); + } + + private _refreshEditorGizmosForCapture(): void { + try { + const gizmo = Service.Gizmo as any; + const selectedPaths: string[] = (Service.Selection as any)?.query?.() ?? []; + // refreshSelectedGizmos only updates already-associated gizmos. Rebind + // the current selection first so a just-opened/reloaded editor cannot + // reach capture with selection state but without its render models. + for (const path of selectedPaths) { + gizmo?.onSelectionSelect?.(path); + } + for (const node of gizmo?.querySelectNodes?.() ?? []) { + gizmo?.showAllGizmoOfNode?.(node); + } + gizmo?.refreshSelectedGizmos?.(); + gizmo?.onUpdate?.(0); + } catch (error) { + console.warn('[Screenshot] Failed to refresh editor gizmos before capture.', error); + } + } + private get _editorMask(): number { return Layers.makeMaskInclude([ Layers.Enum.GIZMOS, diff --git a/src/core/scene/scene-process/service/selection.ts b/src/core/scene/scene-process/service/selection.ts index d70a8841b..e6772e5db 100644 --- a/src/core/scene/scene-process/service/selection.ts +++ b/src/core/scene/scene-process/service/selection.ts @@ -1,11 +1,14 @@ import { BaseService } from './core'; -import { register } from './core/decorator'; +import { register, Service } from './core/decorator'; import { ServiceEvents } from './core/global-events'; import type { ISelectionService, ISelectionEvents, IChangeNodeOptions } from '../../common'; import { NodeEventType } from '../../common'; import type { Node } from 'cc'; import { getEditorNodeByUuid, getEditorNodePath, getEditorNodeUuidByPath } from './gizmo/utils/editor-node'; import { normalizeNodePath } from '../../../engine/editor-extends/manager/path-utils'; +import { Rpc } from '../rpc'; +import type { IBrowserNodeSnapshotState, IBrowserNodeTransformState } from '../../browser-scene-state'; +import dumpUtil from './dump'; function pathToUuid(path: string): string { return getEditorNodeUuidByPath(path); @@ -26,9 +29,15 @@ interface SelectionEntry { export class SelectionService extends BaseService implements ISelectionService { private _selections: SelectionEntry[] = []; private _onNodeChangedHandler?: (node: Node, opts?: IChangeNodeOptions) => void; + private _browserNodeRevision = 0; init() { this._onNodeChangedHandler = (node: Node, opts: IChangeNodeOptions = {}) => { + if (opts.type === NodeEventType.TRANSFORM_CHANGED) { + this._publishBrowserNodeTransform(node); + } else { + this._publishBrowserNodeSnapshot(node); + } if (opts.type === NodeEventType.SET_PROPERTY && opts.propPath === 'name') { this._onNodePathChanged(node); } else if (opts.type === NodeEventType.PARENT_CHANGED) { @@ -68,6 +77,7 @@ export class SelectionService extends BaseService implements I this._callFocusInEditor(uuid); } this.broadcast('selection:select', normalized, this._getPaths()); + this._publishBrowserSelection(); } unselect(path: string): void { @@ -80,6 +90,7 @@ export class SelectionService extends BaseService implements I this._callLostFocusInEditor(entry.uuid); } this.broadcast('selection:unselect', normalized, this._getPaths()); + this._publishBrowserSelection(); } clear(): void { @@ -93,6 +104,7 @@ export class SelectionService extends BaseService implements I } } this.broadcast('selection:clear'); + this._publishBrowserSelection(); } query(): string[] { @@ -112,6 +124,126 @@ export class SelectionService extends BaseService implements I return this._selections.map(e => e.path); } + /** Keep the headless screenshot worker in sync with the browser editor selection. */ + private _publishBrowserSelection(): void { + if (!(Rpc as any).isWebTransport?.()) { + return; + } + try { + const editorUuid = (Service.Editor as any)?.getCurrentEditorUuid?.(); + if (!editorUuid) { + return; + } + void Rpc.getInstance() + .request('browserSceneState', 'setEditorState', [editorUuid, { + selection: this._getPaths(), + camera: (Service.Camera as any)?.getScreenshotState?.(), + }]) + .catch((error: unknown) => { + console.warn('[Selection] Failed to publish PinK selection.', error); + }); + } catch (error) { + console.warn('[Selection] Failed to publish PinK selection.', error); + } + } + + /** Keep unsaved browser-side node movement visible to the screenshot worker. */ + private _publishBrowserNodeTransform(node: Node): void { + if (!(Rpc as any).isWebTransport?.()) { + return; + } + try { + const editor = Service.Editor as any; + const editorUuid = editor?.getCurrentEditorUuid?.(); + const root = editor?.getRootNode?.() as Node | null; + if (!editorUuid || !root || !this._belongsToEditor(node, root)) { + return; + } + const path = getEditorNodePath(node); + if (!path) { + return; + } + const position = (node as any).position; + const rotation = (node as any).rotation; + const scale = (node as any).scale; + if (!position || !rotation || !scale) { + return; + } + const transform: IBrowserNodeTransformState = { + uuid: node.uuid, + path, + revision: ++this._browserNodeRevision, + position: { x: position.x, y: position.y, z: position.z }, + rotation: { x: rotation.x, y: rotation.y, z: rotation.z, w: rotation.w }, + scale: { x: scale.x, y: scale.y, z: scale.z }, + }; + void Rpc.getInstance() + .request('browserSceneState', 'setEditorState', [editorUuid, { + nodeTransforms: [transform], + }]) + .catch((error: unknown) => { + console.warn('[Selection] Failed to publish PinK node transform.', error); + }); + } catch (error) { + console.warn('[Selection] Failed to publish PinK node transform.', error); + } + } + + /** Publish a JSON-safe dump so all unsaved Node and Component inspector values + * can be restored in the separate screenshot worker without saving the asset. */ + private _publishBrowserNodeSnapshot(node: Node): void { + if (!(Rpc as any).isWebTransport?.()) { + return; + } + try { + const context = this._getBrowserNodeContext(node); + if (!context) { + return; + } + const dump = dumpUtil.dumpNode(node); + if (!dump) { + return; + } + const snapshot: IBrowserNodeSnapshotState = { + uuid: node.uuid, + path: context.path, + revision: ++this._browserNodeRevision, + dump, + }; + void Rpc.getInstance() + .request('browserSceneState', 'setEditorState', [context.editorUuid, { + nodeSnapshots: [snapshot], + }]) + .catch((error: unknown) => { + console.warn('[Selection] Failed to publish PinK node inspector state.', error); + }); + } catch (error) { + console.warn('[Selection] Failed to publish PinK node inspector state.', error); + } + } + + private _getBrowserNodeContext(node: Node): { editorUuid: string; path: string } | null { + const editor = Service.Editor as any; + const editorUuid = editor?.getCurrentEditorUuid?.(); + const root = editor?.getRootNode?.() as Node | null; + if (!editorUuid || !root || !this._belongsToEditor(node, root)) { + return null; + } + const path = getEditorNodePath(node); + return path ? { editorUuid, path } : null; + } + + private _belongsToEditor(node: Node, root: Node): boolean { + let current: Node | null = node; + while (current) { + if (current === root) { + return true; + } + current = current.parent; + } + return false; + } + private _callFocusInEditor(uuid: string): void { try { const node = getEditorNodeByUuid(uuid) as any; diff --git a/src/core/scene/test/browser-scene-state.test.ts b/src/core/scene/test/browser-scene-state.test.ts index ddea02380..16f637cc4 100644 --- a/src/core/scene/test/browser-scene-state.test.ts +++ b/src/core/scene/test/browser-scene-state.test.ts @@ -14,10 +14,37 @@ describe('BrowserSceneState', () => { }); expect(stored).toMatchObject({ uuid: 'scene-uuid', url: 'db://assets/main.scene' }); + expect(stored.selection).toEqual([]); expect(browserSceneState.getCurrent()).toEqual(stored); expect(stored.updatedAt).toEqual(expect.any(Number)); }); + it('tracks browser selection only for the currently published scene', () => { + browserSceneState.setCurrent({ + uuid: 'scene-uuid', + url: 'db://assets/main.scene', + type: 'scene', + name: 'main.scene', + }); + + expect(browserSceneState.setEditorState('other-scene', { selection: ['Canvas/Other'] })).toBe(false); + expect(browserSceneState.setEditorState('scene-uuid', { + selection: ['Canvas/GreenBtn'], + camera: { + is2D: true, + position: { x: 640, y: 360, z: 5000 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + projection: 0, + fov: 45, + orthoHeight: 360, + near: 6, + far: 10000, + }, + })).toBe(true); + expect(browserSceneState.getCurrent()?.selection).toEqual(['Canvas/GreenBtn']); + expect(browserSceneState.getCurrent()?.camera?.orthoHeight).toBe(360); + }); + it('does not let an older browser scene clear a newer selection', () => { browserSceneState.setCurrent({ uuid: 'new-scene-uuid', @@ -31,4 +58,53 @@ describe('BrowserSceneState', () => { expect(browserSceneState.clearCurrent('new-scene-uuid')).toBe(true); expect(browserSceneState.getCurrent()).toBeNull(); }); + + it('merges live node transforms and ignores an older update for the same node', () => { + browserSceneState.setCurrent({ + uuid: 'scene-uuid', + url: 'db://assets/main.scene', + type: 'scene', + name: 'main.scene', + }); + const transform = (revision: number, x: number) => ({ + uuid: 'button-uuid', + path: 'Canvas/Button', + revision, + position: { x, y: 20, z: 0 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + scale: { x: 1, y: 1, z: 1 }, + }); + + browserSceneState.setEditorState('scene-uuid', { nodeTransforms: [transform(2, 200)] }); + browserSceneState.setEditorState('scene-uuid', { nodeTransforms: [transform(1, 100)] }); + + const current = browserSceneState.getCurrent()!; + expect(current.nodeTransforms).toEqual([transform(2, 200)]); + current.nodeTransforms[0].position.x = 999; + expect(browserSceneState.getCurrent()!.nodeTransforms[0].position.x).toBe(200); + }); + + it('stores independent copies of the latest inspector dump for each node', () => { + browserSceneState.setCurrent({ + uuid: 'scene-uuid', + url: 'db://assets/main.scene', + type: 'scene', + name: 'main.scene', + }); + const newer = { + uuid: 'button-uuid', + path: 'Canvas/Button', + revision: 2, + dump: { active: { value: true }, __comps__: [{ value: { color: { value: '#ff0000' } } }] }, + }; + browserSceneState.setEditorState('scene-uuid', { nodeSnapshots: [newer] }); + browserSceneState.setEditorState('scene-uuid', { + nodeSnapshots: [{ ...newer, revision: 1, dump: { active: { value: false } } }], + }); + + const current = browserSceneState.getCurrent()!; + expect(current.nodeSnapshots).toEqual([newer]); + (current.nodeSnapshots[0].dump as any).active.value = false; + expect((browserSceneState.getCurrent()!.nodeSnapshots[0].dump as any).active.value).toBe(true); + }); }); diff --git a/src/core/scene/test/editor-close-options.test.ts b/src/core/scene/test/editor-close-options.test.ts index 718312a15..db8f6c4aa 100644 --- a/src/core/scene/test/editor-close-options.test.ts +++ b/src/core/scene/test/editor-close-options.test.ts @@ -190,14 +190,14 @@ describe('Screenshot scene preparation', () => { mockRpcRequest.mockReset(); }); - it('reopens a clean current scene without saving before a screenshot', async () => { + it('keeps a clean current scene open so editor-only screenshot state is preserved', async () => { const { service, close, open } = createService('scene-uuid', false); const operation = jest.fn(async () => 'captured'); await expect(service.withScreenshotScene(undefined, operation)).resolves.toBe('captured'); - expect(close).toHaveBeenCalledWith({ urlOrUUID: 'scene-uuid', save: false }); - expect(open).toHaveBeenCalledWith({ urlOrUUID: 'scene-uuid' }); + expect(close).not.toHaveBeenCalled(); + expect(open).not.toHaveBeenCalled(); expect(operation).toHaveBeenCalledTimes(1); }); diff --git a/src/core/scene/test/screenshot-camera-params.test.ts b/src/core/scene/test/screenshot-camera-params.test.ts index 5096629ba..5889e199f 100644 --- a/src/core/scene/test/screenshot-camera-params.test.ts +++ b/src/core/scene/test/screenshot-camera-params.test.ts @@ -1,9 +1,17 @@ jest.mock('cc', () => { class Vec3 { constructor(public x = 0, public y = 0, public z = 0) {} + + clone() { + return new Vec3(this.x, this.y, this.z); + } } class Quat { constructor(public x = 0, public y = 0, public z = 0, public w = 1) {} + + clone() { + return new Quat(this.x, this.y, this.z, this.w); + } } class Rect { constructor( @@ -50,10 +58,17 @@ jest.mock('cc', () => { default: {}, Camera, Canvas: class {}, - Color: class {}, + Color: class { + constructor(public r = 0, public g = 0, public b = 0, public a = 255) {} + + clone() { + return new (this.constructor as any)(this.r, this.g, this.b, this.a); + } + }, + gfx: { ClearFlagBit: { NONE: 0, DEPTH_STENCIL: 6 } }, Layers: { BitMask: { PROFILER: 1 }, - Enum: { EDITOR: 2, GIZMOS: 4, SCENE_GIZMO: 8 }, + Enum: { EDITOR: 2, GIZMOS: 4, SCENE_GIZMO: 8, IGNORE_RAYCAST: 16 }, makeMaskExclude: jest.fn(() => 0), makeMaskInclude: jest.fn((layers: number[]) => layers.reduce((mask, layer) => mask | layer, 0)), }, @@ -80,6 +95,14 @@ jest.mock('../scene-process/service/screenshot/screenshot-buffer', () => ({ ScreenshotBuffer: class {}, })); +jest.mock('../scene-process/service/dump', () => ({ + __esModule: true, + default: { + restoreNodeSnapshotProperties: jest.fn(() => Promise.resolve()), + restoreComponentSnapshotProperties: jest.fn(() => Promise.resolve()), + }, +})); + import { Canvas, Node, UITransform } from 'cc'; import { ScreenshotService } from '../scene-process/service/screenshot'; @@ -320,6 +343,374 @@ describe('Screenshot camera parameter normalization', () => { expect(findSceneCameras).not.toHaveBeenCalled(); }); + it('preserves the normal scene-camera stack and appends gizmo overlays', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const makeNode = (name: string, position = new (jest.requireMock('cc').Vec3)(0, 0, 10)) => ({ + name, + getWorldPosition: jest.fn(() => position), + getWorldRotation: jest.fn(() => new (jest.requireMock('cc').Quat)()), + }); + const mainCamera = { + node: makeNode('Main Camera', new (jest.requireMock('cc').Vec3)(1, 2, 30)), + enabledInHierarchy: true, + targetTexture: null, + projection: 1, + priority: 0, + fov: 55, + near: 0.1, + far: 10000, + clearFlags: 1, + visibility: 0xffff, + rect: new (jest.requireMock('cc').Rect)(0, 0, 1, 1), + }; + const uiCamera = { + node: makeNode('UI Camera', new (jest.requireMock('cc').Vec3)(640, 360, 1000)), + enabledInHierarchy: true, + targetTexture: null, + projection: 0, + priority: 100, + orthoHeight: 360, + near: 0.1, + far: 10000, + clearFlags: 1, + visibility: 0xffffffff, + rect: new (jest.requireMock('cc').Rect)(0, 0, 1, 1), + }; + const sceneGizmoCamera = { + node: makeNode('Live Scene Gizmo Camera', new (jest.requireMock('cc').Vec3)(0, 0, 40)), + enabledInHierarchy: true, + projection: 1, + priority: 1000, + fov: 45, + near: 0.1, + far: 1000, + clearFlags: 6, + visibility: 8, + rect: new (jest.requireMock('cc').Rect)(0.7, 0.8, 0.2, 0.2), + }; + jest.spyOn(service as any, '_findSceneCameras').mockReturnValue([mainCamera, uiCamera]); + Service.Gizmo = { sceneGizmoCamera }; + + const contentFramings = (service as any)._resolveFramings({ includeGizmos: true }, 1200, 600); + const reference = (service as any)._topmostFraming(contentFramings); + const overlays = (service as any)._resolveGizmoOverlayFramings( + reference, + { includeGizmos: true }, + 1200, + 600, + ); + + expect(contentFramings.map((framing: any) => framing.info.nodeName)).toEqual([ + 'Main Camera', + 'UI Camera', + ]); + expect(reference.info.nodeName).toBe('UI Camera'); + expect(overlays).toHaveLength(2); + expect(overlays.map((framing: any) => framing.info.nodeName)).toEqual([ + 'Editor UIGizmoCamera', + 'Scene Gizmo Camera', + ]); + expect(overlays.map((framing: any) => framing.runtimeCameraName)).toEqual([ + 'Editor UIGizmoCamera', + 'Scene Gizmo Camera', + ]); + expect(overlays[0].params).toMatchObject({ + position: { x: 640, y: 360, z: 1000 }, + projection: 0, + orthoHeight: 360, + priority: 102, + clearFlags: 0, + visibility: 20, + rect: { x: 0, y: 0, width: 1, height: 1 }, + usePostProcess: false, + }); + expect(overlays[1].params).toMatchObject({ + clearFlags: 6, + visibility: 8, + rect: { width: 1 / 6, height: 1 / 6 }, + }); + }); + + it('rejects an explicit game camera combined with editor gizmos', () => { + const service = new ScreenshotService(); + + expect(() => (service as any)._resolveFramings({ + camera: 'Main Camera', + includeGizmos: true, + })).toThrow('不能与 camera 同时指定'); + }); + + it('omits the 3D scene-axis camera in the current 2D editor view', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const { Quat, Rect, Vec3 } = jest.requireMock('cc'); + const node = { + name: 'Editor Camera', + getWorldPosition: () => new Vec3(0, 0, 10), + getWorldRotation: () => new Quat(), + }; + Service.Gizmo = { + is2D: true, + sceneGizmoCamera: { node: {}, enabledInHierarchy: true }, + }; + const params = (service as any)._paramsFromComponent({ + node, + projection: 0, + priority: 0, + rect: new Rect(0, 0, 1, 1), + }); + const reference = { + info: (service as any)._cameraInfoFromParams(params, 'scene', 'Canvas Camera'), + source: 'scene', + params, + }; + + const framings = (service as any)._resolveGizmoOverlayFramings( + reference, + { includeGizmos: true }, + 800, + 600, + ); + + expect(framings).toHaveLength(1); + expect(framings[0].info.nodeName).toBe('Editor UIGizmoCamera'); + }); + + it('temporarily aligns the editor camera to the content framing and restores it', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const { Quat, Rect, Vec3 } = jest.requireMock('cc'); + const previousState = { + is2D: false, + position: { x: 0, y: 0, z: 20 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + projection: 1, + fov: 45, + fovAxis: 0, + orthoHeight: 10, + near: 0.1, + far: 1000, + }; + const applyScreenshotState = jest.fn(); + Service.Camera = { + getScreenshotState: jest.fn(() => previousState), + applyScreenshotState, + }; + const reference = { + info: { nodeName: 'Canvas Camera' }, + source: 'scene', + params: { + position: new Vec3(640, 360, 1000), + rotation: new Quat(), + projection: 0, + priority: 100, + fov: 60, + fovAxis: 1, + orthoHeight: 360, + near: 0.01, + far: 10000, + clearFlags: 1, + clearDepth: 1, + clearStencil: 0, + visibility: 0xffff, + rect: new Rect(0, 0, 1, 1), + }, + }; + + const restore = (service as any)._alignEditorCameraForGizmoCapture(reference); + + expect(applyScreenshotState).toHaveBeenCalledWith(expect.objectContaining({ + is2D: true, + position: { x: 640, y: 360, z: 1000 }, + projection: 0, + fovAxis: 1, + orthoHeight: 360, + })); + restore(); + expect(applyScreenshotState).toHaveBeenLastCalledWith(previousState); + }); + + it('rebinds and refreshes selected gizmos before capture', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const selectedNode = {}; + Service.Selection = { query: jest.fn(() => ['/Canvas/Button']) }; + Service.Gizmo = { + onSelectionSelect: jest.fn(), + querySelectNodes: jest.fn(() => [selectedNode]), + showAllGizmoOfNode: jest.fn(), + refreshSelectedGizmos: jest.fn(), + onUpdate: jest.fn(), + }; + + (service as any)._refreshEditorGizmosForCapture(); + + expect(Service.Gizmo.onSelectionSelect).toHaveBeenCalledWith('/Canvas/Button'); + expect(Service.Gizmo.showAllGizmoOfNode).toHaveBeenCalledWith(selectedNode); + expect(Service.Gizmo.refreshSelectedGizmos).toHaveBeenCalledTimes(1); + expect(Service.Gizmo.onUpdate).toHaveBeenCalledWith(0); + }); + + it('automatically includes editor gizmos when a node is selected', () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + Service.Selection = { query: jest.fn(() => ['/Canvas/Button']) }; + + expect((service as any)._shouldIncludeEditorGizmos({})).toBe(true); + expect((service as any)._shouldIncludeEditorGizmos({ includeGizmos: false })).toBe(false); + expect((service as any)._shouldIncludeEditorGizmos({ camera: 'Main Camera' })).toBe(false); + }); + + it('restores the browser editor selection in the headless worker before capture', async () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const clear = jest.fn(); + const select = jest.fn(); + const applyScreenshotState = jest.fn(); + Service.Editor = { getCurrentEditorUuid: jest.fn(() => 'scene-uuid') }; + Service.Camera = { applyScreenshotState }; + Service.Selection = { query: jest.fn(() => []), clear, select }; + + const camera = { is2D: true, orthoHeight: 360 }; + await (service as any)._applyBrowserEditorStateForCapture({ + uuid: 'scene-uuid', + selection: ['Canvas/GreenBtn'], + camera, + }); + + expect(applyScreenshotState).toHaveBeenCalledWith(camera); + expect(clear).toHaveBeenCalledTimes(1); + expect(select).toHaveBeenCalledWith('Canvas/GreenBtn'); + }); + + it('applies unsaved browser node transforms before capture', async () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const previousEditorExtends = (globalThis as any).EditorExtends; + const setPosition = jest.fn(); + const setRotation = jest.fn(); + const setScale = jest.fn(); + const updateWorldTransform = jest.fn(); + const button: any = { + uuid: 'button-uuid', + name: 'Button', + layer: 0, + children: [], + position: { x: 0, y: 0, z: 0 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + scale: { x: 1, y: 1, z: 1 }, + setPosition, + setRotation, + setScale, + updateWorldTransform, + }; + const root: any = { + uuid: 'scene-uuid', + name: 'Scene', + layer: 0, + children: [button], + parent: null, + }; + button.parent = root; + (globalThis as any).EditorExtends = { + Node: { getNode: jest.fn(() => button) }, + }; + Service.Editor = { + getCurrentEditorUuid: jest.fn(() => 'scene-uuid'), + getCurrentEditorType: jest.fn(() => 'scene'), + getRootNode: jest.fn(() => root), + }; + Service.Camera = {}; + Service.Selection = { query: jest.fn(() => []) }; + + try { + await (service as any)._applyBrowserEditorStateForCapture({ + uuid: 'scene-uuid', + nodeTransforms: [{ + uuid: 'button-uuid', + path: 'Canvas/Button', + revision: 1, + position: { x: -360, y: 140, z: 0 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + scale: { x: 1.5, y: 1.5, z: 1 }, + }], + }); + + expect(setPosition).toHaveBeenCalledWith(expect.objectContaining({ x: -360, y: 140, z: 0 })); + expect(setRotation).toHaveBeenCalledWith(expect.objectContaining({ x: 0, y: 0, z: 0, w: 1 })); + expect(setScale).toHaveBeenCalledWith(expect.objectContaining({ x: 1.5, y: 1.5, z: 1 })); + expect(updateWorldTransform).toHaveBeenCalledTimes(1); + } finally { + (globalThis as any).EditorExtends = previousEditorExtends; + } + }); + + it('restores unsaved node and component inspector dumps before capture', async () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const dumpUtil = jest.requireMock('../scene-process/service/dump').default; + const previousEditorExtends = (globalThis as any).EditorExtends; + const component = { uuid: 'sprite-uuid' }; + const node: any = { + uuid: 'button-uuid', + name: 'Button', + layer: 0, + children: [], + components: [component], + updateWorldTransform: jest.fn(), + }; + (globalThis as any).EditorExtends = { + Node: { getNode: jest.fn(() => node) }, + }; + Service.Editor = { getCurrentEditorUuid: jest.fn(() => 'scene-uuid') }; + Service.Camera = {}; + Service.Selection = { query: jest.fn(() => []) }; + const componentDump = { + type: 'cc.Sprite', + value: { uuid: { value: 'sprite-uuid' }, color: { value: '#ff0000' } }, + }; + const nodeDump = { + active: { value: true }, + __comps__: [componentDump], + }; + + try { + await (service as any)._applyBrowserEditorStateForCapture({ + uuid: 'scene-uuid', + nodeSnapshots: [{ + uuid: 'button-uuid', + path: 'Canvas/Button', + revision: 2, + dump: nodeDump, + }], + }); + + expect(dumpUtil.restoreNodeSnapshotProperties).toHaveBeenCalledWith(node, nodeDump); + expect(dumpUtil.restoreComponentSnapshotProperties).toHaveBeenCalledWith(component, componentDump); + expect(node.updateWorldTransform).toHaveBeenCalledTimes(1); + } finally { + (globalThis as any).EditorExtends = previousEditorExtends; + } + }); + + it('does not apply a browser selection to a different screenshot scene', async () => { + const service = new ScreenshotService(); + const { Service } = jest.requireMock('../scene-process/service/core'); + const clear = jest.fn(); + const select = jest.fn(); + Service.Editor = { getCurrentEditorUuid: jest.fn(() => 'target-scene-uuid') }; + Service.Selection = { query: jest.fn(() => []), clear, select }; + + await (service as any)._applyBrowserEditorStateForCapture({ + uuid: 'browser-scene-uuid', + selection: ['Canvas/GreenBtn'], + }); + + expect(clear).not.toHaveBeenCalled(); + expect(select).not.toHaveBeenCalled(); + }); + it('tears down a temporary camera when parameter assignment fails', () => { const service = new ScreenshotService(); const tempCamera: any = { enabled: true }; diff --git a/src/core/scene/test/selection-prefab-path.test.ts b/src/core/scene/test/selection-prefab-path.test.ts index a31dad22d..68188f8d9 100644 --- a/src/core/scene/test/selection-prefab-path.test.ts +++ b/src/core/scene/test/selection-prefab-path.test.ts @@ -1,6 +1,7 @@ const mockService = { Editor: { getCurrentEditorType: jest.fn(), + getCurrentEditorUuid: jest.fn(), getRootNode: jest.fn(), }, }; @@ -8,6 +9,13 @@ const mockCc = { director: { getScene: jest.fn(), }, + Layers: { + Enum: { + GIZMOS: 1 << 21, + SCENE_GIZMO: 1 << 24, + EDITOR: 1 << 22, + }, + }, }; jest.mock('cc', () => ({ @@ -20,11 +28,21 @@ jest.mock('../scene-process/service/core/decorator', () => ({ Service: mockService, })); +const mockDumpNode = jest.fn((node: any) => ({ + active: { value: true }, + __comps__: node.components ?? [], +})); +jest.mock('../scene-process/service/dump', () => ({ + __esModule: true, + default: { dumpNode: mockDumpNode }, +})); + describe('SelectionService prefab path resolution', () => { afterEach(() => { jest.resetModules(); jest.clearAllMocks(); mockService.Editor.getCurrentEditorType.mockReturnValue('unknown'); + mockService.Editor.getCurrentEditorUuid.mockReturnValue(null); mockService.Editor.getRootNode.mockReturnValue(null); mockCc.director.getScene.mockReturnValue(null); delete (globalThis as any).EditorExtends; @@ -206,6 +224,118 @@ describe('SelectionService prefab path resolution', () => { expect((selection as any)._selections).toEqual([{ path: 'Other/Node/Child', uuid: '' }]); }); + + it('prefers the node in the current editor when an editor helper has the same runtime uuid', () => { + const selectedNode = { name: 'GreenBtn', uuid: 'Node.520', layer: 1 << 25, children: [], components: [] }; + const canvas = { name: 'Canvas', uuid: 'canvas-uuid', layer: 1 << 25, children: [selectedNode], components: [] }; + const editorGrid = { + name: 'internal/editor/grid-2d', + uuid: 'Node.520', + layer: 1 << 22, + children: [], + components: [], + }; + const root = { + name: 'scene-2d', + uuid: 'scene-uuid', + layer: 0, + children: [canvas, editorGrid], + components: [], + }; + (globalThis as any).EditorExtends = { + Node: { + // Reproduces the engine lookup collision observed in the scene + // process: the generated editor node shadows the scene node. + getNode: jest.fn(() => editorGrid), + }, + }; + mockService.Editor.getCurrentEditorType.mockReturnValue('scene'); + mockService.Editor.getRootNode.mockReturnValue(root); + + const { getEditorNodeByPath, getEditorNodeByUuid } = require('../scene-process/service/gizmo/utils/editor-node'); + + expect(getEditorNodeByUuid('Node.520')).toBe(selectedNode); + expect(getEditorNodeByPath('Canvas/GreenBtn')).toBe(selectedNode); + }); + + it('publishes the latest unsaved node transform for screenshot capture', async () => { + const root: any = { name: 'Scene', uuid: 'scene-uuid', children: [], parent: null }; + const node: any = { + name: 'Button', + uuid: 'button-uuid', + children: [], + parent: root, + position: { x: -360, y: 140, z: 0 }, + rotation: { x: 0, y: 0, z: 0, w: 1 }, + scale: { x: 1, y: 1, z: 1 }, + }; + root.children.push(node); + (globalThis as any).EditorExtends = { + Node: { getNodePath: jest.fn(() => 'Canvas/Button') }, + }; + mockService.Editor.getCurrentEditorUuid.mockReturnValue('scene-uuid'); + mockService.Editor.getRootNode.mockReturnValue(root); + const { Rpc } = require('../scene-process/rpc'); + const request = jest.fn(() => Promise.resolve(true)); + jest.spyOn(Rpc, 'isWebTransport').mockReturnValue(true); + jest.spyOn(Rpc, 'getInstance').mockReturnValue({ request }); + const { SelectionService } = require('../scene-process/service/selection'); + const selection = new SelectionService(); + + (selection as any)._publishBrowserNodeTransform(node); + await Promise.resolve(); + + expect(request).toHaveBeenCalledWith('browserSceneState', 'setEditorState', [ + 'scene-uuid', + { + nodeTransforms: [expect.objectContaining({ + uuid: 'button-uuid', + path: 'Canvas/Button', + revision: 1, + position: { x: -360, y: 140, z: 0 }, + })], + }, + ]); + }); + + it('publishes unsaved node and component inspector values for screenshot capture', async () => { + const root: any = { name: 'Scene', uuid: 'scene-uuid', children: [], parent: null }; + const node: any = { + name: 'Button', + uuid: 'button-uuid', + children: [], + components: [{ type: 'cc.Sprite', value: { color: { value: '#ff0000' } } }], + parent: root, + }; + root.children.push(node); + (globalThis as any).EditorExtends = { + Node: { getNodePath: jest.fn(() => 'Canvas/Button') }, + }; + mockService.Editor.getCurrentEditorUuid.mockReturnValue('scene-uuid'); + mockService.Editor.getRootNode.mockReturnValue(root); + const { Rpc } = require('../scene-process/rpc'); + const request = jest.fn(() => Promise.resolve(true)); + jest.spyOn(Rpc, 'isWebTransport').mockReturnValue(true); + jest.spyOn(Rpc, 'getInstance').mockReturnValue({ request }); + const { SelectionService } = require('../scene-process/service/selection'); + const selection = new SelectionService(); + + (selection as any)._publishBrowserNodeSnapshot(node); + await Promise.resolve(); + + expect(mockDumpNode).toHaveBeenCalledWith(node); + expect(request).toHaveBeenCalledWith('browserSceneState', 'setEditorState', [ + 'scene-uuid', + { + nodeSnapshots: [expect.objectContaining({ + uuid: 'button-uuid', + path: 'Canvas/Button', + revision: 1, + dump: expect.objectContaining({ active: { value: true } }), + })], + }, + ]); + }); }); describe('SelectionService 前导 / 归一化', () => { diff --git a/tests/scene-screenshot-api.test.ts b/tests/scene-screenshot-api.test.ts index b2e497846..4beadf4bb 100644 --- a/tests/scene-screenshot-api.test.ts +++ b/tests/scene-screenshot-api.test.ts @@ -111,6 +111,9 @@ describe('SceneScreenshotApi image transport', () => { attached: true, }); expect(mockUnlink).toHaveBeenCalledWith(shot.filePath); + expect(mockCapture).toHaveBeenCalledWith(expect.objectContaining({ + includeGizmos: undefined, + })); const structured = SchemaScreenshotResult.parse(result.data); expect(structured.image).toEqual({ mimeType: 'image/jpeg', attached: true }); @@ -129,4 +132,16 @@ describe('SceneScreenshotApi image transport', () => { consoleError.mockRestore(); } }); + + it('passes the editor-gizmo option to the scene process', async () => { + await new SceneScreenshotApi().getSceneScreenshot({ + format: 'png', + quality: 80, + includeGizmos: true, + }); + + expect(mockCapture).toHaveBeenCalledWith(expect.objectContaining({ + includeGizmos: true, + })); + }); }); From be2a898a1232e6bf253f016c596099a6c8c3fdc7 Mon Sep 17 00:00:00 2001 From: Decvil <2534450611@qq.com> Date: Tue, 25 Aug 2026 11:47:44 +0800 Subject: [PATCH 3/4] refactor(scene): simplify screenshot response --- .../__snapshots__/dts-snapshot.test.ts.snap | 11 -- src/api/scene/scene.ts | 99 +++++++++++++++- src/api/scene/screenshot-schema.ts | 13 --- src/api/scene/screenshot.ts | 108 ------------------ src/core/scene/common/screenshot.ts | 16 +-- .../scene/scene-process/service/screenshot.ts | 41 ------- .../test/screenshot-camera-params.test.ts | 1 - src/mcp/image-result.ts | 19 --- src/mcp/mcp.middleware.ts | 8 +- tests/mcp-image-result.test.ts | 34 ------ tests/scene-screenshot-api.test.ts | 34 ++---- 11 files changed, 108 insertions(+), 276 deletions(-) delete mode 100644 src/api/scene/screenshot.ts delete mode 100644 src/mcp/image-result.ts delete mode 100644 tests/mcp-image-result.test.ts diff --git a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap index 21fbdf05d..1f7fd01da 100644 --- a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap +++ b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap @@ -6659,12 +6659,6 @@ export declare interface IScreenshotCameraInfo { height: number; }; } -export declare interface IScreenshotNodeSummary { - name: string; - active: boolean; - components: string[]; - children?: IScreenshotNodeSummary[]; -} export declare interface IScreenshotOptions { sceneUrlOrUUID?: string; width?: number; @@ -6675,15 +6669,10 @@ export declare interface IScreenshotOptions { } export declare interface IScreenshotResult { filePath: string; - width: number; - height: number; sceneUrl?: string; sceneName?: string; mtime?: number; - actualCamera: IScreenshotCameraInfo; actualCameras: IScreenshotCameraInfo[]; - nodeSummary?: IScreenshotNodeSummary; - renderNote?: string; } export declare interface IScreenshotService { capture(options: IScreenshotOptions): Promise; diff --git a/src/api/scene/scene.ts b/src/api/scene/scene.ts index 58e63560c..c3a9bf24c 100644 --- a/src/api/scene/scene.ts +++ b/src/api/scene/scene.ts @@ -1,3 +1,5 @@ +import sharp from 'sharp'; +import { readFile, unlink } from 'fs/promises'; import { SchemaOpenOptions, SchemaCloseResult, @@ -16,27 +18,32 @@ import { TReload, TSaveResult, } from './schema'; +import { + SchemaScreenshotOptions, + SchemaScreenshotResult, + TScreenshotOptions, + TScreenshotResult, +} from './screenshot-schema'; import { description, param, result, title, tool } from '../decorator/decorator.js'; import { COMMON_STATUS, CommonResultType, getCommonErrorStatus } from '../base/schema-base'; import { Scene, TSceneTemplateType } from '../../core/scene'; import { ComponentApi } from './component'; import { NodeApi } from './node'; import { PrefabApi } from './prefab'; -import { SceneScreenshotApi } from './screenshot'; -import { options } from '../../core/builder/platforms/android/i18n/en'; + +type TScreenshotResultWithPayload = TScreenshotResult & { + image: TScreenshotResult['image'] & { base64: string }; +}; export class SceneApi { public component: ComponentApi; public node: NodeApi; public prefab: PrefabApi; - public screenshot: SceneScreenshotApi; constructor() { this.component = new ComponentApi(); this.node = new NodeApi(); this.prefab = new PrefabApi(); - // 场景截图工具（实例化以触发 @tool 装饰器注册） - this.screenshot = new SceneScreenshotApi(); } @tool('scene-query-current') @@ -136,7 +143,7 @@ export class SceneApi { targetDirectory: options.dbURL, templateType: options.templateType as TSceneTemplateType, }); - + return { code: COMMON_STATUS.SUCCESS, data: data as TCreateResult, @@ -169,4 +176,84 @@ export class SceneApi { }; } } + + @tool('scene-get-screenshot') + @title('Capture scene screenshot') // 截取当前场景画面 + @description( + 'Headlessly render the currently open scene from project files and return the image (base64, for multimodal input) plus scene metadata ' + + '(scene name, file mtime, actual camera params). ' + + 'Use it to visually reason about layout, spacing, occlusion and spatial relations. ' + + 'The image is scaled/compressed and is a tool-chain rendered preview, not pixel-accurate; confirm exact values with structured data.', + ) // 以 headless 离屏渲染当前场景，返回图片 + 场景元数据，供多模态视觉分析布局/遮挡/间距 + @result(SchemaScreenshotResult) + async getSceneScreenshot( + @param(SchemaScreenshotOptions) options: TScreenshotOptions, + ): Promise> { + let tempFilePath: string | undefined; + try { + // 场景进程在截图 RPC 内安全刷新目标，避免旧实例在重新打开前被保存。 + const shot = await Scene.Screenshot.capture({ + sceneUrlOrUUID: options.sceneUrlOrUUID, + width: options.width, + height: options.height, + camera: options.camera, + viewMode: options.viewMode, + includeGizmos: options.includeGizmos, + }); + tempFilePath = shot.filePath; + + const format = options.format ?? 'jpeg'; + const quality = options.quality ?? 80; + const mimeType = format === 'png' ? 'image/png' : 'image/jpeg'; + + // 默认保持设计分辨率；调用方显式指定 maxSize 时才等比缩小。 + const raw = await readFile(shot.filePath); + let pipeline = sharp(raw); + if (options.maxSize != null) { + pipeline = pipeline.resize({ + width: options.maxSize, + height: options.maxSize, + fit: 'inside', + withoutEnlargement: true, + }); + } + pipeline = format === 'png' + ? pipeline.png({ quality, compressionLevel: 9 }) + : pipeline.jpeg({ quality }); + + const { data: out, info: outputInfo } = await pipeline.toBuffer({ resolveWithObject: true }); + const base64 = out.toString('base64'); + + return { + code: COMMON_STATUS.SUCCESS, + data: { + image: { base64, mimeType }, + meta: { + sceneUrl: shot.sceneUrl, + sceneName: shot.sceneName, + mtime: shot.mtime, + actualCameras: shot.actualCameras, + width: outputInfo.width, + height: outputInfo.height, + }, + }, + }; + } catch (e) { + console.error(e); + return { + code: getCommonErrorStatus(e), + reason: e instanceof Error ? e.message : String(e), + }; + } finally { + if (tempFilePath) { + try { + await unlink(tempFilePath); + } catch (cleanupError: any) { + if (cleanupError?.code !== 'ENOENT') { + console.warn(`[Screenshot] Failed to remove temporary image ${tempFilePath}:`, cleanupError); + } + } + } + } + } } diff --git a/src/api/scene/screenshot-schema.ts b/src/api/scene/screenshot-schema.ts index 417080fee..9fc6d782a 100644 --- a/src/api/scene/screenshot-schema.ts +++ b/src/api/scene/screenshot-schema.ts @@ -54,30 +54,17 @@ const SchemaScreenshotCameraInfo = z.object({ }), }).describe('Actual camera parameters used for the shot'); -const SchemaScreenshotNodeSummary: z.ZodType = z.lazy(() => z.object({ - name: z.string(), - active: z.boolean(), - components: z.array(z.string()), - children: z.array(SchemaScreenshotNodeSummary).optional(), -})); - export const SchemaScreenshotResult = z.object({ image: z.object({ mimeType: z.string().describe('e.g. image/jpeg or image/png'), - attached: z.literal(true).describe('The rendered image is attached as an MCP image content block'), }).describe('Metadata for the rendered image attached to the MCP response'), meta: z.object({ sceneUrl: z.string().optional(), sceneName: z.string().optional(), mtime: z.number().optional().describe('Scene file mtime in ms; use it to judge whether the shot matches the on-disk file'), - actualCamera: SchemaScreenshotCameraInfo, actualCameras: z.array(SchemaScreenshotCameraInfo).describe('All rendered cameras in ascending priority order'), - nodeSummary: SchemaScreenshotNodeSummary.optional(), - renderNote: z.string().optional().describe('Notes about the render, e.g. editor-camera fallback'), width: z.number().describe('Width of the returned image after maxSize processing'), height: z.number().describe('Height of the returned image after maxSize processing'), - renderWidth: z.number().describe('Original offscreen render width before output resizing'), - renderHeight: z.number().describe('Original offscreen render height before output resizing'), }).describe('Scene metadata bound to the screenshot'), }).describe('Scene screenshot result'); diff --git a/src/api/scene/screenshot.ts b/src/api/scene/screenshot.ts deleted file mode 100644 index cd0bfb94f..000000000 --- a/src/api/scene/screenshot.ts +++ /dev/null @@ -1,108 +0,0 @@ -import sharp from 'sharp'; -import { readFile, unlink } from 'fs/promises'; -import { description, param, result, title, tool } from '../decorator/decorator.js'; -import { COMMON_STATUS, CommonResultType, getCommonErrorStatus } from '../base/schema-base'; -import { Scene } from '../../core/scene'; -import { - SchemaScreenshotOptions, - SchemaScreenshotResult, - TScreenshotOptions, - TScreenshotResult, -} from './screenshot-schema'; - -type TScreenshotResultWithPayload = TScreenshotResult & { - image: TScreenshotResult['image'] & { base64: string }; -}; - -/** - * 场景截图 MCP 工具。 - * - * 让具备视觉能力的 Agent 以 headless 离屏渲染的方式「看到」当前场景画面： - * 场景进程离屏渲染 → 临时 PNG → 这里用 sharp 缩放压缩 → base64 内联返回（MCP image 内容块）。 - */ -export class SceneScreenshotApi { - @tool('scene-get-screenshot') - @title('Capture scene screenshot') // 截取当前场景画面 - @description( - 'Headlessly render the currently open scene from project files and return the image (base64, for multimodal input) plus scene metadata ' - + '(scene name, file mtime, actual camera params, node-tree summary). ' - + 'Use it to visually reason about layout, spacing, occlusion and spatial relations. ' - + 'The image is scaled/compressed and is a tool-chain rendered preview, not pixel-accurate; confirm exact values with structured data.', - ) // 以 headless 离屏渲染当前场景，返回图片 + 场景元数据，供多模态视觉分析布局/遮挡/间距 - @result(SchemaScreenshotResult) - async getSceneScreenshot( - @param(SchemaScreenshotOptions) options: TScreenshotOptions, - ): Promise> { - let tempFilePath: string | undefined; - try { - // 场景进程在截图 RPC 内安全刷新目标，避免旧实例在重新打开前被保存。 - const shot = await Scene.Screenshot.capture({ - sceneUrlOrUUID: options.sceneUrlOrUUID, - width: options.width, - height: options.height, - camera: options.camera, - viewMode: options.viewMode, - includeGizmos: options.includeGizmos, - }); - tempFilePath = shot.filePath; - - const format = options.format ?? 'jpeg'; - const quality = options.quality ?? 80; - const mimeType = format === 'png' ? 'image/png' : 'image/jpeg'; - - // 默认保持设计分辨率；调用方显式指定 maxSize 时才等比缩小。 - const raw = await readFile(shot.filePath); - let pipeline = sharp(raw); - if (options.maxSize != null) { - pipeline = pipeline.resize({ - width: options.maxSize, - height: options.maxSize, - fit: 'inside', - withoutEnlargement: true, - }); - } - pipeline = format === 'png' - ? pipeline.png({ quality, compressionLevel: 9 }) - : pipeline.jpeg({ quality }); - - const { data: out, info: outputInfo } = await pipeline.toBuffer({ resolveWithObject: true }); - const base64 = out.toString('base64'); - - return { - code: COMMON_STATUS.SUCCESS, - data: { - image: { base64, mimeType, attached: true }, - meta: { - sceneUrl: shot.sceneUrl, - sceneName: shot.sceneName, - mtime: shot.mtime, - actualCamera: shot.actualCamera, - actualCameras: shot.actualCameras, - nodeSummary: shot.nodeSummary, - renderNote: shot.renderNote, - width: outputInfo.width, - height: outputInfo.height, - renderWidth: shot.width, - renderHeight: shot.height, - }, - }, - }; - } catch (e) { - console.error(e); - return { - code: getCommonErrorStatus(e), - reason: e instanceof Error ? e.message : String(e), - }; - } finally { - if (tempFilePath) { - try { - await unlink(tempFilePath); - } catch (cleanupError: any) { - if (cleanupError?.code !== 'ENOENT') { - console.warn(`[Screenshot] Failed to remove temporary image ${tempFilePath}:`, cleanupError); - } - } - } - } - } -} diff --git a/src/core/scene/common/screenshot.ts b/src/core/scene/common/screenshot.ts index 365c36fd4..1781ff891 100644 --- a/src/core/scene/common/screenshot.ts +++ b/src/core/scene/common/screenshot.ts @@ -2,7 +2,7 @@ * 场景截图服务的公共类型定义。 * * 截图由场景进程以 headless 方式离屏渲染当前打开的场景生成， - * 返回临时 PNG 文件路径 + 元数据（不跨进程传裸像素，见 screenshot.ts 说明）。 + * 返回临时 PNG 文件路径 + 元数据（不跨进程传裸像素，见 API scene.ts 说明）。 */ /** 取景 / 视图模式 */ @@ -51,32 +51,18 @@ export interface IScreenshotCameraInfo { viewport: { x: number; y: number; width: number; height: number }; } -/** 节点树概要中的单个节点 */ -export interface IScreenshotNodeSummary { - name: string; - active: boolean; - components: string[]; - children?: IScreenshotNodeSummary[]; -} - /** 场景进程返回的原始截图结果（JSON-safe） */ export interface IScreenshotResult { /** 离屏渲染写出的临时 PNG 绝对路径 */ filePath: string; - width: number; - height: number; /** 当前场景在 asset-db 中的 url */ sceneUrl?: string; /** 场景文件名 */ sceneName?: string; /** 场景磁盘文件修改时间（ms），供 Agent 判断时效 */ mtime?: number; - actualCamera: IScreenshotCameraInfo; /** All cameras rendered into the offscreen target, ordered by priority. */ actualCameras: IScreenshotCameraInfo[]; - nodeSummary?: IScreenshotNodeSummary; - /** 渲染备注（如无场景相机、退回编辑器相机等提示） */ - renderNote?: string; } export interface IScreenshotEvents {} diff --git a/src/core/scene/scene-process/service/screenshot.ts b/src/core/scene/scene-process/service/screenshot.ts index c795014e2..524e35a2d 100644 --- a/src/core/scene/scene-process/service/screenshot.ts +++ b/src/core/scene/scene-process/service/screenshot.ts @@ -15,7 +15,6 @@ import type { import type { IScreenshotCameraInfo, IScreenshotEvents, - IScreenshotNodeSummary, IScreenshotOptions, IScreenshotResult, IScreenshotService, @@ -23,8 +22,6 @@ import type { const DEFAULT_SIZE = 1024; const MAX_SIZE = 4096; -const NODE_SUMMARY_MAX_DEPTH = 4; -const NODE_SUMMARY_MAX_CHILDREN = 40; /** 临时截图相机在场景树里的节点名，销毁后不残留 */ const TEMP_CAMERA_NODE_NAME = '__cli_screenshot_camera__'; @@ -62,7 +59,6 @@ interface ICameraParams { interface IResolvedFraming { info: IScreenshotCameraInfo; source: 'scene' | 'editor'; - renderNote?: string; params: ICameraParams; /** Engine-recognized camera name used by editor render-pipeline ordering. */ runtimeCameraName?: string; @@ -342,20 +338,13 @@ export class ScreenshotService extends BaseService implements const filePath = this._writePng(result.width, result.height, result.buffer); const meta = await this._collectSceneMeta(); const cameraInfos = framings.map(framing => framing.info); - const renderNotes = [...new Set(framings.map(framing => framing.renderNote).filter(Boolean))]; return { filePath, - width: result.width, - height: result.height, sceneUrl: meta.url, sceneName: meta.name, mtime: meta.mtime, - // Keep the historical singular field as the top-most/only camera. - actualCamera: cameraInfos[cameraInfos.length - 1], actualCameras: cameraInfos, - nodeSummary: this._summarizeNode(scene, 0), - renderNote: renderNotes.length ? renderNotes.join(' ') : undefined, }; } @@ -502,7 +491,6 @@ export class ScreenshotService extends BaseService implements info: this._cameraInfoFromParams(params, 'editor', editorCam.node.name), source: 'editor', params, - renderNote: '场景内无相机，已使用编辑器相机当前视角取景（不含 Gizmo/网格）。', }]; } @@ -526,7 +514,6 @@ export class ScreenshotService extends BaseService implements source: 'editor', params: gizmoParams, runtimeCameraName: EDITOR_GIZMO_CAMERA_NAME, - renderNote: '已保持场景相机取景，并叠加选中节点/组件 Gizmo。', }]; const gizmoService = Service.Gizmo as any; @@ -549,7 +536,6 @@ export class ScreenshotService extends BaseService implements source: 'editor', params: sceneGizmoParams, runtimeCameraName: SCENE_GIZMO_CAMERA_NAME, - renderNote: '已按截图尺寸叠加右上角 3D 场景坐标轴。', }); } @@ -696,7 +682,6 @@ export class ScreenshotService extends BaseService implements private _resolvePrefab2DFraming(options: IScreenshotOptions): { info: IScreenshotCameraInfo; source: 'editor'; - renderNote: string; params: ICameraParams; canvas: Canvas; } | null { @@ -763,7 +748,6 @@ export class ScreenshotService extends BaseService implements source: 'editor', params, canvas, - renderNote: 'UI Prefab 已按预览 Canvas 边界自动取景（不含 Gizmo/网格）。', }; } @@ -803,8 +787,6 @@ export class ScreenshotService extends BaseService implements framing.info.nodeName, ); - const note = 'Canvas 相机已按工程设计分辨率重新取景。'; - framing.renderNote = framing.renderNote ? `${framing.renderNote} ${note}` : note; } private _findAlignedCanvasForCamera(camera: Camera): Canvas | null { @@ -1055,29 +1037,6 @@ export class ScreenshotService extends BaseService implements } } - private _summarizeNode(node: CCNode, depth: number): IScreenshotNodeSummary { - const components = (node.components ?? []) - .map((c) => cc.js.getClassName(c)) - .filter(Boolean); - const summary: IScreenshotNodeSummary = { - name: node.name, - active: node.active, - components, - }; - if (depth < NODE_SUMMARY_MAX_DEPTH && node.children?.length) { - const kids = node.children.slice(0, NODE_SUMMARY_MAX_CHILDREN); - summary.children = kids.map((child) => this._summarizeNode(child, depth + 1)); - if (node.children.length > NODE_SUMMARY_MAX_CHILDREN) { - summary.children.push({ - name: `…(+${node.children.length - NODE_SUMMARY_MAX_CHILDREN} more)`, - active: true, - components: [], - }); - } - } - return summary; - } - private _clampSize(value: number | undefined, fallback: number): number { if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { return Math.min(MAX_SIZE, Math.max(1, Math.floor(fallback))); diff --git a/src/core/scene/test/screenshot-camera-params.test.ts b/src/core/scene/test/screenshot-camera-params.test.ts index 5889e199f..4a5a27e51 100644 --- a/src/core/scene/test/screenshot-camera-params.test.ts +++ b/src/core/scene/test/screenshot-camera-params.test.ts @@ -241,7 +241,6 @@ describe('Screenshot camera parameter normalization', () => { projection: 'ortho', orthoHeight: 360, }); - expect(framing.renderNote).toContain('工程设计分辨率'); } finally { (globalThis as any).cc = previousCc; } diff --git a/src/mcp/image-result.ts b/src/mcp/image-result.ts deleted file mode 100644 index 0bc6319db..000000000 --- a/src/mcp/image-result.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Remove binary image data from JSON/text channels after the full tool result - * has passed schema validation. The original result remains untouched so its - * base64 payload can still be emitted as an MCP image content block. - */ -export function redactInlineImageData(result: any): any { - const image = result?.data?.image; - if (!image || typeof image.base64 !== 'string') { - return result; - } - const { base64: _base64, ...imageMetadata } = image; - return { - ...result, - data: { - ...result.data, - image: imageMetadata, - }, - }; -} diff --git a/src/mcp/mcp.middleware.ts b/src/mcp/mcp.middleware.ts index edb54d7f4..ecb773a62 100644 --- a/src/mcp/mcp.middleware.ts +++ b/src/mcp/mcp.middleware.ts @@ -14,7 +14,6 @@ import stripAnsi from 'strip-ansi'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; import { assetManager } from '../core/assets'; -import { redactInlineImageData } from './image-result'; export function isToolErrorCode(code: unknown): boolean { return typeof code === 'number' && code >= 500 && code < 600; @@ -159,11 +158,8 @@ export class McpMiddleware { // structured output schema and is emitted only as image content. const image = result?.data?.image; const validatedResult = this.validateToolResult(meta, result); - const responseResult = image?.base64 && typeof image.base64 === 'string' - ? redactInlineImageData(validatedResult) - : validatedResult; - const formattedResult = this.formatToolResult(responseResult); - const structuredContent = { result: responseResult }; + const formattedResult = this.formatToolResult(validatedResult); + const structuredContent = { result: validatedResult }; console.debug(`call ${toolName} with args:${methodArgs.toString()} result: ${formattedResult}`); diff --git a/tests/mcp-image-result.test.ts b/tests/mcp-image-result.test.ts deleted file mode 100644 index 8025291ad..000000000 --- a/tests/mcp-image-result.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { redactInlineImageData } from '../src/mcp/image-result'; - -describe('MCP inline image result', () => { - it('keeps base64 only in the original image payload', () => { - const result = { - code: 200, - data: { - image: { - base64: 'large-image-payload', - mimeType: 'image/jpeg', - attached: true, - }, - meta: { width: 1280, height: 720 }, - }, - }; - - const redacted = redactInlineImageData(result); - - expect(redacted).toEqual({ - code: 200, - data: { - image: { mimeType: 'image/jpeg', attached: true }, - meta: { width: 1280, height: 720 }, - }, - }); - expect(result.data.image.base64).toBe('large-image-payload'); - expect(JSON.stringify(redacted)).not.toContain('large-image-payload'); - }); - - it('does not clone ordinary tool results', () => { - const result = { code: 200, data: { value: 1 } }; - expect(redactInlineImageData(result)).toBe(result); - }); -}); diff --git a/tests/scene-screenshot-api.test.ts b/tests/scene-screenshot-api.test.ts index 4beadf4bb..7acf7183e 100644 --- a/tests/scene-screenshot-api.test.ts +++ b/tests/scene-screenshot-api.test.ts @@ -26,6 +26,8 @@ jest.mock('../src/api/decorator/decorator.js', () => ({ }), { virtual: true }); jest.mock('../src/core/scene', () => ({ + NodeType: { EMPTY: 'Empty' }, + SCENE_TEMPLATE_TYPE: ['empty'], Scene: { Screenshot: { capture: (...args: unknown[]) => mockCapture(...args), @@ -43,26 +45,17 @@ jest.mock('sharp', () => ({ default: jest.fn(() => mockPipeline), })); -import { SceneScreenshotApi } from '../src/api/scene/screenshot'; +jest.mock('../src/api/scene/component', () => ({ ComponentApi: jest.fn() })); +jest.mock('../src/api/scene/node', () => ({ NodeApi: jest.fn() })); +jest.mock('../src/api/scene/prefab', () => ({ PrefabApi: jest.fn() })); + +import { SceneApi } from '../src/api/scene/scene'; import { SchemaScreenshotResult } from '../src/api/scene/screenshot-schema'; const shot = { filePath: 'C:\\Temp\\cocos-cli-screenshot-test.png', - width: 1200, - height: 600, sceneUrl: 'db://assets/main.scene', sceneName: 'main.scene', - actualCamera: { - source: 'scene' as const, - projection: 'perspective' as const, - position: { x: 0, y: 0, z: 10 }, - rotation: { x: 0, y: 0, z: 0, w: 1 }, - fov: 45, - priority: 0, - clearFlags: 1, - visibility: 0xffffffff, - viewport: { x: 0, y: 0, width: 1, height: 1 }, - }, actualCameras: [{ source: 'scene' as const, projection: 'perspective' as const, @@ -76,7 +69,7 @@ const shot = { }], }; -describe('SceneScreenshotApi image transport', () => { +describe('SceneApi screenshot image transport', () => { beforeEach(() => { jest.clearAllMocks(); mockResize.mockReturnValue(mockPipeline); @@ -92,7 +85,7 @@ describe('SceneScreenshotApi image transport', () => { }); it('returns output dimensions and removes the temporary PNG', async () => { - const result = await new SceneScreenshotApi().getSceneScreenshot({ + const result = await new SceneApi().getSceneScreenshot({ format: 'jpeg', quality: 80, maxSize: 400, @@ -102,13 +95,10 @@ describe('SceneScreenshotApi image transport', () => { expect(result.data?.meta).toMatchObject({ width: 400, height: 200, - renderWidth: 1200, - renderHeight: 600, }); expect(result.data?.image).toMatchObject({ base64: Buffer.from('encoded-image').toString('base64'), mimeType: 'image/jpeg', - attached: true, }); expect(mockUnlink).toHaveBeenCalledWith(shot.filePath); expect(mockCapture).toHaveBeenCalledWith(expect.objectContaining({ @@ -116,7 +106,7 @@ describe('SceneScreenshotApi image transport', () => { })); const structured = SchemaScreenshotResult.parse(result.data); - expect(structured.image).toEqual({ mimeType: 'image/jpeg', attached: true }); + expect(structured.image).toEqual({ mimeType: 'image/jpeg' }); expect(structured).not.toHaveProperty('meta.filePath'); }); @@ -125,7 +115,7 @@ describe('SceneScreenshotApi image transport', () => { mockToBuffer.mockRejectedValue(new Error('encode failed')); try { - const result = await new SceneScreenshotApi().getSceneScreenshot({ format: 'jpeg', quality: 80 }); + const result = await new SceneApi().getSceneScreenshot({ format: 'jpeg', quality: 80 }); expect(result.code).toBeGreaterThanOrEqual(500); expect(mockUnlink).toHaveBeenCalledWith(shot.filePath); } finally { @@ -134,7 +124,7 @@ describe('SceneScreenshotApi image transport', () => { }); it('passes the editor-gizmo option to the scene process', async () => { - await new SceneScreenshotApi().getSceneScreenshot({ + await new SceneApi().getSceneScreenshot({ format: 'png', quality: 80, includeGizmos: true, From a000029d21f9dd58320f2ff4293b0435f721083a Mon Sep 17 00:00:00 2001 From: Decvil <2534450611@qq.com> Date: Wed, 26 Aug 2026 15:26:29 +0800 Subject: [PATCH 4/4] fix(scene): harden screenshot state and result handling - add typed camera snapshot and restore APIs - avoid orphaned temporary PNGs when metadata collection fails - keep base64 image data out of structured MCP responses and logs - update public type snapshots --- .../__snapshots__/dts-snapshot.test.ts.snap | 23 +++++++++++++ src/api/scene/screenshot-schema.ts | 5 +++ src/core/scene/common/camera.ts | 19 +++++++++++ .../scene/scene-process/service/camera.ts | 6 ++-- .../scene/scene-process/service/editor.ts | 2 +- .../scene/scene-process/service/screenshot.ts | 9 ++++-- .../scene/scene-process/service/selection.ts | 2 +- src/mcp/mcp.middleware.ts | 32 +++++++++---------- 8 files changed, 74 insertions(+), 24 deletions(-) diff --git a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap index 1f7fd01da..bfb2099e6 100644 --- a/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap +++ b/packages/cocos-cli-types/__tests__/__snapshots__/dts-snapshot.test.ts.snap @@ -6108,6 +6108,26 @@ export declare interface ICameraConfig { near2D?: number; wheelSpeed2D?: number; } +export declare interface ICameraScreenshotState { + is2D: boolean; + position: { + x: number; + y: number; + z: number; + }; + rotation: { + x: number; + y: number; + z: number; + w: number; + }; + projection: number; + fov: number; + fovAxis: number; + orthoHeight: number; + near: number; + far: number; +} export declare interface ICameraService { init(): void; initFromConfig(): Promise; @@ -6132,6 +6152,9 @@ export declare interface ICameraService { setOriginAxes2D(config: IOriginAxesConfig): void; setOriginAxes3D(config: IOriginAxesConfig): void; onUpdate(deltaTime: number): void; + waitForRestore(): Promise; + getScreenshotState(): ICameraScreenshotState | undefined; + applyScreenshotState(state: ICameraScreenshotState): void; } export declare interface IChangeNodeLockParams { paths: string[]; diff --git a/src/api/scene/screenshot-schema.ts b/src/api/scene/screenshot-schema.ts index 9fc6d782a..15a8aa5e1 100644 --- a/src/api/scene/screenshot-schema.ts +++ b/src/api/scene/screenshot-schema.ts @@ -56,6 +56,11 @@ const SchemaScreenshotCameraInfo = z.object({ export const SchemaScreenshotResult = z.object({ image: z.object({ + // NOTE: the raw base64 payload is deliberately NOT part of this schema. + // It is carried on the tool result's `data.image.base64` field and emitted + // only as an MCP image content block (see mcp.middleware.ts), so it stays + // out of structuredContent and logs. Do not add `.strict()` here or the + // extra base64 key would make schema validation throw. mimeType: z.string().describe('e.g. image/jpeg or image/png'), }).describe('Metadata for the rendered image attached to the MCP response'), meta: z.object({ diff --git a/src/core/scene/common/camera.ts b/src/core/scene/common/camera.ts index 9bd450efd..161be6f71 100644 --- a/src/core/scene/common/camera.ts +++ b/src/core/scene/common/camera.ts @@ -3,6 +3,19 @@ import type { ICameraConfig, IOriginAxesConfig } from '../scene-configs'; export type { ICameraConfig, IOriginAxesConfig }; +/** Editor-camera view snapshot used to align/restore the camera around a screenshot. */ +export interface ICameraScreenshotState { + is2D: boolean; + position: { x: number; y: number; z: number }; + rotation: { x: number; y: number; z: number; w: number }; + projection: number; + fov: number; + fovAxis: number; + orthoHeight: number; + near: number; + far: number; +} + export interface ICameraService { init(): void; initFromConfig(): Promise; @@ -27,6 +40,12 @@ export interface ICameraService { setOriginAxes2D(config: IOriginAxesConfig): void; setOriginAxes3D(config: IOriginAxesConfig): void; onUpdate(deltaTime: number): void; + /** Wait until the async camera-view restore triggered by opening an editor has settled. */ + waitForRestore(): Promise; + /** Snapshot the current editor-camera view, or undefined when no camera is available. */ + getScreenshotState(): ICameraScreenshotState | undefined; + /** Apply a previously captured editor-camera view (used to align/restore around a screenshot). */ + applyScreenshotState(state: ICameraScreenshotState): void; } export type IPublicCameraService = Pick implements ICamera await this._restorePromise; } - getScreenshotState(): any { + getScreenshotState(): ICameraScreenshotState | undefined { const camera = this._camera; if (!camera?.node) { return undefined; @@ -510,7 +510,7 @@ export class CameraService extends BaseService implements ICamera }; } - applyScreenshotState(state: any): void { + applyScreenshotState(state: ICameraScreenshotState): void { if (!state || !this._camera?.node) { return; } diff --git a/src/core/scene/scene-process/service/editor.ts b/src/core/scene/scene-process/service/editor.ts index 5f0bd5500..e6247236b 100644 --- a/src/core/scene/scene-process/service/editor.ts +++ b/src/core/scene/scene-process/service/editor.ts @@ -589,7 +589,7 @@ export class EditorService extends BaseService implements IEditor type: assetInfo.type, name: assetInfo.name, selection: Service.Selection?.query?.() ?? [], - camera: (Service.Camera as any)?.getScreenshotState?.(), + camera: Service.Camera?.getScreenshotState?.(), }]); } catch (error) { console.warn('[Scene] Failed to publish PinK current scene.', error); diff --git a/src/core/scene/scene-process/service/screenshot.ts b/src/core/scene/scene-process/service/screenshot.ts index 524e35a2d..324cb70de 100644 --- a/src/core/scene/scene-process/service/screenshot.ts +++ b/src/core/scene/scene-process/service/screenshot.ts @@ -18,6 +18,7 @@ import type { IScreenshotOptions, IScreenshotResult, IScreenshotService, + ICameraScreenshotState, } from '../../common'; const DEFAULT_SIZE = 1024; @@ -121,7 +122,7 @@ export class ScreenshotService extends BaseService implements return Service.Editor.withScreenshotScene( options.sceneUrlOrUUID, async () => { - await (Service.Camera as any)?.waitForRestore?.(); + await Service.Camera?.waitForRestore?.(); await this._applyBrowserEditorStateForCapture(browserState); return this._capturePreparedScene(options); }, @@ -151,7 +152,7 @@ export class ScreenshotService extends BaseService implements return; } if (browserState.camera) { - (Service.Camera as any)?.applyScreenshotState?.(browserState.camera); + Service.Camera?.applyScreenshotState?.(browserState.camera as ICameraScreenshotState); } const snapshotRevisions = await this._applyBrowserNodeSnapshots(browserState.nodeSnapshots); this._applyBrowserNodeTransforms(browserState.nodeTransforms, snapshotRevisions); @@ -335,9 +336,11 @@ export class ScreenshotService extends BaseService implements } } - const filePath = this._writePng(result.width, result.height, result.buffer); + // Collect everything that can still fail before writing the file, so a + // late error cannot orphan a temp PNG the API layer never learns about. const meta = await this._collectSceneMeta(); const cameraInfos = framings.map(framing => framing.info); + const filePath = this._writePng(result.width, result.height, result.buffer); return { filePath, diff --git a/src/core/scene/scene-process/service/selection.ts b/src/core/scene/scene-process/service/selection.ts index e6772e5db..dd6ef5bd5 100644 --- a/src/core/scene/scene-process/service/selection.ts +++ b/src/core/scene/scene-process/service/selection.ts @@ -137,7 +137,7 @@ export class SelectionService extends BaseService implements I void Rpc.getInstance() .request('browserSceneState', 'setEditorState', [editorUuid, { selection: this._getPaths(), - camera: (Service.Camera as any)?.getScreenshotState?.(), + camera: Service.Camera?.getScreenshotState?.(), }]) .catch((error: unknown) => { console.warn('[Selection] Failed to publish PinK selection.', error); diff --git a/src/mcp/mcp.middleware.ts b/src/mcp/mcp.middleware.ts index ecb773a62..c35809bc4 100644 --- a/src/mcp/mcp.middleware.ts +++ b/src/mcp/mcp.middleware.ts @@ -161,25 +161,25 @@ export class McpMiddleware { const formattedResult = this.formatToolResult(validatedResult); const structuredContent = { result: validatedResult }; - console.debug(`call ${toolName} with args:${methodArgs.toString()} result: ${formattedResult}`); - - // 图片二进制只放进 MCP image 内容块。文本、structuredContent 和日志 - // 仅保留图片元数据，避免同一份 base64 被重复传输和写入日志。 - const content: any[] = [ - { type: 'text' as const, text: formattedResult }, - ]; - if (image?.base64 && typeof image.base64 === 'string') { - content.push({ - type: 'image' as const, - data: image.base64, - mimeType: image.mimeType || 'image/png', - }); - } - return { + console.debug(`call ${toolName} with args:${methodArgs.toString()} result: ${formattedResult}`); + + // 图片二进制只放进 MCP image 内容块。文本、structuredContent 和日志 + // 仅保留图片元数据，避免同一份 base64 被重复传输和写入日志。 + const content: any[] = [ + { type: 'text' as const, text: formattedResult }, + ]; + if (image?.base64 && typeof image.base64 === 'string') { + content.push({ + type: 'image' as const, + data: image.base64, + mimeType: image.mimeType || 'image/png', + }); + } + return { content, structuredContent: structuredContent, isError: isToolErrorCode(validatedResult?.code) - }; + }; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error);