From 3c6fd887d9614468dec13300167e8d738e74b910 Mon Sep 17 00:00:00 2001 From: looopmax Date: Tue, 18 Aug 2026 14:50:22 +0800 Subject: [PATCH 1/2] feat(runtime-cache): isolate compiled cache by tool versions --- src/lib/runtime-module-cache.ts | 961 ++++++++++++++++++++++++++++++++ tests/runtime-bundle.test.ts | 523 +++++++++++++++++ 2 files changed, 1484 insertions(+) create mode 100644 src/lib/runtime-module-cache.ts create mode 100644 tests/runtime-bundle.test.ts diff --git a/src/lib/runtime-module-cache.ts b/src/lib/runtime-module-cache.ts new file mode 100644 index 000000000..0ef3398d8 --- /dev/null +++ b/src/lib/runtime-module-cache.ts @@ -0,0 +1,961 @@ +import * as fs from 'fs'; +import * as vm from 'vm'; +import * as os from 'os'; +import { createRequire } from 'module'; +import { basename, dirname, extname, join, resolve } from 'path'; + +const HEADER_SIZE = 16; +const CACHE_VERSION = 2; +const CACHE_MAGIC = Buffer.from('RTBCACH1'); +const CACHE_ENABLED_ENV = 'COCOS_RUNTIME_BUNDLE_CACHE'; +const CACHE_SCOPE_ENV = 'COCOS_RUNTIME_BUNDLE_CACHE_SCOPE'; +const CACHE_FILE_ENV = 'COCOS_RUNTIME_BUNDLE_CACHE_FILE'; +const PINK_VERSION_ENV = 'VSCODE_PINK_VERSION'; +const COCOS_CLI_VERSION_ENV = 'COCOS_CLI_VERSION'; +const V8_CACHED_DATA_VERSION = process.versions.v8; +const COMMONJS_PARAMETERS = ['exports', 'require', 'module', '__filename', '__dirname']; +const GLOBAL_INSTALL_KEY = '__cocos_runtime_module_cache_installed__'; +const globalState = globalThis as typeof globalThis & { + [GLOBAL_INSTALL_KEY]?: boolean; +}; + +export interface RuntimeBundleCacheEntry { + offset: number; + length: number; + codeCacheOffset: number; + codeCacheLength: number; + mtimeMs: number; +} + +export interface RuntimeBundleResolutionEntry { + filename: string; + mtimeMs: number; +} + +export interface RuntimeBundlerOptions { + cachePath?: string; + isRuntimeBundle?: (filePath: string) => boolean; + enabled?: boolean; + moveToTrash?: (filePath: string) => Promise; +} + +interface RuntimeModule { + filename?: string; + paths?: string[]; + exports: unknown; + loaded: boolean; + require(request: string): unknown; +} + +type ModuleLoad = (this: RuntimeModule, filename: string) => void; +type ResolveFilename = ( + request: string, + parent?: RuntimeModule, + isMain?: boolean, + options?: unknown, +) => string; + +const nodeRequire = createRequire(resolve(process.cwd(), 'package.json')); +const nodeModule = nodeRequire('module') as { + prototype: { + load: ModuleLoad; + }; + _resolveFilename: ResolveFilename; + _nodeModulePaths(path: string): string[]; + builtinModules?: string[]; +}; +const builtinModules = new Set(nodeModule.builtinModules ?? []); +const packageTypeCache = new Map(); + +interface PendingModule { + source: Buffer; + codeCache: Buffer; + mtimeMs: number; +} + +interface CacheManifest { + v8CachedDataVersion: string; + modules: Record; + resolutions: Record; +} + +interface CompiledModule { + cachedDataRejected?: boolean; + cachedData?: Buffer; + (...args: unknown[]): unknown; +} + +/** + * Caches runtime-bundle source and V8 CommonJS compiled data at the module + * resolution/load boundaries. + * + * The cache file is laid out as: + * fixed header | JSON manifest | source/code-cache byte ranges + * + * A cache miss reads and compiles the source once, then records the source and + * V8 cachedData asynchronously. A cache hit skips the source-file read and + * executes a wrapper compiled from the cachedData. Cache writes are serialized + * and use a stream. + */ +export class RuntimeBundler { + private readonly cachePath: string; + private readonly isRuntimeBundle: (filePath: string) => boolean; + private readonly enabled: boolean; + private readonly cleanupStaleCaches: boolean; + private readonly moveToTrash: (filePath: string) => Promise; + + private cacheBytes: Buffer | undefined; + private cacheContentOffset = 0; + private cacheModules = new Map(); + private cacheResolutions = new Map(); + private pendingModules = new Map(); + private pendingResolutions = new Map(); + private freshnessChecks = new Map>(); + private resolutionChecks = new Map>(); + private flushPromise: Promise | undefined; + private flushScheduled = false; + private installed = false; + private originalModuleLoad: ModuleLoad | undefined; + private hookedModuleLoad: ModuleLoad | undefined; + private originalResolveFilename: ResolveFilename | undefined; + private hookedResolveFilename: ResolveFilename | undefined; + + constructor(options: RuntimeBundlerOptions = {}) { + this.cleanupStaleCaches = options.cachePath === undefined && !process.env[CACHE_FILE_ENV]?.trim(); + this.cachePath = options.cachePath ?? getDefaultCachePath(); + this.enabled = options.enabled ?? isCacheEnabled(process.env[CACHE_ENABLED_ENV]); + this.isRuntimeBundle = options.isRuntimeBundle ?? isCacheableModulePath; + this.moveToTrash = options.moveToTrash ?? moveCacheFileToTrash; + } + + install(): void { + if (!this.enabled || this.installed) { + return; + } + + this.loadCacheSync(); + if (this.cleanupStaleCaches) { + scheduleStaleCacheCleanup(this.cachePath, this.moveToTrash); + } + this.originalModuleLoad = nodeModule.prototype.load; + this.originalResolveFilename = nodeModule._resolveFilename; + const originalModuleLoad = this.originalModuleLoad; + const originalResolveFilename = this.originalResolveFilename; + const bundler = this; + this.hookedModuleLoad = function hookedModuleLoad(this: RuntimeModule, filename: string): void { + bundler.loadModule(this, filename, originalModuleLoad); + }; + this.hookedResolveFilename = function hookedResolveFilename( + request: string, + parent?: RuntimeModule, + isMain?: boolean, + options?: unknown, + ): string { + return bundler.resolveFilename(request, parent, isMain, options, originalResolveFilename); + }; + nodeModule._resolveFilename = this.hookedResolveFilename; + nodeModule.prototype.load = this.hookedModuleLoad; + this.installed = true; + } + + uninstall(): void { + if (!this.installed) { + return; + } + if (this.hookedModuleLoad && nodeModule.prototype.load === this.hookedModuleLoad && this.originalModuleLoad) { + nodeModule.prototype.load = this.originalModuleLoad; + } + if (this.hookedResolveFilename && nodeModule._resolveFilename === this.hookedResolveFilename && this.originalResolveFilename) { + nodeModule._resolveFilename = this.originalResolveFilename; + } + this.hookedModuleLoad = undefined; + this.originalModuleLoad = undefined; + this.hookedResolveFilename = undefined; + this.originalResolveFilename = undefined; + this.installed = false; + } + + async flush(): Promise { + const checks = [...this.freshnessChecks.values(), ...this.resolutionChecks.values()]; + if (checks.length > 0) { + await Promise.all(checks); + } + if (this.flushPromise) { + await this.flushPromise; + if (this.hasPending()) { + await this.flush(); + } + return; + } + if (!this.hasPending()) { + return; + } + + const pendingModules = this.pendingModules; + const pendingResolutions = this.pendingResolutions; + this.pendingModules = new Map(); + this.pendingResolutions = new Map(); + this.flushPromise = this.writeCache(pendingModules, pendingResolutions) + .catch((error) => { + this.restorePending(pendingModules, pendingResolutions); + throw error; + }) + .finally(() => { + this.flushPromise = undefined; + }); + await this.flushPromise; + if (this.hasPending()) { + await this.flush(); + } + } + + private hasPending(): boolean { + return this.pendingModules.size > 0 || this.pendingResolutions.size > 0; + } + + private resolveFilename( + request: string, + parent: RuntimeModule | undefined, + isMain: boolean | undefined, + options: unknown, + originalResolveFilename: ResolveFilename, + ): string { + const key = getResolutionKey(request, parent, isMain, options); + if (key) { + const cached = this.pendingResolutions.get(key) ?? this.cacheResolutions.get(key); + if (cached) { + this.scheduleResolutionFreshnessCheck(key, cached); + return cached.filename; + } + } + + const filename = originalResolveFilename.call(nodeModule, request, parent, isMain, options); + if (key && typeof filename === 'string' && isCacheableResolvedPath(filename)) { + this.scheduleResolutionRecord(key, filename); + } + return filename; + } + + private loadModule(module: RuntimeModule, filename: string, originalModuleLoad: ModuleLoad): void { + const filePath = resolve(filename); + if (!this.isRuntimeBundle(filePath)) { + originalModuleLoad.call(module, filename); + return; + } + + const pending = this.pendingModules.get(filePath); + if (pending) { + this.scheduleFreshnessCheck(filePath, pending.mtimeMs); + this.executeCompiledModule(module, filename, pending.source, pending.codeCache); + return; + } + + const cached = this.cacheModules.get(filePath); + if (cached) { + const content = this.getCachedModule(cached); + if (content) { + this.scheduleFreshnessCheck(filePath, cached.mtimeMs); + const codeCache = this.executeCompiledModule(module, filename, content.source, content.codeCache); + if (codeCache !== undefined) { + this.recordAsync(filePath, content.source, codeCache, cached.mtimeMs); + } + return; + } + } + + // Cache miss: keep the synchronous require contract and read the + // source once inside the module-loading path. + let mtimeMs: number; + let content: Buffer; + try { + mtimeMs = fs.statSync(filePath).mtimeMs; + content = fs.readFileSync(filePath); + } catch { + originalModuleLoad.call(module, filename); + return; + } + const codeCache = this.executeCompiledModule(module, filename, content); + this.recordAsync(filePath, content, codeCache ?? Buffer.alloc(0), mtimeMs); + } + + private scheduleFreshnessCheck(filePath: string, cachedMtimeMs: number): void { + if (this.freshnessChecks.has(filePath)) { + return; + } + const check = this.scheduleIdle(() => this.refreshIfStale(filePath, cachedMtimeMs)) + .finally(() => { + this.freshnessChecks.delete(filePath); + }); + this.freshnessChecks.set(filePath, check); + } + + private scheduleIdle(task: () => Promise): Promise { + return new Promise((resolvePromise) => { + setImmediate(() => { + void task() + .catch(() => undefined) + .finally(resolvePromise); + }); + }); + } + + private async refreshIfStale(filePath: string, cachedMtimeMs: number): Promise { + const stats = await fs.promises.stat(filePath); + if (!stats.isFile() || stats.mtimeMs === cachedMtimeMs) { + return; + } + + const source = await fs.promises.readFile(filePath); + const codeCache = createCompiledData(source, filePath); + const currentPending = this.pendingModules.get(filePath); + if (currentPending && currentPending.mtimeMs > stats.mtimeMs) { + return; + } + this.recordAsync(filePath, source, codeCache, stats.mtimeMs); + } + + private executeCompiledModule(module: RuntimeModule, filename: string, source: Buffer, cachedData?: Buffer): Buffer | undefined { + // Module.load normally initializes these before invoking the extension. + module.filename ??= filename; + module.paths ??= nodeModule._nodeModulePaths(dirname(filename)); + if (getModuleExtension(filename) === '.json') { + module.exports = JSON.parse(stripJsonBom(source.toString('utf8'))); + module.loaded = true; + return undefined; + } + + const sourceText = stripShebang(source.toString('utf8')); + let compiled = compileCommonJs(sourceText, filename, cachedData); + const cachedDataRejected = compiled.cachedDataRejected === true; + if (cachedDataRejected) { + compiled = compileCommonJs(sourceText, filename); + } + const moduleExports = module.exports; + const localRequire = createModuleRequire(module); + Reflect.apply(compiled, moduleExports, [moduleExports, localRequire, module, filename, dirname(filename)]); + module.loaded = true; + if (cachedData !== undefined && !cachedDataRejected) { + return undefined; + } + return compiled.cachedData ?? Buffer.alloc(0); + } + + private loadCacheSync(): void { + let bytes: Buffer; + try { + bytes = fs.readFileSync(this.cachePath); + } catch { + return; + } + + if (bytes.length < HEADER_SIZE || !bytes.subarray(0, 8).equals(CACHE_MAGIC)) { + return; + } + if (bytes.readUInt32LE(8) !== CACHE_VERSION) { + return; + } + + const mappingLength = bytes.readUInt32LE(12); + const contentOffset = HEADER_SIZE + mappingLength; + if (contentOffset > bytes.length) { + return; + } + + try { + const mapping = JSON.parse(bytes.subarray(HEADER_SIZE, contentOffset).toString('utf8')) as unknown; + if (!isCacheManifest(mapping, bytes.length - contentOffset)) { + return; + } + this.cacheBytes = bytes; + this.cacheContentOffset = contentOffset; + this.cacheModules = new Map(Object.entries(mapping.modules)); + this.cacheResolutions = new Map(Object.entries(mapping.resolutions)); + } catch { + this.cacheBytes = undefined; + this.cacheModules.clear(); + this.cacheResolutions.clear(); + } + } + + private getCachedModule(entry: RuntimeBundleCacheEntry): { source: Buffer; codeCache: Buffer } | undefined { + if (!this.cacheBytes || !isCacheEntry(entry, this.cacheBytes.length - this.cacheContentOffset)) { + return undefined; + } + const start = this.cacheContentOffset + entry.offset; + const codeCacheStart = this.cacheContentOffset + entry.codeCacheOffset; + return { + source: this.cacheBytes.subarray(start, start + entry.length), + codeCache: this.cacheBytes.subarray(codeCacheStart, codeCacheStart + entry.codeCacheLength), + }; + } + + private recordAsync(filePath: string, source: Buffer, codeCache: Buffer, mtimeMs: number): void { + this.pendingModules.set(filePath, { + source: Buffer.from(source), + codeCache: Buffer.from(codeCache), + mtimeMs, + }); + if (this.flushScheduled) { + return; + } + this.flushScheduled = true; + setImmediate(() => { + this.flushScheduled = false; + void this.flush().catch(() => undefined); + }); + } + + private recordResolutionAsync(key: string, entry: RuntimeBundleResolutionEntry): void { + this.pendingResolutions.set(key, entry); + if (this.flushScheduled) { + return; + } + this.flushScheduled = true; + setImmediate(() => { + this.flushScheduled = false; + void this.flush().catch(() => undefined); + }); + } + + private scheduleResolutionRecord(key: string, filename: string): void { + if (this.resolutionChecks.has(key)) { + return; + } + const check = this.scheduleIdle(async () => { + const stats = await fs.promises.stat(filename); + if (stats.isFile()) { + this.recordResolutionAsync(key, { filename, mtimeMs: stats.mtimeMs }); + } + }) + .finally(() => { + this.resolutionChecks.delete(key); + }); + this.resolutionChecks.set(key, check); + } + + private scheduleResolutionFreshnessCheck(key: string, entry: RuntimeBundleResolutionEntry): void { + if (this.resolutionChecks.has(key)) { + return; + } + const check = this.scheduleIdle(async () => { + try { + const stats = await fs.promises.stat(entry.filename); + const current = this.pendingResolutions.get(key) ?? this.cacheResolutions.get(key); + if (current !== entry) { + return; + } + if (!stats.isFile()) { + this.pendingResolutions.delete(key); + this.cacheResolutions.delete(key); + return; + } + if (stats.mtimeMs !== entry.mtimeMs) { + this.recordResolutionAsync(key, { filename: entry.filename, mtimeMs: stats.mtimeMs }); + } + } catch { + const current = this.pendingResolutions.get(key) ?? this.cacheResolutions.get(key); + if (current === entry) { + this.pendingResolutions.delete(key); + this.cacheResolutions.delete(key); + } + } + }) + .finally(() => { + this.resolutionChecks.delete(key); + }); + this.resolutionChecks.set(key, check); + } + + private restorePending( + pendingModules: Map, + pendingResolutions: Map, + ): void { + const restoredModules = new Map(pendingModules); + for (const [filePath, content] of this.pendingModules) { + restoredModules.set(filePath, content); + } + this.pendingModules = restoredModules; + + const restoredResolutions = new Map(pendingResolutions); + for (const [key, entry] of this.pendingResolutions) { + restoredResolutions.set(key, entry); + } + this.pendingResolutions = restoredResolutions; + } + + private async writeCache( + pendingModules: Map, + pendingResolutions: Map, + ): Promise { + const contentByPath = new Map(); + const entries = new Map(); + + for (const [filePath, entry] of this.cacheModules) { + const content = this.getCachedModule(entry); + if (content) { + contentByPath.set(filePath, { + source: Buffer.from(content.source), + codeCache: Buffer.from(content.codeCache), + }); + entries.set(filePath, { ...entry }); + } + } + for (const [filePath, value] of pendingModules) { + contentByPath.set(filePath, { source: value.source, codeCache: value.codeCache }); + entries.set(filePath, { + offset: 0, + length: value.source.length, + codeCacheOffset: value.source.length, + codeCacheLength: value.codeCache.length, + mtimeMs: value.mtimeMs, + }); + } + + let offset = 0; + const modules: Record = {}; + for (const [filePath, entry] of entries) { + const content = contentByPath.get(filePath)!; + modules[filePath] = { + offset, + length: content.source.length, + codeCacheOffset: offset + content.source.length, + codeCacheLength: content.codeCache.length, + mtimeMs: entry.mtimeMs, + }; + offset += content.source.length + content.codeCache.length; + } + + const resolutions = new Map(this.cacheResolutions); + for (const [key, entry] of pendingResolutions) { + resolutions.set(key, entry); + } + const manifest: CacheManifest = { + v8CachedDataVersion: V8_CACHED_DATA_VERSION, + modules, + resolutions: Object.fromEntries(resolutions), + }; + + const mappingBytes = Buffer.from(JSON.stringify(manifest), 'utf8'); + const header = Buffer.alloc(HEADER_SIZE); + CACHE_MAGIC.copy(header, 0); + header.writeUInt32LE(CACHE_VERSION, 8); + header.writeUInt32LE(mappingBytes.length, 12); + + const tempPath = `${this.cachePath}.${process.pid}.${Date.now()}.tmp`; + await fs.promises.mkdir(dirname(this.cachePath), { recursive: true }); + const stream = fs.createWriteStream(tempPath, { flags: 'w' }); + try { + await this.writeStream(stream, [ + header, + mappingBytes, + ...Object.keys(modules).flatMap((filePath) => { + const content = contentByPath.get(filePath)!; + return [content.source, content.codeCache]; + }), + ]); + await fs.promises.rename(tempPath, this.cachePath); + } catch (error) { + stream.destroy(); + await fs.promises.rm(tempPath, { force: true }).catch(() => undefined); + throw error; + } + + this.cacheModules = new Map(Object.entries(modules)); + this.cacheResolutions = resolutions; + this.cacheContentOffset = HEADER_SIZE + mappingBytes.length; + this.cacheBytes = Buffer.concat([ + header, + mappingBytes, + ...Object.keys(modules).flatMap((filePath) => { + const content = contentByPath.get(filePath)!; + return [content.source, content.codeCache]; + }), + ], HEADER_SIZE + mappingBytes.length + offset); + } + + private async writeStream(stream: fs.WriteStream, chunks: Buffer[]): Promise { + await new Promise((resolvePromise, rejectPromise) => { + let index = 0; + let settled = false; + + const cleanup = (): void => { + stream.removeListener('error', onError); + stream.removeListener('finish', onFinish); + stream.removeListener('drain', onDrain); + }; + const resolveOnce = (): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + resolvePromise(); + }; + const rejectOnce = (error: Error): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + rejectPromise(error); + }; + const onError = (error: Error): void => rejectOnce(error); + const onFinish = (): void => resolveOnce(); + const writeNext = (): void => { + if (settled) { + return; + } + try { + while (index < chunks.length && stream.write(chunks[index]!)) { + index += 1; + } + if (index < chunks.length) { + index += 1; + stream.once('drain', onDrain); + } else { + stream.end(); + } + } catch (error) { + rejectOnce(error as Error); + } + }; + const onDrain = (): void => writeNext(); + + stream.once('error', onError); + stream.once('finish', onFinish); + writeNext(); + }); + } +} + +function isCacheEnabled(value: string | undefined): boolean { + if (value === undefined) { + return true; + } + return !['0', 'false', 'off', 'no'].includes(value.trim().toLowerCase()); +} + +function getDefaultCachePath(): string { + const configuredFile = process.env[CACHE_FILE_ENV]?.trim(); + if (configuredFile) { + return resolve(process.cwd(), configuredFile); + } + + const configuredScope = process.env[CACHE_SCOPE_ENV]?.trim().toLowerCase(); + const isScene = configuredScope === 'scene' || (configuredScope !== 'host' && isSceneProcess()); + const baseName = isScene ? '.runtime-bundle-cache.scene' : '.runtime-bundle-cache'; + return resolve(process.cwd(), `${baseName}-[${getRuntimeCacheVersion()}]`); +} + +function isSceneProcess(): boolean { + return process.argv.some((argument) => /[\\/]scene-process[\\/]main\.(?:c|m)?js$/.test(argument)); +} + +function getRuntimeCacheVersion(): string { + const pinkVersion = process.env[PINK_VERSION_ENV]?.trim() + || readNearestVersion(__dirname, 'product.json', 'pinkVersion') + || readNearestVersion(process.cwd(), 'product.json', 'pinkVersion') + || 'unknown'; + const cocosCliPath = getArgumentValue('--cocos-path', '--enginePath'); + const cocosCliVersion = process.env[COCOS_CLI_VERSION_ENV]?.trim() + || (cocosCliPath ? readNearestVersion(cocosCliPath, 'package.json', 'version') : undefined) + || 'unknown'; + return `${sanitizeVersion(pinkVersion)}+${sanitizeVersion(cocosCliVersion)}`; +} + +function getArgumentValue(...names: string[]): string | undefined { + for (let index = 0; index < process.argv.length; index++) { + const argument = process.argv[index]; + for (const name of names) { + if (argument === name) { + return process.argv[index + 1]; + } + if (argument.startsWith(`${name}=`)) { + return argument.slice(name.length + 1); + } + } + } + return undefined; +} + +function readNearestVersion(startDirectory: string, fileName: string, property: string): string | undefined { + let directory = resolve(startDirectory); + while (true) { + try { + const filePath = join(directory, fileName); + const value = JSON.parse(fs.readFileSync(filePath, 'utf8')) as Record; + const version = value[property]; + if (typeof version === 'string' && version.trim()) { + return version.trim(); + } + } catch { + // Continue searching parent directories when the file is absent or invalid. + } + + const parent = dirname(directory); + if (parent === directory) { + return undefined; + } + directory = parent; + } +} + +function sanitizeVersion(version: string): string { + return version.replace(/[^A-Za-z0-9._-]/g, '_'); +} + +/* +```mermaid +sequenceDiagram + participant Runtime as RuntimeBundler + participant Cache as 当前版本 cache + participant FS as 文件系统 + participant Trash as 系统回收站 + Runtime->>Cache: 同步读取当前版本 cache + Runtime-->>FS: setImmediate 扫描旧版本 cache + FS-->>Runtime: 返回旧 cache 文件 + Runtime->>Trash: 异步移入回收站 +``` +*/ +function scheduleStaleCacheCleanup(cachePath: string, moveToTrash: (filePath: string) => Promise): void { + setImmediate(() => { + void cleanupStaleCacheFiles(cachePath, moveToTrash).catch(() => undefined); + }); +} + +async function cleanupStaleCacheFiles(cachePath: string, moveToTrash: (filePath: string) => Promise): Promise { + const currentName = basename(cachePath); + const baseName = currentName.split('-[', 1)[0]; + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(dirname(cachePath), { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (!entry.isFile() || entry.name === currentName || !isStaleCacheName(entry.name, baseName)) { + continue; + } + await moveToTrash(join(dirname(cachePath), entry.name)).catch(() => undefined); + } +} + +function isStaleCacheName(fileName: string, baseName: string): boolean { + return fileName === baseName || (fileName.startsWith(`${baseName}-[`) && fileName.endsWith(']')); +} + +async function moveCacheFileToTrash(filePath: string): Promise { + const electronTrash = getElectronTrashItem(); + if (electronTrash) { + await electronTrash(filePath); + return; + } + + if (process.platform === 'win32') { + return; + } + + const trashDirectory = process.platform === 'darwin' + ? join(os.homedir(), '.Trash') + : join(os.homedir(), '.local', 'share', 'Trash', 'files'); + await fs.promises.mkdir(trashDirectory, { recursive: true }); + await fs.promises.rename(filePath, await getUniqueTrashPath(trashDirectory, basename(filePath))); +} + +function getElectronTrashItem(): ((filePath: string) => Promise) | undefined { + try { + const electron = nodeRequire('electron') as { shell?: { trashItem?: (filePath: string) => Promise } }; + return electron.shell?.trashItem; + } catch { + return undefined; + } +} + +async function getUniqueTrashPath(trashDirectory: string, fileName: string): Promise { + const extension = extname(fileName); + const stem = basename(fileName, extension); + for (let index = 0; ; index++) { + const candidate = join(trashDirectory, index === 0 ? fileName : `${stem}-${index}${extension}`); + try { + await fs.promises.access(candidate); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return candidate; + } + throw error; + } + } +} + +function isCacheableModulePath(filePath: string): boolean { + const normalized = filePath.replaceAll('\\', '/'); + if (normalized.startsWith('node:') || builtinModules.has(normalized)) { + return false; + } + const moduleName = normalized.startsWith('node:') ? normalized.slice('node:'.length) : normalized; + if (builtinModules.has(moduleName)) { + return false; + } + if (!normalized.startsWith('/') && !/^[A-Za-z]:\//.test(normalized)) { + return false; + } + const extension = extname(normalized).toLowerCase(); + // Module.load is also used for extensionless CommonJS files and JSON + // modules. ESM (.mjs) and native addons (.node) bypass this source loader + // and must continue through Node's normal loading path. + if (extension === '.json' || extension === '.cjs') { + return true; + } + if (extension !== '' && extension !== '.js') { + return false; + } + return !isModulePackagePath(resolve(filePath)); +} + +function isModulePackagePath(filePath: string): boolean { + let directory = dirname(filePath); + while (true) { + const packageType = packageTypeForDirectory(directory); + if (packageType) { + return packageType === 'module'; + } + const parent = dirname(directory); + if (parent === directory) { + return false; + } + directory = parent; + } +} + +function packageTypeForDirectory(directory: string): 'commonjs' | 'module' | undefined { + if (packageTypeCache.has(directory)) { + return packageTypeCache.get(directory); + } + + try { + const packageJson = JSON.parse(fs.readFileSync(resolve(directory, 'package.json'), 'utf8')) as { type?: unknown }; + const packageType = packageJson.type === 'module' ? 'module' : 'commonjs'; + packageTypeCache.set(directory, packageType); + return packageType; + } catch { + packageTypeCache.set(directory, undefined); + return undefined; + } +} + +function isCacheableResolvedPath(filePath: string): boolean { + return isCacheableModulePath(resolve(filePath)); +} + +function getResolutionKey( + request: string, + parent: RuntimeModule | undefined, + isMain: boolean | undefined, + options: unknown, +): string | undefined { + if (!parent?.filename || options !== undefined || isBuiltinRequest(request)) { + return undefined; + } + return JSON.stringify({ parent: resolve(parent.filename), request, isMain: Boolean(isMain) }); +} + +function isBuiltinRequest(request: string): boolean { + const moduleName = request.startsWith('node:') ? request.slice('node:'.length) : request; + return request.startsWith('node:') || builtinModules.has(moduleName); +} + +function compileCommonJs(source: string, filename: string, cachedData?: Buffer): CompiledModule { + return vm.compileFunction(source, COMMONJS_PARAMETERS, { + filename, + cachedData, + produceCachedData: cachedData === undefined, + }) as unknown as CompiledModule; +} + +function createCompiledData(source: Buffer, filename: string): Buffer { + if (getModuleExtension(filename) === '.json') { + return Buffer.alloc(0); + } + const compiled = compileCommonJs(stripShebang(source.toString('utf8')), filename); + return compiled.cachedData ?? Buffer.alloc(0); +} + +function createModuleRequire(module: RuntimeModule): NodeRequire { + const localRequire = module.require.bind(module) as NodeRequire; + localRequire.resolve = ((request: string, options?: { paths?: string[] }): string => ( + nodeModule._resolveFilename(request, module, false, options) + )) as NodeRequire['resolve']; + localRequire.cache = nodeRequire.cache; + localRequire.extensions = nodeRequire.extensions; + localRequire.main = nodeRequire.main; + return localRequire; +} + +function getModuleExtension(filePath: string): string { + return extname(filePath).toLowerCase(); +} + +function stripJsonBom(content: string): string { + return content.charCodeAt(0) === 0xFEFF ? content.slice(1) : content; +} + +function stripShebang(content: string): string { + return content.replace(/^#![^\r\n]*(?:\r\n|\n|\r|$)/, (line) => line.replace(/[^\r\n]/g, ' ')); +} + +function isCacheManifest(value: unknown, contentLength: number): value is CacheManifest { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + const manifest = value as Partial; + if (manifest.v8CachedDataVersion !== V8_CACHED_DATA_VERSION + || !manifest.modules + || !manifest.resolutions + || typeof manifest.modules !== 'object' + || typeof manifest.resolutions !== 'object') { + return false; + } + return Object.values(manifest.modules).every((entry) => isCacheEntry(entry, contentLength)) + && Object.values(manifest.resolutions).every(isResolutionEntry); +} + +function isCacheEntry(value: unknown, contentLength: number): value is RuntimeBundleCacheEntry { + if (!value || typeof value !== 'object') { + return false; + } + const entry = value as Partial; + const { offset, length, codeCacheOffset, codeCacheLength, mtimeMs } = entry; + return typeof offset === 'number' + && typeof length === 'number' + && typeof codeCacheOffset === 'number' + && typeof codeCacheLength === 'number' + && Number.isSafeInteger(offset) + && Number.isSafeInteger(length) + && Number.isSafeInteger(codeCacheOffset) + && Number.isSafeInteger(codeCacheLength) + && offset >= 0 + && length >= 0 + && codeCacheOffset >= 0 + && codeCacheLength >= 0 + && offset + length <= contentLength + && codeCacheOffset + codeCacheLength <= contentLength + && typeof mtimeMs === 'number' + && Number.isFinite(mtimeMs); +} + +function isResolutionEntry(value: unknown): value is RuntimeBundleResolutionEntry { + if (!value || typeof value !== 'object') { + return false; + } + const entry = value as Partial; + return typeof entry.filename === 'string' + && typeof entry.mtimeMs === 'number' + && Number.isFinite(entry.mtimeMs); +} + +export const runtimeBundler = new RuntimeBundler(); +if (!globalState[GLOBAL_INSTALL_KEY]) { + globalState[GLOBAL_INSTALL_KEY] = true; + runtimeBundler.install(); +} diff --git a/tests/runtime-bundle.test.ts b/tests/runtime-bundle.test.ts new file mode 100644 index 000000000..9a6d17bdd --- /dev/null +++ b/tests/runtime-bundle.test.ts @@ -0,0 +1,523 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { dirname, join, resolve } from 'path'; +import * as vm from 'vm'; +import { RuntimeBundler, runtimeBundler } from '../src/lib/runtime-module-cache'; + +jest.mock('vm', () => { + const actual = jest.requireActual('vm'); + return { + ...actual, + compileFunction: jest.fn(actual.compileFunction), + }; +}); + +const nodeFs = require('fs') as typeof import('fs'); +const nodeModule = require('module') as typeof import('module'); + +describe('RuntimeBundler', () => { + let tempRoot: string; + let bundler: RuntimeBundler | undefined; + + beforeEach(() => { + tempRoot = mkdtempSync(join(tmpdir(), 'cocos-runtime-bundle-')); + }); + + afterEach(() => { + bundler?.uninstall(); + runtimeBundler.install(); + jest.restoreAllMocks(); + rmSync(tempRoot, { recursive: true, force: true }); + }); + + function createModule(sourcePath: string, value: string): void { + writeFileSync(sourcePath, `module.exports = ${JSON.stringify(value)};`); + } + + function requireFresh(sourcePath: string): unknown { + const moduleInstance = new (nodeModule as any)(sourcePath, module) as { + filename: string; + paths: string[]; + exports: unknown; + load(filename: string): void; + }; + moduleInstance.filename = sourcePath; + moduleInstance.paths = (nodeModule as any)._nodeModulePaths(dirname(sourcePath)); + moduleInstance.load(sourcePath); + return moduleInstance.exports; + } + + it('matches CommonJS and JSON modules by default but excludes built-ins and unsupported module types', () => { + runtimeBundler.uninstall(); + bundler = new RuntimeBundler(); + const isRuntimeBundle = (bundler as any).isRuntimeBundle as (filePath: string) => boolean; + + expect(isRuntimeBundle('/tmp/project/module.js')).toBe(true); + expect(isRuntimeBundle('/tmp/project/module.cjs')).toBe(true); + expect(isRuntimeBundle('/tmp/project/module.json')).toBe(true); + expect(isRuntimeBundle('/tmp/project/module')).toBe(true); + expect(isRuntimeBundle('node:fs')).toBe(false); + expect(isRuntimeBundle('/tmp/project/module.mjs')).toBe(false); + expect(isRuntimeBundle('/tmp/project/module.node')).toBe(false); + }); + + it('skips JavaScript files inside type=module package scopes', () => { + const packageRoot = join(tempRoot, 'esm-package'); + mkdirSync(join(packageRoot, 'nested'), { recursive: true }); + writeFileSync(join(packageRoot, 'package.json'), JSON.stringify({ type: 'module' })); + writeFileSync(join(packageRoot, 'index.js'), 'export default true;'); + writeFileSync(join(packageRoot, 'nested', 'common.cjs'), 'module.exports = true;'); + + runtimeBundler.uninstall(); + const bundler = new RuntimeBundler(); + const isRuntimeBundle = (bundler as any).isRuntimeBundle as (filePath: string) => boolean; + + expect(isRuntimeBundle(join(packageRoot, 'index.js'))).toBe(false); + expect(isRuntimeBundle(join(packageRoot, 'nested', 'extensionless'))).toBe(false); + expect(isRuntimeBundle(join(packageRoot, 'nested', 'common.cjs'))).toBe(true); + }); + + it('separates host and scene cache files and supports an explicit cache file', () => { + const previousScope = process.env.COCOS_RUNTIME_BUNDLE_CACHE_SCOPE; + const previousFile = process.env.COCOS_RUNTIME_BUNDLE_CACHE_FILE; + const previousPinkVersion = process.env.VSCODE_PINK_VERSION; + const previousCocosCliVersion = process.env.COCOS_CLI_VERSION; + try { + delete process.env.COCOS_RUNTIME_BUNDLE_CACHE_FILE; + process.env.VSCODE_PINK_VERSION = 'pink-test'; + process.env.COCOS_CLI_VERSION = 'cli-test'; + process.env.COCOS_RUNTIME_BUNDLE_CACHE_SCOPE = 'host'; + expect((new RuntimeBundler() as any).cachePath).toBe(resolve(process.cwd(), '.runtime-bundle-cache-[pink-test+cli-test]')); + + process.env.COCOS_RUNTIME_BUNDLE_CACHE_SCOPE = 'scene'; + expect((new RuntimeBundler() as any).cachePath).toBe(resolve(process.cwd(), '.runtime-bundle-cache.scene-[pink-test+cli-test]')); + + const explicitPath = join(tempRoot, 'scene-runtime-bundle.cache'); + process.env.COCOS_RUNTIME_BUNDLE_CACHE_FILE = explicitPath; + expect((new RuntimeBundler() as any).cachePath).toBe(explicitPath); + } finally { + if (previousScope === undefined) { + delete process.env.COCOS_RUNTIME_BUNDLE_CACHE_SCOPE; + } else { + process.env.COCOS_RUNTIME_BUNDLE_CACHE_SCOPE = previousScope; + } + if (previousFile === undefined) { + delete process.env.COCOS_RUNTIME_BUNDLE_CACHE_FILE; + } else { + process.env.COCOS_RUNTIME_BUNDLE_CACHE_FILE = previousFile; + } + if (previousPinkVersion === undefined) { + delete process.env.VSCODE_PINK_VERSION; + } else { + process.env.VSCODE_PINK_VERSION = previousPinkVersion; + } + if (previousCocosCliVersion === undefined) { + delete process.env.COCOS_CLI_VERSION; + } else { + process.env.COCOS_CLI_VERSION = previousCocosCliVersion; + } + } + }); + + it('moves legacy and stale versioned default caches to trash asynchronously', async () => { + const previousCwd = process.cwd(); + const previousPinkVersion = process.env.VSCODE_PINK_VERSION; + const previousCocosCliVersion = process.env.COCOS_CLI_VERSION; + const movedFiles: string[] = []; + try { + process.chdir(tempRoot); + process.env.VSCODE_PINK_VERSION = 'pink-current'; + process.env.COCOS_CLI_VERSION = 'cli-current'; + const legacyPath = resolve(process.cwd(), '.runtime-bundle-cache'); + const stalePath = resolve(process.cwd(), '.runtime-bundle-cache-[pink-old+cli-old]'); + const currentPath = resolve(process.cwd(), '.runtime-bundle-cache-[pink-current+cli-current]'); + writeFileSync(legacyPath, 'legacy'); + writeFileSync(stalePath, 'stale'); + writeFileSync(currentPath, 'current'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ + moveToTrash: async (filePath) => { + movedFiles.push(filePath); + rmSync(filePath, { force: true }); + }, + }); + bundler.install(); + await new Promise((resolvePromise) => setImmediate(() => setImmediate(resolvePromise))); + + expect(movedFiles.sort()).toEqual([legacyPath, stalePath].sort()); + expect(existsSync(legacyPath)).toBe(false); + expect(existsSync(stalePath)).toBe(false); + expect(existsSync(currentPath)).toBe(true); + } finally { + process.chdir(previousCwd); + if (previousPinkVersion === undefined) { + delete process.env.VSCODE_PINK_VERSION; + } else { + process.env.VSCODE_PINK_VERSION = previousPinkVersion; + } + if (previousCocosCliVersion === undefined) { + delete process.env.COCOS_CLI_VERSION; + } else { + process.env.COCOS_CLI_VERSION = previousCocosCliVersion; + } + } + }); + + it('keeps stale caches when moving them to trash fails', async () => { + const previousCwd = process.cwd(); + const previousPinkVersion = process.env.VSCODE_PINK_VERSION; + const previousCocosCliVersion = process.env.COCOS_CLI_VERSION; + try { + process.chdir(tempRoot); + process.env.VSCODE_PINK_VERSION = 'pink-current'; + process.env.COCOS_CLI_VERSION = 'cli-current'; + const stalePath = resolve(process.cwd(), '.runtime-bundle-cache-[pink-old+cli-old]'); + writeFileSync(stalePath, 'stale'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ + moveToTrash: async () => { + throw new Error('trash unavailable'); + }, + }); + bundler.install(); + await new Promise((resolvePromise) => setImmediate(() => setImmediate(resolvePromise))); + + expect(existsSync(stalePath)).toBe(true); + } finally { + process.chdir(previousCwd); + if (previousPinkVersion === undefined) { + delete process.env.VSCODE_PINK_VERSION; + } else { + process.env.VSCODE_PINK_VERSION = previousPinkVersion; + } + if (previousCocosCliVersion === undefined) { + delete process.env.COCOS_CLI_VERSION; + } else { + process.env.COCOS_CLI_VERSION = previousCocosCliVersion; + } + } + }); + + it('records module bytes on a miss and serves a cache hit through Module.load', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'first'); + + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + const originalLoad = (nodeModule.prototype as any).load; + bundler.install(); + expect((nodeModule.prototype as any).load).not.toBe(originalLoad); + + const originalReadFileSync = nodeFs.readFileSync; + expect(nodeFs.readFileSync).toBe(originalReadFileSync); + expect(requireFresh(sourcePath)).toBe('first'); + expect(requireFresh(sourcePath)).toBe('first'); + await bundler.flush(); + bundler.uninstall(); + + const cache = readFileSync(cachePath); + const mappingLength = cache.readUInt32LE(12); + const mapping = JSON.parse(cache.subarray(16, 16 + mappingLength).toString('utf8')); + const entry = mapping.modules[Object.keys(mapping.modules)[0]]; + expect(entry).toEqual(expect.objectContaining({ + length: Buffer.byteLength(`module.exports = ${JSON.stringify('first')};`), + mtimeMs: statSync(sourcePath).mtimeMs, + })); + expect(cache.subarray(16 + mappingLength + entry.offset, 16 + mappingLength + entry.offset + entry.length).toString()).toBe( + `module.exports = ${JSON.stringify('first')};`, + ); + + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('first'); + }); + + it('persists compiled data and does not call Module._compile on a cache hit', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'compiled'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('compiled'); + await bundler.flush(); + bundler.uninstall(); + + const cache = readFileSync(cachePath); + const mappingLength = cache.readUInt32LE(12); + const mapping = JSON.parse(cache.subarray(16, 16 + mappingLength).toString('utf8')); + const entry = mapping.modules[sourcePath]; + expect(entry).toEqual(expect.objectContaining({ + codeCacheLength: expect.any(Number), + })); + expect(entry.codeCacheLength).toBeGreaterThan(0); + + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + const compile = jest.spyOn((nodeModule.prototype as any), '_compile'); + expect(requireFresh(sourcePath)).toBe('compiled'); + expect(compile).not.toHaveBeenCalled(); + compile.mockRestore(); + }); + + it('reuses the compiled CommonJS wrapper for repeated loads in one process', () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'compiled-once'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + const compileFunction = vm.compileFunction as jest.MockedFunction; + compileFunction.mockClear(); + bundler.install(); + + expect(requireFresh(sourcePath)).toBe('compiled-once'); + expect(requireFresh(sourcePath)).toBe('compiled-once'); + + expect(compileFunction).toHaveBeenCalledTimes(1); + compileFunction.mockClear(); + }); + + it('returns cached content before asynchronously checking and refreshing a changed source', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'first'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('first'); + await bundler.flush(); + bundler.uninstall(); + + writeFileSync(sourcePath, 'module.exports = "second";'); + const changedMtime = statSync(sourcePath).mtimeMs + 2000; + utimesSync(sourcePath, new Date(changedMtime), new Date(changedMtime)); + + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + const statSyncSpy = jest.spyOn(nodeFs, 'statSync'); + const asyncStatSpy = jest.spyOn(nodeFs.promises, 'stat'); + expect(requireFresh(sourcePath)).toBe('first'); + expect(statSyncSpy).not.toHaveBeenCalledWith(sourcePath); + expect(asyncStatSpy).not.toHaveBeenCalledWith(sourcePath); + + await new Promise((resolvePromise) => setImmediate(resolvePromise)); + expect(asyncStatSpy).toHaveBeenCalledWith(sourcePath); + await bundler.flush(); + expect(requireFresh(sourcePath)).toBe('second'); + statSyncSpy.mockRestore(); + asyncStatSpy.mockRestore(); + }); + + it('does not compile solely to refresh an expired cache entry during idle maintenance', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'first'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('first'); + await bundler.flush(); + bundler.uninstall(); + + writeFileSync(sourcePath, 'module.exports = "second";'); + const changedMtime = statSync(sourcePath).mtimeMs + 2000; + utimesSync(sourcePath, new Date(changedMtime), new Date(changedMtime)); + + bundler = new RuntimeBundler({ cachePath }); + const compileFunction = vm.compileFunction as jest.MockedFunction; + compileFunction.mockClear(); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('first'); + expect(compileFunction).toHaveBeenCalledTimes(1); + + await new Promise((resolvePromise) => setImmediate(resolvePromise)); + await bundler.flush(); + + expect(compileFunction).toHaveBeenCalledTimes(1); + compileFunction.mockClear(); + }); + + it('defers the automatic cache flush until the event-loop idle phase', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'first'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + const writeCache = jest.spyOn(bundler as any, 'writeCache'); + expect(requireFresh(sourcePath)).toBe('first'); + await Promise.resolve(); + expect(writeCache).not.toHaveBeenCalled(); + + await new Promise((resolvePromise) => setImmediate(resolvePromise)); + expect(writeCache).toHaveBeenCalled(); + await bundler.flush(); + writeCache.mockRestore(); + }); + + it('reuses a cached dependency resolution without calling the original resolver', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const dependencyPath = join(tempRoot, 'runtime', 'dependency.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + writeFileSync(dependencyPath, 'module.exports = "dependency";'); + writeFileSync(sourcePath, 'module.exports = require("./dependency.js");'); + + runtimeBundler.uninstall(); + const resolveFilename = jest.spyOn(nodeModule as any, '_resolveFilename'); + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('dependency'); + await bundler.flush(); + bundler.uninstall(); + + const cache = readFileSync(cachePath); + const mappingLength = cache.readUInt32LE(12); + const mapping = JSON.parse(cache.subarray(16, 16 + mappingLength).toString('utf8')); + expect(mapping.resolutions).toEqual(expect.any(Object)); + + resolveFilename.mockClear(); + delete (nodeModule as any)._cache[dependencyPath]; + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + const statSyncSpy = jest.spyOn(nodeFs, 'statSync'); + expect(requireFresh(sourcePath)).toBe('dependency'); + expect(resolveFilename).not.toHaveBeenCalled(); + expect(statSyncSpy).not.toHaveBeenCalledWith(sourcePath); + expect(statSyncSpy).not.toHaveBeenCalledWith(dependencyPath); + statSyncSpy.mockRestore(); + resolveFilename.mockRestore(); + }); + + it('invalidates a cached module when its mtime changes', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'first'); + + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('first'); + await bundler.flush(); + bundler.uninstall(); + + writeFileSync(sourcePath, 'module.exports = "second";'); + const changedMtime = statSync(sourcePath).mtimeMs + 2000; + utimesSync(sourcePath, new Date(changedMtime), new Date(changedMtime)); + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + + expect(requireFresh(sourcePath)).toBe('first'); + await bundler.flush(); + expect(requireFresh(sourcePath)).toBe('second'); + }); + + it('serves cached JSON through the native JSON module contract', async () => { + const sourcePath = join(tempRoot, 'runtime', 'metadata.json'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + writeFileSync(sourcePath, '\uFEFF{"name":"first","enabled":true}'); + + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + expect(requireFresh(sourcePath)).toEqual({ name: 'first', enabled: true }); + await bundler.flush(); + bundler.uninstall(); + + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + const readFileSync = jest.spyOn(nodeFs, 'readFileSync'); + expect(requireFresh(sourcePath)).toEqual({ name: 'first', enabled: true }); + expect(readFileSync).not.toHaveBeenCalledWith(sourcePath); + readFileSync.mockRestore(); + }); + + it('can be disabled with COCOS_RUNTIME_BUNDLE_CACHE', () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'content'); + + runtimeBundler.uninstall(); + const previousValue = process.env.COCOS_RUNTIME_BUNDLE_CACHE; + process.env.COCOS_RUNTIME_BUNDLE_CACHE = '0'; + try { + bundler = new RuntimeBundler({ + cachePath: join(tempRoot, '.runtime-bundle-cache'), + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + + expect(requireFresh(sourcePath)).toBe('content'); + writeFileSync(sourcePath, 'module.exports = "changed";'); + expect(requireFresh(sourcePath)).toBe('changed'); + } finally { + if (previousValue === undefined) { + delete process.env.COCOS_RUNTIME_BUNDLE_CACHE; + } else { + process.env.COCOS_RUNTIME_BUNDLE_CACHE = previousValue; + } + } + }); + + it('keeps pending bytes when an asynchronous cache write fails', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'content'); + + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + const writeCache = jest.spyOn(bundler as any, 'writeCache').mockRejectedValue(new Error('write failed')); + + expect(requireFresh(sourcePath)).toBe('content'); + await expect(bundler.flush()).rejects.toThrow('write failed'); + + writeCache.mockRestore(); + await bundler.flush(); + bundler.uninstall(); + + bundler = new RuntimeBundler({ + cachePath, + isRuntimeBundle: (filePath) => filePath.endsWith('/runtime/index.js'), + }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('content'); + }); +}); From 9b50465da90cbb4bae0089b109835898a4dccf9d Mon Sep 17 00:00:00 2001 From: looopmax Date: Tue, 18 Aug 2026 15:49:05 +0800 Subject: [PATCH 2/2] perf: optimize runtime cache and asset refresh --- .gitignore | 1 + .../source/libs/filesystem/local-provider.ts | 11 ++- .../test/16.filesystem-provider.spec.js | 17 ++++ src/api/index.ts | 26 +++-- src/core/assets/manager/asset-db.ts | 62 +++++++----- .../test/auto-refresh-asset-lazy.test.ts | 55 +++++++++++ src/core/scene/scene-process/main.ts | 12 ++- .../startup-cpu-profiler.test.ts | 63 +++++++++++++ .../scene-process/startup-cpu-profiler.ts | 94 +++++++++++++++++++ src/lib/runtime-module-cache.ts | 76 ++++++++++----- tests/runtime-bundle.test.ts | 32 +++++++ 11 files changed, 392 insertions(+), 57 deletions(-) create mode 100644 src/core/assets/test/auto-refresh-asset-lazy.test.ts create mode 100644 src/core/scene/scene-process/startup-cpu-profiler.test.ts create mode 100644 src/core/scene/scene-process/startup-cpu-profiler.ts diff --git a/.gitignore b/.gitignore index 657faa349..1935d0e53 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .user.json .DS_Store .temp +.runtime-bundle-cache* # tsconfig tsconfig.tsbuildinfo diff --git a/packages/asset-db/source/libs/filesystem/local-provider.ts b/packages/asset-db/source/libs/filesystem/local-provider.ts index 7f3186e4f..fc4cfcb03 100644 --- a/packages/asset-db/source/libs/filesystem/local-provider.ts +++ b/packages/asset-db/source/libs/filesystem/local-provider.ts @@ -1,12 +1,17 @@ 'use strict'; import { dirname } from 'path'; -import { copy, ensureDir, existsSync, move, outputFile, readFile, remove, stat } from 'fs-extra'; +import { access, copy, ensureDir, move, outputFile, readFile, remove, stat } from 'fs-extra'; import { IAssetDeleteOptions, IAssetFileSystemProvider, IAssetRenameOptions, IAssetWriteFileOptions } from './provider'; export class LocalAssetFileSystemProvider implements IAssetFileSystemProvider { - exists(path: string) { - return existsSync(path); + async exists(path: string) { + try { + await access(path); + return true; + } catch { + return false; + } } async stat(path: string) { diff --git a/packages/asset-db/test/16.filesystem-provider.spec.js b/packages/asset-db/test/16.filesystem-provider.spec.js index 894f359d5..7a489f3c7 100644 --- a/packages/asset-db/test/16.filesystem-provider.spec.js +++ b/packages/asset-db/test/16.filesystem-provider.spec.js @@ -5,6 +5,7 @@ const fse = require('fs-extra'); const path = require('path'); const assetdb = require('../dist'); +const { LocalAssetFileSystemProvider } = require('../dist/libs/filesystem/local-provider'); const { fsCopy, fsReadFile, @@ -44,6 +45,22 @@ describe('AssetDB 文件系统 Provider', () => { return { db, asset }; } + it('LocalAssetFileSystemProvider.exists 使用异步 access 检查路径', async () => { + const provider = new LocalAssetFileSystemProvider(); + const existingPath = path.join(PATH.ROOT, 'exists.txt'); + const missingPath = path.join(PATH.ROOT, 'missing.txt'); + + fse.outputFileSync(existingPath, 'exists'); + + const existingResult = provider.exists(existingPath); + const missingResult = provider.exists(missingPath); + + expect(existingResult).to.be.instanceOf(Promise); + expect(missingResult).to.be.instanceOf(Promise); + expect(await existingResult).to.equal(true); + expect(await missingResult).to.equal(false); + }); + afterEach(() => { if (typeof assetdb.resetFileSystemProvider === 'function') { assetdb.resetFileSystemProvider(); diff --git a/src/api/index.ts b/src/api/index.ts index 32d547f13..bd7e52dca 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,3 +1,4 @@ +import '../lib/runtime-module-cache'; import type { EngineApi } from '../api/engine/engine'; import type { ProjectApi } from '../api/project/project'; import type { AssetsApi } from '../api/assets/assets'; @@ -32,19 +33,30 @@ export class CocosAPI { * 初始化 API 实例,主要是为了实现按需加载 */ private async _init() { - const { SceneApi } = await import('../api/scene/scene'); + // 各模块之间无实例级依赖,可并行加载(模块加载器保证共享依赖只求值一次) + const [ + { SceneApi }, + { EngineApi }, + { ProjectApi }, + { AssetsApi }, + { BuilderApi }, + { ConfigurationApi }, + { SystemApi }, + ] = await Promise.all([ + import('../api/scene/scene'), + import('../api/engine/engine'), + import('../api/project/project'), + import('../api/assets/assets'), + import('../api/builder/builder'), + import('../api/configuration/configuration'), + import('../api/system/system'), + ]); this.scene = new SceneApi(); - const { EngineApi } = await import('../api/engine/engine'); this.engine = new EngineApi(); - const { ProjectApi } = await import('../api/project/project'); this.project = new ProjectApi(); - const { AssetsApi } = await import('../api/assets/assets'); this.assets = new AssetsApi(); - const { BuilderApi } = await import('../api/builder/builder'); this.builder = new BuilderApi(); - const { ConfigurationApi } = await import('../api/configuration/configuration'); this.configuration = new ConfigurationApi(); - const { SystemApi } = await import('../api/system/system'); this.system = new SystemApi(); } diff --git a/src/core/assets/manager/asset-db.ts b/src/core/assets/manager/asset-db.ts index eea836292..3981f9ee5 100644 --- a/src/core/assets/manager/asset-db.ts +++ b/src/core/assets/manager/asset-db.ts @@ -69,9 +69,9 @@ class AssetDBManager extends EventEmitter { private state: RefreshState = 'free'; public assetDBInfo: Record = {}; private waitingTaskQueue: IWaitingTaskInfo[] = []; - private waitingRefreshAsset: string[] = []; - private pendingAutoRefreshResolves: Function[] = []; - private autoRefreshTimer?: NodeJS.Timeout; + private waringRefreshAsset: string[] = []; + private autoRefreshAssetLazyPending = false; + private waringRefreshAssetPendingMap = new Map(); private get assetBusy() { return this.assetBusyTask.size > 0; } @@ -492,26 +492,44 @@ class AssetDBManager extends EventEmitter { * 懒刷新资源,请勿使用,目前的逻辑是针对重刷文件夹定制的 * @param file */ - public async autoRefreshAssetLazy(pathOrUrlOrUUID: string) { - if (!this.waitingRefreshAsset.includes(pathOrUrlOrUUID)) { - this.waitingRefreshAsset.push(pathOrUrlOrUUID); - } - - this.autoRefreshTimer && clearTimeout(this.autoRefreshTimer); - return new Promise((resolve) => { - this.pendingAutoRefreshResolves.push(resolve); - this.autoRefreshTimer = setTimeout(async () => { - const taskId = 'autoRefreshAssetLazy' + Date.now(); - this.assetBusyTask.add(taskId); - const files = JSON.parse(JSON.stringify(this.waitingRefreshAsset)); - this.waitingRefreshAsset.length = 0; - await Promise.all(files.map((file: string) => assetdb.refresh(file))); - this.assetBusyTask.delete(taskId); - this.step(); - this.pendingAutoRefreshResolves.forEach((resolve) => resolve(true)); - this.pendingAutoRefreshResolves.length = 0; - }, 100); + public autoRefreshAssetLazy(pathOrUrlOrUUID: string): Promise { + if (!this.waringRefreshAsset.includes(pathOrUrlOrUUID)) { + this.waringRefreshAsset.push(pathOrUrlOrUUID); + } + + const promise = new Promise((resolve) => { + const pending = this.waringRefreshAssetPendingMap.get(pathOrUrlOrUUID) || []; + pending.push(resolve); + this.waringRefreshAssetPendingMap.set(pathOrUrlOrUUID, pending); }); + + if (this.autoRefreshAssetLazyPending) { + return promise; + } + + this.autoRefreshAssetLazyPending = true; + void (async () => { + try { + while (this.waringRefreshAsset.length > 0) { + const files = Array.from(this.waringRefreshAsset); + this.waringRefreshAsset.length = 0; + const taskId = 'autoRefreshAssetLazy' + Date.now(); + this.assetBusyTask.add(taskId); + await Promise.all(files.map((file) => assetdb.refresh(file))); + this.assetBusyTask.delete(taskId); + this.step(); + + files.forEach((file) => { + const pending = this.waringRefreshAssetPendingMap.get(file); + pending?.forEach((resolve) => resolve(true)); + this.waringRefreshAssetPendingMap.delete(file); + }); + } + } finally { + this.autoRefreshAssetLazyPending = false; + } + })(); + return promise; } /** diff --git a/src/core/assets/test/auto-refresh-asset-lazy.test.ts b/src/core/assets/test/auto-refresh-asset-lazy.test.ts new file mode 100644 index 000000000..518b1cb73 --- /dev/null +++ b/src/core/assets/test/auto-refresh-asset-lazy.test.ts @@ -0,0 +1,55 @@ +import * as assetdb from '@cocos/asset-db'; +import assetDBManager from '../manager/asset-db'; + +describe('AssetDBManager.autoRefreshAssetLazy', () => { + const manager = assetDBManager as any; + + beforeEach(() => { + manager.assetBusyTask.clear(); + manager.waringRefreshAsset.length = 0; + manager.autoRefreshAssetLazyPending = false; + manager.waringRefreshAssetPendingMap.clear(); + jest.spyOn(manager, 'step').mockResolvedValue(undefined); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('starts immediately and resolves queued calls after each refresh', async () => { + const releases: Record void> = {}; + let second!: Promise; + let third!: Promise; + jest.spyOn(assetdb, 'refresh').mockImplementation(async (file) => { + if (file === 'first') { + second = manager.autoRefreshAssetLazy('second'); + third = manager.autoRefreshAssetLazy('third'); + } + await new Promise((resolve) => { + releases[file] = resolve; + }); + return 0; + }); + const refresh = assetdb.refresh as jest.Mock; + + const first = manager.autoRefreshAssetLazy('first'); + await Promise.resolve(); + expect(refresh).toHaveBeenCalledWith('first'); + + releases.first(); + await new Promise((resolve) => setImmediate(resolve)); + expect(refresh.mock.calls.map(([file]) => file)).toEqual(['first', 'second', 'third']); + + releases.second(); + await new Promise((resolve) => setImmediate(resolve)); + let secondResolved = false; + void second.then(() => { secondResolved = true; }); + expect(secondResolved).toBe(false); + + releases.third(); + await expect(second).resolves.toBe(true); + await expect(third).resolves.toBe(true); + await expect(first).resolves.toBe(true); + expect(refresh).toHaveBeenCalledTimes(3); + }); +}); diff --git a/src/core/scene/scene-process/main.ts b/src/core/scene/scene-process/main.ts index 47d7ba8e2..cf508741a 100644 --- a/src/core/scene/scene-process/main.ts +++ b/src/core/scene/scene-process/main.ts @@ -1,3 +1,4 @@ +import '../../../lib/runtime-module-cache'; import { SceneReadyChannel } from '../common'; import { Rpc } from './rpc'; import { parseCommandLineArgs, resolveSceneAssetBase } from './utils'; @@ -5,6 +6,7 @@ import { Engine } from '../../engine'; import { join } from 'path'; import { serviceManager } from './service/service-manager'; import { installSceneEditorShim } from './editor-shim'; +import { StartupCpuProfiler } from './startup-cpu-profiler'; async function startup() { // 监听进程退出事件 @@ -69,7 +71,9 @@ async function startup() { console.log(`[Scene] startup worker success, cocos version: ${cc.ENGINE_VERSION}`); } -startup().catch(err => { - console.error('[Scene] Startup fatal error:', err); - process.exit(1); -}); +new StartupCpuProfiler('VSCODE_COCOS_SCENE_PROCESS_CPU_PROFILE') + .run(startup) + .catch(err => { + console.error('[Scene] Startup fatal error:', err); + process.exit(1); + }); diff --git a/src/core/scene/scene-process/startup-cpu-profiler.test.ts b/src/core/scene/scene-process/startup-cpu-profiler.test.ts new file mode 100644 index 000000000..a3da5c13b --- /dev/null +++ b/src/core/scene/scene-process/startup-cpu-profiler.test.ts @@ -0,0 +1,63 @@ +import { Session } from 'inspector'; +import * as fs from 'fs'; +import { StartupCpuProfiler } from './startup-cpu-profiler'; + +jest.mock('inspector', () => ({ + Session: jest.fn().mockImplementation(() => ({ + connect: jest.fn(), + disconnect: jest.fn(), + post: jest.fn((method: string, callback: (error: Error | null, params?: object) => void) => { + if (method === 'Profiler.stop') { + callback(null, { profile: { nodes: [] } }); + } else { + callback(null); + } + }), + })), +})); + +jest.mock('fs', () => ({ + writeFileSync: jest.fn(), +})); + +const profileEnvKey = 'VSCODE_COCOS_SCENE_PROCESS_CPU_PROFILE'; +const originalProfilePath = process.env[profileEnvKey]; + +afterEach(() => { + jest.clearAllMocks(); + if (originalProfilePath === undefined) { + delete process.env[profileEnvKey]; + } else { + process.env[profileEnvKey] = originalProfilePath; + } +}); + +it('runs transparently when the environment variable is not a .cpuprofile path', async () => { + delete process.env[profileEnvKey]; + const task = jest.fn(async () => 'ready'); + + const result = await new StartupCpuProfiler(profileEnvKey).run(task); + + expect(result).toBe('ready'); + expect(task).toHaveBeenCalledTimes(1); + expect(Session).not.toHaveBeenCalled(); +}); + +it('records startup and writes the profile to the configured path', async () => { + const outputPath = '/tmp/scene-process.cpuprofile'; + process.env[profileEnvKey] = outputPath; + const task = jest.fn(async () => 'ready'); + + const result = await new StartupCpuProfiler(profileEnvKey).run(task); + + expect(result).toBe('ready'); + expect(task).toHaveBeenCalledTimes(1); + expect(Session).toHaveBeenCalledTimes(1); + const session = (Session as unknown as jest.Mock).mock.results[0].value; + expect(session.connect).toHaveBeenCalledTimes(1); + expect(session.post).toHaveBeenNthCalledWith(1, 'Profiler.enable', expect.any(Function)); + expect(session.post).toHaveBeenNthCalledWith(2, 'Profiler.start', expect.any(Function)); + expect(session.post).toHaveBeenNthCalledWith(3, 'Profiler.stop', expect.any(Function)); + expect(fs.writeFileSync).toHaveBeenCalledWith(outputPath, JSON.stringify({ nodes: [] })); + expect(session.disconnect).toHaveBeenCalledTimes(1); +}); diff --git a/src/core/scene/scene-process/startup-cpu-profiler.ts b/src/core/scene/scene-process/startup-cpu-profiler.ts new file mode 100644 index 000000000..3d9522e10 --- /dev/null +++ b/src/core/scene/scene-process/startup-cpu-profiler.ts @@ -0,0 +1,94 @@ +import { Session } from 'inspector'; +import * as fs from 'fs'; + +/** + * Records the scene-process startup CPU profile when an output path is provided. + */ +export class StartupCpuProfiler { + private session: Session | undefined; + private readonly outputPath: string; + private started = false; + private finished = false; + private startPromise: Promise = Promise.resolve(); + + constructor(envKey: string) { + this.outputPath = process.env[envKey] ?? ''; + } + + get enabled(): boolean { + return this.outputPath.endsWith('.cpuprofile'); + } + + async run(task: () => Promise): Promise { + this.start(); + try { + return await task(); + } finally { + try { + await this.finish(); + } catch (error) { + console.error('[Scene] Failed to finish CPU profile:', error); + } + } + } + + private start(): void { + if (!this.enabled || this.started) { + return; + } + this.started = true; + const session = this.session = new Session(); + session.connect(); + this.startPromise = new Promise((resolve, reject) => { + session.post('Profiler.enable', (error) => { + if (error) { + reject(error); + return; + } + session.post('Profiler.start', (startError) => { + if (startError) { + reject(startError); + return; + } + resolve(); + }); + }); + }); + } + + private async finish(): Promise { + if (!this.started || this.finished) { + return; + } + this.finished = true; + const session = this.session; + if (!session) { + return; + } + try { + await this.startPromise; + await new Promise((resolve, reject) => { + session.post('Profiler.stop', (error, params) => { + if (error) { + reject(error); + return; + } + const profile = (params as { profile?: object } | undefined)?.profile; + if (!profile) { + reject(new Error('[Scene] Profiler.stop returned no profile')); + return; + } + try { + fs.writeFileSync(this.outputPath, JSON.stringify(profile)); + console.log(`[Scene] CPU profile written to ${this.outputPath}`); + } catch (writeError) { + console.error('[Scene] Failed to write CPU profile:', writeError); + } + resolve(); + }); + }); + } finally { + session.disconnect(); + } + } +} diff --git a/src/lib/runtime-module-cache.ts b/src/lib/runtime-module-cache.ts index 0ef3398d8..8d80d5940 100644 --- a/src/lib/runtime-module-cache.ts +++ b/src/lib/runtime-module-cache.ts @@ -85,6 +85,12 @@ interface CompiledModule { (...args: unknown[]): unknown; } +interface CompiledModuleCacheEntry { + mtimeMs: number; + sourceLength: number; + compiled: CompiledModule; +} + /** * Caches runtime-bundle source and V8 CommonJS compiled data at the module * resolution/load boundaries. @@ -110,6 +116,7 @@ export class RuntimeBundler { private cacheResolutions = new Map(); private pendingModules = new Map(); private pendingResolutions = new Map(); + private compiledModules = new Map(); private freshnessChecks = new Map>(); private resolutionChecks = new Map>(); private flushPromise: Promise | undefined; @@ -172,6 +179,7 @@ export class RuntimeBundler { this.originalModuleLoad = undefined; this.hookedResolveFilename = undefined; this.originalResolveFilename = undefined; + this.compiledModules.clear(); this.installed = false; } @@ -246,7 +254,7 @@ export class RuntimeBundler { const pending = this.pendingModules.get(filePath); if (pending) { this.scheduleFreshnessCheck(filePath, pending.mtimeMs); - this.executeCompiledModule(module, filename, pending.source, pending.codeCache); + this.executeCompiledModule(module, filename, pending.source, pending.codeCache, pending.mtimeMs); return; } @@ -255,7 +263,7 @@ export class RuntimeBundler { const content = this.getCachedModule(cached); if (content) { this.scheduleFreshnessCheck(filePath, cached.mtimeMs); - const codeCache = this.executeCompiledModule(module, filename, content.source, content.codeCache); + const codeCache = this.executeCompiledModule(module, filename, content.source, content.codeCache, cached.mtimeMs); if (codeCache !== undefined) { this.recordAsync(filePath, content.source, codeCache, cached.mtimeMs); } @@ -274,7 +282,7 @@ export class RuntimeBundler { originalModuleLoad.call(module, filename); return; } - const codeCache = this.executeCompiledModule(module, filename, content); + const codeCache = this.executeCompiledModule(module, filename, content, undefined, mtimeMs); this.recordAsync(filePath, content, codeCache ?? Buffer.alloc(0), mtimeMs); } @@ -306,15 +314,38 @@ export class RuntimeBundler { } const source = await fs.promises.readFile(filePath); - const codeCache = createCompiledData(source, filePath); const currentPending = this.pendingModules.get(filePath); if (currentPending && currentPending.mtimeMs > stats.mtimeMs) { return; } - this.recordAsync(filePath, source, codeCache, stats.mtimeMs); - } - - private executeCompiledModule(module: RuntimeModule, filename: string, source: Buffer, cachedData?: Buffer): Buffer | undefined { + this.compiledModules.delete(filePath); + this.recordAsync(filePath, source, Buffer.alloc(0), stats.mtimeMs); + } + + /* + ```mermaid + sequenceDiagram + participant Load as Module.load + participant Memory as compiledModules + participant V8 as vm.compileFunction + participant Disk as idle refresh + Load->>Memory: lookup by path, mtime and source length + alt compiled wrapper exists + Memory-->>Load: reuse Function + else wrapper missing + Load->>V8: compile once with cachedData when available + V8-->>Memory: store executable Function + end + Disk-->>Memory: invalidate only after source changes + ``` + */ + private executeCompiledModule( + module: RuntimeModule, + filename: string, + source: Buffer, + cachedData: Buffer | undefined, + mtimeMs: number, + ): Buffer | undefined { // Module.load normally initializes these before invoking the extension. module.filename ??= filename; module.paths ??= nodeModule._nodeModulePaths(dirname(filename)); @@ -325,16 +356,27 @@ export class RuntimeBundler { } const sourceText = stripShebang(source.toString('utf8')); - let compiled = compileCommonJs(sourceText, filename, cachedData); - const cachedDataRejected = compiled.cachedDataRejected === true; - if (cachedDataRejected) { - compiled = compileCommonJs(sourceText, filename); + const cacheKey = resolve(filename); + const reusableCachedData = cachedData && cachedData.length > 0 ? cachedData : undefined; + const existing = this.compiledModules.get(cacheKey); + let compiled = existing && existing.mtimeMs === mtimeMs && existing.sourceLength === source.length + ? existing.compiled + : undefined; + let cachedDataRejected = false; + if (!compiled) { + compiled = compileCommonJs(sourceText, filename, reusableCachedData); + cachedDataRejected = compiled.cachedDataRejected === true; + this.compiledModules.set(cacheKey, { + mtimeMs, + sourceLength: source.length, + compiled, + }); } const moduleExports = module.exports; const localRequire = createModuleRequire(module); Reflect.apply(compiled, moduleExports, [moduleExports, localRequire, module, filename, dirname(filename)]); module.loaded = true; - if (cachedData !== undefined && !cachedDataRejected) { + if (compiled === existing?.compiled || (reusableCachedData !== undefined && !cachedDataRejected)) { return undefined; } return compiled.cachedData ?? Buffer.alloc(0); @@ -873,14 +915,6 @@ function compileCommonJs(source: string, filename: string, cachedData?: Buffer): }) as unknown as CompiledModule; } -function createCompiledData(source: Buffer, filename: string): Buffer { - if (getModuleExtension(filename) === '.json') { - return Buffer.alloc(0); - } - const compiled = compileCommonJs(stripShebang(source.toString('utf8')), filename); - return compiled.cachedData ?? Buffer.alloc(0); -} - function createModuleRequire(module: RuntimeModule): NodeRequire { const localRequire = module.require.bind(module) as NodeRequire; localRequire.resolve = ((request: string, options?: { paths?: string[] }): string => ( diff --git a/tests/runtime-bundle.test.ts b/tests/runtime-bundle.test.ts index 9a6d17bdd..ca287e524 100644 --- a/tests/runtime-bundle.test.ts +++ b/tests/runtime-bundle.test.ts @@ -299,6 +299,38 @@ describe('RuntimeBundler', () => { compileFunction.mockClear(); }); + it('executes once when persisted V8 cached data is rejected', async () => { + const sourcePath = join(tempRoot, 'runtime', 'index.js'); + const cachePath = join(tempRoot, '.runtime-bundle-cache'); + mkdirSync(join(tempRoot, 'runtime'), { recursive: true }); + createModule(sourcePath, 'rejected-cache'); + + runtimeBundler.uninstall(); + bundler = new RuntimeBundler({ cachePath }); + bundler.install(); + expect(requireFresh(sourcePath)).toBe('rejected-cache'); + await bundler.flush(); + bundler.uninstall(); + + const cache = readFileSync(cachePath); + const mappingLength = cache.readUInt32LE(12); + const mapping = JSON.parse(cache.subarray(16, 16 + mappingLength).toString('utf8')); + const entry = mapping.modules[sourcePath]; + const codeCacheOffset = 16 + mappingLength + entry.codeCacheOffset; + const corruptedCache = Buffer.from(cache); + corruptedCache[codeCacheOffset] ^= 0xff; + writeFileSync(cachePath, corruptedCache); + + bundler = new RuntimeBundler({ cachePath }); + const compileFunction = vm.compileFunction as jest.MockedFunction; + compileFunction.mockClear(); + bundler.install(); + + expect(requireFresh(sourcePath)).toBe('rejected-cache'); + expect(compileFunction).toHaveBeenCalledTimes(1); + compileFunction.mockClear(); + }); + it('returns cached content before asynchronously checking and refreshing a changed source', async () => { const sourcePath = join(tempRoot, 'runtime', 'index.js'); const cachePath = join(tempRoot, '.runtime-bundle-cache');