diff --git a/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap b/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap index 873036f4c0..725211ba84 100644 --- a/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap +++ b/packages/metro-file-map/src/__tests__/__snapshots__/index-test.js.snap @@ -16,7 +16,7 @@ exports[`FileMap file system changes processing recovery from duplicate module I " `; -exports[`FileMap throws on duplicate module ids if "throwOnModuleCollision" is set to true 1`] = ` +exports[`FileMap throws on duplicate module ids if "failValidationOnConflicts" is set to true 1`] = ` "Advice: Resolve conflicts of type \\"duplicate\\" by renaming one or both of the conflicting modules, or by excluding conflicting paths from Haste. 1. Strawberry (duplicate) diff --git a/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js b/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js index 72ab125d69..981a9cb8e2 100644 --- a/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js +++ b/packages/metro-file-map/src/__tests__/includes_dotfiles-test.js @@ -19,7 +19,6 @@ const rootDir = path.join(__dirname, './test_dotfiles_root'); const commonOptions = { extensions: ['js'], maxWorkers: 1, - platforms: [], resetCache: true, retainAllFiles: true, rootDir, diff --git a/packages/metro-file-map/src/__tests__/index-test.js b/packages/metro-file-map/src/__tests__/index-test.js index 6db43b0238..2608f98f40 100644 --- a/packages/metro-file-map/src/__tests__/index-test.js +++ b/packages/metro-file-map/src/__tests__/index-test.js @@ -11,14 +11,20 @@ import type {InputOptions} from '..'; import type { + BuildResult, ChangeEvent, ChangeEventMetadata, FileData, FileMetadata, FileSystem, + HasteMap, + MockMap, WatcherBackendOptions, WorkerSetupArgs, } from '../flow-types'; +import type {default as FileMapT} from '../index'; +import type {HasteMapOptions} from '../plugins/HastePlugin'; +import type {MockMapOptions} from '../plugins/MockPlugin'; import typeof WorkerModule from '../worker'; import {AbstractWatcher} from '../watchers/AbstractWatcher'; @@ -94,7 +100,7 @@ jest.mock('../crawlers/watchman', () => ({ '', // dependencies hash, typeof contentOrLink !== 'string' ? 1 : 0, - '', // Haste name + null, // Haste name ]); } } else { @@ -251,6 +257,16 @@ let mockClocks; let mockEmitters: {[root: string]: MockWatcher, __proto__: null}; let mockEnd; let mockProcessFile; +let buildNewFileMap: ( + overrides?: Partial, + hasteOverrides?: Partial, + mocksOverrides?: Partial, +) => Promise<{ + ...BuildResult, + fileMap: FileMapT, + hasteMap: HasteMap, + mockMap: ?MockMap, +}>; let cacheContent = null; describe('FileMap', () => { @@ -322,7 +338,6 @@ describe('FileMap', () => { defaultConfig = { enableSymlinks: false, extensions: ['js', 'json'], - hasteImplModulePath, healthCheck: { enabled: false, interval: 10000, @@ -330,7 +345,6 @@ describe('FileMap', () => { filePrefix: '.metro-file-map-health-check', }, maxWorkers: 1, - platforms: ['ios', 'android'], resetCache: false, retainAllFiles: false, rootDir: path.join('/', 'project'), @@ -341,6 +355,49 @@ describe('FileMap', () => { useWatchman: true, cacheManagerFactory: () => mockCacheManager, }; + + const defaultHasteConfig: HasteMapOptions = { + console: globalThis.console, + enableHastePackages: true, + rootDir: defaultConfig.rootDir, + hasteImplModulePath, + platforms: new Set(['ios', 'android']), + failValidationOnConflicts: false, + }; + + const defaultMockConfig: MockMapOptions = { + console: globalThis.console, + rootDir: defaultConfig.rootDir, + mocksPattern: /__mocks__/, + throwOnModuleCollision: false, + }; + + buildNewFileMap = async ( + overrides = {}, + hasteOverrides = {}, + mockOverrides = {}, + ) => { + const hasteMap = new (require('../plugins/HastePlugin').default)({ + ...defaultHasteConfig, + ...hasteOverrides, + }); + const mockMap = new (require('../plugins/MockPlugin').default)({ + ...defaultMockConfig, + ...mockOverrides, + }); + const fileMap = new FileMap({ + ...defaultConfig, + ...overrides, + plugins: [hasteMap, mockMap], + }); + const {fileSystem} = await fileMap.build(); + return { + fileMap, + fileSystem, + hasteMap, + mockMap, + }; + }; }); afterEach(() => { @@ -355,11 +412,10 @@ describe('FileMap', () => { }); test('ignores files given a pattern', async () => { - const config = {...defaultConfig, ignorePattern: /Kiwi/}; mockFs[path.join('/', 'project', 'fruits', 'Kiwi.js')] = ` // Kiwi! `; - const {fileSystem} = await new FileMap(config).build(); + const {fileSystem} = await buildNewFileMap({ignorePattern: /Kiwi/}); expect([...fileSystem.matchFiles({filter: /Kiwi/})]).toEqual([]); }); @@ -367,32 +423,30 @@ describe('FileMap', () => { mockFs[path.join('/', 'project', 'fruits', '.git', 'fruit-history.js')] = ` // test `; - const {fileSystem} = await new FileMap(defaultConfig).build(); + const {fileSystem} = await buildNewFileMap(); expect([...fileSystem.matchFiles({filter: /\.git/})]).toEqual([]); }); test('ignores vcs directories with ignore pattern regex', async () => { - const config = {...defaultConfig, ignorePattern: /Kiwi/}; mockFs[path.join('/', 'project', 'fruits', 'Kiwi.js')] = ` // Kiwi! `; mockFs[path.join('/', 'project', 'fruits', '.git', 'fruit-history.js')] = ` // test `; - const {fileSystem} = await new FileMap(config).build(); + const {fileSystem} = await buildNewFileMap({ignorePattern: /Kiwi/}); expect([...fileSystem.matchFiles({filter: /Kiwi/})]).toEqual([]); expect([...fileSystem.matchFiles({filter: /\.git/})]).toEqual([]); }); test('throw on ignore pattern except for regex', async () => { - const config = {ignorePattern: 'Kiwi', ...defaultConfig}; mockFs['/project/fruits/Kiwi.js'] = ` // Kiwi! `; try { - // $FlowExpectedError[incompatible-type] - await new FileMap(config).build(); + // $FlowExpectedError[incompatible-type] testing runtime validation + await buildNewFileMap({ignorePattern: 'Kiwi'}); } catch (err) { expect(err.message).toBe( 'metro-file-map: the `ignorePattern` option must be a RegExp', @@ -478,12 +532,13 @@ describe('FileMap', () => { // fbjs2 `; - const fileMap = new FileMap({ - ...defaultConfig, - mocksPattern: '__mocks__', - }); - - const {fileSystem, hasteMap, mockMap} = await fileMap.build(); + const {fileMap, fileSystem, hasteMap, mockMap} = await buildNewFileMap( + {}, + {}, + { + mocksPattern: /__mocks__/, + }, + ); expect(cacheContent?.clocks).toEqual(mockClocks); @@ -524,7 +579,7 @@ describe('FileMap', () => { 'Melon', null, 0, - '', + null, ], [path.join('vegetables', 'Melon.js')]: [ 32, @@ -582,7 +637,7 @@ describe('FileMap', () => { // $FlowFixMe[missing-local-annot] node.mockImplementation(options => { // The node crawler returns "null" for the SHA-1. - const changedFiles = createMap({ + const changedFiles = createMap({ [path.join('fruits', 'Banana.js')]: [ 32, 42, @@ -617,7 +672,7 @@ describe('FileMap', () => { 'Melon', null, 0, - '', + null, ], [path.join('vegetables', 'Melon.js')]: [ 32, @@ -637,7 +692,7 @@ describe('FileMap', () => { '', null, 1, - '', + null, ], } : null), @@ -649,16 +704,13 @@ describe('FileMap', () => { }); }); - const fileMap = new FileMap({ - ...defaultConfig, + const {fileMap} = await buildNewFileMap({ computeSha1: true, maxWorkers: 1, enableSymlinks, useWatchman, }); - await fileMap.build(); - expect( createMap({ [path.join('fruits', 'Banana.js')]: [ @@ -695,7 +747,7 @@ describe('FileMap', () => { 'Melon', '8d40afbb6e2dc78e1ba383b6d02cafad35cceef2', 0, - '', + null, ], [path.join('vegetables', 'Melon.js')]: [ 32, @@ -714,7 +766,7 @@ describe('FileMap', () => { 1, '', null, - '', + null, ], } : null), @@ -733,7 +785,7 @@ describe('FileMap', () => { `, }); - const originalData = await new FileMap(defaultConfig).build(); + const originalData = await buildNewFileMap(); // Haste Melon present in its original location. expect(originalData.hasteMap.getModule('Melon')).toEqual( @@ -748,7 +800,7 @@ describe('FileMap', () => { [path.join('/', 'project', 'vegetables', 'Melon.js')]: null, // Mock deletion }); - const newData = await new FileMap(defaultConfig).build(); + const newData = await buildNewFileMap(); expect(console.warn).not.toHaveBeenCalled(); expect(console.error).not.toHaveBeenCalled(); @@ -764,14 +816,10 @@ describe('FileMap', () => { module.exports = require("./video.mp4"); `; - const fileMap = new FileMap({ - ...defaultConfig, - extensions: [...defaultConfig.extensions], + const {fileSystem, hasteMap} = await buildNewFileMap({ roots: [...defaultConfig.roots, path.join('/', 'project', 'video')], }); - const {fileSystem, hasteMap} = await fileMap.build(); - expect(hasteMap.getModule('IRequireAVideo')).toEqual( path.join(defaultConfig.rootDir, 'video', 'IRequireAVideo.js'), ); @@ -793,13 +841,11 @@ describe('FileMap', () => { // fbjs! `; - const fileMap = new FileMap({ - ...defaultConfig, - mocksPattern: '__mocks__', - retainAllFiles: true, - }); - - const {fileSystem, hasteMap} = await fileMap.build(); + const {fileSystem, hasteMap} = await buildNewFileMap( + {retainAllFiles: true}, + {}, + {mocksPattern: /__mocks__/}, + ); // Expect the node module to be part of files but make sure it wasn't // read. @@ -826,26 +872,19 @@ describe('FileMap', () => { ); mockFs[pathToMock] = '/* empty */'; - const {mockMap} = await new FileMap({ - mocksPattern: '__mocks__', - throwOnModuleCollision: true, - ...defaultConfig, - }).build(); + const {mockMap} = await buildNewFileMap( + {}, + {}, + { + mocksPattern: /__mocks__/, + throwOnModuleCollision: true, + }, + ); expect(mockMap).not.toBeNull(); expect(mockMap?.getMockModule('Blueberry')).toEqual(pathToMock); }); - test('returns null mockMap if mocksPattern is empty', async () => { - const {mockMap} = await new FileMap({ - mocksPattern: '', - throwOnModuleCollision: true, - ...defaultConfig, - }).build(); - - expect(mockMap).toBeNull(); - }); - test('throws on duplicate mock files when throwOnModuleCollision', async () => { // Duplicate mock files for blueberry mockFs[ @@ -881,15 +920,24 @@ describe('FileMap', () => { ' * /../../fruits2/__mocks__/subdir/Blueberry.js\n'; await expect(() => - new FileMap({ - mocksPattern: '__mocks__', - throwOnModuleCollision: true, - ...defaultConfig, - console: { - ...globalThis.console, - warn: mockWarn, + buildNewFileMap( + {}, + { + console: { + ...globalThis.console, + warn: mockWarn, + }, + failValidationOnConflicts: true, }, - }).build(), + { + console: { + ...globalThis.console, + warn: mockWarn, + }, + mocksPattern: /__mocks__/, + throwOnModuleCollision: true, + }, + ), ).rejects.toThrowError('Mock map has 1 error:\n' + expectedError); expect(mockWarn).toHaveBeenCalledWith(expectedError); }); @@ -899,7 +947,7 @@ describe('FileMap', () => { const Banana = require("Banana"); `; - const {hasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap} = await buildNewFileMap(); expect(() => hasteMap.getModule('Strawberry')).toThrow( DuplicateHasteCandidatesError, @@ -911,7 +959,7 @@ describe('FileMap', () => { ).toMatchSnapshot(); }); - test('throws on duplicate module ids if "throwOnModuleCollision" is set to true', async () => { + test('throws on duplicate module ids if "failValidationOnConflicts" is set to true', async () => { expect.assertions(2); // Raspberry thinks it is a Strawberry mockFs[path.join('/', 'project', 'fruits', 'another', 'Strawberry.js')] = ` @@ -919,10 +967,7 @@ describe('FileMap', () => { `; try { - await new FileMap({ - throwOnModuleCollision: true, - ...defaultConfig, - }).build(); + await buildNewFileMap({}, {failValidationOnConflicts: true}); } catch (err) { expect(err).toBeInstanceOf(HasteConflictsError); expect(err.getDetailedMessage()).toMatchSnapshot(); @@ -944,7 +989,7 @@ describe('FileMap', () => { const Blackberry = require("Blackberry"); `; - const {fileSystem, hasteMap} = await new FileMap(defaultConfig).build(); + const {fileSystem, hasteMap} = await buildNewFileMap(); assertFileSystemEqual( fileSystem, @@ -993,7 +1038,7 @@ describe('FileMap', () => { }); test('does not access the file system on a warm cache with no changes', async () => { - await new FileMap(defaultConfig).build(); + await buildNewFileMap(); const initialData = cacheContent; // First run should attempt to read the cache, but there will be no result @@ -1018,7 +1063,7 @@ describe('FileMap', () => { vegetables: 'c:fake-clock:4', }); - await new FileMap(defaultConfig).build(); + await buildNewFileMap(); const data = cacheContent; // Expect the cache to have been read again @@ -1034,9 +1079,7 @@ describe('FileMap', () => { test('only does minimal file system access when files change', async () => { // Run with a cold cache initially - const {fileSystem: initialFileSystem} = await new FileMap( - defaultConfig, - ).build(); + const {fileSystem: initialFileSystem} = await buildNewFileMap(); expect( initialFileSystem.getDependencies(path.join('fruits', 'Banana.js')), @@ -1059,7 +1102,7 @@ describe('FileMap', () => { vegetables: 'c:fake-clock:2', }); - const {fileSystem} = await new FileMap(defaultConfig).build(); + const {fileSystem} = await buildNewFileMap(); const data = cacheContent; expect(mockCacheManager.read).toHaveBeenCalledTimes(2); @@ -1076,7 +1119,7 @@ describe('FileMap', () => { }); test('correctly handles file deletions', async () => { - await new FileMap(defaultConfig).build(); + await buildNewFileMap(); // $FlowFixMe[incompatible-type] fs.readFileSync.mockClear(); @@ -1091,7 +1134,7 @@ describe('FileMap', () => { fruits: 'c:fake-clock:3', vegetables: 'c:fake-clock:2', }); - const {fileSystem, hasteMap} = await new FileMap(defaultConfig).build(); + const {fileSystem, hasteMap} = await buildNewFileMap(); expect(fileSystem.exists(path.join('fruits', 'Banana.js'))).toEqual(false); expect(hasteMap.getModule('Banana')).toBeNull(); @@ -1103,7 +1146,7 @@ describe('FileMap', () => { mockFs[path.join('/', 'project', 'fruits', 'Strawberry.js')] = ` const Banana = require("Banana"); `; - const {hasteMap: firstHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: firstHasteMap} = await buildNewFileMap(); // Generic and ios return the generic implementation. expect(firstHasteMap.getModule('Strawberry')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.js'), @@ -1119,7 +1162,7 @@ describe('FileMap', () => { `, }); mockClocks = createMap({fruits: 'c:fake-clock:3'}); - const {hasteMap: secondHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: secondHasteMap} = await buildNewFileMap(); expect(secondHasteMap.getModule('Strawberry')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.js'), ); @@ -1138,7 +1181,7 @@ describe('FileMap', () => { mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')] = ` const Raspberry = require("Raspberry"); `; - const {hasteMap: firstHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: firstHasteMap} = await buildNewFileMap(); expect(firstHasteMap.getModule('Strawberry', 'ios')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.ios.js'), ); @@ -1152,7 +1195,7 @@ describe('FileMap', () => { [path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]: null, }); mockClocks = createMap({fruits: 'c:fake-clock:3'}); - const {hasteMap: secondHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: secondHasteMap} = await buildNewFileMap(); // Expect both ios and generic return generic. expect(secondHasteMap.getModule('Strawberry', 'ios')).toEqual( @@ -1168,7 +1211,7 @@ describe('FileMap', () => { [path.join('/', 'project', 'fruits', 'Strawberry.js')]: null, }); mockClocks = createMap({fruits: 'c:fake-clock:4'}); - const {hasteMap: thirdHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: thirdHasteMap} = await buildNewFileMap(); // No implementation of Strawberry remains. expect(thirdHasteMap.getModule('Strawberry', 'ios')).toBeNull(); @@ -1180,7 +1223,7 @@ describe('FileMap', () => { mockFs[path.join('/', 'project', 'fruits', 'Strawberry.ios.js')] = ` const Raspberry = require("Raspberry"); `; - const {hasteMap: firstHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: firstHasteMap} = await buildNewFileMap(); expect(firstHasteMap.getModule('Strawberry', 'ios')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.ios.js'), ); @@ -1195,7 +1238,7 @@ describe('FileMap', () => { [path.join('/', 'project', 'fruits', 'Strawberry.ios.js')]: null, }); mockClocks = createMap({fruits: 'c:fake-clock:3'}); - const {hasteMap: secondHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: secondHasteMap} = await buildNewFileMap(); expect(secondHasteMap.getModule('Strawberry')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.js'), ); @@ -1216,7 +1259,7 @@ describe('FileMap', () => { mockFs[path.join('/', 'project', 'fruits', 'another', 'Banana.ios.js')] = '//'; - const {hasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap} = await buildNewFileMap(); expect(() => hasteMap.getModule('Strawberry')).toThrow( new DuplicateHasteCandidatesError( 'Strawberry', @@ -1276,7 +1319,7 @@ describe('FileMap', () => { vegetables: 'c:fake-clock:2', }); - const {hasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap} = await buildNewFileMap(); expect(hasteMap.getModule('Strawberry')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.js'), @@ -1299,7 +1342,7 @@ describe('FileMap', () => { vegetables: 'c:fake-clock:2', }); - const {hasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap} = await buildNewFileMap(); expect(hasteMap.getModule('Banana')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Banana.js'), ); @@ -1318,9 +1361,7 @@ describe('FileMap', () => { {"name": "Strawberry"} `; - const {hasteMap: initialHasteMap} = await new FileMap( - defaultConfig, - ).build(); + const {hasteMap: initialHasteMap} = await buildNewFileMap(); let initialStrawberryError; try { @@ -1380,7 +1421,7 @@ describe('FileMap', () => { fruits: 'c:fake-clock:4', }); - const {hasteMap: newHasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap: newHasteMap} = await buildNewFileMap(); expect(newHasteMap.getModule('Strawberry')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.js'), @@ -1399,7 +1440,7 @@ describe('FileMap', () => { vegetables: 'c:fake-clock:2', }); - const {hasteMap} = await new FileMap(defaultConfig).build(); + const {hasteMap} = await buildNewFileMap(); expect(hasteMap.getModule('Strawberry')).toEqual( path.join(defaultConfig.rootDir, 'fruits', 'Strawberry.js'), ); @@ -1422,14 +1463,14 @@ describe('FileMap', () => { // $FlowFixMe[missing-local-annot] watchman.mockImplementation(async options => { const {changedFiles} = await mockImpl(options); - changedFiles.set(invalidFilePath, [34, 44, 0, '', null, 0, '']); + changedFiles.set(invalidFilePath, [34, 44, 0, '', null, 0, null]); return { changedFiles, removedFiles: new Set(), }; }); - const {fileSystem} = await new FileMap(defaultConfig).build(); + const {fileSystem} = await buildNewFileMap(); expect(fileSystem.getDifference(new Map()).removedFiles.size).toBe(5); // Ensure this file is not part of the file list. @@ -1440,13 +1481,16 @@ describe('FileMap', () => { const jestWorker = require('jest-worker').Worker; const path = require('path'); const dependencyExtractor = path.join(__dirname, 'dependencyExtractor.js'); - await new FileMap({ - ...defaultConfig, - dependencyExtractor, - hasteImplModulePath: undefined, - maxWorkers: 4, - maxFilesPerWorker: 2, - }).build(); + await buildNewFileMap( + { + dependencyExtractor, + maxWorkers: 4, + maxFilesPerWorker: 2, + }, + { + hasteImplModulePath: undefined, + }, + ); expect(jestWorker).toHaveBeenCalledTimes(1); @@ -1455,6 +1499,23 @@ describe('FileMap', () => { expect.objectContaining({ // With maxFilesPerWorker = 2 and 5 files, we should have 3 workers. numWorkers: 3, + setupArgs: [ + { + dependencyExtractor, + plugins: [ + { + match: /[/\\^]package\.json$/, + workerModulePath: expect.stringMatching( + /src[/\\]plugins[/\\]haste[/\\]worker\.js$/, + ), + workerSetupArgs: { + enableHastePackages: true, + hasteImplModulePath: null, + }, + }, + ], + }, + ], }), ); @@ -1465,10 +1526,8 @@ describe('FileMap', () => { { computeDependencies: true, computeSha1: false, - dependencyExtractor, - enableHastePackages: true, filePath: path.join('/', 'project', 'fruits', 'Banana.js'), - hasteImplModulePath: undefined, + isNodeModules: false, maybeReturnContent: false, }, ], @@ -1476,10 +1535,8 @@ describe('FileMap', () => { { computeDependencies: true, computeSha1: false, - dependencyExtractor, - enableHastePackages: true, filePath: path.join('/', 'project', 'fruits', 'Pear.js'), - hasteImplModulePath: undefined, + isNodeModules: false, maybeReturnContent: false, }, ], @@ -1487,10 +1544,8 @@ describe('FileMap', () => { { computeDependencies: true, computeSha1: false, - dependencyExtractor, - enableHastePackages: true, filePath: path.join('/', 'project', 'fruits', 'Strawberry.js'), - hasteImplModulePath: undefined, + isNodeModules: false, maybeReturnContent: false, }, ], @@ -1498,10 +1553,8 @@ describe('FileMap', () => { { computeDependencies: true, computeSha1: false, - dependencyExtractor, - enableHastePackages: true, filePath: path.join('/', 'project', 'fruits', '__mocks__', 'Pear.js'), - hasteImplModulePath: undefined, + isNodeModules: false, maybeReturnContent: false, }, ], @@ -1509,10 +1562,8 @@ describe('FileMap', () => { { computeDependencies: true, computeSha1: false, - dependencyExtractor, - enableHastePackages: true, filePath: path.join('/', 'project', 'vegetables', 'Melon.js'), - hasteImplModulePath: undefined, + isNodeModules: false, maybeReturnContent: false, }, ], @@ -1533,13 +1584,13 @@ describe('FileMap', () => { node.mockImplementation((() => { return Promise.resolve({ changedFiles: createMap({ - [path.join('fruits', 'Banana.js')]: [32, 42, 0, '', null, 0, ''], + [path.join('fruits', 'Banana.js')]: [32, 42, 0, '', null, 0, null], }), removedFiles: new Set(), }); }) as typeof node); - const {fileSystem} = await new FileMap(defaultConfig).build(); + const {fileSystem} = await buildNewFileMap(); expect(watchman).toBeCalled(); expect(node).toBeCalled(); @@ -1575,13 +1626,13 @@ describe('FileMap', () => { node.mockImplementation(() => { return Promise.resolve({ changedFiles: createMap({ - [path.join('fruits', 'Banana.js')]: [32, 42, 0, '', null, 0, ''], + [path.join('fruits', 'Banana.js')]: [32, 42, 0, '', null, 0, null], }), removedFiles: new Set(), }); }); - const {fileSystem} = await new FileMap(defaultConfig).build(); + const {fileSystem} = await buildNewFileMap(); expect(watchman).toBeCalled(); expect(node).toBeCalled(); @@ -1616,7 +1667,7 @@ describe('FileMap', () => { node.mockImplementation(() => Promise.reject(new Error('node error'))); try { - await new FileMap(defaultConfig).build(); + await buildNewFileMap(); } catch (error) { expect(error.message).toEqual( 'Crawler retry failed:\n' + @@ -1644,11 +1695,12 @@ describe('FileMap', () => { only?: boolean, mockFs?: MockFS, config?: Partial, + hasteConfig?: Partial, }>; function fm_it( title: string, - fn: (fm: FileMap) => mixed, + fn: (fm: $ReadOnly<{fileMap: FileMap, hasteMap: HasteMap}>) => mixed, options?: FileMapTestOptions = {}, ): void { options = options || {}; @@ -1656,51 +1708,53 @@ describe('FileMap', () => { if (options.mockFs) { mockFs = options.mockFs; } - const config = { - ...defaultConfig, + const {fileMap, hasteMap} = await buildNewFileMap({ watch: true, ...options.config, - }; - const hm = new FileMap(config); - await hm.build(); + }); try { - await fn(hm); + await fn({fileMap, hasteMap}); } finally { // $FlowFixMe[unused-promise] - hm.end(); + fileMap.end(); } }); } fm_it.only = ( title: string, - fn: () => mixed, + fn: (fm: $ReadOnly<{fileMap: FileMap, hasteMap: HasteMap}>) => mixed, options?: FileMapTestOptions, ): void => fm_it(title, fn, {...options, only: true}); - fm_it('build returns a "live" fileSystem and hasteMap', async hm => { - const {fileSystem, hasteMap} = await hm.build(); - const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); - expect(fileSystem.getModuleName(filePath)).toBeDefined(); - expect(hasteMap.getModule('Banana')).toBe(filePath); - mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); - mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); - const {eventsQueue} = await waitForItToChange(hm); - expect(eventsQueue).toHaveLength(1); - const deletedBanana = { - filePath, - metadata: { - modifiedTime: null, - size: null, - type: 'f', - }, - type: 'delete', - }; - expect(eventsQueue).toEqual([deletedBanana]); - // Verify that the initial result has been updated - expect(fileSystem.getModuleName(filePath)).toBeNull(); - expect(hasteMap.getModule('Banana')).toBeNull(); - }); + fm_it( + 'build returns a "live" fileSystem and hasteMap', + async ({fileMap, hasteMap}) => { + const {fileSystem} = await fileMap.build(); + const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); + expect(fileSystem.exists(filePath)).toBe(true); + expect(hasteMap.getModuleNameByPath(filePath)).toBe('Banana'); + expect(hasteMap.getModule('Banana')).toBe(filePath); + mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); + mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); + const {eventsQueue} = await waitForItToChange(fileMap); + expect(eventsQueue).toHaveLength(1); + const deletedBanana = { + filePath, + metadata: { + modifiedTime: null, + size: null, + type: 'f', + }, + type: 'delete', + }; + expect(eventsQueue).toEqual([deletedBanana]); + // Verify that the initial result has been updated + expect(fileSystem.exists(filePath)).toBe(false); + expect(hasteMap.getModuleNameByPath(filePath)).toBeNull(); + expect(hasteMap.getModule('Banana')).toBeNull(); + }, + ); const MOCK_CHANGE_FILE: ChangeEventMetadata = { type: 'f', @@ -1732,50 +1786,51 @@ describe('FileMap', () => { size: 55, }; - fm_it('handles several change events at once', async hm => { - const {fileSystem, hasteMap} = await hm.build(); - mockFs[path.join('/', 'project', 'fruits', 'Tomato.js')] = ` + fm_it( + 'handles several change events at once', + async ({fileMap, hasteMap}) => { + const {fileSystem} = await fileMap.build(); + mockFs[path.join('/', 'project', 'fruits', 'Tomato.js')] = ` // Tomato! `; - mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = ` + mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = ` // Pear! `; - const e = mockEmitters[path.join('/', 'project', 'fruits')]; - e.emitFileEvent({ - event: 'touch', - relativePath: 'Tomato.js', - metadata: MOCK_CHANGE_FILE, - }); - e.emitFileEvent({ - event: 'touch', - relativePath: 'Pear.js', - metadata: MOCK_CHANGE_FILE, - }); - const {eventsQueue} = await waitForItToChange(hm); - expect(eventsQueue).toEqual([ - { - filePath: path.join('/', 'project', 'fruits', 'Tomato.js'), + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emitFileEvent({ + event: 'touch', + relativePath: 'Tomato.js', metadata: MOCK_CHANGE_FILE, - type: 'add', - }, - { - filePath: path.join('/', 'project', 'fruits', 'Pear.js'), + }); + e.emitFileEvent({ + event: 'touch', + relativePath: 'Pear.js', metadata: MOCK_CHANGE_FILE, - type: 'change', - }, - ]); - expect( - fileSystem.getModuleName( - path.join('/', 'project', 'fruits', 'Tomato.js'), - ), - ).not.toBeNull(); - expect(hasteMap.getModule('Tomato')).toBeDefined(); - expect(hasteMap.getModule('Pear')).toBe( - path.join('/', 'project', 'fruits', 'Pear.js'), - ); - }); + }); + const {eventsQueue} = await waitForItToChange(fileMap); + expect(eventsQueue).toEqual([ + { + filePath: path.join('/', 'project', 'fruits', 'Tomato.js'), + metadata: MOCK_CHANGE_FILE, + type: 'add', + }, + { + filePath: path.join('/', 'project', 'fruits', 'Pear.js'), + metadata: MOCK_CHANGE_FILE, + type: 'change', + }, + ]); + expect( + fileSystem.exists(path.join('/', 'project', 'fruits', 'Tomato.js')), + ).toBe(true); + expect(hasteMap.getModule('Tomato')).toBeDefined(); + expect(hasteMap.getModule('Pear')).toBe( + path.join('/', 'project', 'fruits', 'Pear.js'), + ); + }, + ); - fm_it('does not emit duplicate change events', async hm => { + fm_it('does not emit duplicate change events', async ({fileMap}) => { const e = mockEmitters[path.join('/', 'project', 'fruits')]; mockFs[path.join('/', 'project', 'fruits', 'Tomato.js')] = ` // Tomato! @@ -1790,15 +1845,15 @@ describe('FileMap', () => { relativePath: 'Tomato.js', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); expect(eventsQueue).toHaveLength(1); }); fm_it( 'file data is still available during processing', - async hm => { + async ({fileMap, hasteMap}) => { const e = mockEmitters[path.join('/', 'project', 'fruits')]; - const {fileSystem, hasteMap} = await hm.build(); + const {fileSystem} = await fileMap.build(); // Pre-existing file const bananaPath = path.join('/', 'project', 'fruits', 'Banana.js'); expect(fileSystem.linkStats(bananaPath)).toEqual({ @@ -1836,7 +1891,7 @@ describe('FileMap', () => { expect(fileSystem.getSha1(bananaPath)).toBe(originalHash); expect(hasteMap.getModule('Banana')).toBe(bananaPath); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); expect(eventsQueue).toHaveLength(1); // After the 'change' event is emitted, we should have new data @@ -1854,8 +1909,8 @@ describe('FileMap', () => { fm_it( 'suppresses backend symlink events if enableSymlinks: false', - async hm => { - const {fileSystem} = await hm.build(); + async ({fileMap}) => { + const {fileSystem} = await fileMap.build(); const fruitsRoot = path.join('/', 'project', 'fruits'); const e = mockEmitters[fruitsRoot]; e.emitFileEvent({ @@ -1868,7 +1923,7 @@ describe('FileMap', () => { relativePath: 'LinkToStrawberry.js', metadata: MOCK_CHANGE_LINK, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); expect(eventsQueue).toEqual([ { filePath: path.join(fruitsRoot, 'Strawberry.js'), @@ -1884,8 +1939,8 @@ describe('FileMap', () => { fm_it( 'emits symlink events if enableSymlinks: true', - async hm => { - const {fileSystem} = await hm.build(); + async ({fileMap}) => { + const {fileSystem} = await fileMap.build(); const fruitsRoot = path.join('/', 'project', 'fruits'); const e = mockEmitters[fruitsRoot]; e.emitFileEvent({ @@ -1898,7 +1953,7 @@ describe('FileMap', () => { relativePath: 'LinkToStrawberry.js', metadata: MOCK_CHANGE_LINK, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); expect(eventsQueue).toEqual([ { filePath: path.join(fruitsRoot, 'Strawberry.js'), @@ -1920,15 +1975,15 @@ describe('FileMap', () => { fm_it( 'emits a change even if a file in node_modules has changed', - async hm => { - const {fileSystem} = await hm.build(); + async ({fileMap}) => { + const {fileSystem} = await fileMap.build(); const e = mockEmitters[path.join('/', 'project', 'fruits')]; e.emitFileEvent({ event: 'touch', relativePath: path.join('node_modules', 'apple.js'), metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); const filePath = path.join( '/', 'project', @@ -1940,14 +1995,14 @@ describe('FileMap', () => { expect(eventsQueue).toEqual([ {filePath, metadata: MOCK_CHANGE_FILE, type: 'add'}, ]); - expect(fileSystem.getModuleName(filePath)).toBeDefined(); + expect(fileSystem.exists(filePath)).toBe(true); }, ); fm_it( 'does not emit changes for regular files with unwatched extensions', - async hm => { - const {fileSystem} = await hm.build(); + async ({fileMap}) => { + const {fileSystem} = await fileMap.build(); mockFs[path.join('/', 'project', 'fruits', 'Banana.unwatched')] = ''; const e = mockEmitters[path.join('/', 'project', 'fruits')]; @@ -1961,44 +2016,47 @@ describe('FileMap', () => { relativePath: 'Banana.unwatched', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); expect(eventsQueue).toHaveLength(1); expect(eventsQueue).toEqual([ {filePath, metadata: MOCK_CHANGE_FILE, type: 'change'}, ]); - expect(fileSystem.getModuleName(filePath)).toBeDefined(); + expect(fileSystem.exists(filePath)).toBe(true); }, ); - fm_it('does not emit delete events for unknown files', async hm => { - const {fileSystem} = await hm.build(); - mockFs[path.join('/', 'project', 'fruits', 'Banana.unwatched')] = ''; + fm_it( + 'does not emit delete events for unknown files', + async ({fileMap}) => { + const {fileSystem} = await fileMap.build(); + mockFs[path.join('/', 'project', 'fruits', 'Banana.unwatched')] = ''; - const e = mockEmitters[path.join('/', 'project', 'fruits')]; - e.emitFileEvent({ - event: 'delete', - relativePath: 'Banana.js', - }); - e.emitFileEvent({ - event: 'delete', - relativePath: 'Unknown.ext', - }); - const {eventsQueue} = await waitForItToChange(hm); - const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); - expect(eventsQueue).toHaveLength(1); - expect(eventsQueue).toEqual([ - {filePath, metadata: MOCK_DELETE_FILE, type: 'delete'}, - ]); - expect(fileSystem.getModuleName(filePath)).toBeDefined(); - expect(console.warn).not.toHaveBeenCalled(); - expect(console.error).not.toHaveBeenCalled(); - }); + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emitFileEvent({ + event: 'delete', + relativePath: 'Banana.js', + }); + e.emitFileEvent({ + event: 'delete', + relativePath: 'Unknown.ext', + }); + const {eventsQueue} = await waitForItToChange(fileMap); + const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); + expect(eventsQueue).toHaveLength(1); + expect(eventsQueue).toEqual([ + {filePath, metadata: MOCK_DELETE_FILE, type: 'delete'}, + ]); + expect(fileSystem.exists(filePath)).toBe(false); + expect(console.warn).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + }, + ); fm_it( 'does emit changes for symlinks with unlisted extensions', - async hm => { - const {fileSystem} = await hm.build(); + async ({fileMap}) => { + const {fileSystem} = await fileMap.build(); const e = mockEmitters[path.join('/', 'project', 'fruits')]; mockFs[path.join('/', 'project', 'fruits', 'LinkToStrawberry.ext')] = { link: 'Strawberry.js', @@ -2008,7 +2066,7 @@ describe('FileMap', () => { relativePath: 'LinkToStrawberry.ext', metadata: MOCK_CHANGE_LINK, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); const filePath = path.join( '/', 'project', @@ -2025,16 +2083,21 @@ describe('FileMap', () => { modifiedTime: 46, size: 5, }); - // getModuleName traverses the symlink, verifying the link is read. - expect(fileSystem.getModuleName(filePath)).toEqual('Strawberry'); + // lookup traverses the symlink, verifying the link is read. + expect(fileSystem.lookup(filePath)).toEqual( + expect.objectContaining({ + exists: true, + realPath: expect.stringMatching(/Strawberry\.js$/), + }), + ); }, {config: {enableSymlinks: true}}, ); fm_it( 'symlink deletion is handled without affecting the symlink target', - async hm => { - const {fileSystem, hasteMap} = await hm.build(); + async ({fileMap, hasteMap}) => { + const {fileSystem} = await fileMap.build(); const symlinkPath = path.join( '/', @@ -2044,8 +2107,8 @@ describe('FileMap', () => { ); const realPath = path.join('/', 'project', 'fruits', 'Strawberry.js'); - expect(fileSystem.getModuleName(symlinkPath)).toEqual('Strawberry'); - expect(fileSystem.getModuleName(realPath)).toEqual('Strawberry'); + expect(hasteMap.getModuleNameByPath(symlinkPath)).toEqual('Strawberry'); + expect(hasteMap.getModuleNameByPath(realPath)).toEqual('Strawberry'); expect(hasteMap.getModule('Strawberry', 'g')).toEqual(realPath); // Delete the symlink @@ -2055,7 +2118,7 @@ describe('FileMap', () => { event: 'delete', relativePath: 'LinkToStrawberry.js', }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); expect(eventsQueue).toHaveLength(1); expect(eventsQueue).toEqual([ @@ -2065,8 +2128,8 @@ describe('FileMap', () => { // Symlink is deleted without affecting the Haste module or real file. expect(fileSystem.exists(symlinkPath)).toBe(false); expect(fileSystem.exists(realPath)).toBe(true); - expect(fileSystem.getModuleName(symlinkPath)).toEqual(null); - expect(fileSystem.getModuleName(realPath)).toEqual('Strawberry'); + expect(hasteMap.getModuleNameByPath(symlinkPath)).toEqual(null); + expect(hasteMap.getModuleNameByPath(realPath)).toEqual('Strawberry'); expect(hasteMap.getModule('Strawberry', 'g')).toEqual(realPath); }, {config: {enableSymlinks: true}}, @@ -2074,8 +2137,7 @@ describe('FileMap', () => { fm_it( 'correctly tracks changes to both platform-specific versions of a single module name', - async hm => { - const {hasteMap, fileSystem} = await hm.build(); + async ({fileMap, hasteMap}) => { expect(hasteMap.getModule('Orange', 'ios')).toBeTruthy(); expect(hasteMap.getModule('Orange', 'android')).toBeTruthy(); const e = mockEmitters[path.join('/', 'project', 'fruits')]; @@ -2089,7 +2151,7 @@ describe('FileMap', () => { relativePath: 'Orange.android.js', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); expect(eventsQueue).toHaveLength(2); expect(eventsQueue).toEqual([ { @@ -2104,12 +2166,12 @@ describe('FileMap', () => { }, ]); expect( - fileSystem.getModuleName( + hasteMap.getModuleNameByPath( path.join('/', 'project', 'fruits', 'Orange.ios.js'), ), ).toBeTruthy(); expect( - fileSystem.getModuleName( + hasteMap.getModuleNameByPath( path.join('/', 'project', 'fruits', 'Orange.android.js'), ), ).toBeTruthy(); @@ -2134,52 +2196,54 @@ describe('FileMap', () => { }, ); - fm_it('correctly handles moving a Haste module', async hm => { - const oldPath = path.join('/', 'project', 'vegetables', 'Melon.js'); - const newPath = path.join('/', 'project', 'fruits', 'Melon.js'); + fm_it( + 'correctly handles moving a Haste module', + async ({fileMap, hasteMap}) => { + const oldPath = path.join('/', 'project', 'vegetables', 'Melon.js'); + const newPath = path.join('/', 'project', 'fruits', 'Melon.js'); - const {hasteMap} = await hm.build(); - expect(hasteMap.getModule('Melon')).toEqual(oldPath); + expect(hasteMap.getModule('Melon')).toEqual(oldPath); - // Move vegetables/Melon.js -> fruits/Melon.js - mockFs[newPath] = mockFs[oldPath]; - mockFs[oldPath] = null; + // Move vegetables/Melon.js -> fruits/Melon.js + mockFs[newPath] = mockFs[oldPath]; + mockFs[oldPath] = null; - mockEmitters[path.join('/', 'project', 'vegetables')].emitFileEvent({ - event: 'delete', - relativePath: 'Melon.js', - }); - mockEmitters[path.join('/', 'project', 'fruits')].emitFileEvent({ - event: 'touch', - relativePath: 'Melon.js', - metadata: MOCK_CHANGE_FILE, - }); + mockEmitters[path.join('/', 'project', 'vegetables')].emitFileEvent({ + event: 'delete', + relativePath: 'Melon.js', + }); + mockEmitters[path.join('/', 'project', 'fruits')].emitFileEvent({ + event: 'touch', + relativePath: 'Melon.js', + metadata: MOCK_CHANGE_FILE, + }); - const {eventsQueue} = await waitForItToChange(hm); + const {eventsQueue} = await waitForItToChange(fileMap); - // No duplicate warnings or errors should be printed. - expect(console.warn).not.toHaveBeenCalled(); - expect(console.error).not.toHaveBeenCalled(); + // No duplicate warnings or errors should be printed. + expect(console.warn).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); - expect(eventsQueue).toHaveLength(2); - expect(eventsQueue).toEqual([ - { - filePath: path.join('/', 'project', 'vegetables', 'Melon.js'), - metadata: MOCK_DELETE_FILE, - type: 'delete', - }, - { - filePath: path.join('/', 'project', 'fruits', 'Melon.js'), - metadata: MOCK_CHANGE_FILE, - type: 'add', - }, - ]); - expect(hasteMap.getModule('Melon')).toEqual(newPath); - }); + expect(eventsQueue).toHaveLength(2); + expect(eventsQueue).toEqual([ + { + filePath: path.join('/', 'project', 'vegetables', 'Melon.js'), + metadata: MOCK_DELETE_FILE, + type: 'delete', + }, + { + filePath: path.join('/', 'project', 'fruits', 'Melon.js'), + metadata: MOCK_CHANGE_FILE, + type: 'add', + }, + ]); + expect(hasteMap.getModule('Melon')).toEqual(newPath); + }, + ); describe('recovery from duplicate module IDs', () => { - async function setupDuplicates(hm: FileMap) { - const {fileSystem, hasteMap} = await hm.build(); + async function setupDuplicates(fm: FileMap, hasteMap: HasteMap) { + const {fileSystem} = await fm.build(); mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = ` // Pear! `; @@ -2197,7 +2261,7 @@ describe('FileMap', () => { relativePath: path.join('another', 'Pear.js'), metadata: MOCK_CHANGE_FILE, }); - await waitForItToChange(hm); + await waitForItToChange(fm); expect( fileSystem.exists( path.join('/', 'project', 'fruits', 'another', 'Pear.js'), @@ -2223,15 +2287,15 @@ describe('FileMap', () => { } fm_it( - 'does not throw on a duplicate created at runtime even if throwOnModuleCollision: true', - async hm => { + 'does not throw on a duplicate created at runtime even if failValidationOnConflicts: true', + async ({fileMap}) => { mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = ` // Pear! `; mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear.js')] = ` // Pear too! `; - const {fileSystem} = await hm.build(); + const {fileSystem} = await fileMap.build(); const e = mockEmitters[path.join('/', 'project', 'fruits')]; e.emitFileEvent({ event: 'touch', @@ -2248,7 +2312,7 @@ describe('FileMap', () => { console.error.mockImplementationOnce(() => { reject(new Error('should not print error')); }); - hm.once('change', resolve); + fileMap.once('change', resolve); }); // Expect a warning to be printed, but no error. expect(console.warn).toHaveBeenCalledWith( @@ -2268,17 +2332,16 @@ describe('FileMap', () => { ).toBe(true); }, { - config: { - throwOnModuleCollision: true, + hasteConfig: { + failValidationOnConflicts: true, }, }, ); fm_it( 'recovers when the oldest version of the duplicates is fixed', - async hm => { - const {hasteMap} = await hm.build(); - await setupDuplicates(hm); + async ({fileMap, hasteMap}) => { + await setupDuplicates(fileMap, hasteMap); mockFs[path.join('/', 'project', 'fruits', 'Pear.js')] = null; mockFs[path.join('/', 'project', 'fruits', 'Pear2.js')] = ` // Pear! @@ -2293,7 +2356,7 @@ describe('FileMap', () => { relativePath: 'Pear2.js', metadata: MOCK_CHANGE_FILE, }); - await waitForItToChange(hm); + await waitForItToChange(fileMap); expect(hasteMap.getModule('Pear')).toBe( path.join('/', 'project', 'fruits', 'another', 'Pear.js'), ); @@ -2303,51 +2366,57 @@ describe('FileMap', () => { }, ); - fm_it('recovers when the most recent duplicate is fixed', async hm => { - const {hasteMap} = await hm.build(); - await setupDuplicates(hm); - mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear.js')] = - null; - mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear2.js')] = ` + fm_it( + 'recovers when the most recent duplicate is fixed', + async ({fileMap, hasteMap}) => { + await setupDuplicates(fileMap, hasteMap); + mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear.js')] = + null; + mockFs[path.join('/', 'project', 'fruits', 'another', 'Pear2.js')] = ` // Pear too! `; - const e = mockEmitters[path.join('/', 'project', 'fruits')]; - e.emitFileEvent({ - event: 'touch', - relativePath: path.join('another', 'Pear2.js'), - metadata: MOCK_CHANGE_FILE, - }); - e.emitFileEvent({ - event: 'delete', - relativePath: path.join('another', 'Pear.js'), - }); - await waitForItToChange(hm); - expect(hasteMap.getModule('Pear')).toBe( - path.join('/', 'project', 'fruits', 'Pear.js'), - ); - expect(hasteMap.getModule('Pear2')).toBe( - path.join('/', 'project', 'fruits', 'another', 'Pear2.js'), - ); - }); + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emitFileEvent({ + event: 'touch', + relativePath: path.join('another', 'Pear2.js'), + metadata: MOCK_CHANGE_FILE, + }); + e.emitFileEvent({ + event: 'delete', + relativePath: path.join('another', 'Pear.js'), + }); + await waitForItToChange(fileMap); + expect(hasteMap.getModule('Pear')).toBe( + path.join('/', 'project', 'fruits', 'Pear.js'), + ); + expect(hasteMap.getModule('Pear2')).toBe( + path.join('/', 'project', 'fruits', 'another', 'Pear2.js'), + ); + }, + ); - fm_it('ignore directory events (even with file-ish names)', async hm => { - const e = mockEmitters[path.join('/', 'project', 'fruits')]; - mockFs[path.join('/', 'project', 'fruits', 'tomato.js', 'index.js')] = ` + fm_it( + 'ignore directory events (even with file-ish names)', + async ({fileMap}) => { + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + mockFs[path.join('/', 'project', 'fruits', 'tomato.js', 'index.js')] = + ` // Tomato! `; - e.emitFileEvent({ - event: 'touch', - relativePath: 'tomato.js', - metadata: MOCK_CHANGE_FOLDER, - }); - e.emitFileEvent({ - event: 'touch', - relativePath: path.join('tomato.js', 'index.js'), - metadata: MOCK_CHANGE_FILE, - }); - const {eventsQueue} = await waitForItToChange(hm); - expect(eventsQueue).toHaveLength(1); - }); + e.emitFileEvent({ + event: 'touch', + relativePath: 'tomato.js', + metadata: MOCK_CHANGE_FOLDER, + }); + e.emitFileEvent({ + event: 'touch', + relativePath: path.join('tomato.js', 'index.js'), + metadata: MOCK_CHANGE_FILE, + }); + const {eventsQueue} = await waitForItToChange(fileMap); + expect(eventsQueue).toHaveLength(1); + }, + ); }); }); }); diff --git a/packages/metro-file-map/src/__tests__/worker-test.js b/packages/metro-file-map/src/__tests__/worker-test.js index 899dc654af..436dfd3362 100644 --- a/packages/metro-file-map/src/__tests__/worker-test.js +++ b/packages/metro-file-map/src/__tests__/worker-test.js @@ -13,6 +13,7 @@ import type {WorkerMessage, WorkerMetadata} from '../flow-types'; import typeof TWorker from '../worker'; import typeof FS from 'fs'; +import {HastePlugin} from '..'; import {Worker} from '../worker'; import * as fs from 'fs'; import * as path from 'path'; @@ -65,26 +66,48 @@ jest.mock('fs', () => { }); const defaults: WorkerMessage = { + isNodeModules: false, computeDependencies: false, computeSha1: false, - enableHastePackages: false, filePath: path.join('/project', 'notexist.js'), maybeReturnContent: false, }; +const defaultHasteConfig = { + enableHastePackages: true, + failValidationOnConflicts: false, + hasteImplModulePath: require.resolve('./haste_impl.js'), + platforms: new Set(['ios', 'android']), + rootDir: path.normalize('/project'), +}; + +function workerWithHaste( + message: WorkerMessage, + hasteOverrides: Partial = {}, +) { + return new Worker({ + plugins: [ + new HastePlugin({ + ...defaultHasteConfig, + ...hasteOverrides, + }).getWorker(), + ], + }).processFile(message); +} + describe('worker', () => { let worker: (message: WorkerMessage) => Promise; beforeEach(() => { jest.clearAllMocks(); - const workerInstance = new Worker({}); + const workerInstance = new Worker({plugins: []}); worker = async message => workerInstance.processFile(message); }); const defaults: WorkerMessage = { computeDependencies: false, computeSha1: false, - enableHastePackages: false, + isNodeModules: false, filePath: path.join('/project', 'notexist.js'), maybeReturnContent: false, }; @@ -98,6 +121,7 @@ describe('worker', () => { }), ).toEqual({ dependencies: ['Banana', 'Strawberry'], + pluginData: [], }); expect( @@ -108,75 +132,79 @@ describe('worker', () => { }), ).toEqual({ dependencies: [], + pluginData: [], }); }); test('accepts a custom dependency extractor', async () => { expect( - new Worker({}).processFile({ + await new Worker({ + dependencyExtractor: path.join(__dirname, 'dependencyExtractor.js'), + plugins: [], + }).processFile({ ...defaults, computeDependencies: true, - dependencyExtractor: path.join(__dirname, 'dependencyExtractor.js'), filePath: path.join('/project', 'fruits', 'Pear.js'), }), ).toEqual({ dependencies: ['Banana', 'Strawberry', 'Lime'], + pluginData: [], }); }); test('delegates to hasteImplModulePath for getting the id', async () => { expect( - await worker({ + await workerWithHaste({ ...defaults, computeDependencies: true, filePath: path.join('/project', 'fruits', 'Pear.js'), - hasteImplModulePath: require.resolve('./haste_impl.js'), }), ).toEqual({ dependencies: ['Banana', 'Strawberry'], - id: 'Pear', + pluginData: ['Pear'], }); expect( - await worker({ + await workerWithHaste({ ...defaults, computeDependencies: true, filePath: path.join('/project', 'fruits', 'Strawberry.js'), - hasteImplModulePath: require.resolve('./haste_impl.js'), }), ).toEqual({ dependencies: [], - id: 'Strawberry', + pluginData: ['Strawberry'], }); }); test('parses package.json files as haste packages when enableHastePackages=true', async () => { - const worker = new Worker({}); expect( - worker.processFile({ - ...defaults, - computeDependencies: true, - enableHastePackages: true, - filePath: path.join('/project', 'package.json'), - }), + await workerWithHaste( + { + ...defaults, + computeDependencies: true, + filePath: path.join('/project', 'package.json'), + }, + {enableHastePackages: true}, + ), ).toEqual({ dependencies: undefined, - id: 'haste-package', + pluginData: ['haste-package'], }); }); test('does not parse package.json files as haste packages when enableHastePackages=false', async () => { - const worker = new Worker({}); expect( - worker.processFile({ - ...defaults, - computeDependencies: true, - enableHastePackages: false, - filePath: path.join('/project', 'package.json'), - }), + await workerWithHaste( + { + ...defaults, + computeDependencies: true, + filePath: path.join('/project', 'package.json'), + }, + {enableHastePackages: false}, + ), ).toEqual({ dependencies: undefined, - id: undefined, + pluginData: [null], }); }); @@ -203,7 +231,10 @@ describe('worker', () => { computeSha1: true, filePath: path.join('/project', 'fruits', 'apple.png'), }), - ).toEqual({sha1: '4caece539b039b16e16206ea2478f8c5ffb2ca05'}); + ).toEqual({ + pluginData: [], + sha1: '4caece539b039b16e16206ea2478f8c5ffb2ca05', + }); expect( await worker({ @@ -211,7 +242,7 @@ describe('worker', () => { computeSha1: false, filePath: path.join('/project', 'fruits', 'Banana.js'), }), - ).toEqual({sha1: undefined}); + ).toEqual({pluginData: [], sha1: undefined}); expect( await worker({ @@ -219,7 +250,10 @@ describe('worker', () => { computeSha1: true, filePath: path.join('/project', 'fruits', 'Banana.js'), }), - ).toEqual({sha1: '7772b628e422e8cf59c526be4bb9f44c0898e3d1'}); + ).toEqual({ + pluginData: [], + sha1: '7772b628e422e8cf59c526be4bb9f44c0898e3d1', + }); expect( await worker({ @@ -227,7 +261,10 @@ describe('worker', () => { computeSha1: true, filePath: path.join('/project', 'fruits', 'Pear.js'), }), - ).toEqual({sha1: 'c7a7a68a1c8aaf452669dd2ca52ac4a434d25552'}); + ).toEqual({ + pluginData: [], + sha1: 'c7a7a68a1c8aaf452669dd2ca52ac4a434d25552', + }); await expect(() => worker({...defaults, computeSha1: true, filePath: '/i/dont/exist.js'}), @@ -236,15 +273,14 @@ describe('worker', () => { test('avoids computing dependencies if not requested and Haste does not need it', async () => { expect( - await worker({ + await workerWithHaste({ ...defaults, computeDependencies: false, filePath: path.join('/project', 'fruits', 'Pear.js'), - hasteImplModulePath: path.resolve(__dirname, 'haste_impl.js'), }), ).toEqual({ dependencies: undefined, - id: 'Pear', + pluginData: ['Pear'], sha1: undefined, }); @@ -255,7 +291,7 @@ describe('worker', () => { test('returns content if requested and content is read', async () => { expect( - await worker({ + await workerWithHaste({ ...defaults, computeSha1: true, filePath: path.join('/project', 'fruits', 'Pear.js'), @@ -263,23 +299,23 @@ describe('worker', () => { }), ).toEqual({ content: expect.any(Buffer), + pluginData: ['Pear'], sha1: 'c7a7a68a1c8aaf452669dd2ca52ac4a434d25552', }); }); test('does not return content if maybeReturnContent but content is not read', async () => { expect( - await worker({ + await workerWithHaste({ ...defaults, computeSha1: false, filePath: path.join('/project', 'fruits', 'Pear.js'), - hasteImplModulePath: path.resolve(__dirname, 'haste_impl.js'), maybeReturnContent: true, }), ).toEqual({ content: undefined, dependencies: undefined, - id: 'Pear', + pluginData: ['Pear'], sha1: undefined, }); }); @@ -307,15 +343,15 @@ describe('jest-worker interface', () => { }); test('setup cannot be called twice', () => { - workerModule.setup({}); - expect(() => workerModule.setup({})).toThrow( + workerModule.setup({plugins: []}); + expect(() => workerModule.setup({plugins: []})).toThrow( new Error('metro-file-map: setup() should only be called once'), ); }); test('processFile may be called after setup', () => { jest.mock('mock-haste-impl', () => {}, {virtual: true}); - workerModule.setup({}); + workerModule.setup({plugins: []}); workerModule.processFile(defaults); }); }); diff --git a/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js b/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js index e8dff89aba..61075e502d 100644 --- a/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js +++ b/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js @@ -43,13 +43,11 @@ const buildParameters: BuildParameters = { computeDependencies: true, computeSha1: true, dependencyExtractor: null, - enableHastePackages: true, enableSymlinks: false, forceNodeFilesystemAPI: true, ignorePattern: /ignored/, retainAllFiles: false, extensions: ['js', 'json'], - hasteImplModulePath: require.resolve('../../__tests__/haste_impl'), plugins: [], rootDir: path.join('/', 'project'), roots: [ @@ -137,6 +135,9 @@ describe('cacheManager', () => { getSerializableSnapshot() { return {}; }, + getWorker() { + return null; + }, onNewOrModifiedFile() {}, onRemovedFile() {}, getCacheKey() { @@ -189,23 +190,6 @@ describe('cacheManager', () => { ); }); - test('creates different cache file paths for different hasteImplModulePath cache keys', () => { - const hasteImpl = require('../../__tests__/haste_impl'); - hasteImpl.setCacheKey('foo'); - const cacheManager1 = new DiskCacheManager( - {buildParameters}, - defaultConfig, - ); - hasteImpl.setCacheKey('bar'); - const cacheManager2 = new DiskCacheManager( - {buildParameters}, - defaultConfig, - ); - expect(cacheManager1.getCacheFilePath()).not.toBe( - cacheManager2.getCacheFilePath(), - ); - }); - test('creates different cache file paths for different projects', () => { const cacheManager1 = new DiskCacheManager( {buildParameters}, diff --git a/packages/metro-file-map/src/constants.js b/packages/metro-file-map/src/constants.js index f2c11f4488..78fd7a58a8 100644 --- a/packages/metro-file-map/src/constants.js +++ b/packages/metro-file-map/src/constants.js @@ -36,7 +36,7 @@ const constants/*: HType */ = { DEPENDENCIES: 3, SHA1: 4, SYMLINK: 5, - ID: 6, + PLUGINDATA: 6, /* module map attributes */ PATH: 0, diff --git a/packages/metro-file-map/src/crawlers/__tests__/integration-test.js b/packages/metro-file-map/src/crawlers/__tests__/integration-test.js index 876ba633fc..775aaaed88 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/integration-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/integration-test.js @@ -84,18 +84,26 @@ const CASES = [ [ true, new Map([ - ['foo.js', [expect.any(Number), 245, 0, '', null, 0, '']], + ['foo.js', [expect.any(Number), 245, 0, '', null, 0, null]], [ join('directory', 'bar.js'), - [expect.any(Number), 245, 0, '', null, 0, ''], + [expect.any(Number), 245, 0, '', null, 0, null], ], [ 'link-to-directory', - [expect.any(Number), 9, 0, '', null, expect.oneOf(1, 'directory'), ''], + [ + expect.any(Number), + 9, + 0, + '', + null, + expect.oneOf(1, 'directory'), + null, + ], ], [ 'link-to-foo.js', - [expect.any(Number), 6, 0, '', null, expect.oneOf(1, 'foo.js'), ''], + [expect.any(Number), 6, 0, '', null, expect.oneOf(1, 'foo.js'), null], ], ]), ], @@ -104,9 +112,9 @@ const CASES = [ new Map([ [ join('directory', 'bar.js'), - [expect.any(Number), 245, 0, '', null, 0, ''], + [expect.any(Number), 245, 0, '', null, 0, null], ], - ['foo.js', [expect.any(Number), 245, 0, '', null, 0, '']], + ['foo.js', [expect.any(Number), 245, 0, '', null, 0, null]], ]), ], ]; @@ -126,7 +134,9 @@ describe.each(Object.keys(CRAWLERS))( previousState: { fileSystem: new TreeFS({ rootDir: FIXTURES_DIR, - files: new Map([['removed.js', [123, 234, 0, '', null, 0, '']]]), + files: new Map([ + ['removed.js', [123, 234, 0, '', null, 0, null]], + ]), processFile: () => { throw new Error('Not implemented'); }, diff --git a/packages/metro-file-map/src/crawlers/__tests__/node-test.js b/packages/metro-file-map/src/crawlers/__tests__/node-test.js index 1bc720543b..6fbba2fa70 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/node-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/node-test.js @@ -185,9 +185,9 @@ describe('node crawler', () => { expect(changedFiles).toEqual( createMap({ - 'fruits/strawberry.js': [32, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [33, 42, 0, '', null, 0, ''], - 'vegetables/melon.json': [34, 42, 0, '', null, 0, ''], + 'fruits/strawberry.js': [32, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [33, 42, 0, '', null, 0, null], + 'vegetables/melon.json': [34, 42, 0, '', null, 0, null], }), ); @@ -198,9 +198,9 @@ describe('node crawler', () => { nodeCrawl = require('../node').default; // In this test sample, strawberry is changed and tomato is unchanged - const tomato = [33, 42, 1, '', null, 0, '']; + const tomato = [33, 42, 1, '', null, 0, null]; const files = createMap({ - 'fruits/strawberry.js': [30, 40, 1, '', null, 0, ''], + 'fruits/strawberry.js': [30, 40, 1, '', null, 0, null], 'fruits/tomato.js': tomato, }); @@ -215,7 +215,7 @@ describe('node crawler', () => { // Tomato is not included because it is unchanged expect(changedFiles).toEqual( createMap({ - 'fruits/strawberry.js': [32, 42, 0, '', null, 0, ''], + 'fruits/strawberry.js': [32, 42, 0, '', null, 0, null], }), ); @@ -228,9 +228,9 @@ describe('node crawler', () => { // In this test sample, previouslyExisted was present before and will not be // when crawling this directory. const files = createMap({ - 'fruits/previouslyExisted.js': [30, 40, 1, '', null, 0, ''], - 'fruits/strawberry.js': [33, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [32, 42, 0, '', null, 0, ''], + 'fruits/previouslyExisted.js': [30, 40, 1, '', null, 0, null], + 'fruits/strawberry.js': [33, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [32, 42, 0, '', null, 0, null], }); const {changedFiles, removedFiles} = await nodeCrawl({ @@ -243,8 +243,8 @@ describe('node crawler', () => { expect(changedFiles).toEqual( createMap({ - 'fruits/strawberry.js': [32, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [33, 42, 0, '', null, 0, ''], + 'fruits/strawberry.js': [32, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [33, 42, 0, '', null, 0, null], }), ); expect(removedFiles).toEqual(new Set(['fruits/previouslyExisted.js'])); @@ -272,8 +272,8 @@ describe('node crawler', () => { ); expect(changedFiles).toEqual( createMap({ - 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [32, 42, 0, '', null, 0, ''], + 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [32, 42, 0, '', null, 0, null], }), ); expect(removedFiles).toEqual(new Set()); @@ -297,8 +297,8 @@ describe('node crawler', () => { expect(changedFiles).toEqual( createMap({ - 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [32, 42, 0, '', null, 0, ''], + 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [32, 42, 0, '', null, 0, null], }), ); expect(removedFiles).toEqual(new Set()); @@ -321,8 +321,8 @@ describe('node crawler', () => { expect(childProcess.spawn).toHaveBeenCalledTimes(0); expect(changedFiles).toEqual( createMap({ - 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [32, 42, 0, '', null, 0, ''], + 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [32, 42, 0, '', null, 0, null], }), ); expect(removedFiles).toEqual(new Set()); @@ -386,8 +386,8 @@ describe('node crawler', () => { expect(changedFiles).toEqual( createMap({ - 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, ''], - 'fruits/tomato.js': [32, 42, 0, '', null, 0, ''], + 'fruits/directory/strawberry.js': [33, 42, 0, '', null, 0, null], + 'fruits/tomato.js': [32, 42, 0, '', null, 0, null], }), ); expect(removedFiles).toEqual(new Set()); diff --git a/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js b/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js index 2d971458cd..ee4b25363f 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/watchman-test.js @@ -118,9 +118,9 @@ describe('watchman watch', () => { }; mockFiles = createMap({ - [MELON_RELATIVE]: [33, 43, 0, '', null, 0, ''], - [STRAWBERRY_RELATIVE]: [30, 40, 0, '', null, 0, ''], - [TOMATO_RELATIVE]: [31, 41, 0, '', null, 0, ''], + [MELON_RELATIVE]: [33, 43, 0, '', null, 0, null], + [STRAWBERRY_RELATIVE]: [30, 40, 0, '', null, 0, null], + [TOMATO_RELATIVE]: [31, 41, 0, '', null, 0, null], }); }); @@ -223,7 +223,7 @@ describe('watchman watch', () => { expect(changedFiles).toEqual( createMap({ - [KIWI_RELATIVE]: [42, 40, 0, '', null, 0, ''], + [KIWI_RELATIVE]: [42, 40, 0, '', null, 0, null], }), ); @@ -296,7 +296,7 @@ describe('watchman watch', () => { // banana is not included because it is unchanged expect(changedFiles).toEqual( createMap({ - [KIWI_RELATIVE]: [42, 52, 0, '', null, 0, ''], + [KIWI_RELATIVE]: [42, 52, 0, '', null, 0, null], [TOMATO_RELATIVE]: [76, 41, 1, '', mockTomatoSha1, 0, 'Tomato'], }), ); @@ -373,7 +373,7 @@ describe('watchman watch', () => { // Melon is not included because it is unchanged. expect(changedFiles).toEqual( createMap({ - [KIWI_RELATIVE]: [42, 52, 0, '', null, 0, ''], + [KIWI_RELATIVE]: [42, 52, 0, '', null, 0, null], }), ); @@ -542,7 +542,7 @@ describe('watchman watch', () => { expect(changedFiles).toEqual( createMap({ - [KIWI_RELATIVE]: [42, 40, 0, '', null, 0, ''], + [KIWI_RELATIVE]: [42, 40, 0, '', null, 0, null], }), ); diff --git a/packages/metro-file-map/src/crawlers/node/index.js b/packages/metro-file-map/src/crawlers/node/index.js index a21fb02b1e..075a87516c 100644 --- a/packages/metro-file-map/src/crawlers/node/index.js +++ b/packages/metro-file-map/src/crawlers/node/index.js @@ -82,7 +82,7 @@ function find( '', null, stat.isSymbolicLink() ? 1 : 0, - '', + null, ]); } } @@ -160,7 +160,7 @@ function findNative( '', null, stat.isSymbolicLink() ? 1 : 0, - '', + null, ]); } if (--count === 0) { diff --git a/packages/metro-file-map/src/crawlers/watchman/index.js b/packages/metro-file-map/src/crawlers/watchman/index.js index 5a3d15868e..68baacc658 100644 --- a/packages/metro-file-map/src/crawlers/watchman/index.js +++ b/packages/metro-file-map/src/crawlers/watchman/index.js @@ -339,7 +339,7 @@ export default async function watchmanCrawl({ '', sha1hex ?? null, symlinkInfo, - '', + null, ]; // If watchman is fresh, the removed files map starts with all files diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js index c38b2c564e..487d9ce1ce 100644 --- a/packages/metro-file-map/src/flow-types.js +++ b/packages/metro-file-map/src/flow-types.js @@ -18,7 +18,6 @@ export type {PerfLoggerFactory, PerfLogger}; export type BuildParameters = $ReadOnly<{ computeDependencies: boolean, computeSha1: boolean, - enableHastePackages: boolean, enableSymlinks: boolean, extensions: $ReadOnlyArray, forceNodeFilesystemAPI: boolean, @@ -30,15 +29,12 @@ export type BuildParameters = $ReadOnly<{ // Module paths that should export a 'getCacheKey' method dependencyExtractor: ?string, - hasteImplModulePath: ?string, cacheBreaker: string, }>; export type BuildResult = { fileSystem: FileSystem, - hasteMap: HasteMap, - mockMap: ?MockMap, }; export type CacheData = $ReadOnly<{ @@ -164,45 +160,77 @@ export type EventsQueue = Array<{ type: string, }>; -export type FileMapDelta = $ReadOnly<{ - removed: Iterable<[CanonicalPath, FileMetadata]>, - addedOrModified: Iterable<[CanonicalPath, FileMetadata]>, +export type FileMapDelta = $ReadOnly<{ + removed: Iterable<[CanonicalPath, T]>, + addedOrModified: Iterable<[CanonicalPath, T]>, }>; -interface FileSystemState { - metadataIterator( - opts: $ReadOnly<{ - includeNodeModules: boolean, - includeSymlinks: boolean, +export type FileMapPluginInitOptions< + SerializableState, + PerFileData = void, +> = $ReadOnly<{ + files: $ReadOnly<{ + fileIterator( + opts: $ReadOnly<{ + includeNodeModules: boolean, + includeSymlinks: boolean, + }>, + ): Iterable<{ + baseName: string, + canonicalPath: string, + pluginData: ?PerFileData, }>, - ): Iterable<{ - baseName: string, - canonicalPath: string, - metadata: FileMetadata, - }>; -} - -export type FileMapPluginInitOptions = $ReadOnly<{ - files: FileSystemState, + lookup( + mixedPath: string, + ): + | {exists: false} + | {exists: true, type: 'f', pluginData: PerFileData} + | {exists: true, type: 'd'}, + }>, pluginState: ?SerializableState, + ...PerFileData extends void + ? {} + : {processFile: (mixedPath: string) => PerFileData}, }>; -type V8Serializable = interface {}; +export type FileMapPluginWorker = $ReadOnly<{ + match: boolean | RegExp, + workerModulePath: string, + workerSetupArgs: JsonData, +}>; -export interface FileMapPlugin { +export type V8Serializable = + | string + | number + | boolean + | null + | $ReadOnlyArray + | $ReadOnlySet + | $ReadOnlyMap + | {[key: string]: V8Serializable}; + +export interface FileMapPlugin< + SerializableState = V8Serializable, + PerFileData = void, +> { +name: string; initialize( - initOptions: FileMapPluginInitOptions, + initOptions: FileMapPluginInitOptions, ): Promise; assertValid(): void; - bulkUpdate(delta: FileMapDelta): Promise; + bulkUpdate(delta: FileMapDelta): Promise; getSerializableSnapshot(): SerializableState; - onRemovedFile(relativeFilePath: string, fileMetadata: FileMetadata): void; - onNewOrModifiedFile( - relativeFilePath: string, - fileMetadata: FileMetadata, - ): void; + onRemovedFile(relativeFilePath: string, pluginData: ?PerFileData): void; + onNewOrModifiedFile(relativeFilePath: string, pluginData: ?PerFileData): void; getCacheKey(): string; + getWorker(): ?FileMapPluginWorker; +} + +export interface MetadataWorker { + processFile( + WorkerMessage, + $ReadOnly<{getContent: () => Buffer}>, + ): V8Serializable; } export type HType = { @@ -212,7 +240,7 @@ export type HType = { DEPENDENCIES: 3, SHA1: 4, SYMLINK: 5, - ID: 6, + PLUGINDATA: number, PATH: 0, TYPE: 1, MODULE: 0, @@ -235,7 +263,8 @@ export type FileMetadata = [ /* dependencies */ string, /* sha1 */ ?string, /* symlink */ 0 | 1 | string, // string specifies target, if known - /* id */ string, + /* plugindata */ + ... ]; export type FileStats = $ReadOnly<{ @@ -252,7 +281,6 @@ export interface FileSystem { changedFiles: FileData, removedFiles: Set, }; - getModuleName(file: Path): ?string; getSerializableSnapshot(): CacheData['fileSystemData']; getSha1(file: Path): ?string; getOrComputeSha1(file: Path): Promise; @@ -323,6 +351,14 @@ export interface FileSystem { export type Glob = string; +export type JsonData = + | string + | number + | boolean + | null + | Array + | {[key: string]: JsonData}; + export type LookupResult = | { // The node is missing from the FileSystem implementation (note this @@ -339,11 +375,23 @@ export type LookupResult = exists: true, // The real, normal, absolute paths of any symlinks traversed. links: $ReadOnlySet, - // The real, normal, absolute path of the file or directory. + // The real, normal, absolute path of the directory. + realPath: string, + // Currently lookup always follows symlinks, so can only return + // directories or regular files, but this may be extended. + type: 'd', + } + | { + exists: true, + // The real, normal, absolute paths of any symlinks traversed. + links: $ReadOnlySet, + // The real, normal, absolute path of the file. realPath: string, // Currently lookup always follows symlinks, so can only return // directories or regular files, but this may be extended. - type: 'd' | 'f', + type: 'f', + // The file's metadata tuple. Must only be mutated via FileProcessor. + metadata: FileMetadata, }; export interface MockMap { @@ -365,6 +413,8 @@ export interface HasteMap { type?: ?HTypeValue, ): ?Path; + getModuleNameByPath(file: Path): ?string; + getPackage( name: string, platform: ?string, @@ -391,9 +441,9 @@ export interface MutableFileSystem extends FileSystem { export type Path = string; export type ProcessFileFunction = ( - absolutePath: string, + normalPath: string, metadata: FileMetadata, - request: $ReadOnly<{computeSha1: boolean}>, + fields: $ReadOnlyArray, ) => ?Buffer; export type RawMockMap = $ReadOnly<{ @@ -458,18 +508,19 @@ export type WatchmanClocks = Map; export type WorkerMessage = $ReadOnly<{ computeDependencies: boolean, computeSha1: boolean, - dependencyExtractor?: ?string, - enableHastePackages: boolean, + isNodeModules: boolean, filePath: string, - hasteImplModulePath?: ?string, maybeReturnContent: boolean, }>; export type WorkerMetadata = $ReadOnly<{ dependencies?: ?$ReadOnlyArray, - id?: ?string, sha1?: ?string, content?: ?Buffer, + pluginData?: $ReadOnlyArray, }>; -export type WorkerSetupArgs = $ReadOnly<{}>; +export type WorkerSetupArgs = $ReadOnly<{ + dependencyExtractor?: ?string, + plugins: $ReadOnlyArray, +}>; diff --git a/packages/metro-file-map/src/index.js b/packages/metro-file-map/src/index.js index 59bf0d6f93..019ed635d6 100644 --- a/packages/metro-file-map/src/index.js +++ b/packages/metro-file-map/src/index.js @@ -25,6 +25,7 @@ import type { EventsQueue, FileData, FileMapPlugin, + FileMapPluginWorker, FileMetadata, FileSystem, HasteMapData, @@ -75,22 +76,22 @@ export type { export type InputOptions = $ReadOnly<{ computeDependencies?: ?boolean, computeSha1?: ?boolean, - enableHastePackages?: boolean, + // enableHastePackages?: boolean, enableSymlinks?: ?boolean, enableWorkerThreads?: ?boolean, extensions: $ReadOnlyArray, forceNodeFilesystemAPI?: ?boolean, ignorePattern?: ?RegExp, - mocksPattern?: ?string, - platforms: $ReadOnlyArray, - plugins?: $ReadOnlyArray>, + // mocksPattern?: ?string, + // platforms: $ReadOnlyArray, + plugins?: $ReadOnlyArray, retainAllFiles: boolean, rootDir: string, roots: $ReadOnlyArray, // Module paths that should export a 'getCacheKey' method dependencyExtractor?: ?string, - hasteImplModulePath?: ?string, + // hasteImplModulePath?: ?string, cacheManagerFactory?: ?CacheManagerFactory, console?: Console, @@ -99,7 +100,7 @@ export type InputOptions = $ReadOnly<{ maxWorkers: number, perfLoggerFactory?: ?PerfLoggerFactory, resetCache?: ?boolean, - throwOnModuleCollision?: ?boolean, + // throwOnModuleCollision?: ?boolean, useWatchman?: ?boolean, watch?: ?boolean, watchmanDeferStates?: $ReadOnlyArray, @@ -122,6 +123,13 @@ type InternalOptions = $ReadOnly<{ watchmanDeferStates: $ReadOnlyArray, }>; +// $FlowFixMe[unclear-type] Plugin types cannot be known statically +type AnyFileMapPlugin = FileMapPlugin; +type IndexedPlugin = $ReadOnly<{ + plugin: AnyFileMapPlugin, + dataIdx: ?number, +}>; + export {DiskCacheManager} from './cache/DiskCacheManager'; export {DuplicateHasteCandidatesError} from './plugins/haste/DuplicateHasteCandidatesError'; export {HasteConflictsError} from './plugins/haste/HasteConflictsError'; @@ -142,12 +150,11 @@ export type { // This should be bumped whenever a code change to `metro-file-map` itself // would cause a change to the cache data structure and/or content (for a given // filesystem state and build parameters). -const CACHE_BREAKER = '10'; +const CACHE_BREAKER = '11'; const CHANGE_INTERVAL = 30; const NODE_MODULES = path.sep + 'node_modules' + path.sep; -const PACKAGE_JSON = path.sep + 'package.json'; const VCS_DIRECTORIES = /[/\\]\.(git|hg)[/\\]/.source; const WATCHMAN_REQUIRED_CAPABILITIES = [ 'field-content.sha1hex', @@ -249,9 +256,7 @@ export default class FileMap extends EventEmitter { _healthCheckInterval: ?IntervalID; _startupPerfLogger: ?PerfLogger; - #hastePlugin: HastePlugin; - #mockPlugin: ?MockPlugin = null; - #plugins: $ReadOnlyArray>; + #plugins: $ReadOnlyArray; static create(options: InputOptions): FileMap { return new FileMap(options); @@ -285,32 +290,34 @@ export default class FileMap extends EventEmitter { } this._console = options.console || global.console; - const throwOnModuleCollision = Boolean(options.throwOnModuleCollision); - - const enableHastePackages = options.enableHastePackages ?? true; - - this.#hastePlugin = new HastePlugin({ - console: this._console, - enableHastePackages, - perfLogger: this._startupPerfLogger, - platforms: new Set(options.platforms), - rootDir: options.rootDir, - failValidationOnConflicts: throwOnModuleCollision, - }); - - const plugins: Array> = [this.#hastePlugin]; - - if (options.mocksPattern != null && options.mocksPattern !== '') { - this.#mockPlugin = new MockPlugin({ - console: this._console, - mocksPattern: new RegExp(options.mocksPattern), - rootDir: options.rootDir, - throwOnModuleCollision, + // const throwOnModuleCollision = Boolean(options.throwOnModuleCollision); + + // this.#hastePlugin = new HastePlugin({ + // console: this._console, + // enableHastePackages, + // hasteImplModulePath: options.hasteImplModulePath, + // perfLogger: this._startupPerfLogger, + // platforms: new Set(options.platforms), + // rootDir: options.rootDir, + // failValidationOnConflicts: throwOnModuleCollision, + // }); + + let dataSlot: number = H.PLUGINDATA; + + const indexedPlugins: Array = []; + const pluginWorkers: Array = []; + const plugins = options.plugins ?? []; + for (const plugin of plugins) { + const maybeWorker = plugin.getWorker(); + indexedPlugins.push({ + plugin, + dataIdx: maybeWorker != null ? dataSlot++ : null, }); - plugins.push(this.#mockPlugin); + if (maybeWorker != null) { + pluginWorkers.push(maybeWorker); + } } - - this.#plugins = plugins; + this.#plugins = indexedPlugins; const buildParameters: BuildParameters = { computeDependencies: @@ -319,13 +326,11 @@ export default class FileMap extends EventEmitter { : options.computeDependencies, computeSha1: options.computeSha1 || false, dependencyExtractor: options.dependencyExtractor ?? null, - enableHastePackages, enableSymlinks: options.enableSymlinks || false, extensions: options.extensions, forceNodeFilesystemAPI: !!options.forceNodeFilesystemAPI, - hasteImplModulePath: options.hasteImplModulePath, ignorePattern, - plugins: options.plugins ?? [], + plugins, retainAllFiles: options.retainAllFiles, rootDir: options.rootDir, roots: Array.from(new Set(options.roots)), @@ -351,12 +356,12 @@ export default class FileMap extends EventEmitter { this._fileProcessor = new FileProcessor({ dependencyExtractor: buildParameters.dependencyExtractor, - enableHastePackages: buildParameters.enableHastePackages, enableWorkerThreads: options.enableWorkerThreads ?? false, - hasteImplModulePath: buildParameters.hasteImplModulePath, maxFilesPerWorker: options.maxFilesPerWorker, maxWorkers: options.maxWorkers, perfLogger: this._startupPerfLogger, + pluginWorkers, + rootDir: options.rootDir, }); this._buildPromise = null; @@ -383,20 +388,21 @@ export default class FileMap extends EventEmitter { const rootDir = this._options.rootDir; this._startupPerfLogger?.point('constructFileSystem_start'); const processFile: ProcessFileFunction = ( - absolutePath, + normalPath, metadata, - opts, + dataIdx, ) => { const result = this._fileProcessor.processRegularFile( - absolutePath, + normalPath, metadata, { - computeSha1: opts.computeSha1, + computeSha1: true, computeDependencies: false, + dataIdx, maybeReturnContent: true, }, ); - debug('Lazily processed file: %s', absolutePath); + debug('Lazily processed file: %s', normalPath); // Emit an event to inform caches that there is new data to save. this.emit('metadata'); return result?.content; @@ -426,10 +432,51 @@ export default class FileMap extends EventEmitter { clocks: initialData?.clocks ?? new Map(), }), Promise.all( - plugins.map(plugin => + plugins.map(({plugin, dataIdx}) => plugin.initialize({ - files: fileSystem, + files: { + lookup: mixedPath => { + const result = fileSystem.lookup(mixedPath); + if (!result.exists) { + return {exists: false}; + } + if (result.type === 'd') { + return {exists: true, type: 'd'}; + } + return { + exists: true, + type: 'f', + pluginData: + dataIdx != null ? result.metadata[dataIdx] : null, + }; + }, + fileIterator: opts => + mapIterator( + fileSystem.metadataIterator(opts), + ({baseName, canonicalPath, metadata}) => ({ + baseName, + canonicalPath, + pluginData: dataIdx != null ? metadata[dataIdx] : null, + }), + ), + }, pluginState: initialData?.plugins.get(plugin.name), + processFile: (mixedPath: string) => { + if (dataIdx == null) { + throw new Error( + `File map plugin "${plugin.name}" does not provide a worker.`, + ); + } + const result = fileSystem.lookup(mixedPath); + if (!result.exists) { + throw new Error('File does not exist'); + } + if (result.type !== 'f') { + throw new Error(`${result.realPath} is not a regular file`); + } + processFile(result.realPath, result.metadata); + return result.metadata[dataIdx]; + }, }), ), ), @@ -439,7 +486,7 @@ export default class FileMap extends EventEmitter { await this._applyFileDelta(fileSystem, plugins, fileDelta); // Validate the mock and Haste maps before persisting them. - plugins.forEach(plugin => plugin.assertValid()); + plugins.forEach(({plugin}) => plugin.assertValid()); const watchmanClocks = new Map(fileDelta.clocks ?? []); await this._takeSnapshotAndPersist( @@ -456,11 +503,7 @@ export default class FileMap extends EventEmitter { ); await this._watch(fileSystem, watchmanClocks, plugins); - return { - fileSystem, - hasteMap: this.#hastePlugin, - mockMap: this.#mockPlugin, - }; + return {fileSystem}; })(); } return this._buildPromise.then(result => { @@ -549,21 +592,23 @@ export default class FileMap extends EventEmitter { }); } - _maybeReadLink(filePath: Path, fileMetadata: FileMetadata): ?Promise { + _maybeReadLink(normalPath: Path, fileMetadata: FileMetadata): ?Promise { // If we only need to read a link, it's more efficient to do it in-band // (with async file IO) than to have the overhead of worker IO. if (fileMetadata[H.SYMLINK] === 1) { - return fsPromises.readlink(filePath).then(symlinkTarget => { - fileMetadata[H.VISITED] = 1; - fileMetadata[H.SYMLINK] = symlinkTarget; - }); + return fsPromises + .readlink(this._pathUtils.normalToAbsolute(normalPath)) + .then(symlinkTarget => { + fileMetadata[H.VISITED] = 1; + fileMetadata[H.SYMLINK] = symlinkTarget; + }); } return null; } async _applyFileDelta( fileSystem: MutableFileSystem, - plugins: $ReadOnlyArray>, + plugins: $ReadOnlyArray, delta: $ReadOnly<{ changedFiles: FileData, removedFiles: $ReadOnlySet, @@ -589,50 +634,26 @@ export default class FileMap extends EventEmitter { const readLinkPromises = []; const readLinkErrors: Array<{ - absolutePath: string, + normalFilePath: string, error: Error & {code?: string}, }> = []; const filesToProcess: Array<[string, FileMetadata]> = []; - for (const [relativeFilePath, fileData] of changedFiles) { + for (const [normalFilePath, fileData] of changedFiles) { // A crawler may preserve the H.VISITED flag to indicate that the file // contents are unchaged and it doesn't need visiting again. if (fileData[H.VISITED] === 1) { continue; } - if ( - !this._options.enableHastePackages && - relativeFilePath.endsWith(PACKAGE_JSON) - ) { - continue; - } - - if ( - fileData[H.SYMLINK] === 0 && - !this._options.computeDependencies && - !this._options.computeSha1 && - this._options.hasteImplModulePath == null && - !( - this._options.enableHastePackages && - relativeFilePath.endsWith(PACKAGE_JSON) - ) - ) { - // Nothing to process - continue; - } - - // SHA-1, if requested, should already be present thanks to the crawler. - const absolutePath = this._pathUtils.normalToAbsolute(relativeFilePath); - if (fileData[H.SYMLINK] === 0) { - filesToProcess.push([absolutePath, fileData]); + filesToProcess.push([normalFilePath, fileData]); } else { - const maybeReadLink = this._maybeReadLink(absolutePath, fileData); + const maybeReadLink = this._maybeReadLink(normalFilePath, fileData); if (maybeReadLink) { readLinkPromises.push( maybeReadLink.catch(error => - readLinkErrors.push({absolutePath, error}), + readLinkErrors.push({normalFilePath, error}), ), ); } @@ -641,7 +662,7 @@ export default class FileMap extends EventEmitter { this._startupPerfLogger?.point('applyFileDelta_preprocess_end'); debug( - 'Visiting %d added/modified files and %d symlinks.', + 'Found %d added/modified files and %d symlinks.', filesToProcess.length, readLinkPromises.length, ); @@ -667,13 +688,13 @@ export default class FileMap extends EventEmitter { // it if it already exists. We're not emitting events at this point in // startup, so there's nothing more to do. this._startupPerfLogger?.point('applyFileDelta_missing_start'); - for (const {absolutePath, error} of batchResult.errors.concat( + for (const {normalFilePath, error} of batchResult.errors.concat( readLinkErrors, )) { /* $FlowFixMe[incompatible-type] Error exposed after improved typing of * Array.{includes,indexOf,lastIndexOf} */ if (['ENOENT', 'EACCESS'].includes(error.code)) { - missingFiles.add(this._pathUtils.absoluteToNormal(absolutePath)); + missingFiles.add(normalFilePath); } else { // Anything else is fatal. throw error; @@ -693,13 +714,18 @@ export default class FileMap extends EventEmitter { this._startupPerfLogger?.point('applyFileDelta_add_end'); this._startupPerfLogger?.point('applyFileDelta_updatePlugins_start'); + await Promise.all([ - plugins.map(plugin => - plugin.bulkUpdate({ - addedOrModified: changedFiles, - removed, - }), - ), + plugins.map(({plugin, dataIdx}) => { + const mapFn: ([CanonicalPath, FileMetadata]) => [CanonicalPath, mixed] = + dataIdx != null + ? ([relativePath, fileData]) => [relativePath, fileData[dataIdx]] + : ([relativePath, fileData]) => [relativePath, null]; + return plugin.bulkUpdate({ + addedOrModified: mapIterator(changedFiles.entries(), mapFn), + removed: mapIterator(removed.values(), mapFn), + }); + }), ]); this._startupPerfLogger?.point('applyFileDelta_updatePlugins_end'); this._startupPerfLogger?.point('applyFileDelta_end'); @@ -711,7 +737,7 @@ export default class FileMap extends EventEmitter { async _takeSnapshotAndPersist( fileSystem: FileSystem, clocks: WatchmanClocks, - plugins: $ReadOnlyArray>, + plugins: $ReadOnlyArray, changed: FileData, removed: Set, ) { @@ -721,7 +747,7 @@ export default class FileMap extends EventEmitter { fileSystemData: fileSystem.getSerializableSnapshot(), clocks: new Map(clocks), plugins: new Map( - plugins.map(plugin => [ + plugins.map(({plugin}) => [ plugin.name, plugin.getSerializableSnapshot(), ]), @@ -756,7 +782,7 @@ export default class FileMap extends EventEmitter { async _watch( fileSystem: MutableFileSystem, clocks: WatchmanClocks, - plugins: $ReadOnlyArray>, + plugins: $ReadOnlyArray, ): Promise { this._startupPerfLogger?.point('watch_start'); if (!this._options.watch) { @@ -911,15 +937,15 @@ export default class FileMap extends EventEmitter { '', null, change.metadata.type === 'l' ? 1 : 0, - '', + null, ]; try { if (change.metadata.type === 'l') { - await this._maybeReadLink(absoluteFilePath, fileMetadata); + await this._maybeReadLink(relativeFilePath, fileMetadata); } else { await this._fileProcessor.processRegularFile( - absoluteFilePath, + relativeFilePath, fileMetadata, { computeSha1: this._options.computeSha1, @@ -930,8 +956,13 @@ export default class FileMap extends EventEmitter { } fileSystem.addOrModify(relativeFilePath, fileMetadata); this._updateClock(clocks, change.clock); - plugins.forEach(plugin => - plugin.onNewOrModifiedFile(relativeFilePath, fileMetadata), + plugins.forEach(({plugin, dataIdx}) => + dataIdx != null + ? plugin.onNewOrModifiedFile( + relativeFilePath, + fileMetadata[dataIdx], + ) + : plugin.onNewOrModifiedFile(relativeFilePath), ); enqueueEvent(change.metadata); } catch (e) { @@ -955,8 +986,10 @@ export default class FileMap extends EventEmitter { // exists in the file map and remove should always return metadata. const metadata = nullthrows(fileSystem.remove(relativeFilePath)); this._updateClock(clocks, change.clock); - plugins.forEach(plugin => - plugin.onRemovedFile(relativeFilePath, metadata), + plugins.forEach(({plugin, dataIdx}) => + dataIdx != null + ? plugin.onRemovedFile(relativeFilePath, metadata[dataIdx]) + : plugin.onRemovedFile(relativeFilePath), ); enqueueEvent({ @@ -1072,3 +1105,13 @@ export default class FileMap extends EventEmitter { static H: HType = H; } + +// TODO: Replace with it.map() from Node 22+ +const mapIterator: (Iterator, (T) => S) => Iterable = (it, fn) => + 'map' in it + ? it.map(fn) + : (function* mapped() { + for (const item of it) { + yield fn(item); + } + })(); diff --git a/packages/metro-file-map/src/lib/FileProcessor.js b/packages/metro-file-map/src/lib/FileProcessor.js index d22d4e9cd5..a8f87d6721 100644 --- a/packages/metro-file-map/src/lib/FileProcessor.js +++ b/packages/metro-file-map/src/lib/FileProcessor.js @@ -10,6 +10,7 @@ */ import type { + FileMapPluginWorker, FileMetadata, PerfLogger, WorkerMessage, @@ -19,6 +20,7 @@ import type { import H from '../constants'; import {Worker} from '../worker'; +import {RootPathUtils} from './RootPathUtils'; import {Worker as JestWorker} from 'jest-worker'; import {sep} from 'path'; @@ -35,6 +37,10 @@ type ProcessFileRequest = $ReadOnly<{ * using the dependencyExtractor provided to the constructor. */ computeDependencies: boolean, + /** + * The specific plugin that requested the worker, if any. + */ + dataIdx?: ?number, /** * Only if processing has already required reading the file's contents, return * the contents as a Buffer - null otherwise. Not supported for batches. @@ -51,55 +57,71 @@ interface MaybeCodedError extends Error { code?: string; } -const NODE_MODULES = sep + 'node_modules' + sep; +const NODE_MODULES_SEP = 'node_modules' + sep; const MAX_FILES_PER_WORKER = 100; export class FileProcessor { #dependencyExtractor: ?string; - #enableHastePackages: boolean; - #hasteImplModulePath: ?string; #enableWorkerThreads: boolean; #maxFilesPerWorker: number; #maxWorkers: number; #perfLogger: ?PerfLogger; #workerArgs: WorkerSetupArgs; #inBandWorker: Worker; + #rootPathUtils: RootPathUtils; constructor( opts: $ReadOnly<{ dependencyExtractor: ?string, - enableHastePackages: boolean, enableWorkerThreads: boolean, - hasteImplModulePath: ?string, maxFilesPerWorker?: ?number, maxWorkers: number, + pluginWorkers: ?$ReadOnlyArray, perfLogger: ?PerfLogger, + rootDir: string, }>, ) { this.#dependencyExtractor = opts.dependencyExtractor; - this.#enableHastePackages = opts.enableHastePackages; this.#enableWorkerThreads = opts.enableWorkerThreads; - this.#hasteImplModulePath = opts.hasteImplModulePath; this.#maxFilesPerWorker = opts.maxFilesPerWorker ?? MAX_FILES_PER_WORKER; this.#maxWorkers = opts.maxWorkers; - this.#workerArgs = {}; + this.#workerArgs = { + dependencyExtractor: this.#dependencyExtractor ?? null, + plugins: [...(opts.pluginWorkers ?? [])], + }; this.#inBandWorker = new Worker(this.#workerArgs); this.#perfLogger = opts.perfLogger; + this.#rootPathUtils = new RootPathUtils(opts.rootDir); } async processBatch( - files: $ReadOnlyArray<[string /*absolutePath*/, FileMetadata]>, + files: $ReadOnlyArray<[string /*relativePath*/, FileMetadata]>, req: ProcessFileRequest, ): Promise<{ errors: Array<{ - absolutePath: string, + normalFilePath: string, error: MaybeCodedError, }>, }> { const errors = []; + + const workerJobs = files + .map(([relativePath, fileMetadata]) => { + const maybeWorkerInput = this.#getWorkerInput( + relativePath, + fileMetadata, + req, + ); + if (!maybeWorkerInput) { + return null; + } + return [maybeWorkerInput, fileMetadata]; + }) + .filter(Boolean); + const numWorkers = Math.min( this.#maxWorkers, - Math.ceil(files.length / this.#maxFilesPerWorker), + Math.ceil(workerJobs.length / this.#maxFilesPerWorker), ); const batchWorker = this.#getBatchWorker(numWorkers); @@ -110,20 +132,17 @@ export class FileProcessor { } await Promise.all( - files.map(([absolutePath, fileMetadata]) => { - const maybeWorkerInput = this.#getWorkerInput( - absolutePath, - fileMetadata, - req, - ); - if (!maybeWorkerInput) { - return null; - } + workerJobs.map(([workerInput, fileMetadata]) => { return batchWorker - .processFile(maybeWorkerInput) + .processFile(workerInput) .then(reply => processWorkerReply(reply, fileMetadata)) .catch(error => - errors.push({absolutePath, error: normalizeWorkerError(error)}), + errors.push({ + normalFilePath: this.#rootPathUtils.absoluteToNormal( + workerInput.filePath, + ), + error: normalizeWorkerError(error), + }), ); }), ); @@ -132,11 +151,11 @@ export class FileProcessor { } processRegularFile( - absolutePath: string, + normalPath: string, fileMetadata: FileMetadata, req: ProcessFileRequest, ): ?{content: ?Buffer} { - const workerInput = this.#getWorkerInput(absolutePath, fileMetadata, req); + const workerInput = this.#getWorkerInput(normalPath, fileMetadata, req); return workerInput ? { content: processWorkerReply( @@ -148,13 +167,36 @@ export class FileProcessor { } #getWorkerInput( - absolutePath: string, + normalPath: string, fileMetadata: FileMetadata, req: ProcessFileRequest, ): ?WorkerMessage { + if (fileMetadata[H.SYMLINK] !== 0) { + // Only process regular files + return null; + } + const computeSha1 = req.computeSha1 && fileMetadata[H.SHA1] == null; + const {computeDependencies, dataIdx, maybeReturnContent} = req; + + if ( + !computeDependencies && + !computeSha1 && + !this.#workerArgs.plugins.some(plugin => + typeof plugin.match === 'boolean' + ? plugin.match + : plugin.match.test(normalPath), + ) + ) { + // Nothing to process + return null; + } - const {computeDependencies, maybeReturnContent} = req; + const nodeModulesIdx = normalPath.indexOf(NODE_MODULES_SEP); + // Path may begin 'node_modules/' or contain '/node_modules/'. + const isNodeModules = + nodeModulesIdx === 0 || + (nodeModulesIdx > 0 && normalPath[nodeModulesIdx - 1] === sep); // Use a cheaper worker configuration for node_modules files, because we // never care about extracting dependencies, and they may never be Haste @@ -162,15 +204,13 @@ export class FileProcessor { // // Note that we'd only expect node_modules files to reach this point if // retainAllFiles is true, or they're touched during watch mode. - if (absolutePath.includes(NODE_MODULES)) { - if (computeSha1) { + if (isNodeModules) { + if (computeSha1 || dataIdx != null) { return { computeDependencies: false, computeSha1: true, - dependencyExtractor: null, - enableHastePackages: false, - filePath: absolutePath, - hasteImplModulePath: null, + isNodeModules: true, + filePath: this.#rootPathUtils.normalToAbsolute(normalPath), maybeReturnContent, }; } @@ -180,10 +220,8 @@ export class FileProcessor { return { computeDependencies, computeSha1, - dependencyExtractor: this.#dependencyExtractor, - enableHastePackages: this.#enableHastePackages, - filePath: absolutePath, - hasteImplModulePath: this.#hasteImplModulePath, + isNodeModules, + filePath: this.#rootPathUtils.normalToAbsolute(normalPath), maybeReturnContent, }; } @@ -235,11 +273,13 @@ function processWorkerReply( fileMetadata: FileMetadata, ) { fileMetadata[H.VISITED] = 1; - - const metadataId = metadata.id; - - if (metadataId != null) { - fileMetadata[H.ID] = metadataId; + if (metadata.pluginData) { + // $FlowFixMe[incompatible-type] - treat inexact tuple as array to set tail entries + (fileMetadata as Array).splice( + H.PLUGINDATA, + metadata.pluginData.length, + ...metadata.pluginData, + ); } fileMetadata[H.DEPENDENCIES] = metadata.dependencies diff --git a/packages/metro-file-map/src/lib/TreeFS.js b/packages/metro-file-map/src/lib/TreeFS.js index 74ebb70681..b77ea844de 100644 --- a/packages/metro-file-map/src/lib/TreeFS.js +++ b/packages/metro-file-map/src/lib/TreeFS.js @@ -134,11 +134,6 @@ export default class TreeFS implements MutableFileSystem { return tfs; } - getModuleName(mixedPath: Path): ?string { - const fileMetadata = this._getFileData(mixedPath); - return (fileMetadata && fileMetadata[H.ID]) ?? null; - } - getSize(mixedPath: Path): ?number { const fileMetadata = this._getFileData(mixedPath); return (fileMetadata && fileMetadata[H.SIZE]) ?? null; @@ -222,20 +217,19 @@ export default class TreeFS implements MutableFileSystem { if (existing != null && existing.length > 0) { return {sha1: existing}; } - const absolutePath = this.#pathUtils.normalToAbsolute(canonicalPath); // Mutate the metadata we first retrieved. This may be orphaned or about // to be overwritten if the file changes while we are processing it - // by only mutating the original metadata, we don't risk caching a stale // SHA-1 after a change event. - const maybeContent = await this.#processFile(absolutePath, fileMetadata, { - computeSha1: true, + const maybeContent = await this.#processFile(canonicalPath, fileMetadata, { + dataIdx: H.SHA1, }); const sha1 = fileMetadata[H.SHA1]; invariant( sha1 != null && sha1.length > 0, "File processing didn't populate a SHA-1 hash for %s", - absolutePath, + canonicalPath, ); return maybeContent @@ -267,19 +261,17 @@ export default class TreeFS implements MutableFileSystem { }; } const {canonicalPath, node} = result; - const type = isDirectory(node) ? 'd' : isRegularFile(node) ? 'f' : 'l'; + const realPath = this.#pathUtils.normalToAbsolute(canonicalPath); + if (isDirectory(node)) { + return {exists: true, links, realPath, type: 'd'}; + } invariant( - type !== 'l', + isRegularFile(node), 'lookup follows symlinks, so should never return one (%s -> %s)', mixedPath, canonicalPath, ); - return { - exists: true, - links, - realPath: this.#pathUtils.normalToAbsolute(canonicalPath), - type, - }; + return {exists: true, links, realPath, type: 'f', metadata: node}; } getAllFiles(): Array { @@ -1006,7 +998,7 @@ export default class TreeFS implements MutableFileSystem { includeSymlinks: boolean, includeNodeModules: boolean, }>, - ): Iterable<{ + ): Iterator<{ baseName: string, canonicalPath: string, metadata: FileMetadata, diff --git a/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js b/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js index 16c66bcc45..393862517f 100644 --- a/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js +++ b/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js @@ -10,12 +10,14 @@ */ import type { + FileMapPluginWorker, FileMetadata, WorkerMessage, WorkerMetadata, } from '../../flow-types'; import H from '../../constants'; +import path from 'path'; const MockJestWorker = jest.fn().mockImplementation(() => ({ processFile: async () => ({}), @@ -25,11 +27,11 @@ const mockWorkerFn = jest.fn().mockReturnValue({}); const defaultOptions = { dependencyExtractor: null, - enableHastePackages: false, enableWorkerThreads: true, - hasteImplModulePath: null, maxWorkers: 5, perfLogger: null, + pluginWorkers: [] as $ReadOnlyArray, + rootDir: process.platform === 'win32' ? 'C:\\root' : '/root', }; describe('processBatch', () => { @@ -106,19 +108,21 @@ describe('processRegularFile', () => { test('synchronously populates metadata', () => { const processor = new FileProcessor(defaultOptions); - const [filename, metadata] = getNMockFiles(1)[0]; + const [normalFilePath, metadata] = getNMockFiles(1)[0]; expect(metadata[H.SHA1]).toBeFalsy(); const fileContent = Buffer.from('hello world'); mockReadFileSync.mockReturnValue(fileContent); - const result = processor.processRegularFile(filename, metadata, { + const result = processor.processRegularFile(normalFilePath, metadata, { computeSha1: true, computeDependencies: false, maybeReturnContent: true, }); - expect(mockReadFileSync).toHaveBeenCalledWith(filename); + expect(mockReadFileSync).toHaveBeenCalledWith( + path.resolve(defaultOptions.rootDir, normalFilePath), + ); expect(result).toEqual({ content: fileContent, @@ -133,6 +137,6 @@ function getNMockFiles(numFiles: number): Array<[string, FileMetadata]> { .fill(null) .map((_, i) => [ `file${i}.js`, - [123, 234, 0, '', null, 0, ''] as FileMetadata, + [123, 234, 0, '', null, 0, null] as FileMetadata, ]); } diff --git a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js index eea9f09f98..0e640cf205 100644 --- a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js +++ b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js @@ -36,18 +36,18 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { rootDir: p('/project'), files: new Map([ [p('foo/another.js'), [123, 2, 0, '', '', 0, 'another']], - [p('foo/owndir'), [0, 0, 0, '', '', '.', '']], - [p('foo/link-to-bar.js'), [0, 0, 0, '', '', p('../bar.js'), '']], - [p('foo/link-to-another.js'), [0, 0, 0, '', '', p('another.js'), '']], - [p('../outside/external.js'), [0, 0, 0, '', '', 0, '']], + [p('foo/owndir'), [0, 0, 0, '', '', '.', null]], + [p('foo/link-to-bar.js'), [0, 0, 0, '', '', p('../bar.js'), null]], + [p('foo/link-to-another.js'), [0, 0, 0, '', '', p('another.js'), null]], + [p('../outside/external.js'), [0, 0, 0, '', '', 0, null]], [p('bar.js'), [234, 3, 0, '', '', 0, 'bar']], - [p('link-to-foo'), [456, 0, 0, '', '', p('./../project/foo'), '']], - [p('abs-link-out'), [456, 0, 0, '', '', p('/outside/./baz/..'), '']], - [p('root'), [0, 0, 0, '', '', '..', '']], - [p('link-to-nowhere'), [123, 0, 0, '', '', p('./nowhere'), '']], - [p('link-to-self'), [123, 0, 0, '', '', p('./link-to-self'), '']], - [p('link-cycle-1'), [123, 0, 0, '', '', p('./link-cycle-2'), '']], - [p('link-cycle-2'), [123, 0, 0, '', '', p('./link-cycle-1'), '']], + [p('link-to-foo'), [456, 0, 0, '', '', p('./../project/foo'), null]], + [p('abs-link-out'), [456, 0, 0, '', '', p('/outside/./baz/..'), null]], + [p('root'), [0, 0, 0, '', '', '..', null]], + [p('link-to-nowhere'), [123, 0, 0, '', '', p('./nowhere'), null]], + [p('link-to-self'), [123, 0, 0, '', '', p('./link-to-self'), null]], + [p('link-cycle-1'), [123, 0, 0, '', '', p('./link-cycle-2'), null]], + [p('link-cycle-2'), [123, 0, 0, '', '', p('./link-cycle-1'), null]], [p('node_modules/pkg/a.js'), [123, 0, 0, '', '', 0, 'a']], [p('node_modules/pkg/package.json'), [123, 0, 0, '', '', 0, 'pkg']], ]), @@ -135,6 +135,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { links: new Set(expectedSymlinks), realPath: expectedRealPath, type: 'f', + metadata: expect.any(Array), }), ); @@ -187,8 +188,8 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { const tfs = new TreeFS({ rootDir: p('/deep/project/root'), files: new Map([ - [p('foo/index.js'), [123, 0, 0, '', '', 0, '']], - [p('link-up'), [123, 0, 0, '', '', p('..'), '']], + [p('foo/index.js'), [123, 0, 0, '', '', 0, null]], + [p('link-up'), [123, 0, 0, '', '', p('..'), null]], ]), processFile: () => { throw new Error('Not implemented'); @@ -215,7 +216,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { describe('symlinks to an ancestor of the project root', () => { beforeEach(() => { - tfs.addOrModify(p('foo/link-up-2'), [0, 0, 0, '', '', p('../..'), '']); + tfs.addOrModify(p('foo/link-up-2'), [0, 0, 0, '', '', p('../..'), null]); }); test.each([ @@ -242,6 +243,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { realPath: expectedRealPath, links: new Set(expectedSymlinks), type: 'f', + metadata: expect.any(Array), }); }, ); @@ -269,23 +271,23 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { describe('getDifference', () => { test('returns changed (inc. new) and removed files in given FileData', () => { const newFiles: FileData = new Map([ - [p('new-file'), [789, 0, 0, '', '', 0, '']], - [p('link-to-foo'), [456, 0, 0, '', '', p('./foo'), '']], + [p('new-file'), [789, 0, 0, '', '', 0, null]], + [p('link-to-foo'), [456, 0, 0, '', '', p('./foo'), null]], // Different modified time, expect new mtime in changedFiles - [p('foo/another.js'), [124, 0, 0, '', '', 0, '']], - [p('link-cycle-1'), [123, 0, 0, '', '', p('./link-cycle-2'), '']], - [p('link-cycle-2'), [123, 0, 0, '', '', p('./link-cycle-1'), '']], + [p('foo/another.js'), [124, 0, 0, '', '', 0, null]], + [p('link-cycle-1'), [123, 0, 0, '', '', p('./link-cycle-2'), null]], + [p('link-cycle-2'), [123, 0, 0, '', '', p('./link-cycle-1'), null]], // Was a symlink, now a regular file - [p('link-to-self'), [123, 0, 0, '', '', 0, '']], - [p('link-to-nowhere'), [123, 0, 0, '', '', p('./nowhere'), '']], + [p('link-to-self'), [123, 0, 0, '', '', 0, null]], + [p('link-to-nowhere'), [123, 0, 0, '', '', p('./nowhere'), null]], [p('node_modules/pkg/a.js'), [123, 0, 0, '', '', 0, 'a']], [p('node_modules/pkg/package.json'), [123, 0, 0, '', '', 0, 'pkg']], ]); expect(tfs.getDifference(newFiles)).toEqual({ changedFiles: new Map([ - [p('new-file'), [789, 0, 0, '', '', 0, '']], - [p('foo/another.js'), [124, 0, 0, '', '', 0, '']], - [p('link-to-self'), [123, 0, 0, '', '', 0, '']], + [p('new-file'), [789, 0, 0, '', '', 0, null]], + [p('foo/another.js'), [124, 0, 0, '', '', 0, null]], + [p('link-to-self'), [123, 0, 0, '', '', 0, null]], ]), removedFiles: new Set([ p('foo/owndir'), @@ -311,24 +313,27 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { [ [ p('a/1/package.json'), - [0, 0, 0, '', '', './real-package.json', ''], + [0, 0, 0, '', '', './real-package.json', null], ], [ p('a/2/package.json'), - [0, 0, 0, '', '', './notexist-package.json', ''], + [0, 0, 0, '', '', './notexist-package.json', null], + ], + [ + p('a/b/c/d/link-to-C'), + [0, 0, 0, '', '', p('../../../..'), null], ], - [p('a/b/c/d/link-to-C'), [0, 0, 0, '', '', p('../../../..'), '']], [ p('a/b/c/d/link-to-B'), - [0, 0, 0, '', '', p('../../../../..'), ''], + [0, 0, 0, '', '', p('../../../../..'), null], ], [ p('a/b/c/d/link-to-A'), - [0, 0, 0, '', '', p('../../../../../..'), ''], + [0, 0, 0, '', '', p('../../../../../..'), null], ], [ p('n_m/workspace/link-to-pkg'), - [0, 0, 0, '', '', p('../../../workspace-pkg'), ''], + [0, 0, 0, '', '', p('../../../workspace-pkg'), null], ], ] as Array<[CanonicalPath, FileMetadata]> ).concat( @@ -349,7 +354,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { '../../package.json', '../../../a/b/package.json', '../workspace-pkg/package.json', - ].map(posixPath => [p(posixPath), [0, 0, 0, '', '', 0, '']]), + ].map(posixPath => [p(posixPath), [0, 0, 0, '', '', 0, null]]), ), ), processFile: () => { @@ -713,8 +718,16 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { describe('mutation', () => { describe('addOrModify', () => { test('accepts non-real and absolute paths', () => { - tfs.addOrModify(p('link-to-foo/new.js'), [0, 1, 0, '', '', 0, '']); - tfs.addOrModify(p('/project/fileatroot.js'), [0, 2, 0, '', '', 0, '']); + tfs.addOrModify(p('link-to-foo/new.js'), [0, 1, 0, '', '', 0, null]); + tfs.addOrModify(p('/project/fileatroot.js'), [ + 0, + 2, + 0, + '', + '', + 0, + null, + ]); expect(tfs.getAllFiles().sort()).toEqual([ p('/outside/external.js'), p('/project/bar.js'), @@ -735,10 +748,10 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { new Map([ [ p('newdir/link-to-link-to-bar.js'), - [0, 0, 0, '', '', p('../foo/link-to-bar.js'), ''], + [0, 0, 0, '', '', p('../foo/link-to-bar.js'), null], ], - [p('foo/baz.js'), [0, 0, 0, '', '', 0, '']], - [p('bar.js'), [999, 1, 0, '', '', 0, '']], + [p('foo/baz.js'), [0, 0, 0, '', '', 0, null]], + [p('bar.js'), [999, 1, 0, '', '', 0, null]], ]), ); @@ -832,7 +845,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { { baseName: 'external.js', canonicalPath: p('../outside/external.js'), - metadata: [0, 0, 0, '', '', 0, ''], + metadata: [0, 0, 0, '', '', 0, null], }, { baseName: 'bar.js', @@ -870,7 +883,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { { baseName: 'link-to-bar.js', canonicalPath: p('foo/link-to-bar.js'), - metadata: [0, 0, 0, '', '', p('../bar.js'), ''], + metadata: [0, 0, 0, '', '', p('../bar.js'), null], }, ]), ); @@ -884,9 +897,9 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { tfs = new TreeFS({ rootDir: p('/project'), files: new Map([ - [p('foo.js'), [123, 0, 0, '', 'def456', 0, '']], - [p('bar.js'), [123, 0, 0, '', '', 0, '']], - [p('link-to-bar'), [456, 0, 0, '', '', p('./bar.js'), '']], + [p('foo.js'), [123, 0, 0, '', 'def456', 0, null]], + [p('bar.js'), [123, 0, 0, '', '', 0, null]], + [p('link-to-bar'), [456, 0, 0, '', '', p('./bar.js'), null]], ]), processFile: mockProcessFile, }); @@ -905,7 +918,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { test('calls processFile exactly once if SHA-1 not initially set', async () => { expect(await tfs.getOrComputeSha1(p('bar.js'))).toEqual({sha1: 'abc123'}); expect(mockProcessFile).toHaveBeenCalledWith( - p('/project/bar.js'), + p('bar.js'), expect.any(Array), {computeSha1: true}, ); @@ -924,7 +937,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { content: Buffer.from('content'), }); expect(mockProcessFile).toHaveBeenCalledWith( - p('/project/bar.js'), + p('bar.js'), expect.any(Array), {computeSha1: true}, ); @@ -941,7 +954,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { sha1: 'abc123', }); expect(mockProcessFile).toHaveBeenCalledWith( - p('/project/bar.js'), + p('bar.js'), expect.any(Array), {computeSha1: true}, ); @@ -955,12 +968,12 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { }); const getOrComputePromise = tfs.getOrComputeSha1(p('bar.js')); expect(mockProcessFile).toHaveBeenCalledWith( - p('/project/bar.js'), + p('bar.js'), expect.any(Array), {computeSha1: true}, ); // Simulate the file being modified while we're waiting for the SHA1. - tfs.addOrModify(p('bar.js'), [123, 0, 0, '', '', 0, '']); + tfs.addOrModify(p('bar.js'), [123, 0, 0, '', '', 0, null]); resolve?.('newsha1'); expect(await getOrComputePromise).toEqual({sha1: 'newsha1'}); // A second call re-computes diff --git a/packages/metro-file-map/src/lib/__tests__/rootRelativeCacheKeys-test.js b/packages/metro-file-map/src/lib/__tests__/rootRelativeCacheKeys-test.js index 3288581dda..51ef34a9b2 100644 --- a/packages/metro-file-map/src/lib/__tests__/rootRelativeCacheKeys-test.js +++ b/packages/metro-file-map/src/lib/__tests__/rootRelativeCacheKeys-test.js @@ -8,22 +8,25 @@ * @format */ -import type {BuildParameters} from '../../flow-types'; +import type {BuildParameters, FileMapPlugin} from '../../flow-types'; import typeof PathModule from 'path'; import rootRelativeCacheKeys from '../rootRelativeCacheKeys'; +// $FlowExpectedError[incompatible-type] Partial mock +const getMockPlugin = (cacheKey: string): FileMapPlugin<> => ({ + getCacheKey: jest.fn(() => cacheKey), +}); + const buildParameters: BuildParameters = { computeDependencies: false, computeSha1: false, dependencyExtractor: null, - enableHastePackages: true, enableSymlinks: false, extensions: ['a'], forceNodeFilesystemAPI: false, - hasteImplModulePath: null, ignorePattern: /a/, - plugins: [], + plugins: [getMockPlugin('1')], retainAllFiles: false, rootDir: '/root', roots: ['a', 'b'], @@ -61,10 +64,9 @@ jest.mock( test('returns a distinct cache key for any change', () => { const { - hasteImplModulePath: _, - dependencyExtractor: __, - rootDir: ___, - plugins: ____, + dependencyExtractor: _, + rootDir: __, + plugins: ___, ...simpleParameters } = buildParameters; @@ -82,7 +84,6 @@ test('returns a distinct cache key for any change', () => { // Boolean case 'computeDependencies': case 'computeSha1': - case 'enableHastePackages': case 'enableSymlinks': case 'forceNodeFilesystemAPI': case 'retainAllFiles': @@ -105,8 +106,8 @@ test('returns a distinct cache key for any change', () => { configs.push(buildParameters); configs.push({...buildParameters, dependencyExtractor: '/extractor/1'}); configs.push({...buildParameters, dependencyExtractor: '/extractor/2'}); - configs.push({...buildParameters, hasteImplModulePath: '/haste/1'}); - configs.push({...buildParameters, hasteImplModulePath: '/haste/2'}); + configs.push({...buildParameters, plugins: []}); + configs.push({...buildParameters, plugins: [getMockPlugin('2')]}); // Generate hashes for each config const configHashes = configs.map( diff --git a/packages/metro-file-map/src/lib/rootRelativeCacheKeys.js b/packages/metro-file-map/src/lib/rootRelativeCacheKeys.js index a7eefdc019..d68d1e674e 100644 --- a/packages/metro-file-map/src/lib/rootRelativeCacheKeys.js +++ b/packages/metro-file-map/src/lib/rootRelativeCacheKeys.js @@ -55,14 +55,12 @@ export default function rootRelativeCacheKeys( case 'extensions': case 'computeDependencies': case 'computeSha1': - case 'enableHastePackages': case 'enableSymlinks': case 'forceNodeFilesystemAPI': case 'retainAllFiles': return buildParameters[key] ?? null; case 'ignorePattern': return buildParameters[key].toString(); - case 'hasteImplModulePath': case 'dependencyExtractor': return moduleCacheKey(buildParameters[key]); default: diff --git a/packages/metro-file-map/src/plugins/AbstractDataPlugin.js b/packages/metro-file-map/src/plugins/AbstractDataPlugin.js new file mode 100644 index 0000000000..3150414fde --- /dev/null +++ b/packages/metro-file-map/src/plugins/AbstractDataPlugin.js @@ -0,0 +1,93 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type { + FileMapDelta, + FileMapPlugin, + FileMapPluginInitOptions, + FileMapPluginWorker, +} from '../flow-types'; + +import invariant from 'invariant'; + +export type AbstractDataPluginOptions = $ReadOnly<{ + name: string, + workerParams: FileMapPluginWorker, +}>; + +type LookupFn = FileMapPluginInitOptions['files']['lookup']; + +export default class AbstractDataPlugin implements FileMapPlugin { + +name: string; + +#workerParams: FileMapPluginWorker; + #initialized: boolean = false; + + #lookup: ?LookupFn; + #processFile: ?(mixedPath: string) => T; + + constructor(options: AbstractDataPluginOptions) { + this.name = options.name; + this.#workerParams = options.workerParams; + } + + async initialize({ + files: {lookup}, + processFile, + }: FileMapPluginInitOptions): Promise { + this.#initialized = true; + this.#lookup = lookup; + this.#processFile = processFile; + } + + lookup(mixedPath: string): ReturnType> { + invariant( + this.#lookup != null, + 'Plugin must be initialized before lookup()', + ); + return this.#lookup(mixedPath); + } + + processFile(mixedPath: string): T { + invariant( + this.#processFile != null, + 'Plugin must be initialized before lookup()', + ); + return this.#processFile(mixedPath); + } + + getSerializableSnapshot() {} + + async bulkUpdate(delta: FileMapDelta): Promise { + for (const [normalPath, data] of delta.removed) { + this.onRemovedFile(normalPath, data); + } + for (const [normalPath, data] of delta.addedOrModified) { + this.onNewOrModifiedFile(normalPath, data); + } + } + + onNewOrModifiedFile(relativeFilePath: string, data: ?T) {} + + onRemovedFile(relativeFilePath: string, data: ?T) {} + + assertValid(): void {} + + getCacheKey(): string { + throw new Error( + 'AbstractDataPlugin: getCacheKey must be implemented by subclass: ' + + this.name, + ); + } + + getWorker(): FileMapPluginWorker { + return this.#workerParams; + } +} diff --git a/packages/metro-file-map/src/plugins/HastePlugin.js b/packages/metro-file-map/src/plugins/HastePlugin.js index c70937f6df..06ac8a0b40 100644 --- a/packages/metro-file-map/src/plugins/HastePlugin.js +++ b/packages/metro-file-map/src/plugins/HastePlugin.js @@ -16,7 +16,7 @@ import type { FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, - FileMetadata, + FileMapPluginWorker, HasteConflict, HasteMap, HasteMapItem, @@ -36,21 +36,25 @@ import path from 'path'; const EMPTY_OBJ: $ReadOnly<{[string]: HasteMapItemMetadata}> = {}; const EMPTY_MAP: $ReadOnlyMap = new Map(); +const PACKAGE_JSON = /[/\\^]package\.json$/; // Periodically yield to the event loop to allow parallel I/O, etc. // Based on 200k files taking up to 800ms => max 40ms between yields. const YIELD_EVERY_NUM_HASTE_FILES = 10000; -type HasteMapOptions = $ReadOnly<{ +export type HasteMapOptions = $ReadOnly<{ console?: ?Console, enableHastePackages: boolean, - perfLogger: ?PerfLogger, + hasteImplModulePath?: ?string, + perfLogger?: ?PerfLogger, platforms: $ReadOnlySet, rootDir: Path, - failValidationOnConflicts: boolean, + failValidationOnConflicts?: boolean, }>; -export default class HastePlugin implements HasteMap, FileMapPlugin { +export default class HastePlugin + implements HasteMap, FileMapPlugin +{ +name = 'haste'; +#rootDir: Path; @@ -59,41 +63,74 @@ export default class HastePlugin implements HasteMap, FileMapPlugin { +#console: ?Console; +#enableHastePackages: boolean; + +#hasteImplCacheKey: ?string; + +#hasteImplModulePath: ?string; +#perfLogger: ?PerfLogger; +#pathUtils: RootPathUtils; +#platforms: $ReadOnlySet; +#failValidationOnConflicts: boolean; + #getModuleNameByPath: string => ?string; constructor(options: HasteMapOptions) { this.#console = options.console ?? null; this.#enableHastePackages = options.enableHastePackages; + const hasteImplPath = options.hasteImplModulePath; + + if (hasteImplPath != null) { + // $FlowFixMe[unsupported-syntax] - dynamic require + const hasteImpl = require(hasteImplPath); + if (typeof hasteImpl.getCacheKey !== 'function') { + throw new Error( + `HasteImpl module ${hasteImplPath} must export a function named "getCacheKey"`, + ); + } + this.#hasteImplCacheKey = hasteImpl.getCacheKey(); + this.#hasteImplModulePath = hasteImplPath; + } + this.#perfLogger = options.perfLogger; this.#platforms = options.platforms; this.#rootDir = options.rootDir; this.#pathUtils = new RootPathUtils(options.rootDir); - this.#failValidationOnConflicts = options.failValidationOnConflicts; + this.#failValidationOnConflicts = + options.failValidationOnConflicts ?? false; } - async initialize({files}: FileMapPluginInitOptions): Promise { + async initialize({ + files, + }: FileMapPluginInitOptions): Promise { this.#perfLogger?.point('constructHasteMap_start'); let hasteFiles = 0; - for (const {baseName, canonicalPath, metadata} of files.metadataIterator({ + for (const { + baseName, + canonicalPath, + pluginData: hasteId, + } of files.fileIterator({ // Symlinks and node_modules are never Haste modules or packages. includeNodeModules: false, includeSymlinks: false, })) { - if (metadata[H.ID]) { - this.setModule(metadata[H.ID], [ - canonicalPath, - this.#enableHastePackages && baseName === 'package.json' - ? H.PACKAGE - : H.MODULE, - ]); - if (++hasteFiles % YIELD_EVERY_NUM_HASTE_FILES === 0) { - await new Promise(setImmediate); - } + if (hasteId == null) { + continue; + } + this.setModule(hasteId, [ + canonicalPath, + this.#enableHastePackages && baseName === 'package.json' + ? H.PACKAGE + : H.MODULE, + ]); + if (++hasteFiles % YIELD_EVERY_NUM_HASTE_FILES === 0) { + await new Promise(setImmediate); } } + this.#getModuleNameByPath = mixedPath => { + const result = files.lookup(mixedPath); + return result.exists && + result.type === 'f' && + typeof result.pluginData === 'string' + ? result.pluginData + : null; + }; this.#perfLogger?.point('constructHasteMap_end'); this.#perfLogger?.annotate({int: {hasteFiles}}); } @@ -126,6 +163,15 @@ export default class HastePlugin implements HasteMap, FileMapPlugin { return null; } + getModuleNameByPath(mixedPath: Path): ?string { + if (this.#getModuleNameByPath == null) { + throw new Error( + 'HastePlugin has not been initialized before getModuleNameByPath', + ); + } + return this.#getModuleNameByPath(mixedPath) ?? null; + } + getPackage( name: string, platform: ?string, @@ -207,19 +253,19 @@ export default class HastePlugin implements HasteMap, FileMapPlugin { ); } - async bulkUpdate(delta: FileMapDelta): Promise { + async bulkUpdate(delta: FileMapDelta): Promise { // Process removals first so that moves aren't treated as duplicates. - for (const [normalPath, metadata] of delta.removed) { - this.onRemovedFile(normalPath, metadata); + for (const [normalPath, maybeHasteId] of delta.removed) { + this.onRemovedFile(normalPath, maybeHasteId); } - for (const [normalPath, metadata] of delta.addedOrModified) { - this.onNewOrModifiedFile(normalPath, metadata); + for (const [normalPath, maybeHasteId] of delta.addedOrModified) { + this.onNewOrModifiedFile(normalPath, maybeHasteId); } } - onNewOrModifiedFile(relativeFilePath: string, fileMetadata: FileMetadata) { - const id = fileMetadata[H.ID] || null; // Empty string indicates no module + onNewOrModifiedFile(relativeFilePath: string, id: ?string) { if (id == null) { + // Not a Haste module or package return; } @@ -294,9 +340,9 @@ export default class HastePlugin implements HasteMap, FileMapPlugin { hasteMapItem[platform] = module; } - onRemovedFile(relativeFilePath: string, fileMetadata: FileMetadata) { - const moduleName = fileMetadata[H.ID] || null; // Empty string indicates no module + onRemovedFile(relativeFilePath: string, moduleName: ?string) { if (moduleName == null) { + // Not a Haste module or package return; } @@ -454,7 +500,24 @@ export default class HastePlugin implements HasteMap, FileMapPlugin { getCacheKey(): string { return JSON.stringify([ this.#enableHastePackages, + this.#hasteImplCacheKey, [...this.#platforms].sort(), ]); } + + getWorker(): FileMapPluginWorker { + return { + match: + this.#hasteImplModulePath != null + ? true + : this.#enableHastePackages + ? PACKAGE_JSON + : false, + workerModulePath: require.resolve('./haste/worker.js'), + workerSetupArgs: { + enableHastePackages: this.#enableHastePackages, + hasteImplModulePath: this.#hasteImplModulePath ?? null, + }, + }; + } } diff --git a/packages/metro-file-map/src/plugins/MockPlugin.js b/packages/metro-file-map/src/plugins/MockPlugin.js index 43ad69f589..20926e5ec5 100644 --- a/packages/metro-file-map/src/plugins/MockPlugin.js +++ b/packages/metro-file-map/src/plugins/MockPlugin.js @@ -13,6 +13,7 @@ import type { FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, + FileMapPluginWorker, MockMap as IMockMap, Path, RawMockMap, @@ -27,6 +28,14 @@ import path from 'path'; export const CACHE_VERSION = 2; +export type MockMapOptions = $ReadOnly<{ + console: typeof console, + mocksPattern: RegExp, + rawMockMap?: RawMockMap, + rootDir: Path, + throwOnModuleCollision: boolean, + }>; + export default class MockPlugin implements FileMapPlugin, IMockMap { +name = 'mocks'; @@ -47,13 +56,7 @@ export default class MockPlugin implements FileMapPlugin, IMockMap { }, rootDir, throwOnModuleCollision, - }: $ReadOnly<{ - console: typeof console, - mocksPattern: RegExp, - rawMockMap?: RawMockMap, - rootDir: Path, - throwOnModuleCollision: boolean, - }>) { + }: MockMapOptions) { this.#mocksPattern = mocksPattern; if (rawMockMap.version !== CACHE_VERSION) { throw new Error('Incompatible state passed to MockPlugin'); @@ -76,11 +79,11 @@ export default class MockPlugin implements FileMapPlugin, IMockMap { // Otherwise, traverse all files to rebuild await this.bulkUpdate({ addedOrModified: [ - ...files.metadataIterator({ + ...files.fileIterator({ includeNodeModules: false, includeSymlinks: false, }), - ].map(({canonicalPath, metadata}) => [canonicalPath, metadata]), + ].map(({canonicalPath}) => [canonicalPath, null]), removed: [], }); } @@ -97,7 +100,7 @@ export default class MockPlugin implements FileMapPlugin, IMockMap { ); } - async bulkUpdate(delta: FileMapDelta): Promise { + async bulkUpdate(delta: FileMapDelta<>): Promise { // Process removals first so that moves aren't treated as duplicates. for (const [relativeFilePath] of delta.removed) { this.onRemovedFile(relativeFilePath); @@ -213,4 +216,8 @@ export default class MockPlugin implements FileMapPlugin, IMockMap { this.#mocksPattern.flags ); } + + getWorker(): ?FileMapPluginWorker { + return null; + } } diff --git a/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js b/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js index 513994be01..508d83ec5a 100644 --- a/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js +++ b/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js @@ -9,7 +9,6 @@ * @oncall react_native */ -import type {FileMetadata} from '../../../flow-types'; import type HasteMapType from '../../HastePlugin'; let mockPathModule; @@ -25,22 +24,22 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { { canonicalPath: p('project/Foo.js'), baseName: 'Foo.js', - metadata: hasteMetadata('NameForFoo'), + pluginData: 'NameForFoo', }, { canonicalPath: p('project/Bar.js'), baseName: 'Bar.js', - metadata: hasteMetadata('Bar'), + pluginData: 'Bar', }, { canonicalPath: p('project/Duplicate.js'), baseName: 'Duplicate.js', - metadata: hasteMetadata('Duplicate'), + pluginData: 'Duplicate', }, { canonicalPath: p('project/other/Duplicate.js'), baseName: 'Duplicate.js', - metadata: hasteMetadata('Duplicate'), + pluginData: 'Duplicate', }, ]; @@ -69,18 +68,20 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { const hasteMap = new HasteMap(opts); const initialState = { files: { - metadataIterator: jest.fn().mockReturnValue([ + fileIterator: jest.fn().mockReturnValue([ { canonicalPath: p('project/Foo.js'), baseName: 'Foo.js', - metadata: hasteMetadata('NameForFoo'), + pluginData: 'NameForFoo', }, ]), + lookup: jest.fn(), }, pluginState: null, + processFile: jest.fn(), }; await hasteMap.initialize(initialState); - expect(initialState.files.metadataIterator).toHaveBeenCalledWith({ + expect(initialState.files.fileIterator).toHaveBeenCalledWith({ includeNodeModules: false, includeSymlinks: false, }); @@ -93,14 +94,18 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { beforeEach(async () => { hasteMap = new HasteMap(opts); await hasteMap.initialize({ - files: {metadataIterator: jest.fn().mockReturnValue(INITIAL_FILES)}, + files: { + fileIterator: jest.fn().mockReturnValue(INITIAL_FILES), + lookup: jest.fn(), + }, pluginState: null, + processFile: jest.fn(), }); }); test('removes a module, without affecting others', () => { expect(hasteMap.getModule('NameForFoo')).not.toBeNull(); - hasteMap.onRemovedFile(p('project/Foo.js'), hasteMetadata('NameForFoo')); + hasteMap.onRemovedFile(p('project/Foo.js'), 'NameForFoo'); expect(hasteMap.getModule('NameForFoo')).toBeNull(); expect(hasteMap.getModule('Bar')).not.toBeNull(); }); @@ -109,10 +114,7 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { expect(() => hasteMap.getModule('Duplicate')).toThrow( DuplicateHasteCandidatesError, ); - hasteMap.onRemovedFile( - p('project/Duplicate.js'), - hasteMetadata('Duplicate'), - ); + hasteMap.onRemovedFile(p('project/Duplicate.js'), 'Duplicate'); expect(hasteMap.getModule('Duplicate')).toBe( p('/root/project/other/Duplicate.js'), ); @@ -125,14 +127,18 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { beforeEach(async () => { hasteMap = new HasteMap(opts); await hasteMap.initialize({ - files: {metadataIterator: jest.fn().mockReturnValue(INITIAL_FILES)}, + files: { + fileIterator: jest.fn().mockReturnValue(INITIAL_FILES), + lookup: jest.fn(), + }, pluginState: null, + processFile: jest.fn(), }); }); test('removes a module, without affecting others', () => { expect(hasteMap.getModule('NameForFoo')).not.toBeNull(); - hasteMap.onRemovedFile(p('project/Foo.js'), hasteMetadata('NameForFoo')); + hasteMap.onRemovedFile(p('project/Foo.js'), 'NameForFoo'); expect(hasteMap.getModule('NameForFoo')).toBeNull(); expect(hasteMap.getModule('Bar')).not.toBeNull(); }); @@ -143,12 +149,12 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { ); await hasteMap.bulkUpdate({ removed: [ - [p('project/Duplicate.js'), hasteMetadata('Duplicate')], - [p('project/Foo.js'), hasteMetadata('NameForFoo')], + [p('project/Duplicate.js'), 'Duplicate'], + [p('project/Foo.js'), 'NameForFoo'], ], addedOrModified: [ - [p('project/Baz.js'), hasteMetadata('Baz')], // New - [p('project/other/Bar.js'), hasteMetadata('Bar')], // New duplicate + [p('project/Baz.js'), 'Baz'], // New + [p('project/other/Bar.js'), 'Bar'], // New duplicate ], }); expect(hasteMap.getModule('Duplicate')).toBe( @@ -161,8 +167,44 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { ); }); }); -}); -function hasteMetadata(hasteName: string): FileMetadata { - return [0, 0, 0, '', '', 0, hasteName]; -} + describe('getModuleNameByPath', () => { + let hasteMap: HasteMapType; + let lookup; + + beforeEach(async () => { + hasteMap = new HasteMap(opts); + lookup = jest.fn().mockReturnValue(null); + + await hasteMap.initialize({ + files: { + fileIterator: jest.fn().mockReturnValue(INITIAL_FILES), + lookup, + }, + pluginState: null, + processFile: jest.fn(), + }); + }); + + test('returns the correct module name', () => { + lookup.mockImplementation( + filePath => + ({ + [p('/root/Foo.js')]: { + exists: true, + type: 'f', + pluginData: 'Foo' as ?string, + }, + [p('/root/not-haste.js')]: { + exists: true, + type: 'f', + pluginData: null as ?string, + }, + })[filePath] ?? {exists: false}, + ); + expect(hasteMap.getModuleNameByPath(p('/root/Foo.js'))).toBe('Foo'); + expect(hasteMap.getModuleNameByPath(p('/root/not-haste.js'))).toBe(null); + expect(hasteMap.getModuleNameByPath(p('/root/not-exists.js'))).toBe(null); + }); + }); +}); diff --git a/packages/metro-file-map/src/plugins/haste/types.js b/packages/metro-file-map/src/plugins/haste/types.js new file mode 100644 index 0000000000..044ab781c2 --- /dev/null +++ b/packages/metro-file-map/src/plugins/haste/types.js @@ -0,0 +1,14 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +export type WorkerSetupArgs = $ReadOnly<{ + hasteImplModulePath: string, + enableHastePackages: boolean, +}>; diff --git a/packages/metro-file-map/src/plugins/haste/worker.js b/packages/metro-file-map/src/plugins/haste/worker.js new file mode 100644 index 0000000000..2a4779e13b --- /dev/null +++ b/packages/metro-file-map/src/plugins/haste/worker.js @@ -0,0 +1,63 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +/* eslint-disable import/no-commonjs */ + +'use strict'; + +const excludedExtensions = require('../../workerExclusionList'); +const path = require('path'); + +/*:: +import type {WorkerSetupArgs} from './types'; +import type {MetadataWorker, WorkerMessage, V8Serializable} from '../../flow-types'; +*/ + +const PACKAGE_JSON = path.sep + 'package.json'; + +module.exports = class Worker /*:: implements MetadataWorker */ { + #enableHastePackages /*: boolean */; + #hasteImpl /*: ?$ReadOnly<{getHasteName: string => ?string}> */; + #hasteImplModulePath /*: ?string */ = null; + + constructor(setupArgs /*: WorkerSetupArgs */) { + this.#enableHastePackages = setupArgs.enableHastePackages; + if (setupArgs.hasteImplModulePath) { + // $FlowFixMe[unsupported-syntax] - dynamic require + this.#hasteImpl = require(setupArgs.hasteImplModulePath); + } + } + + processFile( + data /*: WorkerMessage */, + utils /*: $ReadOnly<{getContent: () => Buffer }> */, + ) /*: V8Serializable */ { + let hasteName /*: string | null */ = null; + const {filePath} = data; + if (this.#enableHastePackages && filePath.endsWith(PACKAGE_JSON)) { + // Process a package.json that is returned as a PACKAGE type with its name. + try { + const fileData = JSON.parse(utils.getContent().toString()); + if (fileData.name) { + hasteName = fileData.name; + } + } catch (err) { + throw new Error(`Cannot parse ${filePath} as JSON: ${err.message}`); + } + } else if ( + this.#hasteImpl != null && + !excludedExtensions.has(filePath.substr(filePath.lastIndexOf('.'))) + ) { + // Process a random file that is returned as a MODULE. + hasteName = this.#hasteImpl?.getHasteName(filePath) || null; + } + return hasteName; + } +}; diff --git a/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js b/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js index 33fbbd3ef7..6796c29430 100644 --- a/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js +++ b/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js @@ -97,7 +97,10 @@ Duplicate manual mock found for \`foo\`: const mockMap = new MockMap(opts); await mockMap.initialize({ files: { - metadataIterator: () => { + fileIterator: () => { + throw new Error('should not be used'); + }, + lookup: () => { throw new Error('should not be used'); }, }, diff --git a/packages/metro-file-map/src/worker.js b/packages/metro-file-map/src/worker.js index 5c135dd397..d111a36a43 100644 --- a/packages/metro-file-map/src/worker.js +++ b/packages/metro-file-map/src/worker.js @@ -13,6 +13,8 @@ /*:: import type { DependencyExtractor, + FileMapPluginWorker, + MetadataWorker, WorkerMessage, WorkerMetadata, WorkerSetupArgs, @@ -25,43 +27,30 @@ const defaultDependencyExtractor = require('./lib/dependencyExtractor'); const excludedExtensions = require('./workerExclusionList'); const {createHash} = require('crypto'); const fs = require('graceful-fs'); -const path = require('path'); - -const PACKAGE_JSON = path.sep + 'package.json'; - -let hasteImpl /*: ?{getHasteName: string => ?string} */ = null; -let hasteImplModulePath /*: ?string */ = null; - -function getHasteImpl( - requestedModulePath /*: string */, -) /*: {getHasteName: string => ?string} */ { - if (hasteImpl) { - if (requestedModulePath !== hasteImplModulePath) { - throw new Error('metro-file-map: hasteImplModulePath changed'); - } - return hasteImpl; - } - hasteImplModulePath = requestedModulePath; - // $FlowFixMe[unsupported-syntax] - dynamic require - hasteImpl = require(hasteImplModulePath); - return hasteImpl; -} function sha1hex(content /*: string | Buffer */) /*: string */ { return createHash('sha1').update(content).digest('hex'); } class Worker { - constructor(args /*: WorkerSetupArgs */) {} + #dependencyExtractorPath /*: ?string */; + #plugins /*: $ReadOnlyArray */; + + constructor({plugins = [], dependencyExtractor} /*: WorkerSetupArgs */) { + this.#dependencyExtractorPath = dependencyExtractor; + this.#plugins = plugins.map(({workerModulePath, workerSetupArgs}) => { + // $FlowFixMe[unsupported-syntax] - dynamic require + const PluginWorker = require(workerModulePath); + return new PluginWorker(workerSetupArgs); + }); + } processFile(data /*: WorkerMessage */) /*: WorkerMetadata */ { let content /*: ?Buffer */; let dependencies /*: WorkerMetadata['dependencies'] */; - let id /*: WorkerMetadata['id'] */; let sha1 /*: WorkerMetadata['sha1'] */; - const {computeDependencies, computeSha1, enableHastePackages, filePath} = - data; + const {computeDependencies, computeSha1, filePath} = data; const getContent = () /*: Buffer */ => { if (content == null) { @@ -71,43 +60,30 @@ class Worker { return content; }; - if (enableHastePackages && filePath.endsWith(PACKAGE_JSON)) { - // Process a package.json that is returned as a PACKAGE type with its name. - try { - const fileData = JSON.parse(getContent().toString()); + const workerUtils = {getContent}; + const pluginData = this.#plugins.map(plugin => + plugin.processFile(data, workerUtils), + ); - if (fileData.name) { - id = fileData.name; - } - } catch (err) { - throw new Error(`Cannot parse ${filePath} as JSON: ${err.message}`); - } - } else if ( - (data.hasteImplModulePath != null || computeDependencies) && + if ( + computeDependencies && !excludedExtensions.has(filePath.substr(filePath.lastIndexOf('.'))) ) { - // Process a random file that is returned as a MODULE. - if (data.hasteImplModulePath != null) { - id = getHasteImpl(data.hasteImplModulePath).getHasteName(filePath); - } - - if (computeDependencies) { - const dependencyExtractor /*: ?DependencyExtractor */ = - data.dependencyExtractor != null - ? // $FlowFixMe[unsupported-syntax] - dynamic require - require(data.dependencyExtractor) - : null; - - dependencies = Array.from( - dependencyExtractor != null - ? dependencyExtractor.extract( - getContent().toString(), - filePath, - defaultDependencyExtractor.extract, - ) - : defaultDependencyExtractor.extract(getContent().toString()), - ); - } + const dependencyExtractor /*: ?DependencyExtractor */ = + this.#dependencyExtractorPath != null + ? // $FlowFixMe[unsupported-syntax] - dynamic require + require(this.#dependencyExtractorPath) + : null; + + dependencies = Array.from( + dependencyExtractor != null + ? dependencyExtractor.extract( + getContent().toString(), + filePath, + defaultDependencyExtractor.extract, + ) + : defaultDependencyExtractor.extract(getContent().toString()), + ); } // If a SHA-1 is requested on update, compute it. @@ -116,8 +92,8 @@ class Worker { } return content && data.maybeReturnContent - ? {content, dependencies, id, sha1} - : {dependencies, id, sha1}; + ? {content, dependencies, pluginData, sha1} + : {dependencies, pluginData, sha1}; } } diff --git a/packages/metro/src/node-haste/DependencyGraph.js b/packages/metro/src/node-haste/DependencyGraph.js index 1dc470932a..5182f59498 100644 --- a/packages/metro/src/node-haste/DependencyGraph.js +++ b/packages/metro/src/node-haste/DependencyGraph.js @@ -103,9 +103,12 @@ export default class DependencyGraph extends EventEmitter { type: 'dep_graph_loading', hasReducedPerformance: !!hasReducedPerformance, }); - const fileMap = createFileMap(config, { + this.#packageCache = this._createPackageCache(); + + const {fileMap, hasteMap} = createFileMap(config, { throwOnModuleCollision: false, watch, + extraPlugins: [this.#packageCache], }); // We can have a lot of graphs listening to Haste for changes. @@ -115,25 +118,20 @@ export default class DependencyGraph extends EventEmitter { this._haste = fileMap; this._haste.on('status', status => this._onWatcherStatus(status)); - this._initializedPromise = fileMap - .build() - .then(({fileSystem, hasteMap}) => { - log(createActionEndEntry(initializingMetroLogEntry)); - config.reporter.update({type: 'dep_graph_loaded'}); + this._initializedPromise = fileMap.build().then(({fileSystem}) => { + log(createActionEndEntry(initializingMetroLogEntry)); + config.reporter.update({type: 'dep_graph_loaded'}); - this._fileSystem = fileSystem; - this._hasteMap = hasteMap; + this._fileSystem = fileSystem; + this._hasteMap = hasteMap; - this._haste.on('change', changeEvent => - this._onHasteChange(changeEvent), - ); - this._haste.on('healthCheck', result => - this._onWatcherHealthCheck(result), - ); - this._resolutionCache = new Map(); - this.#packageCache = this._createPackageCache(); - this._createModuleResolver(); - }); + this._haste.on('change', changeEvent => this._onHasteChange(changeEvent)); + this._haste.on('healthCheck', result => + this._onWatcherHealthCheck(result), + ); + this._resolutionCache = new Map(); + this._createModuleResolver(); + }); } _onWatcherHealthCheck(result: HealthCheckResult) { @@ -383,7 +381,7 @@ export default class DependencyGraph extends EventEmitter { }; getHasteName(filePath: string): string { - const hasteName = this._fileSystem.getModuleName(filePath); + const hasteName = this._hasteMap.getModuleNameByPath(filePath); if (hasteName) { return hasteName; diff --git a/packages/metro/src/node-haste/DependencyGraph/createFileMap.js b/packages/metro/src/node-haste/DependencyGraph/createFileMap.js index cde3e4c933..c005ca33b0 100644 --- a/packages/metro/src/node-haste/DependencyGraph/createFileMap.js +++ b/packages/metro/src/node-haste/DependencyGraph/createFileMap.js @@ -10,9 +10,10 @@ */ import type {ConfigT} from 'metro-config'; +import type {HasteMap, InputOptions} from 'metro-file-map'; import ci from 'ci-info'; -import MetroFileMap, {DiskCacheManager} from 'metro-file-map'; +import MetroFileMap, {DiskCacheManager, HastePlugin} from 'metro-file-map'; function getIgnorePattern(config: ConfigT): RegExp { // For now we support both options @@ -59,8 +60,9 @@ export default function createFileMap( watch?: boolean, throwOnModuleCollision?: boolean, cacheFilePrefix?: string, + extraPlugins?: InputOptions['plugins'], }>, -): MetroFileMap { +): {fileMap: MetroFileMap, hasteMap: HasteMap} { const dependencyExtractor = options?.extractDependencies === false ? null @@ -72,7 +74,18 @@ export default function createFileMap( config.watcher.unstable_autoSaveCache ?? {}; const autoSave = watch && autoSaveEnabled ? autoSaveOpts : false; - return MetroFileMap.create({ + const hasteMap = new HastePlugin({ + platforms: new Set([ + ...config.resolver.platforms, + MetroFileMap.H.NATIVE_PLATFORM, + ]), + hasteImplModulePath: config.resolver.hasteImplModulePath, + enableHastePackages: config?.resolver.enableGlobalPackages, + rootDir: config.projectRoot, + failValidationOnConflicts: options?.throwOnModuleCollision ?? true, + }); + + const fileMap = new MetroFileMap({ cacheManagerFactory: config?.unstable_fileMapCacheManagerFactory ?? (factoryParams => @@ -86,7 +99,6 @@ export default function createFileMap( computeDependencies, computeSha1: !config.watcher.unstable_lazySha1, dependencyExtractor: config.resolver.dependencyExtractor, - enableHastePackages: config?.resolver.enableGlobalPackages, enableSymlinks: true, enableWorkerThreads: config.watcher.unstable_workerThreads, extensions: Array.from( @@ -97,19 +109,17 @@ export default function createFileMap( ]), ), forceNodeFilesystemAPI: !config.resolver.useWatchman, - hasteImplModulePath: config.resolver.hasteImplModulePath, healthCheck: config.watcher.healthCheck, ignorePattern: getIgnorePattern(config), maxWorkers: config.maxWorkers, - mocksPattern: '', - platforms: [...config.resolver.platforms, MetroFileMap.H.NATIVE_PLATFORM], + plugins: [hasteMap, ...(options?.extraPlugins ?? [])], retainAllFiles: true, resetCache: config.resetCache, rootDir: config.projectRoot, roots: config.watchFolders, - throwOnModuleCollision: options?.throwOnModuleCollision ?? true, useWatchman: config.resolver.useWatchman, watch, watchmanDeferStates: config.watcher.watchman.deferStates, }); + return {fileMap, hasteMap}; } diff --git a/packages/metro/src/node-haste/Package.js b/packages/metro/src/node-haste/Package.js index ad906ee3f4..9037211a87 100644 --- a/packages/metro/src/node-haste/Package.js +++ b/packages/metro/src/node-haste/Package.js @@ -11,7 +11,6 @@ import type {PackageJson} from 'metro-resolver/private/types'; -import fs from 'fs'; import path from 'path'; export default class Package { @@ -19,11 +18,20 @@ export default class Package { _root: string; _content: ?PackageJson; + #readAndParse: () => PackageJson; - constructor({file}: {file: string, ...}) { + constructor({ + file, + readAndParse, + }: { + file: string, + readAndParse: () => PackageJson, + ... + }) { this.path = path.resolve(file); this._root = path.dirname(this.path); this._content = null; + this.#readAndParse = readAndParse; } invalidate() { @@ -32,7 +40,7 @@ export default class Package { read(): PackageJson { if (this._content == null) { - this._content = JSON.parse(fs.readFileSync(this.path, 'utf8')); + this._content = this.#readAndParse(); } return this._content; } diff --git a/packages/metro/src/node-haste/PackageCache.js b/packages/metro/src/node-haste/PackageCache.js index 74323a719b..efc2e7d51f 100644 --- a/packages/metro/src/node-haste/PackageCache.js +++ b/packages/metro/src/node-haste/PackageCache.js @@ -9,14 +9,17 @@ * @oncall react_native */ +import type {PackageJson} from 'metro-resolver/private/types'; + import Package from './Package'; +import AbstractDataPlugin from 'metro-file-map/private/plugins/AbstractDataPlugin'; type GetClosestPackageFn = (absoluteFilePath: string) => ?{ packageJsonPath: string, packageRelativePath: string, }; -export class PackageCache { +export class PackageCache extends AbstractDataPlugin { _getClosestPackage: GetClosestPackageFn; _packageCache: { [filePath: string]: Package, @@ -40,6 +43,14 @@ export class PackageCache { }; constructor(options: {getClosestPackage: GetClosestPackageFn, ...}) { + super({ + name: 'package-cache', + workerParams: { + workerModulePath: require.resolve('./lib/packageJsonWorker'), + workerSetupArgs: {}, + filter: filePath => filePath.endsWith('package.json'), + }, + }); this._getClosestPackage = options.getClosestPackage; this._packageCache = Object.create(null); this._packagePathAndSubpathByModulePath = Object.create(null); @@ -50,11 +61,26 @@ export class PackageCache { if (!this._packageCache[filePath]) { this._packageCache[filePath] = new Package({ file: filePath, + readAndParse: () => { + const result = this.lookup(filePath); + if (result.exists == false || result.type !== 'f') { + throw new Error('missing'); + } + // In lazy mode, we haven't read the file yet. + if (typeof result.pluginData === 'undefined') { + return this.processFile(filePath); + } + return result.pluginData; + }, }); } return this._packageCache[filePath]; } + getCacheKey(): string { + return 'package-cache-1'; + } + getPackageOf( absoluteModulePath: string, ): ?{pkg: Package, packageRelativePath: string} { diff --git a/packages/metro/src/node-haste/lib/packageJsonWorker.js b/packages/metro/src/node-haste/lib/packageJsonWorker.js new file mode 100644 index 0000000000..a912c4b822 --- /dev/null +++ b/packages/metro/src/node-haste/lib/packageJsonWorker.js @@ -0,0 +1,38 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +/* eslint-disable import/no-commonjs */ + +'use strict'; + +const path = require('path'); + +/*:: +import type {MetadataWorker, WorkerMessage, V8Serializable} from 'metro-file-map/private/flow-types'; +*/ + +const PACKAGE_JSON = path.sep + 'package.json'; + +module.exports = class Worker /*:: implements MetadataWorker */ { + processFile( + data /*: WorkerMessage */, + {getContent} /*: $ReadOnly<{getContent: () => Buffer }> */, + ) /*: V8Serializable */ { + if (!data.filePath.endsWith(PACKAGE_JSON)) { + return null; + } + const content = getContent(); + try { + return JSON.parse(content.toString()); + } catch (e) { + return null; + } + } +};