diff --git a/src/meta/__tests__/build.test.ts b/src/meta/__tests__/build.test.ts index 468f80c0..10d402fb 100644 --- a/src/meta/__tests__/build.test.ts +++ b/src/meta/__tests__/build.test.ts @@ -7,6 +7,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest import { buildProd } from '../build.js'; import * as externalPluginModule from '../plugins/external.js'; +import * as nativeDependenciesPluginModule from '../plugins/native-dependencies.js'; import type { UserConfigOptions } from '../../user-config.js'; @@ -154,6 +155,84 @@ describe('buildProd', () => { vi.doUnmock('esbuild'); }); + it('should not register the native dependencies plugin when none are declared', { timeout: 10000 }, async () => { + const nativeDependenciesPlugin = vi.spyOn(nativeDependenciesPluginModule, 'nativeDependenciesPlugin'); + const outfile = path.join(outdir, 'module-no-native.js'); + loadUserConfig.mockReturnValue( + okAsync({ + build: { + mode: 'module', + nativeDependencies: [], + outfile + }, + entry: vi.fn() + } satisfies UserConfigOptions) + ); + parseEntryFromFunction.mockReturnValueOnce(ok('./example/app.js')); + const result = await buildProd({ configFile }); + expect(result.isOk()).toBe(true); + expect(nativeDependenciesPlugin).not.toHaveBeenCalled(); + }); + + it('should emit a declared native dependency beside the bundle', { timeout: 10000 }, async () => { + const artifact = path.join(outdir, 'artifact-source'); + await fs.promises.writeFile(artifact, 'native artifact'); + const outfile = path.join(outdir, 'module-native.js'); + loadUserConfig.mockReturnValue( + okAsync({ + build: { + mode: 'module', + // `neverthrow` stands in for a native package: it is already in the example app's graph, so + // the plugin observes a real resolution rather than a stubbed one. + nativeDependencies: [ + { + locate: () => artifact, + outputName: 'artifact', + packageName: 'neverthrow', + runtimeEnvVar: 'ARTIFACT_BINARY_PATH' + } + ], + outfile + }, + entry: vi.fn() + } satisfies UserConfigOptions) + ); + parseEntryFromFunction.mockReturnValueOnce(ok('./example/app.js')); + + const result = await buildProd({ configFile }); + + expect(result.isOk()).toBe(true); + expect(fs.existsSync(path.join(outdir, 'artifact'))).toBe(true); + expect(fs.readFileSync(outfile, 'utf-8')).toContain('process.env.ARTIFACT_BINARY_PATH ??='); + }); + + it('should fail the build when a declared native dependency is never imported', async () => { + const outfile = path.join(outdir, 'module-missing-native.js'); + loadUserConfig.mockReturnValue( + okAsync({ + build: { + mode: 'module', + nativeDependencies: [ + { + locate: () => '/dev/null', + outputName: 'absent', + packageName: '@scope/never-imported', + runtimeEnvVar: 'ABSENT_BINARY_PATH' + } + ], + outfile + }, + entry: vi.fn() + } satisfies UserConfigOptions) + ); + parseEntryFromFunction.mockReturnValueOnce(ok('./example/app.js')); + + const result = await buildProd({ configFile }); + + expect(result.isErr()).toBe(true); + expect(result).toMatchObject({ error: { message: 'Failed to build application' } }); + }); + it('should bundle with bundle:false to mark node_modules as external', { timeout: 10000 }, async () => { const externalPlugin = vi.spyOn(externalPluginModule, 'externalPlugin'); const outfile = path.join(outdir, 'module-unbundled.js'); diff --git a/src/meta/__tests__/native-dependencies.test.ts b/src/meta/__tests__/native-dependencies.test.ts new file mode 100644 index 00000000..bb33f425 --- /dev/null +++ b/src/meta/__tests__/native-dependencies.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + KNOWN_NATIVE_DEPENDENCIES, + KNOWN_NATIVE_DEPENDENCY_NAMES, + resolveNativeDependency +} from '../native-dependencies.js'; + +import type { NativeDependency } from '../../user-config.js'; + +describe('resolveNativeDependency', () => { + it('should expand a known name to its built-in recipe', () => { + expect(resolveNativeDependency('esbuild')).toBe(KNOWN_NATIVE_DEPENDENCIES.esbuild); + }); + + it('should pass a custom dependency through unchanged', () => { + const custom: NativeDependency = { + locate: () => '/artifact', + outputName: 'artifact', + packageName: 'custom', + runtimeEnvVar: 'CUSTOM_BINARY_PATH' + }; + expect(resolveNativeDependency(custom)).toBe(custom); + }); +}); + +describe('KNOWN_NATIVE_DEPENDENCY_NAMES', () => { + it('should list every built-in recipe, so the config validator accepts each one', () => { + expect(KNOWN_NATIVE_DEPENDENCY_NAMES).toStrictEqual(Object.keys(KNOWN_NATIVE_DEPENDENCIES)); + }); +}); + +describe('esbuild recipe', () => { + it('should resolve the executable of the platform package for the current host', () => { + const resolve = vi.fn().mockReturnValue('/pkgs/@esbuild/target/bin/esbuild'); + + const artifact = KNOWN_NATIVE_DEPENDENCIES.esbuild.locate({ + entryPath: '/pkgs/esbuild/lib/main.js', + require: { resolve } as any + }); + + expect(resolve).toHaveBeenCalledWith(`@esbuild/${process.platform}-${process.arch}/bin/esbuild`); + expect(artifact).toBe('/pkgs/@esbuild/target/bin/esbuild'); + }); +}); diff --git a/src/meta/build.ts b/src/meta/build.ts index 0ad8ea34..1774ce47 100644 --- a/src/meta/build.ts +++ b/src/meta/build.ts @@ -4,9 +4,11 @@ import { RuntimeException } from '@douglasneuroinformatics/libjs'; import { fromAsyncThrowable, ok, ResultAsync } from 'neverthrow'; import { loadUserConfig } from './load.js'; +import { resolveNativeDependency } from './native-dependencies.js'; import { parseEntryFromFunction } from './parse.js'; import { docsPlugin } from './plugins/docs.js'; import { externalPlugin } from './plugins/external.js'; +import { nativeDependenciesPlugin } from './plugins/native-dependencies.js'; import { prismaPlugin } from './plugins/prisma.js'; import { swcPlugin } from './plugins/swc.js'; @@ -55,6 +57,10 @@ export function buildProd({ plugins.push(externalPlugin()); } + if (config.build.nativeDependencies?.length) { + plugins.push(nativeDependenciesPlugin(config.build.nativeDependencies.map(resolveNativeDependency))); + } + await esbuild.build({ banner: { js: "Object.defineProperties(globalThis, { __dirname: { value: import.meta.dirname, writable: false }, __filename: { value: import.meta.filename, writable: false }, require: { value: (await import('module')).createRequire(import.meta.url), writable: false } });" diff --git a/src/meta/load.ts b/src/meta/load.ts index 64280303..203042ec 100644 --- a/src/meta/load.ts +++ b/src/meta/load.ts @@ -7,6 +7,7 @@ import { z } from 'zod/v4'; import { AbstractAppContainer } from '../app/app.base.js'; import { importDefault } from './import.js'; +import { KNOWN_NATIVE_DEPENDENCY_NAMES } from './native-dependencies.js'; import { parseEntryFromFunction } from './parse.js'; import type { UserConfigOptions } from '../user-config.js'; @@ -14,11 +15,18 @@ import type { UserConfigOptions } from '../user-config.js'; // we cannot use zod function here as we cannot have any wrappers apply and screw up toString representation const $AnyFunction = z.custom<(...args: any[]) => any>((arg) => typeof arg === 'function', 'must be function'); +const $NativeDependency = z.object({ + locate: $AnyFunction, + outputName: z.string().min(1), + packageName: z.string().min(1), + runtimeEnvVar: z.string().regex(/^[A-Z_][A-Z0-9_]*$/, 'must be a valid environment variable name') +}); + const $UserConfigOptions: z.ZodType = z.object({ build: z.object({ bundle: z.boolean().optional(), - esbuildOptions: z.record(z.string(), z.any()).optional(), mode: z.enum(['module', 'server']).optional(), + nativeDependencies: z.array(z.union([z.enum(KNOWN_NATIVE_DEPENDENCY_NAMES), $NativeDependency])).optional(), onComplete: $AnyFunction.optional(), outfile: z.string().min(1) }), diff --git a/src/meta/native-dependencies.ts b/src/meta/native-dependencies.ts new file mode 100644 index 00000000..e01c9e25 --- /dev/null +++ b/src/meta/native-dependencies.ts @@ -0,0 +1,31 @@ +import type { KnownNativeDependencyName, NativeDependency } from '../user-config.js'; + +/** + * Built-in recipes, so naming the package is enough for the common cases. + * + * Each `locate` resolves through the `require` of the module that imported the package, never through + * `libnest`'s own. The application and `libnest` routinely resolve different versions of the same + * dependency, and pairing an artifact with a different version of its own JavaScript fails at runtime. + */ +const KNOWN_NATIVE_DEPENDENCIES = { + esbuild: { + // The executable lives in a per-platform sibling package listed in esbuild's optionalDependencies. + // `ESBUILD_BINARY_PATH` both points esbuild at it and suppresses esbuild's refusal to run from a + // bundle, which it otherwise detects by comparing __filename against its own layout. + locate: ({ require }): string => require.resolve(`@esbuild/${process.platform}-${process.arch}/bin/esbuild`), + outputName: 'esbuild', + packageName: 'esbuild', + runtimeEnvVar: 'ESBUILD_BINARY_PATH' + } +} satisfies { [K in KnownNativeDependencyName]: NativeDependency }; + +const KNOWN_NATIVE_DEPENDENCY_NAMES = Object.keys(KNOWN_NATIVE_DEPENDENCIES) as [ + KnownNativeDependencyName, + ...KnownNativeDependencyName[] +]; + +function resolveNativeDependency(dependency: KnownNativeDependencyName | NativeDependency): NativeDependency { + return typeof dependency === 'string' ? KNOWN_NATIVE_DEPENDENCIES[dependency] : dependency; +} + +export { KNOWN_NATIVE_DEPENDENCIES, KNOWN_NATIVE_DEPENDENCY_NAMES, resolveNativeDependency }; diff --git a/src/meta/plugins/__tests__/native-dependencies.test.ts b/src/meta/plugins/__tests__/native-dependencies.test.ts new file mode 100644 index 00000000..f30fa7b6 --- /dev/null +++ b/src/meta/plugins/__tests__/native-dependencies.test.ts @@ -0,0 +1,158 @@ +import * as path from 'node:path'; + +import type { PluginBuild } from 'esbuild'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { nativeDependenciesPlugin } from '../native-dependencies.js'; + +import type { NativeDependency } from '../../../user-config.js'; + +const fs = vi.hoisted(() => ({ + chmod: vi.fn(), + copyFile: vi.fn() +})); + +vi.mock('node:fs/promises', () => fs); + +const createRequire = vi.hoisted(() => vi.fn()); + +vi.mock('node:module', () => ({ createRequire })); + +const ENTRY_PATH = '/pkgs/widget/lib/main.js'; +const ARTIFACT_PATH = '/pkgs/widget-linux-x64/bin/widget'; + +const dependency: NativeDependency = { + locate: vi.fn(() => ARTIFACT_PATH), + outputName: 'widget', + packageName: 'widget', + runtimeEnvVar: 'WIDGET_BINARY_PATH' +}; + +/** + * A stub of the subset of `PluginBuild` the plugin touches, exposing the two registered callbacks so a + * test can drive resolution and completion directly. + */ +function createBuild({ resolvedPath = ENTRY_PATH }: { resolvedPath?: string } = {}) { + const build = { + initialOptions: { + banner: { js: '' }, + outdir: '/app' + }, + onEnd: vi.fn(), + onResolve: vi.fn(), + resolve: vi.fn().mockResolvedValue({ path: resolvedPath }) + } satisfies Partial<{ [K in keyof PluginBuild]: any }>; + return { + build, + onEnd: () => build.onEnd.mock.lastCall![0] as () => Promise, + onResolve: () => + build.onResolve.mock.lastCall![1] as (args: { kind: string; path: string; resolveDir: string }) => Promise + }; +} + +const resolveArgs = (specifier: string) => ({ kind: 'import-statement', path: specifier, resolveDir: '/app' }); + +describe('nativeDependenciesPlugin', () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it('should assign the environment variable without overwriting one supplied by the operator', async () => { + const { build } = createBuild(); + await nativeDependenciesPlugin([dependency]).setup(build as any); + expect(build.initialOptions.banner.js).toBe("process.env.WIDGET_BINARY_PATH ??= import.meta.dirname + '/widget';"); + }); + + it('should terminate its banner statement, so a second appending plugin still parses', async () => { + const { build } = createBuild(); + await nativeDependenciesPlugin([dependency]).setup(build as any); + expect(build.initialOptions.banner.js.endsWith(';')).toBe(true); + }); + + it('should emit the artifact beside the bundle and make it executable', async () => { + const { build, onEnd, onResolve } = createBuild(); + createRequire.mockReturnValue({ resolve: vi.fn() }); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await onResolve()(resolveArgs('widget')); + await onEnd()(); + + expect(dependency.locate).toHaveBeenCalledWith(expect.objectContaining({ entryPath: ENTRY_PATH })); + expect(fs.copyFile).toHaveBeenCalledExactlyOnceWith(ARTIFACT_PATH, path.join('/app', 'widget')); + expect(fs.chmod).toHaveBeenCalledExactlyOnceWith(path.join('/app', 'widget'), 0o755); + }); + + it('should write beside the bundle when the output is an outfile rather than an outdir', async () => { + const { build, onEnd, onResolve } = createBuild(); + createRequire.mockReturnValue({ resolve: vi.fn() }); + const outfileBuild = { + ...build, + initialOptions: { banner: { js: '' }, outfile: '/dist/server.js' } + }; + await nativeDependenciesPlugin([dependency]).setup(outfileBuild as any); + + await onResolve()(resolveArgs('widget')); + await onEnd()(); + + expect(fs.copyFile).toHaveBeenCalledExactlyOnceWith(ARTIFACT_PATH, path.join('/dist', 'widget')); + }); + + it('should locate the artifact through a require rooted at the importing module, not at libnest', async () => { + const { build, onEnd, onResolve } = createBuild(); + const requireFn = { resolve: vi.fn() }; + createRequire.mockReturnValue(requireFn); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await onResolve()(resolveArgs('widget')); + await onEnd()(); + + expect(createRequire).toHaveBeenCalledWith(ENTRY_PATH); + expect(dependency.locate).toHaveBeenCalledWith({ entryPath: ENTRY_PATH, require: requireFn }); + }); + + it('should match a subpath import of a declared dependency', async () => { + const { build, onResolve } = createBuild(); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await onResolve()(resolveArgs('widget/lib/main.js')); + + expect(build.resolve).toHaveBeenCalledOnce(); + }); + + it('should ignore an import that is not a declared dependency', async () => { + const { build, onResolve } = createBuild(); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await expect(onResolve()(resolveArgs('widgetry'))).resolves.toBeNull(); + expect(build.resolve).not.toHaveBeenCalled(); + }); + + it('should resolve a dependency only once however many times it is imported', async () => { + const { build, onResolve } = createBuild(); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await onResolve()(resolveArgs('widget')); + await onResolve()(resolveArgs('widget')); + + expect(build.resolve).toHaveBeenCalledOnce(); + }); + + it('should throw when a declared dependency is never imported, rather than emitting a bundle pointing at nothing', async () => { + const { build, onEnd } = createBuild(); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await expect(onEnd()()).rejects.toThrowError( + "Declared native dependency 'widget' was never imported by the application, so its native artifact cannot be located" + ); + expect(fs.copyFile).not.toHaveBeenCalled(); + }); + + it('should throw when a declared dependency cannot be resolved, so an unresolved import is not silently skipped', async () => { + const { build, onEnd, onResolve } = createBuild({ resolvedPath: '' }); + await nativeDependenciesPlugin([dependency]).setup(build as any); + + await onResolve()(resolveArgs('widget')); + + await expect(onEnd()()).rejects.toThrowError("Declared native dependency 'widget' was never imported"); + }); +}); diff --git a/src/meta/plugins/__tests__/prisma.test.ts b/src/meta/plugins/__tests__/prisma.test.ts index 93dcba16..746cf9e1 100644 --- a/src/meta/plugins/__tests__/prisma.test.ts +++ b/src/meta/plugins/__tests__/prisma.test.ts @@ -63,6 +63,12 @@ describe('prismaPlugin', () => { ); }); + it('should terminate its banner statement, so a second appending plugin still parses', async () => { + fs.readdir.mockResolvedValueOnce([mockTargetFile]); + await prismaPlugin().setup(build as any); + expect(build.initialOptions.banner.js.endsWith(';')).toBe(true); + }); + it('should copy the binary to the target directory', async () => { fs.readdir.mockResolvedValueOnce([mockTargetFile]); const plugin = prismaPlugin(); diff --git a/src/meta/plugins/native-dependencies.ts b/src/meta/plugins/native-dependencies.ts new file mode 100644 index 00000000..90d693e3 --- /dev/null +++ b/src/meta/plugins/native-dependencies.ts @@ -0,0 +1,73 @@ +import * as fs from 'node:fs/promises'; +import * as module from 'node:module'; +import * as path from 'node:path'; + +import type { Plugin } from 'esbuild'; + +import type { NativeDependency } from '../../user-config.js'; + +/** + * Emits the native artifact of each declared dependency beside the bundle, and prepends an assignment + * pointing the package at it, since a bundled package can no longer find an artifact stored relative + * to its own source. + * + * The artifact is located from the module that actually imported the package during this build, rather + * than from a lookup rooted in `libnest`. A transitive dependency frequently resolves to a different + * version than `libnest`'s own, and shipping an artifact that does not match its JavaScript produces a + * version-mismatch failure on first use. + */ +export function nativeDependenciesPlugin(dependencies: NativeDependency[]): Plugin { + return { + name: 'native-dependencies', + setup(build): void { + const options = build.initialOptions; + const outdir = options.outfile ? path.dirname(options.outfile) : options.outdir!; + + // Populated during resolution, consumed once the graph is complete. + const entryPaths = new Map(); + + for (const { outputName, runtimeEnvVar } of dependencies) { + // `??=` so a value supplied by the operator still wins at runtime. + options.banner!.js! += `process.env.${runtimeEnvVar} ??= import.meta.dirname + '/${outputName}';`; + } + + build.onResolve({ filter: /.*/, namespace: 'file' }, async (args) => { + const dependency = dependencies.find(({ packageName }) => { + return args.path === packageName || args.path.startsWith(`${packageName}/`); + }); + if (!dependency || entryPaths.has(dependency.packageName)) { + return null; + } + // esbuild passes `pluginName` on this call, so it cannot re-enter this plugin. + const resolved = await build.resolve(args.path, { + kind: args.kind, + resolveDir: args.resolveDir + }); + if (resolved.path) { + entryPaths.set(dependency.packageName, resolved.path); + } + // Resolution is observed, never altered -- the package itself is still bundled as normal. + return null; + }); + + build.onEnd(async () => { + for (const dependency of dependencies) { + const entryPath = entryPaths.get(dependency.packageName); + if (!entryPath) { + throw new Error( + `Declared native dependency '${dependency.packageName}' was never imported by the application, so its native artifact cannot be located` + ); + } + const artifact = await dependency.locate({ + entryPath, + require: module.createRequire(entryPath) + }); + const destination = path.join(outdir, dependency.outputName); + await fs.copyFile(artifact, destination); + // Copying preserves the source mode, but only when the destination does not already exist. + await fs.chmod(destination, 0o755); + } + }); + } + }; +} diff --git a/src/meta/plugins/prisma.ts b/src/meta/plugins/prisma.ts index cc17da95..a748338d 100644 --- a/src/meta/plugins/prisma.ts +++ b/src/meta/plugins/prisma.ts @@ -26,7 +26,10 @@ export function prismaPlugin(): Plugin { throw new Error(`Failed to find prisma query engine for target '${binaryTarget}' in directory '${enginesDir}'`); } - build.initialOptions.banner!.js! += `process.env.PRISMA_QUERY_ENGINE_LIBRARY = import.meta.dirname + '/' + '${engineFile}'`; + // The trailing semicolon is load-bearing: the banner concatenates statements from every plugin + // that appends to it, with no separator between them. Without it, a second appending plugin + // produces a bundle that does not parse. + build.initialOptions.banner!.js! += `process.env.PRISMA_QUERY_ENGINE_LIBRARY = import.meta.dirname + '/' + '${engineFile}';`; build.onEnd(async () => { return fs.copyFile(path.join(enginesDir, engineFile), path.join(outdir, engineFile)); diff --git a/src/user-config.ts b/src/user-config.ts index d9c024ba..fae3d800 100644 --- a/src/user-config.ts +++ b/src/user-config.ts @@ -2,6 +2,35 @@ import type { ConditionalKeys, IfEmptyObject, Jsonifiable, Promisable } from 'ty import type { BaseEnv } from './schemas/env.schema.js'; +/** A `require` rooted at a specific module, so lookups resolve in the application's dependency graph. */ +type ResolvedRequire = ReturnType; + +/** Packages `libnest` ships a built-in {@link NativeDependency} recipe for. */ +export type KnownNativeDependencyName = 'esbuild'; + +export type NativeDependencyContext = { + /** Absolute path of the resolved module that pulled the dependency into the bundle. */ + entryPath: string; + /** A `require` rooted at {@link NativeDependencyContext.entryPath}. */ + require: ResolvedRequire; +}; + +/** + * A dependency that loads a native artifact through a path relative to its own source, and so cannot + * survive being bundled. The artifact is emitted beside the bundle, and the package is pointed back at + * it by an environment variable set before the application starts. + */ +export type NativeDependency = { + /** Returns the absolute path of the artifact to emit. */ + locate: (context: NativeDependencyContext) => Promisable; + /** The filename to write into the output directory, beside the bundle. */ + outputName: string; + /** The bare specifier the application imports, e.g. `esbuild`. */ + packageName: string; + /** The environment variable the package reads at runtime to locate its artifact. */ + runtimeEnvVar: string; +}; + /** * Configuration options for a `libnest` application. */ @@ -10,6 +39,12 @@ export type UserConfigOptions = { build: { bundle?: boolean; mode?: 'module' | 'server'; + /** + * Dependencies that cannot survive bundling, because they load a native artifact through a path + * relative to their own source. Each is emitted next to the bundle and redirected there at + * startup. Naming one the application never imports fails the build. + */ + nativeDependencies?: (KnownNativeDependencyName | NativeDependency)[]; /** A callback function to invoke when the build is complete */ onComplete?: () => Promisable; /** The path where the bundle should be written */