From c5e0a09a13764875eafd84ca869f23565e4ff758 Mon Sep 17 00:00:00 2001 From: Ethan Soh Date: Thu, 4 Jun 2026 11:54:40 +0800 Subject: [PATCH 1/2] feat: db:generate works without an external Postgres (auto PGlite) (#2) When no external DB env is configured, generate now starts an in-process PGlite, runs migrations, serves it over a local pg-gateway wire port, and points Zapatos + pg-schema-dump at it (then tears down). External Postgres path preserved when DB env vars are present. Adds the missing db:generate script and awaits the async CLI calls. Co-Authored-By: Claude Opus 4.8 --- README.md | 2 +- bun.lock | 1 + package.json | 1 + src/cli.ts | 4 +- src/generate.ts | 120 ++++++++++++++++++++++------------ tests/generate.pglite.test.ts | 67 +++++++++++++------ 6 files changed, 129 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 20e3127..db4e47a 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ npm install pgstrap --save-dev - `npm run db:migrate` - Run pending migrations - `npm run db:reset` - Drop and recreate the database, then run all migrations -- `npm run db:generate` - Generate types and structure dumps. Use `pgstrap generate --pglite` to run migrations against an in-memory PGlite instance. +- `npm run db:generate` - Generate types and structure dumps. When no Postgres connection env is configured, pgstrap runs migrations against an in-memory PGlite instance automatically. - `npm run db:create-migration` - Create a new migration file ### Configuration diff --git a/bun.lock b/bun.lock index 83407d9..e43bb07 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "pgstrap", diff --git a/package.json b/package.json index 18721c9..b01b599 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "scripts": { "test": "bun test", "build": "tsup ./src/index.ts ./src/cli.ts --outDir ./dist --dts --sourcemap inline", + "db:generate": "bun ./src/cli.ts generate", "format": "biome format . --write", "format:check": "biome format ." }, diff --git a/src/cli.ts b/src/cli.ts index 9a9bdec..7208056 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -29,7 +29,7 @@ import { getProjectContext } from "./get-project-context" process.exit(0) }) .command("migrate", "migrates the database", {}, async () => { - migrate(await getProjectContext()) + await migrate(await getProjectContext()) }) .command( "generate", @@ -38,7 +38,7 @@ import { getProjectContext } from "./get-project-context" yargs.option("pglite", { type: "boolean", default: false }) }, async (argv) => { - generate({ ...(await getProjectContext()), pglite: !!argv.pglite }) + await generate({ ...(await getProjectContext()), pglite: !!argv.pglite }) }, ) .parse() diff --git a/src/generate.ts b/src/generate.ts index f337094..b4eedc5 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -1,33 +1,63 @@ import * as zg from "zapatos/generate" -import { - getConnectionStringFromEnv, - getPgConnectionFromEnv, -} from "pg-connection-from-env" +import { getConnectionStringFromEnv } from "pg-connection-from-env" import { Context } from "./get-project-context" import { dumpTree } from "pg-schema-dump" import path from "path" import { migrate } from "./migrate" -export const generate = async ({ +const postgresEnvKeys = [ + "DATABASE_URL", + "PGHOST", + "PGPORT", + "PGDATABASE", + "PGUSER", + "PGPASSWORD", +] + +const hasExternalDatabaseConfig = ( + env: Record = process.env, +) => postgresEnvKeys.some((key) => Boolean(env[key])) + +const generateWithPglite = async ({ schemas, defaultDatabase, dbDir, - pglite = false, migrationsDir, }: Pick & { - pglite?: boolean - migrationsDir?: string + migrationsDir: string }) => { - dbDir = dbDir ?? "./src/db" - migrationsDir = migrationsDir ?? path.join(dbDir, "migrations") + const { PGlite } = await import("@electric-sql/pglite") + const { fromNodeSocket } = await import("pg-gateway/node") + const net = await import("node:net") - if (pglite) { - const { PGlite } = await import("@electric-sql/pglite") - const { fromNodeSocket } = await import("pg-gateway/node") - const net = await import("node:net") + const db = new PGlite() + const server = net.createServer(async (socket) => { + await fromNodeSocket(socket, { + serverVersion: "16.3 (PGlite)", + auth: { + method: "password", + validateCredentials: ({ username, password }: any) => + username === "postgres" && password === "postgres", + getClearTextPassword: () => "postgres", + }, + async onStartup() { + await (db as any).waitReady + }, + async onMessage(data: Uint8Array, { isAuthenticated }: any) { + if (!isAuthenticated) return + try { + const { data: responseData } = await (db as any).execProtocol(data) + return responseData + } catch { + return undefined + } + }, + }) + }) - const db = new PGlite() + const prevDbUrl = process.env.DATABASE_URL + try { await migrate({ client: db as any, migrationsDir, @@ -36,35 +66,10 @@ export const generate = async ({ schemas, }) - const server = net.createServer(async (socket) => { - const connection = await fromNodeSocket(socket, { - serverVersion: "16.3 (PGlite)", - auth: { - method: "password", - validateCredentials: ({ username, password }: any) => - username === "postgres" && password === "postgres", - getClearTextPassword: () => "postgres", - }, - async onStartup() { - await (db as any).waitReady - }, - async onMessage(data: Uint8Array, { isAuthenticated }: any) { - if (!isAuthenticated) return - try { - const { data: responseData } = await (db as any).execProtocol(data) - return responseData - } catch { - return undefined - } - }, - }) - }) - await new Promise((resolve) => server.listen(0, resolve)) const port = (server.address() as any).port const connectionString = `postgres://postgres:postgres@127.0.0.1:${port}/postgres` - const prevDbUrl = process.env.DATABASE_URL process.env.DATABASE_URL = connectionString await zg.generate({ @@ -82,10 +87,41 @@ export const generate = async ({ defaultDatabase: "postgres", schemas, }) - - server.close() + } finally { if (prevDbUrl === undefined) delete process.env.DATABASE_URL else process.env.DATABASE_URL = prevDbUrl + + await new Promise((resolve, reject) => { + server.close((err) => { + if (err) reject(err) + else resolve() + }) + }).catch(() => null) + + await (db as any).close?.() + } +} + +export const generate = async ({ + schemas, + defaultDatabase, + dbDir, + pglite = false, + migrationsDir, +}: Pick & { + pglite?: boolean + migrationsDir?: string +}) => { + dbDir = dbDir ?? "./src/db" + migrationsDir = migrationsDir ?? path.join(dbDir, "migrations") + + if (pglite || !hasExternalDatabaseConfig()) { + await generateWithPglite({ + schemas, + defaultDatabase, + dbDir, + migrationsDir, + }) return } diff --git a/tests/generate.pglite.test.ts b/tests/generate.pglite.test.ts index 56dcd53..79cd1a8 100644 --- a/tests/generate.pglite.test.ts +++ b/tests/generate.pglite.test.ts @@ -13,35 +13,60 @@ exports.down = async (pgm) => { } ` -test("generate with pglite runs migrations and dumps structure", async () => { +const postgresEnvKeys = [ + "DATABASE_URL", + "PGHOST", + "PGPORT", + "PGDATABASE", + "PGUSER", + "PGPASSWORD", +] + +test("generate without postgres env uses pglite for migrations and dumps structure", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pgstrap-generate-")) const migrationsDir = path.join(tmp, "migrations") + const previousEnv: Record = {} + + for (const key of postgresEnvKeys) { + previousEnv[key] = process.env[key] + } + fs.mkdirSync(migrationsDir, { recursive: true }) fs.writeFileSync( path.join(migrationsDir, "001_create_table.js"), migrationFile, ) - await generate({ - schemas: ["public"], - defaultDatabase: "postgres", - dbDir: path.join(tmp, "db"), - migrationsDir, - pglite: true, - }) - - const zapatosFile = path.join(tmp, "db", "zapatos", "schema.d.ts") - const structureDir = path.join( - tmp, - "db", - "structure", - "public", - "tables", - "foo", - ) + try { + for (const key of postgresEnvKeys) { + delete process.env[key] + } + + await generate({ + schemas: ["public"], + defaultDatabase: "postgres", + dbDir: path.join(tmp, "db"), + migrationsDir, + }) + + const zapatosFile = path.join(tmp, "db", "zapatos", "schema.d.ts") + const structureDir = path.join( + tmp, + "db", + "structure", + "public", + "tables", + "foo", + ) - expect(fs.existsSync(zapatosFile)).toBe(true) - expect(fs.existsSync(path.join(structureDir, "table.sql"))).toBe(true) + expect(fs.existsSync(zapatosFile)).toBe(true) + expect(fs.existsSync(path.join(structureDir, "table.sql"))).toBe(true) + } finally { + for (const [key, value] of Object.entries(previousEnv)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } - fs.rmSync(tmp, { recursive: true, force: true }) + fs.rmSync(tmp, { recursive: true, force: true }) + } }) From a6448cf53a1c42833601ae646a9c8415b7620a10 Mon Sep 17 00:00:00 2001 From: Ethan Soh Date: Thu, 4 Jun 2026 12:03:47 +0800 Subject: [PATCH 2/2] Address PGlite fallback review feedback --- src/generate.ts | 80 +++++++++++++++++++--------- tests/generate.pglite.test.ts | 98 +++++++++++++++++++++++++++-------- 2 files changed, 130 insertions(+), 48 deletions(-) diff --git a/src/generate.ts b/src/generate.ts index b4eedc5..1868f78 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -14,16 +14,34 @@ const postgresEnvKeys = [ "PGPASSWORD", ] -const hasExternalDatabaseConfig = ( +export const hasExternalDatabaseConfig = ( env: Record = process.env, ) => postgresEnvKeys.some((key) => Boolean(env[key])) +let pgliteEnvLock = Promise.resolve() + +const withPgliteEnvLock = async (fn: () => Promise) => { + const previous = pgliteEnvLock + let release!: () => void + pgliteEnvLock = new Promise((resolve) => { + release = resolve + }) + + await previous + try { + return await fn() + } finally { + release() + } +} + const generateWithPglite = async ({ schemas, defaultDatabase, dbDir, migrationsDir, -}: Pick & { +}: Pick & { + dbDir: string migrationsDir: string }) => { const { PGlite } = await import("@electric-sql/pglite") @@ -48,15 +66,16 @@ const generateWithPglite = async ({ try { const { data: responseData } = await (db as any).execProtocol(data) return responseData - } catch { + } catch (error) { + if (process.env.DEBUG?.includes("pgstrap")) { + console.error("PGlite protocol execution failed", error) + } return undefined } }, }) }) - const prevDbUrl = process.env.DATABASE_URL - try { await migrate({ client: db as any, @@ -66,31 +85,42 @@ const generateWithPglite = async ({ schemas, }) - await new Promise((resolve) => server.listen(0, resolve)) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) const port = (server.address() as any).port const connectionString = `postgres://postgres:postgres@127.0.0.1:${port}/postgres` - process.env.DATABASE_URL = connectionString - - await zg.generate({ - db: { - connectionString, - }, - schemas: Object.fromEntries( - schemas.map((s) => [s, { include: "*", exclude: [] }]), - ), - outDir: dbDir, - }) - - await dumpTree({ - targetDir: path.join(dbDir, "structure"), - defaultDatabase: "postgres", - schemas, + await withPgliteEnvLock(async () => { + const previousEnv = Object.fromEntries( + postgresEnvKeys.map((key) => [key, process.env[key]]), + ) as Record + + try { + for (const key of postgresEnvKeys) delete process.env[key] + process.env.DATABASE_URL = connectionString + + await zg.generate({ + db: { + connectionString, + }, + schemas: Object.fromEntries( + schemas.map((s) => [s, { include: "*", exclude: [] }]), + ), + outDir: dbDir, + }) + + await dumpTree({ + targetDir: path.join(dbDir, "structure"), + defaultDatabase: "postgres", + schemas, + }) + } finally { + for (const [key, value] of Object.entries(previousEnv)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } }) } finally { - if (prevDbUrl === undefined) delete process.env.DATABASE_URL - else process.env.DATABASE_URL = prevDbUrl - await new Promise((resolve, reject) => { server.close((err) => { if (err) reject(err) diff --git a/tests/generate.pglite.test.ts b/tests/generate.pglite.test.ts index 79cd1a8..4eb21d0 100644 --- a/tests/generate.pglite.test.ts +++ b/tests/generate.pglite.test.ts @@ -2,7 +2,7 @@ import { test, expect } from "bun:test" import fs from "fs" import os from "os" import path from "path" -import { generate } from "../src/generate" +import { generate, hasExternalDatabaseConfig } from "../src/generate" const migrationFile = ` exports.up = async (pgm) => { @@ -22,22 +22,59 @@ const postgresEnvKeys = [ "PGPASSWORD", ] -test("generate without postgres env uses pglite for migrations and dumps structure", async () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pgstrap-generate-")) - const migrationsDir = path.join(tmp, "migrations") - const previousEnv: Record = {} +const snapshotPostgresEnv = () => + Object.fromEntries( + postgresEnvKeys.map((key) => [key, process.env[key]]), + ) as Record - for (const key of postgresEnvKeys) { - previousEnv[key] = process.env[key] +const restorePostgresEnv = ( + previousEnv: Record, +) => { + for (const [key, value] of Object.entries(previousEnv)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value } +} +const writeMigration = (migrationsDir: string) => { fs.mkdirSync(migrationsDir, { recursive: true }) fs.writeFileSync( path.join(migrationsDir, "001_create_table.js"), migrationFile, ) +} + +const expectGeneratedArtifacts = (tmp: string) => { + const zapatosFile = path.join(tmp, "db", "zapatos", "schema.d.ts") + const structureDir = path.join( + tmp, + "db", + "structure", + "public", + "tables", + "foo", + ) + + expect(fs.existsSync(zapatosFile)).toBe(true) + expect(fs.existsSync(path.join(structureDir, "table.sql"))).toBe(true) +} + +test("hasExternalDatabaseConfig detects postgres env settings", () => { + expect(hasExternalDatabaseConfig({})).toBe(false) + expect( + hasExternalDatabaseConfig({ DATABASE_URL: "postgres://example" }), + ).toBe(true) + expect(hasExternalDatabaseConfig({ PGHOST: "127.0.0.1" })).toBe(true) +}) + +test("generate without postgres env uses pglite for migrations and dumps structure", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pgstrap-generate-")) + const migrationsDir = path.join(tmp, "migrations") + const previousEnv = snapshotPostgresEnv() try { + writeMigration(migrationsDir) + for (const key of postgresEnvKeys) { delete process.env[key] } @@ -49,24 +86,39 @@ test("generate without postgres env uses pglite for migrations and dumps structu migrationsDir, }) - const zapatosFile = path.join(tmp, "db", "zapatos", "schema.d.ts") - const structureDir = path.join( - tmp, - "db", - "structure", - "public", - "tables", - "foo", - ) - - expect(fs.existsSync(zapatosFile)).toBe(true) - expect(fs.existsSync(path.join(structureDir, "table.sql"))).toBe(true) + expectGeneratedArtifacts(tmp) } finally { - for (const [key, value] of Object.entries(previousEnv)) { - if (value === undefined) delete process.env[key] - else process.env[key] = value - } + restorePostgresEnv(previousEnv) + fs.rmSync(tmp, { recursive: true, force: true }) + } +}) + +test("generate with pglite true ignores external postgres env settings", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pgstrap-generate-")) + const migrationsDir = path.join(tmp, "migrations") + const previousEnv = snapshotPostgresEnv() + + try { + writeMigration(migrationsDir) + + process.env.DATABASE_URL = "postgres://external:external@example.invalid/db" + process.env.PGHOST = "example.invalid" + process.env.PGPORT = "5432" + process.env.PGDATABASE = "external" + process.env.PGUSER = "external" + process.env.PGPASSWORD = "external" + await generate({ + schemas: ["public"], + defaultDatabase: "postgres", + dbDir: path.join(tmp, "db"), + migrationsDir, + pglite: true, + }) + + expectGeneratedArtifacts(tmp) + } finally { + restorePostgresEnv(previousEnv) fs.rmSync(tmp, { recursive: true, force: true }) } })