From 3c16c814d7c84b92e7e7bee4e77e01e98f4055a0 Mon Sep 17 00:00:00 2001 From: RyuseiTaniguchi Date: Sat, 29 Aug 2026 14:54:12 -0700 Subject: [PATCH] fix(tests): stop exhausting test-db connections and migrate once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pnpm test` at vitest's default concurrency reported ~137 failures that were not real. Each integration file opens its own pg.Pool against the single shared container, and pg pools default to 10 connections. At 14 workers that demands ~140 against a server whose max_connections is 100, so Postgres terminates connections mid-run (57P01) and the failures land on whichever tests happened to be in flight. That also explains the suite's flakiness: consecutive runs on identical code gave 5, then 1, then 0 failures. Raise max_connections to 300 and cap each pool at 4. While here, cut the dominant cost: every one of the 52 integration files replayed the full migration chain. Migrate once into a template database in global setup and have each file clone it with CREATE DATABASE ... TEMPLATE. Concurrent clones briefly conflict (55006), so retry. Falls back to replaying migrations when no template is present, keeping a bare DATABASE_URL working. Durability settings are also disabled — the server is destroyed at teardown. pnpm test, default concurrency before: 339s, 137 failed / 479 passed after: 21s, 727 passed Five consecutive integration runs are now green where the suite previously varied run to run. Also ignore .stryker-tmp/** in ESLint. Stryker leaves sandbox copies of the tree behind, and linting them reported 981 of 1024 errors, failing `pnpm lint` on code that is not ours. Lint is now 0 errors. --- eslint.config.mjs | 4 +++ tests/global-setup.ts | 51 ++++++++++++++++++++++++++++++++++++-- tests/integration/setup.ts | 47 +++++++++++++++++++++++++++++------ 3 files changed, 93 insertions(+), 9 deletions(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index c5dd669b..fdbdf3fa 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -19,6 +19,10 @@ const eslintConfig = defineConfig([ "out/**", "build/**", "next-env.d.ts", + // Stryker leaves sandbox copies of the whole tree behind. Linting those + // repeats every finding once per sandbox and fails the run on code that + // is not ours to fix. + ".stryker-tmp/**", ]), ]); diff --git a/tests/global-setup.ts b/tests/global-setup.ts index d779515b..c4d428eb 100644 --- a/tests/global-setup.ts +++ b/tests/global-setup.ts @@ -1,10 +1,57 @@ import { PostgreSqlContainer, type StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { Pool } from "pg"; +import path from "node:path"; let container: StartedPostgreSqlContainer; +// Every test file clones this database instead of replaying the migration +// chain itself. Shared with tests/integration/setup.ts via TEST_TEMPLATE_DB. +const TEMPLATE_DB = "ledgr_test_template"; + export async function setup() { - container = await new PostgreSqlContainer("postgres:17-alpine").start(); - process.env.DATABASE_URL = container.getConnectionUri(); + container = await new PostgreSqlContainer("postgres:17-alpine") + .withCommand([ + "postgres", + // One throwaway database per test file, all on this single server. The + // default max_connections (100) sits below what the fork pool can demand + // — 14 workers holding a pool each exhausts it, and Postgres starts + // terminating connections mid-run (57P01). + "-c", + "max_connections=300", + // Durability buys nothing for a server destroyed at teardown. + "-c", + "fsync=off", + "-c", + "synchronous_commit=off", + "-c", + "full_page_writes=off", + ]) + .start(); + + const connectionString = container.getConnectionUri(); + process.env.DATABASE_URL = connectionString; + process.env.TEST_TEMPLATE_DB = TEMPLATE_DB; + + // Migrate once, here, into a template database. Cloning that template per + // test file replaces one full migration run per file with a file copy. + const admin = new Pool({ connectionString, max: 1 }); + await admin.query(`CREATE DATABASE "${TEMPLATE_DB}"`); + await admin.end(); + + const templateUrl = new URL(connectionString); + templateUrl.pathname = `/${TEMPLATE_DB}`; + const pool = new Pool({ connectionString: templateUrl.toString(), max: 1 }); + try { + await migrate(drizzle({ client: pool }), { + migrationsFolder: path.join(process.cwd(), "src/db/migrations"), + }); + } finally { + // The template must have no open connections, or CREATE DATABASE ... + // TEMPLATE fails with 55006 for every test file that follows. + await pool.end(); + } } export async function teardown() { diff --git a/tests/integration/setup.ts b/tests/integration/setup.ts index 8b3483bf..10f1c7f6 100644 --- a/tests/integration/setup.ts +++ b/tests/integration/setup.ts @@ -5,11 +5,35 @@ import { randomUUID } from "crypto"; import * as schema from "../../src/db/schema"; import path from "node:path"; +// Each worker holds a pool for the life of its test file. Left at pg's default +// of 10 these overrun the server's connection limit once vitest scales forks to +// the core count; the suite needs only a couple of concurrent queries per file. +const POOL_MAX = 4; + +// CREATE DATABASE ... TEMPLATE briefly conflicts when several workers clone the +// same template at once (55006). It clears on its own, so retry rather than +// serialize every worker behind a lock. +const CLONE_RETRIES = 10; + +async function cloneTemplate(admin: Pool, dbName: string, template: string) { + for (let attempt = 0; ; attempt++) { + try { + await admin.query(`CREATE DATABASE "${dbName}" TEMPLATE "${template}"`); + return; + } catch (error) { + const code = (error as { code?: string }).code; + if (code !== "55006" || attempt >= CLONE_RETRIES) throw error; + await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1))); + } + } +} + export async function createTestDb() { const connectionString = process.env.DATABASE_URL || "postgresql://ledgr:ledgr@localhost:5432/ledgr_test"; const dbName = `test_${randomUUID().replace(/-/g, "")}`; + const template = process.env.TEST_TEMPLATE_DB; // Isolate each test file in its own *database* (not a schema). Migrations // reference tables as `"public".""`, which only resolves when the @@ -17,18 +41,27 @@ export async function createTestDb() { // on those qualified references. A throwaway database per file gives every test // its own public schema, keeps the concurrent suite isolated, and is robust to // future migrations regardless of how they qualify identifiers. - const admin = new Pool({ connectionString }); - await admin.query(`CREATE DATABASE "${dbName}"`); + const admin = new Pool({ connectionString, max: 1 }); + if (template) { + await cloneTemplate(admin, dbName, template); + } else { + await admin.query(`CREATE DATABASE "${dbName}"`); + } await admin.end(); const url = new URL(connectionString); url.pathname = `/${dbName}`; - const pool = new Pool({ connectionString: url.toString() }); + const pool = new Pool({ connectionString: url.toString(), max: POOL_MAX }); const db = drizzle({ client: pool, schema }); - await migrate(db, { - migrationsFolder: path.join(process.cwd(), "src/db/migrations"), - }); + + // The template arrives already migrated. Without one (a direct DATABASE_URL, + // no global setup) fall back to replaying the chain. + if (!template) { + await migrate(db, { + migrationsFolder: path.join(process.cwd(), "src/db/migrations"), + }); + } return { db, @@ -36,7 +69,7 @@ export async function createTestDb() { await pool.end(); // DROP DATABASE cannot run while connections are open; FORCE terminates any // stragglers (Postgres 13+). - const admin2 = new Pool({ connectionString }); + const admin2 = new Pool({ connectionString, max: 1 }); await admin2.query(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`); await admin2.end(); },