From 4c29c68d601cc966045c2d5ca7b3215cd2ab53a5 Mon Sep 17 00:00:00 2001 From: swsarancodes Date: Sun, 25 Oct 2026 17:30:51 +0530 Subject: [PATCH] A check of the Modernizer's script path against a real PostgreSQL scripts/modernizer-batch.ts writes the golden corpus into one script per source dialect, splits and converts it the way a user's file is, and has PostgreSQL check every statement with EXPLAIN in a read-only transaction. The corpus already knows what each query must do, so the statuses are compared with that: a statement the corpus knows is not converted, or can return different rows, must never end up as plain converted, and one whose rows were proved identical must never end up failed. Exit code 1 on any disagreement. npm run test:batch. --- package.json | 1 + scripts/modernizer-batch.ts | 126 ++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 scripts/modernizer-batch.ts diff --git a/package.json b/package.json index 5468037..f8f78e5 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "mcp": "bun run src/mcp/cli.ts", "test:corpus": "server/node_modules/.bin/tsx scripts/modernizer-corpus.ts", "corpus:golden": "server/node_modules/.bin/tsx scripts/modernizer-golden.ts", + "test:batch": "server/node_modules/.bin/tsx scripts/modernizer-batch.ts", "test:schema-pg": "server/node_modules/.bin/tsx scripts/schema-change-pg.ts", "test:schema-pg:versions": "bash scripts/schema-change-versions.sh", "queryshift": "server/node_modules/.bin/tsx queryshift/run.ts", diff --git a/scripts/modernizer-batch.ts b/scripts/modernizer-batch.ts new file mode 100644 index 0000000..0def836 --- /dev/null +++ b/scripts/modernizer-batch.ts @@ -0,0 +1,126 @@ +/** + * Runs the Modernizer's golden corpus through the script (batch) path, against a real PostgreSQL 16: + * + * docker run -d --name goforge-corpus -e POSTGRES_PASSWORD=corpus -e POSTGRES_DB=corpus -p 54329:5432 postgres:16 + * npm run test:batch + * + * Every corpus query is written into one script per source dialect, split by the same splitter a user's file goes through, + * converted, and checked by PostgreSQL itself (EXPLAIN in a read-only transaction). The corpus already knows what each query must + * do: convert and return the reference rows, report a behaviour trap, or report that something is not converted. The check is that + * the statuses agree with that knowledge. A statement the corpus knows is not converted, or can return different rows, must never + * end up as plain "converted"; a statement whose rows the corpus proved identical must never end up "failed". + * Exit code 1 on any such disagreement. + */ +import { createRequire } from "node:module"; +import { BATCH_STATUSES, BatchItem, BatchStatus, runBatch } from "../src/features/query-modernizer/batch"; +import { checkPostgresSyntax } from "../src/features/query-modernizer/engine/checks"; +import { CASES, type CorpusCase } from "../src/features/query-modernizer/engine/corpus/cases"; +import { FIXTURE_SCHEMA, FIXTURE_SQL } from "../src/features/query-modernizer/engine/corpus/fixture"; +import { LEGACY_DIALECTS, type LegacyDialect } from "../src/features/query-modernizer/types"; +import { validateSql } from "../server/src/services/pg-validate"; + +const require = createRequire(new URL("../server/package.json", import.meta.url)); +const { Client } = require("pg") as typeof import("pg"); +const URL_ = process.env.CORPUS_DATABASE_URL ?? "postgres://postgres:corpus@127.0.0.1:54329/corpus"; + +const rank: Record = { failed: 4, review: 3, "out-of-scope": 2, converted: 1, unchanged: 0 }; +const worst = (items: BatchItem[]): BatchStatus => items.reduce((best, item) => (rank[item.status] > rank[best] ? item.status : best), "unchanged" as BatchStatus); + +interface Verdict { + c: CorpusCase; + status: BatchStatus; + items: BatchItem[]; +} + +async function main() { + const client = new Client({ connectionString: URL_ }); + await client.connect(); + await client.query("SET TIME ZONE 'UTC'"); + await client.query(FIXTURE_SQL); + + // validateSql runs one statement per call; after a rejection the connection must leave the aborted transaction before the next one. + const run = async (fn: (c: { query(text: string): Promise }) => Promise): Promise => { + try { + return await fn(client as never); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } + }; + + const verdicts: Verdict[] = []; + const skipped: CorpusCase[] = []; + let units = 0; + let startedAt = performance.now(); + + for (const dialect of LEGACY_DIALECTS.map((option) => option.value) as LegacyDialect[]) { + for (const withSchema of [true, false]) { + // cases that build their own tables first can't be checked by EXPLAIN against the fixture: they are left out and counted + const cases = CASES.filter((c) => c.from === dialect && Boolean(c.schema) === withSchema && !c.setup); + skipped.push(...CASES.filter((c) => c.from === dialect && Boolean(c.schema) === withSchema && c.setup)); + if (cases.length === 0) continue; + + const script = cases.map((c) => `-- @@case ${c.id}\n${c.sql.replace(/;\s*$/, "")}\n;\n`).join("\n"); + const result = await runBatch( + { script, dialect, schema: withSchema ? FIXTURE_SCHEMA : undefined }, + { syntax: checkPostgresSyntax, validate: (sql) => validateSql(sql, run), connectionName: "corpus" }, + { concurrency: 1 }, + ); + units += result.items.length; + + // a case starts at its marker and runs until the next one + const byCase = new Map(); + let current: string | undefined; + for (const item of result.items) { + const marker = /@@case (\S+)/.exec(item.unit.comments)?.[1]; + if (marker) current = marker; + if (current) byCase.set(current, [...(byCase.get(current) ?? []), item]); + } + for (const c of cases) { + const items = byCase.get(c.id) ?? []; + verdicts.push({ c, status: items.length > 0 ? worst(items) : "failed", items }); + } + } + } + await client.end(); + const elapsed = performance.now() - startedAt; + + // ---- report ------------------------------------------------------------------------------------------------- + const families = new Map>(); + for (const { c, status } of verdicts) { + const row = families.get(c.family) ?? { converted: 0, unchanged: 0, review: 0, failed: 0, "out-of-scope": 0 }; + row[status]++; + families.set(c.family, row); + } + console.log("\n" + "Family".padEnd(26) + BATCH_STATUSES.map((s) => s.padStart(14)).join("")); + const total = { converted: 0, unchanged: 0, review: 0, failed: 0, "out-of-scope": 0 } as Record; + for (const [family, row] of [...families.entries()].sort()) { + console.log(family.padEnd(26) + BATCH_STATUSES.map((s) => String(row[s]).padStart(14)).join("")); + for (const s of BATCH_STATUSES) total[s] += row[s]; + } + console.log("TOTAL".padEnd(26) + BATCH_STATUSES.map((s) => String(total[s]).padStart(14)).join("")); + console.log(`\n${verdicts.length} corpus queries in ${units} statements, converted as scripts and checked by PostgreSQL in ${(elapsed / 1000).toFixed(1)} s (${skipped.length} that need their own setup tables were left out).`); + + // ---- the agreement with what the corpus knows ------------------------------------------------------------------- + const violations: string[] = []; + const plain = (items: BatchItem[]) => items.every((item) => item.status === "converted" || item.status === "unchanged"); + for (const { c, status, items } of verdicts) { + const trapsThatChangeResults = (c.trap ? [c.trap] : []).concat(c.traps ?? []).filter((id) => items.some((item) => item.outcome?.engine.traps.some((trap) => trap.id === id && trap.severity === "changes-results"))); + if (c.unconverted && plain(items)) violations.push(`SILENT ${c.id}: the corpus knows something is not converted, the script path says "${status}"\n ${c.sql.replace(/\s+/g, " ")}`); + else if (trapsThatChangeResults.length > 0 && plain(items)) violations.push(`SILENT ${c.id}: can return different rows (${trapsThatChangeResults.join(", ")}), the script path says "${status}"`); + else if (!c.unconverted && !c.trap && !(c.traps?.length) && c.reference !== undefined && status === "failed") violations.push(`WRONG ${c.id}: rows proven identical by the corpus, the script path says "failed": ${items.flatMap((item) => item.reasons).join(" | ").slice(0, 300)}\n ${c.sql.replace(/\s+/g, " ")}`); + } + const unexpectedReview = verdicts.filter(({ c, status }) => status === "review" && !c.trap && !c.traps?.length && !c.unconverted && c.reference !== undefined); + if (unexpectedReview.length > 0) { + console.log(`\n${unexpectedReview.length} statement(s) the corpus proved correct are marked "needs review" (not a disagreement, listed for reading):`); + for (const { c, items } of unexpectedReview.slice(0, 15)) console.log(` ${c.id}: ${items.flatMap((item) => item.reasons).join(" | ").slice(0, 200)}`); + } + console.log(`\ndisagreements with the corpus: ${violations.length}`); + for (const violation of violations) console.log(` ${violation}`); + process.exit(violations.length > 0 ? 1 : 0); +} + +main().catch((error) => { + console.error(error); + process.exit(2); +});