diff --git a/package.json b/package.json index 5468037..f8f78e5 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "mcp": "bun run src/mcp/cli.ts", "test:corpus": "server/node_modules/.bin/tsx scripts/modernizer-corpus.ts", "corpus:golden": "server/node_modules/.bin/tsx scripts/modernizer-golden.ts", + "test:batch": "server/node_modules/.bin/tsx scripts/modernizer-batch.ts", "test:schema-pg": "server/node_modules/.bin/tsx scripts/schema-change-pg.ts", "test:schema-pg:versions": "bash scripts/schema-change-versions.sh", "queryshift": "server/node_modules/.bin/tsx queryshift/run.ts", diff --git a/scripts/modernizer-batch.ts b/scripts/modernizer-batch.ts new file mode 100644 index 0000000..0def836 --- /dev/null +++ b/scripts/modernizer-batch.ts @@ -0,0 +1,126 @@ +/** + * Runs the Modernizer's golden corpus through the script (batch) path, against a real PostgreSQL 16: + * + * docker run -d --name goforge-corpus -e POSTGRES_PASSWORD=corpus -e POSTGRES_DB=corpus -p 54329:5432 postgres:16 + * npm run test:batch + * + * Every corpus query is written into one script per source dialect, split by the same splitter a user's file goes through, + * converted, and checked by PostgreSQL itself (EXPLAIN in a read-only transaction). The corpus already knows what each query must + * do: convert and return the reference rows, report a behaviour trap, or report that something is not converted. The check is that + * the statuses agree with that knowledge. A statement the corpus knows is not converted, or can return different rows, must never + * end up as plain "converted"; a statement whose rows the corpus proved identical must never end up "failed". + * Exit code 1 on any such disagreement. + */ +import { createRequire } from "node:module"; +import { BATCH_STATUSES, BatchItem, BatchStatus, runBatch } from "../src/features/query-modernizer/batch"; +import { checkPostgresSyntax } from "../src/features/query-modernizer/engine/checks"; +import { CASES, type CorpusCase } from "../src/features/query-modernizer/engine/corpus/cases"; +import { FIXTURE_SCHEMA, FIXTURE_SQL } from "../src/features/query-modernizer/engine/corpus/fixture"; +import { LEGACY_DIALECTS, type LegacyDialect } from "../src/features/query-modernizer/types"; +import { validateSql } from "../server/src/services/pg-validate"; + +const require = createRequire(new URL("../server/package.json", import.meta.url)); +const { Client } = require("pg") as typeof import("pg"); +const URL_ = process.env.CORPUS_DATABASE_URL ?? "postgres://postgres:corpus@127.0.0.1:54329/corpus"; + +const rank: Record = { failed: 4, review: 3, "out-of-scope": 2, converted: 1, unchanged: 0 }; +const worst = (items: BatchItem[]): BatchStatus => items.reduce((best, item) => (rank[item.status] > rank[best] ? item.status : best), "unchanged" as BatchStatus); + +interface Verdict { + c: CorpusCase; + status: BatchStatus; + items: BatchItem[]; +} + +async function main() { + const client = new Client({ connectionString: URL_ }); + await client.connect(); + await client.query("SET TIME ZONE 'UTC'"); + await client.query(FIXTURE_SQL); + + // validateSql runs one statement per call; after a rejection the connection must leave the aborted transaction before the next one. + const run = async (fn: (c: { query(text: string): Promise }) => Promise): Promise => { + try { + return await fn(client as never); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } + }; + + const verdicts: Verdict[] = []; + const skipped: CorpusCase[] = []; + let units = 0; + let startedAt = performance.now(); + + for (const dialect of LEGACY_DIALECTS.map((option) => option.value) as LegacyDialect[]) { + for (const withSchema of [true, false]) { + // cases that build their own tables first can't be checked by EXPLAIN against the fixture: they are left out and counted + const cases = CASES.filter((c) => c.from === dialect && Boolean(c.schema) === withSchema && !c.setup); + skipped.push(...CASES.filter((c) => c.from === dialect && Boolean(c.schema) === withSchema && c.setup)); + if (cases.length === 0) continue; + + const script = cases.map((c) => `-- @@case ${c.id}\n${c.sql.replace(/;\s*$/, "")}\n;\n`).join("\n"); + const result = await runBatch( + { script, dialect, schema: withSchema ? FIXTURE_SCHEMA : undefined }, + { syntax: checkPostgresSyntax, validate: (sql) => validateSql(sql, run), connectionName: "corpus" }, + { concurrency: 1 }, + ); + units += result.items.length; + + // a case starts at its marker and runs until the next one + const byCase = new Map(); + let current: string | undefined; + for (const item of result.items) { + const marker = /@@case (\S+)/.exec(item.unit.comments)?.[1]; + if (marker) current = marker; + if (current) byCase.set(current, [...(byCase.get(current) ?? []), item]); + } + for (const c of cases) { + const items = byCase.get(c.id) ?? []; + verdicts.push({ c, status: items.length > 0 ? worst(items) : "failed", items }); + } + } + } + await client.end(); + const elapsed = performance.now() - startedAt; + + // ---- report ------------------------------------------------------------------------------------------------- + const families = new Map>(); + for (const { c, status } of verdicts) { + const row = families.get(c.family) ?? { converted: 0, unchanged: 0, review: 0, failed: 0, "out-of-scope": 0 }; + row[status]++; + families.set(c.family, row); + } + console.log("\n" + "Family".padEnd(26) + BATCH_STATUSES.map((s) => s.padStart(14)).join("")); + const total = { converted: 0, unchanged: 0, review: 0, failed: 0, "out-of-scope": 0 } as Record; + for (const [family, row] of [...families.entries()].sort()) { + console.log(family.padEnd(26) + BATCH_STATUSES.map((s) => String(row[s]).padStart(14)).join("")); + for (const s of BATCH_STATUSES) total[s] += row[s]; + } + console.log("TOTAL".padEnd(26) + BATCH_STATUSES.map((s) => String(total[s]).padStart(14)).join("")); + console.log(`\n${verdicts.length} corpus queries in ${units} statements, converted as scripts and checked by PostgreSQL in ${(elapsed / 1000).toFixed(1)} s (${skipped.length} that need their own setup tables were left out).`); + + // ---- the agreement with what the corpus knows ------------------------------------------------------------------- + const violations: string[] = []; + const plain = (items: BatchItem[]) => items.every((item) => item.status === "converted" || item.status === "unchanged"); + for (const { c, status, items } of verdicts) { + const trapsThatChangeResults = (c.trap ? [c.trap] : []).concat(c.traps ?? []).filter((id) => items.some((item) => item.outcome?.engine.traps.some((trap) => trap.id === id && trap.severity === "changes-results"))); + if (c.unconverted && plain(items)) violations.push(`SILENT ${c.id}: the corpus knows something is not converted, the script path says "${status}"\n ${c.sql.replace(/\s+/g, " ")}`); + else if (trapsThatChangeResults.length > 0 && plain(items)) violations.push(`SILENT ${c.id}: can return different rows (${trapsThatChangeResults.join(", ")}), the script path says "${status}"`); + else if (!c.unconverted && !c.trap && !(c.traps?.length) && c.reference !== undefined && status === "failed") violations.push(`WRONG ${c.id}: rows proven identical by the corpus, the script path says "failed": ${items.flatMap((item) => item.reasons).join(" | ").slice(0, 300)}\n ${c.sql.replace(/\s+/g, " ")}`); + } + const unexpectedReview = verdicts.filter(({ c, status }) => status === "review" && !c.trap && !c.traps?.length && !c.unconverted && c.reference !== undefined); + if (unexpectedReview.length > 0) { + console.log(`\n${unexpectedReview.length} statement(s) the corpus proved correct are marked "needs review" (not a disagreement, listed for reading):`); + for (const { c, items } of unexpectedReview.slice(0, 15)) console.log(` ${c.id}: ${items.flatMap((item) => item.reasons).join(" | ").slice(0, 200)}`); + } + console.log(`\ndisagreements with the corpus: ${violations.length}`); + for (const violation of violations) console.log(` ${violation}`); + process.exit(violations.length > 0 ? 1 : 0); +} + +main().catch((error) => { + console.error(error); + process.exit(2); +});