From c698f4b70bafa726bd000f4b755f1ebc36a9440e Mon Sep 17 00:00:00 2001 From: krkarma777 Date: Mon, 31 Aug 2026 10:54:37 +0900 Subject: [PATCH] feat: Add heuristic option with git-style cost cap --- CHANGELOG.md | 5 ++ README.md | 3 +- src/chars.ts | 4 +- src/index.ts | 29 ++++++++--- src/myers.ts | 67 +++++++++++++++++++++++-- src/scan.ts | 4 +- test/heuristic.test.ts | 110 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 206 insertions(+), 16 deletions(-) create mode 100644 test/heuristic.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5948e33..600550e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and the project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- `heuristic` option: caps the search cost per subproblem (the git xdiff + strategy) so pathological inputs stay fast — the 8 KB completely-different + worst case drops from ~217 ms to ~8 ms for an edit script ~8% above + minimal. Output is identical to exact mode while the edit distance is + under the cap. Also accepted by `diffTokens` via a new options argument. - `refine` option: re-diffs each delete/insert pair one granularity finer (`line` pairs by word, `word` pairs by char), so `quick` → `quicker` reports the shared prefix as equal and just `+er` as the change. diff --git a/README.md b/README.md index 4ad75ff..20bce1d 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ Returns `DiffEntry[]` — the shortest edit script between `a` and `b`. |---|---|---|---| | `mode` | `'word' \| 'char' \| 'line'` | `'word'` | tokenization granularity | | `refine` | `boolean` | `false` | re-diff each delete/insert pair one level finer (`line`→word, `word`→char), e.g. `quick`→`quicker` reports just `+er` | +| `heuristic` | `boolean` | `false` | cap the search cost like git does, keeping pathological inputs fast (the 227 ms worst case below drops to ~8 ms, +8% edit-script size); output stays identical to exact mode while the edit distance is small | - `word` — runs of Unicode letters/digits/underscore, whitespace runs, symbol runs - `char` — individual code points (surrogate-pair safe) @@ -110,7 +111,7 @@ Against the popular npm diff libraries — [`diff` (jsdiff)](https://www.npmjs.c How to read this honestly: - **On typical inputs every library here is sub-millisecond-ish** — the differences are fractions of a millisecond and won't matter to most applications. -- **The worst case is where libraries separate**, and it's the row that decides whether your UI freezes on pathological input: this package is 2.6–10× faster than everything tested, while still returning a provably minimal diff. +- **The worst case is where libraries separate**, and it's the row that decides whether your UI freezes on pathological input: this package is 2.6–10× faster than everything tested, while still returning a provably minimal diff. If you'd rather trade minimality for speed there, `{ heuristic: true }` brings that row to ~8 ms (edit script ~8% larger) — still exact whenever the edit distance is small. - `diff-match-patch` (default) trades exactness for speed by design — its documented timeout heuristics can return non-minimal diffs. This package never does. - `fast-myers-diff` has no tokenizer and emits index ranges rather than text entries, so its rows do less output work (word/line rows reuse our tokenizer); `diff-match-patch` has no built-in word or line API. diff --git a/src/chars.ts b/src/chars.ts index 8908139..5bfc1e0 100644 --- a/src/chars.ts +++ b/src/chars.ts @@ -10,7 +10,7 @@ import { pushEntry, type DiffEntry } from './entries.ts'; * the original input. Common prefix/suffix are stripped at the code-unit * level first, so untouched regions skip the scan entirely. */ -export function diffChars(a: string, b: string): DiffEntry[] { +export function diffChars(a: string, b: string, heuristic = false): DiffEntry[] { const aLen = a.length; const bLen = b.length; const minLen = aLen < bLen ? aLen : bLen; @@ -28,7 +28,7 @@ export function diffChars(a: string, b: string): DiffEntry[] { const aMid = scanCodePoints(a, prefix, aLen - suffix); const bMid = scanCodePoints(b, prefix, bLen - suffix); - const { changedA, changedB } = myersDiff(aMid.points, bMid.points); + const { changedA, changedB } = myersDiff(aMid.points, bMid.points, heuristic); const entries: DiffEntry[] = []; if (prefix > 0) entries.push({ operation: 'equal', text: a.slice(0, prefix) }); diff --git a/src/index.ts b/src/index.ts index cbbac7c..4ea2438 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,20 @@ export interface DiffOptions { * effect in 'char' mode. Defaults to false. */ refine?: boolean; + /** + * Caps the search cost per subproblem (the git xdiff strategy) so that + * pathological inputs — two large, almost entirely different strings — + * stay fast instead of costing O((N+M)·D). The result is always a valid + * diff but is no longer guaranteed minimal; while the edit distance is + * under the cap (64+), output is identical to exact mode. Defaults to + * false (exact, provably minimal). + */ + heuristic?: boolean; +} + +export interface DiffTokensOptions { + /** See {@link DiffOptions.heuristic}. */ + heuristic?: boolean; } /** @@ -30,21 +44,22 @@ export function diff(a: string, b: string, options: DiffOptions = {}): DiffEntry return a.length === 0 ? [] : [{ operation: 'equal', text: a }]; } const mode = options.mode ?? 'word'; - const entries = mode === 'char' ? diffChars(a, b) : diffScanned(a, b, mode); + const heuristic = options.heuristic === true; + const entries = mode === 'char' ? diffChars(a, b, heuristic) : diffScanned(a, b, mode, heuristic); if (options.refine === true && mode !== 'char') { - return refineEntries(entries, mode === 'line' ? 'word' : 'char'); + return refineEntries(entries, mode === 'line' ? 'word' : 'char', heuristic); } return entries; } /** Re-diffs adjacent delete/insert pairs at a finer granularity. */ -function refineEntries(entries: DiffEntry[], finerMode: DiffMode): DiffEntry[] { +function refineEntries(entries: DiffEntry[], finerMode: DiffMode, heuristic: boolean): DiffEntry[] { const out: DiffEntry[] = []; for (let i = 0; i < entries.length; i++) { const entry = entries[i]; const next = entries[i + 1]; if (entry.operation === 'delete' && next !== undefined && next.operation === 'insert') { - for (const sub of diff(entry.text, next.text, { mode: finerMode })) { + for (const sub of diff(entry.text, next.text, { mode: finerMode, heuristic })) { pushEntry(out, sub.operation, sub.text); } i++; @@ -56,7 +71,9 @@ function refineEntries(entries: DiffEntry[], finerMode: DiffMode): DiffEntry[] { } /** Diffs two pre-tokenized sequences. Tokens are compared by exact string equality. */ -export function diffTokens(aTokens: readonly string[], bTokens: readonly string[]): DiffEntry[] { +export function diffTokens( + aTokens: readonly string[], bTokens: readonly string[], options: DiffTokensOptions = {}, +): DiffEntry[] { const n = aTokens.length; const m = bTokens.length; const minLen = n < m ? n : m; @@ -72,7 +89,7 @@ export function diffTokens(aTokens: readonly string[], bTokens: readonly string[ const ids = new Map(); const ia = internRange(aTokens, prefix, n - suffix, ids); const ib = internRange(bTokens, prefix, m - suffix, ids); - const mid = myersDiff(ia, ib); + const mid = myersDiff(ia, ib, options.heuristic === true); const changedA = new Uint8Array(n); const changedB = new Uint8Array(m); diff --git a/src/myers.ts b/src/myers.ts index 6f6b31f..2c1244f 100644 --- a/src/myers.ts +++ b/src/myers.ts @@ -8,8 +8,15 @@ export interface MyersResult { * divide-and-conquer refinement (middle snake), operating on interned * token ids. Scratch buffers are allocated once and reused across the * whole recursion. + * + * With `heuristic` enabled, each subproblem's search is capped at + * max(64, sqrt(N+M)) phases (the git xdiff approach): past the cap it + * splits at the furthest-reaching point found so far instead of the true + * middle snake. The result is always a valid edit script, but no longer + * guaranteed minimal; pathological inputs go from O((N+M)·D) to roughly + * O((N+M)^1.5). */ -export function myersDiff(a: Int32Array, b: Int32Array): MyersResult { +export function myersDiff(a: Int32Array, b: Int32Array, heuristic = false): MyersResult { const n = a.length; const m = b.length; const changedA = new Uint8Array(n); @@ -23,7 +30,7 @@ export function myersDiff(a: Int32Array, b: Int32Array): MyersResult { const vf = new Int32Array(2 * (n + m) + 3); const vb = new Int32Array(2 * (n + m) + 3); - walk(a, 0, n, b, 0, m, changedA, changedB, vf, vb, offset); + walk(a, 0, n, b, 0, m, changedA, changedB, vf, vb, offset, heuristic); return { changedA, changedB }; } @@ -32,6 +39,7 @@ function walk( b: Int32Array, b0: number, b1: number, changedA: Uint8Array, changedB: Uint8Array, vf: Int32Array, vb: Int32Array, offset: number, + heuristic: boolean, ): void { while (a0 < a1 && b0 < b1 && a[a0] === b[b0]) { a0++; b0++; } while (a1 > a0 && b1 > b0 && a[a1 - 1] === b[b1 - 1]) { a1--; b1--; } @@ -45,9 +53,9 @@ function walk( return; } - const [sx, sy, ex, ey] = middleSnake(a, a0, a1, b, b0, b1, vf, vb, offset); - walk(a, a0, a0 + sx, b, b0, b0 + sy, changedA, changedB, vf, vb, offset); - walk(a, a0 + ex, a1, b, b0 + ey, b1, changedA, changedB, vf, vb, offset); + const [sx, sy, ex, ey] = middleSnake(a, a0, a1, b, b0, b1, vf, vb, offset, heuristic); + walk(a, a0, a0 + sx, b, b0, b0 + sy, changedA, changedB, vf, vb, offset, heuristic); + walk(a, a0 + ex, a1, b, b0 + ey, b1, changedA, changedB, vf, vb, offset, heuristic); } /** @@ -68,17 +76,25 @@ function middleSnake( a: Int32Array, a0: number, a1: number, b: Int32Array, b0: number, b1: number, vf: Int32Array, vb: Int32Array, offset: number, + heuristic: boolean, ): [number, number, number, number] { const n = a1 - a0; const m = b1 - b0; const delta = n - m; const deltaOdd = (delta & 1) !== 0; const dMax = Math.ceil((n + m) / 2); + const maxCost = heuristic ? Math.max(64, Math.floor(Math.sqrt(n + m))) : 0; vf[offset + 1] = 0; vb[offset + delta + 1] = n + 1; for (let d = 0; d <= dMax; d++) { + if (maxCost !== 0 && d > maxCost) { + const split = bestEffortSplit(vf, vb, offset, d - 1, n, m, delta); + if (split !== null) return split; + // Both candidates were degenerate corners; fall through and keep + // searching exactly — the overlap must be imminent. + } // Forward pass for phase d. for (let k = -d; k <= d; k += 2) { let x: number; @@ -116,3 +132,44 @@ function middleSnake( } throw new Error('middleSnake: no overlap found (invariant violated)'); } + +/** + * Heuristic bailout: picks the furthest-reaching point recorded in phase + * `d` of either direction and splits there. Any point on a furthest + * forward/backward path is a valid (if not necessarily optimal) split, and + * its distance from the corner is at least `d`, so both children shrink. + */ +function bestEffortSplit( + vf: Int32Array, vb: Int32Array, offset: number, + d: number, n: number, m: number, delta: number, +): [number, number, number, number] | null { + let fx = -1; + let fy = -1; + let fSum = -1; + for (let k = -d; k <= d; k += 2) { + const x = vf[offset + k]; + const y = x - k; + if (x >= 0 && y >= 0 && x <= n && y <= m && x + y > fSum) { + fSum = x + y; + fx = x; + fy = y; + } + } + let bx = -1; + let by = -1; + let bSum = -1; + for (let k = delta - d; k <= delta + d; k += 2) { + const x = vb[offset + k]; + const y = x - k; + if (x >= 0 && y >= 0 && x <= n && y <= m && (n - x) + (m - y) > bSum) { + bSum = (n - x) + (m - y); + bx = x; + by = y; + } + } + const forwardUsable = fSum >= 0 && !(fx === n && fy === m); + const backwardUsable = bSum >= 0 && !(bx === 0 && by === 0); + if (forwardUsable && (!backwardUsable || fSum >= bSum)) return [fx, fy, fx, fy]; + if (backwardUsable) return [bx, by, bx, by]; + return null; +} diff --git a/src/scan.ts b/src/scan.ts index 8a00160..33cf6f7 100644 --- a/src/scan.ts +++ b/src/scan.ts @@ -11,7 +11,7 @@ import { pushEntry, type DiffEntry } from './entries.ts'; * equivalence classes are exactly those of tokenize()+diffTokens(), so the * result is identical — this only removes allocation and hashing overhead. */ -export function diffScanned(a: string, b: string, mode: 'word' | 'line'): DiffEntry[] { +export function diffScanned(a: string, b: string, mode: 'word' | 'line', heuristic = false): DiffEntry[] { const scanA = mode === 'word' ? scanWordTokens(a) : scanLineTokens(a); const scanB = mode === 'word' ? scanWordTokens(b) : scanLineTokens(b); const offA = scanA.offsets; @@ -45,7 +45,7 @@ export function diffScanned(a: string, b: string, mode: 'word' | 'line'): DiffEn a, offA, hashA, prefix, countA - suffix, b, offB, hashB, prefix, countB - suffix, ); - const { changedA, changedB } = myersDiff(ia, ib); + const { changedA, changedB } = myersDiff(ia, ib, heuristic); const entries: DiffEntry[] = []; if (prefix > 0) entries.push({ operation: 'equal', text: a.slice(0, offA[prefix]) }); diff --git a/test/heuristic.test.ts b/test/heuristic.test.ts new file mode 100644 index 0000000..28e471f --- /dev/null +++ b/test/heuristic.test.ts @@ -0,0 +1,110 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { diff, type DiffEntry } from '../src/index.ts'; + +function makeRng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; +} + +function joinSide(entries: DiffEntry[], skip: 'insert' | 'delete'): string { + let s = ''; + for (const e of entries) if (e.operation !== skip) s += e.text; + return s; +} + +function editCount(entries: DiffEntry[]): number { + let count = 0; + for (const e of entries) if (e.operation !== 'equal') count += e.text.length; + return count; +} + +const PIECES = ['a', 'b', 'cd', '가', '😀', ' ', '\n', '.']; + +function randomString(rng: () => number, maxLen: number): string { + const len = Math.floor(rng() * maxLen); + let s = ''; + for (let i = 0; i < len; i++) s += PIECES[Math.floor(rng() * PIECES.length)]; + return s; +} + +function mutate(rng: () => number, source: string, maxEdits: number): string { + const chars = [...source]; + const edits = 1 + Math.floor(rng() * maxEdits); + for (let e = 0; e < edits; e++) { + const pos = Math.floor(rng() * (chars.length + 1)); + const op = rng(); + if (op < 0.34 && chars.length > 0) chars.splice(Math.min(pos, chars.length - 1), 1); + else if (op < 0.67) chars.splice(pos, 0, PIECES[Math.floor(rng() * PIECES.length)]); + else if (chars.length > 0) chars[Math.min(pos, chars.length - 1)] = PIECES[Math.floor(rng() * PIECES.length)]; + } + return chars.join(''); +} + +test('heuristic: identical output to exact mode while D is under the cost cap', () => { + const rng = makeRng(31337); + for (let iter = 0; iter < 200; iter++) { + const a = randomString(rng, 80); + const b = mutate(rng, a, 8); // few edits => D far below the 64 cap + for (const mode of ['word', 'char', 'line'] as const) { + assert.deepEqual( + diff(a, b, { mode, heuristic: true }), + diff(a, b, { mode }), + `mode=${mode} a=${JSON.stringify(a)} b=${JSON.stringify(b)}`, + ); + } + } +}); + +test('heuristic fuzz: round-trip invariant holds on wildly different inputs', () => { + const rng = makeRng(55555); + for (let iter = 0; iter < 150; iter++) { + const a = randomString(rng, 300); + const b = randomString(rng, 300); + for (const mode of ['word', 'char', 'line'] as const) { + const entries = diff(a, b, { mode, heuristic: true }); + assert.equal(joinSide(entries, 'insert'), a, `a mismatch mode=${mode}`); + assert.equal(joinSide(entries, 'delete'), b, `b mismatch mode=${mode}`); + } + } +}); + +test('heuristic: edit script is valid but may be larger than minimal', () => { + const rng = makeRng(777777); + let heuristicTotal = 0; + let exactTotal = 0; + for (let iter = 0; iter < 30; iter++) { + const a = randomString(rng, 600); + const b = randomString(rng, 600); + const h = diff(a, b, { mode: 'char', heuristic: true }); + const x = diff(a, b, { mode: 'char' }); + assert.equal(joinSide(h, 'insert'), a); + assert.equal(joinSide(h, 'delete'), b); + heuristicTotal += editCount(h); + exactTotal += editCount(x); + assert.ok(editCount(h) >= editCount(x), 'heuristic cannot beat the minimum'); + } + // Sanity: the heuristic should stay in the same ballpark, not degenerate + // to delete-everything/insert-everything on every input. + assert.ok(heuristicTotal <= exactTotal * 2, `${heuristicTotal} vs exact ${exactTotal}`); +}); + +test('heuristic: refine sub-diffs inherit the flag without breaking round-trips', () => { + const rng = makeRng(9999); + for (let iter = 0; iter < 50; iter++) { + const a = randomString(rng, 200); + const b = randomString(rng, 200); + const entries = diff(a, b, { refine: true, heuristic: true }); + assert.equal(joinSide(entries, 'insert'), a); + assert.equal(joinSide(entries, 'delete'), b); + } +}); + +test('heuristic: trivial cases are unaffected', () => { + assert.deepEqual(diff('same', 'same', { heuristic: true }), [{ operation: 'equal', text: 'same' }]); + assert.deepEqual(diff('', '', { heuristic: true }), []); + assert.deepEqual(diff('', 'x', { heuristic: true }), [{ operation: 'insert', text: 'x' }]); +});