diff --git a/CHANGELOG.md b/CHANGELOG.md index aa82966..ac8cd62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and the project adheres to [Semantic Versioning](https://semver.org/). option): locale-aware word diffs for unspaced scripts (Japanese, Chinese, Thai) and cluster-safe character diffs (ZWJ emoji, combining sequences). `refine` drops `intl-word` pairs to grapheme granularity. (#15) +- `ignoreCase` and `ignoreWhitespace` options on `diff` and `diffTokens`: + masked differences compare equal, with `equal` texts taken from `b` so + non-delete concatenation always reproduces `b`. In `line` mode + `ignoreWhitespace` compares trimmed lines; elsewhere whitespace runs + match each other but presence still matters. (#16) ## [1.1.0] - 2026-08-31 diff --git a/README.md b/README.md index f7f4ddc..f8ac874 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,10 @@ Returns `DiffEntry[]` — the shortest edit script between `a` and `b`. | `locale` | `string \| string[]` | runtime locale | BCP 47 locale(s) for the `Intl.Segmenter` modes | | `refine` | `boolean` | `false` | re-diff each delete/insert pair one level finer (`line`→word, `word`→char), e.g. `quick`→`quicker` reports just `+er` | | `heuristic` | `boolean` | `false` | cap the search cost like git does, keeping pathological inputs fast (the 227 ms worst case below drops to ~8 ms, +8% edit-script size); output stays identical to exact mode while the edit distance is small | +| `ignoreCase` | `boolean` | `false` | compare tokens case-insensitively | +| `ignoreWhitespace` | `boolean` | `false` | whitespace runs compare equal (`line` mode: lines compared trimmed); whitespace with no counterpart still diffs | + +With `ignoreCase`/`ignoreWhitespace`, `equal` texts are taken from `b`, so concatenating non-`delete` texts reproduces `b` exactly; `a`-side reconstruction holds only up to the ignored differences. - `word` — runs of Unicode letters/digits/underscore, whitespace runs, symbol runs - `char` — individual code points (surrogate-pair safe) @@ -100,7 +104,7 @@ diffRanges('the quick fox', 'the slow fox'); ### `diffTokens(aTokens, bTokens, options?)` -Lower-level API: diff two pre-tokenized `string[]` sequences with any tokenization you like (`options.heuristic` supported). +Lower-level API: diff two pre-tokenized `string[]` sequences with any tokenization you like (`heuristic`, `ignoreCase`, and `ignoreWhitespace` supported). ### `tokenize(text, mode?)` diff --git a/src/index.ts b/src/index.ts index 0b8e272..6d339ac 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,11 +33,30 @@ export interface DiffOptions { * false (exact, provably minimal). */ heuristic?: boolean; + /** + * Compares tokens case-insensitively. When tokens differ only in ways an + * ignore option masks, the equal entry's text is taken from `b`, so + * concatenating non-delete texts always reproduces `b` (but not + * necessarily `a`). Defaults to false. + */ + ignoreCase?: boolean; + /** + * Treats whitespace differences as equal: in 'line' mode lines are + * compared with leading/trailing whitespace trimmed; in other modes any + * whitespace-only token matches any other. Presence still matters — a + * whitespace token with no counterpart remains an insert/delete. Same + * `b`-side text rule as ignoreCase. Defaults to false. + */ + ignoreWhitespace?: boolean; } export interface DiffTokensOptions { /** See {@link DiffOptions.heuristic}. */ heuristic?: boolean; + /** See {@link DiffOptions.ignoreCase}. */ + ignoreCase?: boolean; + /** Whitespace-only tokens compare equal. See {@link DiffOptions.ignoreWhitespace}. */ + ignoreWhitespace?: boolean; } /** @@ -53,17 +72,22 @@ export function diff(a: string, b: string, options: DiffOptions = {}): DiffEntry } const mode = options.mode ?? 'word'; const heuristic = options.heuristic === true; + const normalize = buildNormalizer(mode, options.ignoreCase === true, options.ignoreWhitespace === true); let entries: DiffEntry[]; - if (mode === 'char') { + if (normalize !== null || mode === 'intl-word' || mode === 'grapheme') { + // Generic token pipeline: needed for Segmenter tokens and whenever + // token comparison is normalized. + entries = diffTokensCore( + tokenize(a, mode, options.locale), tokenize(b, mode, options.locale), heuristic, normalize, + ); + } else if (mode === 'char') { entries = diffChars(a, b, heuristic); - } else if (mode === 'intl-word' || mode === 'grapheme') { - entries = diffTokens(tokenize(a, mode, options.locale), tokenize(b, mode, options.locale), { heuristic }); } else { entries = diffScanned(a, b, mode, heuristic); } const finer = REFINE_TARGET[mode]; if (options.refine === true && finer !== undefined) { - return refineEntries(entries, finer, heuristic, options.locale); + return refineEntries(entries, finer, options); } return entries; } @@ -118,15 +142,20 @@ export function diffRanges(a: string, b: string, options: DiffOptions = {}): Dif } /** Re-diffs adjacent delete/insert pairs at a finer granularity. */ -function refineEntries( - entries: DiffEntry[], finerMode: DiffMode, heuristic: boolean, locale?: string | string[], -): DiffEntry[] { +function refineEntries(entries: DiffEntry[], finerMode: DiffMode, options: DiffOptions): DiffEntry[] { + const subOptions: DiffOptions = { + mode: finerMode, + heuristic: options.heuristic, + locale: options.locale, + ignoreCase: options.ignoreCase, + ignoreWhitespace: options.ignoreWhitespace, + }; const out: DiffEntry[] = []; for (let i = 0; i < entries.length; i++) { const entry = entries[i]; const next = entries[i + 1]; if (entry.operation === 'delete' && next !== undefined && next.operation === 'insert') { - for (const sub of diff(entry.text, next.text, { mode: finerMode, heuristic, locale })) { + for (const sub of diff(entry.text, next.text, subOptions)) { pushEntry(out, sub.operation, sub.text); } i++; @@ -140,6 +169,36 @@ function refineEntries( /** Diffs two pre-tokenized sequences. Tokens are compared by exact string equality. */ export function diffTokens( aTokens: readonly string[], bTokens: readonly string[], options: DiffTokensOptions = {}, +): DiffEntry[] { + const normalize = buildNormalizer(null, options.ignoreCase === true, options.ignoreWhitespace === true); + return diffTokensCore(aTokens, bTokens, options.heuristic === true, normalize); +} + +type Normalizer = (token: string) => string; + +const WS_ONLY = /^\s+$/; + +/** + * Builds the token normalizer for the ignore options, or null when tokens + * compare verbatim. 'line' mode compares trimmed lines; other modes (and + * caller-supplied tokens, mode = null) equate whitespace-only tokens. + */ +function buildNormalizer(mode: DiffMode | null, ignoreCase: boolean, ignoreWhitespace: boolean): Normalizer | null { + if (!ignoreCase && !ignoreWhitespace) return null; + return token => { + let t = token; + if (ignoreWhitespace) { + if (mode === 'line') t = t.trim(); + else if (WS_ONLY.test(t)) t = ' '; + } + if (ignoreCase) t = t.toLowerCase(); + return t; + }; +} + +function diffTokensCore( + aTokens: readonly string[], bTokens: readonly string[], + heuristic: boolean, normalize: Normalizer | null, ): DiffEntry[] { const n = aTokens.length; const m = bTokens.length; @@ -147,22 +206,26 @@ export function diffTokens( // Strip common affixes before interning so the Map only ever sees the // changed region — for a localized edit this skips almost all hashing. + // With a normalizer the ids already encode normalized equality, so the + // Myers walk handles affixes and the verbatim pre-strip is skipped. let prefix = 0; - while (prefix < minLen && aTokens[prefix] === bTokens[prefix]) prefix++; let suffix = 0; - const maxSuffix = minLen - prefix; - while (suffix < maxSuffix && aTokens[n - 1 - suffix] === bTokens[m - 1 - suffix]) suffix++; + if (normalize === null) { + while (prefix < minLen && aTokens[prefix] === bTokens[prefix]) prefix++; + const maxSuffix = minLen - prefix; + while (suffix < maxSuffix && aTokens[n - 1 - suffix] === bTokens[m - 1 - suffix]) suffix++; + } const ids = new Map(); - const ia = internRange(aTokens, prefix, n - suffix, ids); - const ib = internRange(bTokens, prefix, m - suffix, ids); - const mid = myersDiff(ia, ib, options.heuristic === true); + const ia = internRange(aTokens, prefix, n - suffix, ids, normalize); + const ib = internRange(bTokens, prefix, m - suffix, ids, normalize); + const mid = myersDiff(ia, ib, heuristic); const changedA = new Uint8Array(n); const changedB = new Uint8Array(m); changedA.set(mid.changedA, prefix); changedB.set(mid.changedB, prefix); - return buildEntries(aTokens, bTokens, changedA, changedB); + return buildEntries(aTokens, bTokens, changedA, changedB, normalize !== null); } /** @@ -170,11 +233,12 @@ export function diffTokens( * Int32Array elements instead of hashing/comparing strings. */ function internRange( - tokens: readonly string[], start: number, end: number, ids: Map, + tokens: readonly string[], start: number, end: number, + ids: Map, normalize: Normalizer | null, ): Int32Array { const out = new Int32Array(end - start); for (let i = start; i < end; i++) { - const token = tokens[i]; + const token = normalize === null ? tokens[i] : normalize(tokens[i]); let id = ids.get(token); if (id === undefined) { id = ids.size; @@ -188,6 +252,7 @@ function internRange( function buildEntries( aTokens: readonly string[], bTokens: readonly string[], changedA: Uint8Array, changedB: Uint8Array, + equalFromB: boolean, ): DiffEntry[] { const entries: DiffEntry[] = []; const n = aTokens.length; @@ -195,10 +260,14 @@ function buildEntries( let i = 0; let j = 0; while (i < n || j < m) { - const eqStart = i; + const eqStartA = i; + const eqStartB = j; while (i < n && j < m && changedA[i] === 0 && changedB[j] === 0) { i++; j++; } - if (i > eqStart) { - entries.push({ operation: 'equal', text: joinRange(aTokens, eqStart, i) }); + if (i > eqStartA) { + // Under an ignore option the paired tokens may differ in masked ways; + // taking b's text guarantees non-delete concatenation reproduces b. + const text = equalFromB ? joinRange(bTokens, eqStartB, j) : joinRange(aTokens, eqStartA, i); + entries.push({ operation: 'equal', text }); } const delStart = i; while (i < n && changedA[i] === 1) i++; diff --git a/test/ignore-options.test.ts b/test/ignore-options.test.ts new file mode 100644 index 0000000..ead43da --- /dev/null +++ b/test/ignore-options.test.ts @@ -0,0 +1,92 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { diff, diffTokens, type DiffEntry } from '../src/index.ts'; + +function joinSide(entries: DiffEntry[], skip: 'insert' | 'delete'): string { + let s = ''; + for (const e of entries) if (e.operation !== skip) s += e.text; + return s; +} + +test('ignoreCase: case-only differences become equal, text taken from b', () => { + assert.deepEqual(diff('The QUICK Fox', 'the quick fox', { ignoreCase: true }), [ + { operation: 'equal', text: 'the quick fox' }, + ]); + assert.deepEqual(diff('Foo bar', 'foo BAZ', { ignoreCase: true }), [ + { operation: 'equal', text: 'foo ' }, + { operation: 'delete', text: 'bar' }, + { operation: 'insert', text: 'BAZ' }, + ]); +}); + +test('ignoreCase: works in char mode', () => { + assert.deepEqual(diff('AbC', 'abc', { mode: 'char', ignoreCase: true }), [ + { operation: 'equal', text: 'abc' }, + ]); +}); + +test('ignoreWhitespace: whitespace runs compare equal, but presence still matters', () => { + assert.deepEqual(diff('a b', 'a b', { ignoreWhitespace: true }), [ + { operation: 'equal', text: 'a b' }, + ]); + assert.deepEqual(diff('a\t\nb', 'a b', { ignoreWhitespace: true }), [ + { operation: 'equal', text: 'a b' }, + ]); + // A whitespace token with no counterpart is still an insertion. + assert.deepEqual(diff('x', 'x ', { ignoreWhitespace: true }), [ + { operation: 'equal', text: 'x' }, + { operation: 'insert', text: ' ' }, + ]); +}); + +test('ignoreWhitespace: line mode compares trimmed lines', () => { + assert.deepEqual(diff(' hello \nworld', 'hello\nworld', { mode: 'line', ignoreWhitespace: true }), [ + { operation: 'equal', text: 'hello\nworld' }, + ]); +}); + +test('ignoreCase + ignoreWhitespace combine', () => { + assert.deepEqual(diff('Hello World', 'hello world', { ignoreCase: true, ignoreWhitespace: true }), [ + { operation: 'equal', text: 'hello world' }, + ]); +}); + +test('diffTokens: accepts ignore options for custom tokens', () => { + assert.deepEqual(diffTokens(['A'], ['a'], { ignoreCase: true }), [{ operation: 'equal', text: 'a' }]); + assert.deepEqual(diffTokens(['\t'], [' '], { ignoreWhitespace: true }), [{ operation: 'equal', text: ' ' }]); +}); + +test('ignore options: b-side reconstruction always holds', () => { + let state = 20270202 >>> 0; + const rng = () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; + const pieces = ['a', 'A', 'Bb', 'bB', ' ', ' ', '\t', '가', '\n', '.']; + const make = () => { + const len = Math.floor(rng() * 30); + let s = ''; + for (let i = 0; i < len; i++) s += pieces[Math.floor(rng() * pieces.length)]; + return s; + }; + for (let iter = 0; iter < 200; iter++) { + const a = make(); + const b = make(); + for (const mode of ['word', 'char', 'line'] as const) { + for (const opts of [{ ignoreCase: true }, { ignoreWhitespace: true }, { ignoreCase: true, ignoreWhitespace: true }]) { + const entries = diff(a, b, { mode, ...opts }); + assert.equal( + joinSide(entries, 'delete'), b, + `b mismatch mode=${mode} opts=${JSON.stringify(opts)} a=${JSON.stringify(a)} b=${JSON.stringify(b)}`, + ); + } + } + } +}); + +test('ignore options off: behavior is unchanged (both sides reconstruct)', () => { + const entries = diff('The Fox', 'the fox'); + assert.equal(joinSide(entries, 'insert'), 'The Fox'); + assert.equal(joinSide(entries, 'delete'), 'the fox'); + assert.ok(entries.some(e => e.operation !== 'equal')); +});