Add refine option for two-pass sub-token diffs

CHANGELOG.md

@@ -7,6 +7,9 @@ and the project adheres to [Semantic Versioning](https://semver.org/).
 ## [Unreleased]
 
 ### Added
+- `refine` option: re-diffs each delete/insert pair one granularity finer
+  (`line` pairs by word, `word` pairs by char), so `quick` → `quicker`
+  reports the shared prefix as equal and just `+er` as the change.
 - CI workflow: full test suite on Node 24 plus a compatibility matrix that
   smoke-tests the built ESM/CJS output on Node 16/18/20/22, backing the
   `engines: >=16` claim with an actual run.

README.md

@@ -71,6 +71,7 @@ Returns `DiffEntry[]` — the shortest edit script between `a` and `b`.
 | option | type | default | description |
 |---|---|---|---|
 | `mode` | `'word' \| 'char' \| 'line'` | `'word'` | tokenization granularity |
+| `refine` | `boolean` | `false` | re-diff each delete/insert pair one level finer (`line`→word, `word`→char), e.g. `quick`→`quicker` reports just `+er` |
 
 - `word` — runs of Unicode letters/digits/underscore, whitespace runs, symbol runs
 - `char` — individual code points (surrogate-pair safe)

src/chars.ts

@@ -1,5 +1,5 @@
 import { myersDiff } from './myers.ts';
-import type { DiffEntry, DiffOperation } from './index.ts';
+import { pushEntry, type DiffEntry } from './entries.ts';
 
 /**
  * Char-mode diff that never materializes per-character token strings.
@@ -77,13 +77,6 @@ function scanCodePoints(str: string, from: number, to: number): CodePointScan {
   return { points: count === size ? points : points.subarray(0, count), offsets, count };
 }
 
-/** Appends an entry, merging with the previous one when the operation matches. */
-function pushEntry(entries: DiffEntry[], operation: DiffOperation, text: string): void {
-  const last = entries[entries.length - 1];
-  if (last !== undefined && last.operation === operation) last.text += text;
-  else entries.push({ operation, text });
-}
-
 function isHighSurrogate(unit: number): boolean {
   return unit >= 0xd800 && unit <= 0xdbff;
 }

src/entries.ts

@@ -0,0 +1,13 @@
+export type DiffOperation = 'equal' | 'insert' | 'delete';
+
+export interface DiffEntry {
+  operation: DiffOperation;
+  text: string;
+}
+
+/** Appends an entry, merging with the previous one when the operation matches. */
+export function pushEntry(entries: DiffEntry[], operation: DiffOperation, text: string): void {
+  const last = entries[entries.length - 1];
+  if (last !== undefined && last.operation === operation) last.text += text;
+  else entries.push({ operation, text });
+}

src/index.ts

@@ -2,19 +2,20 @@ import { tokenize, type DiffMode } from './tokenize.ts';
 import { myersDiff } from './myers.ts';
 import { diffChars } from './chars.ts';
 import { diffScanned } from './scan.ts';
+import { pushEntry, type DiffEntry, type DiffOperation } from './entries.ts';
 
-export { tokenize, type DiffMode };
-
-export type DiffOperation = 'equal' | 'insert' | 'delete';
-
-export interface DiffEntry {
-  operation: DiffOperation;
-  text: string;
-}
+export { tokenize, type DiffMode, type DiffEntry, type DiffOperation };
 
 export interface DiffOptions {
   /** Tokenization granularity. Defaults to 'word'. */
   mode?: DiffMode;
+  /**
+   * Re-diffs each delete/insert pair one granularity finer ('line' pairs by
+   * word, 'word' pairs by char), so replacing "quick" with "quicker" reports
+   * the shared "quick" as equal instead of replacing the whole word. No
+   * effect in 'char' mode. Defaults to false.
+   */
+  refine?: boolean;
 }
 
 /**
@@ -29,8 +30,29 @@ export function diff(a: string, b: string, options: DiffOptions = {}): DiffEntry
     return a.length === 0 ? [] : [{ operation: 'equal', text: a }];
   }
   const mode = options.mode ?? 'word';
-  if (mode === 'char') return diffChars(a, b);
-  return diffScanned(a, b, mode);
+  const entries = mode === 'char' ? diffChars(a, b) : diffScanned(a, b, mode);
+  if (options.refine === true && mode !== 'char') {
+    return refineEntries(entries, mode === 'line' ? 'word' : 'char');
+  }
+  return entries;
+}
+
+/** Re-diffs adjacent delete/insert pairs at a finer granularity. */
+function refineEntries(entries: DiffEntry[], finerMode: DiffMode): DiffEntry[] {
+  const out: DiffEntry[] = [];
+  for (let i = 0; i < entries.length; i++) {
+    const entry = entries[i];
+    const next = entries[i + 1];
+    if (entry.operation === 'delete' && next !== undefined && next.operation === 'insert') {
+      for (const sub of diff(entry.text, next.text, { mode: finerMode })) {
+        pushEntry(out, sub.operation, sub.text);
+      }
+      i++;
+    } else {
+      pushEntry(out, entry.operation, entry.text);
+    }
+  }
+  return out;
 }
 
 /** Diffs two pre-tokenized sequences. Tokens are compared by exact string equality. */

src/scan.ts

@@ -1,5 +1,5 @@
 import { myersDiff } from './myers.ts';
-import type { DiffEntry, DiffOperation } from './index.ts';
+import { pushEntry, type DiffEntry } from './entries.ts';
 
 /**
  * Fused scan pipeline for word/line modes: instead of materializing token
@@ -76,13 +76,6 @@ export function diffScanned(a: string, b: string, mode: 'word' | 'line'): DiffEn
   return entries;
 }
 
-/** Appends an entry, merging with the previous one when the operation matches. */
-function pushEntry(entries: DiffEntry[], operation: DiffOperation, text: string): void {
-  const last = entries[entries.length - 1];
-  if (last !== undefined && last.operation === operation) last.text += text;
-  else entries.push({ operation, text });
-}
-
 function rangesEqual(a: string, as: number, ae: number, b: string, bs: number, be: number): boolean {
   if (ae - as !== be - bs) return false;
   for (let i = as, j = bs; i < ae; i++, j++) {

test/refine.test.ts

@@ -0,0 +1,73 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { diff, type DiffEntry } from '../src/index.ts';
+
+function joinSide(entries: DiffEntry[], skip: 'insert' | 'delete'): string {
+  let s = '';
+  for (const e of entries) if (e.operation !== skip) s += e.text;
+  return s;
+}
+
+test('refine: word-mode pairs are re-diffed at char level', () => {
+  assert.deepEqual(diff('the quick fox', 'the quicker fox', { refine: true }), [
+    { operation: 'equal', text: 'the quick' },
+    { operation: 'insert', text: 'er' },
+    { operation: 'equal', text: ' fox' },
+  ]);
+});
+
+test('refine: line-mode pairs are re-diffed at word level', () => {
+  assert.deepEqual(diff('alpha beta\ngamma\n', 'alpha zeta\ngamma\n', { mode: 'line', refine: true }), [
+    { operation: 'equal', text: 'alpha ' },
+    { operation: 'delete', text: 'beta' },
+    { operation: 'insert', text: 'zeta' },
+    { operation: 'equal', text: '\ngamma\n' },
+  ]);
+});
+
+test('refine: char mode is unaffected (already finest granularity)', () => {
+  const a = 'kitten';
+  const b = 'sitting';
+  assert.deepEqual(diff(a, b, { mode: 'char', refine: true }), diff(a, b, { mode: 'char' }));
+});
+
+test('refine: solo deletes and inserts pass through unchanged', () => {
+  assert.deepEqual(diff('keep removed keep', 'keep keep', { refine: true }), diff('keep removed keep', 'keep keep'));
+  assert.deepEqual(diff('', 'abc', { refine: true }), [{ operation: 'insert', text: 'abc' }]);
+});
+
+test('refine: entries stay merged and ordered after splicing', () => {
+  const entries = diff('aaa bbb ccc', 'aaa bXb ccc', { refine: true });
+  // No two adjacent entries share an operation.
+  for (let i = 1; i < entries.length; i++) {
+    assert.notEqual(entries[i].operation, entries[i - 1].operation, JSON.stringify(entries));
+  }
+  // Within the changed region, delete still precedes insert.
+  const delIdx = entries.findIndex(e => e.operation === 'delete');
+  const insIdx = entries.findIndex(e => e.operation === 'insert');
+  assert.ok(delIdx < insIdx);
+});
+
+test('refine fuzz: round-trip invariant holds in every mode', () => {
+  let state = 20261101 >>> 0;
+  const rng = () => {
+    state = (state * 1664525 + 1013904223) >>> 0;
+    return state / 0x100000000;
+  };
+  const pieces = ['a', 'bb', '가나', '😀', ' ', '\n', '.', 'word'];
+  const make = () => {
+    const len = Math.floor(rng() * 30);
+    let s = '';
+    for (let i = 0; i < len; i++) s += pieces[Math.floor(rng() * pieces.length)];
+    return s;
+  };
+  for (let iter = 0; iter < 200; iter++) {
+    const a = make();
+    const b = make();
+    for (const mode of ['word', 'char', 'line'] as const) {
+      const entries = diff(a, b, { mode, refine: true });
+      assert.equal(joinSide(entries, 'insert'), a, `a mismatch mode=${mode} a=${JSON.stringify(a)} b=${JSON.stringify(b)}`);
+      assert.equal(joinSide(entries, 'delete'), b, `b mismatch mode=${mode} a=${JSON.stringify(a)} b=${JSON.stringify(b)}`);
+    }
+  }
+});