From f59996ca047d74b74619bf945961826c283f0cda Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 7 Oct 2026 14:27:23 -0700 Subject: [PATCH 01/16] fix(remend): stop appending emphasis closers that match no opener Underscore emphasis was healed by counting delimiter parity, which cannot see flanking, the rule of 3, or partial matches between runs of different lengths. Inputs like `a __ b \__c`, `__ foo`, `a__.`, and `__a____.a` gained a closer that renders as literal text, while `a __ b __bold` lost one it needed. A new module matches _ and __ runs with the CommonMark emphasis algorithm, using remaining run lengths for the rule of 3. The underscore handlers close the innermost opener it leaves open, and keep a heal only if every underscore it added matched. Escape checks for *, _, **, ~~, and backticks now look for an odd run of backslashes, so a marker after an escaped backslash still opens. A trailing unescaped backslash would escape any appended closer, so the text is healed with that backslash escaped. A fast-check property generates marker-dense text and asserts, against mdast-util-from-markdown, that every appended closer matches an opener. --- .changeset/remend-escaped-markers.md | 9 + .../broken-markdown-variants.test.ts | 11 +- .../remend/__tests__/escaped-markers.test.ts | 80 ++++++ packages/remend/__tests__/italic.test.ts | 3 +- packages/remend/__tests__/lists.test.ts | 4 +- .../__tests__/streaming-properties.test.ts | 77 ++++++ .../remend/__tests__/underscore-runs.test.ts | 23 ++ packages/remend/src/emphasis-handlers.ts | 227 +++--------------- packages/remend/src/index.ts | 42 +++- packages/remend/src/scan.ts | 13 +- packages/remend/src/underscore-delimiters.ts | 178 ++++++++++++++ packages/remend/src/utils.ts | 5 + 12 files changed, 454 insertions(+), 218 deletions(-) create mode 100644 .changeset/remend-escaped-markers.md create mode 100644 packages/remend/__tests__/escaped-markers.test.ts create mode 100644 packages/remend/src/underscore-delimiters.ts diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md new file mode 100644 index 00000000..ceb64c00 --- /dev/null +++ b/.changeset/remend-escaped-markers.md @@ -0,0 +1,9 @@ +--- +"remend": patch +--- + +fix(remend): stop appending emphasis closers that match no opener + +- Match `_` and `__` delimiters with the CommonMark emphasis algorithm (flanking, rule of 3, partial matches) instead of counting parity, and drop an underscore closer that would not match +- Treat a marker as escaped only after an odd run of backslashes, so `\\*`, `\\_`, `\\**`, `\\~~`, and `` \\` `` still open +- Escape a trailing backslash before appending closers, so it cannot escape them diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index b483fddc..6fa64cae 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -77,17 +77,14 @@ describe("backslash escapes with incomplete formatting", () => { expect(result).toBe("\\*not italic"); }); - it("should not close double-escaped backslash before asterisk", () => { - // remend sees the char before * as \ (the second backslash) and treats it as escaped + it("should close an asterisk after an escaped backslash", () => { const result = remend("\\\\*actually italic"); - expect(result).toBe("\\\\*actually italic"); + expect(result).toBe("\\\\*actually italic*"); }); - it("should close escaped double asterisks (remend does not track escape depth)", () => { - // remend doesn't understand that \** has the first * escaped; - // it sees ** and closes bold + it("should not close escaped double asterisks", () => { const result = remend("\\**not bold"); - expect(result).toBe("\\**not bold**"); + expect(result).toBe("\\**not bold"); }); it("should handle mixed escaped and real formatting", () => { diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts new file mode 100644 index 00000000..ec924418 --- /dev/null +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("escaped markers", () => { + it.each<{ name: string; input: string; expected: string }>([ + { + name: "closes bold whose content holds an escaped **", + input: "**a\\**b", + expected: "**a\\**b**", + }, + { + name: "skips an escaped ~~", + input: "\\~~strike", + expected: "\\~~strike", + }, + { + name: "closes ** after an escaped backslash", + input: "\\\\**bold", + expected: "\\\\**bold**", + }, + { + name: "closes ~~ after an escaped backslash", + input: "\\\\~~strike", + expected: "\\\\~~strike~~", + }, + { + name: "closes * after an escaped backslash", + input: "\\\\*foo", + expected: "\\\\*foo*", + }, + { + name: "closes _ after an escaped backslash", + input: "\\\\_foo", + expected: "\\\\_foo_", + }, + { + name: "opens a code span after an escaped backslash", + input: "\\\\``a", + expected: "\\\\``a``", + }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); + +describe("trailing backslash", () => { + // A closer appended after a trailing backslash would be escaped by it, so + // the backslash is escaped first + it.each<{ name: string; input: string; expected: string }>([ + { name: "**", input: "**foo\\", expected: "**foo\\\\**" }, + { name: "*", input: "*foo\\", expected: "*foo\\\\*" }, + { name: "_", input: "_foo\\", expected: "_foo\\\\_" }, + { name: "__", input: "__foo\\", expected: "__foo\\\\__" }, + { name: "~~", input: "~~foo\\", expected: "~~foo\\\\~~" }, + { name: "a partial __ match", input: "____.\\", expected: "____.\\\\__" }, + { + name: "a backslash after whitespace", + input: "**foo \\", + expected: "**foo \\\\**", + }, + { + name: "nothing to heal", + input: "plain text\\", + expected: "plain text\\", + }, + { + name: "an opener whose only content is the backslash", + input: "______\n__\\", + expected: "______\n__\\\\__", + }, + { + name: "an escaped trailing backslash", + input: "**foo\\\\", + expected: "**foo\\\\**", + }, + { name: "inline code", input: "`foo\\", expected: "`foo\\`" }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); diff --git a/packages/remend/__tests__/italic.test.ts b/packages/remend/__tests__/italic.test.ts index e0233cda..4ad9807b 100644 --- a/packages/remend/__tests__/italic.test.ts +++ b/packages/remend/__tests__/italic.test.ts @@ -120,8 +120,7 @@ describe("italic formatting with single underscores (_)", () => { expect(remend("naïve_approach")).toBe("naïve_approach"); }); - it("should not count word-internal single underscores in countSingleUnderscores", () => { - // This tests the path where underscore is between word characters (lines 106-108) + it("should not count word-internal single underscores", () => { expect(remend("some_variable_name")).toBe("some_variable_name"); expect(remend("test_123_value")).toBe("test_123_value"); expect(remend("_start with underscore")).toBe("_start with underscore_"); diff --git a/packages/remend/__tests__/lists.test.ts b/packages/remend/__tests__/lists.test.ts index 21f668ea..64a54667 100644 --- a/packages/remend/__tests__/lists.test.ts +++ b/packages/remend/__tests__/lists.test.ts @@ -69,8 +69,10 @@ describe("list handling", () => { expect(remend("* __\n* **")).toBe("* __\n* **"); expect(remend("+ __\n+ **")).toBe("+ __\n+ **"); + // A __ followed by whitespace cannot open emphasis + expect(remend("- __ text after")).toBe("- __ text after"); + // List items with emphasis markers and text should still complete - expect(remend("- __ text after")).toBe("- __ text after__"); expect(remend("- ** text after")).toBe("- ** text after**"); // Mixed list items diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 6199a7c2..10792a9d 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -60,6 +60,7 @@ const preservedPrefixLength = (input: string, output: string): number => { interface CodeNode { lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value: string; } @@ -68,6 +69,7 @@ interface MdastNode { children?: MdastNode[]; lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value?: string; } @@ -267,6 +269,81 @@ describe("streaming properties", () => { }); }); +const UNDERSCORE_RUN_TOKENS = [ + "a", + "word", + " ", + ".", + "`", + "\\", + "_", + "__", + "___", +]; + +const underscoreRunArbitrary = fc + .array(fc.constantFrom(...UNDERSCORE_RUN_TOKENS), { + minLength: 1, + maxLength: 10, + }) + .map((tokens) => tokens.join("")); + +// Gaps outside this property: a closer appended after whitespace cannot +// close, and remend does not recognize indented code blocks +const TRAILING_WHITESPACE = /\s$/; +const INDENTED_CODE = /^ {4}/; + +const MARKER_PATTERN = /[_*]/g; +// Closers, after the backslash that escapes a trailing backslash +const APPENDED_CLOSERS = /^\\?[_*]+$/; + +const countLiteralMarkers = (markdown: string): number => { + let count = 0; + const visit = (node: MdastNode): void => { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (node.type === "text" && start !== undefined && end !== undefined) { + count += markdown.slice(start, end).match(MARKER_PATTERN)?.length ?? 0; + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(fromMarkdown(markdown)); + return count; +}; + +// Each marker healing appends must close one the prefix left literal, so the +// literal count drops by at least as many as were appended. A closer that +// renders as literal text matched nothing, so remend invented it. +const assertAppendedClosersMatch = (prefix: string): void => { + const healed = remend(prefix); + const suffix = healed.slice(prefix.length); + if (!(healed.startsWith(prefix) && APPENDED_CLOSERS.test(suffix))) { + return; + } + const appended = suffix.match(MARKER_PATTERN)?.length ?? 0; + if (countLiteralMarkers(healed) > countLiteralMarkers(prefix) - appended) { + throw new Error( + `healing appended an unmatched closer: ${JSON.stringify(prefix)} -> ${JSON.stringify(healed)}` + ); + } +}; + +describe("appended closers", () => { + it("never appends an underscore closer that matches no opener", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + fc.pre( + !(TRAILING_WHITESPACE.test(prefix) || INDENTED_CODE.test(prefix)) + ); + assertAppendedClosersMatch(prefix); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); +}); + describe("exhaustive prefix sweep", () => { // Every prefix of a fixed corpus, deterministically. The corpus mixes // constructs that interact. diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 120f6a29..467354d0 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -54,4 +54,27 @@ describe("escaped underscores", () => { it("should treat the run after an escaped underscore as a delimiter", () => { expect(remend("\\___bold")).toBe("\\___bold__"); }); + + it("should not treat a run after an escaped backslash as escaped", () => { + expect(remend("\\\\__bold")).toBe("\\\\__bold__"); + }); +}); + +describe("underscore delimiter matching", () => { + it.each<[string, string]>([ + ["a run followed by whitespace", "__ foo"], + ["a spaced run before an escaped run", "a __ b \\__c"], + ["a spaced run before an escaped run at the start", "__ \\__alph"], + ["a run after a letter and before punctuation", "a__."], + ["a closer-only run with nothing open", "__ a__ a"], + ["a longer run that closes the opener", "__a____.a"], + ["an opener the rule of 3 keeps a __ closer from matching", "\\_____\\a"], + ["a leftover single underscore after an escape", "__\\__ a"], + ])("should not append a closer for %s", (_name, input) => { + expect(remend(input)).toBe(input); + }); + + it("should still close an opener after a spaced run", () => { + expect(remend("a __ b __bold")).toBe("a __ b __bold__"); + }); }); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 8905ae68..3d40ecb0 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -17,13 +17,18 @@ import { import { countDoublePairs, getScan, - inHtmlTagAt, inLinkUrlAt, inMathAt, + isEscaped, REGION, type TextScan, } from "./scan"; -import { isHorizontalRule, isWordChar } from "./utils"; +import { + closesAddedUnderscores, + findOpenUnderscoreRuns, + type OpenUnderscoreRun, +} from "./underscore-delimiters"; +import { isHorizontalRule, isWhitespaceChar, isWordChar } from "./utils"; // Helper function to check if an asterisk should be skipped const shouldSkipAsterisk = ( @@ -33,7 +38,7 @@ const shouldSkipAsterisk = ( nextChar: string ): boolean => { // Skip if escaped with backslash - if (prevChar === "\\") { + if (isEscaped(scan.text, index)) { return true; } @@ -75,9 +80,6 @@ const shouldSkipAsterisk = ( return false; }; -const isWhitespaceChar = (char: string): boolean => - char === " " || char === "\t" || char === "\n"; - const isWordInternalAsterisk = (prevChar: string, nextChar: string): boolean => Boolean(prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)); @@ -149,67 +151,6 @@ export const countSingleAsterisks = (text: string): number => { return count; }; -// Helper function to check if an underscore should be skipped -const shouldSkipUnderscore = ( - scan: TextScan, - index: number, - prevChar: string, - nextChar: string -): boolean => { - // Skip if escaped with backslash - if (prevChar === "\\") { - return true; - } - - // Skip if within math block - if (inMathAt(scan, index)) { - return true; - } - - // Skip if within a link or image URL - if (inLinkUrlAt(scan, index)) { - return true; - } - - // Skip if within an HTML tag (e.g. ) - if (inHtmlTagAt(scan, index)) { - return true; - } - - // Skip if part of __ - if (prevChar === "_" || nextChar === "_") { - return true; - } - - // Skip if underscore is word-internal (between word characters) - if (prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)) { - return true; - } - - return false; -}; - -export const countSingleUnderscores = (text: string): number => { - const scan = getScan(text); - let count = 0; - const len = text.length; - - for (let index = 0; index < len; index += 1) { - if (text[index] !== "_" || scan.regions[index] !== REGION.PROSE) { - continue; - } - - const prevChar = index > 0 ? text[index - 1] : ""; - const nextChar = index < len - 1 ? text[index + 1] : ""; - - if (!shouldSkipUnderscore(scan, index, prevChar, nextChar)) { - count += 1; - } - } - - return count; -}; - // Counts triple asterisks that are not part of quadruple or more asterisks // and not inside code regions export const countTripleAsterisks = (text: string): number => { @@ -240,125 +181,18 @@ export const countTripleAsterisks = (text: string): number => { const countDoubleAsterisks = (text: string): number => countDoublePairs(text, "*"); -// Whether the text has an unmatched __ delimiter, counted per maximal -// underscore run. -// -// Counting raw occurrences misreads identifiers: a name like snake__case -// contains __ but cannot open or close emphasis, and counting it either -// invents a closer (odd count) or pairs it against a real delimiter and -// swallows a closer that was needed (even count). Per run: -// -// - A run contributes floor(length / 2) pairs, and flips delimiter parity -// only when that is odd. __ and ___ flip; ____ does not. -// - A word-internal run (word characters on both sides) is part of an -// identifier, never a delimiter. -// - Runs inside code regions, math, link URLs, and HTML tags are skipped, -// matching the single-underscore handler's flanking rules. -// - A run alone on its line is a thematic break, not emphasis. -const isLineBoundaryChar = (char: string): boolean => - char === "" || char === " " || char === "\t" || char === "\n"; - -// isHorizontalRule scans the run's whole line, so its verdict is memoized -// per line to keep run counting linear when many runs share a line -interface ThematicBreakMemo { - lineEnd: number; - result: boolean; -} - -const isThematicBreakRun = ( - text: string, - runStart: number, - prevChar: string, - nextChar: string, - memo: ThematicBreakMemo -): boolean => { - // A thematic break line holds only markers and whitespace, so only runs - // flanked by whitespace or line boundaries can be part of one - if (!(isLineBoundaryChar(prevChar) && isLineBoundaryChar(nextChar))) { - return false; - } - if (runStart > memo.lineEnd) { - const lineEnd = text.indexOf("\n", runStart); - memo.lineEnd = lineEnd === -1 ? text.length : lineEnd; - memo.result = isHorizontalRule(text, runStart, "_"); - } - return memo.result; -}; - -// Whether an underscore run at [runStart, runEnd) flips delimiter parity -const doubleUnderscoreRunFlips = ( - scan: TextScan, - initialRunStart: number, - runEnd: number, - memo: ThematicBreakMemo -): boolean => { - const text = scan.text; - - // A backslash escapes the first underscore of the run. The escaped - // underscore is literal punctuation, so the rest of the run still flanks - // as a delimiter. - let runStart = initialRunStart; - let escaped = false; - if (runStart > 0 && text[runStart - 1] === "\\") { - runStart += 1; - escaped = true; - } - const runLength = runEnd - runStart; - if (runLength < 2) { - return false; - } - - const beforeRun = runStart > 0 ? text[runStart - 1] : ""; - const prevChar = escaped ? "\\" : beforeRun; - const nextChar = runEnd < text.length ? text[runEnd] : ""; - if (isWordChar(prevChar) && isWordChar(nextChar)) { - return false; - } - if (isThematicBreakRun(text, runStart, prevChar, nextChar, memo)) { - return false; - } - if ( - inMathAt(scan, runStart) || - inLinkUrlAt(scan, runStart) || - inHtmlTagAt(scan, runStart) - ) { - return false; - } +const isHalfClosedDoubleUnderscore = ( + run: OpenUnderscoreRun | undefined +): boolean => run !== undefined && run.count === 1 && run.length >= 2; - return Math.floor(runLength / 2) % 2 === 1; -}; +// Healing that adds an underscore closer which cannot match would render it +// as literal text, so such a heal is dropped +const healUnderscores = (text: string, healed: string): string => + closesAddedUnderscores(text, healed) ? healed : text; -const hasUnmatchedDoubleUnderscore = (text: string): boolean => { - const scan = getScan(text); - const n = text.length; - const memo: ThematicBreakMemo = { lineEnd: -1, result: false }; - let unmatched = false; - let i = 0; - - while (i < n) { - if (text[i] !== "_" || scan.regions[i] !== REGION.PROSE) { - i += 1; - continue; - } - - const runStart = i; - let runEnd = i + 1; - while ( - runEnd < n && - text[runEnd] === "_" && - scan.regions[runEnd] === REGION.PROSE - ) { - runEnd += 1; - } - i = runEnd; - - if (doubleUnderscoreRunFlips(scan, runStart, runEnd, memo)) { - unmatched = !unmatched; - } - } - - return unmatched; -}; +const innermostOpenUnderscoreRun = ( + text: string +): OpenUnderscoreRun | undefined => findOpenUnderscoreRuns(text).at(-1); // Helper to check if bold marker should not be completed const shouldSkipBoldCompletion = ( @@ -475,9 +309,9 @@ export const handleIncompleteDoubleUnderscoreItalic = ( isInsideCodeBlock(text, markerIndex) || isWithinCompleteInlineCode(text, markerIndex) ) && - hasUnmatchedDoubleUnderscore(text) + isHalfClosedDoubleUnderscore(innermostOpenUnderscoreRun(text)) ) { - return `${text}_`; + return healUnderscores(text, `${text}_`); } } return text; @@ -498,8 +332,9 @@ export const handleIncompleteDoubleUnderscoreItalic = ( return text; } - if (hasUnmatchedDoubleUnderscore(text)) { - return `${text}__`; + const openRun = innermostOpenUnderscoreRun(text); + if (openRun && openRun.count >= 2) { + return healUnderscores(text, `${text}__`); } return text; @@ -515,7 +350,7 @@ const isLoneProseAsterisk = (scan: TextScan, i: number): boolean => { scan.regions[i] === REGION.PROSE && text[i - 1] !== "*" && text[i + 1] !== "*" && - text[i - 1] !== "\\" && + !isEscaped(text, i) && !inMathAt(scan, i) ); }; @@ -605,7 +440,7 @@ const findFirstSingleUnderscoreIndex = (text: string): number => { scan.regions[i] === REGION.PROSE && text[i - 1] !== "_" && text[i + 1] !== "_" && - text[i - 1] !== "\\" && + !isEscaped(text, i) && !inMathAt(scan, i) && !inLinkUrlAt(scan, i) ) { @@ -713,14 +548,12 @@ export const handleIncompleteSingleUnderscoreItalic = ( return text; } - const singleUnderscores = countSingleUnderscores(text); - if (singleUnderscores % 2 === 1) { + if (innermostOpenUnderscoreRun(text)?.count === 1) { // Check if we need to insert _ before trailing ** for proper nesting - const trailingResult = handleTrailingAsterisksForUnderscore(text); - if (trailingResult !== null) { - return trailingResult; - } - return insertClosingUnderscore(text); + const healed = + handleTrailingAsterisksForUnderscore(text) ?? + insertClosingUnderscore(text); + return healUnderscores(text, healed); } return text; diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 0ecf9137..6a39f53d 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -17,7 +17,7 @@ import { INCOMPLETE_IMAGE_PLACEHOLDER, type LinkMode, } from "./link-image-handler"; -import { clearScanCache } from "./scan"; +import { clearScanCache, getScan, inMathAt, isEscaped, REGION } from "./scan"; import { handleIncompleteSetextHeading } from "./setext-heading-handler"; import { handleSingleTildeEscape } from "./single-tilde-handler"; import { handleIncompleteStrikethrough } from "./strikethrough-handler"; @@ -277,6 +277,16 @@ const getEnabledBuiltInHandlers = ( }); }; +// Whether the text ends with an unescaped backslash in prose +const isTrailingEscape = (text: string): boolean => { + const last = text.length - 1; + if (text[last] !== "\\" || isEscaped(text, last)) { + return false; + } + const scan = getScan(text); + return scan.regions[last] === REGION.PROSE && !inMathAt(scan, last); +}; + // Parses markdown text and removes incomplete tokens to prevent partial rendering const remend = (text: string, options?: RemendOptions): string => { if (!text || typeof text !== "string") { @@ -284,7 +294,7 @@ const remend = (text: string, options?: RemendOptions): string => { } // Remove trailing whitespace if it's not a double space - let result = + const result = text.endsWith(" ") && !text.endsWith(" ") ? text.slice(0, -1) : text; // Get enabled built-in handlers @@ -302,25 +312,39 @@ const remend = (text: string, options?: RemendOptions): string => { (a, b) => (a.handler.priority ?? 0) - (b.handler.priority ?? 0) ); - try { + const runHandlers = (input: string): string => { + let output = input; // Execute handlers in priority order for (const { handler, earlyReturn } of allHandlers) { - result = handler.handle(result); + output = handler.handle(output); // Check for early return condition (e.g., incomplete link marker) - if (earlyReturn?.(result)) { - return result; + if (earlyReturn?.(output)) { + return output; } } // A handler that removes a trailing fragment can expose a trailing space // (e.g. dropping an incomplete image). Strip it the same way the input // was stripped, so healed output re-heals to itself. - if (result.endsWith(" ") && !result.endsWith(" ")) { - return result.slice(0, -1); + if (output.endsWith(" ") && !output.endsWith(" ")) { + return output.slice(0, -1); + } + + return output; + }; + + try { + if (!isTrailingEscape(result)) { + return runHandlers(result); } - return result; + // A trailing backslash escapes whatever follows it, so a closer appended + // after it would render as text. Heal with the backslash escaped, which + // keeps it literal without changing how the markers before it flank. + const escaped = `${result}\\`; + const healed = runHandlers(escaped); + return healed === escaped ? result : healed; } finally { clearScanCache(); } diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 1bb9f74f..5bf68c9b 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -342,7 +342,7 @@ const nextSpanOpener = ( return -1; } const escaped = - next > i && text[next - 1] === "\\" && regions[next - 1] === REGION.PROSE; + next > i && regions[next - 1] === REGION.PROSE && isEscaped(text, next); if (regions[next] === REGION.PROSE && !escaped) { return next; } @@ -473,6 +473,15 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => scan.regions[position] === REGION.CODE_SPAN; +/** Whether the character at position follows an odd run of backslashes */ +export const isEscaped = (text: string, position: number): boolean => { + let backslashes = 0; + for (let i = position - 1; i >= 0 && text[i] === "\\"; i -= 1) { + backslashes += 1; + } + return backslashes % 2 === 1; +}; + /** Counts non-overlapping double-character pairs (**, ~~, $$) in prose */ export const countDoublePairs = (text: string, char: string): number => { const scan = getScan(text); @@ -480,7 +489,7 @@ export const countDoublePairs = (text: string, char: string): number => { let count = 0; for (let i = text.indexOf(pair); i !== -1; i = text.indexOf(pair, i)) { - if (scan.regions[i] === REGION.PROSE) { + if (scan.regions[i] === REGION.PROSE && !isEscaped(text, i)) { count += 1; i += 2; } else { diff --git a/packages/remend/src/underscore-delimiters.ts b/packages/remend/src/underscore-delimiters.ts new file mode 100644 index 00000000..c24c384d --- /dev/null +++ b/packages/remend/src/underscore-delimiters.ts @@ -0,0 +1,178 @@ +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isEscaped, + REGION, + type TextScan, +} from "./scan"; +import { isWhitespaceChar, isWordChar } from "./utils"; + +/** An underscore delimiter run left open at the end of the text */ +export interface OpenUnderscoreRun { + /** Underscores still unmatched */ + count: number; + /** Unescaped length of the run before any were matched */ + length: number; +} + +interface DelimiterRun extends OpenUnderscoreRun { + canClose: boolean; + canOpen: boolean; +} + +// The start and end of the text count as whitespace for flanking +const isWhitespaceOrEdge = (char: string): boolean => + char === "" || isWhitespaceChar(char); + +const isPunctuationChar = (char: string): boolean => + char === "_" || !(isWhitespaceOrEdge(char) || isWordChar(char)); + +// Flanking per CommonMark, with the stricter underscore rules: a run that +// flanks on both sides opens only after punctuation and closes only before +// punctuation, so word-internal runs (snake__case) can't open or close +const toDelimiterRun = ( + text: string, + index: number, + end: number, + prevChar: string +): DelimiterRun => { + const nextChar = end < text.length ? text[end] : ""; + const leftFlanking = !( + isWhitespaceOrEdge(nextChar) || + (isPunctuationChar(nextChar) && + !isWhitespaceOrEdge(prevChar) && + !isPunctuationChar(prevChar)) + ); + const rightFlanking = !( + isWhitespaceOrEdge(prevChar) || + (isPunctuationChar(prevChar) && + !isWhitespaceOrEdge(nextChar) && + !isPunctuationChar(nextChar)) + ); + const length = end - index; + return { + canClose: rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)), + canOpen: leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)), + count: length, + length, + }; +}; + +const isSkippedUnderscore = (scan: TextScan, i: number): boolean => + scan.text[i] !== "_" || + scan.regions[i] !== REGION.PROSE || + inMathAt(scan, i) || + inLinkUrlAt(scan, i) || + inHtmlTagAt(scan, i); + +const collectDelimiterRuns = (scan: TextScan): DelimiterRun[] => { + const { text } = scan; + const runs: DelimiterRun[] = []; + let i = text.indexOf("_"); + + while (i !== -1) { + if (isSkippedUnderscore(scan, i)) { + i = text.indexOf("_", i + 1); + continue; + } + + let end = i + 1; + while (end < text.length && !isSkippedUnderscore(scan, end)) { + end += 1; + } + + // A backslash escapes the run's first underscore, which stays literal + // punctuation before the rest of the run + if (isEscaped(text, i)) { + if (end - i > 1) { + runs.push(toDelimiterRun(text, i + 1, end, "_")); + } + } else { + runs.push(toDelimiterRun(text, i, end, i > 0 ? text[i - 1] : "")); + } + i = text.indexOf("_", end); + } + + return runs; +}; + +// The "rule of 3": when either run can both open and close, they match only +// if their combined length is not a multiple of 3, unless both lengths are. +// Lengths are what remains of each run after earlier partial matches. +const violatesRuleOfThree = ( + opener: DelimiterRun, + closer: DelimiterRun +): boolean => + (closer.canOpen || opener.canClose) && + (opener.count + closer.count) % 3 === 0 && + !(opener.count % 3 === 0 && closer.count % 3 === 0); + +// Matches a closing run against the stack until it runs out of underscores +// or openers, popping every opener it passes +const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { + while (closer.count > 0) { + let openerIndex = stack.length - 1; + while ( + openerIndex >= 0 && + violatesRuleOfThree(stack[openerIndex], closer) + ) { + openerIndex -= 1; + } + if (openerIndex < 0) { + return; + } + + const opener = stack[openerIndex]; + const used = opener.count >= 2 && closer.count >= 2 ? 2 : 1; + opener.count -= used; + closer.count -= used; + stack.length = opener.count > 0 ? openerIndex + 1 : openerIndex; + } +}; + +// Runs the CommonMark emphasis algorithm over the text's underscore runs, +// leaving each run's unmatched count and the openers still open +const matchUnderscoreRuns = ( + text: string +): { runs: DelimiterRun[]; stack: DelimiterRun[] } => { + const runs = collectDelimiterRuns(getScan(text)); + const stack: DelimiterRun[] = []; + + for (const run of runs) { + if (run.canClose) { + matchCloser(stack, run); + } + if (run.canOpen && run.count > 0) { + stack.push(run); + } + } + + return { runs, stack }; +}; + +/** The underscore openers the text leaves unmatched, outermost first */ +export const findOpenUnderscoreRuns = (text: string): OpenUnderscoreRun[] => + matchUnderscoreRuns(text).stack; + +// Underscores that render as literal text: openers still open, openers a +// closer passed over, and closers left over +const countLiteralUnderscores = (text: string): number => + matchUnderscoreRuns(text).runs.reduce((sum, run) => sum + run.count, 0); + +/** + * Whether every underscore that healing added to the text matched one the + * text left literal. An added closer can fail to match, for example under + * the rule of 3. + */ +export const closesAddedUnderscores = ( + text: string, + healed: string +): boolean => { + // Scan the text first, while the scan cache still holds it + const literal = countLiteralUnderscores(text); + return ( + countLiteralUnderscores(healed) <= literal - (healed.length - text.length) + ); +}; diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 06291fd0..59c1898e 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,6 +1,11 @@ import { letterNumberUnderscorePattern } from "./patterns"; import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; +const whitespaceCharPattern = /\s/; + +export const isWhitespaceChar = (char: string): boolean => + whitespaceCharPattern.test(char); + // OPTIMIZATION: Precompute which characters are word characters // Using ASCII fast path before falling back to Unicode regex export const isWordChar = (char: string): boolean => { From 197b451d2db50341e36ec8c60745b6f9b8da0ef4 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 7 Oct 2026 16:18:44 -0700 Subject: [PATCH 02/16] fix(remend): heal every open underscore run in one pass The underscore handlers closed only the innermost open run and ran before the inline code handler, so a second remend pass could append more closers. Heal against the closed code span, repeat until no underscore opener is left, and match asterisk runs alongside underscores so a matched asterisk pair discards the openers inside it. Escapes in countDoublePairs no longer apply to $$, since a backslash inside math is TeX. --- .changeset/remend-escaped-markers.md | 1 + .../remend/__tests__/escaped-markers.test.ts | 11 ++++ .../__tests__/streaming-properties.test.ts | 10 +++ .../remend/__tests__/underscore-runs.test.ts | 23 +++++++ packages/remend/src/emphasis-handlers.ts | 42 +++++++++++-- packages/remend/src/scan.ts | 4 +- packages/remend/src/underscore-delimiters.ts | 63 ++++++++++++------- 7 files changed, 126 insertions(+), 28 deletions(-) diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index ceb64c00..9844d3cb 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -7,3 +7,4 @@ fix(remend): stop appending emphasis closers that match no opener - Match `_` and `__` delimiters with the CommonMark emphasis algorithm (flanking, rule of 3, partial matches) instead of counting parity, and drop an underscore closer that would not match - Treat a marker as escaped only after an odd run of backslashes, so `\\*`, `\\_`, `\\**`, `\\~~`, and `` \\` `` still open - Escape a trailing backslash before appending closers, so it cannot escape them +- Close every open underscore run in one pass, including runs before an unterminated code span, so healed output re-heals to itself diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts index ec924418..4f53aa2a 100644 --- a/packages/remend/__tests__/escaped-markers.test.ts +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -43,6 +43,17 @@ describe("escaped markers", () => { }); }); +describe("math", () => { + // Inside math a backslash is TeX, so it does not escape the closing $$ + it.each<{ name: string; input: string }>([ + { name: "a backslash before $$", input: "$$a\\$$" }, + { name: "a lone backslash", input: "$$\\$$" }, + { name: "inline block math", input: "text $$\\frac\\$$ more" }, + ])("leaves complete math with $name unchanged", ({ input }) => { + expect(remend(input)).toBe(input); + }); +}); + describe("trailing backslash", () => { // A closer appended after a trailing backslash would be escaped by it, so // the backslash is escaped first diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 10792a9d..99237afb 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -342,6 +342,16 @@ describe("appended closers", () => { { numRuns: Number(process.env.NUM_RUNS ?? 2000) } ); }); + + it("re-heals underscore runs to themselves", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + const healed = remend(prefix); + expect(remend(healed)).toBe(healed); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); }); describe("exhaustive prefix sweep", () => { diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 467354d0..2cfb02a6 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -77,4 +77,27 @@ describe("underscore delimiter matching", () => { it("should still close an opener after a spaced run", () => { expect(remend("a __ b __bold")).toBe("a __ b __bold__"); }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "an outer run after an escape", + input: "\\__._a", + expected: "\\__._a__", + }, + { name: "a run across an asterisk", input: "_a *_a", expected: "_a *_a*_" }, + { + name: "a run after an open code span", + input: "__`x", + expected: "__`x`__", + }, + { + name: "a run after an open code span ending in a backslash", + input: "__`x\\", + expected: "__`x\\`__", + }, + ])("should close $name in one pass", ({ input, expected }) => { + const healed = remend(input); + expect(healed).toBe(expected); + expect(remend(healed)).toBe(healed); + }); }); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 3d40ecb0..581547b7 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -2,6 +2,7 @@ import { isInsideCodeBlock, isWithinCompleteInlineCode, } from "./code-block-utils"; +import { handleIncompleteInlineCode } from "./inline-code-handler"; import { boldItalicPattern, boldPattern, @@ -290,9 +291,7 @@ const shouldSkipItalicCompletion = ( }; // Completes incomplete italic formatting with double underscores (__) -export const handleIncompleteDoubleUnderscoreItalic = ( - text: string -): string => { +const healDoubleUnderscore = (text: string): string => { const italicMatch = matchTrailing(text, italicPattern); if (!italicMatch) { // Check for half-complete closing marker: __content_ should become __content__ @@ -512,9 +511,7 @@ const handleTrailingAsterisksForUnderscore = (text: string): string | null => { }; // Completes incomplete italic formatting with single underscores (_) -export const handleIncompleteSingleUnderscoreItalic = ( - text: string -): string => { +const healSingleUnderscore = (text: string): string => { const singleUnderscoreMatch = matchTrailing(text, singleUnderscorePattern); if (!singleUnderscoreMatch) { @@ -621,3 +618,36 @@ export const handleIncompleteBoldItalic = (text: string): string => { return text; }; + +// The inline code handler runs after these, so a closer appended while a code +// span is open would land inside the span. Heal against the closed span and +// keep that closing only when the heal adds a closer. +const healAfterOpenCodeSpan = ( + text: string, + heal: (text: string) => string +): string => { + const closed = handleIncompleteInlineCode(text); + if (closed === text) { + return heal(text); + } + const healed = heal(closed); + return healed === closed ? text : healed; +}; + +export const handleIncompleteDoubleUnderscoreItalic = (text: string): string => + healAfterOpenCodeSpan(text, healDoubleUnderscore); + +// Each heal closes only the innermost open run, so healing repeats until no +// opener is left. Every accepted heal closes a literal, so the loop ends. +const healUnderscoreStack = (text: string): string => { + let healed = text; + let next = healSingleUnderscore(healDoubleUnderscore(healed)); + while (next !== healed) { + healed = next; + next = healSingleUnderscore(healDoubleUnderscore(healed)); + } + return healed; +}; + +export const handleIncompleteSingleUnderscoreItalic = (text: string): string => + healAfterOpenCodeSpan(text, healUnderscoreStack); diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 5bf68c9b..15245737 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -489,7 +489,9 @@ export const countDoublePairs = (text: string, char: string): number => { let count = 0; for (let i = text.indexOf(pair); i !== -1; i = text.indexOf(pair, i)) { - if (scan.regions[i] === REGION.PROSE && !isEscaped(text, i)) { + // Inside math a backslash is TeX, so only markdown markers can be escaped + const escaped = char !== "$" && isEscaped(text, i); + if (scan.regions[i] === REGION.PROSE && !escaped) { count += 1; i += 2; } else { diff --git a/packages/remend/src/underscore-delimiters.ts b/packages/remend/src/underscore-delimiters.ts index c24c384d..e4dceab7 100644 --- a/packages/remend/src/underscore-delimiters.ts +++ b/packages/remend/src/underscore-delimiters.ts @@ -20,8 +20,11 @@ export interface OpenUnderscoreRun { interface DelimiterRun extends OpenUnderscoreRun { canClose: boolean; canOpen: boolean; + marker: Marker; } +type Marker = "*" | "_"; + // The start and end of the text count as whitespace for flanking const isWhitespaceOrEdge = (char: string): boolean => char === "" || isWhitespaceChar(char); @@ -38,6 +41,7 @@ const toDelimiterRun = ( end: number, prevChar: string ): DelimiterRun => { + const marker = text[index] as Marker; const nextChar = end < text.length ? text[end] : ""; const leftFlanking = !( isWhitespaceOrEdge(nextChar) || @@ -52,47 +56,59 @@ const toDelimiterRun = ( !isPunctuationChar(nextChar)) ); const length = end - index; + if (marker === "*") { + return { + canClose: rightFlanking, + canOpen: leftFlanking, + count: length, + length, + marker, + }; + } return { canClose: rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)), canOpen: leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)), count: length, length, + marker, }; }; -const isSkippedUnderscore = (scan: TextScan, i: number): boolean => - scan.text[i] !== "_" || +const isSkippedMarker = (scan: TextScan, i: number, marker: Marker): boolean => + scan.text[i] !== marker || scan.regions[i] !== REGION.PROSE || inMathAt(scan, i) || inLinkUrlAt(scan, i) || inHtmlTagAt(scan, i); -const collectDelimiterRuns = (scan: TextScan): DelimiterRun[] => { +const collectDelimiterRuns = ( + scan: TextScan, + markers: readonly Marker[] +): DelimiterRun[] => { const { text } = scan; const runs: DelimiterRun[] = []; - let i = text.indexOf("_"); - while (i !== -1) { - if (isSkippedUnderscore(scan, i)) { - i = text.indexOf("_", i + 1); + for (let i = 0; i < text.length; i += 1) { + const marker = text[i] as Marker; + if (!markers.includes(marker) || isSkippedMarker(scan, i, marker)) { continue; } let end = i + 1; - while (end < text.length && !isSkippedUnderscore(scan, end)) { + while (end < text.length && !isSkippedMarker(scan, end, marker)) { end += 1; } - // A backslash escapes the run's first underscore, which stays literal + // A backslash escapes the run's first marker, which stays literal // punctuation before the rest of the run if (isEscaped(text, i)) { if (end - i > 1) { - runs.push(toDelimiterRun(text, i + 1, end, "_")); + runs.push(toDelimiterRun(text, i + 1, end, marker)); } } else { runs.push(toDelimiterRun(text, i, end, i > 0 ? text[i - 1] : "")); } - i = text.indexOf("_", end); + i = end - 1; } return runs; @@ -109,14 +125,15 @@ const violatesRuleOfThree = ( (opener.count + closer.count) % 3 === 0 && !(opener.count % 3 === 0 && closer.count % 3 === 0); -// Matches a closing run against the stack until it runs out of underscores -// or openers, popping every opener it passes +// Matches a closing run against the stack until it runs out of markers or +// openers, popping every opener it passes const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { while (closer.count > 0) { let openerIndex = stack.length - 1; while ( openerIndex >= 0 && - violatesRuleOfThree(stack[openerIndex], closer) + (stack[openerIndex].marker !== closer.marker || + violatesRuleOfThree(stack[openerIndex], closer)) ) { openerIndex -= 1; } @@ -132,12 +149,13 @@ const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { } }; -// Runs the CommonMark emphasis algorithm over the text's underscore runs, +// Runs the CommonMark emphasis algorithm over the text's delimiter runs, // leaving each run's unmatched count and the openers still open -const matchUnderscoreRuns = ( - text: string +const matchDelimiterRuns = ( + text: string, + markers: readonly Marker[] ): { runs: DelimiterRun[]; stack: DelimiterRun[] } => { - const runs = collectDelimiterRuns(getScan(text)); + const runs = collectDelimiterRuns(getScan(text), markers); const stack: DelimiterRun[] = []; for (const run of runs) { @@ -154,12 +172,15 @@ const matchUnderscoreRuns = ( /** The underscore openers the text leaves unmatched, outermost first */ export const findOpenUnderscoreRuns = (text: string): OpenUnderscoreRun[] => - matchUnderscoreRuns(text).stack; + matchDelimiterRuns(text, ["_"]).stack; // Underscores that render as literal text: openers still open, openers a -// closer passed over, and closers left over +// closer passed over, and closers left over. Asterisk runs take part because +// a matched asterisk pair discards the underscore openers between them. const countLiteralUnderscores = (text: string): number => - matchUnderscoreRuns(text).runs.reduce((sum, run) => sum + run.count, 0); + matchDelimiterRuns(text, ["*", "_"]) + .runs.filter((run) => run.marker === "_") + .reduce((sum, run) => sum + run.count, 0); /** * Whether every underscore that healing added to the text matched one the From ebd0094d0eff26db9badcdaccf318f7569ce03a3 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Oct 2026 08:15:02 -0700 Subject: [PATCH 03/16] fix(remend): heal emphasis with one CommonMark delimiter engine Replace the per-marker emphasis handlers, which decided by parity counts and appended closers in a fixed priority order, with one delimiter engine that matches runs as micromark does. Healing now closes only openers that exist, in nesting order, within the block and link label that holds them. Fold the bold, italic, boldItalic and strikethrough handlers into a single emphasis handler at priority 30, and add a micromark oracle that checks healed output is idempotent and faithful. --- .changeset/remend-escaped-markers.md | 11 +- apps/website/content/docs/termination.mdx | 13 +- packages/remend/README.md | 5 +- packages/remend/__tests__/bold-italic.test.ts | 2 +- .../broken-markdown-variants.test.ts | 51 +- .../remend/__tests__/coverage-gaps.test.ts | 19 - .../__tests__/emphasis-fast-path.test.ts | 75 ++ .../remend/__tests__/emphasis-nesting.test.ts | 140 +++ .../remend/__tests__/emphasis-oracle.test.ts | 213 +++++ .../remend/__tests__/escaped-markers.test.ts | 6 +- packages/remend/__tests__/italic.test.ts | 3 +- packages/remend/__tests__/lists.test.ts | 19 +- .../remend/__tests__/mixed-formatting.test.ts | 11 +- packages/remend/__tests__/streaming.test.ts | 5 +- .../__tests__/trailing-patterns.test.ts | 67 -- .../remend/__tests__/underscore-runs.test.ts | 22 +- packages/remend/package.json | 2 + packages/remend/src/emphasis-delimiters.ts | 550 +++++++++++ packages/remend/src/emphasis-handlers.ts | 885 ++++++------------ packages/remend/src/index.ts | 88 +- packages/remend/src/inline-block.ts | 56 ++ packages/remend/src/inline-code-handler.ts | 18 +- packages/remend/src/patterns.ts | 64 -- packages/remend/src/scan.ts | 4 +- packages/remend/src/strikethrough-handler.ts | 66 -- packages/remend/src/underscore-delimiters.ts | 199 ---- pnpm-lock.yaml | 6 + 27 files changed, 1436 insertions(+), 1164 deletions(-) create mode 100644 packages/remend/__tests__/emphasis-fast-path.test.ts create mode 100644 packages/remend/__tests__/emphasis-nesting.test.ts create mode 100644 packages/remend/__tests__/emphasis-oracle.test.ts delete mode 100644 packages/remend/__tests__/trailing-patterns.test.ts create mode 100644 packages/remend/src/emphasis-delimiters.ts create mode 100644 packages/remend/src/inline-block.ts delete mode 100644 packages/remend/src/strikethrough-handler.ts delete mode 100644 packages/remend/src/underscore-delimiters.ts diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index 9844d3cb..88c713bf 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -1,10 +1,9 @@ --- -"remend": patch +"remend": minor --- -fix(remend): stop appending emphasis closers that match no opener +Heal emphasis the way CommonMark and GFM parse it -- Match `_` and `__` delimiters with the CommonMark emphasis algorithm (flanking, rule of 3, partial matches) instead of counting parity, and drop an underscore closer that would not match -- Treat a marker as escaped only after an odd run of backslashes, so `\\*`, `\\_`, `\\**`, `\\~~`, and `` \\` `` still open -- Escape a trailing backslash before appending closers, so it cannot escape them -- Close every open underscore run in one pass, including runs before an unterminated code span, so healed output re-heals to itself +- Closes only openers that exist, innermost first, within the paragraph, list item or link text that holds them. An open code span closes before the emphasis around it. +- Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. +- The `bold`, `italic`, `boldItalic` and `strikethrough` options still apply, but the built-in handlers they controlled are now one `emphasis` handler at priority 30. Custom handlers ordered against the old priorities may need new ones. diff --git a/apps/website/content/docs/termination.mdx b/apps/website/content/docs/termination.mdx index 6a1327f8..ad68f445 100644 --- a/apps/website/content/docs/termination.mdx +++ b/apps/website/content/docs/termination.mdx @@ -278,27 +278,24 @@ Built-in handlers use priorities 0-75: | `htmlTags` | 10 | | `setextHeadings` | 15 | | `links` | 20 | -| `boldItalic` | 30 | -| `bold` | 35 | -| `italic` | 40-42 | +| `emphasis` (bold, italic, bold italic, strikethrough) | 30 | | `inlineCode` | 50 | -| `strikethrough` | 60 | | `katex` | 70 | Custom handlers default to priority 100, running after all built-ins. Set a lower priority to run before specific built-ins. -**Example: Running Before Bold Handling** +**Example: Running Before Emphasis Handling** -To process content before bold formatting is completed (priority 30), set a lower priority: +To process content before emphasis is completed (priority 30), set a lower priority: ```typescript const preprocessHandler: RemendHandler = { name: 'preprocess', handle: (text) => { - // Transform content before bold is completed + // Transform content before emphasis is completed return text.replace(/\{\{highlight\}\}/g, '**'); }, - priority: 25, // Runs before bold (35) and after links (20) + priority: 25, // Runs before emphasis (30) and after links (20) }; ``` diff --git a/packages/remend/README.md b/packages/remend/README.md index 5b9b44a2..d8f050d3 100644 --- a/packages/remend/README.md +++ b/packages/remend/README.md @@ -131,11 +131,8 @@ Built-in handlers use priorities 0-75. Custom handlers default to 100 (run after | `htmlTags` | 10 | | `setextHeadings` | 15 | | `links` | 20 | -| `boldItalic` | 30 | -| `bold` | 35 | -| `italic` | 40-42 | +| `emphasis` (bold, italic, bold italic, strikethrough) | 30 | | `inlineCode` | 50 | -| `strikethrough` | 60 | | `katex` | 70 | | `inlineKatex` | 75 | | Custom (default) | 100 | diff --git a/packages/remend/__tests__/bold-italic.test.ts b/packages/remend/__tests__/bold-italic.test.ts index b5d62f91..06b868f0 100644 --- a/packages/remend/__tests__/bold-italic.test.ts +++ b/packages/remend/__tests__/bold-italic.test.ts @@ -39,7 +39,7 @@ describe("bold-italic formatting (***)", () => { it("should handle nested formatting with triple asterisks", () => { expect(remend("***bold-italic with `code")).toBe( - "***bold-italic with `code***`" + "***bold-italic with `code`***" ); }); diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index 6fa64cae..f0cb5ff8 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -3,30 +3,24 @@ import remend from "../src"; // 1. Rapid successive formatting switches describe("rapid successive formatting switches", () => { - it("should close italic and strikethrough but not bold when asterisk in content", () => { - // Bold pattern can't match when there's a * in content between ** and end - // Only italic (*) and strikethrough (~~) close + it("should close strikethrough, italic and bold innermost first", () => { const result = remend("**bold then *italic then ~~strike"); - expect(result).toBe("**bold then *italic then ~~strike*~~"); + expect(result).toBe("**bold then *italic then ~~strike~~***"); }); - it("should close italic and strikethrough when bold pattern blocked", () => { - // Bold pattern blocked by * in content; italic and strikethrough close + it("should close italic and bold inside strikethrough", () => { const result = remend("~~strike **bold *italic"); - expect(result).toBe("~~strike **bold *italic*~~"); + expect(result).toBe("~~strike **bold *italic***~~"); }); - it("should close handlers in priority order (bold before strikethrough before code)", () => { - // Bold can't match (asterisk in content), italic appends *, boldItalic appends ***, - // then inline code appends `, then strikethrough appends ~~ + it("should close the code span before the emphasis around it", () => { const result = remend("*italic **bold ~~strike `code"); - expect(result).toBe("*italic **bold ~~strike `code***`~~"); + expect(result).toBe("*italic **bold ~~strike `code`~~***"); }); - it("should close bold before strikethrough (priority order)", () => { - // Bold handler (priority 35) runs before strikethrough (priority 60) + it("should close strikethrough before the bold around it", () => { const result = remend("**bold ~~strike"); - expect(result).toBe("**bold ~~strike**~~"); + expect(result).toBe("**bold ~~strike~~**"); }); it("should close italic then bold via bold-italic handler", () => { @@ -82,7 +76,7 @@ describe("backslash escapes with incomplete formatting", () => { expect(result).toBe("\\\\*actually italic*"); }); - it("should not close escaped double asterisks", () => { + it("should leave the asterisk after an escaped one for the author to close", () => { const result = remend("\\**not bold"); expect(result).toBe("\\**not bold"); }); @@ -317,25 +311,19 @@ describe("formatting at paragraph boundaries", () => { // 13. Deeply nested formatting (4+ levels) describe("deeply nested formatting", () => { - it("should close handlers in priority order with deeply nested formatting", () => { - // Bold can't match (asterisk in content), italic appends *, - // inline code appends `, strikethrough appends ~~ + it("should close deeply nested formatting innermost first", () => { const result = remend("**bold *italic ~~strike `code"); - expect(result).toBe("**bold *italic ~~strike `code*`~~"); + expect(result).toBe("**bold *italic ~~strike `code`~~***"); }); - it("should close bold-italic then code then strikethrough", () => { - // BoldItalic (priority 30) appends ***, then inline code (50) appends `, - // then strikethrough (60) appends ~~ + it("should close code then strikethrough then bold-italic", () => { const result = remend("***bold-italic ~~strike `code"); - expect(result).toBe("***bold-italic ~~strike `code***`~~"); + expect(result).toBe("***bold-italic ~~strike `code`~~***"); }); - it("should close italic but not bold when asterisk blocks bold pattern", () => { - // Bold handler can't match when * appears in content after ** - // Italic handler closes the *, leaving ** unclosed + it("should close italic then the bold around it", () => { const result = remend("**bold and *italic"); - expect(result).toBe("**bold and *italic*"); + expect(result).toBe("**bold and *italic***"); }); }); @@ -439,15 +427,14 @@ describe("back-to-back code blocks", () => { // 19. Confusing asterisk sequences describe("confusing asterisk sequences", () => { - it("should handle four asterisks (bold-italic handler appends ***)", () => { - // BoldItalic sees odd triple-asterisk count and markers not balanced, appends *** + it("should close four asterisks with four", () => { const result = remend("****text"); - expect(result).toBe("****text***"); + expect(result).toBe("****text****"); }); - it("should handle five asterisks (bold-italic handler appends ***)", () => { + it("should close five asterisks with five", () => { const result = remend("*****text"); - expect(result).toBe("*****text***"); + expect(result).toBe("*****text*****"); }); it("should handle mixed asterisk counts", () => { diff --git a/packages/remend/__tests__/coverage-gaps.test.ts b/packages/remend/__tests__/coverage-gaps.test.ts index fa04e73c..1bb88b13 100644 --- a/packages/remend/__tests__/coverage-gaps.test.ts +++ b/packages/remend/__tests__/coverage-gaps.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; import remend from "../src"; -import { countTripleAsterisks } from "../src/emphasis-handlers"; import { isHorizontalRule, isWithinHtmlTag, @@ -36,24 +35,6 @@ describe("bold-italic inside code block", () => { }); }); -describe("countTripleAsterisks", () => { - it("should count trailing *** at end of text", () => { - expect(countTripleAsterisks("text***")).toBe(1); - }); - - it("should skip *** inside code blocks", () => { - expect(countTripleAsterisks("```\n***\n```")).toBe(0); - }); - - it("should count *** outside but not inside code blocks", () => { - expect(countTripleAsterisks("```\n***\n```\n***")).toBe(1); - }); - - it("should flush pending asterisks before code block toggle", () => { - expect(countTripleAsterisks("***```code```")).toBe(1); - }); -}); - describe("single underscore counting with code blocks", () => { it("should skip _ inside fenced code blocks", () => { expect(remend("```\n_code\n```\n_text")).toBe("```\n_code\n```\n_text_"); diff --git a/packages/remend/__tests__/emphasis-fast-path.test.ts b/packages/remend/__tests__/emphasis-fast-path.test.ts new file mode 100644 index 00000000..72eb8696 --- /dev/null +++ b/packages/remend/__tests__/emphasis-fast-path.test.ts @@ -0,0 +1,75 @@ +import fc from "fast-check"; +import { expect, it } from "vitest"; +import { + matchDelimiterRuns, + matchHealed, + prepareHeal, +} from "../src/emphasis-delimiters"; +import { isEscaped } from "../src/scan"; + +const TOKENS = [ + "a", + " ", + ".", + "*", + "**", + "_", + "__", + "~~", + "~", + "`", + "\\", + "\n", + "[", + "](u)", + "ि", + "中", + "：", + "$", + "", + "𠀀", + "\n\n", + "- ", + "> ", +]; + +const WHITESPACE = /\s/; +const LINE_CONTENT = /[^\s*_~>+-]/; + +const summary = (m: ReturnType) => ({ + literal: m.literal, + open: m.open.map((r) => `${r.index}:${r.count}`), + runs: m.runs.map((r) => `${r.index}:${r.length}:${r.count}`), +}); + +it("fast path matches a full rescan", () => { + fc.assert( + fc.property( + fc.array(fc.constantFrom(...TOKENS), { minLength: 1, maxLength: 9 }), + fc.array(fc.constantFrom("*", "_", "~", "**", "~~"), { + minLength: 1, + maxLength: 4, + }), + fc.constantFrom("", " ", "\n", " \n", "\\"), + (tokens, closerParts, tail) => { + const text = tokens.join("") + tail; + let insertAt = text.length; + while (insertAt > 0 && WHITESPACE.test(text[insertAt - 1])) { + insertAt -= 1; + } + if (text[insertAt - 1] === "\\" && !isEscaped(text, insertAt - 1)) { + insertAt -= 1; + } + const lineStart = text.lastIndexOf("\n", insertAt - 1) + 1; + fc.pre(LINE_CONTENT.test(text.slice(lineStart, insertAt))); + const closers = closerParts.join(""); + const healed = text.slice(0, insertAt) + closers + text.slice(insertAt); + const base = prepareHeal(text, insertAt); + expect(summary(matchHealed(base, closers))).toEqual( + summary(matchDelimiterRuns(healed)) + ); + } + ), + { numRuns: Number(process.env.NUM_RUNS ?? 20_000) } + ); +}); diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts new file mode 100644 index 00000000..fe25e0a8 --- /dev/null +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +interface Row { + expected: string; + input: string; + name: string; +} + +describe("emphasis nesting", () => { + it.each([ + { + name: "strikethrough inside bold", + input: "**bold ~~strike", + expected: "**bold ~~strike~~**", + }, + { + name: "italic inside bold inside strikethrough", + input: "~~strike **bold *italic", + expected: "~~strike **bold *italic***~~", + }, + { + name: "code span inside bold", + input: "**bold with `code", + expected: "**bold with `code`**", + }, + { + name: "closed italic after an open bold", + input: "**a_~~*~~*", + expected: "**a_~~*~~***", + }, + ])("closes in nesting order: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { + name: "ordered list items", + input: "1. **a\n2. **b", + expected: "1. **a\n2. **b**", + }, + { + name: "paragraphs", + input: "**Summary\n\nThe *quick", + expected: "**Summary\n\nThe *quick*", + }, + { + name: "blockquotes", + input: "> **a\n\n> **b", + expected: "> **a\n\n> **b**", + }, + { + name: "lazy continuation line", + input: "- **text\nmore text", + expected: "- **text\nmore text**", + }, + ])("pairs within one block: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { + name: "closer before a word", + input: "**Note:**This is important.", + expected: "**Note:**This is important.", + }, + { + name: "closer after fullwidth punctuation", + input: "**注意：**这是重要的。", + expected: "**注意：**这是重要的。", + }, + { + name: "globs", + input: "Matches *.ts and *.tsx files", + expected: "Matches *.ts and *.tsx files", + }, + { + name: "field names", + input: "Fields _id, _rev, _key", + expected: "Fields _id, _rev, _key", + }, + { + name: "word-internal ASCII asterisks", + input: "hello**world", + expected: "hello**world", + }, + { + name: "opener inside a link label", + input: "[*a](u)", + expected: "[*a](u)", + }, + { + name: "strikethrough around a link nested in brackets", + input: "[[](u)~~](u)~~]", + expected: "[[](u)~~](u)~~]", + }, + ])("keeps literal markers: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { + name: "bold after CJK text", + input: "这是**粗体", + expected: "这是**粗体**", + }, + { + name: "bold inside Thai text", + input: "คำ**ที่**ดี", + expected: "คำ**ที่**ดี", + }, + ])("handles Unicode: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { name: "many open italic runs", input: " *x".repeat(200) }, + { + name: "many distinct open runs", + input: [ + ..."123456".split("").map((n) => ` ${"*".repeat(Number(n))}a`), + ..."12345".split("").map((n) => ` ${"_".repeat(Number(n))}b`), + " ~~c", + ].join(""), + }, + { + name: "long prose with an open bold", + input: `${"prose with *emph* and **bold** here. ".repeat(300)}**tail`, + }, + ])("stays fast: $name", ({ input }) => { + remend(input); + const start = performance.now(); + remend(input); + expect(performance.now() - start).toBeLessThan(50); + }); +}); diff --git a/packages/remend/__tests__/emphasis-oracle.test.ts b/packages/remend/__tests__/emphasis-oracle.test.ts new file mode 100644 index 00000000..56b053cd --- /dev/null +++ b/packages/remend/__tests__/emphasis-oracle.test.ts @@ -0,0 +1,213 @@ +import fc from "fast-check"; +import { fromMarkdown } from "mdast-util-from-markdown"; +import { gfmStrikethroughFromMarkdown } from "mdast-util-gfm-strikethrough"; +import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough"; +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +interface Node { + children?: Node[]; + position?: { end: { offset?: number }; start: { offset?: number } }; + type: string; +} + +const parse = (markdown: string): Node => + fromMarkdown(markdown, { + extensions: [gfmStrikethrough()], + mdastExtensions: [gfmStrikethroughFromMarkdown()], + }); + +const span = (node: Node): [number, number] => [ + node.position?.start.offset ?? 0, + node.position?.end.offset ?? 0, +]; + +const INLINE = new Set(["emphasis", "strong", "delete", "inlineCode"]); +const TRANSPARENT = new Set(["link", "image", "text", "break"]); +const MARKER_PATTERN = /[*_~`]/g; +const ASCII_PUNCTUATION = /[!-/:-@[-`{-~]/; +const LEADING_TICKS = /^`+/; +const LEADING_MARKERS = /^[*_~`]+/; +const TRAILING_MARKERS = /[*_~`]+$/; +const CONTENT_PATTERN = /[^\s*_~`]/; + +const literal = (markdown: string): number => { + let count = 0; + const visit = (node: Node) => { + if (node.type === "text") { + const [start, end] = span(node); + count += markdown.slice(start, end).match(MARKER_PATTERN)?.length ?? 0; + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(parse(markdown)); + return count; +}; + +// Inline nodes drop out so a heal can only change structure by changing blocks +const blockShape = (node: Node): string => + INLINE.has(node.type) || TRANSPARENT.has(node.type) + ? "" + : `${node.type}(${(node.children ?? []).map(blockShape).filter(Boolean).join(",")})`; + +const visibleOffsets = (markdown: string, root: Node): Set => { + const seen = new Set(); + const visit = (node: Node) => { + const [start, end] = span(node); + if (node.type === "text") { + for (let i = start; i < end; i++) { + if ( + markdown[i] === "\\" && + i + 1 < end && + ASCII_PUNCTUATION.test(markdown[i + 1]) + ) { + seen.add(markdown[i + 1] === "\\" ? i : i + 1); + i++; + } else { + seen.add(i); + } + } + } else if (node.type === "inlineCode") { + const raw = markdown.slice(start, end); + const ticks = raw.length - raw.replace(LEADING_TICKS, "").length; + for (let i = start + ticks; i < end - ticks; i++) { + seen.add(i); + } + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(root); + return seen; +}; + +const inlineSpans = (root: Node): Set => { + const spans = new Set(); + const visit = (node: Node) => { + if (INLINE.has(node.type)) { + spans.add(`${node.type}:${span(node).join(":")}`); + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(root); + return spans; +}; + +// A heal is faithful when it keeps the block structure, keeps the visible text the same, +// and every span it closes holds authored content +const faithful = (input: string, healed: string): boolean => { + if (healed === input) { + return true; + } + let cut = 0; + while (cut < input.length && input[cut] === healed[cut]) { + cut++; + } + const inputTree = parse(input); + const healedTree = parse(healed); + if (blockShape(healedTree) !== blockShape(inputTree)) { + return false; + } + const shown = visibleOffsets(healed, healedTree); + for (let i = cut; i < cut + healed.length - input.length; i++) { + if (shown.has(i)) { + return false; + } + } + const before = inlineSpans(inputTree); + let ok = true; + const visit = (node: Node) => { + const [start, end] = span(node); + if ( + INLINE.has(node.type) && + (end > cut || !before.has(`${node.type}:${start}:${end}`)) + ) { + const raw = healed.slice(start, end); + const lead = raw.length - raw.replace(LEADING_MARKERS, "").length; + const trail = raw.length - raw.replace(TRAILING_MARKERS, "").length; + if ( + !CONTENT_PATTERN.test( + healed.slice(start + lead, Math.min(end - trail, cut)) + ) + ) { + ok = false; + } + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(healedTree); + return ok; +}; + +const WITHOUT_EMPHASIS = { + bold: false, + boldItalic: false, + italic: false, + strikethrough: false, +}; + +const healsCleanly = (input: string, heal: (text: string) => string) => { + const healed = heal(input); + return heal(healed) === healed && faithful(input, healed); +}; + +const ALPHABETS = [ + { + name: "ASCII", + tokens: ["a", " ", ".", "*", "**", "_", "__", "`", "\\", "~~", "\n"], + }, + { + name: "Unicode", + tokens: ["a", "ि", "：", "中", "𠀀", " ", "*", "**", "_", "~~", "."], + }, + { + name: "links", + tokens: ["a", " ", "*", "**", "_", "~~", "`", "[", "](u)", "]"], + }, +]; + +describe("emphasis oracle", () => { + it.each([ + { name: "plain", input: "a", count: 0 }, + { name: "unmatched opener", input: "*a", count: 1 }, + { name: "matched pair", input: "*a*", count: 0 }, + { name: "code span", input: "`*a`", count: 0 }, + ])("literal counts $name", ({ input, count }) => { + expect(literal(input)).toBe(count); + }); + + it.each([ + { name: "closing a span", input: "*a", healed: "*a*", expected: true }, + { name: "empty span", input: "a *", healed: "a **", expected: false }, + { name: "visible addition", input: "*a", healed: "*a*b", expected: false }, + { name: "block change", input: "a\n*", healed: "a\n***", expected: false }, + ])("faithful rejects or accepts $name", ({ input, healed, expected }) => { + expect(faithful(input, healed)).toBe(expected); + }); + + it.each(ALPHABETS)("heals $name samples idempotently and faithfully", ({ + tokens, + }) => { + fc.assert( + fc.property( + fc.array(fc.constantFrom(...tokens), { minLength: 1, maxLength: 8 }), + (parts) => { + const input = parts.join(""); + // Samples the other handlers already mangle say nothing about emphasis + fc.pre(healsCleanly(input, (text) => remend(text, WITHOUT_EMPHASIS))); + const healed = remend(input); + expect(remend(healed)).toBe(healed); + expect(faithful(input, healed)).toBe(true); + } + ), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }, 120_000); +}); diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts index 4f53aa2a..daa0c0ab 100644 --- a/packages/remend/__tests__/escaped-markers.test.ts +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -63,7 +63,11 @@ describe("trailing backslash", () => { { name: "_", input: "_foo\\", expected: "_foo\\\\_" }, { name: "__", input: "__foo\\", expected: "__foo\\\\__" }, { name: "~~", input: "~~foo\\", expected: "~~foo\\\\~~" }, - { name: "a partial __ match", input: "____.\\", expected: "____.\\\\__" }, + { + name: "a partial __ match", + input: "____.\\", + expected: "____.\\\\____", + }, { name: "a backslash after whitespace", input: "**foo \\", diff --git a/packages/remend/__tests__/italic.test.ts b/packages/remend/__tests__/italic.test.ts index 4ad9807b..60e1c696 100644 --- a/packages/remend/__tests__/italic.test.ts +++ b/packages/remend/__tests__/italic.test.ts @@ -136,7 +136,8 @@ describe("italic formatting with single underscores (_)", () => { it("should handle incomplete single underscore with trailing newlines", () => { expect(remend("Text with _italic\n")).toBe("Text with _italic_\n"); - expect(remend("_incomplete\n\n")).toBe("_incomplete_\n\n"); + // A blank line ends the paragraph, so the opener can no longer close + expect(remend("_incomplete\n\n")).toBe("_incomplete\n\n"); expect(remend("Start _text\n")).toBe("Start _text_\n"); }); }); diff --git a/packages/remend/__tests__/lists.test.ts b/packages/remend/__tests__/lists.test.ts index 64a54667..09573bf5 100644 --- a/packages/remend/__tests__/lists.test.ts +++ b/packages/remend/__tests__/lists.test.ts @@ -22,14 +22,9 @@ describe("list handling", () => { expect(remend(text)).toBe(text); }); - it("should complete incomplete italic even in list items", () => { - // List markers are not counted, but incomplete italic formatting is still completed + it("should not close an opener from an earlier list item", () => { const text = "* Item with *incomplete italic\n* Another item"; - // The function adds an asterisk to complete the italic, though at the end of text - // This is not ideal but matches current behavior - expect(remend(text)).toBe( - "* Item with *incomplete italic\n* Another item*" - ); + expect(remend(text)).toBe(text); }); it("should handle mixed list markers and italic formatting", () => { @@ -72,8 +67,8 @@ describe("list handling", () => { // A __ followed by whitespace cannot open emphasis expect(remend("- __ text after")).toBe("- __ text after"); - // List items with emphasis markers and text should still complete - expect(remend("- ** text after")).toBe("- ** text after**"); + // A ** followed by whitespace cannot open emphasis + expect(remend("- ** text after")).toBe("- ** text after"); // Mixed list items expect(remend("- __\n- Normal item\n- **")).toBe( @@ -88,9 +83,9 @@ describe("list handling", () => { expect(remend("- `")).toBe("- `"); }); - it("should not complete list items with emphasis markers spanning multiple lines", () => { - // When a list item starts with ** followed by content with newline, don't complete - expect(remend("- **text\nmore text")).toBe("- **text\nmore text"); + it("should close emphasis within its own list item only", () => { + // A lazy continuation line stays in the item, but the next item starts a new block + expect(remend("- **text\nmore text")).toBe("- **text\nmore text**"); expect(remend("* **content\n* Another item")).toBe( "* **content\n* Another item" ); diff --git a/packages/remend/__tests__/mixed-formatting.test.ts b/packages/remend/__tests__/mixed-formatting.test.ts index c3cd26b1..72add967 100644 --- a/packages/remend/__tests__/mixed-formatting.test.ts +++ b/packages/remend/__tests__/mixed-formatting.test.ts @@ -8,7 +8,7 @@ describe("mixed formatting", () => { }); it("should complete multiple incomplete formats", () => { - expect(remend("**bold and *italic")).toBe("**bold and *italic*"); + expect(remend("**bold and *italic")).toBe("**bold and *italic***"); }); it("should handle nested formatting", () => { @@ -33,8 +33,7 @@ describe("mixed formatting", () => { }); it("should handle code inside bold", () => { - // Bold gets closed first, then code - expect(remend("**bold with `code")).toBe("**bold with `code**`"); + expect(remend("**bold with `code")).toBe("**bold with `code`**"); }); it("should handle strikethrough with other formatting", () => { @@ -48,11 +47,9 @@ describe("mixed formatting", () => { }); it("should handle deeply nested incomplete formatting", () => { - // Formats are closed in the order they're processed. - // The ~~ isn't closed because it ends up inside the completed inline code span (`code ~~strike`), - // so remend correctly leaves it alone. + // The ~~ is code once the span closes, so it gets no closer expect(remend("**bold *italic `code ~~strike")).toBe( - "**bold *italic `code ~~strike*`" + "**bold *italic `code ~~strike`***" ); }); diff --git a/packages/remend/__tests__/streaming.test.ts b/packages/remend/__tests__/streaming.test.ts index 3409036a..8bb89a4f 100644 --- a/packages/remend/__tests__/streaming.test.ts +++ b/packages/remend/__tests__/streaming.test.ts @@ -4,7 +4,7 @@ import remend from "../src"; describe("chunked streaming scenarios", () => { it("should handle nested formatting cut mid-stream", () => { expect(remend("This is **bold with *ital")).toBe( - "This is **bold with *ital*" + "This is **bold with *ital***" ); // When underscore opens after bold, underscore should close before bold (proper nesting) expect(remend("**bold _und")).toBe("**bold _und_**"); @@ -34,8 +34,7 @@ describe("chunked streaming scenarios", () => { }); it("should handle multiple incomplete formats in one chunk", () => { - // Formats are closed in order they're processed - expect(remend("Text **bold `code")).toBe("Text **bold `code**`"); + expect(remend("Text **bold `code")).toBe("Text **bold `code`**"); }); }); diff --git a/packages/remend/__tests__/trailing-patterns.test.ts b/packages/remend/__tests__/trailing-patterns.test.ts deleted file mode 100644 index 7a64e606..00000000 --- a/packages/remend/__tests__/trailing-patterns.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import fc from "fast-check"; -import { describe, expect, it } from "vitest"; -import { - boldItalicPattern, - boldPattern, - halfCompleteTildePattern, - halfCompleteUnderscorePattern, - italicPattern, - matchTrailing, - singleAsteriskPattern, - singleUnderscorePattern, - strikethroughPattern, - type TrailingPattern, -} from "../src/patterns"; - -const patterns: Record = { - boldItalicPattern, - boldPattern, - halfCompleteTildePattern, - halfCompleteUnderscorePattern, - italicPattern, - singleAsteriskPattern, - singleUnderscorePattern, - strikethroughPattern, -}; - -const markdownish = fc - .array(fc.constantFrom("*", "_", "~", "a", " ", "\n", "\\", "`"), { - maxLength: 40, - }) - .map((chars) => chars.join("")); - -const groups = (match: RegExpMatchArray | null) => - match ? [match[0], match[1], match[2]] : null; - -describe("matchTrailing", () => { - it.each( - Object.entries(patterns) - )("%s matches the tail as the full text does", (_, pattern) => { - fc.assert( - fc.property(markdownish, (text) => { - expect(groups(matchTrailing(text, pattern))).toEqual( - groups(text.match(pattern.regex)) - ); - }), - { numRuns: 5000 } - ); - }); - - it.each<{ name: string; text: string; expected: string[] | null }>([ - { - name: "bold with a half closer", - text: "a **b*", - expected: ["**b*", "**", "b*"], - }, - { - name: "the closing marker of a finished run", - text: "**a** b", - expected: ["** b", "**", " b"], - }, - { name: "no marker", text: "a b", expected: null }, - { name: "marker at the end", text: "a **", expected: ["**", "**", ""] }, - { name: "empty", text: "", expected: null }, - ])("$name", ({ text, expected }) => { - expect(groups(matchTrailing(text, boldPattern))).toEqual(expected); - }); -}); diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 2cfb02a6..3ac2f3c5 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -51,8 +51,8 @@ describe("underscore run lengths", () => { }); describe("escaped underscores", () => { - it("should treat the run after an escaped underscore as a delimiter", () => { - expect(remend("\\___bold")).toBe("\\___bold__"); + it("should leave the run after an escaped underscore for the author to close", () => { + expect(remend("\\___bold")).toBe("\\___bold"); }); it("should not treat a run after an escaped backslash as escaped", () => { @@ -63,13 +63,13 @@ describe("escaped underscores", () => { describe("underscore delimiter matching", () => { it.each<[string, string]>([ ["a run followed by whitespace", "__ foo"], - ["a spaced run before an escaped run", "a __ b \\__c"], - ["a spaced run before an escaped run at the start", "__ \\__alph"], + ["two open runs of one marker and length", "_a *_a"], + ["the run left after an escaped underscore", "a __ b \\__c"], + ["the run left after an escaped underscore at the start", "__ \\__alph"], + ["the run left after an escaped underscore and a backslash", "\\_____\\a"], ["a run after a letter and before punctuation", "a__."], ["a closer-only run with nothing open", "__ a__ a"], ["a longer run that closes the opener", "__a____.a"], - ["an opener the rule of 3 keeps a __ closer from matching", "\\_____\\a"], - ["a leftover single underscore after an escape", "__\\__ a"], ])("should not append a closer for %s", (_name, input) => { expect(remend(input)).toBe(input); }); @@ -80,11 +80,15 @@ describe("underscore delimiter matching", () => { it.each<{ name: string; input: string; expected: string }>([ { - name: "an outer run after an escape", + name: "an inner run after an escaped one", input: "\\__._a", - expected: "\\__._a__", + expected: "\\__._a_", + }, + { + name: "the underscore an escaped run leaves open", + input: "__\\__ a", + expected: "__\\__ a_", }, - { name: "a run across an asterisk", input: "_a *_a", expected: "_a *_a*_" }, { name: "a run after an open code span", input: "__`x", diff --git a/packages/remend/package.json b/packages/remend/package.json index 5980db26..9c81aed7 100644 --- a/packages/remend/package.json +++ b/packages/remend/package.json @@ -36,6 +36,8 @@ "@vitest/coverage-v8": "^4.1.10", "fast-check": "^4.9.0", "mdast-util-from-markdown": "^2.0.3", + "mdast-util-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.1.0", "tsup": "^8.5.1", "vitest": "^5.0.0" } diff --git a/packages/remend/src/emphasis-delimiters.ts b/packages/remend/src/emphasis-delimiters.ts new file mode 100644 index 00000000..12061c8e --- /dev/null +++ b/packages/remend/src/emphasis-delimiters.ts @@ -0,0 +1,550 @@ +import { findInlineBlockStart } from "./inline-block"; +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isEscaped, + isFenceAt, + REGION, + type TextScan, +} from "./scan"; +import { isWhitespaceChar } from "./utils"; + +export type EmphasisMarker = "*" | "_" | "~"; + +const MARKERS: readonly string[] = ["*", "_", "~"]; + +export interface MarkerRun { + /** Whether the last opener this run closed is still partly open */ + closesPartially: boolean; + /** Markers still unmatched */ + count: number; + /** Index of the run's first unescaped marker */ + index: number; + /** Unescaped length of the run before any were matched */ + length: number; + marker: EmphasisMarker; +} + +interface DelimiterRun extends MarkerRun { + canClose: boolean; + canOpen: boolean; + nextChar: string; + pairsClosed: number; + pairsOpened: number; + prevChar: string; + /** Start of the link label holding the run, or -1 outside any label */ + scope: number; +} + +interface DelimiterPair { + closer: MarkerRun; + opener: MarkerRun; + used: number; +} + +export interface DelimiterMatch { + /** Markers that render as literal text */ + literal: number; + /** Runs still open outside link labels, innermost first */ + open: MarkerRun[]; + pairs: DelimiterPair[]; + runs: MarkerRun[]; +} + +// Flanking compares whole characters, so a surrogate pair counts as one +const charBefore = (text: string, i: number): string => { + if (i <= 0) { + return ""; + } + const code = text.charCodeAt(i - 1); + return code >= 0xdc_00 && code <= 0xdf_ff && i >= 2 + ? text.slice(i - 2, i) + : text[i - 1]; +}; + +const charAt = (text: string, i: number): string => + i < text.length ? String.fromCodePoint(text.codePointAt(i) ?? 0) : ""; + +// The start and end of the text count as whitespace for flanking +const isWhitespaceOrEdge = (char: string): boolean => + char === "" || isWhitespaceChar(char); + +const punctuationPattern = /[\p{P}\p{S}]/u; + +const isPunctuationChar = (char: string): boolean => + char !== "" && punctuationPattern.test(char); + +// Flanking per CommonMark, with the stricter underscore rules: a run that +// flanks on both sides opens only after punctuation and closes only before +// punctuation, so word-internal runs (snake__case) can't open or close. +// Tildes follow the asterisk rules, and a tilde run longer than 2 is inert. +const applyFlanking = (run: DelimiterRun): void => { + const { marker, nextChar, prevChar } = run; + // micromark treats every attention marker (* _ ~) as flanking punctuation, + // so an asterisk or underscore run beside one can open or close that side + const besideMarker = (char: string): boolean => + marker !== "~" && MARKERS.includes(char); + const leftFlanking = + besideMarker(nextChar) || + !( + isWhitespaceOrEdge(nextChar) || + (isPunctuationChar(nextChar) && + !isWhitespaceOrEdge(prevChar) && + !isPunctuationChar(prevChar)) + ); + const rightFlanking = + besideMarker(prevChar) || + !( + isWhitespaceOrEdge(prevChar) || + (isPunctuationChar(prevChar) && + !isWhitespaceOrEdge(nextChar) && + !isPunctuationChar(nextChar)) + ); + if (marker === "_") { + run.canClose = + rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)); + run.canOpen = + leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)); + return; + } + const inert = marker === "~" && run.length > 2; + run.canClose = rightFlanking && !inert; + run.canOpen = leftFlanking && !inert; +}; + +const createRun = ( + marker: EmphasisMarker, + index: number, + length: number, + prevChar: string, + nextChar: string, + scope: number +): DelimiterRun => { + const run = { + canClose: false, + canOpen: false, + closesPartially: false, + count: length, + index, + length, + marker, + nextChar, + pairsClosed: 0, + pairsOpened: 0, + prevChar, + scope, + }; + applyFlanking(run); + return run; +}; + +const findOpenInlineMath = (scan: TextScan): number => { + const { text } = scan; + let start = text.length; + while (start > 0 && inMathAt(scan, start - 1)) { + start -= 1; + } + const isInlineDollar = + start < text.length && text[start - 1] === "$" && text[start] !== "$"; + return isInlineDollar ? start : text.length; +}; + +// An unclosed code span or inline math span renders as literal text, so its +// markers still count +const isProseAt = (scan: TextScan, i: number, openMath: number): boolean => + (scan.regions[i] === REGION.PROSE || + scan.regions[i] === REGION.CODE_SPAN_OPEN) && + !(i < openMath && inMathAt(scan, i)) && + !inLinkUrlAt(scan, i) && + !inHtmlTagAt(scan, i); + +interface LinkLabel { + end: number; + start: number; +} + +// Emphasis inside a completed link's text resolves on its own, so the label +// walls its runs off from the rest of the block +const findLinkLabels = ( + scan: TextScan, + from: number, + openMath: number +): LinkLabel[] => { + const { text } = scan; + const labels: LinkLabel[] = []; + const openers: { active: boolean; start: number }[] = []; + for (let i = from; i < text.length; i += 1) { + const char = text[i]; + if ( + (char !== "[" && char !== "]") || + isEscaped(text, i) || + !isProseAt(scan, i, openMath) + ) { + continue; + } + if (char === "[") { + openers.push({ active: true, start: i }); + continue; + } + const opener = openers.pop(); + if (opener?.active && text[i + 1] === "(" && inLinkUrlAt(scan, i + 2)) { + labels.push({ end: i, start: opener.start }); + // Links cannot contain links, so a link deactivates every bracket that could enclose it + if (text[opener.start - 1] !== "!" || isEscaped(text, opener.start - 1)) { + for (const outer of openers) { + outer.active = false; + } + } + } + } + return labels; +}; + +const scopeOf = (labels: readonly LinkLabel[], index: number): number => { + let scope = -1; + for (const label of labels) { + if (label.start < index && index < label.end && label.start > scope) { + scope = label.start; + } + } + return scope; +}; + +const collectDelimiterRuns = (scan: TextScan, from: number): DelimiterRun[] => { + const { text } = scan; + const runs: DelimiterRun[] = []; + const openMath = findOpenInlineMath(scan); + const labels = findLinkLabels(scan, from, openMath); + const isRunMarker = (i: number, marker: string): boolean => + text[i] === marker && isProseAt(scan, i, openMath); + + for (let i = from; i < text.length; i += 1) { + const marker = text[i]; + if (!(MARKERS.includes(marker) && isRunMarker(i, marker))) { + continue; + } + + let end = i + 1; + while (end < text.length && isRunMarker(end, marker)) { + end += 1; + } + + // A backslash escapes the run's first marker, which stays literal + // punctuation before the rest of the run + const escaped = isEscaped(text, i); + const start = escaped ? i + 1 : i; + if (end > start) { + runs.push( + createRun( + marker as EmphasisMarker, + start, + end - start, + escaped ? marker : charBefore(text, i), + charAt(text, end), + scopeOf(labels, start) + ) + ); + } + i = end - 1; + } + + return runs; +}; + +// The "rule of 3": when either run can both open and close, they match only +// if their combined length is not a multiple of 3, unless both lengths are. +// Lengths are what remains of each run after earlier partial matches. +const violatesRuleOfThree = ( + opener: DelimiterRun, + closer: DelimiterRun +): boolean => + (closer.canOpen || opener.canClose) && + (opener.count + closer.count) % 3 === 0 && + !(opener.count % 3 === 0 && closer.count % 3 === 0); + +const canMatch = (opener: DelimiterRun, closer: DelimiterRun): boolean => { + if (opener.marker !== closer.marker) { + return false; + } + if (closer.marker === "~") { + return opener.count === closer.count; + } + return !violatesRuleOfThree(opener, closer); +}; + +const matchCloser = ( + stack: DelimiterRun[], + floor: number, + closer: DelimiterRun, + pairs: DelimiterPair[] +): void => { + while (closer.count > 0) { + let openerIndex = stack.length - 1; + while (openerIndex >= floor && !canMatch(stack[openerIndex], closer)) { + openerIndex -= 1; + } + if (openerIndex < floor) { + return; + } + + const opener = stack[openerIndex]; + let used = 1; + if (closer.marker === "~") { + used = closer.count; + } else if (opener.count >= 2 && closer.count >= 2) { + used = 2; + } + opener.count -= used; + closer.count -= used; + opener.pairsOpened += 1; + closer.pairsClosed += 1; + closer.closesPartially = opener.count > 0; + pairs.push({ closer, opener, used }); + stack.length = opener.count > 0 ? openerIndex + 1 : openerIndex; + } +}; + +// Pairs an earlier pass matched wall off the runs between them, so a run +// inside a pair can't match one outside it. +const matchPass = ( + runs: DelimiterRun[], + inPass: (run: DelimiterRun) => boolean, + pairs: DelimiterPair[] +): DelimiterRun[] => { + const stack: DelimiterRun[] = []; + const floors: number[] = []; + + for (const run of runs) { + if (!inPass(run)) { + for (let i = 0; i < run.pairsClosed; i += 1) { + stack.length = floors.pop() ?? 0; + } + for (let i = 0; i < run.pairsOpened; i += 1) { + floors.push(stack.length); + } + continue; + } + if (run.canClose) { + matchCloser(stack, floors.at(-1) ?? 0, run, pairs); + } + if (run.canOpen && run.count > 0) { + stack.push(run); + } + } + + return stack; +}; + +const isTilde = (run: DelimiterRun): boolean => run.marker === "~"; +const isAttention = (run: DelimiterRun): boolean => run.marker !== "~"; + +const matchScope = ( + runs: DelimiterRun[], + pairs: DelimiterPair[] +): DelimiterRun[] => { + // An inert tilde run never reaches the strikethrough resolver + const first = runs.find((run) => !isTilde(run) || run.length <= 2); + const passes = + first && isTilde(first) ? [isTilde, isAttention] : [isAttention, isTilde]; + return passes.flatMap((inPass) => matchPass(runs, inPass, pairs)); +}; + +const matchRuns = (runs: DelimiterRun[]): DelimiterMatch => { + const scopes = new Map(); + for (const run of runs) { + const scope = scopes.get(run.scope); + if (scope) { + scope.push(run); + } else { + scopes.set(run.scope, [run]); + } + } + const pairs: DelimiterPair[] = []; + let open: DelimiterRun[] = []; + for (const [scope, scoped] of scopes) { + const left = matchScope(scoped, pairs); + if (scope === -1) { + open = left; + } + } + + return { + literal: runs.reduce((sum, run) => sum + run.count, 0), + open: open.sort((a, b) => b.index - a.index), + pairs, + runs, + }; +}; + +/** + * Matches the delimiter runs of the text's last inline block as micromark + * does, per CommonMark and GFM strikethrough + */ +export const matchDelimiterRuns = (text: string): DelimiterMatch => { + const scan = getScan(text); + return matchRuns(collectDelimiterRuns(scan, findInlineBlockStart(scan))); +}; + +/** The authored text's runs, reused to match each candidate heal */ +export interface HealBase { + insertAt: number; + runs: readonly DelimiterRun[]; + /** Whether closers would land where runs aren't collected */ + slow: boolean; + text: string; +} + +export const prepareHeal = (text: string, insertAt: number): HealBase => { + const scan = getScan(text); + const from = findInlineBlockStart(scan); + return { + insertAt, + runs: collectDelimiterRuns(scan, from), + // Past a trailing blank line the closers would start a block of their + // own, and inside a tag, math or a fence their markers aren't collected + slow: + from > insertAt || + (insertAt > 0 && + (inHtmlTagAt(scan, insertAt - 1) || + inMathAt(scan, insertAt - 1) || + isFenceAt(scan, insertAt - 1))), + text, + }; +}; + +const insertClosers = (base: HealBase, closers: string): string => + base.text.slice(0, base.insertAt) + closers + base.text.slice(base.insertAt); + +// Only the runs beside the insertion point change, so just those are rebuilt +export const matchHealed = ( + base: HealBase, + closers: string +): DelimiterMatch => { + if (base.slow) { + return matchDelimiterRuns(insertClosers(base, closers)); + } + const { insertAt, text } = base; + const runs = base.runs.map((run) => ({ + ...run, + closesPartially: false, + count: run.length, + pairsClosed: 0, + pairsOpened: 0, + })); + const after = charAt(text, insertAt); + const charAfter = (i: number): string => + i < closers.length ? closers[i] : after; + + let start = 0; + const last = runs.at(-1); + if (last && last.scope === -1 && last.index + last.length === insertAt) { + while (closers[start] === last.marker) { + start += 1; + } + last.length += start; + last.count = last.length; + last.nextChar = charAfter(start); + applyFlanking(last); + } + + while (start < closers.length) { + let end = start + 1; + while (closers[end] === closers[start]) { + end += 1; + } + runs.push( + createRun( + closers[start] as EmphasisMarker, + insertAt + start, + end - start, + start === 0 ? charBefore(text, insertAt) : closers[start - 1], + charAfter(end), + -1 + ) + ); + start = end; + } + + return matchRuns(runs); +}; + +const kindOf = (pair: DelimiterPair): string => + pair.opener.marker === "~" ? "~" : pair.opener.marker + pair.used; + +const parentOf = ( + pairs: readonly DelimiterPair[], + pair: DelimiterPair +): DelimiterPair | undefined => { + let parent: DelimiterPair | undefined; + for (const other of pairs) { + if ( + other.opener.index < pair.opener.index && + other.closer.index >= pair.closer.index && + (!parent || + other.opener.index > parent.opener.index || + (other.opener.index === parent.opener.index && + other.closer.index < parent.closer.index)) + ) { + parent = other; + } + } + return parent; +}; + +// Authors don't nest emphasis directly inside the same emphasis, so a heal +// that needs that is closing runs written as literal text +const nestsSameKind = (match: DelimiterMatch, insertAt: number): boolean => { + const healed = (pair: DelimiterPair): boolean => + pair.closer.index + pair.closer.length > insertAt; + return match.pairs.some((pair) => { + const parent = parentOf(match.pairs, pair); + return ( + parent !== undefined && + kindOf(parent) === kindOf(pair) && + (healed(pair) || healed(parent)) + ); + }); +}; + +const lineContentPattern = /[^\s*_~>+-]/; + +// Closers on a line with nothing else would turn it into a thematic break or +// a fence +const leavesLineBare = (base: HealBase): boolean => { + const lineStart = base.text.lastIndexOf("\n", base.insertAt - 1) + 1; + return !lineContentPattern.test(base.text.slice(lineStart, base.insertAt)); +}; + +/** + * The healed text's literal marker count, or undefined when healing left a + * target opener open, left a marker it added literal, changed how many + * markers any other run leaves unmatched, nested emphasis directly inside + * the same emphasis, or closed fewer literal markers than it added + */ +export const literalAfterHeal = ( + before: DelimiterMatch, + literalBefore: number, + base: HealBase, + closers: string, + targets: ReadonlySet +): number | undefined => { + if (leavesLineBare(base)) { + return undefined; + } + const after = matchHealed(base, closers); + if (after.literal > literalBefore - closers.length) { + return undefined; + } + const countBefore = new Map(before.runs.map((run) => [run.index, run.count])); + const matchesOnlyTargets = after.runs.every((run) => { + if (targets.has(run.index) || run.index + run.length > base.insertAt) { + return run.count === 0; + } + return run.count === (countBefore.get(run.index) ?? 0); + }); + return matchesOnlyTargets && !nestsSameKind(after, base.insertAt) + ? after.literal + : undefined; +}; diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 581547b7..413eca91 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -1,653 +1,328 @@ import { - isInsideCodeBlock, - isWithinCompleteInlineCode, -} from "./code-block-utils"; + type DelimiterMatch, + type HealBase, + literalAfterHeal, + type MarkerRun, + matchDelimiterRuns, + matchHealed, + prepareHeal, +} from "./emphasis-delimiters"; import { handleIncompleteInlineCode } from "./inline-code-handler"; -import { - boldItalicPattern, - boldPattern, - fourOrMoreAsterisksPattern, - halfCompleteUnderscorePattern, - italicPattern, - listItemPattern, - matchTrailing, - singleAsteriskPattern, - singleUnderscorePattern, - whitespaceOrMarkersPattern, -} from "./patterns"; -import { - countDoublePairs, - getScan, - inLinkUrlAt, - inMathAt, - isEscaped, - REGION, - type TextScan, -} from "./scan"; -import { - closesAddedUnderscores, - findOpenUnderscoreRuns, - type OpenUnderscoreRun, -} from "./underscore-delimiters"; -import { isHorizontalRule, isWhitespaceChar, isWordChar } from "./utils"; - -// Helper function to check if an asterisk should be skipped -const shouldSkipAsterisk = ( - scan: TextScan, - index: number, - prevChar: string, - nextChar: string -): boolean => { - // Skip if escaped with backslash - if (isEscaped(scan.text, index)) { - return true; - } - - // Skip if within math block - if (inMathAt(scan, index)) { - return true; - } - - // Special handling for *** sequences - // If this is the first * in ***, don't skip it - it can close a single * italic - // Example: **bold and *italic*** should count the first * of *** as closing the italic - if (prevChar !== "*" && nextChar === "*") { - const nextNextChar = - index < scan.text.length - 2 ? scan.text[index + 2] : ""; - if (nextNextChar === "*") { - // This is the first * in a *** sequence - // Count it as a single asterisk for matching purposes - return false; - } - // This is the first * in ** (not ***) - return true; - } - - // Skip if this is the second or third * in a sequence - if (prevChar === "*") { - return true; - } - - // Skip if flanked by whitespace on both sides (not a valid emphasis delimiter per CommonMark) - // This also catches list markers (e.g., "* item") since they have whitespace on both sides - const prevIsWhitespace = - !prevChar || prevChar === " " || prevChar === "\t" || prevChar === "\n"; - const nextIsWhitespace = - !nextChar || nextChar === " " || nextChar === "\t" || nextChar === "\n"; - if (prevIsWhitespace && nextIsWhitespace) { - return true; - } - - return false; -}; - -const isWordInternalAsterisk = (prevChar: string, nextChar: string): boolean => - Boolean(prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)); - -// Decide whether a candidate single-asterisk delimiter should participate in the -// open/close parity count used by incomplete-italic completion. -// -// Intraword asterisks only count while resolving an active emphasis chain: -// they can close an open run and may reopen within the same word after a close. -// Cold (unmatched) intraword asterisks stay literal (hello*world / #189). -// Trailing right-flanking-only markers after a closed run also stay literal -// (*foo*bar* / a *b*c*). -const shouldCountSingleAsterisk = ( - prevChar: string, - nextChar: string, - count: number, - inWordAsteriskChain: boolean -): { count: true; inWordAsteriskChain: boolean } | { count: false } => { - const isWordInternal = isWordInternalAsterisk(prevChar, nextChar); - const canOpen = Boolean(nextChar) && !isWhitespaceChar(nextChar); - const canClose = Boolean(prevChar) && !isWhitespaceChar(prevChar); - - // Do not start emphasis from a cold word-internal asterisk. - if (isWordInternal && count % 2 === 0 && !inWordAsteriskChain) { - return { count: false }; - } - - // Prefer closing an active run; only open when left-flanking. - if ((canClose && count % 2 === 1) || canOpen) { - return { count: true, inWordAsteriskChain: isWordInternal }; - } - - return { count: false }; -}; - -export const countSingleAsterisks = (text: string): number => { - const scan = getScan(text); - let count = 0; - let inWordAsteriskChain = false; - const len = text.length; - - for (let index = 0; index < len; index += 1) { - // Code regions neither contribute delimiters nor belong to a word chain - if (text[index] !== "*" || scan.regions[index] !== REGION.PROSE) { - if (!isWordChar(text[index])) { - inWordAsteriskChain = false; - } - continue; - } - - const prevChar = index > 0 ? text[index - 1] : ""; - const nextChar = index < len - 1 ? text[index + 1] : ""; - - if (shouldSkipAsterisk(scan, index, prevChar, nextChar)) { - continue; - } - - const decision = shouldCountSingleAsterisk( - prevChar, - nextChar, - count, - inWordAsteriskChain - ); - if (decision.count) { - count += 1; - inWordAsteriskChain = decision.inWordAsteriskChain; - } - } - - return count; +import { whitespaceOrMarkersPattern } from "./patterns"; +import { getScan, isEscaped } from "./scan"; +import { isWhitespaceChar } from "./utils"; + +export interface EmphasisFlags { + bold: boolean; + boldItalic: boolean; + inlineCode: boolean; + italic: boolean; + strikethrough: boolean; +} + +const flagFor = (run: MarkerRun): keyof EmphasisFlags => { + if (run.marker === "~") { + return "strikethrough"; + } + if (run.count >= 3) { + return "boldItalic"; + } + return run.count === 2 && run.marker === "*" ? "bold" : "italic"; }; -// Counts triple asterisks that are not part of quadruple or more asterisks -// and not inside code regions -export const countTripleAsterisks = (text: string): number => { - const scan = getScan(text); - let count = 0; - let consecutiveAsterisks = 0; - - for (let i = 0; i < text.length; i += 1) { - if (text[i] === "*" && scan.regions[i] === REGION.PROSE) { - consecutiveAsterisks += 1; - } else { - // End of asterisk sequence - if (consecutiveAsterisks >= 3) { - count += Math.floor(consecutiveAsterisks / 3); - } - consecutiveAsterisks = 0; - } - } - - // Handle trailing asterisks - if (consecutiveAsterisks >= 3) { - count += Math.floor(consecutiveAsterisks / 3); +// Closers go before trailing whitespace, which would stop them from closing, +// and before a trailing backslash, which would escape them +const findInsertionPoint = (text: string): number => { + let end = text.length; + while (end > 0 && isWhitespaceChar(text[end - 1])) { + end -= 1; } - - return count; + return text[end - 1] === "\\" && !isEscaped(text, end - 1) ? end - 1 : end; }; -const countDoubleAsterisks = (text: string): number => - countDoublePairs(text, "*"); - -const isHalfClosedDoubleUnderscore = ( - run: OpenUnderscoreRun | undefined -): boolean => run !== undefined && run.count === 1 && run.length >= 2; - -// Healing that adds an underscore closer which cannot match would render it -// as literal text, so such a heal is dropped -const healUnderscores = (text: string, healed: string): string => - closesAddedUnderscores(text, healed) ? healed : text; +const insertAtPoint = (text: string, at: number, closers: string): string => + closers === "" ? text : text.slice(0, at) + closers + text.slice(at); -const innermostOpenUnderscoreRun = ( - text: string -): OpenUnderscoreRun | undefined => findOpenUnderscoreRuns(text).at(-1); - -// Helper to check if bold marker should not be completed -const shouldSkipBoldCompletion = ( +const hasContentBefore = ( text: string, - contentAfterMarker: string, - markerIndex: number + run: MarkerRun, + insertAt: number ): boolean => { - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return true; - } - - // Check if this is in a list item with multiline content - const beforeMarker = text.substring(0, markerIndex); - const lastNewlineBeforeMarker = beforeMarker.lastIndexOf("\n"); - const lineStart = - lastNewlineBeforeMarker === -1 ? 0 : lastNewlineBeforeMarker + 1; - const lineBeforeMarker = text.substring(lineStart, markerIndex); - - if (listItemPattern.test(lineBeforeMarker)) { - const hasNewlineInContent = contentAfterMarker.includes("\n"); - if (hasNewlineInContent) { - return true; - } - } - - return isHorizontalRule(text, markerIndex, "*"); + const content = text.slice(run.index + run.length, insertAt); + return content !== "" && !whitespaceOrMarkersPattern.test(content); }; -// Completes incomplete bold formatting (**) -export const handleIncompleteBold = (text: string): string => { - const boldMatch = matchTrailing(text, boldPattern); - if (!boldMatch) { - return text; - } - - const contentAfterMarker = boldMatch[2]; - const markerIndex = text.lastIndexOf(boldMatch[1]); - - // Check if the bold marker is within a code block (fenced or inline) - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } +const asciiLetterOrDigitPattern = /^[A-Za-z0-9]$/; - if (shouldSkipBoldCompletion(text, contentAfterMarker, markerIndex)) { - return text; - } +const isLetterOrDigit = (char: string): boolean => + asciiLetterOrDigitPattern.test(char); - const asteriskPairs = countDoubleAsterisks(text); - if (asteriskPairs % 2 === 1) { - // Check for half-complete closing marker: **content* should become **content** - // The trailing * is the first char of the closing ** being streamed - if (contentAfterMarker.endsWith("*")) { - return `${text}*`; - } - return `${text}**`; - } - - return text; -}; - -// Helper to check if italic marker should not be completed -const shouldSkipItalicCompletion = ( +// An asterisk inside an ASCII word opens only to continue a chain of emphasis +// in that word, so hello*world and 2**3 stay literal while languages written +// without spaces still heal +const opensColdInWord = ( text: string, - contentAfterMarker: string, - markerIndex: number -): boolean => { - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return true; - } - - // Check if this is in a list item with multiline content - const beforeMarker = text.substring(0, markerIndex); - const lastNewlineBeforeMarker = beforeMarker.lastIndexOf("\n"); - const lineStart = - lastNewlineBeforeMarker === -1 ? 0 : lastNewlineBeforeMarker + 1; - const lineBeforeMarker = text.substring(lineStart, markerIndex); - - if (listItemPattern.test(lineBeforeMarker)) { - const hasNewlineInContent = contentAfterMarker.includes("\n"); - if (hasNewlineInContent) { - return true; - } - } + run: MarkerRun, + previous: MarkerRun | undefined +): boolean => + run.marker === "*" && + isLetterOrDigit(text[run.index - 1] ?? "") && + isLetterOrDigit(text[run.index + run.length] ?? "") && + !( + previous?.marker === "*" && + [...text.slice(previous.index + previous.length, run.index)].every( + isLetterOrDigit + ) + ); - return isHorizontalRule(text, markerIndex, "_"); -}; +// Writing \** usually means a literal ** with a sloppy escape, so the rest of +// the run only closes if the author closes it +const followsEscapedMarker = (text: string, run: MarkerRun): boolean => + text[run.index - 1] === run.marker && isEscaped(text, run.index - 1); -// Completes incomplete italic formatting with double underscores (__) -const healDoubleUnderscore = (text: string): string => { - const italicMatch = matchTrailing(text, italicPattern); - if (!italicMatch) { - // Check for half-complete closing marker: __content_ should become __content__ - // The pattern /(__)([^_]*?)$/ won't match __content_ because it ends with _ - // So we need a separate check for this case - const halfCompleteMatch = matchTrailing( - text, - halfCompleteUnderscorePattern +const isGated = ( + text: string, + run: MarkerRun, + insertAt: number, + previous: MarkerRun | undefined +): boolean => + !hasContentBefore(text, run, insertAt) || + // Prose uses a single tilde for "approximately" + (run.marker === "~" && run.count === 1) || + opensColdInWord(text, run, previous) || + followsEscapedMarker(text, run); + +interface HealContext { + base: HealBase; + flags: EmphasisFlags; + insertAt: number; + literal: number; + match: DelimiterMatch; + text: string; + /** The authored run the first closer would extend */ + trailing: MarkerRun | undefined; +} + +// More open runs than nesting plausibly reaches means the markers are +// literal text, like a list of globs or footnote marks +const MAX_OPEN_RUNS = 4; + +interface Heal { + closers: string; + literal: number; + targets: number[]; +} + +// Closers for a group of open runs, innermost first, where up to `reusable` +// markers of the first closer come from a literal run already in the text +const closerCandidates = (group: MarkerRun[], reusable: number): string[] => { + const [first] = group; + const own = + first.marker.repeat(first.count - reusable) + + group + .slice(1) + .map((run) => run.marker.repeat(run.count)) + .join(""); + // Closers for openers of one marker merge into a single run, which the rule + // of 3 may let close them all only at another length + if ([...own].every((char) => char === first.marker)) { + return Array.from({ length: own.length + reusable + 2 }, (_, i) => + first.marker.repeat(i + 1) ); - if (halfCompleteMatch) { - const markerIndex = text.lastIndexOf(halfCompleteMatch[1]); - if ( - !( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) && - isHalfClosedDoubleUnderscore(innermostOpenUnderscoreRun(text)) - ) { - return healUnderscores(text, `${text}_`); - } - } - return text; - } - - const contentAfterMarker = italicMatch[2]; - const markerIndex = text.lastIndexOf(italicMatch[1]); - - // Check if the italic marker is within a code block (fenced or inline) - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } - - if (shouldSkipItalicCompletion(text, contentAfterMarker, markerIndex)) { - return text; } - - const openRun = innermostOpenUnderscoreRun(text); - if (openRun && openRun.count >= 2) { - return healUnderscores(text, `${text}__`); - } - - return text; -}; - -// Skips code regions when locating the asterisk. -// A lone, unescaped asterisk in prose outside math: the only positions that -// can open incomplete italic -const isLoneProseAsterisk = (scan: TextScan, i: number): boolean => { - const { text } = scan; - return ( - text[i] === "*" && - scan.regions[i] === REGION.PROSE && - text[i - 1] !== "*" && - text[i + 1] !== "*" && - !isEscaped(text, i) && - !inMathAt(scan, i) + return Array.from( + { length: reusable + 1 }, + (_, i) => first.marker.repeat(i) + own ); }; -const findFirstSingleAsteriskIndex = (text: string): number => { - const scan = getScan(text); - - for (let i = text.indexOf("*"); i !== -1; i = text.indexOf("*", i + 1)) { - if (!isLoneProseAsterisk(scan, i)) { - continue; - } - const prevChar = i > 0 ? text[i - 1] : ""; - const nextChar = i < text.length - 1 ? text[i + 1] : ""; - - // Skip if flanked by whitespace on both sides (not a valid emphasis delimiter) - const prevIsWs = !prevChar || isWhitespaceChar(prevChar); - const nextIsWs = !nextChar || isWhitespaceChar(nextChar); - if (prevIsWs && nextIsWs) { +// A closer's flanking can depend on the closers after it, so each step tries +// every remaining closer at once before trying it alone +const extendHeal = ( + context: HealContext, + start: Heal, + open: readonly MarkerRun[] +): Heal => { + const { flags, insertAt, text, trailing } = context; + const runs: MarkerRun[] = []; + for (const run of open) { + const previous = context.match.runs.findLast( + (other) => other.index < run.index + ); + if (isGated(text, run, insertAt, previous)) { continue; } - - // Skip cold word-internal asterisks; they are not openers for completion. - // (Active-run closers are still counted in countSingleAsterisks.) - if (prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)) { - continue; + if (flags[flagFor(run)]) { + runs.push(run); } - - // A right-flanking-only marker cannot open incomplete italic. - if (nextIsWs) { - continue; + if (runs.length > MAX_OPEN_RUNS) { + return start; } - - return i; - } - return -1; -}; - -// Completes incomplete italic formatting with single asterisks (*) -export const handleIncompleteSingleAsteriskItalic = (text: string): string => { - const singleAsteriskMatch = matchTrailing(text, singleAsteriskPattern); - - if (!singleAsteriskMatch) { - return text; - } - - const firstSingleAsteriskIndex = findFirstSingleAsteriskIndex(text); - - if (firstSingleAsteriskIndex === -1) { - return text; - } - - // Don't close if the marker is inside a complete inline code span or fenced code block - if ( - isInsideCodeBlock(text, firstSingleAsteriskIndex) || - isWithinCompleteInlineCode(text, firstSingleAsteriskIndex) - ) { - return text; - } - - // Get content after the first single asterisk - const contentAfterFirstAsterisk = text.substring( - firstSingleAsteriskIndex + 1 - ); - - // Check if there's meaningful content after the asterisk - // Don't close if content is only whitespace or emphasis markers - if ( - !contentAfterFirstAsterisk || - whitespaceOrMarkersPattern.test(contentAfterFirstAsterisk) - ) { - return text; - } - - const singleAsterisks = countSingleAsterisks(text); - if (singleAsterisks % 2 === 1) { - return `${text}*`; } - - return text; -}; - -const findFirstSingleUnderscoreIndex = (text: string): number => { - const scan = getScan(text); - - for (let i = text.indexOf("_"); i !== -1; i = text.indexOf("_", i + 1)) { - if ( - scan.regions[i] === REGION.PROSE && - text[i - 1] !== "_" && - text[i + 1] !== "_" && - !isEscaped(text, i) && - !inMathAt(scan, i) && - !inLinkUrlAt(scan, i) - ) { - // Check if underscore is word-internal (between word characters) - const prevChar = i > 0 ? text[i - 1] : ""; - const nextChar = i < text.length - 1 ? text[i + 1] : ""; - if ( - prevChar && - nextChar && - isWordChar(prevChar) && - isWordChar(nextChar) - ) { - continue; + // Two open runs of one marker and length are literal text, like globs or a + // closer that failed flanking in **Note:**text + const kinds = new Set(runs.map((run) => run.marker + run.count)); + if (kinds.size < runs.length) { + return start; + } + + // Closing one run can strand another behind the rule of 3 or nest it + // inside the same emphasis, so the greedy pass runs over every subset + const closeAll = (chosen: MarkerRun[]): Heal => { + let heal = start; + const tryGroup = (group: MarkerRun[]): boolean => { + const [first] = group; + const targets = [...heal.targets, ...group.map((run) => run.index)]; + const targetSet = new Set(targets); + // A literal run the first closer would extend already supplies markers + const reusable = + heal.closers === "" && trailing?.marker === first.marker + ? Math.min(trailing.count, first.count - 1) + : 0; + let found: Heal | undefined; + for (const candidate of closerCandidates(group, reusable)) { + const closers = heal.closers + candidate; + const literal = literalAfterHeal( + context.match, + context.literal, + context.base, + closers, + targetSet + ); + if (literal !== undefined && (!found || literal < found.literal)) { + found = { closers, literal, targets }; + } + } + if (found) { + heal = found; } + return found !== undefined; + }; - return i; + for (let i = 0; i < chosen.length; i += 1) { + if (i < chosen.length - 1 && tryGroup(chosen.slice(i))) { + break; + } + tryGroup([chosen[i]]); + } + return heal; + }; + + // Largest subsets first, so a tie keeps the heal that closes more runs + const subsets: MarkerRun[][] = [[]]; + for (const run of runs) { + subsets.push(...subsets.map((set) => [...set, run])); + } + subsets.reverse(); + let best = start; + for (const subset of subsets) { + if (subset.length > 0) { + const heal = closeAll(subset); + if (heal.literal < best.literal) { + best = heal; + } } } - return -1; -}; - -// Helper function to insert closing underscore, handling trailing newlines -const insertClosingUnderscore = (text: string): string => { - // If text ends with newline(s), insert underscore before them - // Use string methods instead of regex to avoid ReDoS vulnerability - let endIndex = text.length; - while (endIndex > 0 && text[endIndex - 1] === "\n") { - endIndex -= 1; - } - if (endIndex < text.length) { - const textBeforeNewlines = text.slice(0, endIndex); - const trailingNewlines = text.slice(endIndex); - return `${textBeforeNewlines}_${trailingNewlines}`; - } - return `${text}_`; -}; - -// Helper to handle trailing ** for proper nesting of _ and ** markers -const handleTrailingAsterisksForUnderscore = (text: string): string | null => { - if (!text.endsWith("**")) { - return null; - } - - const textWithoutTrailingAsterisks = text.slice(0, -2); - const asteriskPairsAfterRemoval = countDoubleAsterisks( - textWithoutTrailingAsterisks - ); - - // If removing trailing ** makes the count odd, it was added to close an unclosed ** - if (asteriskPairsAfterRemoval % 2 !== 1) { - return null; - } - - const firstDoubleAsteriskIndex = textWithoutTrailingAsterisks.indexOf("**"); - const underscoreIndex = findFirstSingleUnderscoreIndex( - textWithoutTrailingAsterisks - ); - - // If ** opened before _, then _ should close before ** - if ( - firstDoubleAsteriskIndex !== -1 && - underscoreIndex !== -1 && - firstDoubleAsteriskIndex < underscoreIndex - ) { - return `${textWithoutTrailingAsterisks}_**`; - } - - return null; + return best; }; -// Completes incomplete italic formatting with single underscores (_) -const healSingleUnderscore = (text: string): string => { - const singleUnderscoreMatch = matchTrailing(text, singleUnderscorePattern); - - if (!singleUnderscoreMatch) { - return text; - } - - const firstSingleUnderscoreIndex = findFirstSingleUnderscoreIndex(text); - - if (firstSingleUnderscoreIndex === -1) { - return text; - } - - // Get content after the first single underscore - const contentAfterFirstUnderscore = text.substring( - firstSingleUnderscoreIndex + 1 +// Closers can change how authored runs flank and leave one open, so healing +// repeats until a round adds nothing +const healFrom = ( + context: HealContext, + open: readonly MarkerRun[], + targets: number[] = [] +): Heal => { + let heal = extendHeal( + context, + { closers: "", literal: context.literal, targets }, + open ); - - // Check if there's meaningful content after the underscore - // Don't close if content is only whitespace or emphasis markers - if ( - !contentAfterFirstUnderscore || - whitespaceOrMarkersPattern.test(contentAfterFirstUnderscore) - ) { - return text; - } - - if ( - isInsideCodeBlock(text, firstSingleUnderscoreIndex) || - isWithinCompleteInlineCode(text, firstSingleUnderscoreIndex) - ) { - return text; - } - - if (innermostOpenUnderscoreRun(text)?.count === 1) { - // Check if we need to insert _ before trailing ** for proper nesting - const healed = - handleTrailingAsterisksForUnderscore(text) ?? - insertClosingUnderscore(text); - return healUnderscores(text, healed); - } - - return text; -}; - -// Helper to check if bold-italic markers are already balanced -const areBoldItalicMarkersBalanced = (text: string): boolean => { - const asteriskPairs = countDoubleAsterisks(text); - const singleAsterisks = countSingleAsterisks(text); - return asteriskPairs % 2 === 0 && singleAsterisks % 2 === 0; -}; - -// Helper to check if bold-italic should be skipped -const shouldSkipBoldItalicCompletion = ( - text: string, - contentAfterMarker: string, - markerIndex: number -): boolean => { - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return true; - } - - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return true; + while (heal.closers !== "") { + const reopened = matchHealed(context.base, heal.closers).open.filter( + (run) => run.index < context.insertAt + ); + const next = extendHeal(context, heal, reopened); + if (next.closers === heal.closers) { + break; + } + heal = next; } - - return isHorizontalRule(text, markerIndex, "*"); + return heal; }; -// Completes incomplete bold-italic formatting (***) -export const handleIncompleteBoldItalic = (text: string): string => { - // Don't process if text is only asterisks and has 4 or more consecutive asterisks - if (fourOrMoreAsterisksPattern.test(text)) { - return text; - } - - const boldItalicMatch = matchTrailing(text, boldItalicPattern); - - if (!boldItalicMatch) { - return text; - } - - const contentAfterMarker = boldItalicMatch[2]; - const markerIndex = text.lastIndexOf(boldItalicMatch[1]); - - if (shouldSkipBoldItalicCompletion(text, contentAfterMarker, markerIndex)) { - return text; - } - - const tripleAsteriskCount = countTripleAsterisks(text); - if (tripleAsteriskCount % 2 === 1) { - // If both ** and * are balanced, don't add closing *** - // The *** is likely overlapping markers (e.g., **bold and *italic***) - if (areBoldItalicMarkersBalanced(text)) { - return text; +const healEmphasis = (text: string, flags: EmphasisFlags): string => { + const insertAt = findInsertionPoint(text); + const match = matchDelimiterRuns(text); + const last = match.runs.at(-1); + const trailing = + last && last.index + last.length === insertAt ? last : undefined; + const context = { + base: prepareHeal(text, insertAt), + flags, + insertAt, + literal: match.literal, + match, + text, + trailing, + }; + let heal = healFrom(context, match.open); + + // A trailing run that closes something may instead be literal text, with + // the runs open before it still needing closers + if (trailing && trailing.count < trailing.length) { + const prefix = matchDelimiterRuns(text.slice(0, trailing.index)); + const asText = { ...trailing, count: trailing.length }; + const streaming = trailing.count > 0 || trailing.closesPartially; + const alternative = healFrom( + { + ...context, + literal: prefix.literal + asText.count, + match: { ...prefix, runs: [...prefix.runs, asText] }, + trailing: asText, + }, + prefix.open, + streaming ? [] : [trailing.index] + ); + // Reading the trailing run as text must not repair spans the author + // already closed by re-pairing their runs + const targets = new Set(alternative.targets); + const healed = matchHealed(context.base, alternative.closers); + const countBefore = new Map( + match.runs.map((run) => [run.index, run.count]) + ); + const keepsAuthoredRuns = healed.runs.every( + (run) => + run.index >= trailing.index || + targets.has(run.index) || + run.count === countBefore.get(run.index) + ); + if (alternative.literal < heal.literal && keepsAuthoredRuns) { + heal = alternative; } - return `${text}***`; } - return text; + return insertAtPoint(text, insertAt, heal.closers); }; -// The inline code handler runs after these, so a closer appended while a code -// span is open would land inside the span. Heal against the closed span and -// keep that closing only when the heal adds a closer. -const healAfterOpenCodeSpan = ( +export const handleIncompleteEmphasis = ( text: string, - heal: (text: string) => string + flags: EmphasisFlags ): string => { - const closed = handleIncompleteInlineCode(text); - if (closed === text) { - return heal(text); + if (!(flags.inlineCode && getScan(text).openSpan)) { + return healEmphasis(text, flags); } - const healed = heal(closed); - return healed === closed ? text : healed; -}; - -export const handleIncompleteDoubleUnderscoreItalic = (text: string): string => - healAfterOpenCodeSpan(text, healDoubleUnderscore); -// Each heal closes only the innermost open run, so healing repeats until no -// opener is left. Every accepted heal closes a literal, so the loop ends. -const healUnderscoreStack = (text: string): string => { - let healed = text; - let next = healSingleUnderscore(healDoubleUnderscore(healed)); - while (next !== healed) { - healed = next; - next = healSingleUnderscore(healDoubleUnderscore(healed)); + // Emphasis closers must land after the code span closes, so close it first + // and keep that only when emphasis needed healing too + const closed = handleIncompleteInlineCode(text); + if (closed !== text) { + const healed = healEmphasis(closed, flags); + return healed === closed ? text : healed; } - return healed; -}; -export const handleIncompleteSingleUnderscoreItalic = (text: string): string => - healAfterOpenCodeSpan(text, healUnderscoreStack); + // A closer that breaks up a trailing backtick run would let the span close + // after it, which would swallow the closer + const healed = healEmphasis(text, flags); + return handleIncompleteInlineCode(healed) === healed ? healed : text; +}; diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 6a39f53d..36486265 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -1,10 +1,7 @@ import { handleComparisonOperators } from "./comparison-operator-handler"; import { - handleIncompleteBold, - handleIncompleteBoldItalic, - handleIncompleteDoubleUnderscoreItalic, - handleIncompleteSingleAsteriskItalic, - handleIncompleteSingleUnderscoreItalic, + type EmphasisFlags, + handleIncompleteEmphasis, } from "./emphasis-handlers"; import { handleIncompleteHtmlTag } from "./html-tag-handler"; import { handleIncompleteInlineCode } from "./inline-code-handler"; @@ -20,7 +17,6 @@ import { import { clearScanCache, getScan, inMathAt, isEscaped, REGION } from "./scan"; import { handleIncompleteSetextHeading } from "./setext-heading-handler"; import { handleSingleTildeEscape } from "./single-tilde-handler"; -import { handleIncompleteStrikethrough } from "./strikethrough-handler"; // biome-ignore lint/performance/noBarrelFile: "Re-exports utility functions for public API convenience" export { @@ -104,13 +100,8 @@ const PRIORITY = { HTML_TAGS: 10, SETEXT_HEADINGS: 15, LINKS: 20, - BOLD_ITALIC: 30, - BOLD: 35, - ITALIC_DOUBLE_UNDERSCORE: 40, - ITALIC_SINGLE_ASTERISK: 41, - ITALIC_SINGLE_UNDERSCORE: 42, + EMPHASIS: 30, INLINE_CODE: 50, - STRIKETHROUGH: 60, KATEX: 70, INLINE_KATEX: 75, DEFAULT: 100, @@ -167,41 +158,9 @@ const builtInHandlers: Array<{ }, { handler: { - name: "boldItalic", - handle: handleIncompleteBoldItalic, - priority: PRIORITY.BOLD_ITALIC, - }, - optionKey: "boldItalic", - }, - { - handler: { - name: "bold", - handle: handleIncompleteBold, - priority: PRIORITY.BOLD, - }, - optionKey: "bold", - }, - { - handler: { - name: "italicDoubleUnderscore", - handle: handleIncompleteDoubleUnderscoreItalic, - priority: PRIORITY.ITALIC_DOUBLE_UNDERSCORE, - }, - optionKey: "italic", - }, - { - handler: { - name: "italicSingleAsterisk", - handle: handleIncompleteSingleAsteriskItalic, - priority: PRIORITY.ITALIC_SINGLE_ASTERISK, - }, - optionKey: "italic", - }, - { - handler: { - name: "italicSingleUnderscore", - handle: handleIncompleteSingleUnderscoreItalic, - priority: PRIORITY.ITALIC_SINGLE_UNDERSCORE, + name: "emphasis", + handle: (text: string) => text, + priority: PRIORITY.EMPHASIS, }, optionKey: "italic", }, @@ -213,14 +172,6 @@ const builtInHandlers: Array<{ }, optionKey: "inlineCode", }, - { - handler: { - name: "strikethrough", - handle: handleIncompleteStrikethrough, - priority: PRIORITY.STRIKETHROUGH, - }, - optionKey: "strikethrough", - }, { handler: { name: "katex", @@ -247,6 +198,13 @@ const getEnabledBuiltInHandlers = ( earlyReturn?: (result: string) => boolean; }> => { const linkMode: LinkMode = options?.linkMode ?? "protocol"; + const emphasisFlags: EmphasisFlags = { + bold: isEnabled(options?.bold), + boldItalic: isEnabled(options?.boldItalic), + inlineCode: isEnabled(options?.inlineCode), + italic: isEnabled(options?.italic), + strikethrough: isEnabled(options?.strikethrough), + }; return builtInHandlers .filter(({ handler, optionKey }) => { @@ -254,6 +212,15 @@ const getEnabledBuiltInHandlers = ( if (handler.name === "links") { return isEnabled(options?.links) || isEnabled(options?.images); } + // Special case: emphasis is enabled by any of its options + if (handler.name === "emphasis") { + return ( + emphasisFlags.bold || + emphasisFlags.boldItalic || + emphasisFlags.italic || + emphasisFlags.strikethrough + ); + } // Special case: inlineKatex is opt-in (defaults to false, unlike other options) if (handler.name === "inlineKatex") { return isOptedIn(options?.inlineKatex); @@ -273,6 +240,17 @@ const getEnabledBuiltInHandlers = ( earlyReturn: linkMode === "protocol" ? earlyReturn : undefined, }; } + // Special case: wrap emphasis handler to pass its options + if (handler.name === "emphasis") { + return { + handler: { + ...handler, + handle: (text: string) => + handleIncompleteEmphasis(text, emphasisFlags), + }, + earlyReturn, + }; + } return { handler, earlyReturn }; }); }; diff --git a/packages/remend/src/inline-block.ts b/packages/remend/src/inline-block.ts new file mode 100644 index 00000000..c93cd2dc --- /dev/null +++ b/packages/remend/src/inline-block.ts @@ -0,0 +1,56 @@ +import { isFenceAt, skipContainerPrefix, type TextScan } from "./scan"; + +const headingPattern = /^#{1,6}(?:[ \t]|$)/; +const thematicBreakPattern = + /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +const setextUnderlinePattern = /^(?:=+|-+)[ \t]*$/; + +/** + * Index where the text's last inline block starts, since emphasis never + * pairs across paragraphs, list items, headings, table rows or code blocks + */ +export const findInlineBlockStart = (scan: TextScan): number => { + const { text } = scan; + let start = 0; + let quoteDepth = 0; + let endsBlock = false; + + for (let lineStart = 0; lineStart <= text.length; ) { + let lineEnd = text.indexOf("\n", lineStart); + if (lineEnd === -1) { + lineEnd = text.length; + } + const next = lineEnd + 1; + const prefix = skipContainerPrefix(text, lineStart, lineEnd); + const content = text.slice(prefix.contentStart, lineEnd).trim(); + + // The unterminated last line is where the paragraph keeps streaming + if (lineEnd === text.length && content === "") { + break; + } + if ( + content === "" || + isFenceAt(scan, prefix.contentStart) || + // A list marker walk would read the first star of * * * as an item + thematicBreakPattern.test(text.slice(lineStart, lineEnd).trim()) || + thematicBreakPattern.test(content) || + (setextUnderlinePattern.test(content) && !endsBlock) + ) { + start = next; + endsBlock = false; + } else if ( + endsBlock || + prefix.listIndent > 0 || + prefix.quoteDepth > quoteDepth || + headingPattern.test(content) || + content.startsWith("|") + ) { + start = prefix.contentStart; + endsBlock = headingPattern.test(content) || content.startsWith("|"); + } + quoteDepth = prefix.quoteDepth; + lineStart = next; + } + + return Math.min(start, text.length); +}; diff --git a/packages/remend/src/inline-code-handler.ts b/packages/remend/src/inline-code-handler.ts index ed079a79..67c0dc9a 100644 --- a/packages/remend/src/inline-code-handler.ts +++ b/packages/remend/src/inline-code-handler.ts @@ -1,5 +1,6 @@ import { whitespaceOrMarkersPattern } from "./patterns"; import { getScan } from "./scan"; +import { isWhitespaceChar } from "./utils"; // Completes an unclosed inline code span (`) // @@ -21,15 +22,22 @@ export const handleIncompleteInlineCode = (text: string): string => { return text; } + // The closing run goes before trailing whitespace, since one after a + // trailing newline would start a new line and could open a fence + let end = text.length; + while (end > 0 && isWhitespaceChar(text[end - 1])) { + end -= 1; + } + // Don't close if there's no meaningful content after the opening run - const content = text.slice(span.start + span.runLength); + const content = text.slice(span.start + span.runLength, end); if (!content || whitespaceOrMarkersPattern.test(content)) { return text; } // A trailing backtick run is the closing run being streamed let trailingRun = 0; - let i = text.length - 1; + let i = end - 1; while (i >= 0 && text[i] === "`") { trailingRun += 1; i -= 1; @@ -41,5 +49,9 @@ export const handleIncompleteInlineCode = (text: string): string => { return text; } - return text + "`".repeat(span.runLength - trailingRun); + return ( + text.slice(0, end) + + "`".repeat(span.runLength - trailingRun) + + text.slice(end) + ); }; diff --git a/packages/remend/src/patterns.ts b/packages/remend/src/patterns.ts index 307eb87e..1317359f 100644 --- a/packages/remend/src/patterns.ts +++ b/packages/remend/src/patterns.ts @@ -1,66 +1,2 @@ -/** - * An end-anchored pattern whose delimiter may follow the opening marker only - * as the text's final character. - */ -export interface TrailingPattern { - delimiter: string; - markerLength: number; - regex: RegExp; -} - -// A match's marker ends at or after the last delimiter before the final -// character, or at the final character when there is none, so matching from -// there finds the same match without scanning the rest of the document. -export const matchTrailing = ( - text: string, - { delimiter, markerLength, regex }: TrailingPattern -): RegExpMatchArray | null => { - const last = text.lastIndexOf(delimiter, text.length - 2); - const markerEnd = last === -1 ? text.length - 1 : last; - return text.slice(Math.max(0, markerEnd - markerLength + 1)).match(regex); -}; - -export const boldPattern: TrailingPattern = { - delimiter: "*", - markerLength: 2, - regex: /(\*\*)([^*]*\*?)$/, -}; -export const italicPattern: TrailingPattern = { - delimiter: "_", - markerLength: 2, - regex: /(__)([^_]*?)$/, -}; -export const boldItalicPattern: TrailingPattern = { - delimiter: "*", - markerLength: 3, - regex: /(\*\*\*)([^*]*?)$/, -}; -export const singleAsteriskPattern: TrailingPattern = { - delimiter: "*", - markerLength: 1, - regex: /(\*)([^*]*?)$/, -}; -export const singleUnderscorePattern: TrailingPattern = { - delimiter: "_", - markerLength: 1, - regex: /(_)([^_]*?)$/, -}; -export const strikethroughPattern: TrailingPattern = { - delimiter: "~", - markerLength: 2, - regex: /(~~)([^~]*?)$/, -}; -export const halfCompleteUnderscorePattern: TrailingPattern = { - delimiter: "_", - markerLength: 2, - regex: /(__)([^_]+)_$/, -}; -export const halfCompleteTildePattern: TrailingPattern = { - delimiter: "~", - markerLength: 2, - regex: /(~~)([^~]+)~$/, -}; export const whitespaceOrMarkersPattern = /^[\s_~*`]*$/; -export const listItemPattern = /^[\s]*[-*+][\s]+$/; export const letterNumberUnderscorePattern = /[\p{L}\p{N}_]/u; -export const fourOrMoreAsterisksPattern = /^\*{4,}$/; diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 15245737..96ea1a86 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -98,7 +98,7 @@ const skipListMarker = (text: string, start: number): number => { return text[i] === " " ? i + 1 : -1; }; -interface ContainerPrefix { +export interface ContainerPrefix { /** Index of the first character after the markers and indentation */ contentStart: number; listIndent: number; @@ -108,7 +108,7 @@ interface ContainerPrefix { // Walks the block quote and list markers that open a line. A single forward // pass: a regex with nested repetition over these markers backtracks // exponentially on a line of repeated list markers. -const skipContainerPrefix = ( +export const skipContainerPrefix = ( text: string, lineStart: number, lineEnd: number diff --git a/packages/remend/src/strikethrough-handler.ts b/packages/remend/src/strikethrough-handler.ts deleted file mode 100644 index e981f83c..00000000 --- a/packages/remend/src/strikethrough-handler.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { - isInsideCodeBlock, - isWithinCompleteInlineCode, -} from "./code-block-utils"; -import { - halfCompleteTildePattern, - matchTrailing, - strikethroughPattern, - whitespaceOrMarkersPattern, -} from "./patterns"; -import { countDoublePairs } from "./scan"; - -// Tilde runs that open or close a fence are painted as fence regions by the -// scanner, so a line-start ~~~ fence never counts as strikethrough while a -// mid-line tilde run still does. -const countDoubleTildes = (text: string): number => countDoublePairs(text, "~"); - -// Completes incomplete strikethrough formatting (~~) -export const handleIncompleteStrikethrough = (text: string): string => { - const strikethroughMatch = matchTrailing(text, strikethroughPattern); - - if (strikethroughMatch) { - // Don't close if there's no meaningful content after the opening markers - // strikethroughMatch[2] contains the content after ~~ - // Check if content is only whitespace or other emphasis markers - const contentAfterMarker = strikethroughMatch[2]; - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return text; - } - - // Don't close if the marker is inside an inline code span or fenced code block - const markerIndex = text.lastIndexOf(strikethroughMatch[1]); - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } - - if (countDoubleTildes(text) % 2 === 1) { - return `${text}~~`; - } - } else { - // Check for half-complete closing marker: ~~content~ should become ~~content~~ - // The pattern /(~~)([^~]*?)$/ won't match ~~content~ because it ends with ~ - const halfCompleteMatch = matchTrailing(text, halfCompleteTildePattern); - if (halfCompleteMatch) { - // Don't close if the marker is inside an inline code span or fenced code block - const markerIndex = text.lastIndexOf(halfCompleteMatch[0].slice(0, 2)); - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } - if (countDoubleTildes(text) % 2 === 1) { - return `${text}~`; - } - } - } - - return text; -}; diff --git a/packages/remend/src/underscore-delimiters.ts b/packages/remend/src/underscore-delimiters.ts deleted file mode 100644 index e4dceab7..00000000 --- a/packages/remend/src/underscore-delimiters.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { - getScan, - inHtmlTagAt, - inLinkUrlAt, - inMathAt, - isEscaped, - REGION, - type TextScan, -} from "./scan"; -import { isWhitespaceChar, isWordChar } from "./utils"; - -/** An underscore delimiter run left open at the end of the text */ -export interface OpenUnderscoreRun { - /** Underscores still unmatched */ - count: number; - /** Unescaped length of the run before any were matched */ - length: number; -} - -interface DelimiterRun extends OpenUnderscoreRun { - canClose: boolean; - canOpen: boolean; - marker: Marker; -} - -type Marker = "*" | "_"; - -// The start and end of the text count as whitespace for flanking -const isWhitespaceOrEdge = (char: string): boolean => - char === "" || isWhitespaceChar(char); - -const isPunctuationChar = (char: string): boolean => - char === "_" || !(isWhitespaceOrEdge(char) || isWordChar(char)); - -// Flanking per CommonMark, with the stricter underscore rules: a run that -// flanks on both sides opens only after punctuation and closes only before -// punctuation, so word-internal runs (snake__case) can't open or close -const toDelimiterRun = ( - text: string, - index: number, - end: number, - prevChar: string -): DelimiterRun => { - const marker = text[index] as Marker; - const nextChar = end < text.length ? text[end] : ""; - const leftFlanking = !( - isWhitespaceOrEdge(nextChar) || - (isPunctuationChar(nextChar) && - !isWhitespaceOrEdge(prevChar) && - !isPunctuationChar(prevChar)) - ); - const rightFlanking = !( - isWhitespaceOrEdge(prevChar) || - (isPunctuationChar(prevChar) && - !isWhitespaceOrEdge(nextChar) && - !isPunctuationChar(nextChar)) - ); - const length = end - index; - if (marker === "*") { - return { - canClose: rightFlanking, - canOpen: leftFlanking, - count: length, - length, - marker, - }; - } - return { - canClose: rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)), - canOpen: leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)), - count: length, - length, - marker, - }; -}; - -const isSkippedMarker = (scan: TextScan, i: number, marker: Marker): boolean => - scan.text[i] !== marker || - scan.regions[i] !== REGION.PROSE || - inMathAt(scan, i) || - inLinkUrlAt(scan, i) || - inHtmlTagAt(scan, i); - -const collectDelimiterRuns = ( - scan: TextScan, - markers: readonly Marker[] -): DelimiterRun[] => { - const { text } = scan; - const runs: DelimiterRun[] = []; - - for (let i = 0; i < text.length; i += 1) { - const marker = text[i] as Marker; - if (!markers.includes(marker) || isSkippedMarker(scan, i, marker)) { - continue; - } - - let end = i + 1; - while (end < text.length && !isSkippedMarker(scan, end, marker)) { - end += 1; - } - - // A backslash escapes the run's first marker, which stays literal - // punctuation before the rest of the run - if (isEscaped(text, i)) { - if (end - i > 1) { - runs.push(toDelimiterRun(text, i + 1, end, marker)); - } - } else { - runs.push(toDelimiterRun(text, i, end, i > 0 ? text[i - 1] : "")); - } - i = end - 1; - } - - return runs; -}; - -// The "rule of 3": when either run can both open and close, they match only -// if their combined length is not a multiple of 3, unless both lengths are. -// Lengths are what remains of each run after earlier partial matches. -const violatesRuleOfThree = ( - opener: DelimiterRun, - closer: DelimiterRun -): boolean => - (closer.canOpen || opener.canClose) && - (opener.count + closer.count) % 3 === 0 && - !(opener.count % 3 === 0 && closer.count % 3 === 0); - -// Matches a closing run against the stack until it runs out of markers or -// openers, popping every opener it passes -const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { - while (closer.count > 0) { - let openerIndex = stack.length - 1; - while ( - openerIndex >= 0 && - (stack[openerIndex].marker !== closer.marker || - violatesRuleOfThree(stack[openerIndex], closer)) - ) { - openerIndex -= 1; - } - if (openerIndex < 0) { - return; - } - - const opener = stack[openerIndex]; - const used = opener.count >= 2 && closer.count >= 2 ? 2 : 1; - opener.count -= used; - closer.count -= used; - stack.length = opener.count > 0 ? openerIndex + 1 : openerIndex; - } -}; - -// Runs the CommonMark emphasis algorithm over the text's delimiter runs, -// leaving each run's unmatched count and the openers still open -const matchDelimiterRuns = ( - text: string, - markers: readonly Marker[] -): { runs: DelimiterRun[]; stack: DelimiterRun[] } => { - const runs = collectDelimiterRuns(getScan(text), markers); - const stack: DelimiterRun[] = []; - - for (const run of runs) { - if (run.canClose) { - matchCloser(stack, run); - } - if (run.canOpen && run.count > 0) { - stack.push(run); - } - } - - return { runs, stack }; -}; - -/** The underscore openers the text leaves unmatched, outermost first */ -export const findOpenUnderscoreRuns = (text: string): OpenUnderscoreRun[] => - matchDelimiterRuns(text, ["_"]).stack; - -// Underscores that render as literal text: openers still open, openers a -// closer passed over, and closers left over. Asterisk runs take part because -// a matched asterisk pair discards the underscore openers between them. -const countLiteralUnderscores = (text: string): number => - matchDelimiterRuns(text, ["*", "_"]) - .runs.filter((run) => run.marker === "_") - .reduce((sum, run) => sum + run.count, 0); - -/** - * Whether every underscore that healing added to the text matched one the - * text left literal. An added closer can fail to match, for example under - * the rule of 3. - */ -export const closesAddedUnderscores = ( - text: string, - healed: string -): boolean => { - // Scan the text first, while the scan cache still holds it - const literal = countLiteralUnderscores(text); - return ( - countLiteralUnderscores(healed) <= literal - (healed.length - text.length) - ); -}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7cf04b91..d975afbc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -289,6 +289,12 @@ importers: mdast-util-from-markdown: specifier: ^2.0.3 version: 2.0.3 + mdast-util-gfm-strikethrough: + specifier: ^2.0.0 + version: 2.0.0 + micromark-extension-gfm-strikethrough: + specifier: ^2.1.0 + version: 2.1.0 tsup: specifier: ^8.5.1 version: 8.5.1(jiti@2.7.0)(postcss@8.5.28)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.9.0) From 95f394004df031cdace6eb2ce8ceeb26d028490e Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Oct 2026 08:57:28 -0700 Subject: [PATCH 04/16] =?UTF-8?q?fix(remend):=20heal=20emphasis=20after=20?= =?UTF-8?q?prices=20and=20keep=20A*=E7=AE=97=E6=B3=95=20literal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lone $ opened math in the emphasis engine even with inlineKatex off, so bold after an odd count of prices never healed. The math mask now treats a lone $ as text unless inlineKatex is on, matching the math plugin's singleDollarTextMath default. The in-word gate now keeps an asterisk literal after an ASCII letter or digit when any letter or digit follows, so A*算法 stays literal. Adds a skipped-by-default harness that streams a JSONL corpus of model responses and counts frames that disagree with the final render. --- .../__benchmarks__/stream-glitches.test.ts | 212 ++++++++++++++++++ .../__tests__/emphasis-fast-path.test.ts | 7 +- .../remend/__tests__/emphasis-nesting.test.ts | 32 +++ packages/remend/src/emphasis-delimiters.ts | 54 +++-- packages/remend/src/emphasis-handlers.ts | 20 +- packages/remend/src/index.ts | 1 + packages/remend/src/scan.ts | 33 ++- 7 files changed, 325 insertions(+), 34 deletions(-) create mode 100644 packages/remend/__benchmarks__/stream-glitches.test.ts diff --git a/packages/remend/__benchmarks__/stream-glitches.test.ts b/packages/remend/__benchmarks__/stream-glitches.test.ts new file mode 100644 index 00000000..255633d4 --- /dev/null +++ b/packages/remend/__benchmarks__/stream-glitches.test.ts @@ -0,0 +1,212 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fromMarkdown } from "mdast-util-from-markdown"; +import { gfmStrikethroughFromMarkdown } from "mdast-util-gfm-strikethrough"; +import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough"; +import { it } from "vitest"; +import remend from "../src"; + +// Streams each response in a JSONL corpus through remend and counts frames +// whose render disagrees with the finished response's render. Run with +// CORPUS= [BASELINE=] [OUT=] + +interface Node { + children?: Node[]; + position?: { end: { offset?: number }; start: { offset?: number } }; + type: string; +} + +interface Sample { + model: string; + source: string; + text: string; +} + +type Heal = (text: string) => string; + +const toSample = (line: string): Sample => { + const value: unknown = JSON.parse(line); + if ( + typeof value !== "object" || + value === null || + !("text" in value) || + typeof value.text !== "string" + ) { + throw new Error(`Corpus line has no text: ${line.slice(0, 80)}`); + } + const field = (key: string): string => + key in value ? String(value[key as keyof typeof value]) : "unknown"; + return { model: field("model"), source: field("source"), text: value.text }; +}; + +const STEP = 4; +const STYLED = new Set(["emphasis", "strong", "delete", "inlineCode"]); +const ASCII_PUNCTUATION = /[!-/:-@[-`{-~]/; +const LEADING_TICKS = /^`+/; +const LETTER = /\p{L}/gu; +const DIGIT = /\d/g; +const LETTERS = /a+/g; + +const parse = (markdown: string): Node => + fromMarkdown(markdown, { + extensions: [gfmStrikethrough()], + mdastExtensions: [gfmStrikethroughFromMarkdown()], + }); + +// Each source offset's rendering: "" when it is syntax, otherwise the inline +// styles around the character it shows +const renderStyles = (markdown: string, root: Node, shift = 0): string[] => { + const styles: string[] = new Array(markdown.length + shift).fill(""); + const visit = (node: Node, stack: string) => { + const start = node.position?.start.offset ?? 0; + const end = node.position?.end.offset ?? 0; + const style = STYLED.has(node.type) ? `${stack}/${node.type}` : stack; + if (node.type === "text") { + for (let i = start; i < end; i++) { + if ( + markdown[i] === "\\" && + i + 1 < end && + ASCII_PUNCTUATION.test(markdown[i + 1]) + ) { + i++; + } + styles[shift + i] = `v${style}`; + } + } else if (node.type === "inlineCode") { + const raw = markdown.slice(start, end); + const ticks = raw.length - raw.replace(LEADING_TICKS, "").length; + for (let i = start + ticks; i < end - ticks; i++) { + styles[shift + i] = `v${style}`; + } + } + for (const child of node.children ?? []) { + visit(child, style); + } + }; + visit(root, ""); + return styles; +}; + +// Top-level block starts, since emphasis never crosses them +const blockStarts = (root: Node): number[] => + (root.children ?? []).map((child) => child.position?.start.offset ?? 0); + +// Characters of the prefix whose rendering differs from the final render +const mismatches = ( + prefix: string, + healed: string, + from: number, + final: string[] +): number => { + let cut = 0; + while (cut < prefix.length && prefix[cut] === healed[cut]) { + cut++; + } + const added = healed.length - prefix.length; + const frame = renderStyles( + healed.slice(from), + parse(healed.slice(from)), + from + ); + let count = 0; + for (let i = from; i < prefix.length; i++) { + const at = i < cut ? i : i + added; + if ((frame[at] ?? "") !== final[i]) { + count++; + } + } + return count; +}; + +// A coarse shape of the text before a cut, for grouping examples +const shapeOf = (tail: string): string => + tail.replace(LETTER, "a").replace(DIGIT, "0").replace(LETTERS, "a"); + +interface Arm { + frames: number; + glitches: number; +} + +const measure = (samples: Sample[], arms: Record) => { + const byGroup = new Map>(); + const divergent = { baselineCloser: 0, branchCloser: 0, tie: 0 }; + const worse = new Map(); + for (const sample of samples) { + const group = `${sample.source} ${sample.model}`; + const totals = + byGroup.get(group) ?? + Object.fromEntries( + Object.keys(arms).map((name) => [name, { frames: 0, glitches: 0 }]) + ); + byGroup.set(group, totals); + const finalTree = parse(sample.text); + const final = renderStyles(sample.text, finalTree); + const starts = blockStarts(finalTree); + for (let end = STEP; end < sample.text.length; end += STEP) { + const prefix = sample.text.slice(0, end); + const from = starts.findLast((start) => start < end) ?? 0; + const healed = Object.fromEntries( + Object.entries(arms).map(([name, heal]) => [name, heal(prefix)]) + ); + const counts: Record = {}; + for (const name of Object.keys(arms)) { + counts[name] = mismatches(prefix, healed[name], from, final); + totals[name].frames++; + if (counts[name] > 0) { + totals[name].glitches++; + } + } + if (arms.baseline && healed.baseline !== healed.branch) { + if (counts.branch < counts.baseline) { + divergent.branchCloser++; + } else if (counts.branch > counts.baseline) { + divergent.baselineCloser++; + const tail = prefix.slice(Math.max(from, end - 24)); + const shape = shapeOf(tail); + const entry = worse.get(shape) ?? { + count: 0, + example: JSON.stringify({ + tail, + baseline: healed.baseline.slice(Math.max(from, end - 24)), + branch: healed.branch.slice(Math.max(from, end - 24)), + final: sample.text.slice(Math.max(from, end - 24), end + 24), + }), + }; + entry.count++; + worse.set(shape, entry); + } else { + divergent.tie++; + } + } + } + } + return { + divergent, + groups: Object.fromEntries(byGroup), + worse: [...worse.entries()] + .sort((a, b) => b[1].count - a[1].count) + .slice(0, 40) + .map(([shape, entry]) => ({ shape, ...entry })), + }; +}; + +it.skipIf(!process.env.CORPUS)( + "measures streaming glitches against the final render", + async () => { + const samples: Sample[] = readFileSync(process.env.CORPUS ?? "", "utf8") + .split("\n") + .filter(Boolean) + .map(toSample); + const arms: Record = { branch: remend }; + if (process.env.BASELINE) { + const baseline = await import(resolve(process.env.BASELINE)); + arms.baseline = baseline.default; + } + const report = measure(samples, arms); + writeFileSync( + process.env.OUT ?? "stream-glitches.json", + JSON.stringify(report, null, 2) + ); + }, + 24 * 60 * 60 * 1000 +); diff --git a/packages/remend/__tests__/emphasis-fast-path.test.ts b/packages/remend/__tests__/emphasis-fast-path.test.ts index 72eb8696..930c5013 100644 --- a/packages/remend/__tests__/emphasis-fast-path.test.ts +++ b/packages/remend/__tests__/emphasis-fast-path.test.ts @@ -51,7 +51,8 @@ it("fast path matches a full rescan", () => { maxLength: 4, }), fc.constantFrom("", " ", "\n", " \n", "\\"), - (tokens, closerParts, tail) => { + fc.boolean(), + (tokens, closerParts, tail, inlineMath) => { const text = tokens.join("") + tail; let insertAt = text.length; while (insertAt > 0 && WHITESPACE.test(text[insertAt - 1])) { @@ -64,9 +65,9 @@ it("fast path matches a full rescan", () => { fc.pre(LINE_CONTENT.test(text.slice(lineStart, insertAt))); const closers = closerParts.join(""); const healed = text.slice(0, insertAt) + closers + text.slice(insertAt); - const base = prepareHeal(text, insertAt); + const base = prepareHeal(text, insertAt, inlineMath); expect(summary(matchHealed(base, closers))).toEqual( - summary(matchDelimiterRuns(healed)) + summary(matchDelimiterRuns(healed, inlineMath)) ); } ), diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts index fe25e0a8..570ea965 100644 --- a/packages/remend/__tests__/emphasis-nesting.test.ts +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -60,6 +60,33 @@ describe("emphasis nesting", () => { expect(remend(expected)).toBe(expected); }); + it.each([ + { + name: "after an odd number of prices", + input: "Pay $50 now, then **$1,15", + expected: "Pay $50 now, then **$1,15**", + }, + { + name: "after prices in an earlier paragraph", + input: "From $5 to $9 and $12.\n\n**Total: $276.", + expected: "From $5 to $9 and $12.\n\n**Total: $276.**", + }, + { + name: "beside display math", + input: "$$x**2$$ is **big", + expected: "$$x**2$$ is **big**", + }, + ])("treats a lone dollar sign as text: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it("hides markers inside inline math when it is opted in", () => { + expect(remend("Pay $a **b$ and **c", { inlineKatex: true })).toBe( + "Pay $a **b$ and **c**" + ); + }); + it.each([ { name: "closer before a word", @@ -86,6 +113,11 @@ describe("emphasis nesting", () => { input: "hello**world", expected: "hello**world", }, + { + name: "asterisk between an ASCII letter and CJK", + input: "使用A*算法", + expected: "使用A*算法", + }, { name: "opener inside a link label", input: "[*a](u)", diff --git a/packages/remend/src/emphasis-delimiters.ts b/packages/remend/src/emphasis-delimiters.ts index 12061c8e..2f30606b 100644 --- a/packages/remend/src/emphasis-delimiters.ts +++ b/packages/remend/src/emphasis-delimiters.ts @@ -140,10 +140,10 @@ const createRun = ( return run; }; -const findOpenInlineMath = (scan: TextScan): number => { +const findOpenInlineMath = (scan: TextScan, inlineMath: boolean): number => { const { text } = scan; let start = text.length; - while (start > 0 && inMathAt(scan, start - 1)) { + while (start > 0 && inMathAt(scan, start - 1, inlineMath)) { start -= 1; } const isInlineDollar = @@ -153,10 +153,15 @@ const findOpenInlineMath = (scan: TextScan): number => { // An unclosed code span or inline math span renders as literal text, so its // markers still count -const isProseAt = (scan: TextScan, i: number, openMath: number): boolean => +const isProseAt = ( + scan: TextScan, + i: number, + openMath: number, + inlineMath: boolean +): boolean => (scan.regions[i] === REGION.PROSE || scan.regions[i] === REGION.CODE_SPAN_OPEN) && - !(i < openMath && inMathAt(scan, i)) && + !(i < openMath && inMathAt(scan, i, inlineMath)) && !inLinkUrlAt(scan, i) && !inHtmlTagAt(scan, i); @@ -170,7 +175,8 @@ interface LinkLabel { const findLinkLabels = ( scan: TextScan, from: number, - openMath: number + openMath: number, + inlineMath: boolean ): LinkLabel[] => { const { text } = scan; const labels: LinkLabel[] = []; @@ -180,7 +186,7 @@ const findLinkLabels = ( if ( (char !== "[" && char !== "]") || isEscaped(text, i) || - !isProseAt(scan, i, openMath) + !isProseAt(scan, i, openMath, inlineMath) ) { continue; } @@ -212,13 +218,17 @@ const scopeOf = (labels: readonly LinkLabel[], index: number): number => { return scope; }; -const collectDelimiterRuns = (scan: TextScan, from: number): DelimiterRun[] => { +const collectDelimiterRuns = ( + scan: TextScan, + from: number, + inlineMath: boolean +): DelimiterRun[] => { const { text } = scan; const runs: DelimiterRun[] = []; - const openMath = findOpenInlineMath(scan); - const labels = findLinkLabels(scan, from, openMath); + const openMath = findOpenInlineMath(scan, inlineMath); + const labels = findLinkLabels(scan, from, openMath, inlineMath); const isRunMarker = (i: number, marker: string): boolean => - text[i] === marker && isProseAt(scan, i, openMath); + text[i] === marker && isProseAt(scan, i, openMath, inlineMath); for (let i = from; i < text.length; i += 1) { const marker = text[i]; @@ -382,13 +392,20 @@ const matchRuns = (runs: DelimiterRun[]): DelimiterMatch => { * Matches the delimiter runs of the text's last inline block as micromark * does, per CommonMark and GFM strikethrough */ -export const matchDelimiterRuns = (text: string): DelimiterMatch => { +/** Matches the runs of the text's last block, where `inlineMath` decides if a lone `$` opens math */ +export const matchDelimiterRuns = ( + text: string, + inlineMath: boolean +): DelimiterMatch => { const scan = getScan(text); - return matchRuns(collectDelimiterRuns(scan, findInlineBlockStart(scan))); + return matchRuns( + collectDelimiterRuns(scan, findInlineBlockStart(scan), inlineMath) + ); }; /** The authored text's runs, reused to match each candidate heal */ export interface HealBase { + inlineMath: boolean; insertAt: number; runs: readonly DelimiterRun[]; /** Whether closers would land where runs aren't collected */ @@ -396,19 +413,24 @@ export interface HealBase { text: string; } -export const prepareHeal = (text: string, insertAt: number): HealBase => { +export const prepareHeal = ( + text: string, + insertAt: number, + inlineMath: boolean +): HealBase => { const scan = getScan(text); const from = findInlineBlockStart(scan); return { + inlineMath, insertAt, - runs: collectDelimiterRuns(scan, from), + runs: collectDelimiterRuns(scan, from, inlineMath), // Past a trailing blank line the closers would start a block of their // own, and inside a tag, math or a fence their markers aren't collected slow: from > insertAt || (insertAt > 0 && (inHtmlTagAt(scan, insertAt - 1) || - inMathAt(scan, insertAt - 1) || + inMathAt(scan, insertAt - 1, inlineMath) || isFenceAt(scan, insertAt - 1))), text, }; @@ -423,7 +445,7 @@ export const matchHealed = ( closers: string ): DelimiterMatch => { if (base.slow) { - return matchDelimiterRuns(insertClosers(base, closers)); + return matchDelimiterRuns(insertClosers(base, closers), base.inlineMath); } const { insertAt, text } = base; const runs = base.runs.map((run) => ({ diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 413eca91..12001d92 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -16,6 +16,8 @@ export interface EmphasisFlags { bold: boolean; boldItalic: boolean; inlineCode: boolean; + /** Whether a lone `$` opens math, which hides the markers inside it */ + inlineMath: boolean; italic: boolean; strikethrough: boolean; } @@ -53,13 +55,14 @@ const hasContentBefore = ( }; const asciiLetterOrDigitPattern = /^[A-Za-z0-9]$/; +const letterOrDigitPattern = /^[\p{L}\p{N}]/u; const isLetterOrDigit = (char: string): boolean => asciiLetterOrDigitPattern.test(char); -// An asterisk inside an ASCII word opens only to continue a chain of emphasis -// in that word, so hello*world and 2**3 stay literal while languages written -// without spaces still heal +// An asterisk after an ASCII letter or digit opens only to continue a chain of +// emphasis in that word, so hello*world, 2**3 and A*算法 stay literal while +// languages written without spaces still heal const opensColdInWord = ( text: string, run: MarkerRun, @@ -67,7 +70,7 @@ const opensColdInWord = ( ): boolean => run.marker === "*" && isLetterOrDigit(text[run.index - 1] ?? "") && - isLetterOrDigit(text[run.index + run.length] ?? "") && + letterOrDigitPattern.test(text.slice(run.index + run.length)) && !( previous?.marker === "*" && [...text.slice(previous.index + previous.length, run.index)].every( @@ -253,12 +256,12 @@ const healFrom = ( const healEmphasis = (text: string, flags: EmphasisFlags): string => { const insertAt = findInsertionPoint(text); - const match = matchDelimiterRuns(text); + const match = matchDelimiterRuns(text, flags.inlineMath); const last = match.runs.at(-1); const trailing = last && last.index + last.length === insertAt ? last : undefined; const context = { - base: prepareHeal(text, insertAt), + base: prepareHeal(text, insertAt, flags.inlineMath), flags, insertAt, literal: match.literal, @@ -271,7 +274,10 @@ const healEmphasis = (text: string, flags: EmphasisFlags): string => { // A trailing run that closes something may instead be literal text, with // the runs open before it still needing closers if (trailing && trailing.count < trailing.length) { - const prefix = matchDelimiterRuns(text.slice(0, trailing.index)); + const prefix = matchDelimiterRuns( + text.slice(0, trailing.index), + flags.inlineMath + ); const asText = { ...trailing, count: trailing.length }; const streaming = trailing.count > 0 || trailing.closesPartially; const alternative = healFrom( diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 36486265..ca651614 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -202,6 +202,7 @@ const getEnabledBuiltInHandlers = ( bold: isEnabled(options?.bold), boldItalic: isEnabled(options?.boldItalic), inlineCode: isEnabled(options?.inlineCode), + inlineMath: isOptedIn(options?.inlineKatex), italic: isEnabled(options?.italic), strikethrough: isEnabled(options?.strikethrough), }; diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 96ea1a86..d95c8d6f 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -70,6 +70,8 @@ export interface TextScan { linkUrlMask: Uint8Array | null; /** Lazily computed; stays null until first needed */ mathMask: Uint8Array | null; + /** The math mask when a lone `$` is text, also lazily computed */ + mathMaskWithoutDollar: Uint8Array | null; openFence: OpenFence | null; openSpan: OpenSpan | null; regions: Uint8Array; @@ -414,6 +416,7 @@ const scanText = (text: string): TextScan => { openFence, openSpan, mathMask: null, + mathMaskWithoutDollar: null, linkUrlMask: null, htmlTagMask: null, }; @@ -560,7 +563,8 @@ interface MathDelimiter { const mathDelimiterAt = ( text: string, i: number, - context: MathContext + context: MathContext, + singleDollar: boolean ): MathDelimiter | null => { const next = text[i + 1]; if (text[i] === "\\") { @@ -572,6 +576,9 @@ const mathDelimiterAt = ( } if (text[i] === "$" && !isLatexMathContext(context)) { const isBlockDelimiter = next === "$"; + if (!(isBlockDelimiter || singleDollar)) { + return null; + } return { context: getDollarMathContext(context, isBlockDelimiter), length: isBlockDelimiter ? 2 : 1, @@ -584,7 +591,7 @@ const mathDelimiterAt = ( // \(...\) or \[...\]. Delimiters inside code regions are literal and do // not change math state. A two-character delimiter marks its second // character with the context it establishes. -const buildMathMask = (scan: TextScan): Uint8Array => { +const buildMathMask = (scan: TextScan, singleDollar: boolean): Uint8Array => { const { text, regions } = scan; const n = text.length; const mask = new Uint8Array(n); @@ -594,7 +601,9 @@ const buildMathMask = (scan: TextScan): Uint8Array => { while (i < n) { mask[i] = context === "none" ? 0 : 1; const delimiter: MathDelimiter | null = - regions[i] === REGION.PROSE ? mathDelimiterAt(text, i, context) : null; + regions[i] === REGION.PROSE + ? mathDelimiterAt(text, i, context, singleDollar) + : null; if (delimiter === null) { i += 1; continue; @@ -609,16 +618,24 @@ const buildMathMask = (scan: TextScan): Uint8Array => { return mask; }; -export const inMathAt = (scan: TextScan, position: number): boolean => { +/** Whether position is in math, where `singleDollar` decides if a lone `$` opens it */ +export const inMathAt = ( + scan: TextScan, + position: number, + singleDollar = true +): boolean => { if (position < 0 || position >= scan.text.length) { return false; } - if (scan.mathMask === null) { - scan.mathMask = hasMathDelimiters(scan.text) - ? buildMathMask(scan) + const key = singleDollar ? "mathMask" : "mathMaskWithoutDollar"; + let mask = scan[key]; + if (mask === null) { + mask = hasMathDelimiters(scan.text) + ? buildMathMask(scan, singleDollar) : EMPTY_MASK; + scan[key] = mask; } - return scan.mathMask[position] === 1; + return mask[position] === 1; }; // Marks the URL positions of one line: those between a "](" opener and the From 183d05c84d286df29615a78baef5b49e20894173 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Oct 2026 09:41:52 -0700 Subject: [PATCH 05/16] fix(remend): heal bold after a closer that failed flanking Duplicate ** or __ runs now pair off, so the last open one still heals in **Note:**This is **imp and in CJK text where fullwidth punctuation stops a closer. Single-marker duplicates stay literal (globs, _id). Attention and strikethrough now resolve in the order micromark uses for the whole block, link text included. --- .../remend/__tests__/emphasis-nesting.test.ts | 15 +++++++ packages/remend/src/emphasis-delimiters.ts | 31 +++++++++------ packages/remend/src/emphasis-handlers.ts | 39 ++++++++++++++++--- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts index 570ea965..ed9e4608 100644 --- a/packages/remend/__tests__/emphasis-nesting.test.ts +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -55,6 +55,21 @@ describe("emphasis nesting", () => { input: "- **text\nmore text", expected: "- **text\nmore text**", }, + { + name: "bold after a closer that failed flanking", + input: "**Note:**This is **imp", + expected: "**Note:**This is **imp**", + }, + { + name: "bold around a pair opened after fullwidth punctuation", + input: "**机器学习（ML）**是一个分支，能够**从数", + expected: "**机器学习（ML）**是一个分支，能够**从数**", + }, + { + name: "asterisk in link text resolves before strikethrough", + input: "[*](u)~~*~~*a", + expected: "[*](u)~~*~~*a~~", + }, ])("pairs within one block: $name", ({ input, expected }) => { expect(remend(input)).toBe(expected); expect(remend(expected)).toBe(expected); diff --git a/packages/remend/src/emphasis-delimiters.ts b/packages/remend/src/emphasis-delimiters.ts index 2f30606b..25aec281 100644 --- a/packages/remend/src/emphasis-delimiters.ts +++ b/packages/remend/src/emphasis-delimiters.ts @@ -39,8 +39,8 @@ interface DelimiterRun extends MarkerRun { } interface DelimiterPair { - closer: MarkerRun; - opener: MarkerRun; + closer: DelimiterRun; + opener: DelimiterRun; used: number; } @@ -350,15 +350,14 @@ const matchPass = ( const isTilde = (run: DelimiterRun): boolean => run.marker === "~"; const isAttention = (run: DelimiterRun): boolean => run.marker !== "~"; -const matchScope = ( - runs: DelimiterRun[], - pairs: DelimiterPair[] -): DelimiterRun[] => { - // An inert tilde run never reaches the strikethrough resolver +// micromark resolves attention and strikethrough in the order each first +// appears in the block, link text included, and an inert tilde run never +// reaches the strikethrough resolver +const passOrder = (runs: readonly DelimiterRun[]) => { const first = runs.find((run) => !isTilde(run) || run.length <= 2); - const passes = - first && isTilde(first) ? [isTilde, isAttention] : [isAttention, isTilde]; - return passes.flatMap((inPass) => matchPass(runs, inPass, pairs)); + return first && isTilde(first) + ? [isTilde, isAttention] + : [isAttention, isTilde]; }; const matchRuns = (runs: DelimiterRun[]): DelimiterMatch => { @@ -372,9 +371,10 @@ const matchRuns = (runs: DelimiterRun[]): DelimiterMatch => { } } const pairs: DelimiterPair[] = []; + const passes = passOrder(runs); let open: DelimiterRun[] = []; for (const [scope, scoped] of scopes) { - const left = matchScope(scoped, pairs); + const left = passes.flatMap((inPass) => matchPass(scoped, inPass, pairs)); if (scope === -1) { open = left; } @@ -515,6 +515,13 @@ const parentOf = ( return parent; }; +// An authored pair whose opener follows punctuation, as in **注意：**这是, is +// usually a closer that failed flanking, so emphasis around it is intended +const opensAfterPunctuation = (pair: DelimiterPair): boolean => + isPunctuationChar(pair.opener.prevChar) && + !isWhitespaceOrEdge(pair.opener.nextChar) && + !isPunctuationChar(pair.opener.nextChar); + // Authors don't nest emphasis directly inside the same emphasis, so a heal // that needs that is closing runs written as literal text const nestsSameKind = (match: DelimiterMatch, insertAt: number): boolean => { @@ -525,7 +532,7 @@ const nestsSameKind = (match: DelimiterMatch, insertAt: number): boolean => { return ( parent !== undefined && kindOf(parent) === kindOf(pair) && - (healed(pair) || healed(parent)) + (healed(pair) || (healed(parent) && !opensAfterPunctuation(pair))) ); }); }; diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 12001d92..4c3a79c0 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -139,6 +139,37 @@ const closerCandidates = (group: MarkerRun[], reusable: number): string[] => { ); }; +// Two open runs of one kind are literal text, like globs or _id, _rev. +// When only double * or _ runs are open they pair off instead, since a closer +// that failed flanking in **Note:**text leaves an opener behind it that the +// author may still close. Tildes stay strict because GFM matches them in a +// separate pass. +const withoutDuplicates = ( + runs: readonly MarkerRun[] +): MarkerRun[] | undefined => { + const byKind = new Map(); + for (const run of runs) { + const kind = run.marker + run.count; + byKind.set(kind, [...(byKind.get(kind) ?? []), run]); + } + // Closing another kind can bury one run of a pair and expose the other + if (byKind.size > 1 && byKind.size < runs.length) { + return; + } + const healable = new Set(); + for (const group of byKind.values()) { + if (group.length > 1 && (group[0].count === 1 || group[0].marker === "~")) { + return; + } + if (group.length % 2 === 1) { + healable.add( + group.reduce((last, run) => (run.index > last.index ? run : last)) + ); + } + } + return runs.filter((run) => healable.has(run)); +}; + // A closer's flanking can depend on the closers after it, so each step tries // every remaining closer at once before trying it alone const extendHeal = ( @@ -162,10 +193,8 @@ const extendHeal = ( return start; } } - // Two open runs of one marker and length are literal text, like globs or a - // closer that failed flanking in **Note:**text - const kinds = new Set(runs.map((run) => run.marker + run.count)); - if (kinds.size < runs.length) { + const healable = withoutDuplicates(runs); + if (healable === undefined) { return start; } @@ -213,7 +242,7 @@ const extendHeal = ( // Largest subsets first, so a tie keeps the heal that closes more runs const subsets: MarkerRun[][] = [[]]; - for (const run of runs) { + for (const run of healable) { subsets.push(...subsets.map((set) => [...set, run])); } subsets.reverse(); From 11b92d33ec3e417d08d4427956b3f218ac38e510 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Oct 2026 09:41:52 -0700 Subject: [PATCH 06/16] fix(remend): keep inline code and link heals inside their block A code span opened in an earlier block stays a literal backtick, and a closer that would start a line with three backticks goes after a space so it can't open a fence. A [ before a complete link is literal, since links can't nest, so [a[a no longer heals twice. --- .changeset/remend-escaped-markers.md | 1 + packages/remend/__tests__/inline-code.test.ts | 45 +++++++++++++++++++ packages/remend/__tests__/links.test.ts | 27 +++++++++++ packages/remend/__tests__/streaming.test.ts | 7 ++- packages/remend/src/inline-code-handler.ts | 23 +++++++--- packages/remend/src/link-image-handler.ts | 20 ++++++++- 6 files changed, 112 insertions(+), 11 deletions(-) diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index 88c713bf..93f625f4 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -6,4 +6,5 @@ Heal emphasis the way CommonMark and GFM parse it - Closes only openers that exist, innermost first, within the paragraph, list item or link text that holds them. An open code span closes before the emphasis around it. - Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. +- Closes an inline code span only within its own block, and never with a closer that would start a fence. A `[` before a complete link stays literal instead of healing a second link. - The `bold`, `italic`, `boldItalic` and `strikethrough` options still apply, but the built-in handlers they controlled are now one `emphasis` handler at priority 30. Custom handlers ordered against the old priorities may need new ones. diff --git a/packages/remend/__tests__/inline-code.test.ts b/packages/remend/__tests__/inline-code.test.ts index b24cecc2..24a6fa43 100644 --- a/packages/remend/__tests__/inline-code.test.ts +++ b/packages/remend/__tests__/inline-code.test.ts @@ -91,4 +91,49 @@ describe("emphasis markers inside inline code spans should not leak", () => { it("should complete emphasis after a closed inline code span", () => { expect(remend("`code` **bold")).toBe("`code` **bold**"); }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "opener before a setext underline", + input: "`a\n---", + expected: "`a\n---", + }, + { + name: "opener in an earlier list item", + input: "- `a\n- b", + expected: "- `a\n- b", + }, + { + name: "opener in a heading", + input: "# `a\nb", + expected: "# `a\nb", + }, + { + name: "lazy continuation line", + input: "text `a\nb", + expected: "text `a\nb`", + }, + ])("closes only within the opener's block: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "one trailing backtick", + input: "a```a\n`", + expected: "a```a\n` ```", + }, + { + name: "two trailing backticks", + input: "a```a\n``", + expected: "a```a\n`` ```", + }, + ])("keeps a closer at line start from opening a fence: $name", ({ + input, + expected, + }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); }); diff --git a/packages/remend/__tests__/links.test.ts b/packages/remend/__tests__/links.test.ts index 45433029..6a7dce06 100644 --- a/packages/remend/__tests__/links.test.ts +++ b/packages/remend/__tests__/links.test.ts @@ -69,6 +69,33 @@ describe("link handling", () => { "[link [nested] text](streamdown:incomplete-link)" ); }); + + // Links can't contain links, so a bracket before a complete link is literal + it.each<{ name: string; input: string; expected: string }>([ + { + name: "outer bracket around a healed link", + input: "[a[a", + expected: "[a[a](streamdown:incomplete-link)", + }, + { + name: "outer bracket around a complete link", + input: "[a [b](u) c", + expected: "[a [b](u) c", + }, + { + name: "image around a complete link", + input: "![a [b](u) c", + expected: "![a [b](u) c](streamdown:incomplete-image)", + }, + { + name: "open bracket after a link inside a literal bracket", + input: "[outer [inner](u) and [more", + expected: "[outer [inner](u) and [more](streamdown:incomplete-link)", + }, + ])("heals once around nested links: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); }); describe("link handling with linkMode: text-only", () => { diff --git a/packages/remend/__tests__/streaming.test.ts b/packages/remend/__tests__/streaming.test.ts index 8bb89a4f..c6f2a07b 100644 --- a/packages/remend/__tests__/streaming.test.ts +++ b/packages/remend/__tests__/streaming.test.ts @@ -27,10 +27,9 @@ describe("chunked streaming scenarios", () => { }); it("should handle complex nested structures from chunks", () => { - // Backticks spanning multiple lines need special handling - expect(remend("1. First item\n - Nested with `code\n2. Second")).toBe( - "1. First item\n - Nested with `code\n2. Second`" - ); + // A code span can't cross list items, so the backtick stays literal + const text = "1. First item\n - Nested with `code\n2. Second"; + expect(remend(text)).toBe(text); }); it("should handle multiple incomplete formats in one chunk", () => { diff --git a/packages/remend/src/inline-code-handler.ts b/packages/remend/src/inline-code-handler.ts index 67c0dc9a..56d6555c 100644 --- a/packages/remend/src/inline-code-handler.ts +++ b/packages/remend/src/inline-code-handler.ts @@ -1,3 +1,4 @@ +import { findInlineBlockStart } from "./inline-block"; import { whitespaceOrMarkersPattern } from "./patterns"; import { getScan } from "./scan"; import { isWhitespaceChar } from "./utils"; @@ -17,8 +18,10 @@ export const handleIncompleteInlineCode = (text: string): string => { return text; } + // A code span can't cross into another block, so an opener before the + // last one stays a literal backtick const span = scan.openSpan; - if (!span) { + if (!span || span.start < findInlineBlockStart(scan)) { return text; } @@ -49,9 +52,17 @@ export const handleIncompleteInlineCode = (text: string): string => { return text; } - return ( - text.slice(0, end) + - "`".repeat(span.runLength - trailingRun) + - text.slice(end) - ); + // Completing a run that starts its line with three or more backticks would + // open a fence, so a full closer goes after a space instead + const runStart = end - trailingRun; + const lineStart = text.lastIndexOf("\n", runStart - 1) + 1; + const opensFence = + trailingRun > 0 && + span.runLength >= 3 && + text.slice(lineStart, runStart).trim() === ""; + const closer = opensFence + ? ` ${"`".repeat(span.runLength)}` + : "`".repeat(span.runLength - trailingRun); + + return text.slice(0, end) + closer + text.slice(end); }; diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 66cabd0f..f98a553d 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -142,6 +142,15 @@ const handleIncompleteText = ( return null; }; +const opensCompleteLink = (text: string, open: number): boolean => { + const close = findMatchingClosingBracket(text, open); + return ( + close !== -1 && + text[close + 1] === "(" && + text.indexOf(")", close + 2) !== -1 + ); +}; + // One healing step over the trailing incomplete link or image, if any const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Look for patterns like [text]( or ![text]( at the end of text @@ -158,12 +167,21 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Then check for incomplete link text: [partial-text without closing ] // Search backwards for an opening bracket that doesn't have a matching closing bracket + let enclosesLink = false; for ( let i = text.lastIndexOf("["); i !== -1; i = i === 0 ? -1 : text.lastIndexOf("[", i - 1) ) { - if (!isInsideCodeBlock(text, i)) { + const isLink = !( + (i > 0 && text[i - 1] === "!") || + isInsideCodeBlock(text, i) + ); + const closesHere = isLink && opensCompleteLink(text, i); + // Links can't contain links, so a link bracket before one is literal text + const literal = isLink && enclosesLink; + enclosesLink ||= closesHere; + if (!(closesHere || literal || isInsideCodeBlock(text, i))) { const result = handleIncompleteText(text, i, linkMode); if (result !== null) { return result; From e4fef1bdc0809f4bc2f9add6a77baebc3ad80502 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Oct 2026 09:45:54 -0700 Subject: [PATCH 07/16] fix(remend): end emphasis at a streaming dash and heal [`[ once The setext guard turns a lone dash under text into a dash plus a zero-width space. That dash usually starts a list item or thematic break, so emphasis and code spans no longer close across it. An unclosed backtick renders as text, so a link after one now marks the bracket before it as literal, and [`[ heals once. --- packages/remend/__tests__/emphasis-nesting.test.ts | 5 +++++ packages/remend/__tests__/links.test.ts | 5 +++++ packages/remend/src/inline-block.ts | 4 ++++ packages/remend/src/link-image-handler.ts | 6 +++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts index ed9e4608..dbaaba0f 100644 --- a/packages/remend/__tests__/emphasis-nesting.test.ts +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -70,6 +70,11 @@ describe("emphasis nesting", () => { input: "[*](u)~~*~~*a", expected: "[*](u)~~*~~*a~~", }, + { + name: "dash that starts the next list item", + input: "_*\n- ", + expected: "_*\n-\u200B", + }, ])("pairs within one block: $name", ({ input, expected }) => { expect(remend(input)).toBe(expected); expect(remend(expected)).toBe(expected); diff --git a/packages/remend/__tests__/links.test.ts b/packages/remend/__tests__/links.test.ts index 6a7dce06..28f80dd4 100644 --- a/packages/remend/__tests__/links.test.ts +++ b/packages/remend/__tests__/links.test.ts @@ -92,6 +92,11 @@ describe("link handling", () => { input: "[outer [inner](u) and [more", expected: "[outer [inner](u) and [more](streamdown:incomplete-link)", }, + { + name: "bracket after an unclosed backtick", + input: "[`[", + expected: "[`[](streamdown:incomplete-link)", + }, ])("heals once around nested links: $name", ({ input, expected }) => { expect(remend(input)).toBe(expected); expect(remend(expected)).toBe(expected); diff --git a/packages/remend/src/inline-block.ts b/packages/remend/src/inline-block.ts index c93cd2dc..e9843580 100644 --- a/packages/remend/src/inline-block.ts +++ b/packages/remend/src/inline-block.ts @@ -4,6 +4,9 @@ const headingPattern = /^#{1,6}(?:[ \t]|$)/; const thematicBreakPattern = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; const setextUnderlinePattern = /^(?:=+|-+)[ \t]*$/; +// The setext handler's zero-width space after a streaming dash stands in for +// the list item or thematic break that dash usually starts +const dashPlaceholderPattern = /^-{1,2}\u200B$/; /** * Index where the text's last inline block starts, since emphasis never @@ -42,6 +45,7 @@ export const findInlineBlockStart = (scan: TextScan): number => { endsBlock || prefix.listIndent > 0 || prefix.quoteDepth > quoteDepth || + dashPlaceholderPattern.test(content) || headingPattern.test(content) || content.startsWith("|") ) { diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index f98a553d..dc60601b 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -1,4 +1,5 @@ import { isInsideCodeBlock } from "./code-block-utils"; +import { getScan, isCompleteSpanAt, isFenceAt } from "./scan"; import { findMatchingClosingBracket, findMatchingOpeningBracket, @@ -167,15 +168,18 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Then check for incomplete link text: [partial-text without closing ] // Search backwards for an opening bracket that doesn't have a matching closing bracket + const scan = getScan(text); let enclosesLink = false; for ( let i = text.lastIndexOf("["); i !== -1; i = i === 0 ? -1 : text.lastIndexOf("[", i - 1) ) { + // An unclosed backtick renders as text, so a link after one still counts const isLink = !( (i > 0 && text[i - 1] === "!") || - isInsideCodeBlock(text, i) + isFenceAt(scan, i) || + isCompleteSpanAt(scan, i) ); const closesHere = isLink && opensCompleteLink(text, i); // Links can't contain links, so a link bracket before one is literal text From 627ef97916b2b223ee76f2108dd89f9c09a86440 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Oct 2026 09:58:39 -0700 Subject: [PATCH 08/16] fix(remend): leave fill-in blanks of four or more underscores open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Models write blanks like discuss ______? or 电话：__________（. CommonMark reads those runs as openers, so healing styled the rest of the paragraph while the finished response renders them literally. --- .changeset/remend-escaped-markers.md | 2 +- packages/remend/__tests__/emphasis-nesting.test.ts | 10 ++++++++++ packages/remend/__tests__/escaped-markers.test.ts | 4 ++-- packages/remend/src/emphasis-handlers.ts | 2 ++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index 93f625f4..1f840caf 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -5,6 +5,6 @@ Heal emphasis the way CommonMark and GFM parse it - Closes only openers that exist, innermost first, within the paragraph, list item or link text that holds them. An open code span closes before the emphasis around it. -- Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. +- Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, fill-in blanks like `______`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. - Closes an inline code span only within its own block, and never with a closer that would start a fence. A `[` before a complete link stays literal instead of healing a second link. - The `bold`, `italic`, `boldItalic` and `strikethrough` options still apply, but the built-in handlers they controlled are now one `emphasis` handler at priority 30. Custom handlers ordered against the old priorities may need new ones. diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts index dbaaba0f..29489d6d 100644 --- a/packages/remend/__tests__/emphasis-nesting.test.ts +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -133,6 +133,16 @@ describe("emphasis nesting", () => { input: "hello**world", expected: "hello**world", }, + { + name: "fill-in blank before punctuation", + input: "Can we discuss ______? It is", + expected: "Can we discuss ______? It is", + }, + { + name: "fill-in blank between fullwidth punctuation", + input: "电话：__________（可向", + expected: "电话：__________（可向", + }, { name: "asterisk between an ASCII letter and CJK", input: "使用A*算法", diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts index daa0c0ab..9cb3b220 100644 --- a/packages/remend/__tests__/escaped-markers.test.ts +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -65,8 +65,8 @@ describe("trailing backslash", () => { { name: "~~", input: "~~foo\\", expected: "~~foo\\\\~~" }, { name: "a partial __ match", - input: "____.\\", - expected: "____.\\\\____", + input: "___.\\", + expected: "___.\\\\___", }, { name: "a backslash after whitespace", diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 4c3a79c0..4c072ddb 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -92,6 +92,8 @@ const isGated = ( !hasContentBefore(text, run, insertAt) || // Prose uses a single tilde for "approximately" (run.marker === "~" && run.count === 1) || + // Models write fill-in blanks as long underscore runs, like Name: ______ + (run.marker === "_" && run.length >= 4) || opensColdInWord(text, run, previous) || followsEscapedMarker(text, run); From c6475907fe0cf744a53e0e66b9eb828a0db7f4ed Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Oct 2026 15:44:59 -0700 Subject: [PATCH 09/16] fix(remend): match link brackets outside code spans The bracket matchers skip brackets inside a complete code span, so a code span like `[` no longer unbalances the label around it. When a link heal is a no-op and a code span is open after a [, the link handler heals again with that span closed, as the emphasis handler does. A code span binds tighter than a link, so closing it can expose an outer [ that otherwise healed only on the next call. --- packages/remend/__tests__/links.test.ts | 20 ++++++++++++ packages/remend/src/index.ts | 6 +++- packages/remend/src/link-image-handler.ts | 40 ++++++++++++++++++++--- packages/remend/src/utils.ts | 22 +++++++++++-- 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/packages/remend/__tests__/links.test.ts b/packages/remend/__tests__/links.test.ts index 28f80dd4..cd07c379 100644 --- a/packages/remend/__tests__/links.test.ts +++ b/packages/remend/__tests__/links.test.ts @@ -97,6 +97,26 @@ describe("link handling", () => { input: "[`[", expected: "[`[](streamdown:incomplete-link)", }, + { + name: "code span that swallows a complete link", + input: "[`[](u)", + expected: "[`[](u)`](streamdown:incomplete-link)", + }, + { + name: "brackets inside a complete code span", + input: "a[~~`[[`a", + expected: "a[~~`[[`a](streamdown:incomplete-link)", + }, + { + name: "label closed after a code span holding a bracket", + input: "[`[`]", + expected: "[`[`]", + }, + { + name: "code span holding an open bracket before a link", + input: "Split on `[` and see [the", + expected: "Split on `[` and see [the](streamdown:incomplete-link)", + }, ])("heals once around nested links: $name", ({ input, expected }) => { expect(remend(input)).toBe(expected); expect(remend(expected)).toBe(expected); diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index ca651614..fea4d55a 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -235,7 +235,11 @@ const getEnabledBuiltInHandlers = ( handler: { ...handler, handle: (text: string) => - handleIncompleteLinksAndImages(text, linkMode), + handleIncompleteLinksAndImages( + text, + linkMode, + emphasisFlags.inlineCode + ), }, // Only use early return for protocol mode (text-only won't end with the marker) earlyReturn: linkMode === "protocol" ? earlyReturn : undefined, diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index dc60601b..4bd4d5e0 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -1,4 +1,5 @@ import { isInsideCodeBlock } from "./code-block-utils"; +import { handleIncompleteInlineCode } from "./inline-code-handler"; import { getScan, isCompleteSpanAt, isFenceAt } from "./scan"; import { findMatchingClosingBracket, @@ -201,11 +202,7 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // without a bound. Realistic nesting depth is single digits. const MAX_HEAL_PASSES = 32; -// Preserves incomplete links and images with a special marker. -export const handleIncompleteLinksAndImages = ( - text: string, - linkMode: LinkMode = "protocol" -): string => { +const healLinksAndImages = (text: string, linkMode: LinkMode): string => { let current = text; // Removing a trailing incomplete image can expose another incomplete @@ -223,3 +220,36 @@ export const handleIncompleteLinksAndImages = ( } return current; }; + +const endsWithPlaceholder = (text: string): boolean => + text.endsWith("](streamdown:incomplete-link)") || + text.endsWith(`](${INCOMPLETE_IMAGE_PLACEHOLDER})`); + +// Preserves incomplete links and images with a special marker. +export const handleIncompleteLinksAndImages = ( + text: string, + linkMode: LinkMode = "protocol", + closeInlineCode = false +): string => { + const healed = healLinksAndImages(text, linkMode); + if (healed !== text || !closeInlineCode || endsWithPlaceholder(text)) { + return healed; + } + const span = getScan(text).openSpan; + if (!span || text.lastIndexOf("[", span.start) === -1) { + return text; + } + + // Code spans bind tighter than links, so closing an open span can swallow + // brackets and leave a link opener unclosed. Heal against the span the + // inline code handler will close, and keep that only when a link needed it. + // Trailing whitespace stays outside the closers, as the inline code + // handler leaves it + const body = text.trimEnd(); + const closed = handleIncompleteInlineCode(body); + if (closed === body) { + return text; + } + const relinked = healLinksAndImages(closed, linkMode); + return relinked === closed ? text : relinked + text.slice(body.length); +}; diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 59c1898e..5b091db6 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,5 +1,12 @@ import { letterNumberUnderscorePattern } from "./patterns"; -import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isCompleteSpanAt, + isFenceAt, +} from "./scan"; const whitespaceCharPattern = /\s/; @@ -30,13 +37,18 @@ export const isWithinCodeBlock = (text: string, position: number): boolean => isFenceAt(getScan(text), position); // Helper function to find the matching opening bracket for a closing bracket -// Handles nested brackets correctly by searching backwards +// Handles nested brackets correctly by searching backwards. Code spans bind +// tighter than links, so brackets inside a complete span don't count. export const findMatchingOpeningBracket = ( text: string, closeIndex: number ): number => { + const scan = getScan(text); let depth = 1; for (let i = closeIndex - 1; i >= 0; i -= 1) { + if (isCompleteSpanAt(scan, i)) { + continue; + } if (text[i] === "]") { depth += 1; } else if (text[i] === "[") { @@ -50,13 +62,17 @@ export const findMatchingOpeningBracket = ( }; // Helper function to find the matching closing bracket for an opening bracket -// Handles nested brackets correctly +// Handles nested brackets correctly, skipping brackets inside complete code spans export const findMatchingClosingBracket = ( text: string, openIndex: number ): number => { + const scan = getScan(text); let depth = 1; for (let i = openIndex + 1; i < text.length; i += 1) { + if (isCompleteSpanAt(scan, i)) { + continue; + } if (text[i] === "[") { depth += 1; } else if (text[i] === "]") { From 722015a13272e3944477679d1d5164c74e30dfcd Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Oct 2026 15:44:59 -0700 Subject: [PATCH 10/16] fix(remend): stop healing openers whose line has ended None of 8,263 bold spans in the WildChat and Claude corpora cross a line break, and the italic spans that do are stray * in code written outside a fence. An opener stays open once the next line has content. Also keeps M*A*S*H style chains literal when their first pair opened inside a word, lets **----- heal past a line of dashes, and ends a block after an empty list item so a later paragraph resolves its own runs. --- .changeset/remend-escaped-markers.md | 1 + .../broken-markdown-variants.test.ts | 5 +- .../remend/__tests__/emphasis-nesting.test.ts | 24 ++++++- packages/remend/__tests__/lists.test.ts | 4 +- packages/remend/src/emphasis-delimiters.ts | 10 ++- packages/remend/src/emphasis-handlers.ts | 67 ++++++++++++++----- packages/remend/src/inline-block.ts | 5 +- 7 files changed, 90 insertions(+), 26 deletions(-) diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index 1f840caf..7af3b065 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -6,5 +6,6 @@ Heal emphasis the way CommonMark and GFM parse it - Closes only openers that exist, innermost first, within the paragraph, list item or link text that holds them. An open code span closes before the emphasis around it. - Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, fill-in blanks like `______`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. +- Stops closing an opener once its line has ended and the next line has started, since models close emphasis on the line that opens it. A stray `*` in code written outside a fence, like `char *name`, no longer italicizes what follows. - Closes an inline code span only within its own block, and never with a closer that would start a fence. A `[` before a complete link stays literal instead of healing a second link. - The `bold`, `italic`, `boldItalic` and `strikethrough` options still apply, but the built-in handlers they controlled are now one `emphasis` handler at priority 30. Custom handlers ordered against the old priorities may need new ones. diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index f0cb5ff8..f1d0b4e3 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -450,8 +450,9 @@ describe("whitespace edge cases", () => { expect(remend("**bold\twith\ttabs")).toBe("**bold\twith\ttabs**"); }); - it("should close bold with CRLF", () => { - expect(remend("**bold\r\nwith CRLF")).toBe("**bold\r\nwith CRLF**"); + it("should close bold before a CRLF but not after the next line starts", () => { + expect(remend("**bold\r\n")).toBe("**bold**\r\n"); + expect(remend("**bold\r\nwith CRLF")).toBe("**bold\r\nwith CRLF"); }); it("should close bold after many leading newlines", () => { diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts index 29489d6d..40b8bdb2 100644 --- a/packages/remend/__tests__/emphasis-nesting.test.ts +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -51,9 +51,9 @@ describe("emphasis nesting", () => { expected: "> **a\n\n> **b**", }, { - name: "lazy continuation line", + name: "bold left open when its line ended", input: "- **text\nmore text", - expected: "- **text\nmore text**", + expected: "- **text\nmore text", }, { name: "bold after a closer that failed flanking", @@ -75,6 +75,16 @@ describe("emphasis nesting", () => { input: "_*\n- ", expected: "_*\n-\u200B", }, + { + name: "bold before a dash rule", + input: "**-----", + expected: "**-----**", + }, + { + name: "paragraph after an empty list item", + input: "*\n~~**a ~~`", + expected: "*\n~~**a ~~`**", + }, ])("pairs within one block: $name", ({ input, expected }) => { expect(remend(input)).toBe(expected); expect(remend(expected)).toBe(expected); @@ -143,6 +153,16 @@ describe("emphasis nesting", () => { input: "电话：__________（可向", expected: "电话：__________（可向", }, + { + name: "asterisk chain that opened inside a word", + input: 'Watch "M*A*S*H" for the', + expected: 'Watch "M*A*S*H" for the', + }, + { + name: "pointer in code outside a fence", + input: "int f(car *cars, int n) {\n return", + expected: "int f(car *cars, int n) {\n return", + }, { name: "asterisk between an ASCII letter and CJK", input: "使用A*算法", diff --git a/packages/remend/__tests__/lists.test.ts b/packages/remend/__tests__/lists.test.ts index 09573bf5..8f73aec9 100644 --- a/packages/remend/__tests__/lists.test.ts +++ b/packages/remend/__tests__/lists.test.ts @@ -84,8 +84,8 @@ describe("list handling", () => { }); it("should close emphasis within its own list item only", () => { - // A lazy continuation line stays in the item, but the next item starts a new block - expect(remend("- **text\nmore text")).toBe("- **text\nmore text**"); + // Bold doesn't span lines in model output, and the next item starts a new block + expect(remend("- **text\nmore text")).toBe("- **text\nmore text"); expect(remend("* **content\n* Another item")).toBe( "* **content\n* Another item" ); diff --git a/packages/remend/src/emphasis-delimiters.ts b/packages/remend/src/emphasis-delimiters.ts index 25aec281..fd6b8b24 100644 --- a/packages/remend/src/emphasis-delimiters.ts +++ b/packages/remend/src/emphasis-delimiters.ts @@ -7,6 +7,7 @@ import { isEscaped, isFenceAt, REGION, + skipContainerPrefix, type TextScan, } from "./scan"; import { isWhitespaceChar } from "./utils"; @@ -537,13 +538,18 @@ const nestsSameKind = (match: DelimiterMatch, insertAt: number): boolean => { }); }; -const lineContentPattern = /[^\s*_~>+-]/; +const lineContentPattern = /[^\s*_~]/; // Closers on a line with nothing else would turn it into a thematic break or // a fence const leavesLineBare = (base: HealBase): boolean => { const lineStart = base.text.lastIndexOf("\n", base.insertAt - 1) + 1; - return !lineContentPattern.test(base.text.slice(lineStart, base.insertAt)); + const { contentStart } = skipContainerPrefix( + base.text, + lineStart, + base.insertAt + ); + return !lineContentPattern.test(base.text.slice(contentStart, base.insertAt)); }; /** diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 4c072ddb..5ab43706 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -62,39 +62,71 @@ const isLetterOrDigit = (char: string): boolean => // An asterisk after an ASCII letter or digit opens only to continue a chain of // emphasis in that word, so hello*world, 2**3 and A*算法 stay literal while -// languages written without spaces still heal +// languages written without spaces still heal. A chain whose first pair also +// opened inside a word, as in M*A*S*H, is literal too. const opensColdInWord = ( text: string, run: MarkerRun, - previous: MarkerRun | undefined -): boolean => - run.marker === "*" && - isLetterOrDigit(text[run.index - 1] ?? "") && - letterOrDigitPattern.test(text.slice(run.index + run.length)) && - !( - previous?.marker === "*" && - [...text.slice(previous.index + previous.length, run.index)].every( - isLetterOrDigit + match: DelimiterMatch +): boolean => { + if ( + !( + run.marker === "*" && + isLetterOrDigit(text[run.index - 1] ?? "") && + letterOrDigitPattern.test(text.slice(run.index + run.length)) + ) + ) { + return false; + } + const previous = match.runs.findLast((other) => other.index < run.index); + if ( + !( + previous?.marker === "*" && + [...text.slice(previous.index + previous.length, run.index)].every( + isLetterOrDigit + ) ) + ) { + return true; + } + const closed = match.pairs.find( + (pair) => pair.closer.index === previous.index + ); + return ( + closed !== undefined && isLetterOrDigit(text[closed.opener.index - 1] ?? "") ); +}; // Writing \** usually means a literal ** with a sloppy escape, so the rest of // the run only closes if the author closes it const followsEscapedMarker = (text: string, run: MarkerRun): boolean => text[run.index - 1] === run.marker && isEscaped(text, run.index - 1); +// Authors close emphasis on the line that opens it, so an opener whose line +// ended open is usually code written outside a fence, like char *name +const contentPattern = /\S/g; + +const lineEnded = (text: string, run: MarkerRun): boolean => { + const newline = text.indexOf("\n", run.index + run.length); + if (newline === -1) { + return false; + } + contentPattern.lastIndex = newline; + return contentPattern.test(text); +}; + const isGated = ( text: string, run: MarkerRun, insertAt: number, - previous: MarkerRun | undefined + match: DelimiterMatch ): boolean => !hasContentBefore(text, run, insertAt) || // Prose uses a single tilde for "approximately" (run.marker === "~" && run.count === 1) || // Models write fill-in blanks as long underscore runs, like Name: ______ (run.marker === "_" && run.length >= 4) || - opensColdInWord(text, run, previous) || + opensColdInWord(text, run, match) || followsEscapedMarker(text, run); interface HealContext { @@ -182,10 +214,7 @@ const extendHeal = ( const { flags, insertAt, text, trailing } = context; const runs: MarkerRun[] = []; for (const run of open) { - const previous = context.match.runs.findLast( - (other) => other.index < run.index - ); - if (isGated(text, run, insertAt, previous)) { + if (isGated(text, run, insertAt, context.match)) { continue; } if (flags[flagFor(run)]) { @@ -195,7 +224,11 @@ const extendHeal = ( return start; } } - const healable = withoutDuplicates(runs); + // Openers on ended lines still count as duplicates, which mark the markers + // on later lines as literal too + const healable = withoutDuplicates(runs)?.filter( + (run) => !lineEnded(text, run) + ); if (healable === undefined) { return start; } diff --git a/packages/remend/src/inline-block.ts b/packages/remend/src/inline-block.ts index e9843580..f0551834 100644 --- a/packages/remend/src/inline-block.ts +++ b/packages/remend/src/inline-block.ts @@ -6,6 +6,8 @@ const thematicBreakPattern = const setextUnderlinePattern = /^(?:=+|-+)[ \t]*$/; // The setext handler's zero-width space after a streaming dash stands in for // the list item or thematic break that dash usually starts +// A list marker alone on a finished line is an empty item, which ends there +const emptyItemPattern = /^(?:[*+-]|\d{1,9}[.)])$/; const dashPlaceholderPattern = /^-{1,2}\u200B$/; /** @@ -37,7 +39,8 @@ export const findInlineBlockStart = (scan: TextScan): number => { // A list marker walk would read the first star of * * * as an item thematicBreakPattern.test(text.slice(lineStart, lineEnd).trim()) || thematicBreakPattern.test(content) || - (setextUnderlinePattern.test(content) && !endsBlock) + (setextUnderlinePattern.test(content) && !endsBlock) || + (lineEnd < text.length && emptyItemPattern.test(content)) ) { start = next; endsBlock = false; From 7c9dda7d4129208eced3e76f7f785699e91cc2ba Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Oct 2026 15:58:32 -0700 Subject: [PATCH 11/16] test(remend): count visible heal characters as glitches A frame that appends a marker the parser leaves literal shows the reader junk, but its authored text could still match the final render. Each added character that renders as visible text now counts as a mismatch, except the zero-width space the setext guard adds. --- packages/remend/__benchmarks__/stream-glitches.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/remend/__benchmarks__/stream-glitches.test.ts b/packages/remend/__benchmarks__/stream-glitches.test.ts index 255633d4..27ee7495 100644 --- a/packages/remend/__benchmarks__/stream-glitches.test.ts +++ b/packages/remend/__benchmarks__/stream-glitches.test.ts @@ -46,6 +46,7 @@ const LEADING_TICKS = /^`+/; const LETTER = /\p{L}/gu; const DIGIT = /\d/g; const LETTERS = /a+/g; +const ZERO_WIDTH_SPACE = "\u200B"; const parse = (markdown: string): Node => fromMarkdown(markdown, { @@ -91,7 +92,8 @@ const renderStyles = (markdown: string, root: Node, shift = 0): string[] => { const blockStarts = (root: Node): number[] => (root.children ?? []).map((child) => child.position?.start.offset ?? 0); -// Characters of the prefix whose rendering differs from the final render +// Characters of the prefix whose rendering differs from the final render, +// plus characters the heal added that render as visible text const mismatches = ( prefix: string, healed: string, @@ -109,6 +111,11 @@ const mismatches = ( from ); let count = 0; + for (let at = cut; at < cut + added; at++) { + if ((frame[at] ?? "") !== "" && healed[at] !== ZERO_WIDTH_SPACE) { + count++; + } + } for (let i = from; i < prefix.length; i++) { const at = i < cut ? i : i + added; if ((frame[at] ?? "") !== final[i]) { From 70b2c96e88532e5e1f6a05636c0df6587f16f54d Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Oct 2026 15:59:28 -0700 Subject: [PATCH 12/16] fix(remend): leave a * before a comma or semicolon literal Emphasis doesn't open on closing punctuation, so a * listed between commas, as in (+, -, *, /), is a symbol. On top of the ended-line gate this keeps 46 more WildChat frames matching the final render and moves none away from it. --- .changeset/remend-escaped-markers.md | 2 +- packages/remend/__tests__/emphasis-nesting.test.ts | 5 +++++ packages/remend/src/emphasis-handlers.ts | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index 7af3b065..385c44f4 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -5,7 +5,7 @@ Heal emphasis the way CommonMark and GFM parse it - Closes only openers that exist, innermost first, within the paragraph, list item or link text that holds them. An open code span closes before the emphasis around it. -- Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, fill-in blanks like `______`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. +- Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, `(+, -, *, /)`, fill-in blanks like `______`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. - Stops closing an opener once its line has ended and the next line has started, since models close emphasis on the line that opens it. A stray `*` in code written outside a fence, like `char *name`, no longer italicizes what follows. - Closes an inline code span only within its own block, and never with a closer that would start a fence. A `[` before a complete link stays literal instead of healing a second link. - The `bold`, `italic`, `boldItalic` and `strikethrough` options still apply, but the built-in handlers they controlled are now one `emphasis` handler at priority 30. Custom handlers ordered against the old priorities may need new ones. diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts index 40b8bdb2..6ed88ebb 100644 --- a/packages/remend/__tests__/emphasis-nesting.test.ts +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -158,6 +158,11 @@ describe("emphasis nesting", () => { input: 'Watch "M*A*S*H" for the', expected: 'Watch "M*A*S*H" for the', }, + { + name: "asterisk listed between commas", + input: "Use operators (+, -, *, /), equals, and", + expected: "Use operators (+, -, *, /), equals, and", + }, { name: "pointer in code outside a fence", input: "int f(car *cars, int n) {\n return", diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 5ab43706..e8ece220 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -97,6 +97,14 @@ const opensColdInWord = ( ); }; +// Emphasis doesn't start with closing punctuation, so the * in (+, -, *, /) +// is a symbol +const closingPunctuationPattern = /^[,;]/; + +const opensBeforeClosingPunctuation = (text: string, run: MarkerRun): boolean => + isWhitespaceChar(text[run.index - 1] ?? " ") && + closingPunctuationPattern.test(text.slice(run.index + run.length)); + // Writing \** usually means a literal ** with a sloppy escape, so the rest of // the run only closes if the author closes it const followsEscapedMarker = (text: string, run: MarkerRun): boolean => @@ -127,6 +135,7 @@ const isGated = ( // Models write fill-in blanks as long underscore runs, like Name: ______ (run.marker === "_" && run.length >= 4) || opensColdInWord(text, run, match) || + opensBeforeClosingPunctuation(text, run) || followsEscapedMarker(text, run); interface HealContext { From 200fb727fb1996d07df2c1f33af04ed402842e43 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 14 Oct 2026 18:46:05 -0700 Subject: [PATCH 13/16] fix(remend): let a lone list marker end only blocks it can interrupt An empty list item can't interrupt a paragraph, so a lone * inside one is text and a code span opened before it still closes after it. A paragraph inside a quote or list item still ends at an unindented lone marker, which micromark reads as a new list. --- packages/remend/__tests__/inline-code.test.ts | 15 ++++++++ packages/remend/src/inline-block.ts | 37 +++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/packages/remend/__tests__/inline-code.test.ts b/packages/remend/__tests__/inline-code.test.ts index 24a6fa43..d9ea2d37 100644 --- a/packages/remend/__tests__/inline-code.test.ts +++ b/packages/remend/__tests__/inline-code.test.ts @@ -113,6 +113,21 @@ describe("emphasis markers inside inline code spans should not leak", () => { input: "text `a\nb", expected: "text `a\nb`", }, + { + name: "lone star inside a paragraph", + input: "text `a\n*\nb", + expected: "text `a\n*\nb`", + }, + { + name: "lone star inside a quoted paragraph", + input: "> text `a\n> *\n> b", + expected: "> text `a\n> *\n> b`", + }, + { + name: "opener before an empty item after a quote", + input: "> text `a\n*\nb", + expected: "> text `a\n*\nb", + }, ])("closes only within the opener's block: $name", ({ input, expected }) => { expect(remend(input)).toBe(expected); expect(remend(expected)).toBe(expected); diff --git a/packages/remend/src/inline-block.ts b/packages/remend/src/inline-block.ts index f0551834..c0fd6eb6 100644 --- a/packages/remend/src/inline-block.ts +++ b/packages/remend/src/inline-block.ts @@ -1,15 +1,38 @@ -import { isFenceAt, skipContainerPrefix, type TextScan } from "./scan"; +import { + type ContainerPrefix, + isFenceAt, + skipContainerPrefix, + type TextScan, +} from "./scan"; const headingPattern = /^#{1,6}(?:[ \t]|$)/; const thematicBreakPattern = /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; const setextUnderlinePattern = /^(?:=+|-+)[ \t]*$/; +const emptyItemPattern = /^(?:[*+-]|\d{1,9}[.)])$/; // The setext handler's zero-width space after a streaming dash stands in for // the list item or thematic break that dash usually starts -// A list marker alone on a finished line is an empty item, which ends there -const emptyItemPattern = /^(?:[*+-]|\d{1,9}[.)])$/; const dashPlaceholderPattern = /^-{1,2}\u200B$/; +type Paragraph = "none" | "top" | "contained"; + +const paragraphIn = (prefix: ContainerPrefix): Paragraph => + prefix.listIndent > 0 || prefix.quoteDepth > 0 ? "contained" : "top"; + +// An empty item can't interrupt a paragraph, but a paragraph inside a quote or +// list item ends at an unindented line, which isn't lazy continuation text +const startsEmptyItem = ( + content: string, + paragraph: Paragraph, + prefix: ContainerPrefix, + lineStart: number +): boolean => + emptyItemPattern.test(content) && + (paragraph === "none" || + (paragraph === "contained" && + prefix.quoteDepth === 0 && + prefix.contentStart === lineStart)); + /** * Index where the text's last inline block starts, since emphasis never * pairs across paragraphs, list items, headings, table rows or code blocks @@ -19,6 +42,7 @@ export const findInlineBlockStart = (scan: TextScan): number => { let start = 0; let quoteDepth = 0; let endsBlock = false; + let paragraph: Paragraph = "none"; for (let lineStart = 0; lineStart <= text.length; ) { let lineEnd = text.indexOf("\n", lineStart); @@ -40,10 +64,12 @@ export const findInlineBlockStart = (scan: TextScan): number => { thematicBreakPattern.test(text.slice(lineStart, lineEnd).trim()) || thematicBreakPattern.test(content) || (setextUnderlinePattern.test(content) && !endsBlock) || - (lineEnd < text.length && emptyItemPattern.test(content)) + (lineEnd < text.length && + startsEmptyItem(content, paragraph, prefix, lineStart)) ) { start = next; endsBlock = false; + paragraph = "none"; } else if ( endsBlock || prefix.listIndent > 0 || @@ -54,6 +80,9 @@ export const findInlineBlockStart = (scan: TextScan): number => { ) { start = prefix.contentStart; endsBlock = headingPattern.test(content) || content.startsWith("|"); + paragraph = endsBlock ? "none" : paragraphIn(prefix); + } else if (paragraph === "none") { + paragraph = paragraphIn(prefix); } quoteDepth = prefix.quoteDepth; lineStart = next; From 0f9e19d8fa6a02cd626eebc6f4a92ee0c971ad8b Mon Sep 17 00:00:00 2001 From: Nishil Faldu Date: Mon, 19 Oct 2026 01:19:49 -0700 Subject: [PATCH 14/16] fix(remend): complete links only in the current block Version of #685 on top of #670. Dropped when this branch is rebased onto main. --- .changeset/remend-link-block-scope.md | 5 ++ .../remend/__tests__/link-block-scope.test.ts | 60 +++++++++++++++++++ packages/remend/src/link-image-handler.ts | 48 +++++++++++---- 3 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 .changeset/remend-link-block-scope.md create mode 100644 packages/remend/__tests__/link-block-scope.test.ts diff --git a/.changeset/remend-link-block-scope.md b/.changeset/remend-link-block-scope.md new file mode 100644 index 00000000..fd0b372b --- /dev/null +++ b/.changeset/remend-link-block-scope.md @@ -0,0 +1,5 @@ +--- +"remend": patch +--- + +Complete unfinished links and images only in the current block: the paragraph, list item, heading or table row at the end of the text. An unclosed `[` or `](` in an earlier block no longer adds `](streamdown:incomplete-link)` to later text, or drops the text after it. diff --git a/packages/remend/__tests__/link-block-scope.test.ts b/packages/remend/__tests__/link-block-scope.test.ts new file mode 100644 index 00000000..8dfaf0a5 --- /dev/null +++ b/packages/remend/__tests__/link-block-scope.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("link healing stays in the last inline block", () => { + it("does not close a bracket from an earlier paragraph", () => { + expect(remend("See [1 for details.\n\nThe next paragraph")).toBe( + "See [1 for details.\n\nThe next paragraph" + ); + expect(remend("Arrays use x[i notation.\n\nThen we")).toBe( + "Arrays use x[i notation.\n\nThen we" + ); + }); + + it("keeps later paragraphs after an earlier unfinished URL", () => { + const text = + "Read [the docs](https://example.com/guide\n\nNext paragraph keeps streaming"; + expect(remend(text)).toBe(text); + }); + + it('does not pair a later "](" with a bracket from an earlier block', () => { + const text = "Text [a\n\nb](https://exa"; + expect(remend(text)).toBe(text); + }); + + it("does not move an image's later paragraphs into its alt text", () => { + expect(remend("![diagram\n\nThe caption follows")).toBe( + "![diagram\n\nThe caption follows" + ); + }); + + it("treats list items, headings and table rows as separate blocks", () => { + expect(remend("- item [one\n- item two")).toBe("- item [one\n- item two"); + expect(remend("# Title [x\nBody text")).toBe("# Title [x\nBody text"); + expect(remend("| a [b | c |\n| d | e")).toBe("| a [b | c |\n| d | e"); + }); + + it("still heals an incomplete link in the current block", () => { + expect(remend("Done.\n\nSee [the docs")).toBe( + "Done.\n\nSee [the docs](streamdown:incomplete-link)" + ); + expect(remend("See [1 here.\n\nRead [the docs](https://exa")).toBe( + "See [1 here.\n\nRead [the docs](streamdown:incomplete-link)" + ); + expect(remend("- item [one\n- item [two")).toBe( + "- item [one\n- item [two](streamdown:incomplete-link)" + ); + expect(remend("> quote [link\n> continues")).toBe( + "> quote [link\n> continues](streamdown:incomplete-link)" + ); + expect(remend("- item [link\n continues")).toBe( + "- item [link\n continues](streamdown:incomplete-link)" + ); + }); + + it("leaves earlier brackets alone in text-only mode", () => { + expect( + remend("See [1 for details.\n\nRead [the docs", { linkMode: "text-only" }) + ).toBe("See [1 for details.\n\nRead the docs"); + }); +}); diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 4bd4d5e0..dc86e20c 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -1,4 +1,5 @@ import { isInsideCodeBlock } from "./code-block-utils"; +import { findInlineBlockStart } from "./inline-block"; import { handleIncompleteInlineCode } from "./inline-code-handler"; import { getScan, isCompleteSpanAt, isFenceAt } from "./scan"; import { @@ -18,7 +19,8 @@ export const INCOMPLETE_IMAGE_PLACEHOLDER = "streamdown:incomplete-image"; const handleIncompleteUrl = ( text: string, lastParenIndex: number, - linkMode: LinkMode + linkMode: LinkMode, + blockStart: number ): string | null => { const afterParen = text.substring(lastParenIndex + 2); if (afterParen.includes(")")) { @@ -29,7 +31,11 @@ const handleIncompleteUrl = ( // Now find the matching opening bracket for the ] before ( const openBracketIndex = findMatchingOpeningBracket(text, lastParenIndex); - if (openBracketIndex === -1 || isInsideCodeBlock(text, openBracketIndex)) { + // The link text must start in the same block as its "](" + if ( + openBracketIndex < blockStart || + isInsideCodeBlock(text, openBracketIndex) + ) { return null; } @@ -57,8 +63,12 @@ const handleIncompleteUrl = ( // Helper to find the first incomplete [ (for text-only mode) // Always returns a valid index since callers guarantee text[maxPos] is an incomplete [ -const findFirstIncompleteBracket = (text: string, maxPos: number): number => { - for (let j = 0; j < maxPos; j++) { +const findFirstIncompleteBracket = ( + text: string, + maxPos: number, + blockStart: number +): number => { + for (let j = blockStart; j < maxPos; j++) { if (text[j] === "[" && !isInsideCodeBlock(text, j)) { // Skip if it's an image if (j > 0 && text[j - 1] === "!") { @@ -88,7 +98,8 @@ const findFirstIncompleteBracket = (text: string, maxPos: number): number => { const handleIncompleteText = ( text: string, i: number, - linkMode: LinkMode + linkMode: LinkMode, + blockStart: number ): string | null => { // Check if there's a ! before it const isImage = i > 0 && text[i - 1] === "!"; @@ -109,7 +120,7 @@ const handleIncompleteText = ( // For links, handle based on linkMode if (linkMode === "text-only") { // Find the first incomplete [ and strip just that bracket - const firstIncomplete = findFirstIncompleteBracket(text, i); + const firstIncomplete = findFirstIncompleteBracket(text, i, blockStart); return ( text.substring(0, firstIncomplete) + text.substring(firstIncomplete + 1) ); @@ -133,7 +144,7 @@ const handleIncompleteText = ( if (linkMode === "text-only") { // Find the first incomplete [ and strip just that bracket - const firstIncomplete = findFirstIncompleteBracket(text, i); + const firstIncomplete = findFirstIncompleteBracket(text, i, blockStart); return ( text.substring(0, firstIncomplete) + text.substring(firstIncomplete + 1) ); @@ -158,10 +169,24 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Look for patterns like [text]( or ![text]( at the end of text // We need to handle nested brackets in the link text + // Link text cannot cross blocks, so brackets before the last inline block + // are literal. Healing them would append a closer to the wrong block, or + // drop every block after them. + const scan = getScan(text); + const blockStart = findInlineBlockStart(scan); + // Start from the end and look for ]( pattern const lastParenIndex = text.lastIndexOf("]("); - if (lastParenIndex !== -1 && !isInsideCodeBlock(text, lastParenIndex)) { - const result = handleIncompleteUrl(text, lastParenIndex, linkMode); + if ( + lastParenIndex >= blockStart && + !isInsideCodeBlock(text, lastParenIndex) + ) { + const result = handleIncompleteUrl( + text, + lastParenIndex, + linkMode, + blockStart + ); if (result !== null) { return result; } @@ -169,11 +194,10 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Then check for incomplete link text: [partial-text without closing ] // Search backwards for an opening bracket that doesn't have a matching closing bracket - const scan = getScan(text); let enclosesLink = false; for ( let i = text.lastIndexOf("["); - i !== -1; + i >= blockStart; i = i === 0 ? -1 : text.lastIndexOf("[", i - 1) ) { // An unclosed backtick renders as text, so a link after one still counts @@ -187,7 +211,7 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { const literal = isLink && enclosesLink; enclosesLink ||= closesHere; if (!(closesHere || literal || isInsideCodeBlock(text, i))) { - const result = handleIncompleteText(text, i, linkMode); + const result = handleIncompleteText(text, i, linkMode, blockStart); if (result !== null) { return result; } From 43c6616e69aefcaab0285c700d8cbe844a59c4b4 Mon Sep 17 00:00:00 2001 From: Nishil Faldu Date: Mon, 19 Oct 2026 01:22:04 -0700 Subject: [PATCH 15/16] perf(streamdown): repair only the end of a streaming message On every streaming update, Streamdown ran remend over the whole message and ran three regex checks (footnote references, footnote definitions, link definitions) over the whole message. The work per update grew with the length of the answer. Each message now keeps a stable length: the end of every block except the last two, taken from an update where remend changed nothing. Later updates repair only the text after it. The three checks keep their previous result and scan only where a new match can end. The output equals a full repair because remend now completes syntax only in the current block (#670 for emphasis, #685 for links). --- .changeset/streamdown-tail-repair.md | 5 + .../__tests__/stream-repair.test.ts | 162 ++++++++++++++++++ packages/streamdown/index.tsx | 44 ++++- packages/streamdown/lib/parse-blocks.tsx | 84 ++++++++- packages/streamdown/lib/stream-repair.ts | 58 +++++++ 5 files changed, 340 insertions(+), 13 deletions(-) create mode 100644 .changeset/streamdown-tail-repair.md create mode 100644 packages/streamdown/__tests__/stream-repair.test.ts create mode 100644 packages/streamdown/lib/stream-repair.ts diff --git a/.changeset/streamdown-tail-repair.md b/.changeset/streamdown-tail-repair.md new file mode 100644 index 00000000..304b699c --- /dev/null +++ b/.changeset/streamdown-tail-repair.md @@ -0,0 +1,5 @@ +--- +"streamdown": patch +--- + +While streaming, repair only the end of the message instead of the whole message on every update. The footnote and link-definition checks now scan only the new text. Rendered output is unchanged. diff --git a/packages/streamdown/__tests__/stream-repair.test.ts b/packages/streamdown/__tests__/stream-repair.test.ts new file mode 100644 index 00000000..b7374733 --- /dev/null +++ b/packages/streamdown/__tests__/stream-repair.test.ts @@ -0,0 +1,162 @@ +import remend from "remend"; +import { describe, expect, it } from "vitest"; +import { + hasFootnoteDefinition, + hasFootnoteReference, + hasLinkDefinition, + parseMarkdownIntoBlocks, +} from "../lib/parse-blocks"; +import { + advanceStableLength, + repairStream, + type StreamRepairState, +} from "../lib/stream-repair"; + +// The full-document patterns that the incremental checks replace. +const footnoteReferencePattern = /\[\^[\w-]{1,200}\](?!:)/; +const footnoteDefinitionPattern = /\[\^[\w-]{1,200}\]:/; +const linkDefinitionPattern = /^ {0,3}\[[^\]]+\]:/m; + +// Streamed as Streamdown does: repair, split into blocks, advance. Every +// repaired prefix must equal remend over the whole prefix. +const documents = [ + "# Title\n\nFirst paragraph with **bold** and *italic*.\n\nSecond with a [link](https://example.com) and `code`.\n\nThird **unfinished", + "Intro\n\n- one **a**\n- two [b](/b)\n\n continued\n\n1. first\n2. second *em*\n\nAfter the list `x`\n", + "| a | b |\n| --- | --- |\n| **x** | [y](/y) |\n| 3 | 4 |\n\nText after the table with ~~strike~~.\n", + "Before\n\n```ts\nconst a = '**';\n\nconst b = `x`;\n```\n\nAfter the fence with **bold\n\nand more", + "Math $x$ and\n\n$$\na = b\n\n= c\n$$\n\nThen `inline` and **strong**.\n", + "> quote with *em*\n> more\n\nSetext\n===\n\npara\n---\n\nend [unfinished link", + "Call foo(**kwargs) to forward.\n\nGlob src/**/test.js.\n\nThe next paragraph *streams* on.\n", + "See [ref] here.\n\n[ref]: https://example.com\n\nFootnote[^1] text.\n\n[^1]: note\n", + // A closer appended at the end must not make an earlier open block final. + "Before\n\n$$\nx = 1\n\ny **2\n\nstill math\n$$\n\nafter *em*\n", + "Text [a\n\nb](https://exa\n\nc **d\n\ne", +]; + +const chunked = (text: string, seed: number): string[] => { + const chunks: string[] = []; + let state = seed; + for (let index = 0; index < text.length; ) { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + const size = 1 + (state % 4); + chunks.push(text.slice(index, index + size)); + index += size; + } + return chunks; +}; + +// Returns each input whose tail repair differs from a full repair. +const stream = (inputs: string[]) => { + let state: StreamRepairState | null = null; + const mismatches: string[] = []; + for (const input of inputs) { + state = repairStream(state, input, undefined); + if (state.repaired !== remend(input)) { + mismatches.push(input); + } + advanceStableLength(state, parseMarkdownIntoBlocks(state.repaired)); + } + return { state, mismatches }; +}; + +describe("tail-bounded stream repair", () => { + it("matches a full repair at every prefix", () => { + for (const document of documents) { + for (const seed of [1, 2, 3]) { + let text = ""; + const inputs = chunked(document, seed).map((chunk) => { + text += chunk; + return text; + }); + expect(stream(inputs).mismatches).toEqual([]); + } + } + }); + + it("skips the settled prefix once later blocks exist", () => { + const { state } = stream([ + "One.\n\nTwo.\n\nThree.\n\nFour", + "One.\n\nTwo.\n\nThree.\n\nFour and", + ]); + expect(state?.stableLength).toBeGreaterThan(0); + }); + + it("repairs the whole input after a rewrite, CR input, or new options", () => { + let { state } = stream(["One.\n\nTwo.\n\nThree.\n\nFour **x"]); + state = repairStream(state, "Different **text", undefined); + expect(state.stableLength).toBe(0); + expect(state.repaired).toBe(remend("Different **text")); + + state = repairStream(state, "Different **text\r\nmore", undefined); + expect(state.stableLength).toBe(0); + + const options = { links: false }; + state = repairStream(state, "Different **text\r\nmore [x", options); + expect(state.repaired).toBe(remend("Different **text\r\nmore [x", options)); + }); +}); + +describe("incremental footnote and link-definition checks", () => { + // Parsing an unrelated document first resets each check's memo, so the + // second call is a full check of that prefix. + const fresh = (input: string): string[] => { + parseMarkdownIntoBlocks("unrelated reset document\n"); + return parseMarkdownIntoBlocks(input); + }; + + it("matches a full check at every prefix", () => { + const label = "x".repeat(200); + const inputs = [ + "Intro paragraph.\n\nSee note [^1] here.\n\nMore text.\n", + "Intro.\n\nText [^a]: is a definition.\n\nMore.\n", + `Intro.\n\nLong [^${label}] label.\n\nMore [^${label}x] too long.\n`, + "Intro [x](/u).\n\nThen\n[ref]: https://example.com\n\nafter\n", + "a ] b\n\n[multi\nline]: /u\n\nend\n", + "> [q]: /quoted\n\npara\n", + ]; + for (const document of inputs) { + for (const seed of [1, 2]) { + let text = ""; + for (const chunk of chunked(document, seed)) { + text += chunk; + const expected = fresh(text); + parseMarkdownIntoBlocks(text.slice(0, -chunk.length) || " "); + expect(parseMarkdownIntoBlocks(text)).toEqual(expected); + } + } + } + }); + + it("gives each check the same answer as its full regex", () => { + const checks = [ + [hasFootnoteReference, footnoteReferencePattern], + [hasFootnoteDefinition, footnoteDefinitionPattern], + [hasLinkDefinition, linkDefinitionPattern], + ] as const; + const pieces = [ + "[", + "]", + "^", + ":", + "-", + "a", + " ", + "\n", + "\n\n", + "[^n]", + "]:", + "[x]: /u", + ]; + let state = 7717; + for (let run = 0; run < 300; run += 1) { + let text = ""; + for (let step = 0; step < 80; step += 1) { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + text += pieces[state % pieces.length]; + for (const [check, pattern] of checks) { + expect(check(text)).toBe(pattern.test(text)); + } + } + } + }); +}); diff --git a/packages/streamdown/index.tsx b/packages/streamdown/index.tsx index e25fe043..5de1f73b 100644 --- a/packages/streamdown/index.tsx +++ b/packages/streamdown/index.tsx @@ -53,6 +53,11 @@ import { rehypeBlockDirection } from "./lib/rehype/block-direction"; import { rehypeLiteralTagContent } from "./lib/rehype/literal-tag-content"; import { remarkCodeMeta } from "./lib/remark/code-meta"; import { remarkDisableAutolinkProtocols } from "./lib/remark/disable-autolink-protocols"; +import { + advanceStableLength, + repairStream, + type StreamRepairState, +} from "./lib/stream-repair"; import type { CSVSeparator } from "./lib/table/utils"; import { defaultTranslations, @@ -728,14 +733,32 @@ export const Streamdown = memo( // Apply remend to fix incomplete markdown BEFORE parsing into blocks // This prevents partial list items from being interpreted as setext headings + // Tail-bounded repair state for this message. It is checked against each + // input, so a render React discards cannot affect a later one. + const streamRepairRef = useRef(null); const processedChildren = useMemo(() => { if (typeof children !== "string") { return ""; } - let result = - mode === "streaming" && shouldParseIncompleteMarkdown - ? remend(children, remendOptions) - : children; + let result = children; + if (mode === "streaming" && shouldParseIncompleteMarkdown) { + // Block offsets come from the default parser and must match the + // repaired text, so custom parsers and tag preprocessing repair whole. + const canRepairTail = + parseMarkdownIntoBlocksFn === parseMarkdownIntoBlocks && + !literalTagContent?.length && + allowedTagNames.length === 0; + if (canRepairTail) { + streamRepairRef.current = repairStream( + streamRepairRef.current, + children, + remendOptions + ); + result = streamRepairRef.current.repaired; + } else { + result = remend(children, remendOptions); + } + } // Escape markdown metacharacters inside literal-tag-content tags so that // children are rendered as plain text rather than parsed as markdown. @@ -760,12 +783,17 @@ export const Streamdown = memo( remendOptions, allowedTagNames, literalTagContent, + parseMarkdownIntoBlocksFn, ]); - const blocks = useMemo( - () => parseMarkdownIntoBlocksFn(processedChildren), - [processedChildren, parseMarkdownIntoBlocksFn] - ); + const blocks = useMemo(() => { + const parsed = parseMarkdownIntoBlocksFn(processedChildren); + const repair = streamRepairRef.current; + if (repair?.repaired === processedChildren) { + advanceStableLength(repair, parsed); + } + return parsed; + }, [processedChildren, parseMarkdownIntoBlocksFn]); // Render blocks directly. The previous displayBlocks + useTransition path // could be starved by sibling urgent updates when animated was off (#550), diff --git a/packages/streamdown/lib/parse-blocks.tsx b/packages/streamdown/lib/parse-blocks.tsx index 69029387..6ace83c2 100644 --- a/packages/streamdown/lib/parse-blocks.tsx +++ b/packages/streamdown/lib/parse-blocks.tsx @@ -3,7 +3,6 @@ import { Lexer, type Token } from "marked"; // Regex patterns moved to top level for performance // Footnote identifiers must be alphanumeric, underscore, or hyphen (e.g., [^1], [^note], [^my-note]) // Previously used [^\]\s] which incorrectly matched regex character classes like [^\s...] -const footnoteReferencePattern = /\[\^[\w-]{1,200}\](?!:)/; const footnoteDefinitionPattern = /\[\^[\w-]{1,200}\]:/; // Allow hyphens / colons so custom tags like are tracked across // blank-line interruptions (\w alone only matches [A-Za-z0-9_]). @@ -14,6 +13,81 @@ const openingTagPattern = /<([A-Za-z][\w:-]*)[\s>/]/; // leading spaces before a definition (4+ is an indented code block). const linkDefinitionPattern = /^ {0,3}\[[^\]]+\]:/m; +// These checks run on the whole document for every streaming update. Each +// keeps the previous input and its result; when the new input extends it, +// only the region where a new match could end is tested again. +interface ScanMemo { + found: boolean; + input: string; +} + +const extendsMemo = (memo: ScanMemo | null, input: string): memo is ScanMemo => + memo !== null && + input.length >= memo.input.length && + input.startsWith(memo.input); + +// The longest footnote match: "[^", a 200-character label, "]", one more. +const footnoteWindow = 205; +// A footnote reference is `[^label]` not followed by ":", that is +// /\[\^[\w-]{1,200}\](?!:)/. Its `(?!:)` can drop a match at the very end of +// the input once ":" arrives. +// A match followed by a known non-colon character is permanent, so track +// those and test the end of the input separately. +const followedFootnoteReferencePattern = /\[\^[\w-]{1,200}\][^:]/; +const trailingFootnoteReferencePattern = /\[\^[\w-]{1,200}\]$/; + +let footnoteReferenceMemo: ScanMemo | null = null; +let footnoteDefinitionMemo: ScanMemo | null = null; +let linkDefinitionMemo: ScanMemo | null = null; + +const footnoteTail = (memo: ScanMemo, input: string): string => + input.slice(Math.max(0, memo.input.length - footnoteWindow)); + +export const hasFootnoteReference = (input: string): boolean => { + const memo = footnoteReferenceMemo; + const followed = extendsMemo(memo, input) + ? memo.found || + followedFootnoteReferencePattern.test(footnoteTail(memo, input)) + : followedFootnoteReferencePattern.test(input); + footnoteReferenceMemo = { input, found: followed }; + return ( + followed || + trailingFootnoteReferencePattern.test(input.slice(-footnoteWindow)) + ); +}; + +export const hasFootnoteDefinition = (input: string): boolean => { + const memo = footnoteDefinitionMemo; + const found = extendsMemo(memo, input) + ? memo.found || footnoteDefinitionPattern.test(footnoteTail(memo, input)) + : footnoteDefinitionPattern.test(input); + footnoteDefinitionMemo = { input, found }; + return found; +}; + +export const hasLinkDefinition = (input: string): boolean => { + const memo = linkDefinitionMemo; + let found: boolean; + if (!extendsMemo(memo, input)) { + found = linkDefinitionPattern.test(input); + } else if (memo.found) { + found = true; + } else { + // A new match ends after the previous input, and its label holds no "]", + // so it starts on the line after the previous input's last "]" (ignoring + // its final character, which a new ":" can follow). + const previousLength = memo.input.length; + const bracket = + previousLength >= 2 + ? memo.input.lastIndexOf("]", previousLength - 2) + : -1; + const start = bracket < 0 ? 0 : input.lastIndexOf("\n", bracket) + 1; + found = linkDefinitionPattern.test(input.slice(start)); + } + linkDefinitionMemo = { input, found }; + return found; +}; + // HTML void elements (self-closing tags) that don't need closing tags const voidElements = new Set([ "area", @@ -268,7 +342,7 @@ const reuseParsedBlocks = ( // compared on every tick. if ( input.slice(0, previous.input.length) !== previous.input || - linkDefinitionPattern.test(input) + hasLinkDefinition(input) ) { return null; } @@ -327,14 +401,14 @@ export const parseMarkdownIntoBlocks = (markdown: string): string[] => { // Footnote references: [^1], [^label], etc. // Footnote definitions: [^1]: text, [^label]: text, etc. // Use atomic groups or possessive quantifiers to prevent backtracking - const hasFootnoteReference = footnoteReferencePattern.test(markdown); - const hasFootnoteDefinition = footnoteDefinitionPattern.test(markdown); + const markdownHasFootnoteReference = hasFootnoteReference(markdown); + const markdownHasFootnoteDefinition = hasFootnoteDefinition(markdown); // If footnotes are present, return the entire document as a single block // This ensures footnote references and definitions remain in the same mdast tree. // Do not write the cache: footnote documents are not block-split, and leaving // a prior entry avoids poisoning the next non-footnote stream extension check. - if (hasFootnoteReference || hasFootnoteDefinition) { + if (markdownHasFootnoteReference || markdownHasFootnoteDefinition) { return [markdown]; } diff --git a/packages/streamdown/lib/stream-repair.ts b/packages/streamdown/lib/stream-repair.ts new file mode 100644 index 00000000..094fc4aa --- /dev/null +++ b/packages/streamdown/lib/stream-repair.ts @@ -0,0 +1,58 @@ +import remend, { type RemendOptions } from "remend"; + +// Streaming appends text, and the blocks before the last two cannot change +// once a later block exists. Remend only completes syntax in the last inline +// block, so repairing the text after those blocks gives the same string as +// repairing the whole message, without rescanning the message on every update. +export interface StreamRepairState { + input: string; + options: RemendOptions | undefined; + repaired: string; + stableLength: number; +} + +// Mutable blocks at the end of the document. parseMarkdownIntoBlocks reuses +// the same window for its own cache. +const MUTABLE_BLOCKS = 2; + +export const repairStream = ( + previous: StreamRepairState | null, + input: string, + options: RemendOptions | undefined +): StreamRepairState => { + // parseMarkdownIntoBlocks normalizes CR line endings, which shifts block + // offsets, so repair those inputs whole. + const extendsPrevious = + previous !== null && + previous.options === options && + !input.includes("\r") && + input.length >= previous.input.length && + input.startsWith(previous.input); + const stableLength = extendsPrevious ? previous.stableLength : 0; + const repaired = + stableLength > 0 + ? input.slice(0, stableLength) + + remend(input.slice(stableLength), options) + : remend(input, options); + return { input, options, repaired, stableLength }; +}; + +// Called with the blocks parsed from state.repaired. Moves the stable boundary +// past every block but the last two. Only an unchanged repair is used: then +// the blocks are those of the stream itself, while a closer appended at the +// end (such as "$$") could make an open block look finished. +export const advanceStableLength = ( + state: StreamRepairState, + blocks: string[] +): void => { + if (state.repaired !== state.input || state.input.includes("\r")) { + return; + } + let next = 0; + for (let i = 0; i < blocks.length - MUTABLE_BLOCKS; i += 1) { + next += blocks[i].length; + } + if (next > state.stableLength) { + state.stableLength = next; + } +}; From a785125d9e691c15c323cbc24e72fdc41106edfb Mon Sep 17 00:00:00 2001 From: Nishil Faldu Date: Mon, 19 Oct 2026 14:06:54 -0700 Subject: [PATCH 16/16] fix(streamdown): repair the whole message where remend reads all of it Remend tracks math across the whole message: the count of `$` (with inlineKatex), the count of `$$`, the math state from `$`, `\(` and `\[`, and the first `$$` it finds, even inside code. So a `$` before the stable length could change how the end is repaired. The stable length now stops before the first block with math. A fenced code block counts as math only when it holds `$$`. Custom remend handlers can read the whole message, so with them the whole message is repaired, as before. --- .../__tests__/stream-repair.test.ts | 101 ++++++++++++++++-- packages/streamdown/lib/stream-repair.ts | 39 +++++-- 2 files changed, 120 insertions(+), 20 deletions(-) diff --git a/packages/streamdown/__tests__/stream-repair.test.ts b/packages/streamdown/__tests__/stream-repair.test.ts index b7374733..40f184d4 100644 --- a/packages/streamdown/__tests__/stream-repair.test.ts +++ b/packages/streamdown/__tests__/stream-repair.test.ts @@ -1,4 +1,4 @@ -import remend from "remend"; +import remend, { type RemendOptions } from "remend"; import { describe, expect, it } from "vitest"; import { hasFootnoteDefinition, @@ -31,6 +31,12 @@ const documents = [ // A closer appended at the end must not make an earlier open block final. "Before\n\n$$\nx = 1\n\ny **2\n\nstill math\n$$\n\nafter *em*\n", "Text [a\n\nb](https://exa\n\nc **d\n\ne", + // Remend counts math delimiters across the whole message. + "$$\nx\n$$\n\nA\n\nB\n\nThen $$y$$ inline.\n", + "a `$$` b\n\nc $$ d\n\nE\n\nF\n\ng $$ h $$ i\n", + "A costs $5.\n\nB\n\nC costs $6.\n\nD goes on.\n", + "Inline \\(a\n\nB\n\nC\n\nD **e\n", + "```sh\n$ npm i\n```\n\nA\n\nB\n\nC $x$ and $$y\n", ]; const chunked = (text: string, seed: number): string[] => { @@ -45,13 +51,21 @@ const chunked = (text: string, seed: number): string[] => { return chunks; }; +const prefixes = (document: string, seed: number): string[] => { + let text = ""; + return chunked(document, seed).map((chunk) => { + text += chunk; + return text; + }); +}; + // Returns each input whose tail repair differs from a full repair. -const stream = (inputs: string[]) => { +const stream = (inputs: string[], options?: RemendOptions) => { let state: StreamRepairState | null = null; const mismatches: string[] = []; for (const input of inputs) { - state = repairStream(state, input, undefined); - if (state.repaired !== remend(input)) { + state = repairStream(state, input, options); + if (state.repaired !== remend(input, options)) { mismatches.push(input); } advanceStableLength(state, parseMarkdownIntoBlocks(state.repaired)); @@ -61,18 +75,83 @@ const stream = (inputs: string[]) => { describe("tail-bounded stream repair", () => { it("matches a full repair at every prefix", () => { - for (const document of documents) { - for (const seed of [1, 2, 3]) { + for (const options of [undefined, { inlineKatex: true }]) { + for (const document of documents) { + for (const seed of [1, 2, 3]) { + expect(stream(prefixes(document, seed), options).mismatches).toEqual( + [] + ); + } + } + } + }); + + it("matches a full repair on random math, code and emphasis", () => { + const pieces = [ + "$", + "$$", + "\\(", + "\\)", + "\\[", + "`", + "```\n", + "~~~\n", + "**", + "_", + "[", + "](", + "word ", + "word ", + "word ", + "word.\n\n", + "word.\n\n", + "\n", + ]; + let state = 4242; + for (const options of [undefined, { inlineKatex: true }]) { + for (let run = 0; run < 300; run += 1) { let text = ""; - const inputs = chunked(document, seed).map((chunk) => { - text += chunk; - return text; - }); - expect(stream(inputs).mismatches).toEqual([]); + const inputs: string[] = []; + for (let step = 0; step < 80; step += 1) { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + text += pieces[state % pieces.length]; + inputs.push(text); + } + expect(stream(inputs, options).mismatches).toEqual([]); } } }); + it("stops the settled prefix before math outside fenced code", () => { + const math = stream([ + "One $x$.\n\nTwo.\n\nThree.\n\nFour", + "One $x$.\n\nTwo.\n\nThree.\n\nFour and", + ]); + expect(math.state?.stableLength).toBe(0); + + const fenced = stream([ + "```sh\n$ ls\n```\n\nTwo.\n\nThree.\n\nFour", + "```sh\n$ ls\n```\n\nTwo.\n\nThree.\n\nFour and", + ]); + expect(fenced.state?.stableLength).toBeGreaterThan(0); + }); + + it("repairs the whole message with custom handlers", () => { + // Appends "!" when the message has an even number of blocks. + const options: RemendOptions = { + handlers: [ + { + name: "parity", + handle: (text) => (text.split("\n\n").length % 2 ? text : `${text}!`), + }, + ], + }; + const inputs = prefixes("One.\n\nTwo.\n\nThree.\n\nFour.\n\nFive.\n", 1); + const { state, mismatches } = stream(inputs, options); + expect(mismatches).toEqual([]); + expect(state?.stableLength).toBe(0); + }); + it("skips the settled prefix once later blocks exist", () => { const { state } = stream([ "One.\n\nTwo.\n\nThree.\n\nFour", diff --git a/packages/streamdown/lib/stream-repair.ts b/packages/streamdown/lib/stream-repair.ts index 094fc4aa..184a6b51 100644 --- a/packages/streamdown/lib/stream-repair.ts +++ b/packages/streamdown/lib/stream-repair.ts @@ -15,6 +15,17 @@ export interface StreamRepairState { // the same window for its own cache. const MUTABLE_BLOCKS = 2; +// Remend tracks math ($, $$, \( and \[) across the whole message, so text +// after a math delimiter can repair differently without the text before it. +// Delimiters inside fenced code are literal, except "$$": remend finds the +// first "$$" in the message to choose how to close an open one. +const mathDelimiterPattern = /\$|\\[([]/; +const fencedBlockPattern = /^ {0,3}(([`~])\2{2,})[\s\S]*\n {0,3}\1\2*\s*$/; + +const hasMathDelimiter = (block: string): boolean => + block.includes("$$") || + (mathDelimiterPattern.test(block) && !fencedBlockPattern.test(block)); + export const repairStream = ( previous: StreamRepairState | null, input: string, @@ -38,21 +49,31 @@ export const repairStream = ( }; // Called with the blocks parsed from state.repaired. Moves the stable boundary -// past every block but the last two. Only an unchanged repair is used: then -// the blocks are those of the stream itself, while a closer appended at the -// end (such as "$$") could make an open block look finished. +// past every block but the last two, and stops before the first block with +// math. Only an unchanged repair is used: then the blocks are those of the +// stream itself, while a closer appended at the end (such as "$$") could make +// an open block look finished. Custom handlers may read the whole message, so +// with them the whole message is always repaired. export const advanceStableLength = ( state: StreamRepairState, blocks: string[] ): void => { - if (state.repaired !== state.input || state.input.includes("\r")) { + if ( + state.repaired !== state.input || + state.input.includes("\r") || + state.options?.handlers?.length + ) { return; } - let next = 0; + let end = 0; for (let i = 0; i < blocks.length - MUTABLE_BLOCKS; i += 1) { - next += blocks[i].length; - } - if (next > state.stableLength) { - state.stableLength = next; + end += blocks[i].length; + if (end <= state.stableLength) { + continue; + } + if (hasMathDelimiter(blocks[i])) { + return; + } + state.stableLength = end; } };