From 49b2efffc3cbb395aa2a9a898f9805e77c9ce58f Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 7 Oct 2026 14:27:23 -0700 Subject: [PATCH 1/2] fix(remend): stop appending emphasis closers that match no opener Underscore emphasis was healed by counting delimiter parity, which cannot see flanking, the rule of 3, or partial matches between runs of different lengths. Inputs like `a __ b \__c`, `__ foo`, `a__.`, and `__a____.a` gained a closer that renders as literal text, while `a __ b __bold` lost one it needed. A new module matches _ and __ runs with the CommonMark emphasis algorithm, using remaining run lengths for the rule of 3. The underscore handlers close the innermost opener it leaves open, and keep a heal only if every underscore it added matched. Escape checks for *, _, **, ~~, and backticks now look for an odd run of backslashes, so a marker after an escaped backslash still opens. A trailing unescaped backslash would escape any appended closer, so the text is healed with that backslash escaped. A fast-check property generates marker-dense text and asserts, against mdast-util-from-markdown, that every appended closer matches an opener. --- .changeset/remend-escaped-markers.md | 9 + .../broken-markdown-variants.test.ts | 11 +- .../remend/__tests__/escaped-markers.test.ts | 80 ++++++ packages/remend/__tests__/italic.test.ts | 3 +- packages/remend/__tests__/lists.test.ts | 4 +- .../__tests__/streaming-properties.test.ts | 77 ++++++ .../remend/__tests__/underscore-runs.test.ts | 23 ++ packages/remend/src/emphasis-handlers.ts | 227 +++--------------- packages/remend/src/index.ts | 42 +++- packages/remend/src/scan.ts | 13 +- packages/remend/src/underscore-delimiters.ts | 178 ++++++++++++++ packages/remend/src/utils.ts | 5 + 12 files changed, 454 insertions(+), 218 deletions(-) create mode 100644 .changeset/remend-escaped-markers.md create mode 100644 packages/remend/__tests__/escaped-markers.test.ts create mode 100644 packages/remend/src/underscore-delimiters.ts diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md new file mode 100644 index 00000000..ceb64c00 --- /dev/null +++ b/.changeset/remend-escaped-markers.md @@ -0,0 +1,9 @@ +--- +"remend": patch +--- + +fix(remend): stop appending emphasis closers that match no opener + +- Match `_` and `__` delimiters with the CommonMark emphasis algorithm (flanking, rule of 3, partial matches) instead of counting parity, and drop an underscore closer that would not match +- Treat a marker as escaped only after an odd run of backslashes, so `\\*`, `\\_`, `\\**`, `\\~~`, and `` \\` `` still open +- Escape a trailing backslash before appending closers, so it cannot escape them diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index b483fddc..6fa64cae 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -77,17 +77,14 @@ describe("backslash escapes with incomplete formatting", () => { expect(result).toBe("\\*not italic"); }); - it("should not close double-escaped backslash before asterisk", () => { - // remend sees the char before * as \ (the second backslash) and treats it as escaped + it("should close an asterisk after an escaped backslash", () => { const result = remend("\\\\*actually italic"); - expect(result).toBe("\\\\*actually italic"); + expect(result).toBe("\\\\*actually italic*"); }); - it("should close escaped double asterisks (remend does not track escape depth)", () => { - // remend doesn't understand that \** has the first * escaped; - // it sees ** and closes bold + it("should not close escaped double asterisks", () => { const result = remend("\\**not bold"); - expect(result).toBe("\\**not bold**"); + expect(result).toBe("\\**not bold"); }); it("should handle mixed escaped and real formatting", () => { diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts new file mode 100644 index 00000000..ec924418 --- /dev/null +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("escaped markers", () => { + it.each<{ name: string; input: string; expected: string }>([ + { + name: "closes bold whose content holds an escaped **", + input: "**a\\**b", + expected: "**a\\**b**", + }, + { + name: "skips an escaped ~~", + input: "\\~~strike", + expected: "\\~~strike", + }, + { + name: "closes ** after an escaped backslash", + input: "\\\\**bold", + expected: "\\\\**bold**", + }, + { + name: "closes ~~ after an escaped backslash", + input: "\\\\~~strike", + expected: "\\\\~~strike~~", + }, + { + name: "closes * after an escaped backslash", + input: "\\\\*foo", + expected: "\\\\*foo*", + }, + { + name: "closes _ after an escaped backslash", + input: "\\\\_foo", + expected: "\\\\_foo_", + }, + { + name: "opens a code span after an escaped backslash", + input: "\\\\``a", + expected: "\\\\``a``", + }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); + +describe("trailing backslash", () => { + // A closer appended after a trailing backslash would be escaped by it, so + // the backslash is escaped first + it.each<{ name: string; input: string; expected: string }>([ + { name: "**", input: "**foo\\", expected: "**foo\\\\**" }, + { name: "*", input: "*foo\\", expected: "*foo\\\\*" }, + { name: "_", input: "_foo\\", expected: "_foo\\\\_" }, + { name: "__", input: "__foo\\", expected: "__foo\\\\__" }, + { name: "~~", input: "~~foo\\", expected: "~~foo\\\\~~" }, + { name: "a partial __ match", input: "____.\\", expected: "____.\\\\__" }, + { + name: "a backslash after whitespace", + input: "**foo \\", + expected: "**foo \\\\**", + }, + { + name: "nothing to heal", + input: "plain text\\", + expected: "plain text\\", + }, + { + name: "an opener whose only content is the backslash", + input: "______\n__\\", + expected: "______\n__\\\\__", + }, + { + name: "an escaped trailing backslash", + input: "**foo\\\\", + expected: "**foo\\\\**", + }, + { name: "inline code", input: "`foo\\", expected: "`foo\\`" }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); diff --git a/packages/remend/__tests__/italic.test.ts b/packages/remend/__tests__/italic.test.ts index e0233cda..4ad9807b 100644 --- a/packages/remend/__tests__/italic.test.ts +++ b/packages/remend/__tests__/italic.test.ts @@ -120,8 +120,7 @@ describe("italic formatting with single underscores (_)", () => { expect(remend("naïve_approach")).toBe("naïve_approach"); }); - it("should not count word-internal single underscores in countSingleUnderscores", () => { - // This tests the path where underscore is between word characters (lines 106-108) + it("should not count word-internal single underscores", () => { expect(remend("some_variable_name")).toBe("some_variable_name"); expect(remend("test_123_value")).toBe("test_123_value"); expect(remend("_start with underscore")).toBe("_start with underscore_"); diff --git a/packages/remend/__tests__/lists.test.ts b/packages/remend/__tests__/lists.test.ts index 21f668ea..64a54667 100644 --- a/packages/remend/__tests__/lists.test.ts +++ b/packages/remend/__tests__/lists.test.ts @@ -69,8 +69,10 @@ describe("list handling", () => { expect(remend("* __\n* **")).toBe("* __\n* **"); expect(remend("+ __\n+ **")).toBe("+ __\n+ **"); + // A __ followed by whitespace cannot open emphasis + expect(remend("- __ text after")).toBe("- __ text after"); + // List items with emphasis markers and text should still complete - expect(remend("- __ text after")).toBe("- __ text after__"); expect(remend("- ** text after")).toBe("- ** text after**"); // Mixed list items diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 6199a7c2..10792a9d 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -60,6 +60,7 @@ const preservedPrefixLength = (input: string, output: string): number => { interface CodeNode { lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value: string; } @@ -68,6 +69,7 @@ interface MdastNode { children?: MdastNode[]; lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value?: string; } @@ -267,6 +269,81 @@ describe("streaming properties", () => { }); }); +const UNDERSCORE_RUN_TOKENS = [ + "a", + "word", + " ", + ".", + "`", + "\\", + "_", + "__", + "___", +]; + +const underscoreRunArbitrary = fc + .array(fc.constantFrom(...UNDERSCORE_RUN_TOKENS), { + minLength: 1, + maxLength: 10, + }) + .map((tokens) => tokens.join("")); + +// Gaps outside this property: a closer appended after whitespace cannot +// close, and remend does not recognize indented code blocks +const TRAILING_WHITESPACE = /\s$/; +const INDENTED_CODE = /^ {4}/; + +const MARKER_PATTERN = /[_*]/g; +// Closers, after the backslash that escapes a trailing backslash +const APPENDED_CLOSERS = /^\\?[_*]+$/; + +const countLiteralMarkers = (markdown: string): number => { + let count = 0; + const visit = (node: MdastNode): void => { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (node.type === "text" && start !== undefined && end !== undefined) { + count += markdown.slice(start, end).match(MARKER_PATTERN)?.length ?? 0; + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(fromMarkdown(markdown)); + return count; +}; + +// Each marker healing appends must close one the prefix left literal, so the +// literal count drops by at least as many as were appended. A closer that +// renders as literal text matched nothing, so remend invented it. +const assertAppendedClosersMatch = (prefix: string): void => { + const healed = remend(prefix); + const suffix = healed.slice(prefix.length); + if (!(healed.startsWith(prefix) && APPENDED_CLOSERS.test(suffix))) { + return; + } + const appended = suffix.match(MARKER_PATTERN)?.length ?? 0; + if (countLiteralMarkers(healed) > countLiteralMarkers(prefix) - appended) { + throw new Error( + `healing appended an unmatched closer: ${JSON.stringify(prefix)} -> ${JSON.stringify(healed)}` + ); + } +}; + +describe("appended closers", () => { + it("never appends an underscore closer that matches no opener", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + fc.pre( + !(TRAILING_WHITESPACE.test(prefix) || INDENTED_CODE.test(prefix)) + ); + assertAppendedClosersMatch(prefix); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); +}); + describe("exhaustive prefix sweep", () => { // Every prefix of a fixed corpus, deterministically. The corpus mixes // constructs that interact. diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 120f6a29..467354d0 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -54,4 +54,27 @@ describe("escaped underscores", () => { it("should treat the run after an escaped underscore as a delimiter", () => { expect(remend("\\___bold")).toBe("\\___bold__"); }); + + it("should not treat a run after an escaped backslash as escaped", () => { + expect(remend("\\\\__bold")).toBe("\\\\__bold__"); + }); +}); + +describe("underscore delimiter matching", () => { + it.each<[string, string]>([ + ["a run followed by whitespace", "__ foo"], + ["a spaced run before an escaped run", "a __ b \\__c"], + ["a spaced run before an escaped run at the start", "__ \\__alph"], + ["a run after a letter and before punctuation", "a__."], + ["a closer-only run with nothing open", "__ a__ a"], + ["a longer run that closes the opener", "__a____.a"], + ["an opener the rule of 3 keeps a __ closer from matching", "\\_____\\a"], + ["a leftover single underscore after an escape", "__\\__ a"], + ])("should not append a closer for %s", (_name, input) => { + expect(remend(input)).toBe(input); + }); + + it("should still close an opener after a spaced run", () => { + expect(remend("a __ b __bold")).toBe("a __ b __bold__"); + }); }); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 8905ae68..3d40ecb0 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -17,13 +17,18 @@ import { import { countDoublePairs, getScan, - inHtmlTagAt, inLinkUrlAt, inMathAt, + isEscaped, REGION, type TextScan, } from "./scan"; -import { isHorizontalRule, isWordChar } from "./utils"; +import { + closesAddedUnderscores, + findOpenUnderscoreRuns, + type OpenUnderscoreRun, +} from "./underscore-delimiters"; +import { isHorizontalRule, isWhitespaceChar, isWordChar } from "./utils"; // Helper function to check if an asterisk should be skipped const shouldSkipAsterisk = ( @@ -33,7 +38,7 @@ const shouldSkipAsterisk = ( nextChar: string ): boolean => { // Skip if escaped with backslash - if (prevChar === "\\") { + if (isEscaped(scan.text, index)) { return true; } @@ -75,9 +80,6 @@ const shouldSkipAsterisk = ( return false; }; -const isWhitespaceChar = (char: string): boolean => - char === " " || char === "\t" || char === "\n"; - const isWordInternalAsterisk = (prevChar: string, nextChar: string): boolean => Boolean(prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)); @@ -149,67 +151,6 @@ export const countSingleAsterisks = (text: string): number => { return count; }; -// Helper function to check if an underscore should be skipped -const shouldSkipUnderscore = ( - scan: TextScan, - index: number, - prevChar: string, - nextChar: string -): boolean => { - // Skip if escaped with backslash - if (prevChar === "\\") { - return true; - } - - // Skip if within math block - if (inMathAt(scan, index)) { - return true; - } - - // Skip if within a link or image URL - if (inLinkUrlAt(scan, index)) { - return true; - } - - // Skip if within an HTML tag (e.g. ) - if (inHtmlTagAt(scan, index)) { - return true; - } - - // Skip if part of __ - if (prevChar === "_" || nextChar === "_") { - return true; - } - - // Skip if underscore is word-internal (between word characters) - if (prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)) { - return true; - } - - return false; -}; - -export const countSingleUnderscores = (text: string): number => { - const scan = getScan(text); - let count = 0; - const len = text.length; - - for (let index = 0; index < len; index += 1) { - if (text[index] !== "_" || scan.regions[index] !== REGION.PROSE) { - continue; - } - - const prevChar = index > 0 ? text[index - 1] : ""; - const nextChar = index < len - 1 ? text[index + 1] : ""; - - if (!shouldSkipUnderscore(scan, index, prevChar, nextChar)) { - count += 1; - } - } - - return count; -}; - // Counts triple asterisks that are not part of quadruple or more asterisks // and not inside code regions export const countTripleAsterisks = (text: string): number => { @@ -240,125 +181,18 @@ export const countTripleAsterisks = (text: string): number => { const countDoubleAsterisks = (text: string): number => countDoublePairs(text, "*"); -// Whether the text has an unmatched __ delimiter, counted per maximal -// underscore run. -// -// Counting raw occurrences misreads identifiers: a name like snake__case -// contains __ but cannot open or close emphasis, and counting it either -// invents a closer (odd count) or pairs it against a real delimiter and -// swallows a closer that was needed (even count). Per run: -// -// - A run contributes floor(length / 2) pairs, and flips delimiter parity -// only when that is odd. __ and ___ flip; ____ does not. -// - A word-internal run (word characters on both sides) is part of an -// identifier, never a delimiter. -// - Runs inside code regions, math, link URLs, and HTML tags are skipped, -// matching the single-underscore handler's flanking rules. -// - A run alone on its line is a thematic break, not emphasis. -const isLineBoundaryChar = (char: string): boolean => - char === "" || char === " " || char === "\t" || char === "\n"; - -// isHorizontalRule scans the run's whole line, so its verdict is memoized -// per line to keep run counting linear when many runs share a line -interface ThematicBreakMemo { - lineEnd: number; - result: boolean; -} - -const isThematicBreakRun = ( - text: string, - runStart: number, - prevChar: string, - nextChar: string, - memo: ThematicBreakMemo -): boolean => { - // A thematic break line holds only markers and whitespace, so only runs - // flanked by whitespace or line boundaries can be part of one - if (!(isLineBoundaryChar(prevChar) && isLineBoundaryChar(nextChar))) { - return false; - } - if (runStart > memo.lineEnd) { - const lineEnd = text.indexOf("\n", runStart); - memo.lineEnd = lineEnd === -1 ? text.length : lineEnd; - memo.result = isHorizontalRule(text, runStart, "_"); - } - return memo.result; -}; - -// Whether an underscore run at [runStart, runEnd) flips delimiter parity -const doubleUnderscoreRunFlips = ( - scan: TextScan, - initialRunStart: number, - runEnd: number, - memo: ThematicBreakMemo -): boolean => { - const text = scan.text; - - // A backslash escapes the first underscore of the run. The escaped - // underscore is literal punctuation, so the rest of the run still flanks - // as a delimiter. - let runStart = initialRunStart; - let escaped = false; - if (runStart > 0 && text[runStart - 1] === "\\") { - runStart += 1; - escaped = true; - } - const runLength = runEnd - runStart; - if (runLength < 2) { - return false; - } - - const beforeRun = runStart > 0 ? text[runStart - 1] : ""; - const prevChar = escaped ? "\\" : beforeRun; - const nextChar = runEnd < text.length ? text[runEnd] : ""; - if (isWordChar(prevChar) && isWordChar(nextChar)) { - return false; - } - if (isThematicBreakRun(text, runStart, prevChar, nextChar, memo)) { - return false; - } - if ( - inMathAt(scan, runStart) || - inLinkUrlAt(scan, runStart) || - inHtmlTagAt(scan, runStart) - ) { - return false; - } +const isHalfClosedDoubleUnderscore = ( + run: OpenUnderscoreRun | undefined +): boolean => run !== undefined && run.count === 1 && run.length >= 2; - return Math.floor(runLength / 2) % 2 === 1; -}; +// Healing that adds an underscore closer which cannot match would render it +// as literal text, so such a heal is dropped +const healUnderscores = (text: string, healed: string): string => + closesAddedUnderscores(text, healed) ? healed : text; -const hasUnmatchedDoubleUnderscore = (text: string): boolean => { - const scan = getScan(text); - const n = text.length; - const memo: ThematicBreakMemo = { lineEnd: -1, result: false }; - let unmatched = false; - let i = 0; - - while (i < n) { - if (text[i] !== "_" || scan.regions[i] !== REGION.PROSE) { - i += 1; - continue; - } - - const runStart = i; - let runEnd = i + 1; - while ( - runEnd < n && - text[runEnd] === "_" && - scan.regions[runEnd] === REGION.PROSE - ) { - runEnd += 1; - } - i = runEnd; - - if (doubleUnderscoreRunFlips(scan, runStart, runEnd, memo)) { - unmatched = !unmatched; - } - } - - return unmatched; -}; +const innermostOpenUnderscoreRun = ( + text: string +): OpenUnderscoreRun | undefined => findOpenUnderscoreRuns(text).at(-1); // Helper to check if bold marker should not be completed const shouldSkipBoldCompletion = ( @@ -475,9 +309,9 @@ export const handleIncompleteDoubleUnderscoreItalic = ( isInsideCodeBlock(text, markerIndex) || isWithinCompleteInlineCode(text, markerIndex) ) && - hasUnmatchedDoubleUnderscore(text) + isHalfClosedDoubleUnderscore(innermostOpenUnderscoreRun(text)) ) { - return `${text}_`; + return healUnderscores(text, `${text}_`); } } return text; @@ -498,8 +332,9 @@ export const handleIncompleteDoubleUnderscoreItalic = ( return text; } - if (hasUnmatchedDoubleUnderscore(text)) { - return `${text}__`; + const openRun = innermostOpenUnderscoreRun(text); + if (openRun && openRun.count >= 2) { + return healUnderscores(text, `${text}__`); } return text; @@ -515,7 +350,7 @@ const isLoneProseAsterisk = (scan: TextScan, i: number): boolean => { scan.regions[i] === REGION.PROSE && text[i - 1] !== "*" && text[i + 1] !== "*" && - text[i - 1] !== "\\" && + !isEscaped(text, i) && !inMathAt(scan, i) ); }; @@ -605,7 +440,7 @@ const findFirstSingleUnderscoreIndex = (text: string): number => { scan.regions[i] === REGION.PROSE && text[i - 1] !== "_" && text[i + 1] !== "_" && - text[i - 1] !== "\\" && + !isEscaped(text, i) && !inMathAt(scan, i) && !inLinkUrlAt(scan, i) ) { @@ -713,14 +548,12 @@ export const handleIncompleteSingleUnderscoreItalic = ( return text; } - const singleUnderscores = countSingleUnderscores(text); - if (singleUnderscores % 2 === 1) { + if (innermostOpenUnderscoreRun(text)?.count === 1) { // Check if we need to insert _ before trailing ** for proper nesting - const trailingResult = handleTrailingAsterisksForUnderscore(text); - if (trailingResult !== null) { - return trailingResult; - } - return insertClosingUnderscore(text); + const healed = + handleTrailingAsterisksForUnderscore(text) ?? + insertClosingUnderscore(text); + return healUnderscores(text, healed); } return text; diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 0ecf9137..6a39f53d 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -17,7 +17,7 @@ import { INCOMPLETE_IMAGE_PLACEHOLDER, type LinkMode, } from "./link-image-handler"; -import { clearScanCache } from "./scan"; +import { clearScanCache, getScan, inMathAt, isEscaped, REGION } from "./scan"; import { handleIncompleteSetextHeading } from "./setext-heading-handler"; import { handleSingleTildeEscape } from "./single-tilde-handler"; import { handleIncompleteStrikethrough } from "./strikethrough-handler"; @@ -277,6 +277,16 @@ const getEnabledBuiltInHandlers = ( }); }; +// Whether the text ends with an unescaped backslash in prose +const isTrailingEscape = (text: string): boolean => { + const last = text.length - 1; + if (text[last] !== "\\" || isEscaped(text, last)) { + return false; + } + const scan = getScan(text); + return scan.regions[last] === REGION.PROSE && !inMathAt(scan, last); +}; + // Parses markdown text and removes incomplete tokens to prevent partial rendering const remend = (text: string, options?: RemendOptions): string => { if (!text || typeof text !== "string") { @@ -284,7 +294,7 @@ const remend = (text: string, options?: RemendOptions): string => { } // Remove trailing whitespace if it's not a double space - let result = + const result = text.endsWith(" ") && !text.endsWith(" ") ? text.slice(0, -1) : text; // Get enabled built-in handlers @@ -302,25 +312,39 @@ const remend = (text: string, options?: RemendOptions): string => { (a, b) => (a.handler.priority ?? 0) - (b.handler.priority ?? 0) ); - try { + const runHandlers = (input: string): string => { + let output = input; // Execute handlers in priority order for (const { handler, earlyReturn } of allHandlers) { - result = handler.handle(result); + output = handler.handle(output); // Check for early return condition (e.g., incomplete link marker) - if (earlyReturn?.(result)) { - return result; + if (earlyReturn?.(output)) { + return output; } } // A handler that removes a trailing fragment can expose a trailing space // (e.g. dropping an incomplete image). Strip it the same way the input // was stripped, so healed output re-heals to itself. - if (result.endsWith(" ") && !result.endsWith(" ")) { - return result.slice(0, -1); + if (output.endsWith(" ") && !output.endsWith(" ")) { + return output.slice(0, -1); + } + + return output; + }; + + try { + if (!isTrailingEscape(result)) { + return runHandlers(result); } - return result; + // A trailing backslash escapes whatever follows it, so a closer appended + // after it would render as text. Heal with the backslash escaped, which + // keeps it literal without changing how the markers before it flank. + const escaped = `${result}\\`; + const healed = runHandlers(escaped); + return healed === escaped ? result : healed; } finally { clearScanCache(); } diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 1bb9f74f..5bf68c9b 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -342,7 +342,7 @@ const nextSpanOpener = ( return -1; } const escaped = - next > i && text[next - 1] === "\\" && regions[next - 1] === REGION.PROSE; + next > i && regions[next - 1] === REGION.PROSE && isEscaped(text, next); if (regions[next] === REGION.PROSE && !escaped) { return next; } @@ -473,6 +473,15 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => scan.regions[position] === REGION.CODE_SPAN; +/** Whether the character at position follows an odd run of backslashes */ +export const isEscaped = (text: string, position: number): boolean => { + let backslashes = 0; + for (let i = position - 1; i >= 0 && text[i] === "\\"; i -= 1) { + backslashes += 1; + } + return backslashes % 2 === 1; +}; + /** Counts non-overlapping double-character pairs (**, ~~, $$) in prose */ export const countDoublePairs = (text: string, char: string): number => { const scan = getScan(text); @@ -480,7 +489,7 @@ export const countDoublePairs = (text: string, char: string): number => { let count = 0; for (let i = text.indexOf(pair); i !== -1; i = text.indexOf(pair, i)) { - if (scan.regions[i] === REGION.PROSE) { + if (scan.regions[i] === REGION.PROSE && !isEscaped(text, i)) { count += 1; i += 2; } else { diff --git a/packages/remend/src/underscore-delimiters.ts b/packages/remend/src/underscore-delimiters.ts new file mode 100644 index 00000000..c24c384d --- /dev/null +++ b/packages/remend/src/underscore-delimiters.ts @@ -0,0 +1,178 @@ +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isEscaped, + REGION, + type TextScan, +} from "./scan"; +import { isWhitespaceChar, isWordChar } from "./utils"; + +/** An underscore delimiter run left open at the end of the text */ +export interface OpenUnderscoreRun { + /** Underscores still unmatched */ + count: number; + /** Unescaped length of the run before any were matched */ + length: number; +} + +interface DelimiterRun extends OpenUnderscoreRun { + canClose: boolean; + canOpen: boolean; +} + +// The start and end of the text count as whitespace for flanking +const isWhitespaceOrEdge = (char: string): boolean => + char === "" || isWhitespaceChar(char); + +const isPunctuationChar = (char: string): boolean => + char === "_" || !(isWhitespaceOrEdge(char) || isWordChar(char)); + +// Flanking per CommonMark, with the stricter underscore rules: a run that +// flanks on both sides opens only after punctuation and closes only before +// punctuation, so word-internal runs (snake__case) can't open or close +const toDelimiterRun = ( + text: string, + index: number, + end: number, + prevChar: string +): DelimiterRun => { + const nextChar = end < text.length ? text[end] : ""; + const leftFlanking = !( + isWhitespaceOrEdge(nextChar) || + (isPunctuationChar(nextChar) && + !isWhitespaceOrEdge(prevChar) && + !isPunctuationChar(prevChar)) + ); + const rightFlanking = !( + isWhitespaceOrEdge(prevChar) || + (isPunctuationChar(prevChar) && + !isWhitespaceOrEdge(nextChar) && + !isPunctuationChar(nextChar)) + ); + const length = end - index; + return { + canClose: rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)), + canOpen: leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)), + count: length, + length, + }; +}; + +const isSkippedUnderscore = (scan: TextScan, i: number): boolean => + scan.text[i] !== "_" || + scan.regions[i] !== REGION.PROSE || + inMathAt(scan, i) || + inLinkUrlAt(scan, i) || + inHtmlTagAt(scan, i); + +const collectDelimiterRuns = (scan: TextScan): DelimiterRun[] => { + const { text } = scan; + const runs: DelimiterRun[] = []; + let i = text.indexOf("_"); + + while (i !== -1) { + if (isSkippedUnderscore(scan, i)) { + i = text.indexOf("_", i + 1); + continue; + } + + let end = i + 1; + while (end < text.length && !isSkippedUnderscore(scan, end)) { + end += 1; + } + + // A backslash escapes the run's first underscore, which stays literal + // punctuation before the rest of the run + if (isEscaped(text, i)) { + if (end - i > 1) { + runs.push(toDelimiterRun(text, i + 1, end, "_")); + } + } else { + runs.push(toDelimiterRun(text, i, end, i > 0 ? text[i - 1] : "")); + } + i = text.indexOf("_", end); + } + + return runs; +}; + +// The "rule of 3": when either run can both open and close, they match only +// if their combined length is not a multiple of 3, unless both lengths are. +// Lengths are what remains of each run after earlier partial matches. +const violatesRuleOfThree = ( + opener: DelimiterRun, + closer: DelimiterRun +): boolean => + (closer.canOpen || opener.canClose) && + (opener.count + closer.count) % 3 === 0 && + !(opener.count % 3 === 0 && closer.count % 3 === 0); + +// Matches a closing run against the stack until it runs out of underscores +// or openers, popping every opener it passes +const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { + while (closer.count > 0) { + let openerIndex = stack.length - 1; + while ( + openerIndex >= 0 && + violatesRuleOfThree(stack[openerIndex], closer) + ) { + openerIndex -= 1; + } + if (openerIndex < 0) { + return; + } + + const opener = stack[openerIndex]; + const used = opener.count >= 2 && closer.count >= 2 ? 2 : 1; + opener.count -= used; + closer.count -= used; + stack.length = opener.count > 0 ? openerIndex + 1 : openerIndex; + } +}; + +// Runs the CommonMark emphasis algorithm over the text's underscore runs, +// leaving each run's unmatched count and the openers still open +const matchUnderscoreRuns = ( + text: string +): { runs: DelimiterRun[]; stack: DelimiterRun[] } => { + const runs = collectDelimiterRuns(getScan(text)); + const stack: DelimiterRun[] = []; + + for (const run of runs) { + if (run.canClose) { + matchCloser(stack, run); + } + if (run.canOpen && run.count > 0) { + stack.push(run); + } + } + + return { runs, stack }; +}; + +/** The underscore openers the text leaves unmatched, outermost first */ +export const findOpenUnderscoreRuns = (text: string): OpenUnderscoreRun[] => + matchUnderscoreRuns(text).stack; + +// Underscores that render as literal text: openers still open, openers a +// closer passed over, and closers left over +const countLiteralUnderscores = (text: string): number => + matchUnderscoreRuns(text).runs.reduce((sum, run) => sum + run.count, 0); + +/** + * Whether every underscore that healing added to the text matched one the + * text left literal. An added closer can fail to match, for example under + * the rule of 3. + */ +export const closesAddedUnderscores = ( + text: string, + healed: string +): boolean => { + // Scan the text first, while the scan cache still holds it + const literal = countLiteralUnderscores(text); + return ( + countLiteralUnderscores(healed) <= literal - (healed.length - text.length) + ); +}; diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 06291fd0..59c1898e 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,6 +1,11 @@ import { letterNumberUnderscorePattern } from "./patterns"; import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; +const whitespaceCharPattern = /\s/; + +export const isWhitespaceChar = (char: string): boolean => + whitespaceCharPattern.test(char); + // OPTIMIZATION: Precompute which characters are word characters // Using ASCII fast path before falling back to Unicode regex export const isWordChar = (char: string): boolean => { From b3a5f65c14625c6e602f75141d1ce636136341f4 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 7 Oct 2026 16:18:44 -0700 Subject: [PATCH 2/2] fix(remend): heal every open underscore run in one pass The underscore handlers closed only the innermost open run and ran before the inline code handler, so a second remend pass could append more closers. Heal against the closed code span, repeat until no underscore opener is left, and match asterisk runs alongside underscores so a matched asterisk pair discards the openers inside it. Escapes in countDoublePairs no longer apply to $$, since a backslash inside math is TeX. --- .changeset/remend-escaped-markers.md | 1 + .../remend/__tests__/escaped-markers.test.ts | 11 ++++ .../__tests__/streaming-properties.test.ts | 10 +++ .../remend/__tests__/underscore-runs.test.ts | 23 +++++++ packages/remend/src/emphasis-handlers.ts | 42 +++++++++++-- packages/remend/src/scan.ts | 4 +- packages/remend/src/underscore-delimiters.ts | 63 ++++++++++++------- 7 files changed, 126 insertions(+), 28 deletions(-) diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md index ceb64c00..9844d3cb 100644 --- a/.changeset/remend-escaped-markers.md +++ b/.changeset/remend-escaped-markers.md @@ -7,3 +7,4 @@ fix(remend): stop appending emphasis closers that match no opener - Match `_` and `__` delimiters with the CommonMark emphasis algorithm (flanking, rule of 3, partial matches) instead of counting parity, and drop an underscore closer that would not match - Treat a marker as escaped only after an odd run of backslashes, so `\\*`, `\\_`, `\\**`, `\\~~`, and `` \\` `` still open - Escape a trailing backslash before appending closers, so it cannot escape them +- Close every open underscore run in one pass, including runs before an unterminated code span, so healed output re-heals to itself diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts index ec924418..4f53aa2a 100644 --- a/packages/remend/__tests__/escaped-markers.test.ts +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -43,6 +43,17 @@ describe("escaped markers", () => { }); }); +describe("math", () => { + // Inside math a backslash is TeX, so it does not escape the closing $$ + it.each<{ name: string; input: string }>([ + { name: "a backslash before $$", input: "$$a\\$$" }, + { name: "a lone backslash", input: "$$\\$$" }, + { name: "inline block math", input: "text $$\\frac\\$$ more" }, + ])("leaves complete math with $name unchanged", ({ input }) => { + expect(remend(input)).toBe(input); + }); +}); + describe("trailing backslash", () => { // A closer appended after a trailing backslash would be escaped by it, so // the backslash is escaped first diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 10792a9d..99237afb 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -342,6 +342,16 @@ describe("appended closers", () => { { numRuns: Number(process.env.NUM_RUNS ?? 2000) } ); }); + + it("re-heals underscore runs to themselves", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + const healed = remend(prefix); + expect(remend(healed)).toBe(healed); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); }); describe("exhaustive prefix sweep", () => { diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 467354d0..2cfb02a6 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -77,4 +77,27 @@ describe("underscore delimiter matching", () => { it("should still close an opener after a spaced run", () => { expect(remend("a __ b __bold")).toBe("a __ b __bold__"); }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "an outer run after an escape", + input: "\\__._a", + expected: "\\__._a__", + }, + { name: "a run across an asterisk", input: "_a *_a", expected: "_a *_a*_" }, + { + name: "a run after an open code span", + input: "__`x", + expected: "__`x`__", + }, + { + name: "a run after an open code span ending in a backslash", + input: "__`x\\", + expected: "__`x\\`__", + }, + ])("should close $name in one pass", ({ input, expected }) => { + const healed = remend(input); + expect(healed).toBe(expected); + expect(remend(healed)).toBe(healed); + }); }); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 3d40ecb0..581547b7 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -2,6 +2,7 @@ import { isInsideCodeBlock, isWithinCompleteInlineCode, } from "./code-block-utils"; +import { handleIncompleteInlineCode } from "./inline-code-handler"; import { boldItalicPattern, boldPattern, @@ -290,9 +291,7 @@ const shouldSkipItalicCompletion = ( }; // Completes incomplete italic formatting with double underscores (__) -export const handleIncompleteDoubleUnderscoreItalic = ( - text: string -): string => { +const healDoubleUnderscore = (text: string): string => { const italicMatch = matchTrailing(text, italicPattern); if (!italicMatch) { // Check for half-complete closing marker: __content_ should become __content__ @@ -512,9 +511,7 @@ const handleTrailingAsterisksForUnderscore = (text: string): string | null => { }; // Completes incomplete italic formatting with single underscores (_) -export const handleIncompleteSingleUnderscoreItalic = ( - text: string -): string => { +const healSingleUnderscore = (text: string): string => { const singleUnderscoreMatch = matchTrailing(text, singleUnderscorePattern); if (!singleUnderscoreMatch) { @@ -621,3 +618,36 @@ export const handleIncompleteBoldItalic = (text: string): string => { return text; }; + +// The inline code handler runs after these, so a closer appended while a code +// span is open would land inside the span. Heal against the closed span and +// keep that closing only when the heal adds a closer. +const healAfterOpenCodeSpan = ( + text: string, + heal: (text: string) => string +): string => { + const closed = handleIncompleteInlineCode(text); + if (closed === text) { + return heal(text); + } + const healed = heal(closed); + return healed === closed ? text : healed; +}; + +export const handleIncompleteDoubleUnderscoreItalic = (text: string): string => + healAfterOpenCodeSpan(text, healDoubleUnderscore); + +// Each heal closes only the innermost open run, so healing repeats until no +// opener is left. Every accepted heal closes a literal, so the loop ends. +const healUnderscoreStack = (text: string): string => { + let healed = text; + let next = healSingleUnderscore(healDoubleUnderscore(healed)); + while (next !== healed) { + healed = next; + next = healSingleUnderscore(healDoubleUnderscore(healed)); + } + return healed; +}; + +export const handleIncompleteSingleUnderscoreItalic = (text: string): string => + healAfterOpenCodeSpan(text, healUnderscoreStack); diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 5bf68c9b..15245737 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -489,7 +489,9 @@ export const countDoublePairs = (text: string, char: string): number => { let count = 0; for (let i = text.indexOf(pair); i !== -1; i = text.indexOf(pair, i)) { - if (scan.regions[i] === REGION.PROSE && !isEscaped(text, i)) { + // Inside math a backslash is TeX, so only markdown markers can be escaped + const escaped = char !== "$" && isEscaped(text, i); + if (scan.regions[i] === REGION.PROSE && !escaped) { count += 1; i += 2; } else { diff --git a/packages/remend/src/underscore-delimiters.ts b/packages/remend/src/underscore-delimiters.ts index c24c384d..e4dceab7 100644 --- a/packages/remend/src/underscore-delimiters.ts +++ b/packages/remend/src/underscore-delimiters.ts @@ -20,8 +20,11 @@ export interface OpenUnderscoreRun { interface DelimiterRun extends OpenUnderscoreRun { canClose: boolean; canOpen: boolean; + marker: Marker; } +type Marker = "*" | "_"; + // The start and end of the text count as whitespace for flanking const isWhitespaceOrEdge = (char: string): boolean => char === "" || isWhitespaceChar(char); @@ -38,6 +41,7 @@ const toDelimiterRun = ( end: number, prevChar: string ): DelimiterRun => { + const marker = text[index] as Marker; const nextChar = end < text.length ? text[end] : ""; const leftFlanking = !( isWhitespaceOrEdge(nextChar) || @@ -52,47 +56,59 @@ const toDelimiterRun = ( !isPunctuationChar(nextChar)) ); const length = end - index; + if (marker === "*") { + return { + canClose: rightFlanking, + canOpen: leftFlanking, + count: length, + length, + marker, + }; + } return { canClose: rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)), canOpen: leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)), count: length, length, + marker, }; }; -const isSkippedUnderscore = (scan: TextScan, i: number): boolean => - scan.text[i] !== "_" || +const isSkippedMarker = (scan: TextScan, i: number, marker: Marker): boolean => + scan.text[i] !== marker || scan.regions[i] !== REGION.PROSE || inMathAt(scan, i) || inLinkUrlAt(scan, i) || inHtmlTagAt(scan, i); -const collectDelimiterRuns = (scan: TextScan): DelimiterRun[] => { +const collectDelimiterRuns = ( + scan: TextScan, + markers: readonly Marker[] +): DelimiterRun[] => { const { text } = scan; const runs: DelimiterRun[] = []; - let i = text.indexOf("_"); - while (i !== -1) { - if (isSkippedUnderscore(scan, i)) { - i = text.indexOf("_", i + 1); + for (let i = 0; i < text.length; i += 1) { + const marker = text[i] as Marker; + if (!markers.includes(marker) || isSkippedMarker(scan, i, marker)) { continue; } let end = i + 1; - while (end < text.length && !isSkippedUnderscore(scan, end)) { + while (end < text.length && !isSkippedMarker(scan, end, marker)) { end += 1; } - // A backslash escapes the run's first underscore, which stays literal + // A backslash escapes the run's first marker, which stays literal // punctuation before the rest of the run if (isEscaped(text, i)) { if (end - i > 1) { - runs.push(toDelimiterRun(text, i + 1, end, "_")); + runs.push(toDelimiterRun(text, i + 1, end, marker)); } } else { runs.push(toDelimiterRun(text, i, end, i > 0 ? text[i - 1] : "")); } - i = text.indexOf("_", end); + i = end - 1; } return runs; @@ -109,14 +125,15 @@ const violatesRuleOfThree = ( (opener.count + closer.count) % 3 === 0 && !(opener.count % 3 === 0 && closer.count % 3 === 0); -// Matches a closing run against the stack until it runs out of underscores -// or openers, popping every opener it passes +// Matches a closing run against the stack until it runs out of markers or +// openers, popping every opener it passes const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { while (closer.count > 0) { let openerIndex = stack.length - 1; while ( openerIndex >= 0 && - violatesRuleOfThree(stack[openerIndex], closer) + (stack[openerIndex].marker !== closer.marker || + violatesRuleOfThree(stack[openerIndex], closer)) ) { openerIndex -= 1; } @@ -132,12 +149,13 @@ const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { } }; -// Runs the CommonMark emphasis algorithm over the text's underscore runs, +// Runs the CommonMark emphasis algorithm over the text's delimiter runs, // leaving each run's unmatched count and the openers still open -const matchUnderscoreRuns = ( - text: string +const matchDelimiterRuns = ( + text: string, + markers: readonly Marker[] ): { runs: DelimiterRun[]; stack: DelimiterRun[] } => { - const runs = collectDelimiterRuns(getScan(text)); + const runs = collectDelimiterRuns(getScan(text), markers); const stack: DelimiterRun[] = []; for (const run of runs) { @@ -154,12 +172,15 @@ const matchUnderscoreRuns = ( /** The underscore openers the text leaves unmatched, outermost first */ export const findOpenUnderscoreRuns = (text: string): OpenUnderscoreRun[] => - matchUnderscoreRuns(text).stack; + matchDelimiterRuns(text, ["_"]).stack; // Underscores that render as literal text: openers still open, openers a -// closer passed over, and closers left over +// closer passed over, and closers left over. Asterisk runs take part because +// a matched asterisk pair discards the underscore openers between them. const countLiteralUnderscores = (text: string): number => - matchUnderscoreRuns(text).runs.reduce((sum, run) => sum + run.count, 0); + matchDelimiterRuns(text, ["*", "_"]) + .runs.filter((run) => run.marker === "_") + .reduce((sum, run) => sum + run.count, 0); /** * Whether every underscore that healing added to the text matched one the