diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md new file mode 100644 index 00000000..9844d3cb --- /dev/null +++ b/.changeset/remend-escaped-markers.md @@ -0,0 +1,10 @@ +--- +"remend": patch +--- + +fix(remend): stop appending emphasis closers that match no opener + +- Match `_` and `__` delimiters with the CommonMark emphasis algorithm (flanking, rule of 3, partial matches) instead of counting parity, and drop an underscore closer that would not match +- Treat a marker as escaped only after an odd run of backslashes, so `\\*`, `\\_`, `\\**`, `\\~~`, and `` \\` `` still open +- Escape a trailing backslash before appending closers, so it cannot escape them +- Close every open underscore run in one pass, including runs before an unterminated code span, so healed output re-heals to itself diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index b483fddc..6fa64cae 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -77,17 +77,14 @@ describe("backslash escapes with incomplete formatting", () => { expect(result).toBe("\\*not italic"); }); - it("should not close double-escaped backslash before asterisk", () => { - // remend sees the char before * as \ (the second backslash) and treats it as escaped + it("should close an asterisk after an escaped backslash", () => { const result = remend("\\\\*actually italic"); - expect(result).toBe("\\\\*actually italic"); + expect(result).toBe("\\\\*actually italic*"); }); - it("should close escaped double asterisks (remend does not track escape depth)", () => { - // remend doesn't understand that \** has the first * escaped; - // it sees ** and closes bold + it("should not close escaped double asterisks", () => { const result = remend("\\**not bold"); - expect(result).toBe("\\**not bold**"); + expect(result).toBe("\\**not bold"); }); it("should handle mixed escaped and real formatting", () => { diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts new file mode 100644 index 00000000..4f53aa2a --- /dev/null +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("escaped markers", () => { + it.each<{ name: string; input: string; expected: string }>([ + { + name: "closes bold whose content holds an escaped **", + input: "**a\\**b", + expected: "**a\\**b**", + }, + { + name: "skips an escaped ~~", + input: "\\~~strike", + expected: "\\~~strike", + }, + { + name: "closes ** after an escaped backslash", + input: "\\\\**bold", + expected: "\\\\**bold**", + }, + { + name: "closes ~~ after an escaped backslash", + input: "\\\\~~strike", + expected: "\\\\~~strike~~", + }, + { + name: "closes * after an escaped backslash", + input: "\\\\*foo", + expected: "\\\\*foo*", + }, + { + name: "closes _ after an escaped backslash", + input: "\\\\_foo", + expected: "\\\\_foo_", + }, + { + name: "opens a code span after an escaped backslash", + input: "\\\\``a", + expected: "\\\\``a``", + }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); + +describe("math", () => { + // Inside math a backslash is TeX, so it does not escape the closing $$ + it.each<{ name: string; input: string }>([ + { name: "a backslash before $$", input: "$$a\\$$" }, + { name: "a lone backslash", input: "$$\\$$" }, + { name: "inline block math", input: "text $$\\frac\\$$ more" }, + ])("leaves complete math with $name unchanged", ({ input }) => { + expect(remend(input)).toBe(input); + }); +}); + +describe("trailing backslash", () => { + // A closer appended after a trailing backslash would be escaped by it, so + // the backslash is escaped first + it.each<{ name: string; input: string; expected: string }>([ + { name: "**", input: "**foo\\", expected: "**foo\\\\**" }, + { name: "*", input: "*foo\\", expected: "*foo\\\\*" }, + { name: "_", input: "_foo\\", expected: "_foo\\\\_" }, + { name: "__", input: "__foo\\", expected: "__foo\\\\__" }, + { name: "~~", input: "~~foo\\", expected: "~~foo\\\\~~" }, + { name: "a partial __ match", input: "____.\\", expected: "____.\\\\__" }, + { + name: "a backslash after whitespace", + input: "**foo \\", + expected: "**foo \\\\**", + }, + { + name: "nothing to heal", + input: "plain text\\", + expected: "plain text\\", + }, + { + name: "an opener whose only content is the backslash", + input: "______\n__\\", + expected: "______\n__\\\\__", + }, + { + name: "an escaped trailing backslash", + input: "**foo\\\\", + expected: "**foo\\\\**", + }, + { name: "inline code", input: "`foo\\", expected: "`foo\\`" }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); diff --git a/packages/remend/__tests__/italic.test.ts b/packages/remend/__tests__/italic.test.ts index e0233cda..4ad9807b 100644 --- a/packages/remend/__tests__/italic.test.ts +++ b/packages/remend/__tests__/italic.test.ts @@ -120,8 +120,7 @@ describe("italic formatting with single underscores (_)", () => { expect(remend("naïve_approach")).toBe("naïve_approach"); }); - it("should not count word-internal single underscores in countSingleUnderscores", () => { - // This tests the path where underscore is between word characters (lines 106-108) + it("should not count word-internal single underscores", () => { expect(remend("some_variable_name")).toBe("some_variable_name"); expect(remend("test_123_value")).toBe("test_123_value"); expect(remend("_start with underscore")).toBe("_start with underscore_"); diff --git a/packages/remend/__tests__/lists.test.ts b/packages/remend/__tests__/lists.test.ts index 21f668ea..64a54667 100644 --- a/packages/remend/__tests__/lists.test.ts +++ b/packages/remend/__tests__/lists.test.ts @@ -69,8 +69,10 @@ describe("list handling", () => { expect(remend("* __\n* **")).toBe("* __\n* **"); expect(remend("+ __\n+ **")).toBe("+ __\n+ **"); + // A __ followed by whitespace cannot open emphasis + expect(remend("- __ text after")).toBe("- __ text after"); + // List items with emphasis markers and text should still complete - expect(remend("- __ text after")).toBe("- __ text after__"); expect(remend("- ** text after")).toBe("- ** text after**"); // Mixed list items diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 6199a7c2..99237afb 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -60,6 +60,7 @@ const preservedPrefixLength = (input: string, output: string): number => { interface CodeNode { lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value: string; } @@ -68,6 +69,7 @@ interface MdastNode { children?: MdastNode[]; lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value?: string; } @@ -267,6 +269,91 @@ describe("streaming properties", () => { }); }); +const UNDERSCORE_RUN_TOKENS = [ + "a", + "word", + " ", + ".", + "`", + "\\", + "_", + "__", + "___", +]; + +const underscoreRunArbitrary = fc + .array(fc.constantFrom(...UNDERSCORE_RUN_TOKENS), { + minLength: 1, + maxLength: 10, + }) + .map((tokens) => tokens.join("")); + +// Gaps outside this property: a closer appended after whitespace cannot +// close, and remend does not recognize indented code blocks +const TRAILING_WHITESPACE = /\s$/; +const INDENTED_CODE = /^ {4}/; + +const MARKER_PATTERN = /[_*]/g; +// Closers, after the backslash that escapes a trailing backslash +const APPENDED_CLOSERS = /^\\?[_*]+$/; + +const countLiteralMarkers = (markdown: string): number => { + let count = 0; + const visit = (node: MdastNode): void => { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (node.type === "text" && start !== undefined && end !== undefined) { + count += markdown.slice(start, end).match(MARKER_PATTERN)?.length ?? 0; + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(fromMarkdown(markdown)); + return count; +}; + +// Each marker healing appends must close one the prefix left literal, so the +// literal count drops by at least as many as were appended. A closer that +// renders as literal text matched nothing, so remend invented it. +const assertAppendedClosersMatch = (prefix: string): void => { + const healed = remend(prefix); + const suffix = healed.slice(prefix.length); + if (!(healed.startsWith(prefix) && APPENDED_CLOSERS.test(suffix))) { + return; + } + const appended = suffix.match(MARKER_PATTERN)?.length ?? 0; + if (countLiteralMarkers(healed) > countLiteralMarkers(prefix) - appended) { + throw new Error( + `healing appended an unmatched closer: ${JSON.stringify(prefix)} -> ${JSON.stringify(healed)}` + ); + } +}; + +describe("appended closers", () => { + it("never appends an underscore closer that matches no opener", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + fc.pre( + !(TRAILING_WHITESPACE.test(prefix) || INDENTED_CODE.test(prefix)) + ); + assertAppendedClosersMatch(prefix); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); + + it("re-heals underscore runs to themselves", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + const healed = remend(prefix); + expect(remend(healed)).toBe(healed); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); +}); + describe("exhaustive prefix sweep", () => { // Every prefix of a fixed corpus, deterministically. The corpus mixes // constructs that interact. diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 120f6a29..2cfb02a6 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -54,4 +54,50 @@ describe("escaped underscores", () => { it("should treat the run after an escaped underscore as a delimiter", () => { expect(remend("\\___bold")).toBe("\\___bold__"); }); + + it("should not treat a run after an escaped backslash as escaped", () => { + expect(remend("\\\\__bold")).toBe("\\\\__bold__"); + }); +}); + +describe("underscore delimiter matching", () => { + it.each<[string, string]>([ + ["a run followed by whitespace", "__ foo"], + ["a spaced run before an escaped run", "a __ b \\__c"], + ["a spaced run before an escaped run at the start", "__ \\__alph"], + ["a run after a letter and before punctuation", "a__."], + ["a closer-only run with nothing open", "__ a__ a"], + ["a longer run that closes the opener", "__a____.a"], + ["an opener the rule of 3 keeps a __ closer from matching", "\\_____\\a"], + ["a leftover single underscore after an escape", "__\\__ a"], + ])("should not append a closer for %s", (_name, input) => { + expect(remend(input)).toBe(input); + }); + + it("should still close an opener after a spaced run", () => { + expect(remend("a __ b __bold")).toBe("a __ b __bold__"); + }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "an outer run after an escape", + input: "\\__._a", + expected: "\\__._a__", + }, + { name: "a run across an asterisk", input: "_a *_a", expected: "_a *_a*_" }, + { + name: "a run after an open code span", + input: "__`x", + expected: "__`x`__", + }, + { + name: "a run after an open code span ending in a backslash", + input: "__`x\\", + expected: "__`x\\`__", + }, + ])("should close $name in one pass", ({ input, expected }) => { + const healed = remend(input); + expect(healed).toBe(expected); + expect(remend(healed)).toBe(healed); + }); }); diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 8905ae68..581547b7 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -2,6 +2,7 @@ import { isInsideCodeBlock, isWithinCompleteInlineCode, } from "./code-block-utils"; +import { handleIncompleteInlineCode } from "./inline-code-handler"; import { boldItalicPattern, boldPattern, @@ -17,13 +18,18 @@ import { import { countDoublePairs, getScan, - inHtmlTagAt, inLinkUrlAt, inMathAt, + isEscaped, REGION, type TextScan, } from "./scan"; -import { isHorizontalRule, isWordChar } from "./utils"; +import { + closesAddedUnderscores, + findOpenUnderscoreRuns, + type OpenUnderscoreRun, +} from "./underscore-delimiters"; +import { isHorizontalRule, isWhitespaceChar, isWordChar } from "./utils"; // Helper function to check if an asterisk should be skipped const shouldSkipAsterisk = ( @@ -33,7 +39,7 @@ const shouldSkipAsterisk = ( nextChar: string ): boolean => { // Skip if escaped with backslash - if (prevChar === "\\") { + if (isEscaped(scan.text, index)) { return true; } @@ -75,9 +81,6 @@ const shouldSkipAsterisk = ( return false; }; -const isWhitespaceChar = (char: string): boolean => - char === " " || char === "\t" || char === "\n"; - const isWordInternalAsterisk = (prevChar: string, nextChar: string): boolean => Boolean(prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)); @@ -149,67 +152,6 @@ export const countSingleAsterisks = (text: string): number => { return count; }; -// Helper function to check if an underscore should be skipped -const shouldSkipUnderscore = ( - scan: TextScan, - index: number, - prevChar: string, - nextChar: string -): boolean => { - // Skip if escaped with backslash - if (prevChar === "\\") { - return true; - } - - // Skip if within math block - if (inMathAt(scan, index)) { - return true; - } - - // Skip if within a link or image URL - if (inLinkUrlAt(scan, index)) { - return true; - } - - // Skip if within an HTML tag (e.g. ) - if (inHtmlTagAt(scan, index)) { - return true; - } - - // Skip if part of __ - if (prevChar === "_" || nextChar === "_") { - return true; - } - - // Skip if underscore is word-internal (between word characters) - if (prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)) { - return true; - } - - return false; -}; - -export const countSingleUnderscores = (text: string): number => { - const scan = getScan(text); - let count = 0; - const len = text.length; - - for (let index = 0; index < len; index += 1) { - if (text[index] !== "_" || scan.regions[index] !== REGION.PROSE) { - continue; - } - - const prevChar = index > 0 ? text[index - 1] : ""; - const nextChar = index < len - 1 ? text[index + 1] : ""; - - if (!shouldSkipUnderscore(scan, index, prevChar, nextChar)) { - count += 1; - } - } - - return count; -}; - // Counts triple asterisks that are not part of quadruple or more asterisks // and not inside code regions export const countTripleAsterisks = (text: string): number => { @@ -240,125 +182,18 @@ export const countTripleAsterisks = (text: string): number => { const countDoubleAsterisks = (text: string): number => countDoublePairs(text, "*"); -// Whether the text has an unmatched __ delimiter, counted per maximal -// underscore run. -// -// Counting raw occurrences misreads identifiers: a name like snake__case -// contains __ but cannot open or close emphasis, and counting it either -// invents a closer (odd count) or pairs it against a real delimiter and -// swallows a closer that was needed (even count). Per run: -// -// - A run contributes floor(length / 2) pairs, and flips delimiter parity -// only when that is odd. __ and ___ flip; ____ does not. -// - A word-internal run (word characters on both sides) is part of an -// identifier, never a delimiter. -// - Runs inside code regions, math, link URLs, and HTML tags are skipped, -// matching the single-underscore handler's flanking rules. -// - A run alone on its line is a thematic break, not emphasis. -const isLineBoundaryChar = (char: string): boolean => - char === "" || char === " " || char === "\t" || char === "\n"; - -// isHorizontalRule scans the run's whole line, so its verdict is memoized -// per line to keep run counting linear when many runs share a line -interface ThematicBreakMemo { - lineEnd: number; - result: boolean; -} - -const isThematicBreakRun = ( - text: string, - runStart: number, - prevChar: string, - nextChar: string, - memo: ThematicBreakMemo -): boolean => { - // A thematic break line holds only markers and whitespace, so only runs - // flanked by whitespace or line boundaries can be part of one - if (!(isLineBoundaryChar(prevChar) && isLineBoundaryChar(nextChar))) { - return false; - } - if (runStart > memo.lineEnd) { - const lineEnd = text.indexOf("\n", runStart); - memo.lineEnd = lineEnd === -1 ? text.length : lineEnd; - memo.result = isHorizontalRule(text, runStart, "_"); - } - return memo.result; -}; +const isHalfClosedDoubleUnderscore = ( + run: OpenUnderscoreRun | undefined +): boolean => run !== undefined && run.count === 1 && run.length >= 2; -// Whether an underscore run at [runStart, runEnd) flips delimiter parity -const doubleUnderscoreRunFlips = ( - scan: TextScan, - initialRunStart: number, - runEnd: number, - memo: ThematicBreakMemo -): boolean => { - const text = scan.text; +// Healing that adds an underscore closer which cannot match would render it +// as literal text, so such a heal is dropped +const healUnderscores = (text: string, healed: string): string => + closesAddedUnderscores(text, healed) ? healed : text; - // A backslash escapes the first underscore of the run. The escaped - // underscore is literal punctuation, so the rest of the run still flanks - // as a delimiter. - let runStart = initialRunStart; - let escaped = false; - if (runStart > 0 && text[runStart - 1] === "\\") { - runStart += 1; - escaped = true; - } - const runLength = runEnd - runStart; - if (runLength < 2) { - return false; - } - - const beforeRun = runStart > 0 ? text[runStart - 1] : ""; - const prevChar = escaped ? "\\" : beforeRun; - const nextChar = runEnd < text.length ? text[runEnd] : ""; - if (isWordChar(prevChar) && isWordChar(nextChar)) { - return false; - } - if (isThematicBreakRun(text, runStart, prevChar, nextChar, memo)) { - return false; - } - if ( - inMathAt(scan, runStart) || - inLinkUrlAt(scan, runStart) || - inHtmlTagAt(scan, runStart) - ) { - return false; - } - - return Math.floor(runLength / 2) % 2 === 1; -}; - -const hasUnmatchedDoubleUnderscore = (text: string): boolean => { - const scan = getScan(text); - const n = text.length; - const memo: ThematicBreakMemo = { lineEnd: -1, result: false }; - let unmatched = false; - let i = 0; - - while (i < n) { - if (text[i] !== "_" || scan.regions[i] !== REGION.PROSE) { - i += 1; - continue; - } - - const runStart = i; - let runEnd = i + 1; - while ( - runEnd < n && - text[runEnd] === "_" && - scan.regions[runEnd] === REGION.PROSE - ) { - runEnd += 1; - } - i = runEnd; - - if (doubleUnderscoreRunFlips(scan, runStart, runEnd, memo)) { - unmatched = !unmatched; - } - } - - return unmatched; -}; +const innermostOpenUnderscoreRun = ( + text: string +): OpenUnderscoreRun | undefined => findOpenUnderscoreRuns(text).at(-1); // Helper to check if bold marker should not be completed const shouldSkipBoldCompletion = ( @@ -456,9 +291,7 @@ const shouldSkipItalicCompletion = ( }; // Completes incomplete italic formatting with double underscores (__) -export const handleIncompleteDoubleUnderscoreItalic = ( - text: string -): string => { +const healDoubleUnderscore = (text: string): string => { const italicMatch = matchTrailing(text, italicPattern); if (!italicMatch) { // Check for half-complete closing marker: __content_ should become __content__ @@ -475,9 +308,9 @@ export const handleIncompleteDoubleUnderscoreItalic = ( isInsideCodeBlock(text, markerIndex) || isWithinCompleteInlineCode(text, markerIndex) ) && - hasUnmatchedDoubleUnderscore(text) + isHalfClosedDoubleUnderscore(innermostOpenUnderscoreRun(text)) ) { - return `${text}_`; + return healUnderscores(text, `${text}_`); } } return text; @@ -498,8 +331,9 @@ export const handleIncompleteDoubleUnderscoreItalic = ( return text; } - if (hasUnmatchedDoubleUnderscore(text)) { - return `${text}__`; + const openRun = innermostOpenUnderscoreRun(text); + if (openRun && openRun.count >= 2) { + return healUnderscores(text, `${text}__`); } return text; @@ -515,7 +349,7 @@ const isLoneProseAsterisk = (scan: TextScan, i: number): boolean => { scan.regions[i] === REGION.PROSE && text[i - 1] !== "*" && text[i + 1] !== "*" && - text[i - 1] !== "\\" && + !isEscaped(text, i) && !inMathAt(scan, i) ); }; @@ -605,7 +439,7 @@ const findFirstSingleUnderscoreIndex = (text: string): number => { scan.regions[i] === REGION.PROSE && text[i - 1] !== "_" && text[i + 1] !== "_" && - text[i - 1] !== "\\" && + !isEscaped(text, i) && !inMathAt(scan, i) && !inLinkUrlAt(scan, i) ) { @@ -677,9 +511,7 @@ const handleTrailingAsterisksForUnderscore = (text: string): string | null => { }; // Completes incomplete italic formatting with single underscores (_) -export const handleIncompleteSingleUnderscoreItalic = ( - text: string -): string => { +const healSingleUnderscore = (text: string): string => { const singleUnderscoreMatch = matchTrailing(text, singleUnderscorePattern); if (!singleUnderscoreMatch) { @@ -713,14 +545,12 @@ export const handleIncompleteSingleUnderscoreItalic = ( return text; } - const singleUnderscores = countSingleUnderscores(text); - if (singleUnderscores % 2 === 1) { + if (innermostOpenUnderscoreRun(text)?.count === 1) { // Check if we need to insert _ before trailing ** for proper nesting - const trailingResult = handleTrailingAsterisksForUnderscore(text); - if (trailingResult !== null) { - return trailingResult; - } - return insertClosingUnderscore(text); + const healed = + handleTrailingAsterisksForUnderscore(text) ?? + insertClosingUnderscore(text); + return healUnderscores(text, healed); } return text; @@ -788,3 +618,36 @@ export const handleIncompleteBoldItalic = (text: string): string => { return text; }; + +// The inline code handler runs after these, so a closer appended while a code +// span is open would land inside the span. Heal against the closed span and +// keep that closing only when the heal adds a closer. +const healAfterOpenCodeSpan = ( + text: string, + heal: (text: string) => string +): string => { + const closed = handleIncompleteInlineCode(text); + if (closed === text) { + return heal(text); + } + const healed = heal(closed); + return healed === closed ? text : healed; +}; + +export const handleIncompleteDoubleUnderscoreItalic = (text: string): string => + healAfterOpenCodeSpan(text, healDoubleUnderscore); + +// Each heal closes only the innermost open run, so healing repeats until no +// opener is left. Every accepted heal closes a literal, so the loop ends. +const healUnderscoreStack = (text: string): string => { + let healed = text; + let next = healSingleUnderscore(healDoubleUnderscore(healed)); + while (next !== healed) { + healed = next; + next = healSingleUnderscore(healDoubleUnderscore(healed)); + } + return healed; +}; + +export const handleIncompleteSingleUnderscoreItalic = (text: string): string => + healAfterOpenCodeSpan(text, healUnderscoreStack); diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 0ecf9137..6a39f53d 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -17,7 +17,7 @@ import { INCOMPLETE_IMAGE_PLACEHOLDER, type LinkMode, } from "./link-image-handler"; -import { clearScanCache } from "./scan"; +import { clearScanCache, getScan, inMathAt, isEscaped, REGION } from "./scan"; import { handleIncompleteSetextHeading } from "./setext-heading-handler"; import { handleSingleTildeEscape } from "./single-tilde-handler"; import { handleIncompleteStrikethrough } from "./strikethrough-handler"; @@ -277,6 +277,16 @@ const getEnabledBuiltInHandlers = ( }); }; +// Whether the text ends with an unescaped backslash in prose +const isTrailingEscape = (text: string): boolean => { + const last = text.length - 1; + if (text[last] !== "\\" || isEscaped(text, last)) { + return false; + } + const scan = getScan(text); + return scan.regions[last] === REGION.PROSE && !inMathAt(scan, last); +}; + // Parses markdown text and removes incomplete tokens to prevent partial rendering const remend = (text: string, options?: RemendOptions): string => { if (!text || typeof text !== "string") { @@ -284,7 +294,7 @@ const remend = (text: string, options?: RemendOptions): string => { } // Remove trailing whitespace if it's not a double space - let result = + const result = text.endsWith(" ") && !text.endsWith(" ") ? text.slice(0, -1) : text; // Get enabled built-in handlers @@ -302,25 +312,39 @@ const remend = (text: string, options?: RemendOptions): string => { (a, b) => (a.handler.priority ?? 0) - (b.handler.priority ?? 0) ); - try { + const runHandlers = (input: string): string => { + let output = input; // Execute handlers in priority order for (const { handler, earlyReturn } of allHandlers) { - result = handler.handle(result); + output = handler.handle(output); // Check for early return condition (e.g., incomplete link marker) - if (earlyReturn?.(result)) { - return result; + if (earlyReturn?.(output)) { + return output; } } // A handler that removes a trailing fragment can expose a trailing space // (e.g. dropping an incomplete image). Strip it the same way the input // was stripped, so healed output re-heals to itself. - if (result.endsWith(" ") && !result.endsWith(" ")) { - return result.slice(0, -1); + if (output.endsWith(" ") && !output.endsWith(" ")) { + return output.slice(0, -1); + } + + return output; + }; + + try { + if (!isTrailingEscape(result)) { + return runHandlers(result); } - return result; + // A trailing backslash escapes whatever follows it, so a closer appended + // after it would render as text. Heal with the backslash escaped, which + // keeps it literal without changing how the markers before it flank. + const escaped = `${result}\\`; + const healed = runHandlers(escaped); + return healed === escaped ? result : healed; } finally { clearScanCache(); } diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 1bb9f74f..15245737 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -342,7 +342,7 @@ const nextSpanOpener = ( return -1; } const escaped = - next > i && text[next - 1] === "\\" && regions[next - 1] === REGION.PROSE; + next > i && regions[next - 1] === REGION.PROSE && isEscaped(text, next); if (regions[next] === REGION.PROSE && !escaped) { return next; } @@ -473,6 +473,15 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => scan.regions[position] === REGION.CODE_SPAN; +/** Whether the character at position follows an odd run of backslashes */ +export const isEscaped = (text: string, position: number): boolean => { + let backslashes = 0; + for (let i = position - 1; i >= 0 && text[i] === "\\"; i -= 1) { + backslashes += 1; + } + return backslashes % 2 === 1; +}; + /** Counts non-overlapping double-character pairs (**, ~~, $$) in prose */ export const countDoublePairs = (text: string, char: string): number => { const scan = getScan(text); @@ -480,7 +489,9 @@ export const countDoublePairs = (text: string, char: string): number => { let count = 0; for (let i = text.indexOf(pair); i !== -1; i = text.indexOf(pair, i)) { - if (scan.regions[i] === REGION.PROSE) { + // Inside math a backslash is TeX, so only markdown markers can be escaped + const escaped = char !== "$" && isEscaped(text, i); + if (scan.regions[i] === REGION.PROSE && !escaped) { count += 1; i += 2; } else { diff --git a/packages/remend/src/underscore-delimiters.ts b/packages/remend/src/underscore-delimiters.ts new file mode 100644 index 00000000..e4dceab7 --- /dev/null +++ b/packages/remend/src/underscore-delimiters.ts @@ -0,0 +1,199 @@ +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isEscaped, + REGION, + type TextScan, +} from "./scan"; +import { isWhitespaceChar, isWordChar } from "./utils"; + +/** An underscore delimiter run left open at the end of the text */ +export interface OpenUnderscoreRun { + /** Underscores still unmatched */ + count: number; + /** Unescaped length of the run before any were matched */ + length: number; +} + +interface DelimiterRun extends OpenUnderscoreRun { + canClose: boolean; + canOpen: boolean; + marker: Marker; +} + +type Marker = "*" | "_"; + +// The start and end of the text count as whitespace for flanking +const isWhitespaceOrEdge = (char: string): boolean => + char === "" || isWhitespaceChar(char); + +const isPunctuationChar = (char: string): boolean => + char === "_" || !(isWhitespaceOrEdge(char) || isWordChar(char)); + +// Flanking per CommonMark, with the stricter underscore rules: a run that +// flanks on both sides opens only after punctuation and closes only before +// punctuation, so word-internal runs (snake__case) can't open or close +const toDelimiterRun = ( + text: string, + index: number, + end: number, + prevChar: string +): DelimiterRun => { + const marker = text[index] as Marker; + const nextChar = end < text.length ? text[end] : ""; + const leftFlanking = !( + isWhitespaceOrEdge(nextChar) || + (isPunctuationChar(nextChar) && + !isWhitespaceOrEdge(prevChar) && + !isPunctuationChar(prevChar)) + ); + const rightFlanking = !( + isWhitespaceOrEdge(prevChar) || + (isPunctuationChar(prevChar) && + !isWhitespaceOrEdge(nextChar) && + !isPunctuationChar(nextChar)) + ); + const length = end - index; + if (marker === "*") { + return { + canClose: rightFlanking, + canOpen: leftFlanking, + count: length, + length, + marker, + }; + } + return { + canClose: rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)), + canOpen: leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)), + count: length, + length, + marker, + }; +}; + +const isSkippedMarker = (scan: TextScan, i: number, marker: Marker): boolean => + scan.text[i] !== marker || + scan.regions[i] !== REGION.PROSE || + inMathAt(scan, i) || + inLinkUrlAt(scan, i) || + inHtmlTagAt(scan, i); + +const collectDelimiterRuns = ( + scan: TextScan, + markers: readonly Marker[] +): DelimiterRun[] => { + const { text } = scan; + const runs: DelimiterRun[] = []; + + for (let i = 0; i < text.length; i += 1) { + const marker = text[i] as Marker; + if (!markers.includes(marker) || isSkippedMarker(scan, i, marker)) { + continue; + } + + let end = i + 1; + while (end < text.length && !isSkippedMarker(scan, end, marker)) { + end += 1; + } + + // A backslash escapes the run's first marker, which stays literal + // punctuation before the rest of the run + if (isEscaped(text, i)) { + if (end - i > 1) { + runs.push(toDelimiterRun(text, i + 1, end, marker)); + } + } else { + runs.push(toDelimiterRun(text, i, end, i > 0 ? text[i - 1] : "")); + } + i = end - 1; + } + + return runs; +}; + +// The "rule of 3": when either run can both open and close, they match only +// if their combined length is not a multiple of 3, unless both lengths are. +// Lengths are what remains of each run after earlier partial matches. +const violatesRuleOfThree = ( + opener: DelimiterRun, + closer: DelimiterRun +): boolean => + (closer.canOpen || opener.canClose) && + (opener.count + closer.count) % 3 === 0 && + !(opener.count % 3 === 0 && closer.count % 3 === 0); + +// Matches a closing run against the stack until it runs out of markers or +// openers, popping every opener it passes +const matchCloser = (stack: DelimiterRun[], closer: DelimiterRun): void => { + while (closer.count > 0) { + let openerIndex = stack.length - 1; + while ( + openerIndex >= 0 && + (stack[openerIndex].marker !== closer.marker || + violatesRuleOfThree(stack[openerIndex], closer)) + ) { + openerIndex -= 1; + } + if (openerIndex < 0) { + return; + } + + const opener = stack[openerIndex]; + const used = opener.count >= 2 && closer.count >= 2 ? 2 : 1; + opener.count -= used; + closer.count -= used; + stack.length = opener.count > 0 ? openerIndex + 1 : openerIndex; + } +}; + +// Runs the CommonMark emphasis algorithm over the text's delimiter runs, +// leaving each run's unmatched count and the openers still open +const matchDelimiterRuns = ( + text: string, + markers: readonly Marker[] +): { runs: DelimiterRun[]; stack: DelimiterRun[] } => { + const runs = collectDelimiterRuns(getScan(text), markers); + const stack: DelimiterRun[] = []; + + for (const run of runs) { + if (run.canClose) { + matchCloser(stack, run); + } + if (run.canOpen && run.count > 0) { + stack.push(run); + } + } + + return { runs, stack }; +}; + +/** The underscore openers the text leaves unmatched, outermost first */ +export const findOpenUnderscoreRuns = (text: string): OpenUnderscoreRun[] => + matchDelimiterRuns(text, ["_"]).stack; + +// Underscores that render as literal text: openers still open, openers a +// closer passed over, and closers left over. Asterisk runs take part because +// a matched asterisk pair discards the underscore openers between them. +const countLiteralUnderscores = (text: string): number => + matchDelimiterRuns(text, ["*", "_"]) + .runs.filter((run) => run.marker === "_") + .reduce((sum, run) => sum + run.count, 0); + +/** + * Whether every underscore that healing added to the text matched one the + * text left literal. An added closer can fail to match, for example under + * the rule of 3. + */ +export const closesAddedUnderscores = ( + text: string, + healed: string +): boolean => { + // Scan the text first, while the scan cache still holds it + const literal = countLiteralUnderscores(text); + return ( + countLiteralUnderscores(healed) <= literal - (healed.length - text.length) + ); +}; diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 06291fd0..59c1898e 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,6 +1,11 @@ import { letterNumberUnderscorePattern } from "./patterns"; import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; +const whitespaceCharPattern = /\s/; + +export const isWhitespaceChar = (char: string): boolean => + whitespaceCharPattern.test(char); + // OPTIMIZATION: Precompute which characters are word characters // Using ASCII fast path before falling back to Unicode regex export const isWordChar = (char: string): boolean => {