diff --git a/.changeset/remend-escaped-markers.md b/.changeset/remend-escaped-markers.md new file mode 100644 index 00000000..385c44f4 --- /dev/null +++ b/.changeset/remend-escaped-markers.md @@ -0,0 +1,11 @@ +--- +"remend": minor +--- + +Heal emphasis the way CommonMark and GFM parse it + +- Closes only openers that exist, innermost first, within the paragraph, list item or link text that holds them. An open code span closes before the emphasis around it. +- Leaves literal markers alone: `**Note:**This`, `*.ts`, `x**2`, `(+, -, *, /)`, fill-in blanks like `______`, and runs after an escaped marker like `\**`. A trailing backslash is escaped so it can't escape the closers. +- Stops closing an opener once its line has ended and the next line has started, since models close emphasis on the line that opens it. A stray `*` in code written outside a fence, like `char *name`, no longer italicizes what follows. +- Closes an inline code span only within its own block, and never with a closer that would start a fence. A `[` before a complete link stays literal instead of healing a second link. +- The `bold`, `italic`, `boldItalic` and `strikethrough` options still apply, but the built-in handlers they controlled are now one `emphasis` handler at priority 30. Custom handlers ordered against the old priorities may need new ones. diff --git a/.changeset/remend-link-block-scope.md b/.changeset/remend-link-block-scope.md new file mode 100644 index 00000000..fd0b372b --- /dev/null +++ b/.changeset/remend-link-block-scope.md @@ -0,0 +1,5 @@ +--- +"remend": patch +--- + +Complete unfinished links and images only in the current block: the paragraph, list item, heading or table row at the end of the text. An unclosed `[` or `](` in an earlier block no longer adds `](streamdown:incomplete-link)` to later text, or drops the text after it. diff --git a/.changeset/streamdown-tail-repair.md b/.changeset/streamdown-tail-repair.md new file mode 100644 index 00000000..304b699c --- /dev/null +++ b/.changeset/streamdown-tail-repair.md @@ -0,0 +1,5 @@ +--- +"streamdown": patch +--- + +While streaming, repair only the end of the message instead of the whole message on every update. The footnote and link-definition checks now scan only the new text. Rendered output is unchanged. diff --git a/apps/website/content/docs/termination.mdx b/apps/website/content/docs/termination.mdx index 6a1327f8..ad68f445 100644 --- a/apps/website/content/docs/termination.mdx +++ b/apps/website/content/docs/termination.mdx @@ -278,27 +278,24 @@ Built-in handlers use priorities 0-75: | `htmlTags` | 10 | | `setextHeadings` | 15 | | `links` | 20 | -| `boldItalic` | 30 | -| `bold` | 35 | -| `italic` | 40-42 | +| `emphasis` (bold, italic, bold italic, strikethrough) | 30 | | `inlineCode` | 50 | -| `strikethrough` | 60 | | `katex` | 70 | Custom handlers default to priority 100, running after all built-ins. Set a lower priority to run before specific built-ins. -**Example: Running Before Bold Handling** +**Example: Running Before Emphasis Handling** -To process content before bold formatting is completed (priority 30), set a lower priority: +To process content before emphasis is completed (priority 30), set a lower priority: ```typescript const preprocessHandler: RemendHandler = { name: 'preprocess', handle: (text) => { - // Transform content before bold is completed + // Transform content before emphasis is completed return text.replace(/\{\{highlight\}\}/g, '**'); }, - priority: 25, // Runs before bold (35) and after links (20) + priority: 25, // Runs before emphasis (30) and after links (20) }; ``` diff --git a/packages/remend/README.md b/packages/remend/README.md index 5b9b44a2..d8f050d3 100644 --- a/packages/remend/README.md +++ b/packages/remend/README.md @@ -131,11 +131,8 @@ Built-in handlers use priorities 0-75. Custom handlers default to 100 (run after | `htmlTags` | 10 | | `setextHeadings` | 15 | | `links` | 20 | -| `boldItalic` | 30 | -| `bold` | 35 | -| `italic` | 40-42 | +| `emphasis` (bold, italic, bold italic, strikethrough) | 30 | | `inlineCode` | 50 | -| `strikethrough` | 60 | | `katex` | 70 | | `inlineKatex` | 75 | | Custom (default) | 100 | diff --git a/packages/remend/__benchmarks__/stream-glitches.test.ts b/packages/remend/__benchmarks__/stream-glitches.test.ts new file mode 100644 index 00000000..27ee7495 --- /dev/null +++ b/packages/remend/__benchmarks__/stream-glitches.test.ts @@ -0,0 +1,219 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fromMarkdown } from "mdast-util-from-markdown"; +import { gfmStrikethroughFromMarkdown } from "mdast-util-gfm-strikethrough"; +import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough"; +import { it } from "vitest"; +import remend from "../src"; + +// Streams each response in a JSONL corpus through remend and counts frames +// whose render disagrees with the finished response's render. Run with +// CORPUS= [BASELINE=] [OUT=] + +interface Node { + children?: Node[]; + position?: { end: { offset?: number }; start: { offset?: number } }; + type: string; +} + +interface Sample { + model: string; + source: string; + text: string; +} + +type Heal = (text: string) => string; + +const toSample = (line: string): Sample => { + const value: unknown = JSON.parse(line); + if ( + typeof value !== "object" || + value === null || + !("text" in value) || + typeof value.text !== "string" + ) { + throw new Error(`Corpus line has no text: ${line.slice(0, 80)}`); + } + const field = (key: string): string => + key in value ? String(value[key as keyof typeof value]) : "unknown"; + return { model: field("model"), source: field("source"), text: value.text }; +}; + +const STEP = 4; +const STYLED = new Set(["emphasis", "strong", "delete", "inlineCode"]); +const ASCII_PUNCTUATION = /[!-/:-@[-`{-~]/; +const LEADING_TICKS = /^`+/; +const LETTER = /\p{L}/gu; +const DIGIT = /\d/g; +const LETTERS = /a+/g; +const ZERO_WIDTH_SPACE = "\u200B"; + +const parse = (markdown: string): Node => + fromMarkdown(markdown, { + extensions: [gfmStrikethrough()], + mdastExtensions: [gfmStrikethroughFromMarkdown()], + }); + +// Each source offset's rendering: "" when it is syntax, otherwise the inline +// styles around the character it shows +const renderStyles = (markdown: string, root: Node, shift = 0): string[] => { + const styles: string[] = new Array(markdown.length + shift).fill(""); + const visit = (node: Node, stack: string) => { + const start = node.position?.start.offset ?? 0; + const end = node.position?.end.offset ?? 0; + const style = STYLED.has(node.type) ? `${stack}/${node.type}` : stack; + if (node.type === "text") { + for (let i = start; i < end; i++) { + if ( + markdown[i] === "\\" && + i + 1 < end && + ASCII_PUNCTUATION.test(markdown[i + 1]) + ) { + i++; + } + styles[shift + i] = `v${style}`; + } + } else if (node.type === "inlineCode") { + const raw = markdown.slice(start, end); + const ticks = raw.length - raw.replace(LEADING_TICKS, "").length; + for (let i = start + ticks; i < end - ticks; i++) { + styles[shift + i] = `v${style}`; + } + } + for (const child of node.children ?? []) { + visit(child, style); + } + }; + visit(root, ""); + return styles; +}; + +// Top-level block starts, since emphasis never crosses them +const blockStarts = (root: Node): number[] => + (root.children ?? []).map((child) => child.position?.start.offset ?? 0); + +// Characters of the prefix whose rendering differs from the final render, +// plus characters the heal added that render as visible text +const mismatches = ( + prefix: string, + healed: string, + from: number, + final: string[] +): number => { + let cut = 0; + while (cut < prefix.length && prefix[cut] === healed[cut]) { + cut++; + } + const added = healed.length - prefix.length; + const frame = renderStyles( + healed.slice(from), + parse(healed.slice(from)), + from + ); + let count = 0; + for (let at = cut; at < cut + added; at++) { + if ((frame[at] ?? "") !== "" && healed[at] !== ZERO_WIDTH_SPACE) { + count++; + } + } + for (let i = from; i < prefix.length; i++) { + const at = i < cut ? i : i + added; + if ((frame[at] ?? "") !== final[i]) { + count++; + } + } + return count; +}; + +// A coarse shape of the text before a cut, for grouping examples +const shapeOf = (tail: string): string => + tail.replace(LETTER, "a").replace(DIGIT, "0").replace(LETTERS, "a"); + +interface Arm { + frames: number; + glitches: number; +} + +const measure = (samples: Sample[], arms: Record) => { + const byGroup = new Map>(); + const divergent = { baselineCloser: 0, branchCloser: 0, tie: 0 }; + const worse = new Map(); + for (const sample of samples) { + const group = `${sample.source} ${sample.model}`; + const totals = + byGroup.get(group) ?? + Object.fromEntries( + Object.keys(arms).map((name) => [name, { frames: 0, glitches: 0 }]) + ); + byGroup.set(group, totals); + const finalTree = parse(sample.text); + const final = renderStyles(sample.text, finalTree); + const starts = blockStarts(finalTree); + for (let end = STEP; end < sample.text.length; end += STEP) { + const prefix = sample.text.slice(0, end); + const from = starts.findLast((start) => start < end) ?? 0; + const healed = Object.fromEntries( + Object.entries(arms).map(([name, heal]) => [name, heal(prefix)]) + ); + const counts: Record = {}; + for (const name of Object.keys(arms)) { + counts[name] = mismatches(prefix, healed[name], from, final); + totals[name].frames++; + if (counts[name] > 0) { + totals[name].glitches++; + } + } + if (arms.baseline && healed.baseline !== healed.branch) { + if (counts.branch < counts.baseline) { + divergent.branchCloser++; + } else if (counts.branch > counts.baseline) { + divergent.baselineCloser++; + const tail = prefix.slice(Math.max(from, end - 24)); + const shape = shapeOf(tail); + const entry = worse.get(shape) ?? { + count: 0, + example: JSON.stringify({ + tail, + baseline: healed.baseline.slice(Math.max(from, end - 24)), + branch: healed.branch.slice(Math.max(from, end - 24)), + final: sample.text.slice(Math.max(from, end - 24), end + 24), + }), + }; + entry.count++; + worse.set(shape, entry); + } else { + divergent.tie++; + } + } + } + } + return { + divergent, + groups: Object.fromEntries(byGroup), + worse: [...worse.entries()] + .sort((a, b) => b[1].count - a[1].count) + .slice(0, 40) + .map(([shape, entry]) => ({ shape, ...entry })), + }; +}; + +it.skipIf(!process.env.CORPUS)( + "measures streaming glitches against the final render", + async () => { + const samples: Sample[] = readFileSync(process.env.CORPUS ?? "", "utf8") + .split("\n") + .filter(Boolean) + .map(toSample); + const arms: Record = { branch: remend }; + if (process.env.BASELINE) { + const baseline = await import(resolve(process.env.BASELINE)); + arms.baseline = baseline.default; + } + const report = measure(samples, arms); + writeFileSync( + process.env.OUT ?? "stream-glitches.json", + JSON.stringify(report, null, 2) + ); + }, + 24 * 60 * 60 * 1000 +); diff --git a/packages/remend/__tests__/bold-italic.test.ts b/packages/remend/__tests__/bold-italic.test.ts index b5d62f91..06b868f0 100644 --- a/packages/remend/__tests__/bold-italic.test.ts +++ b/packages/remend/__tests__/bold-italic.test.ts @@ -39,7 +39,7 @@ describe("bold-italic formatting (***)", () => { it("should handle nested formatting with triple asterisks", () => { expect(remend("***bold-italic with `code")).toBe( - "***bold-italic with `code***`" + "***bold-italic with `code`***" ); }); diff --git a/packages/remend/__tests__/broken-markdown-variants.test.ts b/packages/remend/__tests__/broken-markdown-variants.test.ts index b483fddc..f1d0b4e3 100644 --- a/packages/remend/__tests__/broken-markdown-variants.test.ts +++ b/packages/remend/__tests__/broken-markdown-variants.test.ts @@ -3,30 +3,24 @@ import remend from "../src"; // 1. Rapid successive formatting switches describe("rapid successive formatting switches", () => { - it("should close italic and strikethrough but not bold when asterisk in content", () => { - // Bold pattern can't match when there's a * in content between ** and end - // Only italic (*) and strikethrough (~~) close + it("should close strikethrough, italic and bold innermost first", () => { const result = remend("**bold then *italic then ~~strike"); - expect(result).toBe("**bold then *italic then ~~strike*~~"); + expect(result).toBe("**bold then *italic then ~~strike~~***"); }); - it("should close italic and strikethrough when bold pattern blocked", () => { - // Bold pattern blocked by * in content; italic and strikethrough close + it("should close italic and bold inside strikethrough", () => { const result = remend("~~strike **bold *italic"); - expect(result).toBe("~~strike **bold *italic*~~"); + expect(result).toBe("~~strike **bold *italic***~~"); }); - it("should close handlers in priority order (bold before strikethrough before code)", () => { - // Bold can't match (asterisk in content), italic appends *, boldItalic appends ***, - // then inline code appends `, then strikethrough appends ~~ + it("should close the code span before the emphasis around it", () => { const result = remend("*italic **bold ~~strike `code"); - expect(result).toBe("*italic **bold ~~strike `code***`~~"); + expect(result).toBe("*italic **bold ~~strike `code`~~***"); }); - it("should close bold before strikethrough (priority order)", () => { - // Bold handler (priority 35) runs before strikethrough (priority 60) + it("should close strikethrough before the bold around it", () => { const result = remend("**bold ~~strike"); - expect(result).toBe("**bold ~~strike**~~"); + expect(result).toBe("**bold ~~strike~~**"); }); it("should close italic then bold via bold-italic handler", () => { @@ -77,17 +71,14 @@ describe("backslash escapes with incomplete formatting", () => { expect(result).toBe("\\*not italic"); }); - it("should not close double-escaped backslash before asterisk", () => { - // remend sees the char before * as \ (the second backslash) and treats it as escaped + it("should close an asterisk after an escaped backslash", () => { const result = remend("\\\\*actually italic"); - expect(result).toBe("\\\\*actually italic"); + expect(result).toBe("\\\\*actually italic*"); }); - it("should close escaped double asterisks (remend does not track escape depth)", () => { - // remend doesn't understand that \** has the first * escaped; - // it sees ** and closes bold + it("should leave the asterisk after an escaped one for the author to close", () => { const result = remend("\\**not bold"); - expect(result).toBe("\\**not bold**"); + expect(result).toBe("\\**not bold"); }); it("should handle mixed escaped and real formatting", () => { @@ -320,25 +311,19 @@ describe("formatting at paragraph boundaries", () => { // 13. Deeply nested formatting (4+ levels) describe("deeply nested formatting", () => { - it("should close handlers in priority order with deeply nested formatting", () => { - // Bold can't match (asterisk in content), italic appends *, - // inline code appends `, strikethrough appends ~~ + it("should close deeply nested formatting innermost first", () => { const result = remend("**bold *italic ~~strike `code"); - expect(result).toBe("**bold *italic ~~strike `code*`~~"); + expect(result).toBe("**bold *italic ~~strike `code`~~***"); }); - it("should close bold-italic then code then strikethrough", () => { - // BoldItalic (priority 30) appends ***, then inline code (50) appends `, - // then strikethrough (60) appends ~~ + it("should close code then strikethrough then bold-italic", () => { const result = remend("***bold-italic ~~strike `code"); - expect(result).toBe("***bold-italic ~~strike `code***`~~"); + expect(result).toBe("***bold-italic ~~strike `code`~~***"); }); - it("should close italic but not bold when asterisk blocks bold pattern", () => { - // Bold handler can't match when * appears in content after ** - // Italic handler closes the *, leaving ** unclosed + it("should close italic then the bold around it", () => { const result = remend("**bold and *italic"); - expect(result).toBe("**bold and *italic*"); + expect(result).toBe("**bold and *italic***"); }); }); @@ -442,15 +427,14 @@ describe("back-to-back code blocks", () => { // 19. Confusing asterisk sequences describe("confusing asterisk sequences", () => { - it("should handle four asterisks (bold-italic handler appends ***)", () => { - // BoldItalic sees odd triple-asterisk count and markers not balanced, appends *** + it("should close four asterisks with four", () => { const result = remend("****text"); - expect(result).toBe("****text***"); + expect(result).toBe("****text****"); }); - it("should handle five asterisks (bold-italic handler appends ***)", () => { + it("should close five asterisks with five", () => { const result = remend("*****text"); - expect(result).toBe("*****text***"); + expect(result).toBe("*****text*****"); }); it("should handle mixed asterisk counts", () => { @@ -466,8 +450,9 @@ describe("whitespace edge cases", () => { expect(remend("**bold\twith\ttabs")).toBe("**bold\twith\ttabs**"); }); - it("should close bold with CRLF", () => { - expect(remend("**bold\r\nwith CRLF")).toBe("**bold\r\nwith CRLF**"); + it("should close bold before a CRLF but not after the next line starts", () => { + expect(remend("**bold\r\n")).toBe("**bold**\r\n"); + expect(remend("**bold\r\nwith CRLF")).toBe("**bold\r\nwith CRLF"); }); it("should close bold after many leading newlines", () => { diff --git a/packages/remend/__tests__/coverage-gaps.test.ts b/packages/remend/__tests__/coverage-gaps.test.ts index fa04e73c..1bb88b13 100644 --- a/packages/remend/__tests__/coverage-gaps.test.ts +++ b/packages/remend/__tests__/coverage-gaps.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; import remend from "../src"; -import { countTripleAsterisks } from "../src/emphasis-handlers"; import { isHorizontalRule, isWithinHtmlTag, @@ -36,24 +35,6 @@ describe("bold-italic inside code block", () => { }); }); -describe("countTripleAsterisks", () => { - it("should count trailing *** at end of text", () => { - expect(countTripleAsterisks("text***")).toBe(1); - }); - - it("should skip *** inside code blocks", () => { - expect(countTripleAsterisks("```\n***\n```")).toBe(0); - }); - - it("should count *** outside but not inside code blocks", () => { - expect(countTripleAsterisks("```\n***\n```\n***")).toBe(1); - }); - - it("should flush pending asterisks before code block toggle", () => { - expect(countTripleAsterisks("***```code```")).toBe(1); - }); -}); - describe("single underscore counting with code blocks", () => { it("should skip _ inside fenced code blocks", () => { expect(remend("```\n_code\n```\n_text")).toBe("```\n_code\n```\n_text_"); diff --git a/packages/remend/__tests__/emphasis-fast-path.test.ts b/packages/remend/__tests__/emphasis-fast-path.test.ts new file mode 100644 index 00000000..930c5013 --- /dev/null +++ b/packages/remend/__tests__/emphasis-fast-path.test.ts @@ -0,0 +1,76 @@ +import fc from "fast-check"; +import { expect, it } from "vitest"; +import { + matchDelimiterRuns, + matchHealed, + prepareHeal, +} from "../src/emphasis-delimiters"; +import { isEscaped } from "../src/scan"; + +const TOKENS = [ + "a", + " ", + ".", + "*", + "**", + "_", + "__", + "~~", + "~", + "`", + "\\", + "\n", + "[", + "](u)", + "ि", + "中", + ":", + "$", + "", + "𠀀", + "\n\n", + "- ", + "> ", +]; + +const WHITESPACE = /\s/; +const LINE_CONTENT = /[^\s*_~>+-]/; + +const summary = (m: ReturnType) => ({ + literal: m.literal, + open: m.open.map((r) => `${r.index}:${r.count}`), + runs: m.runs.map((r) => `${r.index}:${r.length}:${r.count}`), +}); + +it("fast path matches a full rescan", () => { + fc.assert( + fc.property( + fc.array(fc.constantFrom(...TOKENS), { minLength: 1, maxLength: 9 }), + fc.array(fc.constantFrom("*", "_", "~", "**", "~~"), { + minLength: 1, + maxLength: 4, + }), + fc.constantFrom("", " ", "\n", " \n", "\\"), + fc.boolean(), + (tokens, closerParts, tail, inlineMath) => { + const text = tokens.join("") + tail; + let insertAt = text.length; + while (insertAt > 0 && WHITESPACE.test(text[insertAt - 1])) { + insertAt -= 1; + } + if (text[insertAt - 1] === "\\" && !isEscaped(text, insertAt - 1)) { + insertAt -= 1; + } + const lineStart = text.lastIndexOf("\n", insertAt - 1) + 1; + fc.pre(LINE_CONTENT.test(text.slice(lineStart, insertAt))); + const closers = closerParts.join(""); + const healed = text.slice(0, insertAt) + closers + text.slice(insertAt); + const base = prepareHeal(text, insertAt, inlineMath); + expect(summary(matchHealed(base, closers))).toEqual( + summary(matchDelimiterRuns(healed, inlineMath)) + ); + } + ), + { numRuns: Number(process.env.NUM_RUNS ?? 20_000) } + ); +}); diff --git a/packages/remend/__tests__/emphasis-nesting.test.ts b/packages/remend/__tests__/emphasis-nesting.test.ts new file mode 100644 index 00000000..6ed88ebb --- /dev/null +++ b/packages/remend/__tests__/emphasis-nesting.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +interface Row { + expected: string; + input: string; + name: string; +} + +describe("emphasis nesting", () => { + it.each([ + { + name: "strikethrough inside bold", + input: "**bold ~~strike", + expected: "**bold ~~strike~~**", + }, + { + name: "italic inside bold inside strikethrough", + input: "~~strike **bold *italic", + expected: "~~strike **bold *italic***~~", + }, + { + name: "code span inside bold", + input: "**bold with `code", + expected: "**bold with `code`**", + }, + { + name: "closed italic after an open bold", + input: "**a_~~*~~*", + expected: "**a_~~*~~***", + }, + ])("closes in nesting order: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { + name: "ordered list items", + input: "1. **a\n2. **b", + expected: "1. **a\n2. **b**", + }, + { + name: "paragraphs", + input: "**Summary\n\nThe *quick", + expected: "**Summary\n\nThe *quick*", + }, + { + name: "blockquotes", + input: "> **a\n\n> **b", + expected: "> **a\n\n> **b**", + }, + { + name: "bold left open when its line ended", + input: "- **text\nmore text", + expected: "- **text\nmore text", + }, + { + name: "bold after a closer that failed flanking", + input: "**Note:**This is **imp", + expected: "**Note:**This is **imp**", + }, + { + name: "bold around a pair opened after fullwidth punctuation", + input: "**机器学习(ML)**是一个分支,能够**从数", + expected: "**机器学习(ML)**是一个分支,能够**从数**", + }, + { + name: "asterisk in link text resolves before strikethrough", + input: "[*](u)~~*~~*a", + expected: "[*](u)~~*~~*a~~", + }, + { + name: "dash that starts the next list item", + input: "_*\n- ", + expected: "_*\n-\u200B", + }, + { + name: "bold before a dash rule", + input: "**-----", + expected: "**-----**", + }, + { + name: "paragraph after an empty list item", + input: "*\n~~**a ~~`", + expected: "*\n~~**a ~~`**", + }, + ])("pairs within one block: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { + name: "after an odd number of prices", + input: "Pay $50 now, then **$1,15", + expected: "Pay $50 now, then **$1,15**", + }, + { + name: "after prices in an earlier paragraph", + input: "From $5 to $9 and $12.\n\n**Total: $276.", + expected: "From $5 to $9 and $12.\n\n**Total: $276.**", + }, + { + name: "beside display math", + input: "$$x**2$$ is **big", + expected: "$$x**2$$ is **big**", + }, + ])("treats a lone dollar sign as text: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it("hides markers inside inline math when it is opted in", () => { + expect(remend("Pay $a **b$ and **c", { inlineKatex: true })).toBe( + "Pay $a **b$ and **c**" + ); + }); + + it.each([ + { + name: "closer before a word", + input: "**Note:**This is important.", + expected: "**Note:**This is important.", + }, + { + name: "closer after fullwidth punctuation", + input: "**注意:**这是重要的。", + expected: "**注意:**这是重要的。", + }, + { + name: "globs", + input: "Matches *.ts and *.tsx files", + expected: "Matches *.ts and *.tsx files", + }, + { + name: "field names", + input: "Fields _id, _rev, _key", + expected: "Fields _id, _rev, _key", + }, + { + name: "word-internal ASCII asterisks", + input: "hello**world", + expected: "hello**world", + }, + { + name: "fill-in blank before punctuation", + input: "Can we discuss ______? It is", + expected: "Can we discuss ______? It is", + }, + { + name: "fill-in blank between fullwidth punctuation", + input: "电话:__________(可向", + expected: "电话:__________(可向", + }, + { + name: "asterisk chain that opened inside a word", + input: 'Watch "M*A*S*H" for the', + expected: 'Watch "M*A*S*H" for the', + }, + { + name: "asterisk listed between commas", + input: "Use operators (+, -, *, /), equals, and", + expected: "Use operators (+, -, *, /), equals, and", + }, + { + name: "pointer in code outside a fence", + input: "int f(car *cars, int n) {\n return", + expected: "int f(car *cars, int n) {\n return", + }, + { + name: "asterisk between an ASCII letter and CJK", + input: "使用A*算法", + expected: "使用A*算法", + }, + { + name: "opener inside a link label", + input: "[*a](u)", + expected: "[*a](u)", + }, + { + name: "strikethrough around a link nested in brackets", + input: "[[](u)~~](u)~~]", + expected: "[[](u)~~](u)~~]", + }, + ])("keeps literal markers: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { + name: "bold after CJK text", + input: "这是**粗体", + expected: "这是**粗体**", + }, + { + name: "bold inside Thai text", + input: "คำ**ที่**ดี", + expected: "คำ**ที่**ดี", + }, + ])("handles Unicode: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each([ + { name: "many open italic runs", input: " *x".repeat(200) }, + { + name: "many distinct open runs", + input: [ + ..."123456".split("").map((n) => ` ${"*".repeat(Number(n))}a`), + ..."12345".split("").map((n) => ` ${"_".repeat(Number(n))}b`), + " ~~c", + ].join(""), + }, + { + name: "long prose with an open bold", + input: `${"prose with *emph* and **bold** here. ".repeat(300)}**tail`, + }, + ])("stays fast: $name", ({ input }) => { + remend(input); + const start = performance.now(); + remend(input); + expect(performance.now() - start).toBeLessThan(50); + }); +}); diff --git a/packages/remend/__tests__/emphasis-oracle.test.ts b/packages/remend/__tests__/emphasis-oracle.test.ts new file mode 100644 index 00000000..56b053cd --- /dev/null +++ b/packages/remend/__tests__/emphasis-oracle.test.ts @@ -0,0 +1,213 @@ +import fc from "fast-check"; +import { fromMarkdown } from "mdast-util-from-markdown"; +import { gfmStrikethroughFromMarkdown } from "mdast-util-gfm-strikethrough"; +import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough"; +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +interface Node { + children?: Node[]; + position?: { end: { offset?: number }; start: { offset?: number } }; + type: string; +} + +const parse = (markdown: string): Node => + fromMarkdown(markdown, { + extensions: [gfmStrikethrough()], + mdastExtensions: [gfmStrikethroughFromMarkdown()], + }); + +const span = (node: Node): [number, number] => [ + node.position?.start.offset ?? 0, + node.position?.end.offset ?? 0, +]; + +const INLINE = new Set(["emphasis", "strong", "delete", "inlineCode"]); +const TRANSPARENT = new Set(["link", "image", "text", "break"]); +const MARKER_PATTERN = /[*_~`]/g; +const ASCII_PUNCTUATION = /[!-/:-@[-`{-~]/; +const LEADING_TICKS = /^`+/; +const LEADING_MARKERS = /^[*_~`]+/; +const TRAILING_MARKERS = /[*_~`]+$/; +const CONTENT_PATTERN = /[^\s*_~`]/; + +const literal = (markdown: string): number => { + let count = 0; + const visit = (node: Node) => { + if (node.type === "text") { + const [start, end] = span(node); + count += markdown.slice(start, end).match(MARKER_PATTERN)?.length ?? 0; + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(parse(markdown)); + return count; +}; + +// Inline nodes drop out so a heal can only change structure by changing blocks +const blockShape = (node: Node): string => + INLINE.has(node.type) || TRANSPARENT.has(node.type) + ? "" + : `${node.type}(${(node.children ?? []).map(blockShape).filter(Boolean).join(",")})`; + +const visibleOffsets = (markdown: string, root: Node): Set => { + const seen = new Set(); + const visit = (node: Node) => { + const [start, end] = span(node); + if (node.type === "text") { + for (let i = start; i < end; i++) { + if ( + markdown[i] === "\\" && + i + 1 < end && + ASCII_PUNCTUATION.test(markdown[i + 1]) + ) { + seen.add(markdown[i + 1] === "\\" ? i : i + 1); + i++; + } else { + seen.add(i); + } + } + } else if (node.type === "inlineCode") { + const raw = markdown.slice(start, end); + const ticks = raw.length - raw.replace(LEADING_TICKS, "").length; + for (let i = start + ticks; i < end - ticks; i++) { + seen.add(i); + } + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(root); + return seen; +}; + +const inlineSpans = (root: Node): Set => { + const spans = new Set(); + const visit = (node: Node) => { + if (INLINE.has(node.type)) { + spans.add(`${node.type}:${span(node).join(":")}`); + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(root); + return spans; +}; + +// A heal is faithful when it keeps the block structure, keeps the visible text the same, +// and every span it closes holds authored content +const faithful = (input: string, healed: string): boolean => { + if (healed === input) { + return true; + } + let cut = 0; + while (cut < input.length && input[cut] === healed[cut]) { + cut++; + } + const inputTree = parse(input); + const healedTree = parse(healed); + if (blockShape(healedTree) !== blockShape(inputTree)) { + return false; + } + const shown = visibleOffsets(healed, healedTree); + for (let i = cut; i < cut + healed.length - input.length; i++) { + if (shown.has(i)) { + return false; + } + } + const before = inlineSpans(inputTree); + let ok = true; + const visit = (node: Node) => { + const [start, end] = span(node); + if ( + INLINE.has(node.type) && + (end > cut || !before.has(`${node.type}:${start}:${end}`)) + ) { + const raw = healed.slice(start, end); + const lead = raw.length - raw.replace(LEADING_MARKERS, "").length; + const trail = raw.length - raw.replace(TRAILING_MARKERS, "").length; + if ( + !CONTENT_PATTERN.test( + healed.slice(start + lead, Math.min(end - trail, cut)) + ) + ) { + ok = false; + } + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(healedTree); + return ok; +}; + +const WITHOUT_EMPHASIS = { + bold: false, + boldItalic: false, + italic: false, + strikethrough: false, +}; + +const healsCleanly = (input: string, heal: (text: string) => string) => { + const healed = heal(input); + return heal(healed) === healed && faithful(input, healed); +}; + +const ALPHABETS = [ + { + name: "ASCII", + tokens: ["a", " ", ".", "*", "**", "_", "__", "`", "\\", "~~", "\n"], + }, + { + name: "Unicode", + tokens: ["a", "ि", ":", "中", "𠀀", " ", "*", "**", "_", "~~", "."], + }, + { + name: "links", + tokens: ["a", " ", "*", "**", "_", "~~", "`", "[", "](u)", "]"], + }, +]; + +describe("emphasis oracle", () => { + it.each([ + { name: "plain", input: "a", count: 0 }, + { name: "unmatched opener", input: "*a", count: 1 }, + { name: "matched pair", input: "*a*", count: 0 }, + { name: "code span", input: "`*a`", count: 0 }, + ])("literal counts $name", ({ input, count }) => { + expect(literal(input)).toBe(count); + }); + + it.each([ + { name: "closing a span", input: "*a", healed: "*a*", expected: true }, + { name: "empty span", input: "a *", healed: "a **", expected: false }, + { name: "visible addition", input: "*a", healed: "*a*b", expected: false }, + { name: "block change", input: "a\n*", healed: "a\n***", expected: false }, + ])("faithful rejects or accepts $name", ({ input, healed, expected }) => { + expect(faithful(input, healed)).toBe(expected); + }); + + it.each(ALPHABETS)("heals $name samples idempotently and faithfully", ({ + tokens, + }) => { + fc.assert( + fc.property( + fc.array(fc.constantFrom(...tokens), { minLength: 1, maxLength: 8 }), + (parts) => { + const input = parts.join(""); + // Samples the other handlers already mangle say nothing about emphasis + fc.pre(healsCleanly(input, (text) => remend(text, WITHOUT_EMPHASIS))); + const healed = remend(input); + expect(remend(healed)).toBe(healed); + expect(faithful(input, healed)).toBe(true); + } + ), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }, 120_000); +}); diff --git a/packages/remend/__tests__/escaped-markers.test.ts b/packages/remend/__tests__/escaped-markers.test.ts new file mode 100644 index 00000000..9cb3b220 --- /dev/null +++ b/packages/remend/__tests__/escaped-markers.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("escaped markers", () => { + it.each<{ name: string; input: string; expected: string }>([ + { + name: "closes bold whose content holds an escaped **", + input: "**a\\**b", + expected: "**a\\**b**", + }, + { + name: "skips an escaped ~~", + input: "\\~~strike", + expected: "\\~~strike", + }, + { + name: "closes ** after an escaped backslash", + input: "\\\\**bold", + expected: "\\\\**bold**", + }, + { + name: "closes ~~ after an escaped backslash", + input: "\\\\~~strike", + expected: "\\\\~~strike~~", + }, + { + name: "closes * after an escaped backslash", + input: "\\\\*foo", + expected: "\\\\*foo*", + }, + { + name: "closes _ after an escaped backslash", + input: "\\\\_foo", + expected: "\\\\_foo_", + }, + { + name: "opens a code span after an escaped backslash", + input: "\\\\``a", + expected: "\\\\``a``", + }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); + +describe("math", () => { + // Inside math a backslash is TeX, so it does not escape the closing $$ + it.each<{ name: string; input: string }>([ + { name: "a backslash before $$", input: "$$a\\$$" }, + { name: "a lone backslash", input: "$$\\$$" }, + { name: "inline block math", input: "text $$\\frac\\$$ more" }, + ])("leaves complete math with $name unchanged", ({ input }) => { + expect(remend(input)).toBe(input); + }); +}); + +describe("trailing backslash", () => { + // A closer appended after a trailing backslash would be escaped by it, so + // the backslash is escaped first + it.each<{ name: string; input: string; expected: string }>([ + { name: "**", input: "**foo\\", expected: "**foo\\\\**" }, + { name: "*", input: "*foo\\", expected: "*foo\\\\*" }, + { name: "_", input: "_foo\\", expected: "_foo\\\\_" }, + { name: "__", input: "__foo\\", expected: "__foo\\\\__" }, + { name: "~~", input: "~~foo\\", expected: "~~foo\\\\~~" }, + { + name: "a partial __ match", + input: "___.\\", + expected: "___.\\\\___", + }, + { + name: "a backslash after whitespace", + input: "**foo \\", + expected: "**foo \\\\**", + }, + { + name: "nothing to heal", + input: "plain text\\", + expected: "plain text\\", + }, + { + name: "an opener whose only content is the backslash", + input: "______\n__\\", + expected: "______\n__\\\\__", + }, + { + name: "an escaped trailing backslash", + input: "**foo\\\\", + expected: "**foo\\\\**", + }, + { name: "inline code", input: "`foo\\", expected: "`foo\\`" }, + ])("$name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + }); +}); diff --git a/packages/remend/__tests__/inline-code.test.ts b/packages/remend/__tests__/inline-code.test.ts index b24cecc2..d9ea2d37 100644 --- a/packages/remend/__tests__/inline-code.test.ts +++ b/packages/remend/__tests__/inline-code.test.ts @@ -91,4 +91,64 @@ describe("emphasis markers inside inline code spans should not leak", () => { it("should complete emphasis after a closed inline code span", () => { expect(remend("`code` **bold")).toBe("`code` **bold**"); }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "opener before a setext underline", + input: "`a\n---", + expected: "`a\n---", + }, + { + name: "opener in an earlier list item", + input: "- `a\n- b", + expected: "- `a\n- b", + }, + { + name: "opener in a heading", + input: "# `a\nb", + expected: "# `a\nb", + }, + { + name: "lazy continuation line", + input: "text `a\nb", + expected: "text `a\nb`", + }, + { + name: "lone star inside a paragraph", + input: "text `a\n*\nb", + expected: "text `a\n*\nb`", + }, + { + name: "lone star inside a quoted paragraph", + input: "> text `a\n> *\n> b", + expected: "> text `a\n> *\n> b`", + }, + { + name: "opener before an empty item after a quote", + input: "> text `a\n*\nb", + expected: "> text `a\n*\nb", + }, + ])("closes only within the opener's block: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "one trailing backtick", + input: "a```a\n`", + expected: "a```a\n` ```", + }, + { + name: "two trailing backticks", + input: "a```a\n``", + expected: "a```a\n`` ```", + }, + ])("keeps a closer at line start from opening a fence: $name", ({ + input, + expected, + }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); }); diff --git a/packages/remend/__tests__/italic.test.ts b/packages/remend/__tests__/italic.test.ts index e0233cda..60e1c696 100644 --- a/packages/remend/__tests__/italic.test.ts +++ b/packages/remend/__tests__/italic.test.ts @@ -120,8 +120,7 @@ describe("italic formatting with single underscores (_)", () => { expect(remend("naïve_approach")).toBe("naïve_approach"); }); - it("should not count word-internal single underscores in countSingleUnderscores", () => { - // This tests the path where underscore is between word characters (lines 106-108) + it("should not count word-internal single underscores", () => { expect(remend("some_variable_name")).toBe("some_variable_name"); expect(remend("test_123_value")).toBe("test_123_value"); expect(remend("_start with underscore")).toBe("_start with underscore_"); @@ -137,7 +136,8 @@ describe("italic formatting with single underscores (_)", () => { it("should handle incomplete single underscore with trailing newlines", () => { expect(remend("Text with _italic\n")).toBe("Text with _italic_\n"); - expect(remend("_incomplete\n\n")).toBe("_incomplete_\n\n"); + // A blank line ends the paragraph, so the opener can no longer close + expect(remend("_incomplete\n\n")).toBe("_incomplete\n\n"); expect(remend("Start _text\n")).toBe("Start _text_\n"); }); }); diff --git a/packages/remend/__tests__/link-block-scope.test.ts b/packages/remend/__tests__/link-block-scope.test.ts new file mode 100644 index 00000000..8dfaf0a5 --- /dev/null +++ b/packages/remend/__tests__/link-block-scope.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import remend from "../src"; + +describe("link healing stays in the last inline block", () => { + it("does not close a bracket from an earlier paragraph", () => { + expect(remend("See [1 for details.\n\nThe next paragraph")).toBe( + "See [1 for details.\n\nThe next paragraph" + ); + expect(remend("Arrays use x[i notation.\n\nThen we")).toBe( + "Arrays use x[i notation.\n\nThen we" + ); + }); + + it("keeps later paragraphs after an earlier unfinished URL", () => { + const text = + "Read [the docs](https://example.com/guide\n\nNext paragraph keeps streaming"; + expect(remend(text)).toBe(text); + }); + + it('does not pair a later "](" with a bracket from an earlier block', () => { + const text = "Text [a\n\nb](https://exa"; + expect(remend(text)).toBe(text); + }); + + it("does not move an image's later paragraphs into its alt text", () => { + expect(remend("![diagram\n\nThe caption follows")).toBe( + "![diagram\n\nThe caption follows" + ); + }); + + it("treats list items, headings and table rows as separate blocks", () => { + expect(remend("- item [one\n- item two")).toBe("- item [one\n- item two"); + expect(remend("# Title [x\nBody text")).toBe("# Title [x\nBody text"); + expect(remend("| a [b | c |\n| d | e")).toBe("| a [b | c |\n| d | e"); + }); + + it("still heals an incomplete link in the current block", () => { + expect(remend("Done.\n\nSee [the docs")).toBe( + "Done.\n\nSee [the docs](streamdown:incomplete-link)" + ); + expect(remend("See [1 here.\n\nRead [the docs](https://exa")).toBe( + "See [1 here.\n\nRead [the docs](streamdown:incomplete-link)" + ); + expect(remend("- item [one\n- item [two")).toBe( + "- item [one\n- item [two](streamdown:incomplete-link)" + ); + expect(remend("> quote [link\n> continues")).toBe( + "> quote [link\n> continues](streamdown:incomplete-link)" + ); + expect(remend("- item [link\n continues")).toBe( + "- item [link\n continues](streamdown:incomplete-link)" + ); + }); + + it("leaves earlier brackets alone in text-only mode", () => { + expect( + remend("See [1 for details.\n\nRead [the docs", { linkMode: "text-only" }) + ).toBe("See [1 for details.\n\nRead the docs"); + }); +}); diff --git a/packages/remend/__tests__/links.test.ts b/packages/remend/__tests__/links.test.ts index 45433029..cd07c379 100644 --- a/packages/remend/__tests__/links.test.ts +++ b/packages/remend/__tests__/links.test.ts @@ -69,6 +69,58 @@ describe("link handling", () => { "[link [nested] text](streamdown:incomplete-link)" ); }); + + // Links can't contain links, so a bracket before a complete link is literal + it.each<{ name: string; input: string; expected: string }>([ + { + name: "outer bracket around a healed link", + input: "[a[a", + expected: "[a[a](streamdown:incomplete-link)", + }, + { + name: "outer bracket around a complete link", + input: "[a [b](u) c", + expected: "[a [b](u) c", + }, + { + name: "image around a complete link", + input: "![a [b](u) c", + expected: "![a [b](u) c](streamdown:incomplete-image)", + }, + { + name: "open bracket after a link inside a literal bracket", + input: "[outer [inner](u) and [more", + expected: "[outer [inner](u) and [more](streamdown:incomplete-link)", + }, + { + name: "bracket after an unclosed backtick", + input: "[`[", + expected: "[`[](streamdown:incomplete-link)", + }, + { + name: "code span that swallows a complete link", + input: "[`[](u)", + expected: "[`[](u)`](streamdown:incomplete-link)", + }, + { + name: "brackets inside a complete code span", + input: "a[~~`[[`a", + expected: "a[~~`[[`a](streamdown:incomplete-link)", + }, + { + name: "label closed after a code span holding a bracket", + input: "[`[`]", + expected: "[`[`]", + }, + { + name: "code span holding an open bracket before a link", + input: "Split on `[` and see [the", + expected: "Split on `[` and see [the](streamdown:incomplete-link)", + }, + ])("heals once around nested links: $name", ({ input, expected }) => { + expect(remend(input)).toBe(expected); + expect(remend(expected)).toBe(expected); + }); }); describe("link handling with linkMode: text-only", () => { diff --git a/packages/remend/__tests__/lists.test.ts b/packages/remend/__tests__/lists.test.ts index 21f668ea..8f73aec9 100644 --- a/packages/remend/__tests__/lists.test.ts +++ b/packages/remend/__tests__/lists.test.ts @@ -22,14 +22,9 @@ describe("list handling", () => { expect(remend(text)).toBe(text); }); - it("should complete incomplete italic even in list items", () => { - // List markers are not counted, but incomplete italic formatting is still completed + it("should not close an opener from an earlier list item", () => { const text = "* Item with *incomplete italic\n* Another item"; - // The function adds an asterisk to complete the italic, though at the end of text - // This is not ideal but matches current behavior - expect(remend(text)).toBe( - "* Item with *incomplete italic\n* Another item*" - ); + expect(remend(text)).toBe(text); }); it("should handle mixed list markers and italic formatting", () => { @@ -69,9 +64,11 @@ describe("list handling", () => { expect(remend("* __\n* **")).toBe("* __\n* **"); expect(remend("+ __\n+ **")).toBe("+ __\n+ **"); - // List items with emphasis markers and text should still complete - expect(remend("- __ text after")).toBe("- __ text after__"); - expect(remend("- ** text after")).toBe("- ** text after**"); + // A __ followed by whitespace cannot open emphasis + expect(remend("- __ text after")).toBe("- __ text after"); + + // A ** followed by whitespace cannot open emphasis + expect(remend("- ** text after")).toBe("- ** text after"); // Mixed list items expect(remend("- __\n- Normal item\n- **")).toBe( @@ -86,8 +83,8 @@ describe("list handling", () => { expect(remend("- `")).toBe("- `"); }); - it("should not complete list items with emphasis markers spanning multiple lines", () => { - // When a list item starts with ** followed by content with newline, don't complete + it("should close emphasis within its own list item only", () => { + // Bold doesn't span lines in model output, and the next item starts a new block expect(remend("- **text\nmore text")).toBe("- **text\nmore text"); expect(remend("* **content\n* Another item")).toBe( "* **content\n* Another item" diff --git a/packages/remend/__tests__/mixed-formatting.test.ts b/packages/remend/__tests__/mixed-formatting.test.ts index c3cd26b1..72add967 100644 --- a/packages/remend/__tests__/mixed-formatting.test.ts +++ b/packages/remend/__tests__/mixed-formatting.test.ts @@ -8,7 +8,7 @@ describe("mixed formatting", () => { }); it("should complete multiple incomplete formats", () => { - expect(remend("**bold and *italic")).toBe("**bold and *italic*"); + expect(remend("**bold and *italic")).toBe("**bold and *italic***"); }); it("should handle nested formatting", () => { @@ -33,8 +33,7 @@ describe("mixed formatting", () => { }); it("should handle code inside bold", () => { - // Bold gets closed first, then code - expect(remend("**bold with `code")).toBe("**bold with `code**`"); + expect(remend("**bold with `code")).toBe("**bold with `code`**"); }); it("should handle strikethrough with other formatting", () => { @@ -48,11 +47,9 @@ describe("mixed formatting", () => { }); it("should handle deeply nested incomplete formatting", () => { - // Formats are closed in the order they're processed. - // The ~~ isn't closed because it ends up inside the completed inline code span (`code ~~strike`), - // so remend correctly leaves it alone. + // The ~~ is code once the span closes, so it gets no closer expect(remend("**bold *italic `code ~~strike")).toBe( - "**bold *italic `code ~~strike*`" + "**bold *italic `code ~~strike`***" ); }); diff --git a/packages/remend/__tests__/streaming-properties.test.ts b/packages/remend/__tests__/streaming-properties.test.ts index 6199a7c2..99237afb 100644 --- a/packages/remend/__tests__/streaming-properties.test.ts +++ b/packages/remend/__tests__/streaming-properties.test.ts @@ -60,6 +60,7 @@ const preservedPrefixLength = (input: string, output: string): number => { interface CodeNode { lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value: string; } @@ -68,6 +69,7 @@ interface MdastNode { children?: MdastNode[]; lang?: string | null; meta?: string | null; + position?: { end: { offset?: number }; start: { offset?: number } }; type: string; value?: string; } @@ -267,6 +269,91 @@ describe("streaming properties", () => { }); }); +const UNDERSCORE_RUN_TOKENS = [ + "a", + "word", + " ", + ".", + "`", + "\\", + "_", + "__", + "___", +]; + +const underscoreRunArbitrary = fc + .array(fc.constantFrom(...UNDERSCORE_RUN_TOKENS), { + minLength: 1, + maxLength: 10, + }) + .map((tokens) => tokens.join("")); + +// Gaps outside this property: a closer appended after whitespace cannot +// close, and remend does not recognize indented code blocks +const TRAILING_WHITESPACE = /\s$/; +const INDENTED_CODE = /^ {4}/; + +const MARKER_PATTERN = /[_*]/g; +// Closers, after the backslash that escapes a trailing backslash +const APPENDED_CLOSERS = /^\\?[_*]+$/; + +const countLiteralMarkers = (markdown: string): number => { + let count = 0; + const visit = (node: MdastNode): void => { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (node.type === "text" && start !== undefined && end !== undefined) { + count += markdown.slice(start, end).match(MARKER_PATTERN)?.length ?? 0; + } + for (const child of node.children ?? []) { + visit(child); + } + }; + visit(fromMarkdown(markdown)); + return count; +}; + +// Each marker healing appends must close one the prefix left literal, so the +// literal count drops by at least as many as were appended. A closer that +// renders as literal text matched nothing, so remend invented it. +const assertAppendedClosersMatch = (prefix: string): void => { + const healed = remend(prefix); + const suffix = healed.slice(prefix.length); + if (!(healed.startsWith(prefix) && APPENDED_CLOSERS.test(suffix))) { + return; + } + const appended = suffix.match(MARKER_PATTERN)?.length ?? 0; + if (countLiteralMarkers(healed) > countLiteralMarkers(prefix) - appended) { + throw new Error( + `healing appended an unmatched closer: ${JSON.stringify(prefix)} -> ${JSON.stringify(healed)}` + ); + } +}; + +describe("appended closers", () => { + it("never appends an underscore closer that matches no opener", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + fc.pre( + !(TRAILING_WHITESPACE.test(prefix) || INDENTED_CODE.test(prefix)) + ); + assertAppendedClosersMatch(prefix); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); + + it("re-heals underscore runs to themselves", () => { + fc.assert( + fc.property(underscoreRunArbitrary, (prefix) => { + const healed = remend(prefix); + expect(remend(healed)).toBe(healed); + }), + { numRuns: Number(process.env.NUM_RUNS ?? 2000) } + ); + }); +}); + describe("exhaustive prefix sweep", () => { // Every prefix of a fixed corpus, deterministically. The corpus mixes // constructs that interact. diff --git a/packages/remend/__tests__/streaming.test.ts b/packages/remend/__tests__/streaming.test.ts index 3409036a..c6f2a07b 100644 --- a/packages/remend/__tests__/streaming.test.ts +++ b/packages/remend/__tests__/streaming.test.ts @@ -4,7 +4,7 @@ import remend from "../src"; describe("chunked streaming scenarios", () => { it("should handle nested formatting cut mid-stream", () => { expect(remend("This is **bold with *ital")).toBe( - "This is **bold with *ital*" + "This is **bold with *ital***" ); // When underscore opens after bold, underscore should close before bold (proper nesting) expect(remend("**bold _und")).toBe("**bold _und_**"); @@ -27,15 +27,13 @@ describe("chunked streaming scenarios", () => { }); it("should handle complex nested structures from chunks", () => { - // Backticks spanning multiple lines need special handling - expect(remend("1. First item\n - Nested with `code\n2. Second")).toBe( - "1. First item\n - Nested with `code\n2. Second`" - ); + // A code span can't cross list items, so the backtick stays literal + const text = "1. First item\n - Nested with `code\n2. Second"; + expect(remend(text)).toBe(text); }); it("should handle multiple incomplete formats in one chunk", () => { - // Formats are closed in order they're processed - expect(remend("Text **bold `code")).toBe("Text **bold `code**`"); + expect(remend("Text **bold `code")).toBe("Text **bold `code`**"); }); }); diff --git a/packages/remend/__tests__/trailing-patterns.test.ts b/packages/remend/__tests__/trailing-patterns.test.ts deleted file mode 100644 index 7a64e606..00000000 --- a/packages/remend/__tests__/trailing-patterns.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import fc from "fast-check"; -import { describe, expect, it } from "vitest"; -import { - boldItalicPattern, - boldPattern, - halfCompleteTildePattern, - halfCompleteUnderscorePattern, - italicPattern, - matchTrailing, - singleAsteriskPattern, - singleUnderscorePattern, - strikethroughPattern, - type TrailingPattern, -} from "../src/patterns"; - -const patterns: Record = { - boldItalicPattern, - boldPattern, - halfCompleteTildePattern, - halfCompleteUnderscorePattern, - italicPattern, - singleAsteriskPattern, - singleUnderscorePattern, - strikethroughPattern, -}; - -const markdownish = fc - .array(fc.constantFrom("*", "_", "~", "a", " ", "\n", "\\", "`"), { - maxLength: 40, - }) - .map((chars) => chars.join("")); - -const groups = (match: RegExpMatchArray | null) => - match ? [match[0], match[1], match[2]] : null; - -describe("matchTrailing", () => { - it.each( - Object.entries(patterns) - )("%s matches the tail as the full text does", (_, pattern) => { - fc.assert( - fc.property(markdownish, (text) => { - expect(groups(matchTrailing(text, pattern))).toEqual( - groups(text.match(pattern.regex)) - ); - }), - { numRuns: 5000 } - ); - }); - - it.each<{ name: string; text: string; expected: string[] | null }>([ - { - name: "bold with a half closer", - text: "a **b*", - expected: ["**b*", "**", "b*"], - }, - { - name: "the closing marker of a finished run", - text: "**a** b", - expected: ["** b", "**", " b"], - }, - { name: "no marker", text: "a b", expected: null }, - { name: "marker at the end", text: "a **", expected: ["**", "**", ""] }, - { name: "empty", text: "", expected: null }, - ])("$name", ({ text, expected }) => { - expect(groups(matchTrailing(text, boldPattern))).toEqual(expected); - }); -}); diff --git a/packages/remend/__tests__/underscore-runs.test.ts b/packages/remend/__tests__/underscore-runs.test.ts index 120f6a29..3ac2f3c5 100644 --- a/packages/remend/__tests__/underscore-runs.test.ts +++ b/packages/remend/__tests__/underscore-runs.test.ts @@ -51,7 +51,57 @@ describe("underscore run lengths", () => { }); describe("escaped underscores", () => { - it("should treat the run after an escaped underscore as a delimiter", () => { - expect(remend("\\___bold")).toBe("\\___bold__"); + it("should leave the run after an escaped underscore for the author to close", () => { + expect(remend("\\___bold")).toBe("\\___bold"); + }); + + it("should not treat a run after an escaped backslash as escaped", () => { + expect(remend("\\\\__bold")).toBe("\\\\__bold__"); + }); +}); + +describe("underscore delimiter matching", () => { + it.each<[string, string]>([ + ["a run followed by whitespace", "__ foo"], + ["two open runs of one marker and length", "_a *_a"], + ["the run left after an escaped underscore", "a __ b \\__c"], + ["the run left after an escaped underscore at the start", "__ \\__alph"], + ["the run left after an escaped underscore and a backslash", "\\_____\\a"], + ["a run after a letter and before punctuation", "a__."], + ["a closer-only run with nothing open", "__ a__ a"], + ["a longer run that closes the opener", "__a____.a"], + ])("should not append a closer for %s", (_name, input) => { + expect(remend(input)).toBe(input); + }); + + it("should still close an opener after a spaced run", () => { + expect(remend("a __ b __bold")).toBe("a __ b __bold__"); + }); + + it.each<{ name: string; input: string; expected: string }>([ + { + name: "an inner run after an escaped one", + input: "\\__._a", + expected: "\\__._a_", + }, + { + name: "the underscore an escaped run leaves open", + input: "__\\__ a", + expected: "__\\__ a_", + }, + { + name: "a run after an open code span", + input: "__`x", + expected: "__`x`__", + }, + { + name: "a run after an open code span ending in a backslash", + input: "__`x\\", + expected: "__`x\\`__", + }, + ])("should close $name in one pass", ({ input, expected }) => { + const healed = remend(input); + expect(healed).toBe(expected); + expect(remend(healed)).toBe(healed); }); }); diff --git a/packages/remend/package.json b/packages/remend/package.json index 5980db26..9c81aed7 100644 --- a/packages/remend/package.json +++ b/packages/remend/package.json @@ -36,6 +36,8 @@ "@vitest/coverage-v8": "^4.1.10", "fast-check": "^4.9.0", "mdast-util-from-markdown": "^2.0.3", + "mdast-util-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.1.0", "tsup": "^8.5.1", "vitest": "^5.0.0" } diff --git a/packages/remend/src/emphasis-delimiters.ts b/packages/remend/src/emphasis-delimiters.ts new file mode 100644 index 00000000..fd6b8b24 --- /dev/null +++ b/packages/remend/src/emphasis-delimiters.ts @@ -0,0 +1,585 @@ +import { findInlineBlockStart } from "./inline-block"; +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isEscaped, + isFenceAt, + REGION, + skipContainerPrefix, + type TextScan, +} from "./scan"; +import { isWhitespaceChar } from "./utils"; + +export type EmphasisMarker = "*" | "_" | "~"; + +const MARKERS: readonly string[] = ["*", "_", "~"]; + +export interface MarkerRun { + /** Whether the last opener this run closed is still partly open */ + closesPartially: boolean; + /** Markers still unmatched */ + count: number; + /** Index of the run's first unescaped marker */ + index: number; + /** Unescaped length of the run before any were matched */ + length: number; + marker: EmphasisMarker; +} + +interface DelimiterRun extends MarkerRun { + canClose: boolean; + canOpen: boolean; + nextChar: string; + pairsClosed: number; + pairsOpened: number; + prevChar: string; + /** Start of the link label holding the run, or -1 outside any label */ + scope: number; +} + +interface DelimiterPair { + closer: DelimiterRun; + opener: DelimiterRun; + used: number; +} + +export interface DelimiterMatch { + /** Markers that render as literal text */ + literal: number; + /** Runs still open outside link labels, innermost first */ + open: MarkerRun[]; + pairs: DelimiterPair[]; + runs: MarkerRun[]; +} + +// Flanking compares whole characters, so a surrogate pair counts as one +const charBefore = (text: string, i: number): string => { + if (i <= 0) { + return ""; + } + const code = text.charCodeAt(i - 1); + return code >= 0xdc_00 && code <= 0xdf_ff && i >= 2 + ? text.slice(i - 2, i) + : text[i - 1]; +}; + +const charAt = (text: string, i: number): string => + i < text.length ? String.fromCodePoint(text.codePointAt(i) ?? 0) : ""; + +// The start and end of the text count as whitespace for flanking +const isWhitespaceOrEdge = (char: string): boolean => + char === "" || isWhitespaceChar(char); + +const punctuationPattern = /[\p{P}\p{S}]/u; + +const isPunctuationChar = (char: string): boolean => + char !== "" && punctuationPattern.test(char); + +// Flanking per CommonMark, with the stricter underscore rules: a run that +// flanks on both sides opens only after punctuation and closes only before +// punctuation, so word-internal runs (snake__case) can't open or close. +// Tildes follow the asterisk rules, and a tilde run longer than 2 is inert. +const applyFlanking = (run: DelimiterRun): void => { + const { marker, nextChar, prevChar } = run; + // micromark treats every attention marker (* _ ~) as flanking punctuation, + // so an asterisk or underscore run beside one can open or close that side + const besideMarker = (char: string): boolean => + marker !== "~" && MARKERS.includes(char); + const leftFlanking = + besideMarker(nextChar) || + !( + isWhitespaceOrEdge(nextChar) || + (isPunctuationChar(nextChar) && + !isWhitespaceOrEdge(prevChar) && + !isPunctuationChar(prevChar)) + ); + const rightFlanking = + besideMarker(prevChar) || + !( + isWhitespaceOrEdge(prevChar) || + (isPunctuationChar(prevChar) && + !isWhitespaceOrEdge(nextChar) && + !isPunctuationChar(nextChar)) + ); + if (marker === "_") { + run.canClose = + rightFlanking && (!leftFlanking || isPunctuationChar(nextChar)); + run.canOpen = + leftFlanking && (!rightFlanking || isPunctuationChar(prevChar)); + return; + } + const inert = marker === "~" && run.length > 2; + run.canClose = rightFlanking && !inert; + run.canOpen = leftFlanking && !inert; +}; + +const createRun = ( + marker: EmphasisMarker, + index: number, + length: number, + prevChar: string, + nextChar: string, + scope: number +): DelimiterRun => { + const run = { + canClose: false, + canOpen: false, + closesPartially: false, + count: length, + index, + length, + marker, + nextChar, + pairsClosed: 0, + pairsOpened: 0, + prevChar, + scope, + }; + applyFlanking(run); + return run; +}; + +const findOpenInlineMath = (scan: TextScan, inlineMath: boolean): number => { + const { text } = scan; + let start = text.length; + while (start > 0 && inMathAt(scan, start - 1, inlineMath)) { + start -= 1; + } + const isInlineDollar = + start < text.length && text[start - 1] === "$" && text[start] !== "$"; + return isInlineDollar ? start : text.length; +}; + +// An unclosed code span or inline math span renders as literal text, so its +// markers still count +const isProseAt = ( + scan: TextScan, + i: number, + openMath: number, + inlineMath: boolean +): boolean => + (scan.regions[i] === REGION.PROSE || + scan.regions[i] === REGION.CODE_SPAN_OPEN) && + !(i < openMath && inMathAt(scan, i, inlineMath)) && + !inLinkUrlAt(scan, i) && + !inHtmlTagAt(scan, i); + +interface LinkLabel { + end: number; + start: number; +} + +// Emphasis inside a completed link's text resolves on its own, so the label +// walls its runs off from the rest of the block +const findLinkLabels = ( + scan: TextScan, + from: number, + openMath: number, + inlineMath: boolean +): LinkLabel[] => { + const { text } = scan; + const labels: LinkLabel[] = []; + const openers: { active: boolean; start: number }[] = []; + for (let i = from; i < text.length; i += 1) { + const char = text[i]; + if ( + (char !== "[" && char !== "]") || + isEscaped(text, i) || + !isProseAt(scan, i, openMath, inlineMath) + ) { + continue; + } + if (char === "[") { + openers.push({ active: true, start: i }); + continue; + } + const opener = openers.pop(); + if (opener?.active && text[i + 1] === "(" && inLinkUrlAt(scan, i + 2)) { + labels.push({ end: i, start: opener.start }); + // Links cannot contain links, so a link deactivates every bracket that could enclose it + if (text[opener.start - 1] !== "!" || isEscaped(text, opener.start - 1)) { + for (const outer of openers) { + outer.active = false; + } + } + } + } + return labels; +}; + +const scopeOf = (labels: readonly LinkLabel[], index: number): number => { + let scope = -1; + for (const label of labels) { + if (label.start < index && index < label.end && label.start > scope) { + scope = label.start; + } + } + return scope; +}; + +const collectDelimiterRuns = ( + scan: TextScan, + from: number, + inlineMath: boolean +): DelimiterRun[] => { + const { text } = scan; + const runs: DelimiterRun[] = []; + const openMath = findOpenInlineMath(scan, inlineMath); + const labels = findLinkLabels(scan, from, openMath, inlineMath); + const isRunMarker = (i: number, marker: string): boolean => + text[i] === marker && isProseAt(scan, i, openMath, inlineMath); + + for (let i = from; i < text.length; i += 1) { + const marker = text[i]; + if (!(MARKERS.includes(marker) && isRunMarker(i, marker))) { + continue; + } + + let end = i + 1; + while (end < text.length && isRunMarker(end, marker)) { + end += 1; + } + + // A backslash escapes the run's first marker, which stays literal + // punctuation before the rest of the run + const escaped = isEscaped(text, i); + const start = escaped ? i + 1 : i; + if (end > start) { + runs.push( + createRun( + marker as EmphasisMarker, + start, + end - start, + escaped ? marker : charBefore(text, i), + charAt(text, end), + scopeOf(labels, start) + ) + ); + } + i = end - 1; + } + + return runs; +}; + +// The "rule of 3": when either run can both open and close, they match only +// if their combined length is not a multiple of 3, unless both lengths are. +// Lengths are what remains of each run after earlier partial matches. +const violatesRuleOfThree = ( + opener: DelimiterRun, + closer: DelimiterRun +): boolean => + (closer.canOpen || opener.canClose) && + (opener.count + closer.count) % 3 === 0 && + !(opener.count % 3 === 0 && closer.count % 3 === 0); + +const canMatch = (opener: DelimiterRun, closer: DelimiterRun): boolean => { + if (opener.marker !== closer.marker) { + return false; + } + if (closer.marker === "~") { + return opener.count === closer.count; + } + return !violatesRuleOfThree(opener, closer); +}; + +const matchCloser = ( + stack: DelimiterRun[], + floor: number, + closer: DelimiterRun, + pairs: DelimiterPair[] +): void => { + while (closer.count > 0) { + let openerIndex = stack.length - 1; + while (openerIndex >= floor && !canMatch(stack[openerIndex], closer)) { + openerIndex -= 1; + } + if (openerIndex < floor) { + return; + } + + const opener = stack[openerIndex]; + let used = 1; + if (closer.marker === "~") { + used = closer.count; + } else if (opener.count >= 2 && closer.count >= 2) { + used = 2; + } + opener.count -= used; + closer.count -= used; + opener.pairsOpened += 1; + closer.pairsClosed += 1; + closer.closesPartially = opener.count > 0; + pairs.push({ closer, opener, used }); + stack.length = opener.count > 0 ? openerIndex + 1 : openerIndex; + } +}; + +// Pairs an earlier pass matched wall off the runs between them, so a run +// inside a pair can't match one outside it. +const matchPass = ( + runs: DelimiterRun[], + inPass: (run: DelimiterRun) => boolean, + pairs: DelimiterPair[] +): DelimiterRun[] => { + const stack: DelimiterRun[] = []; + const floors: number[] = []; + + for (const run of runs) { + if (!inPass(run)) { + for (let i = 0; i < run.pairsClosed; i += 1) { + stack.length = floors.pop() ?? 0; + } + for (let i = 0; i < run.pairsOpened; i += 1) { + floors.push(stack.length); + } + continue; + } + if (run.canClose) { + matchCloser(stack, floors.at(-1) ?? 0, run, pairs); + } + if (run.canOpen && run.count > 0) { + stack.push(run); + } + } + + return stack; +}; + +const isTilde = (run: DelimiterRun): boolean => run.marker === "~"; +const isAttention = (run: DelimiterRun): boolean => run.marker !== "~"; + +// micromark resolves attention and strikethrough in the order each first +// appears in the block, link text included, and an inert tilde run never +// reaches the strikethrough resolver +const passOrder = (runs: readonly DelimiterRun[]) => { + const first = runs.find((run) => !isTilde(run) || run.length <= 2); + return first && isTilde(first) + ? [isTilde, isAttention] + : [isAttention, isTilde]; +}; + +const matchRuns = (runs: DelimiterRun[]): DelimiterMatch => { + const scopes = new Map(); + for (const run of runs) { + const scope = scopes.get(run.scope); + if (scope) { + scope.push(run); + } else { + scopes.set(run.scope, [run]); + } + } + const pairs: DelimiterPair[] = []; + const passes = passOrder(runs); + let open: DelimiterRun[] = []; + for (const [scope, scoped] of scopes) { + const left = passes.flatMap((inPass) => matchPass(scoped, inPass, pairs)); + if (scope === -1) { + open = left; + } + } + + return { + literal: runs.reduce((sum, run) => sum + run.count, 0), + open: open.sort((a, b) => b.index - a.index), + pairs, + runs, + }; +}; + +/** + * Matches the delimiter runs of the text's last inline block as micromark + * does, per CommonMark and GFM strikethrough + */ +/** Matches the runs of the text's last block, where `inlineMath` decides if a lone `$` opens math */ +export const matchDelimiterRuns = ( + text: string, + inlineMath: boolean +): DelimiterMatch => { + const scan = getScan(text); + return matchRuns( + collectDelimiterRuns(scan, findInlineBlockStart(scan), inlineMath) + ); +}; + +/** The authored text's runs, reused to match each candidate heal */ +export interface HealBase { + inlineMath: boolean; + insertAt: number; + runs: readonly DelimiterRun[]; + /** Whether closers would land where runs aren't collected */ + slow: boolean; + text: string; +} + +export const prepareHeal = ( + text: string, + insertAt: number, + inlineMath: boolean +): HealBase => { + const scan = getScan(text); + const from = findInlineBlockStart(scan); + return { + inlineMath, + insertAt, + runs: collectDelimiterRuns(scan, from, inlineMath), + // Past a trailing blank line the closers would start a block of their + // own, and inside a tag, math or a fence their markers aren't collected + slow: + from > insertAt || + (insertAt > 0 && + (inHtmlTagAt(scan, insertAt - 1) || + inMathAt(scan, insertAt - 1, inlineMath) || + isFenceAt(scan, insertAt - 1))), + text, + }; +}; + +const insertClosers = (base: HealBase, closers: string): string => + base.text.slice(0, base.insertAt) + closers + base.text.slice(base.insertAt); + +// Only the runs beside the insertion point change, so just those are rebuilt +export const matchHealed = ( + base: HealBase, + closers: string +): DelimiterMatch => { + if (base.slow) { + return matchDelimiterRuns(insertClosers(base, closers), base.inlineMath); + } + const { insertAt, text } = base; + const runs = base.runs.map((run) => ({ + ...run, + closesPartially: false, + count: run.length, + pairsClosed: 0, + pairsOpened: 0, + })); + const after = charAt(text, insertAt); + const charAfter = (i: number): string => + i < closers.length ? closers[i] : after; + + let start = 0; + const last = runs.at(-1); + if (last && last.scope === -1 && last.index + last.length === insertAt) { + while (closers[start] === last.marker) { + start += 1; + } + last.length += start; + last.count = last.length; + last.nextChar = charAfter(start); + applyFlanking(last); + } + + while (start < closers.length) { + let end = start + 1; + while (closers[end] === closers[start]) { + end += 1; + } + runs.push( + createRun( + closers[start] as EmphasisMarker, + insertAt + start, + end - start, + start === 0 ? charBefore(text, insertAt) : closers[start - 1], + charAfter(end), + -1 + ) + ); + start = end; + } + + return matchRuns(runs); +}; + +const kindOf = (pair: DelimiterPair): string => + pair.opener.marker === "~" ? "~" : pair.opener.marker + pair.used; + +const parentOf = ( + pairs: readonly DelimiterPair[], + pair: DelimiterPair +): DelimiterPair | undefined => { + let parent: DelimiterPair | undefined; + for (const other of pairs) { + if ( + other.opener.index < pair.opener.index && + other.closer.index >= pair.closer.index && + (!parent || + other.opener.index > parent.opener.index || + (other.opener.index === parent.opener.index && + other.closer.index < parent.closer.index)) + ) { + parent = other; + } + } + return parent; +}; + +// An authored pair whose opener follows punctuation, as in **注意:**这是, is +// usually a closer that failed flanking, so emphasis around it is intended +const opensAfterPunctuation = (pair: DelimiterPair): boolean => + isPunctuationChar(pair.opener.prevChar) && + !isWhitespaceOrEdge(pair.opener.nextChar) && + !isPunctuationChar(pair.opener.nextChar); + +// Authors don't nest emphasis directly inside the same emphasis, so a heal +// that needs that is closing runs written as literal text +const nestsSameKind = (match: DelimiterMatch, insertAt: number): boolean => { + const healed = (pair: DelimiterPair): boolean => + pair.closer.index + pair.closer.length > insertAt; + return match.pairs.some((pair) => { + const parent = parentOf(match.pairs, pair); + return ( + parent !== undefined && + kindOf(parent) === kindOf(pair) && + (healed(pair) || (healed(parent) && !opensAfterPunctuation(pair))) + ); + }); +}; + +const lineContentPattern = /[^\s*_~]/; + +// Closers on a line with nothing else would turn it into a thematic break or +// a fence +const leavesLineBare = (base: HealBase): boolean => { + const lineStart = base.text.lastIndexOf("\n", base.insertAt - 1) + 1; + const { contentStart } = skipContainerPrefix( + base.text, + lineStart, + base.insertAt + ); + return !lineContentPattern.test(base.text.slice(contentStart, base.insertAt)); +}; + +/** + * The healed text's literal marker count, or undefined when healing left a + * target opener open, left a marker it added literal, changed how many + * markers any other run leaves unmatched, nested emphasis directly inside + * the same emphasis, or closed fewer literal markers than it added + */ +export const literalAfterHeal = ( + before: DelimiterMatch, + literalBefore: number, + base: HealBase, + closers: string, + targets: ReadonlySet +): number | undefined => { + if (leavesLineBare(base)) { + return undefined; + } + const after = matchHealed(base, closers); + if (after.literal > literalBefore - closers.length) { + return undefined; + } + const countBefore = new Map(before.runs.map((run) => [run.index, run.count])); + const matchesOnlyTargets = after.runs.every((run) => { + if (targets.has(run.index) || run.index + run.length > base.insertAt) { + return run.count === 0; + } + return run.count === (countBefore.get(run.index) ?? 0); + }); + return matchesOnlyTargets && !nestsSameKind(after, base.insertAt) + ? after.literal + : undefined; +}; diff --git a/packages/remend/src/emphasis-handlers.ts b/packages/remend/src/emphasis-handlers.ts index 8905ae68..e8ece220 100644 --- a/packages/remend/src/emphasis-handlers.ts +++ b/packages/remend/src/emphasis-handlers.ts @@ -1,790 +1,407 @@ import { - isInsideCodeBlock, - isWithinCompleteInlineCode, -} from "./code-block-utils"; -import { - boldItalicPattern, - boldPattern, - fourOrMoreAsterisksPattern, - halfCompleteUnderscorePattern, - italicPattern, - listItemPattern, - matchTrailing, - singleAsteriskPattern, - singleUnderscorePattern, - whitespaceOrMarkersPattern, -} from "./patterns"; -import { - countDoublePairs, - getScan, - inHtmlTagAt, - inLinkUrlAt, - inMathAt, - REGION, - type TextScan, -} from "./scan"; -import { isHorizontalRule, isWordChar } from "./utils"; - -// Helper function to check if an asterisk should be skipped -const shouldSkipAsterisk = ( - scan: TextScan, - index: number, - prevChar: string, - nextChar: string -): boolean => { - // Skip if escaped with backslash - if (prevChar === "\\") { - return true; - } - - // Skip if within math block - if (inMathAt(scan, index)) { - return true; - } + type DelimiterMatch, + type HealBase, + literalAfterHeal, + type MarkerRun, + matchDelimiterRuns, + matchHealed, + prepareHeal, +} from "./emphasis-delimiters"; +import { handleIncompleteInlineCode } from "./inline-code-handler"; +import { whitespaceOrMarkersPattern } from "./patterns"; +import { getScan, isEscaped } from "./scan"; +import { isWhitespaceChar } from "./utils"; + +export interface EmphasisFlags { + bold: boolean; + boldItalic: boolean; + inlineCode: boolean; + /** Whether a lone `$` opens math, which hides the markers inside it */ + inlineMath: boolean; + italic: boolean; + strikethrough: boolean; +} - // Special handling for *** sequences - // If this is the first * in ***, don't skip it - it can close a single * italic - // Example: **bold and *italic*** should count the first * of *** as closing the italic - if (prevChar !== "*" && nextChar === "*") { - const nextNextChar = - index < scan.text.length - 2 ? scan.text[index + 2] : ""; - if (nextNextChar === "*") { - // This is the first * in a *** sequence - // Count it as a single asterisk for matching purposes - return false; - } - // This is the first * in ** (not ***) - return true; +const flagFor = (run: MarkerRun): keyof EmphasisFlags => { + if (run.marker === "~") { + return "strikethrough"; } - - // Skip if this is the second or third * in a sequence - if (prevChar === "*") { - return true; + if (run.count >= 3) { + return "boldItalic"; } - - // Skip if flanked by whitespace on both sides (not a valid emphasis delimiter per CommonMark) - // This also catches list markers (e.g., "* item") since they have whitespace on both sides - const prevIsWhitespace = - !prevChar || prevChar === " " || prevChar === "\t" || prevChar === "\n"; - const nextIsWhitespace = - !nextChar || nextChar === " " || nextChar === "\t" || nextChar === "\n"; - if (prevIsWhitespace && nextIsWhitespace) { - return true; - } - - return false; + return run.count === 2 && run.marker === "*" ? "bold" : "italic"; }; -const isWhitespaceChar = (char: string): boolean => - char === " " || char === "\t" || char === "\n"; - -const isWordInternalAsterisk = (prevChar: string, nextChar: string): boolean => - Boolean(prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)); - -// Decide whether a candidate single-asterisk delimiter should participate in the -// open/close parity count used by incomplete-italic completion. -// -// Intraword asterisks only count while resolving an active emphasis chain: -// they can close an open run and may reopen within the same word after a close. -// Cold (unmatched) intraword asterisks stay literal (hello*world / #189). -// Trailing right-flanking-only markers after a closed run also stay literal -// (*foo*bar* / a *b*c*). -const shouldCountSingleAsterisk = ( - prevChar: string, - nextChar: string, - count: number, - inWordAsteriskChain: boolean -): { count: true; inWordAsteriskChain: boolean } | { count: false } => { - const isWordInternal = isWordInternalAsterisk(prevChar, nextChar); - const canOpen = Boolean(nextChar) && !isWhitespaceChar(nextChar); - const canClose = Boolean(prevChar) && !isWhitespaceChar(prevChar); - - // Do not start emphasis from a cold word-internal asterisk. - if (isWordInternal && count % 2 === 0 && !inWordAsteriskChain) { - return { count: false }; - } - - // Prefer closing an active run; only open when left-flanking. - if ((canClose && count % 2 === 1) || canOpen) { - return { count: true, inWordAsteriskChain: isWordInternal }; +// Closers go before trailing whitespace, which would stop them from closing, +// and before a trailing backslash, which would escape them +const findInsertionPoint = (text: string): number => { + let end = text.length; + while (end > 0 && isWhitespaceChar(text[end - 1])) { + end -= 1; } - - return { count: false }; + return text[end - 1] === "\\" && !isEscaped(text, end - 1) ? end - 1 : end; }; -export const countSingleAsterisks = (text: string): number => { - const scan = getScan(text); - let count = 0; - let inWordAsteriskChain = false; - const len = text.length; - - for (let index = 0; index < len; index += 1) { - // Code regions neither contribute delimiters nor belong to a word chain - if (text[index] !== "*" || scan.regions[index] !== REGION.PROSE) { - if (!isWordChar(text[index])) { - inWordAsteriskChain = false; - } - continue; - } - - const prevChar = index > 0 ? text[index - 1] : ""; - const nextChar = index < len - 1 ? text[index + 1] : ""; - - if (shouldSkipAsterisk(scan, index, prevChar, nextChar)) { - continue; - } - - const decision = shouldCountSingleAsterisk( - prevChar, - nextChar, - count, - inWordAsteriskChain - ); - if (decision.count) { - count += 1; - inWordAsteriskChain = decision.inWordAsteriskChain; - } - } - - return count; -}; +const insertAtPoint = (text: string, at: number, closers: string): string => + closers === "" ? text : text.slice(0, at) + closers + text.slice(at); -// Helper function to check if an underscore should be skipped -const shouldSkipUnderscore = ( - scan: TextScan, - index: number, - prevChar: string, - nextChar: string +const hasContentBefore = ( + text: string, + run: MarkerRun, + insertAt: number ): boolean => { - // Skip if escaped with backslash - if (prevChar === "\\") { - return true; - } - - // Skip if within math block - if (inMathAt(scan, index)) { - return true; - } - - // Skip if within a link or image URL - if (inLinkUrlAt(scan, index)) { - return true; - } - - // Skip if within an HTML tag (e.g. ) - if (inHtmlTagAt(scan, index)) { - return true; - } - - // Skip if part of __ - if (prevChar === "_" || nextChar === "_") { - return true; - } - - // Skip if underscore is word-internal (between word characters) - if (prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)) { - return true; - } - - return false; -}; - -export const countSingleUnderscores = (text: string): number => { - const scan = getScan(text); - let count = 0; - const len = text.length; - - for (let index = 0; index < len; index += 1) { - if (text[index] !== "_" || scan.regions[index] !== REGION.PROSE) { - continue; - } - - const prevChar = index > 0 ? text[index - 1] : ""; - const nextChar = index < len - 1 ? text[index + 1] : ""; - - if (!shouldSkipUnderscore(scan, index, prevChar, nextChar)) { - count += 1; - } - } - - return count; -}; - -// Counts triple asterisks that are not part of quadruple or more asterisks -// and not inside code regions -export const countTripleAsterisks = (text: string): number => { - const scan = getScan(text); - let count = 0; - let consecutiveAsterisks = 0; - - for (let i = 0; i < text.length; i += 1) { - if (text[i] === "*" && scan.regions[i] === REGION.PROSE) { - consecutiveAsterisks += 1; - } else { - // End of asterisk sequence - if (consecutiveAsterisks >= 3) { - count += Math.floor(consecutiveAsterisks / 3); - } - consecutiveAsterisks = 0; - } - } - - // Handle trailing asterisks - if (consecutiveAsterisks >= 3) { - count += Math.floor(consecutiveAsterisks / 3); - } - - return count; + const content = text.slice(run.index + run.length, insertAt); + return content !== "" && !whitespaceOrMarkersPattern.test(content); }; -const countDoubleAsterisks = (text: string): number => - countDoublePairs(text, "*"); +const asciiLetterOrDigitPattern = /^[A-Za-z0-9]$/; +const letterOrDigitPattern = /^[\p{L}\p{N}]/u; -// Whether the text has an unmatched __ delimiter, counted per maximal -// underscore run. -// -// Counting raw occurrences misreads identifiers: a name like snake__case -// contains __ but cannot open or close emphasis, and counting it either -// invents a closer (odd count) or pairs it against a real delimiter and -// swallows a closer that was needed (even count). Per run: -// -// - A run contributes floor(length / 2) pairs, and flips delimiter parity -// only when that is odd. __ and ___ flip; ____ does not. -// - A word-internal run (word characters on both sides) is part of an -// identifier, never a delimiter. -// - Runs inside code regions, math, link URLs, and HTML tags are skipped, -// matching the single-underscore handler's flanking rules. -// - A run alone on its line is a thematic break, not emphasis. -const isLineBoundaryChar = (char: string): boolean => - char === "" || char === " " || char === "\t" || char === "\n"; - -// isHorizontalRule scans the run's whole line, so its verdict is memoized -// per line to keep run counting linear when many runs share a line -interface ThematicBreakMemo { - lineEnd: number; - result: boolean; -} +const isLetterOrDigit = (char: string): boolean => + asciiLetterOrDigitPattern.test(char); -const isThematicBreakRun = ( +// An asterisk after an ASCII letter or digit opens only to continue a chain of +// emphasis in that word, so hello*world, 2**3 and A*算法 stay literal while +// languages written without spaces still heal. A chain whose first pair also +// opened inside a word, as in M*A*S*H, is literal too. +const opensColdInWord = ( text: string, - runStart: number, - prevChar: string, - nextChar: string, - memo: ThematicBreakMemo + run: MarkerRun, + match: DelimiterMatch ): boolean => { - // A thematic break line holds only markers and whitespace, so only runs - // flanked by whitespace or line boundaries can be part of one - if (!(isLineBoundaryChar(prevChar) && isLineBoundaryChar(nextChar))) { - return false; - } - if (runStart > memo.lineEnd) { - const lineEnd = text.indexOf("\n", runStart); - memo.lineEnd = lineEnd === -1 ? text.length : lineEnd; - memo.result = isHorizontalRule(text, runStart, "_"); - } - return memo.result; -}; - -// Whether an underscore run at [runStart, runEnd) flips delimiter parity -const doubleUnderscoreRunFlips = ( - scan: TextScan, - initialRunStart: number, - runEnd: number, - memo: ThematicBreakMemo -): boolean => { - const text = scan.text; - - // A backslash escapes the first underscore of the run. The escaped - // underscore is literal punctuation, so the rest of the run still flanks - // as a delimiter. - let runStart = initialRunStart; - let escaped = false; - if (runStart > 0 && text[runStart - 1] === "\\") { - runStart += 1; - escaped = true; - } - const runLength = runEnd - runStart; - if (runLength < 2) { - return false; - } - - const beforeRun = runStart > 0 ? text[runStart - 1] : ""; - const prevChar = escaped ? "\\" : beforeRun; - const nextChar = runEnd < text.length ? text[runEnd] : ""; - if (isWordChar(prevChar) && isWordChar(nextChar)) { - return false; - } - if (isThematicBreakRun(text, runStart, prevChar, nextChar, memo)) { - return false; - } if ( - inMathAt(scan, runStart) || - inLinkUrlAt(scan, runStart) || - inHtmlTagAt(scan, runStart) + !( + run.marker === "*" && + isLetterOrDigit(text[run.index - 1] ?? "") && + letterOrDigitPattern.test(text.slice(run.index + run.length)) + ) ) { return false; } - - return Math.floor(runLength / 2) % 2 === 1; -}; - -const hasUnmatchedDoubleUnderscore = (text: string): boolean => { - const scan = getScan(text); - const n = text.length; - const memo: ThematicBreakMemo = { lineEnd: -1, result: false }; - let unmatched = false; - let i = 0; - - while (i < n) { - if (text[i] !== "_" || scan.regions[i] !== REGION.PROSE) { - i += 1; - continue; - } - - const runStart = i; - let runEnd = i + 1; - while ( - runEnd < n && - text[runEnd] === "_" && - scan.regions[runEnd] === REGION.PROSE - ) { - runEnd += 1; - } - i = runEnd; - - if (doubleUnderscoreRunFlips(scan, runStart, runEnd, memo)) { - unmatched = !unmatched; - } - } - - return unmatched; -}; - -// Helper to check if bold marker should not be completed -const shouldSkipBoldCompletion = ( - text: string, - contentAfterMarker: string, - markerIndex: number -): boolean => { + const previous = match.runs.findLast((other) => other.index < run.index); if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) + !( + previous?.marker === "*" && + [...text.slice(previous.index + previous.length, run.index)].every( + isLetterOrDigit + ) + ) ) { return true; } - - // Check if this is in a list item with multiline content - const beforeMarker = text.substring(0, markerIndex); - const lastNewlineBeforeMarker = beforeMarker.lastIndexOf("\n"); - const lineStart = - lastNewlineBeforeMarker === -1 ? 0 : lastNewlineBeforeMarker + 1; - const lineBeforeMarker = text.substring(lineStart, markerIndex); - - if (listItemPattern.test(lineBeforeMarker)) { - const hasNewlineInContent = contentAfterMarker.includes("\n"); - if (hasNewlineInContent) { - return true; - } - } - - return isHorizontalRule(text, markerIndex, "*"); + const closed = match.pairs.find( + (pair) => pair.closer.index === previous.index + ); + return ( + closed !== undefined && isLetterOrDigit(text[closed.opener.index - 1] ?? "") + ); }; -// Completes incomplete bold formatting (**) -export const handleIncompleteBold = (text: string): string => { - const boldMatch = matchTrailing(text, boldPattern); - if (!boldMatch) { - return text; - } +// Emphasis doesn't start with closing punctuation, so the * in (+, -, *, /) +// is a symbol +const closingPunctuationPattern = /^[,;]/; - const contentAfterMarker = boldMatch[2]; - const markerIndex = text.lastIndexOf(boldMatch[1]); +const opensBeforeClosingPunctuation = (text: string, run: MarkerRun): boolean => + isWhitespaceChar(text[run.index - 1] ?? " ") && + closingPunctuationPattern.test(text.slice(run.index + run.length)); - // Check if the bold marker is within a code block (fenced or inline) - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } +// Writing \** usually means a literal ** with a sloppy escape, so the rest of +// the run only closes if the author closes it +const followsEscapedMarker = (text: string, run: MarkerRun): boolean => + text[run.index - 1] === run.marker && isEscaped(text, run.index - 1); - if (shouldSkipBoldCompletion(text, contentAfterMarker, markerIndex)) { - return text; - } +// Authors close emphasis on the line that opens it, so an opener whose line +// ended open is usually code written outside a fence, like char *name +const contentPattern = /\S/g; - const asteriskPairs = countDoubleAsterisks(text); - if (asteriskPairs % 2 === 1) { - // Check for half-complete closing marker: **content* should become **content** - // The trailing * is the first char of the closing ** being streamed - if (contentAfterMarker.endsWith("*")) { - return `${text}*`; - } - return `${text}**`; +const lineEnded = (text: string, run: MarkerRun): boolean => { + const newline = text.indexOf("\n", run.index + run.length); + if (newline === -1) { + return false; } - - return text; + contentPattern.lastIndex = newline; + return contentPattern.test(text); }; -// Helper to check if italic marker should not be completed -const shouldSkipItalicCompletion = ( +const isGated = ( text: string, - contentAfterMarker: string, - markerIndex: number -): boolean => { - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return true; - } - - // Check if this is in a list item with multiline content - const beforeMarker = text.substring(0, markerIndex); - const lastNewlineBeforeMarker = beforeMarker.lastIndexOf("\n"); - const lineStart = - lastNewlineBeforeMarker === -1 ? 0 : lastNewlineBeforeMarker + 1; - const lineBeforeMarker = text.substring(lineStart, markerIndex); + run: MarkerRun, + insertAt: number, + match: DelimiterMatch +): boolean => + !hasContentBefore(text, run, insertAt) || + // Prose uses a single tilde for "approximately" + (run.marker === "~" && run.count === 1) || + // Models write fill-in blanks as long underscore runs, like Name: ______ + (run.marker === "_" && run.length >= 4) || + opensColdInWord(text, run, match) || + opensBeforeClosingPunctuation(text, run) || + followsEscapedMarker(text, run); + +interface HealContext { + base: HealBase; + flags: EmphasisFlags; + insertAt: number; + literal: number; + match: DelimiterMatch; + text: string; + /** The authored run the first closer would extend */ + trailing: MarkerRun | undefined; +} - if (listItemPattern.test(lineBeforeMarker)) { - const hasNewlineInContent = contentAfterMarker.includes("\n"); - if (hasNewlineInContent) { - return true; - } - } +// More open runs than nesting plausibly reaches means the markers are +// literal text, like a list of globs or footnote marks +const MAX_OPEN_RUNS = 4; - return isHorizontalRule(text, markerIndex, "_"); -}; +interface Heal { + closers: string; + literal: number; + targets: number[]; +} -// Completes incomplete italic formatting with double underscores (__) -export const handleIncompleteDoubleUnderscoreItalic = ( - text: string -): string => { - const italicMatch = matchTrailing(text, italicPattern); - if (!italicMatch) { - // Check for half-complete closing marker: __content_ should become __content__ - // The pattern /(__)([^_]*?)$/ won't match __content_ because it ends with _ - // So we need a separate check for this case - const halfCompleteMatch = matchTrailing( - text, - halfCompleteUnderscorePattern +// Closers for a group of open runs, innermost first, where up to `reusable` +// markers of the first closer come from a literal run already in the text +const closerCandidates = (group: MarkerRun[], reusable: number): string[] => { + const [first] = group; + const own = + first.marker.repeat(first.count - reusable) + + group + .slice(1) + .map((run) => run.marker.repeat(run.count)) + .join(""); + // Closers for openers of one marker merge into a single run, which the rule + // of 3 may let close them all only at another length + if ([...own].every((char) => char === first.marker)) { + return Array.from({ length: own.length + reusable + 2 }, (_, i) => + first.marker.repeat(i + 1) ); - if (halfCompleteMatch) { - const markerIndex = text.lastIndexOf(halfCompleteMatch[1]); - if ( - !( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) && - hasUnmatchedDoubleUnderscore(text) - ) { - return `${text}_`; - } - } - return text; } - - const contentAfterMarker = italicMatch[2]; - const markerIndex = text.lastIndexOf(italicMatch[1]); - - // Check if the italic marker is within a code block (fenced or inline) - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } - - if (shouldSkipItalicCompletion(text, contentAfterMarker, markerIndex)) { - return text; - } - - if (hasUnmatchedDoubleUnderscore(text)) { - return `${text}__`; - } - - return text; -}; - -// Skips code regions when locating the asterisk. -// A lone, unescaped asterisk in prose outside math: the only positions that -// can open incomplete italic -const isLoneProseAsterisk = (scan: TextScan, i: number): boolean => { - const { text } = scan; - return ( - text[i] === "*" && - scan.regions[i] === REGION.PROSE && - text[i - 1] !== "*" && - text[i + 1] !== "*" && - text[i - 1] !== "\\" && - !inMathAt(scan, i) + return Array.from( + { length: reusable + 1 }, + (_, i) => first.marker.repeat(i) + own ); }; -const findFirstSingleAsteriskIndex = (text: string): number => { - const scan = getScan(text); - - for (let i = text.indexOf("*"); i !== -1; i = text.indexOf("*", i + 1)) { - if (!isLoneProseAsterisk(scan, i)) { - continue; +// Two open runs of one kind are literal text, like globs or _id, _rev. +// When only double * or _ runs are open they pair off instead, since a closer +// that failed flanking in **Note:**text leaves an opener behind it that the +// author may still close. Tildes stay strict because GFM matches them in a +// separate pass. +const withoutDuplicates = ( + runs: readonly MarkerRun[] +): MarkerRun[] | undefined => { + const byKind = new Map(); + for (const run of runs) { + const kind = run.marker + run.count; + byKind.set(kind, [...(byKind.get(kind) ?? []), run]); + } + // Closing another kind can bury one run of a pair and expose the other + if (byKind.size > 1 && byKind.size < runs.length) { + return; + } + const healable = new Set(); + for (const group of byKind.values()) { + if (group.length > 1 && (group[0].count === 1 || group[0].marker === "~")) { + return; } - const prevChar = i > 0 ? text[i - 1] : ""; - const nextChar = i < text.length - 1 ? text[i + 1] : ""; - - // Skip if flanked by whitespace on both sides (not a valid emphasis delimiter) - const prevIsWs = !prevChar || isWhitespaceChar(prevChar); - const nextIsWs = !nextChar || isWhitespaceChar(nextChar); - if (prevIsWs && nextIsWs) { - continue; + if (group.length % 2 === 1) { + healable.add( + group.reduce((last, run) => (run.index > last.index ? run : last)) + ); } + } + return runs.filter((run) => healable.has(run)); +}; - // Skip cold word-internal asterisks; they are not openers for completion. - // (Active-run closers are still counted in countSingleAsterisks.) - if (prevChar && nextChar && isWordChar(prevChar) && isWordChar(nextChar)) { +// A closer's flanking can depend on the closers after it, so each step tries +// every remaining closer at once before trying it alone +const extendHeal = ( + context: HealContext, + start: Heal, + open: readonly MarkerRun[] +): Heal => { + const { flags, insertAt, text, trailing } = context; + const runs: MarkerRun[] = []; + for (const run of open) { + if (isGated(text, run, insertAt, context.match)) { continue; } - - // A right-flanking-only marker cannot open incomplete italic. - if (nextIsWs) { - continue; + if (flags[flagFor(run)]) { + runs.push(run); + } + if (runs.length > MAX_OPEN_RUNS) { + return start; } - - return i; - } - return -1; -}; - -// Completes incomplete italic formatting with single asterisks (*) -export const handleIncompleteSingleAsteriskItalic = (text: string): string => { - const singleAsteriskMatch = matchTrailing(text, singleAsteriskPattern); - - if (!singleAsteriskMatch) { - return text; - } - - const firstSingleAsteriskIndex = findFirstSingleAsteriskIndex(text); - - if (firstSingleAsteriskIndex === -1) { - return text; - } - - // Don't close if the marker is inside a complete inline code span or fenced code block - if ( - isInsideCodeBlock(text, firstSingleAsteriskIndex) || - isWithinCompleteInlineCode(text, firstSingleAsteriskIndex) - ) { - return text; } - - // Get content after the first single asterisk - const contentAfterFirstAsterisk = text.substring( - firstSingleAsteriskIndex + 1 + // Openers on ended lines still count as duplicates, which mark the markers + // on later lines as literal too + const healable = withoutDuplicates(runs)?.filter( + (run) => !lineEnded(text, run) ); - - // Check if there's meaningful content after the asterisk - // Don't close if content is only whitespace or emphasis markers - if ( - !contentAfterFirstAsterisk || - whitespaceOrMarkersPattern.test(contentAfterFirstAsterisk) - ) { - return text; - } - - const singleAsterisks = countSingleAsterisks(text); - if (singleAsterisks % 2 === 1) { - return `${text}*`; - } - - return text; -}; - -const findFirstSingleUnderscoreIndex = (text: string): number => { - const scan = getScan(text); - - for (let i = text.indexOf("_"); i !== -1; i = text.indexOf("_", i + 1)) { - if ( - scan.regions[i] === REGION.PROSE && - text[i - 1] !== "_" && - text[i + 1] !== "_" && - text[i - 1] !== "\\" && - !inMathAt(scan, i) && - !inLinkUrlAt(scan, i) - ) { - // Check if underscore is word-internal (between word characters) - const prevChar = i > 0 ? text[i - 1] : ""; - const nextChar = i < text.length - 1 ? text[i + 1] : ""; - if ( - prevChar && - nextChar && - isWordChar(prevChar) && - isWordChar(nextChar) - ) { - continue; + if (healable === undefined) { + return start; + } + + // Closing one run can strand another behind the rule of 3 or nest it + // inside the same emphasis, so the greedy pass runs over every subset + const closeAll = (chosen: MarkerRun[]): Heal => { + let heal = start; + const tryGroup = (group: MarkerRun[]): boolean => { + const [first] = group; + const targets = [...heal.targets, ...group.map((run) => run.index)]; + const targetSet = new Set(targets); + // A literal run the first closer would extend already supplies markers + const reusable = + heal.closers === "" && trailing?.marker === first.marker + ? Math.min(trailing.count, first.count - 1) + : 0; + let found: Heal | undefined; + for (const candidate of closerCandidates(group, reusable)) { + const closers = heal.closers + candidate; + const literal = literalAfterHeal( + context.match, + context.literal, + context.base, + closers, + targetSet + ); + if (literal !== undefined && (!found || literal < found.literal)) { + found = { closers, literal, targets }; + } + } + if (found) { + heal = found; } + return found !== undefined; + }; - return i; + for (let i = 0; i < chosen.length; i += 1) { + if (i < chosen.length - 1 && tryGroup(chosen.slice(i))) { + break; + } + tryGroup([chosen[i]]); + } + return heal; + }; + + // Largest subsets first, so a tie keeps the heal that closes more runs + const subsets: MarkerRun[][] = [[]]; + for (const run of healable) { + subsets.push(...subsets.map((set) => [...set, run])); + } + subsets.reverse(); + let best = start; + for (const subset of subsets) { + if (subset.length > 0) { + const heal = closeAll(subset); + if (heal.literal < best.literal) { + best = heal; + } } } - return -1; -}; - -// Helper function to insert closing underscore, handling trailing newlines -const insertClosingUnderscore = (text: string): string => { - // If text ends with newline(s), insert underscore before them - // Use string methods instead of regex to avoid ReDoS vulnerability - let endIndex = text.length; - while (endIndex > 0 && text[endIndex - 1] === "\n") { - endIndex -= 1; - } - if (endIndex < text.length) { - const textBeforeNewlines = text.slice(0, endIndex); - const trailingNewlines = text.slice(endIndex); - return `${textBeforeNewlines}_${trailingNewlines}`; - } - return `${text}_`; + return best; }; -// Helper to handle trailing ** for proper nesting of _ and ** markers -const handleTrailingAsterisksForUnderscore = (text: string): string | null => { - if (!text.endsWith("**")) { - return null; - } - - const textWithoutTrailingAsterisks = text.slice(0, -2); - const asteriskPairsAfterRemoval = countDoubleAsterisks( - textWithoutTrailingAsterisks +// Closers can change how authored runs flank and leave one open, so healing +// repeats until a round adds nothing +const healFrom = ( + context: HealContext, + open: readonly MarkerRun[], + targets: number[] = [] +): Heal => { + let heal = extendHeal( + context, + { closers: "", literal: context.literal, targets }, + open ); - - // If removing trailing ** makes the count odd, it was added to close an unclosed ** - if (asteriskPairsAfterRemoval % 2 !== 1) { - return null; - } - - const firstDoubleAsteriskIndex = textWithoutTrailingAsterisks.indexOf("**"); - const underscoreIndex = findFirstSingleUnderscoreIndex( - textWithoutTrailingAsterisks - ); - - // If ** opened before _, then _ should close before ** - if ( - firstDoubleAsteriskIndex !== -1 && - underscoreIndex !== -1 && - firstDoubleAsteriskIndex < underscoreIndex - ) { - return `${textWithoutTrailingAsterisks}_**`; + while (heal.closers !== "") { + const reopened = matchHealed(context.base, heal.closers).open.filter( + (run) => run.index < context.insertAt + ); + const next = extendHeal(context, heal, reopened); + if (next.closers === heal.closers) { + break; + } + heal = next; } - - return null; + return heal; }; -// Completes incomplete italic formatting with single underscores (_) -export const handleIncompleteSingleUnderscoreItalic = ( - text: string -): string => { - const singleUnderscoreMatch = matchTrailing(text, singleUnderscorePattern); - - if (!singleUnderscoreMatch) { - return text; - } - - const firstSingleUnderscoreIndex = findFirstSingleUnderscoreIndex(text); - - if (firstSingleUnderscoreIndex === -1) { - return text; - } - - // Get content after the first single underscore - const contentAfterFirstUnderscore = text.substring( - firstSingleUnderscoreIndex + 1 - ); - - // Check if there's meaningful content after the underscore - // Don't close if content is only whitespace or emphasis markers - if ( - !contentAfterFirstUnderscore || - whitespaceOrMarkersPattern.test(contentAfterFirstUnderscore) - ) { - return text; - } - - if ( - isInsideCodeBlock(text, firstSingleUnderscoreIndex) || - isWithinCompleteInlineCode(text, firstSingleUnderscoreIndex) - ) { - return text; - } - - const singleUnderscores = countSingleUnderscores(text); - if (singleUnderscores % 2 === 1) { - // Check if we need to insert _ before trailing ** for proper nesting - const trailingResult = handleTrailingAsterisksForUnderscore(text); - if (trailingResult !== null) { - return trailingResult; +const healEmphasis = (text: string, flags: EmphasisFlags): string => { + const insertAt = findInsertionPoint(text); + const match = matchDelimiterRuns(text, flags.inlineMath); + const last = match.runs.at(-1); + const trailing = + last && last.index + last.length === insertAt ? last : undefined; + const context = { + base: prepareHeal(text, insertAt, flags.inlineMath), + flags, + insertAt, + literal: match.literal, + match, + text, + trailing, + }; + let heal = healFrom(context, match.open); + + // A trailing run that closes something may instead be literal text, with + // the runs open before it still needing closers + if (trailing && trailing.count < trailing.length) { + const prefix = matchDelimiterRuns( + text.slice(0, trailing.index), + flags.inlineMath + ); + const asText = { ...trailing, count: trailing.length }; + const streaming = trailing.count > 0 || trailing.closesPartially; + const alternative = healFrom( + { + ...context, + literal: prefix.literal + asText.count, + match: { ...prefix, runs: [...prefix.runs, asText] }, + trailing: asText, + }, + prefix.open, + streaming ? [] : [trailing.index] + ); + // Reading the trailing run as text must not repair spans the author + // already closed by re-pairing their runs + const targets = new Set(alternative.targets); + const healed = matchHealed(context.base, alternative.closers); + const countBefore = new Map( + match.runs.map((run) => [run.index, run.count]) + ); + const keepsAuthoredRuns = healed.runs.every( + (run) => + run.index >= trailing.index || + targets.has(run.index) || + run.count === countBefore.get(run.index) + ); + if (alternative.literal < heal.literal && keepsAuthoredRuns) { + heal = alternative; } - return insertClosingUnderscore(text); } - return text; + return insertAtPoint(text, insertAt, heal.closers); }; -// Helper to check if bold-italic markers are already balanced -const areBoldItalicMarkersBalanced = (text: string): boolean => { - const asteriskPairs = countDoubleAsterisks(text); - const singleAsterisks = countSingleAsterisks(text); - return asteriskPairs % 2 === 0 && singleAsterisks % 2 === 0; -}; - -// Helper to check if bold-italic should be skipped -const shouldSkipBoldItalicCompletion = ( +export const handleIncompleteEmphasis = ( text: string, - contentAfterMarker: string, - markerIndex: number -): boolean => { - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return true; - } - - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return true; - } - - return isHorizontalRule(text, markerIndex, "*"); -}; - -// Completes incomplete bold-italic formatting (***) -export const handleIncompleteBoldItalic = (text: string): string => { - // Don't process if text is only asterisks and has 4 or more consecutive asterisks - if (fourOrMoreAsterisksPattern.test(text)) { - return text; - } - - const boldItalicMatch = matchTrailing(text, boldItalicPattern); - - if (!boldItalicMatch) { - return text; - } - - const contentAfterMarker = boldItalicMatch[2]; - const markerIndex = text.lastIndexOf(boldItalicMatch[1]); - - if (shouldSkipBoldItalicCompletion(text, contentAfterMarker, markerIndex)) { - return text; + flags: EmphasisFlags +): string => { + if (!(flags.inlineCode && getScan(text).openSpan)) { + return healEmphasis(text, flags); } - const tripleAsteriskCount = countTripleAsterisks(text); - if (tripleAsteriskCount % 2 === 1) { - // If both ** and * are balanced, don't add closing *** - // The *** is likely overlapping markers (e.g., **bold and *italic***) - if (areBoldItalicMarkersBalanced(text)) { - return text; - } - return `${text}***`; + // Emphasis closers must land after the code span closes, so close it first + // and keep that only when emphasis needed healing too + const closed = handleIncompleteInlineCode(text); + if (closed !== text) { + const healed = healEmphasis(closed, flags); + return healed === closed ? text : healed; } - return text; + // A closer that breaks up a trailing backtick run would let the span close + // after it, which would swallow the closer + const healed = healEmphasis(text, flags); + return handleIncompleteInlineCode(healed) === healed ? healed : text; }; diff --git a/packages/remend/src/index.ts b/packages/remend/src/index.ts index 0ecf9137..fea4d55a 100644 --- a/packages/remend/src/index.ts +++ b/packages/remend/src/index.ts @@ -1,10 +1,7 @@ import { handleComparisonOperators } from "./comparison-operator-handler"; import { - handleIncompleteBold, - handleIncompleteBoldItalic, - handleIncompleteDoubleUnderscoreItalic, - handleIncompleteSingleAsteriskItalic, - handleIncompleteSingleUnderscoreItalic, + type EmphasisFlags, + handleIncompleteEmphasis, } from "./emphasis-handlers"; import { handleIncompleteHtmlTag } from "./html-tag-handler"; import { handleIncompleteInlineCode } from "./inline-code-handler"; @@ -17,10 +14,9 @@ import { INCOMPLETE_IMAGE_PLACEHOLDER, type LinkMode, } from "./link-image-handler"; -import { clearScanCache } from "./scan"; +import { clearScanCache, getScan, inMathAt, isEscaped, REGION } from "./scan"; import { handleIncompleteSetextHeading } from "./setext-heading-handler"; import { handleSingleTildeEscape } from "./single-tilde-handler"; -import { handleIncompleteStrikethrough } from "./strikethrough-handler"; // biome-ignore lint/performance/noBarrelFile: "Re-exports utility functions for public API convenience" export { @@ -104,13 +100,8 @@ const PRIORITY = { HTML_TAGS: 10, SETEXT_HEADINGS: 15, LINKS: 20, - BOLD_ITALIC: 30, - BOLD: 35, - ITALIC_DOUBLE_UNDERSCORE: 40, - ITALIC_SINGLE_ASTERISK: 41, - ITALIC_SINGLE_UNDERSCORE: 42, + EMPHASIS: 30, INLINE_CODE: 50, - STRIKETHROUGH: 60, KATEX: 70, INLINE_KATEX: 75, DEFAULT: 100, @@ -167,41 +158,9 @@ const builtInHandlers: Array<{ }, { handler: { - name: "boldItalic", - handle: handleIncompleteBoldItalic, - priority: PRIORITY.BOLD_ITALIC, - }, - optionKey: "boldItalic", - }, - { - handler: { - name: "bold", - handle: handleIncompleteBold, - priority: PRIORITY.BOLD, - }, - optionKey: "bold", - }, - { - handler: { - name: "italicDoubleUnderscore", - handle: handleIncompleteDoubleUnderscoreItalic, - priority: PRIORITY.ITALIC_DOUBLE_UNDERSCORE, - }, - optionKey: "italic", - }, - { - handler: { - name: "italicSingleAsterisk", - handle: handleIncompleteSingleAsteriskItalic, - priority: PRIORITY.ITALIC_SINGLE_ASTERISK, - }, - optionKey: "italic", - }, - { - handler: { - name: "italicSingleUnderscore", - handle: handleIncompleteSingleUnderscoreItalic, - priority: PRIORITY.ITALIC_SINGLE_UNDERSCORE, + name: "emphasis", + handle: (text: string) => text, + priority: PRIORITY.EMPHASIS, }, optionKey: "italic", }, @@ -213,14 +172,6 @@ const builtInHandlers: Array<{ }, optionKey: "inlineCode", }, - { - handler: { - name: "strikethrough", - handle: handleIncompleteStrikethrough, - priority: PRIORITY.STRIKETHROUGH, - }, - optionKey: "strikethrough", - }, { handler: { name: "katex", @@ -247,6 +198,14 @@ const getEnabledBuiltInHandlers = ( earlyReturn?: (result: string) => boolean; }> => { const linkMode: LinkMode = options?.linkMode ?? "protocol"; + const emphasisFlags: EmphasisFlags = { + bold: isEnabled(options?.bold), + boldItalic: isEnabled(options?.boldItalic), + inlineCode: isEnabled(options?.inlineCode), + inlineMath: isOptedIn(options?.inlineKatex), + italic: isEnabled(options?.italic), + strikethrough: isEnabled(options?.strikethrough), + }; return builtInHandlers .filter(({ handler, optionKey }) => { @@ -254,6 +213,15 @@ const getEnabledBuiltInHandlers = ( if (handler.name === "links") { return isEnabled(options?.links) || isEnabled(options?.images); } + // Special case: emphasis is enabled by any of its options + if (handler.name === "emphasis") { + return ( + emphasisFlags.bold || + emphasisFlags.boldItalic || + emphasisFlags.italic || + emphasisFlags.strikethrough + ); + } // Special case: inlineKatex is opt-in (defaults to false, unlike other options) if (handler.name === "inlineKatex") { return isOptedIn(options?.inlineKatex); @@ -267,16 +235,41 @@ const getEnabledBuiltInHandlers = ( handler: { ...handler, handle: (text: string) => - handleIncompleteLinksAndImages(text, linkMode), + handleIncompleteLinksAndImages( + text, + linkMode, + emphasisFlags.inlineCode + ), }, // Only use early return for protocol mode (text-only won't end with the marker) earlyReturn: linkMode === "protocol" ? earlyReturn : undefined, }; } + // Special case: wrap emphasis handler to pass its options + if (handler.name === "emphasis") { + return { + handler: { + ...handler, + handle: (text: string) => + handleIncompleteEmphasis(text, emphasisFlags), + }, + earlyReturn, + }; + } return { handler, earlyReturn }; }); }; +// Whether the text ends with an unescaped backslash in prose +const isTrailingEscape = (text: string): boolean => { + const last = text.length - 1; + if (text[last] !== "\\" || isEscaped(text, last)) { + return false; + } + const scan = getScan(text); + return scan.regions[last] === REGION.PROSE && !inMathAt(scan, last); +}; + // Parses markdown text and removes incomplete tokens to prevent partial rendering const remend = (text: string, options?: RemendOptions): string => { if (!text || typeof text !== "string") { @@ -284,7 +277,7 @@ const remend = (text: string, options?: RemendOptions): string => { } // Remove trailing whitespace if it's not a double space - let result = + const result = text.endsWith(" ") && !text.endsWith(" ") ? text.slice(0, -1) : text; // Get enabled built-in handlers @@ -302,25 +295,39 @@ const remend = (text: string, options?: RemendOptions): string => { (a, b) => (a.handler.priority ?? 0) - (b.handler.priority ?? 0) ); - try { + const runHandlers = (input: string): string => { + let output = input; // Execute handlers in priority order for (const { handler, earlyReturn } of allHandlers) { - result = handler.handle(result); + output = handler.handle(output); // Check for early return condition (e.g., incomplete link marker) - if (earlyReturn?.(result)) { - return result; + if (earlyReturn?.(output)) { + return output; } } // A handler that removes a trailing fragment can expose a trailing space // (e.g. dropping an incomplete image). Strip it the same way the input // was stripped, so healed output re-heals to itself. - if (result.endsWith(" ") && !result.endsWith(" ")) { - return result.slice(0, -1); + if (output.endsWith(" ") && !output.endsWith(" ")) { + return output.slice(0, -1); + } + + return output; + }; + + try { + if (!isTrailingEscape(result)) { + return runHandlers(result); } - return result; + // A trailing backslash escapes whatever follows it, so a closer appended + // after it would render as text. Heal with the backslash escaped, which + // keeps it literal without changing how the markers before it flank. + const escaped = `${result}\\`; + const healed = runHandlers(escaped); + return healed === escaped ? result : healed; } finally { clearScanCache(); } diff --git a/packages/remend/src/inline-block.ts b/packages/remend/src/inline-block.ts new file mode 100644 index 00000000..c0fd6eb6 --- /dev/null +++ b/packages/remend/src/inline-block.ts @@ -0,0 +1,92 @@ +import { + type ContainerPrefix, + isFenceAt, + skipContainerPrefix, + type TextScan, +} from "./scan"; + +const headingPattern = /^#{1,6}(?:[ \t]|$)/; +const thematicBreakPattern = + /^(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/; +const setextUnderlinePattern = /^(?:=+|-+)[ \t]*$/; +const emptyItemPattern = /^(?:[*+-]|\d{1,9}[.)])$/; +// The setext handler's zero-width space after a streaming dash stands in for +// the list item or thematic break that dash usually starts +const dashPlaceholderPattern = /^-{1,2}\u200B$/; + +type Paragraph = "none" | "top" | "contained"; + +const paragraphIn = (prefix: ContainerPrefix): Paragraph => + prefix.listIndent > 0 || prefix.quoteDepth > 0 ? "contained" : "top"; + +// An empty item can't interrupt a paragraph, but a paragraph inside a quote or +// list item ends at an unindented line, which isn't lazy continuation text +const startsEmptyItem = ( + content: string, + paragraph: Paragraph, + prefix: ContainerPrefix, + lineStart: number +): boolean => + emptyItemPattern.test(content) && + (paragraph === "none" || + (paragraph === "contained" && + prefix.quoteDepth === 0 && + prefix.contentStart === lineStart)); + +/** + * Index where the text's last inline block starts, since emphasis never + * pairs across paragraphs, list items, headings, table rows or code blocks + */ +export const findInlineBlockStart = (scan: TextScan): number => { + const { text } = scan; + let start = 0; + let quoteDepth = 0; + let endsBlock = false; + let paragraph: Paragraph = "none"; + + for (let lineStart = 0; lineStart <= text.length; ) { + let lineEnd = text.indexOf("\n", lineStart); + if (lineEnd === -1) { + lineEnd = text.length; + } + const next = lineEnd + 1; + const prefix = skipContainerPrefix(text, lineStart, lineEnd); + const content = text.slice(prefix.contentStart, lineEnd).trim(); + + // The unterminated last line is where the paragraph keeps streaming + if (lineEnd === text.length && content === "") { + break; + } + if ( + content === "" || + isFenceAt(scan, prefix.contentStart) || + // A list marker walk would read the first star of * * * as an item + thematicBreakPattern.test(text.slice(lineStart, lineEnd).trim()) || + thematicBreakPattern.test(content) || + (setextUnderlinePattern.test(content) && !endsBlock) || + (lineEnd < text.length && + startsEmptyItem(content, paragraph, prefix, lineStart)) + ) { + start = next; + endsBlock = false; + paragraph = "none"; + } else if ( + endsBlock || + prefix.listIndent > 0 || + prefix.quoteDepth > quoteDepth || + dashPlaceholderPattern.test(content) || + headingPattern.test(content) || + content.startsWith("|") + ) { + start = prefix.contentStart; + endsBlock = headingPattern.test(content) || content.startsWith("|"); + paragraph = endsBlock ? "none" : paragraphIn(prefix); + } else if (paragraph === "none") { + paragraph = paragraphIn(prefix); + } + quoteDepth = prefix.quoteDepth; + lineStart = next; + } + + return Math.min(start, text.length); +}; diff --git a/packages/remend/src/inline-code-handler.ts b/packages/remend/src/inline-code-handler.ts index ed079a79..56d6555c 100644 --- a/packages/remend/src/inline-code-handler.ts +++ b/packages/remend/src/inline-code-handler.ts @@ -1,5 +1,7 @@ +import { findInlineBlockStart } from "./inline-block"; import { whitespaceOrMarkersPattern } from "./patterns"; import { getScan } from "./scan"; +import { isWhitespaceChar } from "./utils"; // Completes an unclosed inline code span (`) // @@ -16,20 +18,29 @@ export const handleIncompleteInlineCode = (text: string): string => { return text; } + // A code span can't cross into another block, so an opener before the + // last one stays a literal backtick const span = scan.openSpan; - if (!span) { + if (!span || span.start < findInlineBlockStart(scan)) { return text; } + // The closing run goes before trailing whitespace, since one after a + // trailing newline would start a new line and could open a fence + let end = text.length; + while (end > 0 && isWhitespaceChar(text[end - 1])) { + end -= 1; + } + // Don't close if there's no meaningful content after the opening run - const content = text.slice(span.start + span.runLength); + const content = text.slice(span.start + span.runLength, end); if (!content || whitespaceOrMarkersPattern.test(content)) { return text; } // A trailing backtick run is the closing run being streamed let trailingRun = 0; - let i = text.length - 1; + let i = end - 1; while (i >= 0 && text[i] === "`") { trailingRun += 1; i -= 1; @@ -41,5 +52,17 @@ export const handleIncompleteInlineCode = (text: string): string => { return text; } - return text + "`".repeat(span.runLength - trailingRun); + // Completing a run that starts its line with three or more backticks would + // open a fence, so a full closer goes after a space instead + const runStart = end - trailingRun; + const lineStart = text.lastIndexOf("\n", runStart - 1) + 1; + const opensFence = + trailingRun > 0 && + span.runLength >= 3 && + text.slice(lineStart, runStart).trim() === ""; + const closer = opensFence + ? ` ${"`".repeat(span.runLength)}` + : "`".repeat(span.runLength - trailingRun); + + return text.slice(0, end) + closer + text.slice(end); }; diff --git a/packages/remend/src/link-image-handler.ts b/packages/remend/src/link-image-handler.ts index 66cabd0f..dc86e20c 100644 --- a/packages/remend/src/link-image-handler.ts +++ b/packages/remend/src/link-image-handler.ts @@ -1,4 +1,7 @@ import { isInsideCodeBlock } from "./code-block-utils"; +import { findInlineBlockStart } from "./inline-block"; +import { handleIncompleteInlineCode } from "./inline-code-handler"; +import { getScan, isCompleteSpanAt, isFenceAt } from "./scan"; import { findMatchingClosingBracket, findMatchingOpeningBracket, @@ -16,7 +19,8 @@ export const INCOMPLETE_IMAGE_PLACEHOLDER = "streamdown:incomplete-image"; const handleIncompleteUrl = ( text: string, lastParenIndex: number, - linkMode: LinkMode + linkMode: LinkMode, + blockStart: number ): string | null => { const afterParen = text.substring(lastParenIndex + 2); if (afterParen.includes(")")) { @@ -27,7 +31,11 @@ const handleIncompleteUrl = ( // Now find the matching opening bracket for the ] before ( const openBracketIndex = findMatchingOpeningBracket(text, lastParenIndex); - if (openBracketIndex === -1 || isInsideCodeBlock(text, openBracketIndex)) { + // The link text must start in the same block as its "](" + if ( + openBracketIndex < blockStart || + isInsideCodeBlock(text, openBracketIndex) + ) { return null; } @@ -55,8 +63,12 @@ const handleIncompleteUrl = ( // Helper to find the first incomplete [ (for text-only mode) // Always returns a valid index since callers guarantee text[maxPos] is an incomplete [ -const findFirstIncompleteBracket = (text: string, maxPos: number): number => { - for (let j = 0; j < maxPos; j++) { +const findFirstIncompleteBracket = ( + text: string, + maxPos: number, + blockStart: number +): number => { + for (let j = blockStart; j < maxPos; j++) { if (text[j] === "[" && !isInsideCodeBlock(text, j)) { // Skip if it's an image if (j > 0 && text[j - 1] === "!") { @@ -86,7 +98,8 @@ const findFirstIncompleteBracket = (text: string, maxPos: number): number => { const handleIncompleteText = ( text: string, i: number, - linkMode: LinkMode + linkMode: LinkMode, + blockStart: number ): string | null => { // Check if there's a ! before it const isImage = i > 0 && text[i - 1] === "!"; @@ -107,7 +120,7 @@ const handleIncompleteText = ( // For links, handle based on linkMode if (linkMode === "text-only") { // Find the first incomplete [ and strip just that bracket - const firstIncomplete = findFirstIncompleteBracket(text, i); + const firstIncomplete = findFirstIncompleteBracket(text, i, blockStart); return ( text.substring(0, firstIncomplete) + text.substring(firstIncomplete + 1) ); @@ -131,7 +144,7 @@ const handleIncompleteText = ( if (linkMode === "text-only") { // Find the first incomplete [ and strip just that bracket - const firstIncomplete = findFirstIncompleteBracket(text, i); + const firstIncomplete = findFirstIncompleteBracket(text, i, blockStart); return ( text.substring(0, firstIncomplete) + text.substring(firstIncomplete + 1) ); @@ -142,15 +155,38 @@ const handleIncompleteText = ( return null; }; +const opensCompleteLink = (text: string, open: number): boolean => { + const close = findMatchingClosingBracket(text, open); + return ( + close !== -1 && + text[close + 1] === "(" && + text.indexOf(")", close + 2) !== -1 + ); +}; + // One healing step over the trailing incomplete link or image, if any const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Look for patterns like [text]( or ![text]( at the end of text // We need to handle nested brackets in the link text + // Link text cannot cross blocks, so brackets before the last inline block + // are literal. Healing them would append a closer to the wrong block, or + // drop every block after them. + const scan = getScan(text); + const blockStart = findInlineBlockStart(scan); + // Start from the end and look for ]( pattern const lastParenIndex = text.lastIndexOf("]("); - if (lastParenIndex !== -1 && !isInsideCodeBlock(text, lastParenIndex)) { - const result = handleIncompleteUrl(text, lastParenIndex, linkMode); + if ( + lastParenIndex >= blockStart && + !isInsideCodeBlock(text, lastParenIndex) + ) { + const result = handleIncompleteUrl( + text, + lastParenIndex, + linkMode, + blockStart + ); if (result !== null) { return result; } @@ -158,13 +194,24 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // Then check for incomplete link text: [partial-text without closing ] // Search backwards for an opening bracket that doesn't have a matching closing bracket + let enclosesLink = false; for ( let i = text.lastIndexOf("["); - i !== -1; + i >= blockStart; i = i === 0 ? -1 : text.lastIndexOf("[", i - 1) ) { - if (!isInsideCodeBlock(text, i)) { - const result = handleIncompleteText(text, i, linkMode); + // An unclosed backtick renders as text, so a link after one still counts + const isLink = !( + (i > 0 && text[i - 1] === "!") || + isFenceAt(scan, i) || + isCompleteSpanAt(scan, i) + ); + const closesHere = isLink && opensCompleteLink(text, i); + // Links can't contain links, so a link bracket before one is literal text + const literal = isLink && enclosesLink; + enclosesLink ||= closesHere; + if (!(closesHere || literal || isInsideCodeBlock(text, i))) { + const result = handleIncompleteText(text, i, linkMode, blockStart); if (result !== null) { return result; } @@ -179,11 +226,7 @@ const healTrailingLinkOrImage = (text: string, linkMode: LinkMode): string => { // without a bound. Realistic nesting depth is single digits. const MAX_HEAL_PASSES = 32; -// Preserves incomplete links and images with a special marker. -export const handleIncompleteLinksAndImages = ( - text: string, - linkMode: LinkMode = "protocol" -): string => { +const healLinksAndImages = (text: string, linkMode: LinkMode): string => { let current = text; // Removing a trailing incomplete image can expose another incomplete @@ -201,3 +244,36 @@ export const handleIncompleteLinksAndImages = ( } return current; }; + +const endsWithPlaceholder = (text: string): boolean => + text.endsWith("](streamdown:incomplete-link)") || + text.endsWith(`](${INCOMPLETE_IMAGE_PLACEHOLDER})`); + +// Preserves incomplete links and images with a special marker. +export const handleIncompleteLinksAndImages = ( + text: string, + linkMode: LinkMode = "protocol", + closeInlineCode = false +): string => { + const healed = healLinksAndImages(text, linkMode); + if (healed !== text || !closeInlineCode || endsWithPlaceholder(text)) { + return healed; + } + const span = getScan(text).openSpan; + if (!span || text.lastIndexOf("[", span.start) === -1) { + return text; + } + + // Code spans bind tighter than links, so closing an open span can swallow + // brackets and leave a link opener unclosed. Heal against the span the + // inline code handler will close, and keep that only when a link needed it. + // Trailing whitespace stays outside the closers, as the inline code + // handler leaves it + const body = text.trimEnd(); + const closed = handleIncompleteInlineCode(body); + if (closed === body) { + return text; + } + const relinked = healLinksAndImages(closed, linkMode); + return relinked === closed ? text : relinked + text.slice(body.length); +}; diff --git a/packages/remend/src/patterns.ts b/packages/remend/src/patterns.ts index 307eb87e..1317359f 100644 --- a/packages/remend/src/patterns.ts +++ b/packages/remend/src/patterns.ts @@ -1,66 +1,2 @@ -/** - * An end-anchored pattern whose delimiter may follow the opening marker only - * as the text's final character. - */ -export interface TrailingPattern { - delimiter: string; - markerLength: number; - regex: RegExp; -} - -// A match's marker ends at or after the last delimiter before the final -// character, or at the final character when there is none, so matching from -// there finds the same match without scanning the rest of the document. -export const matchTrailing = ( - text: string, - { delimiter, markerLength, regex }: TrailingPattern -): RegExpMatchArray | null => { - const last = text.lastIndexOf(delimiter, text.length - 2); - const markerEnd = last === -1 ? text.length - 1 : last; - return text.slice(Math.max(0, markerEnd - markerLength + 1)).match(regex); -}; - -export const boldPattern: TrailingPattern = { - delimiter: "*", - markerLength: 2, - regex: /(\*\*)([^*]*\*?)$/, -}; -export const italicPattern: TrailingPattern = { - delimiter: "_", - markerLength: 2, - regex: /(__)([^_]*?)$/, -}; -export const boldItalicPattern: TrailingPattern = { - delimiter: "*", - markerLength: 3, - regex: /(\*\*\*)([^*]*?)$/, -}; -export const singleAsteriskPattern: TrailingPattern = { - delimiter: "*", - markerLength: 1, - regex: /(\*)([^*]*?)$/, -}; -export const singleUnderscorePattern: TrailingPattern = { - delimiter: "_", - markerLength: 1, - regex: /(_)([^_]*?)$/, -}; -export const strikethroughPattern: TrailingPattern = { - delimiter: "~", - markerLength: 2, - regex: /(~~)([^~]*?)$/, -}; -export const halfCompleteUnderscorePattern: TrailingPattern = { - delimiter: "_", - markerLength: 2, - regex: /(__)([^_]+)_$/, -}; -export const halfCompleteTildePattern: TrailingPattern = { - delimiter: "~", - markerLength: 2, - regex: /(~~)([^~]+)~$/, -}; export const whitespaceOrMarkersPattern = /^[\s_~*`]*$/; -export const listItemPattern = /^[\s]*[-*+][\s]+$/; export const letterNumberUnderscorePattern = /[\p{L}\p{N}_]/u; -export const fourOrMoreAsterisksPattern = /^\*{4,}$/; diff --git a/packages/remend/src/scan.ts b/packages/remend/src/scan.ts index 1bb9f74f..d95c8d6f 100644 --- a/packages/remend/src/scan.ts +++ b/packages/remend/src/scan.ts @@ -70,6 +70,8 @@ export interface TextScan { linkUrlMask: Uint8Array | null; /** Lazily computed; stays null until first needed */ mathMask: Uint8Array | null; + /** The math mask when a lone `$` is text, also lazily computed */ + mathMaskWithoutDollar: Uint8Array | null; openFence: OpenFence | null; openSpan: OpenSpan | null; regions: Uint8Array; @@ -98,7 +100,7 @@ const skipListMarker = (text: string, start: number): number => { return text[i] === " " ? i + 1 : -1; }; -interface ContainerPrefix { +export interface ContainerPrefix { /** Index of the first character after the markers and indentation */ contentStart: number; listIndent: number; @@ -108,7 +110,7 @@ interface ContainerPrefix { // Walks the block quote and list markers that open a line. A single forward // pass: a regex with nested repetition over these markers backtracks // exponentially on a line of repeated list markers. -const skipContainerPrefix = ( +export const skipContainerPrefix = ( text: string, lineStart: number, lineEnd: number @@ -342,7 +344,7 @@ const nextSpanOpener = ( return -1; } const escaped = - next > i && text[next - 1] === "\\" && regions[next - 1] === REGION.PROSE; + next > i && regions[next - 1] === REGION.PROSE && isEscaped(text, next); if (regions[next] === REGION.PROSE && !escaped) { return next; } @@ -414,6 +416,7 @@ const scanText = (text: string): TextScan => { openFence, openSpan, mathMask: null, + mathMaskWithoutDollar: null, linkUrlMask: null, htmlTagMask: null, }; @@ -473,6 +476,15 @@ export const isFenceAt = (scan: TextScan, position: number): boolean => { export const isCompleteSpanAt = (scan: TextScan, position: number): boolean => scan.regions[position] === REGION.CODE_SPAN; +/** Whether the character at position follows an odd run of backslashes */ +export const isEscaped = (text: string, position: number): boolean => { + let backslashes = 0; + for (let i = position - 1; i >= 0 && text[i] === "\\"; i -= 1) { + backslashes += 1; + } + return backslashes % 2 === 1; +}; + /** Counts non-overlapping double-character pairs (**, ~~, $$) in prose */ export const countDoublePairs = (text: string, char: string): number => { const scan = getScan(text); @@ -480,7 +492,9 @@ export const countDoublePairs = (text: string, char: string): number => { let count = 0; for (let i = text.indexOf(pair); i !== -1; i = text.indexOf(pair, i)) { - if (scan.regions[i] === REGION.PROSE) { + // Inside math a backslash is TeX, so only markdown markers can be escaped + const escaped = char !== "$" && isEscaped(text, i); + if (scan.regions[i] === REGION.PROSE && !escaped) { count += 1; i += 2; } else { @@ -549,7 +563,8 @@ interface MathDelimiter { const mathDelimiterAt = ( text: string, i: number, - context: MathContext + context: MathContext, + singleDollar: boolean ): MathDelimiter | null => { const next = text[i + 1]; if (text[i] === "\\") { @@ -561,6 +576,9 @@ const mathDelimiterAt = ( } if (text[i] === "$" && !isLatexMathContext(context)) { const isBlockDelimiter = next === "$"; + if (!(isBlockDelimiter || singleDollar)) { + return null; + } return { context: getDollarMathContext(context, isBlockDelimiter), length: isBlockDelimiter ? 2 : 1, @@ -573,7 +591,7 @@ const mathDelimiterAt = ( // \(...\) or \[...\]. Delimiters inside code regions are literal and do // not change math state. A two-character delimiter marks its second // character with the context it establishes. -const buildMathMask = (scan: TextScan): Uint8Array => { +const buildMathMask = (scan: TextScan, singleDollar: boolean): Uint8Array => { const { text, regions } = scan; const n = text.length; const mask = new Uint8Array(n); @@ -583,7 +601,9 @@ const buildMathMask = (scan: TextScan): Uint8Array => { while (i < n) { mask[i] = context === "none" ? 0 : 1; const delimiter: MathDelimiter | null = - regions[i] === REGION.PROSE ? mathDelimiterAt(text, i, context) : null; + regions[i] === REGION.PROSE + ? mathDelimiterAt(text, i, context, singleDollar) + : null; if (delimiter === null) { i += 1; continue; @@ -598,16 +618,24 @@ const buildMathMask = (scan: TextScan): Uint8Array => { return mask; }; -export const inMathAt = (scan: TextScan, position: number): boolean => { +/** Whether position is in math, where `singleDollar` decides if a lone `$` opens it */ +export const inMathAt = ( + scan: TextScan, + position: number, + singleDollar = true +): boolean => { if (position < 0 || position >= scan.text.length) { return false; } - if (scan.mathMask === null) { - scan.mathMask = hasMathDelimiters(scan.text) - ? buildMathMask(scan) + const key = singleDollar ? "mathMask" : "mathMaskWithoutDollar"; + let mask = scan[key]; + if (mask === null) { + mask = hasMathDelimiters(scan.text) + ? buildMathMask(scan, singleDollar) : EMPTY_MASK; + scan[key] = mask; } - return scan.mathMask[position] === 1; + return mask[position] === 1; }; // Marks the URL positions of one line: those between a "](" opener and the diff --git a/packages/remend/src/strikethrough-handler.ts b/packages/remend/src/strikethrough-handler.ts deleted file mode 100644 index e981f83c..00000000 --- a/packages/remend/src/strikethrough-handler.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { - isInsideCodeBlock, - isWithinCompleteInlineCode, -} from "./code-block-utils"; -import { - halfCompleteTildePattern, - matchTrailing, - strikethroughPattern, - whitespaceOrMarkersPattern, -} from "./patterns"; -import { countDoublePairs } from "./scan"; - -// Tilde runs that open or close a fence are painted as fence regions by the -// scanner, so a line-start ~~~ fence never counts as strikethrough while a -// mid-line tilde run still does. -const countDoubleTildes = (text: string): number => countDoublePairs(text, "~"); - -// Completes incomplete strikethrough formatting (~~) -export const handleIncompleteStrikethrough = (text: string): string => { - const strikethroughMatch = matchTrailing(text, strikethroughPattern); - - if (strikethroughMatch) { - // Don't close if there's no meaningful content after the opening markers - // strikethroughMatch[2] contains the content after ~~ - // Check if content is only whitespace or other emphasis markers - const contentAfterMarker = strikethroughMatch[2]; - if ( - !contentAfterMarker || - whitespaceOrMarkersPattern.test(contentAfterMarker) - ) { - return text; - } - - // Don't close if the marker is inside an inline code span or fenced code block - const markerIndex = text.lastIndexOf(strikethroughMatch[1]); - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } - - if (countDoubleTildes(text) % 2 === 1) { - return `${text}~~`; - } - } else { - // Check for half-complete closing marker: ~~content~ should become ~~content~~ - // The pattern /(~~)([^~]*?)$/ won't match ~~content~ because it ends with ~ - const halfCompleteMatch = matchTrailing(text, halfCompleteTildePattern); - if (halfCompleteMatch) { - // Don't close if the marker is inside an inline code span or fenced code block - const markerIndex = text.lastIndexOf(halfCompleteMatch[0].slice(0, 2)); - if ( - isInsideCodeBlock(text, markerIndex) || - isWithinCompleteInlineCode(text, markerIndex) - ) { - return text; - } - if (countDoubleTildes(text) % 2 === 1) { - return `${text}~`; - } - } - } - - return text; -}; diff --git a/packages/remend/src/utils.ts b/packages/remend/src/utils.ts index 06291fd0..5b091db6 100644 --- a/packages/remend/src/utils.ts +++ b/packages/remend/src/utils.ts @@ -1,5 +1,17 @@ import { letterNumberUnderscorePattern } from "./patterns"; -import { getScan, inHtmlTagAt, inLinkUrlAt, inMathAt, isFenceAt } from "./scan"; +import { + getScan, + inHtmlTagAt, + inLinkUrlAt, + inMathAt, + isCompleteSpanAt, + isFenceAt, +} from "./scan"; + +const whitespaceCharPattern = /\s/; + +export const isWhitespaceChar = (char: string): boolean => + whitespaceCharPattern.test(char); // OPTIMIZATION: Precompute which characters are word characters // Using ASCII fast path before falling back to Unicode regex @@ -25,13 +37,18 @@ export const isWithinCodeBlock = (text: string, position: number): boolean => isFenceAt(getScan(text), position); // Helper function to find the matching opening bracket for a closing bracket -// Handles nested brackets correctly by searching backwards +// Handles nested brackets correctly by searching backwards. Code spans bind +// tighter than links, so brackets inside a complete span don't count. export const findMatchingOpeningBracket = ( text: string, closeIndex: number ): number => { + const scan = getScan(text); let depth = 1; for (let i = closeIndex - 1; i >= 0; i -= 1) { + if (isCompleteSpanAt(scan, i)) { + continue; + } if (text[i] === "]") { depth += 1; } else if (text[i] === "[") { @@ -45,13 +62,17 @@ export const findMatchingOpeningBracket = ( }; // Helper function to find the matching closing bracket for an opening bracket -// Handles nested brackets correctly +// Handles nested brackets correctly, skipping brackets inside complete code spans export const findMatchingClosingBracket = ( text: string, openIndex: number ): number => { + const scan = getScan(text); let depth = 1; for (let i = openIndex + 1; i < text.length; i += 1) { + if (isCompleteSpanAt(scan, i)) { + continue; + } if (text[i] === "[") { depth += 1; } else if (text[i] === "]") { diff --git a/packages/streamdown/__tests__/stream-repair.test.ts b/packages/streamdown/__tests__/stream-repair.test.ts new file mode 100644 index 00000000..40f184d4 --- /dev/null +++ b/packages/streamdown/__tests__/stream-repair.test.ts @@ -0,0 +1,241 @@ +import remend, { type RemendOptions } from "remend"; +import { describe, expect, it } from "vitest"; +import { + hasFootnoteDefinition, + hasFootnoteReference, + hasLinkDefinition, + parseMarkdownIntoBlocks, +} from "../lib/parse-blocks"; +import { + advanceStableLength, + repairStream, + type StreamRepairState, +} from "../lib/stream-repair"; + +// The full-document patterns that the incremental checks replace. +const footnoteReferencePattern = /\[\^[\w-]{1,200}\](?!:)/; +const footnoteDefinitionPattern = /\[\^[\w-]{1,200}\]:/; +const linkDefinitionPattern = /^ {0,3}\[[^\]]+\]:/m; + +// Streamed as Streamdown does: repair, split into blocks, advance. Every +// repaired prefix must equal remend over the whole prefix. +const documents = [ + "# Title\n\nFirst paragraph with **bold** and *italic*.\n\nSecond with a [link](https://example.com) and `code`.\n\nThird **unfinished", + "Intro\n\n- one **a**\n- two [b](/b)\n\n continued\n\n1. first\n2. second *em*\n\nAfter the list `x`\n", + "| a | b |\n| --- | --- |\n| **x** | [y](/y) |\n| 3 | 4 |\n\nText after the table with ~~strike~~.\n", + "Before\n\n```ts\nconst a = '**';\n\nconst b = `x`;\n```\n\nAfter the fence with **bold\n\nand more", + "Math $x$ and\n\n$$\na = b\n\n= c\n$$\n\nThen `inline` and **strong**.\n", + "> quote with *em*\n> more\n\nSetext\n===\n\npara\n---\n\nend [unfinished link", + "Call foo(**kwargs) to forward.\n\nGlob src/**/test.js.\n\nThe next paragraph *streams* on.\n", + "See [ref] here.\n\n[ref]: https://example.com\n\nFootnote[^1] text.\n\n[^1]: note\n", + // A closer appended at the end must not make an earlier open block final. + "Before\n\n$$\nx = 1\n\ny **2\n\nstill math\n$$\n\nafter *em*\n", + "Text [a\n\nb](https://exa\n\nc **d\n\ne", + // Remend counts math delimiters across the whole message. + "$$\nx\n$$\n\nA\n\nB\n\nThen $$y$$ inline.\n", + "a `$$` b\n\nc $$ d\n\nE\n\nF\n\ng $$ h $$ i\n", + "A costs $5.\n\nB\n\nC costs $6.\n\nD goes on.\n", + "Inline \\(a\n\nB\n\nC\n\nD **e\n", + "```sh\n$ npm i\n```\n\nA\n\nB\n\nC $x$ and $$y\n", +]; + +const chunked = (text: string, seed: number): string[] => { + const chunks: string[] = []; + let state = seed; + for (let index = 0; index < text.length; ) { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + const size = 1 + (state % 4); + chunks.push(text.slice(index, index + size)); + index += size; + } + return chunks; +}; + +const prefixes = (document: string, seed: number): string[] => { + let text = ""; + return chunked(document, seed).map((chunk) => { + text += chunk; + return text; + }); +}; + +// Returns each input whose tail repair differs from a full repair. +const stream = (inputs: string[], options?: RemendOptions) => { + let state: StreamRepairState | null = null; + const mismatches: string[] = []; + for (const input of inputs) { + state = repairStream(state, input, options); + if (state.repaired !== remend(input, options)) { + mismatches.push(input); + } + advanceStableLength(state, parseMarkdownIntoBlocks(state.repaired)); + } + return { state, mismatches }; +}; + +describe("tail-bounded stream repair", () => { + it("matches a full repair at every prefix", () => { + for (const options of [undefined, { inlineKatex: true }]) { + for (const document of documents) { + for (const seed of [1, 2, 3]) { + expect(stream(prefixes(document, seed), options).mismatches).toEqual( + [] + ); + } + } + } + }); + + it("matches a full repair on random math, code and emphasis", () => { + const pieces = [ + "$", + "$$", + "\\(", + "\\)", + "\\[", + "`", + "```\n", + "~~~\n", + "**", + "_", + "[", + "](", + "word ", + "word ", + "word ", + "word.\n\n", + "word.\n\n", + "\n", + ]; + let state = 4242; + for (const options of [undefined, { inlineKatex: true }]) { + for (let run = 0; run < 300; run += 1) { + let text = ""; + const inputs: string[] = []; + for (let step = 0; step < 80; step += 1) { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + text += pieces[state % pieces.length]; + inputs.push(text); + } + expect(stream(inputs, options).mismatches).toEqual([]); + } + } + }); + + it("stops the settled prefix before math outside fenced code", () => { + const math = stream([ + "One $x$.\n\nTwo.\n\nThree.\n\nFour", + "One $x$.\n\nTwo.\n\nThree.\n\nFour and", + ]); + expect(math.state?.stableLength).toBe(0); + + const fenced = stream([ + "```sh\n$ ls\n```\n\nTwo.\n\nThree.\n\nFour", + "```sh\n$ ls\n```\n\nTwo.\n\nThree.\n\nFour and", + ]); + expect(fenced.state?.stableLength).toBeGreaterThan(0); + }); + + it("repairs the whole message with custom handlers", () => { + // Appends "!" when the message has an even number of blocks. + const options: RemendOptions = { + handlers: [ + { + name: "parity", + handle: (text) => (text.split("\n\n").length % 2 ? text : `${text}!`), + }, + ], + }; + const inputs = prefixes("One.\n\nTwo.\n\nThree.\n\nFour.\n\nFive.\n", 1); + const { state, mismatches } = stream(inputs, options); + expect(mismatches).toEqual([]); + expect(state?.stableLength).toBe(0); + }); + + it("skips the settled prefix once later blocks exist", () => { + const { state } = stream([ + "One.\n\nTwo.\n\nThree.\n\nFour", + "One.\n\nTwo.\n\nThree.\n\nFour and", + ]); + expect(state?.stableLength).toBeGreaterThan(0); + }); + + it("repairs the whole input after a rewrite, CR input, or new options", () => { + let { state } = stream(["One.\n\nTwo.\n\nThree.\n\nFour **x"]); + state = repairStream(state, "Different **text", undefined); + expect(state.stableLength).toBe(0); + expect(state.repaired).toBe(remend("Different **text")); + + state = repairStream(state, "Different **text\r\nmore", undefined); + expect(state.stableLength).toBe(0); + + const options = { links: false }; + state = repairStream(state, "Different **text\r\nmore [x", options); + expect(state.repaired).toBe(remend("Different **text\r\nmore [x", options)); + }); +}); + +describe("incremental footnote and link-definition checks", () => { + // Parsing an unrelated document first resets each check's memo, so the + // second call is a full check of that prefix. + const fresh = (input: string): string[] => { + parseMarkdownIntoBlocks("unrelated reset document\n"); + return parseMarkdownIntoBlocks(input); + }; + + it("matches a full check at every prefix", () => { + const label = "x".repeat(200); + const inputs = [ + "Intro paragraph.\n\nSee note [^1] here.\n\nMore text.\n", + "Intro.\n\nText [^a]: is a definition.\n\nMore.\n", + `Intro.\n\nLong [^${label}] label.\n\nMore [^${label}x] too long.\n`, + "Intro [x](/u).\n\nThen\n[ref]: https://example.com\n\nafter\n", + "a ] b\n\n[multi\nline]: /u\n\nend\n", + "> [q]: /quoted\n\npara\n", + ]; + for (const document of inputs) { + for (const seed of [1, 2]) { + let text = ""; + for (const chunk of chunked(document, seed)) { + text += chunk; + const expected = fresh(text); + parseMarkdownIntoBlocks(text.slice(0, -chunk.length) || " "); + expect(parseMarkdownIntoBlocks(text)).toEqual(expected); + } + } + } + }); + + it("gives each check the same answer as its full regex", () => { + const checks = [ + [hasFootnoteReference, footnoteReferencePattern], + [hasFootnoteDefinition, footnoteDefinitionPattern], + [hasLinkDefinition, linkDefinitionPattern], + ] as const; + const pieces = [ + "[", + "]", + "^", + ":", + "-", + "a", + " ", + "\n", + "\n\n", + "[^n]", + "]:", + "[x]: /u", + ]; + let state = 7717; + for (let run = 0; run < 300; run += 1) { + let text = ""; + for (let step = 0; step < 80; step += 1) { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + text += pieces[state % pieces.length]; + for (const [check, pattern] of checks) { + expect(check(text)).toBe(pattern.test(text)); + } + } + } + }); +}); diff --git a/packages/streamdown/index.tsx b/packages/streamdown/index.tsx index e25fe043..5de1f73b 100644 --- a/packages/streamdown/index.tsx +++ b/packages/streamdown/index.tsx @@ -53,6 +53,11 @@ import { rehypeBlockDirection } from "./lib/rehype/block-direction"; import { rehypeLiteralTagContent } from "./lib/rehype/literal-tag-content"; import { remarkCodeMeta } from "./lib/remark/code-meta"; import { remarkDisableAutolinkProtocols } from "./lib/remark/disable-autolink-protocols"; +import { + advanceStableLength, + repairStream, + type StreamRepairState, +} from "./lib/stream-repair"; import type { CSVSeparator } from "./lib/table/utils"; import { defaultTranslations, @@ -728,14 +733,32 @@ export const Streamdown = memo( // Apply remend to fix incomplete markdown BEFORE parsing into blocks // This prevents partial list items from being interpreted as setext headings + // Tail-bounded repair state for this message. It is checked against each + // input, so a render React discards cannot affect a later one. + const streamRepairRef = useRef(null); const processedChildren = useMemo(() => { if (typeof children !== "string") { return ""; } - let result = - mode === "streaming" && shouldParseIncompleteMarkdown - ? remend(children, remendOptions) - : children; + let result = children; + if (mode === "streaming" && shouldParseIncompleteMarkdown) { + // Block offsets come from the default parser and must match the + // repaired text, so custom parsers and tag preprocessing repair whole. + const canRepairTail = + parseMarkdownIntoBlocksFn === parseMarkdownIntoBlocks && + !literalTagContent?.length && + allowedTagNames.length === 0; + if (canRepairTail) { + streamRepairRef.current = repairStream( + streamRepairRef.current, + children, + remendOptions + ); + result = streamRepairRef.current.repaired; + } else { + result = remend(children, remendOptions); + } + } // Escape markdown metacharacters inside literal-tag-content tags so that // children are rendered as plain text rather than parsed as markdown. @@ -760,12 +783,17 @@ export const Streamdown = memo( remendOptions, allowedTagNames, literalTagContent, + parseMarkdownIntoBlocksFn, ]); - const blocks = useMemo( - () => parseMarkdownIntoBlocksFn(processedChildren), - [processedChildren, parseMarkdownIntoBlocksFn] - ); + const blocks = useMemo(() => { + const parsed = parseMarkdownIntoBlocksFn(processedChildren); + const repair = streamRepairRef.current; + if (repair?.repaired === processedChildren) { + advanceStableLength(repair, parsed); + } + return parsed; + }, [processedChildren, parseMarkdownIntoBlocksFn]); // Render blocks directly. The previous displayBlocks + useTransition path // could be starved by sibling urgent updates when animated was off (#550), diff --git a/packages/streamdown/lib/parse-blocks.tsx b/packages/streamdown/lib/parse-blocks.tsx index 69029387..6ace83c2 100644 --- a/packages/streamdown/lib/parse-blocks.tsx +++ b/packages/streamdown/lib/parse-blocks.tsx @@ -3,7 +3,6 @@ import { Lexer, type Token } from "marked"; // Regex patterns moved to top level for performance // Footnote identifiers must be alphanumeric, underscore, or hyphen (e.g., [^1], [^note], [^my-note]) // Previously used [^\]\s] which incorrectly matched regex character classes like [^\s...] -const footnoteReferencePattern = /\[\^[\w-]{1,200}\](?!:)/; const footnoteDefinitionPattern = /\[\^[\w-]{1,200}\]:/; // Allow hyphens / colons so custom tags like are tracked across // blank-line interruptions (\w alone only matches [A-Za-z0-9_]). @@ -14,6 +13,81 @@ const openingTagPattern = /<([A-Za-z][\w:-]*)[\s>/]/; // leading spaces before a definition (4+ is an indented code block). const linkDefinitionPattern = /^ {0,3}\[[^\]]+\]:/m; +// These checks run on the whole document for every streaming update. Each +// keeps the previous input and its result; when the new input extends it, +// only the region where a new match could end is tested again. +interface ScanMemo { + found: boolean; + input: string; +} + +const extendsMemo = (memo: ScanMemo | null, input: string): memo is ScanMemo => + memo !== null && + input.length >= memo.input.length && + input.startsWith(memo.input); + +// The longest footnote match: "[^", a 200-character label, "]", one more. +const footnoteWindow = 205; +// A footnote reference is `[^label]` not followed by ":", that is +// /\[\^[\w-]{1,200}\](?!:)/. Its `(?!:)` can drop a match at the very end of +// the input once ":" arrives. +// A match followed by a known non-colon character is permanent, so track +// those and test the end of the input separately. +const followedFootnoteReferencePattern = /\[\^[\w-]{1,200}\][^:]/; +const trailingFootnoteReferencePattern = /\[\^[\w-]{1,200}\]$/; + +let footnoteReferenceMemo: ScanMemo | null = null; +let footnoteDefinitionMemo: ScanMemo | null = null; +let linkDefinitionMemo: ScanMemo | null = null; + +const footnoteTail = (memo: ScanMemo, input: string): string => + input.slice(Math.max(0, memo.input.length - footnoteWindow)); + +export const hasFootnoteReference = (input: string): boolean => { + const memo = footnoteReferenceMemo; + const followed = extendsMemo(memo, input) + ? memo.found || + followedFootnoteReferencePattern.test(footnoteTail(memo, input)) + : followedFootnoteReferencePattern.test(input); + footnoteReferenceMemo = { input, found: followed }; + return ( + followed || + trailingFootnoteReferencePattern.test(input.slice(-footnoteWindow)) + ); +}; + +export const hasFootnoteDefinition = (input: string): boolean => { + const memo = footnoteDefinitionMemo; + const found = extendsMemo(memo, input) + ? memo.found || footnoteDefinitionPattern.test(footnoteTail(memo, input)) + : footnoteDefinitionPattern.test(input); + footnoteDefinitionMemo = { input, found }; + return found; +}; + +export const hasLinkDefinition = (input: string): boolean => { + const memo = linkDefinitionMemo; + let found: boolean; + if (!extendsMemo(memo, input)) { + found = linkDefinitionPattern.test(input); + } else if (memo.found) { + found = true; + } else { + // A new match ends after the previous input, and its label holds no "]", + // so it starts on the line after the previous input's last "]" (ignoring + // its final character, which a new ":" can follow). + const previousLength = memo.input.length; + const bracket = + previousLength >= 2 + ? memo.input.lastIndexOf("]", previousLength - 2) + : -1; + const start = bracket < 0 ? 0 : input.lastIndexOf("\n", bracket) + 1; + found = linkDefinitionPattern.test(input.slice(start)); + } + linkDefinitionMemo = { input, found }; + return found; +}; + // HTML void elements (self-closing tags) that don't need closing tags const voidElements = new Set([ "area", @@ -268,7 +342,7 @@ const reuseParsedBlocks = ( // compared on every tick. if ( input.slice(0, previous.input.length) !== previous.input || - linkDefinitionPattern.test(input) + hasLinkDefinition(input) ) { return null; } @@ -327,14 +401,14 @@ export const parseMarkdownIntoBlocks = (markdown: string): string[] => { // Footnote references: [^1], [^label], etc. // Footnote definitions: [^1]: text, [^label]: text, etc. // Use atomic groups or possessive quantifiers to prevent backtracking - const hasFootnoteReference = footnoteReferencePattern.test(markdown); - const hasFootnoteDefinition = footnoteDefinitionPattern.test(markdown); + const markdownHasFootnoteReference = hasFootnoteReference(markdown); + const markdownHasFootnoteDefinition = hasFootnoteDefinition(markdown); // If footnotes are present, return the entire document as a single block // This ensures footnote references and definitions remain in the same mdast tree. // Do not write the cache: footnote documents are not block-split, and leaving // a prior entry avoids poisoning the next non-footnote stream extension check. - if (hasFootnoteReference || hasFootnoteDefinition) { + if (markdownHasFootnoteReference || markdownHasFootnoteDefinition) { return [markdown]; } diff --git a/packages/streamdown/lib/stream-repair.ts b/packages/streamdown/lib/stream-repair.ts new file mode 100644 index 00000000..184a6b51 --- /dev/null +++ b/packages/streamdown/lib/stream-repair.ts @@ -0,0 +1,79 @@ +import remend, { type RemendOptions } from "remend"; + +// Streaming appends text, and the blocks before the last two cannot change +// once a later block exists. Remend only completes syntax in the last inline +// block, so repairing the text after those blocks gives the same string as +// repairing the whole message, without rescanning the message on every update. +export interface StreamRepairState { + input: string; + options: RemendOptions | undefined; + repaired: string; + stableLength: number; +} + +// Mutable blocks at the end of the document. parseMarkdownIntoBlocks reuses +// the same window for its own cache. +const MUTABLE_BLOCKS = 2; + +// Remend tracks math ($, $$, \( and \[) across the whole message, so text +// after a math delimiter can repair differently without the text before it. +// Delimiters inside fenced code are literal, except "$$": remend finds the +// first "$$" in the message to choose how to close an open one. +const mathDelimiterPattern = /\$|\\[([]/; +const fencedBlockPattern = /^ {0,3}(([`~])\2{2,})[\s\S]*\n {0,3}\1\2*\s*$/; + +const hasMathDelimiter = (block: string): boolean => + block.includes("$$") || + (mathDelimiterPattern.test(block) && !fencedBlockPattern.test(block)); + +export const repairStream = ( + previous: StreamRepairState | null, + input: string, + options: RemendOptions | undefined +): StreamRepairState => { + // parseMarkdownIntoBlocks normalizes CR line endings, which shifts block + // offsets, so repair those inputs whole. + const extendsPrevious = + previous !== null && + previous.options === options && + !input.includes("\r") && + input.length >= previous.input.length && + input.startsWith(previous.input); + const stableLength = extendsPrevious ? previous.stableLength : 0; + const repaired = + stableLength > 0 + ? input.slice(0, stableLength) + + remend(input.slice(stableLength), options) + : remend(input, options); + return { input, options, repaired, stableLength }; +}; + +// Called with the blocks parsed from state.repaired. Moves the stable boundary +// past every block but the last two, and stops before the first block with +// math. Only an unchanged repair is used: then the blocks are those of the +// stream itself, while a closer appended at the end (such as "$$") could make +// an open block look finished. Custom handlers may read the whole message, so +// with them the whole message is always repaired. +export const advanceStableLength = ( + state: StreamRepairState, + blocks: string[] +): void => { + if ( + state.repaired !== state.input || + state.input.includes("\r") || + state.options?.handlers?.length + ) { + return; + } + let end = 0; + for (let i = 0; i < blocks.length - MUTABLE_BLOCKS; i += 1) { + end += blocks[i].length; + if (end <= state.stableLength) { + continue; + } + if (hasMathDelimiter(blocks[i])) { + return; + } + state.stableLength = end; + } +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2f91638..7e1ff523 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -289,6 +289,12 @@ importers: mdast-util-from-markdown: specifier: ^2.0.3 version: 2.0.3 + mdast-util-gfm-strikethrough: + specifier: ^2.0.0 + version: 2.0.0 + micromark-extension-gfm-strikethrough: + specifier: ^2.1.0 + version: 2.1.0 tsup: specifier: ^8.5.1 version: 8.5.1(jiti@2.7.0)(postcss@8.5.31)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.9.0)