feat(comark)!: decode HTML attributes with attribute semantics, pluggable textContent decoder

benchmarks/fixtures.ts

@@ -253,6 +253,28 @@ ${Array.from(
 That's it — let me know if you want a full example!
 `
 
+/**
+ * Raw HTML with many attributes, as found in docs written for GitHub/VitePress.
+ * Every tag goes through `parseHtmlAttributes`, half of the values carry
+ * entities so both the plain and the decoding path are measured.
+ */
+export const htmlAttributesMarkdown = `# Raw HTML
+
+${Array.from(
+  { length: 30 },
+  (_, i) => `
+<div class="card card-${i}" id="card-${i}" data-index="${i}" hidden>
+  <a href="https://example.com/search?q=${i}&amp;page=2&amp;sort=asc" title="Tom &amp; Jerry &quot;${i}&quot;" target="_blank" rel="noopener noreferrer">
+    <img src="https://example.com/img/${i}.png" alt="Image &#8220;${i}&#8221; &mdash; preview" width="64" height="64" loading="lazy">
+  </a>
+  <span class='label' data-tooltip='It&#39;s item ${i}'>Item ${i}</span>
+</div>
+
+Inline <kbd class="key">Ctrl</kbd> + <abbr title="Rock &amp; Roll">R&amp;R</abbr> and <a href="/docs/${i}" class="link">docs ${i}</a>.
+`
+).join('\n')}
+`
+
 /** Splits `markdown` into `count` growing prefixes, ending with the full text. */
 export function toStreamChunks(markdown: string, count: number): string[] {
   const chunks: string[] = []

benchmarks/html.bench.ts

@@ -0,0 +1,21 @@
+import { bench, describe } from 'vitest'
+import { createMarkdownParser } from 'comark'
+import { htmlAttributesMarkdown } from './fixtures.ts'
+
+// Raw HTML attributes are decoded by `parseHtmlAttributes`; the generic parse
+// fixtures barely contain HTML, so they cannot catch changes on that path.
+const parse = createMarkdownParser()
+const parseLiteralHtml = createMarkdownParser({ autoClose: false })
+
+await parse(htmlAttributesMarkdown)
+await parseLiteralHtml(htmlAttributesMarkdown)
+
+describe('parse html', () => {
+  bench('html attributes document', async () => {
+    await parse(htmlAttributesMarkdown)
+  })
+
+  bench('html attributes document without auto-close', async () => {
+    await parseLiteralHtml(htmlAttributesMarkdown)
+  })
+})

docs/content/1.getting-started/2.document-model.md

@@ -250,6 +250,15 @@ const heading = document.nodes[0]
 console.log(textContent(heading)) // Hello world
 ```
 
+Text in parsed documents is already entity-decoded. For trees whose text may still contain character references, such as hand-built nodes, pass a `decodeHTML` function. It runs on each text node:
+
+```typescript [text-content-decode.ts]
+import { decodeHTML } from 'entities'
+import { textContent } from 'comark/utils'
+
+textContent(['p', {}, 'Tom & Jerry'], { decodeHTML }) // Tom & Jerry
+```
+
 ### Visit nodes
 
 `visit()` walks every node that matches a checker. A visitor can inspect, replace, or remove matching nodes:

packages/comark/src/internal/parse/utils.ts

@@ -1,5 +1,6 @@
 import type { Node, ElementNode, ElementNodeAttributes, MarkdownDocument } from 'comark'
-import { decodeHTML, isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils'
+import { decodeHTMLAttribute } from 'entities'
+import { isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils'
 
 /** Space, tab, line feed, form feed or carriage return. Defined per file so it stays inlinable on hot paths. */
 function isSpaceCode(c: number): boolean {
@@ -185,6 +186,15 @@ export function parseHtmlInline(content: string): ParsedHtmlTag {
   return { kind: 'other', content: trimmed }
 }
 
+/**
+ * Decode character references in a raw attribute value the way browsers do:
+ * legacy semicolon-less references (`&amp`) are decoded, but not when followed by
+ * `=` or an alphanumeric (`?a=1&copy=2` stays literal). Backslashes are literal.
+ */
+function decodeAttributeValue(value: string): string {
+  return value.includes('&') ? decodeHTMLAttribute(value) : value
+}
+
 /**
  * Parse HTML attribute string into a key/value map.
  * Supports `key="value"`, `key='value'`, bare `key=value`, and boolean attributes.
@@ -199,7 +209,7 @@ export function parseHtmlAttributes(attrsStr: string): Record<string, unknown> {
   while ((match = re.exec(attrsStr)) !== null) {
     const key = match[1].toLowerCase()
     const value = match[2] ?? match[3] ?? match[4]
-    attrs[key] = value === undefined ? true : decodeHTML(value)
+    attrs[key] = value === undefined ? true : decodeAttributeValue(value)
   }
 
   return attrs

packages/comark/src/utils/index.ts

@@ -1,10 +1,7 @@
 // #region Tree Utils
 
-import { decodeHTML } from 'entities'
 import type { Node, MarkdownDocument } from 'comark'
 
-export { decodeHTML } from 'entities'
-
 type VisitResult = Node | false | undefined | void
 
 /**
@@ -14,25 +11,35 @@ export function isMarkdownDocument(value: unknown): value is MarkdownDocument {
   return !!value && typeof value === 'object' && Array.isArray((value as MarkdownDocument).nodes)
 }
 
+export interface TextContentOptions {
+  /**
+   * Called on every text node before it is appended, e.g. `decodeHTML` from `entities`.
+   * Parsed documents already have decoded text, so only pass this for trees whose text
+   * may still contain character references (hand-built or from another source).
+   */
+  decodeHTML?: (text: string) => string
+}
+
 /**
  * Get the text content of a Markdown AST node
  *
  * @param node - The node
  * @param options - The options
- * @param options.decodeUnicodeEntities - Whether to decode Unicode entities
+ * @param options.decodeHTML - Function applied to each text node
  * @returns The text content
  */
-export function textContent(node: Node, options: { decodeUnicodeEntities?: boolean } = {}): string {
+export function textContent(node: Node, options?: TextContentOptions): string {
+  return collectText(node, options?.decodeHTML)
+}
+
+function collectText(node: Node, decode: ((text: string) => string) | undefined): string {
   if (typeof node === 'string') {
-    if (options.decodeUnicodeEntities) {
-      return decodeHTML(node)
-    }
-    return node as string
+    return decode ? decode(node) : node
   }
   let out = ''
   const len = node.length
   for (let i = 2; i < len; i++) {
-    out += textContent(node[i] as Node, options)
+    out += collectText(node[i] as Node, decode)
   }
   return out
 }

packages/comark/test/html-escape.test.ts

@@ -39,6 +39,29 @@ describe('HTML attribute escaping', () => {
     expect(html).not.toContain('title="A "quote""')
   })
 
+  it('keeps backslashes literal while decoding HTML attribute entities', async () => {
+    const tree = await parseMarkdown('<span title="C:\\*dir\\ &amp; &copy;">hi</span>')
+    const p = tree.nodes[0] as [string, Record<string, unknown>, ...Node[]]
+    const span = (Array.isArray(p[2]) ? p[2] : p) as [string, Record<string, unknown>, ...Node[]]
+    expect(span[1].title).toBe('C:\\*dir\\ & ©')
+  })
+
+  it('decodes HTML attribute entities with browser attribute semantics', async () => {
+    const attrsOf = async (markdown: string) => {
+      const tree = await parseMarkdown(markdown)
+      const p = tree.nodes[0] as [string, Record<string, unknown>, ...Node[]]
+      return ((Array.isArray(p[2]) ? p[2] : p) as [string, Record<string, unknown>, ...Node[]])[1]
+    }
+
+    // Legacy semicolon-less reference is decoded...
+    expect((await attrsOf('<span title="A &amp B">hi</span>')).title).toBe('A & B')
+    // ...but not when followed by `=` or an alphanumeric, so query strings survive.
+    expect((await attrsOf('<a href="?a=1&copy=2">hi</a>')).href).toBe('?a=1&copy=2')
+    expect((await attrsOf('<span title="&notit;">hi</span>')).title).toBe('&notit;')
+    // Invalid code points become U+FFFD; references are decoded once.
+    expect((await attrsOf('<span title="&#0;|&amp;lt;">hi</span>')).title).toBe('\uFFFD|&lt;')
+  })
+
   it('escapes quotes in component attribute props', async () => {
     const html = await renderHtml(`:span[hi]{title='a" onmouseover=alert(1) x="b'}`)
     expect(html).toContain('title="a&quot; onmouseover=alert(1) x=&quot;b"')

packages/comark/test/text-content.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from 'vitest'
+import { decodeHTML } from 'entities'
+import type { Node } from 'comark'
+import { textContent } from '../src/utils'
+
+describe('textContent', () => {
+  const tree: Node = ['p', {}, 'Tom & ', ['strong', {}, 'Jerry'], ' ©']
+
+  it('concatenates text nodes', () => {
+    expect(textContent('plain')).toBe('plain')
+    expect(textContent(tree)).toBe('Tom & Jerry ©')
+  })
+
+  it('applies decodeHTML to every text node', () => {
+    expect(textContent(tree, { decodeHTML })).toBe('Tom & Jerry ©')
+    expect(textContent('&lt;', { decodeHTML })).toBe('<')
+  })
+
+  it('decodes each text node separately', () => {
+    const calls: string[] = []
+    const split: Node = ['p', {}, '&am', 'p;']
+    const result = textContent(split, {
+      decodeHTML: (text) => {
+        calls.push(text)
+        return decodeHTML(text)
+      },
+    })
+    expect(calls).toEqual(['&am', 'p;'])
+    expect(result).toBe('&amp;')
+  })
+})

pnpm-workspace.yaml

@@ -90,7 +90,8 @@ catalog:
   comark: ^0.8.1
 
   # Core comark dependencies
-  entities: ^8.1.0
+  # Keep in the same major as markdown-exit's `entities` so installs dedupe to one copy.
+  entities: ^7.0.0
   github-slugger: ^2.0.0
   hast-util-to-string: ^3.0.1
   js-yaml: ^5.5.0