Port upstream 4942: drive the token walk with a linear walker

third_party/muya/src/utils/marked/__tests__/walkTokensLinear.spec.ts

@@ -0,0 +1,65 @@
+import { Marked } from 'marked';
+import { describe, expect, it } from 'vitest';
+import { lexBlock } from '../lexBlock';
+
+// lexBlock walks tokens with a local linear driver instead of
+// `Marked.walkTokens`, whose per-token result concat is O(tokens²)
+// (marktext#4887). The drivers must keep visiting the same tokens.
+
+const NESTED = [
+    '# Heading with **bold**',
+    '',
+    '> quote with a [link](https://example.com)',
+    '',
+    '- item one\n- item two\n  - nested `code`',
+    '',
+    '| a | b |\n| - | - |\n| **c** | d |',
+    '',
+    'Paragraph with *em* text.',
+].join('\n');
+
+function visitedTypes(markdown: string): string[] {
+    const seen: string[] = [];
+    const m = new Marked();
+    const tokens = new m.Lexer(m.defaults).blockTokens(markdown);
+    m.walkTokens(tokens, token => void seen.push(token.type));
+    return seen.sort();
+}
+
+describe('lexBlock token walk', () => {
+    it('visits the same token set as Marked.walkTokens', () => {
+        const seen: string[] = [];
+        // lexBlock runs its walker internally; re-walk its output with the
+        // reference driver to compare coverage on identical token trees.
+        const tokens = lexBlock(NESTED, {
+            footnote: false,
+            isGitlabCompatibilityEnabled: true,
+            frontMatter: false,
+            math: false,
+        });
+        const m = new Marked();
+        m.walkTokens(tokens as never, token => void seen.push(token.type));
+
+        expect(seen.sort()).toEqual(visitedTypes(NESTED));
+        expect(seen).toContain('table');
+        expect(seen).toContain('list_item');
+    });
+
+    it('applies muya token augmentation to nested tokens', () => {
+        const tokens = lexBlock(
+            '> # quoted heading\n\n- item\n\n  ```js\n  x\n  ```\n',
+            {
+                footnote: false,
+                isGitlabCompatibilityEnabled: true,
+                frontMatter: false,
+                math: false,
+            },
+        );
+        const json = JSON.stringify(tokens);
+        // Both augmentations live on tokens that are only reachable through
+        // child arrays (blockquote > heading, list item > fenced code); a
+        // walker that misses child arrays would leave them untouched.
+        expect(json).toContain('"headingStyle":"atx"');
+        expect(json).toContain('"codeBlockStyle":"fenced"');
+    });
+});

third_party/muya/src/utils/marked/lexBlock.ts

@@ -8,6 +8,42 @@ import fm from './frontMatter';
 import { DEFAULT_OPTIONS } from './options';
 import walkTokens from './walkTokens';
 
+/**
+ * Depth-first token walk with the same visit order as `Marked.walkTokens`,
+ * minus its return-value accumulation: marked's driver `concat`s every
+ * callback result into one array, which clones that array once per token —
+ * O(tokens²) on large documents (the dominant parse cost in marktext#4887;
+ * our callbacks return nothing, so the accumulation is pure waste).
+ */
+function walkTokensLinear(
+    tokens: Token[],
+    callback: (token: Token) => void,
+): void {
+    for (const token of tokens) {
+        callback(token);
+
+        switch (token.type) {
+            case 'table': {
+                for (const cell of token.header)
+                    walkTokensLinear(cell.tokens, callback);
+                for (const row of token.rows) {
+                    for (const cell of row)
+                        walkTokensLinear(cell.tokens, callback);
+                }
+                break;
+            }
+            case 'list': {
+                walkTokensLinear(token.items, callback);
+                break;
+            }
+            default: {
+                if ('tokens' in token && token.tokens)
+                    walkTokensLinear(token.tokens, callback);
+            }
+        }
+    }
+}
+
 export function lexBlock(
     src: string,
     options: ILexOption = DEFAULT_OPTIONS,
@@ -46,7 +82,7 @@ export function lexBlock(
     // are picked up; the no-arg constructor would fall back to global defaults.
     tokens.push(...new m.Lexer(m.defaults).blockTokens(src));
     tokens = compatibleTaskList(tokens as Token[]);
-    m.walkTokens(tokens as Token[], walkTokens(options));
+    walkTokensLinear(tokens as Token[], walkTokens(options));
 
     // After walkTokens / compatibleTaskList run, marked's Heading/List/ListItem
     // tokens have been augmented with muya-specific fields (headingStyle,