fix(chunkers): preserve FAQ and component-attribute prose in docs chunks

cleanContent stripped every JSX brace expression and whole tags, which
deleted the FAQ component's question/answer strings (595 Q&As across
108 docs pages) and prose-bearing title/description/alt attributes from
the embedding index — while leaving mangled JSX debris behind. Consume
FAQ blocks whole before the tag strip and emit their Q&A text as plain
prose, keep title/description/alt values when tags are dropped, and
leave the brace strip untouched for genuinely code-ish expressions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: j15z

apps/sim/lib/chunkers/docs-chunker.test.ts

@@ -0,0 +1,124 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/knowledge/embeddings', () => ({
+  generateEmbeddings: vi.fn(async () => ({ embeddings: [] })),
+  getConfiguredEmbeddingModel: vi.fn(() => 'test-model'),
+}))
+
+import { DocsChunker } from '@/lib/chunkers/docs-chunker'
+
+function cleanContent(content: string): string {
+  const chunker = new DocsChunker()
+  return (chunker as unknown as { cleanContent(content: string): string }).cleanContent.call(
+    chunker,
+    content
+  )
+}
+
+describe('cleanContent FAQ extraction', () => {
+  it('keeps FAQ question/answer prose that the tag and brace strips would otherwise delete', () => {
+    const cleaned = cleanContent(
+      [
+        'Some intro prose.',
+        '',
+        'import { FAQ } from "@/components/ui/faq"',
+        '',
+        '<FAQ items={[',
+        '  { question: "What is the maximum file size for uploads?", answer: "The maximum file size for files processed during a workflow run is 20 MB." },',
+        '  { question: "How are files passed between blocks internally?", answer: "Files are represented as standardized UserFile objects." },',
+        ']} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('What is the maximum file size for uploads?')
+    expect(cleaned).toContain('20 MB')
+    expect(cleaned).toContain('standardized UserFile objects')
+    expect(cleaned).toContain('Some intro prose.')
+    expect(cleaned).not.toContain('items=')
+    expect(cleaned).not.toContain('question:')
+  })
+
+  it('survives braces and angle-bracket tokens inside answer strings', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ items={[',
+        `  { question: "What input formats work?", answer: "Use a data URI with the format 'data:{mime};base64,{data}' or a URL." },`,
+        '  { question: "Do I extract base64 manually?", answer: "No. Pass the entire file reference (e.g., <gmail.attachments[0]>) and the block extracts what it needs." },',
+        ']} />',
+      ].join('\n')
+    )
+
+    // Brace placeholders keep their token text; the wrapper chars are dropped
+    // so the later brace strip cannot punch holes in the sentence.
+    expect(cleaned).toContain("'data:mime;base64,data'")
+    // Angle brackets are dropped so the tag strip cannot re-eat the sentence.
+    expect(cleaned).toContain('(e.g., gmail.attachments[0]) and the block extracts')
+  })
+
+  it('extracts items formatted across multiple lines', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ items={[',
+        '  {',
+        '    question: "Is SSO supported?",',
+        '    answer: "Yes, on enterprise plans."',
+        '  },',
+        ']} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('Is SSO supported?')
+    expect(cleaned).toContain('Yes, on enterprise plans.')
+  })
+
+  it('unescapes escaped quotes in extracted strings', () => {
+    const cleaned = cleanContent(
+      '<FAQ items={[ { question: "What does \\"draft\\" mean?", answer: "An unsaved workflow." } ]} />'
+    )
+
+    expect(cleaned).toContain('What does "draft" mean?')
+  })
+})
+
+describe('cleanContent prose attributes', () => {
+  it('keeps title, description, and alt values while dropping the tag chrome', () => {
+    const cleaned = cleanContent(
+      [
+        '<Card title="Using tables in workflows" description="Read, write, and update rows with the Table block." href="/tables/using-in-workflows" className="p-2">',
+        '</Card>',
+        '<Image src="/static/tables.png" alt="A table of typed columns" width={800} height={500} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('Using tables in workflows')
+    expect(cleaned).toContain('Read, write, and update rows with the Table block.')
+    expect(cleaned).toContain('A table of typed columns')
+    expect(cleaned).not.toContain('href')
+    expect(cleaned).not.toContain('p-2')
+    expect(cleaned).not.toContain('/static/tables.png')
+  })
+})
+
+describe('cleanContent scaffolding strips', () => {
+  it('still strips imports, exports, comments, and code-ish brace expressions', () => {
+    const cleaned = cleanContent(
+      [
+        'import { Callout } from "fumadocs-ui/components/callout"',
+        'export const dynamic = "force-static"',
+        '{/* editorial note */}',
+        'Visible prose {props.title} continues here.',
+        '<Callout>Inside text stays</Callout>',
+      ].join('\n')
+    )
+
+    expect(cleaned).not.toContain('import')
+    expect(cleaned).not.toContain('force-static')
+    expect(cleaned).not.toContain('editorial note')
+    expect(cleaned).not.toContain('props.title')
+    expect(cleaned).toContain('Visible prose')
+    expect(cleaned).toContain('Inside text stays')
+  })
+})

apps/sim/lib/chunkers/docs-chunker.ts

@@ -21,6 +21,61 @@ interface Frontmatter {
 
 const logger = createLogger('DocsChunker')
 
+/**
+ * The FAQ component and its items: `<FAQ items={[{ question: "...", answer:
+ * "..." }, ...]} />`. The component is consumed whole BEFORE the tag strip —
+ * its opening tag has no `>` until the closing `]} />`, so the multiline tag
+ * regex would otherwise swallow the items (ending at the first `>` inside an
+ * answer). The item regex consumes quoted strings escape-aware, so braces or
+ * quotes inside an answer never end a match early.
+ */
+const FAQ_COMPONENT_PATTERN = /<FAQ\s+items=\{\[([\s\S]*?)\]\}\s*\/>/g
+const FAQ_ITEM_PATTERN =
+  /\{\s*question:\s*"((?:[^"\\]|\\.)*)"\s*,\s*answer:\s*"((?:[^"\\]|\\.)*)"\s*\}/g
+
+/**
+ * Tag attributes whose string values are page prose (fumadocs Card/Image/
+ * Accordion vocabulary), kept as text when the surrounding tag is stripped.
+ * Everything else inside JSX syntax is scaffolding. Extend this vocabulary
+ * when a new docs component carries prose in its props.
+ */
+const PROSE_TAG_ATTRIBUTE_PATTERN = /\b(?:title|description|alt)="([^"]*)"/g
+
+function unescapeJsxString(value: string): string {
+  return value.replace(/\\(.)/g, (_, char: string) =>
+    char === 'n' ? '\n' : char === 't' ? '\t' : char
+  )
+}
+
+/**
+ * Emit an FAQ block's question/answer strings as plain prose lines. Angle
+ * brackets and braces around inline tokens (`<gmail.attachments[0]>`,
+ * `data:{mime}`) are dropped so the later tag and brace strips cannot
+ * re-consume parts of the emitted text.
+ */
+function extractFaqProse(items: string): string {
+  const lines: string[] = []
+  for (const match of items.matchAll(FAQ_ITEM_PATTERN)) {
+    lines.push(unescapeJsxString(match[1]), unescapeJsxString(match[2]))
+  }
+  if (lines.length === 0) return ' '
+  return `\n${lines.join('\n').replace(/[<>{}]/g, '')}\n`
+}
+
+/**
+ * Tag-strip replacer keeping prose-bearing attribute values (title,
+ * description, alt) while the tag and its remaining attributes are dropped.
+ */
+function keepProseAttributes(tag: string): string {
+  const kept: string[] = []
+  for (const match of tag.matchAll(PROSE_TAG_ATTRIBUTE_PATTERN)) {
+    const value = match[1].trim()
+    if (value) kept.push(value)
+  }
+  if (kept.length === 0) return ' '
+  return ` ${kept.join('. ')} `
+}
+
 export class DocsChunker {
   private readonly textChunker: TextChunker
   private readonly baseUrl: string
@@ -216,7 +271,8 @@ export class DocsChunker {
       .replace(/\r/g, '\n')
       .replace(/^import\s+.*$/gm, '')
       .replace(/^export\s+.*$/gm, '')
-      .replace(/<\/?[a-zA-Z][^>]*>/g, ' ')
+      .replace(FAQ_COMPONENT_PATTERN, (_match, items: string) => extractFaqProse(items))
+      .replace(/<\/?[a-zA-Z][^>]*>/g, keepProseAttributes)
       .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
       .replace(/\{[^{}]*\}/g, ' ')
       .replace(/\n{3,}/g, '\n\n')