fix(chunkers): preserve FAQ prose in docs chunks

cleanContent deleted every FAQ section from the embedding index: the
multiline tag strip swallows an entire <FAQ items={[...]}/> block (it
matches from <FAQ to the first ">", often inside an answer string), and
the brace strip eats any surviving { question, answer } items — 637
Q&As across the docs never reached search, with mangled JSX fragments
embedded in their place. Consume FAQ blocks whole before the tag strip
and emit their question/answer text as plain prose, escape-aware so
braces and quotes inside answers survive. Tag and brace stripping are
otherwise unchanged — a corpus survey showed FAQ props are the only
place real page prose lives inside JSX syntax on searchable pages.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: j15z

apps/sim/lib/chunkers/docs-chunker.test.ts

@@ -0,0 +1,105 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/knowledge/embeddings', () => ({
+  generateEmbeddings: vi.fn(async () => ({ embeddings: [] })),
+  getConfiguredEmbeddingModel: vi.fn(() => 'test-model'),
+}))
+
+import { DocsChunker } from '@/lib/chunkers/docs-chunker'
+
+function cleanContent(content: string): string {
+  const chunker = new DocsChunker()
+  return (chunker as unknown as { cleanContent(content: string): string }).cleanContent.call(
+    chunker,
+    content
+  )
+}
+
+describe('cleanContent FAQ extraction', () => {
+  it('keeps FAQ question/answer prose that the tag and brace strips would otherwise delete', () => {
+    const cleaned = cleanContent(
+      [
+        'Some intro prose.',
+        '',
+        'import { FAQ } from "@/components/ui/faq"',
+        '',
+        '<FAQ items={[',
+        '  { question: "What is the maximum file size for uploads?", answer: "The maximum file size for files processed during a workflow run is 20 MB." },',
+        '  { question: "How are files passed between blocks internally?", answer: "Files are represented as standardized UserFile objects." },',
+        ']} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('What is the maximum file size for uploads?')
+    expect(cleaned).toContain('20 MB')
+    expect(cleaned).toContain('standardized UserFile objects')
+    expect(cleaned).toContain('Some intro prose.')
+    expect(cleaned).not.toContain('items=')
+    expect(cleaned).not.toContain('question:')
+  })
+
+  it('survives braces and angle-bracket tokens inside answer strings', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ items={[',
+        `  { question: "What input formats work?", answer: "Use a data URI with the format 'data:{mime};base64,{data}' or a URL." },`,
+        '  { question: "Do I extract base64 manually?", answer: "No. Pass the entire file reference (e.g., <gmail.attachments[0]>) and the block extracts what it needs." },',
+        ']} />',
+      ].join('\n')
+    )
+
+    // Brace placeholders keep their token text; the wrapper chars are dropped
+    // so the later brace strip cannot punch holes in the sentence.
+    expect(cleaned).toContain("'data:mime;base64,data'")
+    // Angle brackets are dropped so the tag strip cannot re-eat the sentence.
+    expect(cleaned).toContain('(e.g., gmail.attachments[0]) and the block extracts')
+  })
+
+  it('extracts items formatted across multiple lines', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ items={[',
+        '  {',
+        '    question: "Is SSO supported?",',
+        '    answer: "Yes, on enterprise plans."',
+        '  },',
+        ']} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('Is SSO supported?')
+    expect(cleaned).toContain('Yes, on enterprise plans.')
+  })
+
+  it('unescapes escaped quotes in extracted strings', () => {
+    const cleaned = cleanContent(
+      '<FAQ items={[ { question: "What does \\"draft\\" mean?", answer: "An unsaved workflow." } ]} />'
+    )
+
+    expect(cleaned).toContain('What does "draft" mean?')
+  })
+})
+
+describe('cleanContent scaffolding strips', () => {
+  it('still strips imports, exports, comments, and code-ish brace expressions', () => {
+    const cleaned = cleanContent(
+      [
+        'import { Callout } from "fumadocs-ui/components/callout"',
+        'export const dynamic = "force-static"',
+        '{/* editorial note */}',
+        'Visible prose {props.title} continues here.',
+        '<Callout>Inside text stays</Callout>',
+      ].join('\n')
+    )
+
+    expect(cleaned).not.toContain('import')
+    expect(cleaned).not.toContain('force-static')
+    expect(cleaned).not.toContain('editorial note')
+    expect(cleaned).not.toContain('props.title')
+    expect(cleaned).toContain('Visible prose')
+    expect(cleaned).toContain('Inside text stays')
+  })
+})

apps/sim/lib/chunkers/docs-chunker.ts

@@ -21,6 +21,38 @@ interface Frontmatter {
 
 const logger = createLogger('DocsChunker')
 
+/**
+ * One `{ question: "...", answer: "..." }` FAQ item. Quoted strings are
+ * consumed escape-aware, so braces or quotes inside an answer never end a
+ * match early.
+ */
+const FAQ_ITEM_PATTERN =
+  /\{\s*question:\s*"((?:[^"\\]|\\.)*)"\s*,\s*answer:\s*"((?:[^"\\]|\\.)*)"\s*\}/g
+
+function unescapeJsxString(value: string): string {
+  return value.replace(/\\(.)/g, (_, char: string) =>
+    char === 'n' ? '\n' : char === 't' ? '\t' : char
+  )
+}
+
+/**
+ * Emit an FAQ block's question/answer strings as plain prose lines. Must run
+ * BEFORE the tag strip: a `<FAQ items={[` opening tag has no `>` until the
+ * closing `]} />`, so the multiline tag regex would otherwise swallow the
+ * items whole (ending at the first `>` inside an answer). Angle brackets and
+ * braces around inline tokens (`<gmail.attachments[0]>`, `data:{mime}`) are
+ * dropped so the later tag and brace strips cannot re-consume the emitted
+ * text.
+ */
+function extractFaqProse(items: string): string {
+  const lines: string[] = []
+  for (const match of items.matchAll(FAQ_ITEM_PATTERN)) {
+    lines.push(unescapeJsxString(match[1]), unescapeJsxString(match[2]))
+  }
+  if (lines.length === 0) return ' '
+  return `\n${lines.join('\n').replace(/[<>{}]/g, '')}\n`
+}
+
 export class DocsChunker {
   private readonly textChunker: TextChunker
   private readonly baseUrl: string
@@ -216,6 +248,7 @@ export class DocsChunker {
       .replace(/\r/g, '\n')
       .replace(/^import\s+.*$/gm, '')
       .replace(/^export\s+.*$/gm, '')
+      .replace(/<FAQ\s+items=\{\[([\s\S]*?)\]\}\s*\/>/g, (_m, items: string) => extractFaqProse(items))
       .replace(/<\/?[a-zA-Z][^>]*>/g, ' ')
       .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
       .replace(/\{[^{}]*\}/g, ' ')