feat(docs): fail CI when generated integration docs are stale

Author: j15z

apps/docs/content/docs/en/integrations/ashby.mdx

@@ -1493,7 +1493,7 @@ Trigger workflow when a new job is created
 |   ↳ `title` | string | Job title |
 |   ↳ `confidential` | boolean | Whether the job is confidential |
 |   ↳ `status` | string | Job status \(Open, Closed, Draft, Archived\) |
-|   ↳ `employmentType` | string | Employment type \(FullTime, PartTime, Intern, Contract\) |
+|   ↳ `employmentType` | string | Employment type \(FullTime, PartTime, Intern, Contract, Temporary\) |
 
 
 ---

apps/docs/content/docs/en/integrations/logrocket.mdx

@@ -188,3 +188,4 @@ Register a release version in LogRocket so uploaded source maps can decode stack
 | --------- | ---- | ----------- |
 | `version` | string | Release version that was registered |
 
+

apps/docs/content/docs/en/integrations/netsuite.mdx

@@ -5,7 +5,7 @@ description: Manage NetSuite records, queries, datasets, batches, and async jobs
 
 import { BlockInfoCard } from "@/components/ui/block-info-card"
 
-<BlockInfoCard
+<BlockInfoCard 
   type="netsuite"
   color="#FFFFFF"
 />

apps/docs/content/docs/en/integrations/snowflake.mdx

@@ -5,7 +5,7 @@ description: Query data and manage warehouses and tasks in Snowflake
 
 import { BlockInfoCard } from "@/components/ui/block-info-card"
 
-<BlockInfoCard
+<BlockInfoCard 
   type="snowflake"
   color="#FFFFFF"
 />

apps/docs/content/docs/en/integrations/zoho_desk.mdx

@@ -5,7 +5,7 @@ description: Manage Zoho Desk tickets, comments, threads, and contacts
 
 import { BlockInfoCard } from "@/components/ui/block-info-card"
 
-<BlockInfoCard
+<BlockInfoCard 
   type="zoho_desk"
   color="#FFFFFF"
 />
@@ -522,3 +522,4 @@ Trigger a workflow when a Zoho Desk event occurs (ticket, comment, thread, conta
 | `orgId` | string | Zoho Desk organization ID |
 | `payload` | json | The full resource that changed \(ticket, comment, thread, etc.\). Comment and thread events gain a derived plain-text `contentText` alongside the raw `content` + `contentType`; ticket events gain `descriptionText` alongside `description`. |
 | `prevState` | json | Previous state of the resource \(update events only\) |
+

package.json

@@ -62,6 +62,7 @@
     "tool-metadata:generate": "bun run scripts/sync-tool-metadata.ts",
     "tool-metadata:check": "bun run scripts/sync-tool-metadata.ts --check",
     "integration-catalog:check": "bun run scripts/check-integration-catalog.ts",
+    "docs:check": "bun run scripts/generate-docs.ts --check",
     "mship-tools:generate": "bun run scripts/sync-tool-catalog.ts",
     "mship-tools:check": "bun run scripts/sync-tool-catalog.ts --check",
     "trace-spans-contract:generate": "bun run scripts/sync-trace-spans-contract.ts",

scripts/generate-docs.ts

@@ -358,23 +358,71 @@ interface IconRef {
   source: string
 }
 
+/**
+ * Check mode (`--check`): render every generated artifact in memory and compare
+ * it against the committed file instead of writing, so CI can fail on docs
+ * drift the same way `tool-metadata:check` fails on stale tool metadata. Check
+ * mode performs no filesystem mutations.
+ *
+ * The pipeline writes some pages twice per run — the block pass writes the base
+ * page, then the trigger pass reads it back and appends/merges the Triggers
+ * section — so check mode keeps an in-memory overlay of everything "written"
+ * this run (`emittedByPath`), readers consult the overlay before disk
+ * (`readGeneratedFile`), and staleness is judged once at the end against each
+ * artifact's FINAL content. Comparing at emit time would flag the intermediate
+ * block-pass content of every trigger-owning page as a false positive.
+ *
+ * Known limitation: `updateMetaJson` derives the sidebar from the mdx files on
+ * disk, so in check mode a brand-new block's missing page is reported directly
+ * while the corresponding meta.json entry is not — regenerating fixes both.
+ */
+let CHECK_ONLY = false
+const staleArtifacts: string[] = []
+const emittedByPath = new Map<string, string>()
+
+/** Writes a generated artifact, or in check mode records its final content for the end-of-run comparison. */
+function emitGeneratedFile(filePath: string, content: string): void {
+  if (CHECK_ONLY) {
+    emittedByPath.set(filePath, content)
+    return
+  }
+  fs.writeFileSync(filePath, content)
+}
+
+/** Reads a generated artifact as the pipeline would see it mid-run: overlay first in check mode, then disk. */
+function readGeneratedFile(filePath: string): string | null {
+  const emitted = emittedByPath.get(filePath)
+  if (emitted !== undefined) return emitted
+  return fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : null
+}
+
+/** Compares every overlay entry against the committed file; returns repo-relative stale paths. */
+function collectStaleEmissions(): string[] {
+  const stale: string[] = []
+  for (const [filePath, content] of emittedByPath) {
+    const committed = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : null
+    if (committed !== content) stale.push(path.relative(rootDir, filePath))
+  }
+  return stale
+}
+
 /**
  * Copy the icons.tsx file from the main sim app to the docs app
  * This ensures icons are rendered consistently across both apps
  */
 function copyIconsFile(): void {
   try {
-    console.log('Copying icons from sim app to docs app...')
+    if (!CHECK_ONLY) console.log('Copying icons from sim app to docs app...')
 
     if (!fs.existsSync(ICONS_PATH)) {
       console.error(`Source icons file not found: ${ICONS_PATH}`)
       return
     }
 
     const iconsContent = fs.readFileSync(ICONS_PATH, 'utf-8')
-    fs.writeFileSync(DOCS_ICONS_PATH, iconsContent)
+    emitGeneratedFile(DOCS_ICONS_PATH, iconsContent)
 
-    console.log('✓ Icons successfully copied to docs app')
+    if (!CHECK_ONLY) console.log('✓ Icons successfully copied to docs app')
   } catch (error) {
     console.error('Error copying icons file:', error)
   }
@@ -579,8 +627,8 @@ ${mappingEntries}
 }
 `
 
-    fs.writeFileSync(iconMappingPath, content)
-    console.log('✓ Icon mapping file written to docs app')
+    emitGeneratedFile(iconMappingPath, content)
+    if (!CHECK_ONLY) console.log('✓ Icon mapping file written to docs app')
   } catch (error) {
     console.error('Error writing icon mapping:', error)
   }
@@ -938,8 +986,8 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
 ${mappingEntries}
 }
 `
-    fs.writeFileSync(iconMappingPath, content)
-    console.log('✓ Integration icon mapping written')
+    emitGeneratedFile(iconMappingPath, content)
+    if (!CHECK_ONLY) console.log('✓ Integration icon mapping written')
   } catch (error) {
     console.error('Error writing integration icon mapping:', error)
   }
@@ -1122,6 +1170,11 @@ async function writeIntegrationsJson(iconMapping: Record<string, IconRef>): Prom
       return
     }
 
+    if (CHECK_ONLY) {
+      staleArtifacts.push(path.relative(rootDir, jsonPath))
+      return
+    }
+
     const updatedAt = new Date().toISOString().slice(0, 10)
     fs.writeFileSync(jsonPath, `${serialize({ updatedAt, integrations })}\n`)
     console.log(`✓ Integration data written: ${integrations.length} integrations → ${jsonPath}`)
@@ -3117,10 +3170,7 @@ async function generateBlockDoc(blockPath: string) {
       const displayType = stripVersionSuffix(blockConfig.type)
       const outputFilePath = path.join(DOCS_OUTPUT_PATH, `${displayType}.mdx`)
 
-      let existingContent: string | null = null
-      if (fs.existsSync(outputFilePath)) {
-        existingContent = fs.readFileSync(outputFilePath, 'utf-8')
-      }
+      const existingContent = readGeneratedFile(outputFilePath)
 
       const manualSections = existingContent ? extractManualContent(existingContent) : {}
 
@@ -3131,10 +3181,14 @@ async function generateBlockDoc(blockPath: string) {
         finalContent = mergeWithManualContent(markdown, existingContent, manualSections)
       }
 
-      fs.writeFileSync(outputFilePath, finalContent)
-      const logType =
-        displayType !== blockConfig.type ? `${displayType} (from ${blockConfig.type})` : displayType
-      console.log(`✓ Generated docs for ${logType}`)
+      emitGeneratedFile(outputFilePath, finalContent)
+      if (!CHECK_ONLY) {
+        const logType =
+          displayType !== blockConfig.type
+            ? `${displayType} (from ${blockConfig.type})`
+            : displayType
+        console.log(`✓ Generated docs for ${logType}`)
+      }
     }
   } catch (error) {
     console.error(`Error processing ${blockPath}:`, error)
@@ -3376,6 +3430,13 @@ function cleanupStaleToolDocs(validToolDocs: Set<string>): void {
       continue
     }
 
+    if (CHECK_ONLY) {
+      staleArtifacts.push(
+        `${path.relative(rootDir, docPath)} (stale page — regeneration would delete it)`
+      )
+      continue
+    }
+
     fs.unlinkSync(docPath)
     console.log(`✓ Removed stale tool doc: ${blockType}.mdx`)
     removedCount++
@@ -3900,14 +3961,16 @@ async function generateAllTriggerDocs(): Promise<void> {
         continue
       }
 
-      const existing = fs.existsSync(outputFilePath)
-        ? fs.readFileSync(outputFilePath, 'utf-8')
-        : null
+      const existing = readGeneratedFile(outputFilePath)
 
       if (existing?.includes('\n## Actions')) {
         // Actions page generated this run by the block pass — append the Triggers section.
         if (!existing.includes('\n## Triggers')) {
-          fs.appendFileSync(outputFilePath, `\n${buildTriggersSection(triggers)}`)
+          if (CHECK_ONLY) {
+            emittedByPath.set(outputFilePath, `${existing}\n${buildTriggersSection(triggers)}`)
+          } else {
+            fs.appendFileSync(outputFilePath, `\n${buildTriggersSection(triggers)}`)
+          }
         }
       } else {
         // Trigger-only service (no actions block) — (re)write the standalone page,
@@ -3923,13 +3986,15 @@ async function generateAllTriggerDocs(): Promise<void> {
           Object.keys(manualSections).length > 0
             ? mergeWithManualContent(markdown, existing, manualSections)
             : markdown
-        fs.writeFileSync(outputFilePath, finalContent)
+        emitGeneratedFile(outputFilePath, finalContent)
       }
 
       generatedProviders.push(blockType)
-      console.log(
-        `✓ Triggers for ${formatTriggerProviderName(provider)} (${triggers.length} trigger${triggers.length === 1 ? '' : 's'})`
-      )
+      if (!CHECK_ONLY) {
+        console.log(
+          `✓ Triggers for ${formatTriggerProviderName(provider)} (${triggers.length} trigger${triggers.length === 1 ? '' : 's'})`
+        )
+      }
     }
 
     console.log(`✓ Trigger sections merged into ${generatedProviders.length} integration pages`)
@@ -3990,21 +4055,37 @@ function updateMetaJson() {
     pages: items,
   }
 
-  fs.writeFileSync(metaJsonPath, `${JSON.stringify(metaJson, null, 2)}\n`)
-  console.log(`Updated meta.json with ${items.length} entries`)
+  emitGeneratedFile(metaJsonPath, `${JSON.stringify(metaJson, null, 2)}\n`)
+  if (!CHECK_ONLY) console.log(`Updated meta.json with ${items.length} entries`)
 }
 
 if (import.meta.main) {
-  console.log('Starting documentation generator...')
+  CHECK_ONLY = process.argv.includes('--check')
+  console.log(
+    CHECK_ONLY
+      ? 'Checking generated documentation freshness...'
+      : 'Starting documentation generator...'
+  )
   generateAllBlockDocs()
     .then((success) => {
-      if (success) {
-        console.log('Documentation generation completed successfully')
-        process.exit(0)
-      } else {
+      if (!success) {
         console.error('Documentation generation failed')
         process.exit(1)
       }
+      if (CHECK_ONLY) {
+        const stale = [...collectStaleEmissions(), ...staleArtifacts]
+        if (stale.length > 0) {
+          console.error(
+            `Generated integration docs are stale:\n- ${stale.join('\n- ')}\n` +
+              'Run `bun run scripts/generate-docs.ts` and commit the result.'
+          )
+          process.exit(1)
+        }
+        console.log('✓ Generated integration docs are in sync')
+        process.exit(0)
+      }
+      console.log('Documentation generation completed successfully')
+      process.exit(0)
     })
     .catch((error) => {
       console.error('Fatal error:', error)

scripts/run-audits.ts

@@ -27,6 +27,7 @@ const EXCLUDED: Record<string, string> = {
 const EXTRA_AUDITS = [
   'tool-metadata:check',
   'integration-catalog:check',
+  'docs:check',
   'skills:check',
   'agent-stream-docs:check',
 ] as const