Skip to content

Commit 9b8d72d

Browse files
authored
improvement(library): check sim's integration counts against the catalog (#10113)
* improvement(library): hold sim's integration, action, and trigger counts to the catalog in check:library-content * fix(library): match bold and title-cased sim counts, and track code fences by length
1 parent f836354 commit 9b8d72d

2 files changed

Lines changed: 127 additions & 2 deletions

File tree

‎scripts/check-library-content.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ beforeEach(() => {
7575
'https://www.sim.ai/models/anthropic',
7676
'https://www.sim.ai/models/anthropic/claude-opus-4-6',
7777
]),
78+
simCatalog: { integrations: 280, actions: 5826, triggers: 538 },
7879
}
7980
mkdirSync(path.join(config.contentDir, 'authors'), { recursive: true })
8081
writeFileSync(path.join(config.contentDir, 'authors', 'sim.json'), '{"id":"sim","name":"Sim"}')
@@ -318,6 +319,61 @@ describe('check-library-content', () => {
318319
expect(findings[0]).toMatchObject({ line: 12, rule: 'mdx' })
319320
})
320321

322+
it.each([
323+
['Sim connects 1,000+ integrations and every major LLM.', '1,000+ integrations', 'overstates'],
324+
[
325+
"Sim's [100+ integrations](https://www.sim.ai/integrations) cover the basics.",
326+
'100+ integrations',
327+
'stale',
328+
],
329+
['Sim ships 9,000+ actions across its catalog.', '9,000+ actions', 'overstates'],
330+
['Sim listens for events with 300+ triggers.', '300+ triggers', 'stale'],
331+
['Sim has **1,000+** integrations.', '1,000+ integrations', 'overstates'],
332+
['Sim has 1,000+ Integrations.', '1,000+ Integrations', 'overstates'],
333+
])('rejects a Sim count the catalog does not support: %s', async (body, claim, verdict) => {
334+
writePost('library', 'post', body)
335+
expect(await findingsFor('library', 'post')).toEqual([
336+
{
337+
line: 10,
338+
rule: 'sim-count',
339+
message: expect.stringMatching(new RegExp(`${claim.replace('+', '\\+')}.*${verdict}`)),
340+
},
341+
])
342+
})
343+
344+
it('rejects an unsupported Sim count in a frontmatter FAQ answer, at its line', async () => {
345+
writePost('library', 'post', 'Body.', {
346+
faq: '\n - q: "How many integrations does Sim have?"\n a: "Sim has 1,000+ integrations."',
347+
})
348+
expect(await findingsFor('library', 'post')).toEqual([
349+
expect.objectContaining({ line: 10, rule: 'sim-count' }),
350+
])
351+
})
352+
353+
it('accepts true Sim counts and leaves competitor counts alone', async () => {
354+
writePost(
355+
'library',
356+
'post',
357+
[
358+
'Sim has 280+ integrations, 5,800+ actions, and 530+ triggers.',
359+
"Sim's pricing page documents more than 270 integrations.",
360+
'Sim fits teams that want a collaborative visual builder, 270+ integrations, and an Apache 2.0 core.',
361+
"Zapier lists 9,000+ integrations, while Sim's 280+ integrations focus on agents.",
362+
'Compared with Sim, MindStudio advertises 1,000+ integrations.',
363+
'Sim is open source. Workato cites 14,000+ integrations.',
364+
'Sim can call any tool that the connector marketplaces built on open protocols expose, often 1,000+ integrations.',
365+
'```text\nSim has 1,000+ integrations\n```',
366+
'````md\n```text\nSim has 1,000+ integrations\n````',
367+
].join('\n\n')
368+
)
369+
expect(await findingsFor('library', 'post')).toEqual([])
370+
})
371+
372+
it('does not check dated blog or changelog posts, whose counts were true when published', async () => {
373+
writePost('blog', 'v0-5', 'Sim v0.5 adds 100+ integrations spanning 300+ actions.')
374+
expect(await findingsFor('blog', 'v0-5')).toEqual([])
375+
})
376+
321377
it('rejects an FAQ heading in the body', async () => {
322378
writePost('library', 'post', 'Intro.\n\n## FAQs\n\nQ and A.')
323379
expect(await findingsFor('library', 'post')).toEqual([

‎scripts/check-library-content.ts‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,22 @@
2929
* - `search-title` / `search-description`: a published post's search title (`seoTitle ?? title`
3030
* with the " | Sim" suffix) fits in 60 characters and its search description
3131
* (`seoDescription ?? description`) in 160, the lengths Google shows before truncating.
32+
* - `sim-count`: a library post's count of Sim's integrations, actions, or triggers is true
33+
* against the generated catalog (`INTEGRATION_TOTALS`): no higher than the total, and no
34+
* lower than 90% of it. Answer engines quote these figures, so an overclaim such as
35+
* "1,000+ integrations" or a figure left from an older catalog spreads to third-party sites.
36+
* A count belongs to Sim when the nearest "Sim" before it on the line is at most 12 words back
37+
* with no capitalized word between, which leaves competitor counts alone.
38+
* Blog and changelog posts are dated, so their counts are true as of publication.
3239
*
3340
* Run one post with `--slug <section>/<slug>` or `--slug <slug>`.
3441
*/
3542
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
3643
import path from 'node:path'
3744
import { compile } from '@mdx-js/mdx'
45+
import { INTEGRATION_TOTALS } from '@sim/deployment-config/integration-totals'
3846
import integrationsJson from '@sim/deployment-config/integrations.json'
47+
import { formatLowerBound } from '@sim/utils/formatting'
3948
import { SIM_DOCS_URL, SIM_SITE_URL } from '@sim/utils/site'
4049
import matter from 'gray-matter'
4150
import type { Root } from 'mdast'
@@ -65,6 +74,10 @@ export type Rule =
6574
| 'internal-link'
6675
| 'search-title'
6776
| 'search-description'
77+
| 'sim-count'
78+
79+
/** The catalog units a Sim count claim can name. */
80+
type SimCountUnit = 'integrations' | 'actions' | 'triggers'
6881

6982
/** Appended to every post title by the root layout's `title.template`. */
7083
const SEARCH_TITLE_SUFFIX = ' | Sim'
@@ -94,6 +107,8 @@ export interface ContentCheckConfig {
94107
customerSlugs: readonly string[]
95108
/** Canonical integration, model, and documentation pages served by the current source tree. */
96109
linkedPages: ReadonlySet<string>
110+
/** Sim's catalog totals that `sim-count` holds library claims to. */
111+
simCatalog: Readonly<Record<SimCountUnit, number>>
97112
}
98113

99114
/** Post folders per section, keyed by slug, with each post's `draft` flag. */
@@ -116,8 +131,43 @@ const TRAILING_PUNCTUATION = /[.,;:!]+$/
116131
const LINK_TARGET_OPENER = /(?:\]\(\s*|href=\{?["'`])$/
117132
const MARKDOWN_LINK = /\[[^\]\n]*\]\([^)\n]*\)/
118133
const FAQ_HEADING = /^#{1,6}\s+FAQs?\s*:?\s*$/i
119-
const CODE_FENCE = /^\s*(```|~~~)/
134+
/** An opening or closing code fence: three or more backticks or tildes. */
135+
const CODE_FENCE = /^\s*(`{3,}|~{3,})/
120136
const ROUTE_FILES = ['page.tsx', 'page.ts', 'route.tsx', 'route.ts']
137+
/** A catalog count such as "1,000+ integrations": group 1 is the number, group 2 the unit. */
138+
const COUNT_CLAIM = /\b(\d{1,3}(?:,\d{3})+|\d+)\+?\s+(integrations|actions|triggers)\b/gi
139+
/** Inline emphasis and code markers, which would otherwise split "**1,000+** integrations". */
140+
const INLINE_MARKERS = /\*\*|__|[*`]/g
141+
const SIM_NAME = /\bSim\b/g
142+
const SIM_CLAIM_MAX_WORDS = 12
143+
/** A Sim count below this share of the catalog total is left from an older catalog. */
144+
const SIM_COUNT_FLOOR = 0.9
145+
146+
/**
147+
* The counts on one line of text that belong to Sim, with the claim as written. A sentence or
148+
* clause that moves on to another product starts with a capitalized word, which ends Sim's claim.
149+
*/
150+
function simCountClaims(line: string): { claim: string; count: number; unit: SimCountUnit }[] {
151+
const claims: { claim: string; count: number; unit: SimCountUnit }[] = []
152+
const text = line.replace(INLINE_MARKERS, '')
153+
for (const match of text.matchAll(COUNT_CLAIM)) {
154+
const before = text.slice(0, match.index)
155+
const lastSim = [...before.matchAll(SIM_NAME)].at(-1)
156+
if (lastSim?.index === undefined) continue
157+
const words = before
158+
.slice(lastSim.index + lastSim[0].length)
159+
.split(/\s+/)
160+
.map((word) => word.replace(/^[^A-Za-z0-9]+/, ''))
161+
.filter(Boolean)
162+
if (words.length > SIM_CLAIM_MAX_WORDS || words.some((word) => /^[A-Z]/.test(word))) continue
163+
claims.push({
164+
claim: match[0],
165+
count: Number.parseInt(match[1].replaceAll(',', ''), 10),
166+
unit: match[2].toLowerCase() as SimCountUnit,
167+
})
168+
}
169+
return claims
170+
}
121171

122172
function isSection(value: string): value is Section {
123173
return (SECTIONS as readonly string[]).includes(value)
@@ -397,6 +447,23 @@ export async function checkPost(
397447

398448
const data = parsed.data as Record<string, unknown>
399449

450+
const checkSimCounts = (text: string, line: number) => {
451+
if (section !== 'library') return
452+
for (const { claim, count, unit } of simCountClaims(text)) {
453+
const total = config.simCatalog[unit]
454+
if (count <= total && count >= total * SIM_COUNT_FLOOR) continue
455+
report(
456+
line,
457+
'sim-count',
458+
count > total
459+
? `"${claim}" overstates Sim's catalog, which has ${total} ${unit}.`
460+
: `"${claim}" is stale: Sim's catalog has ${total} ${unit}.`,
461+
`Write ${formatLowerBound(total)} ${unit}, the catalog-derived figure (INTEGRATION_TOTALS in packages/deployment-config/src/integration-totals.ts).`
462+
)
463+
}
464+
}
465+
frontmatterLines.forEach((text, index) => checkSimCounts(text, index + 2))
466+
400467
if (section === 'changelog' && result.success) {
401468
const { date, updated, draft, ogAlt } = result.data
402469
if (!draft && (date.getTime() > Date.now() || (updated && updated.getTime() > Date.now()))) {
@@ -535,11 +602,12 @@ export async function checkPost(
535602
const fenceMatch = CODE_FENCE.exec(text)
536603
if (fenceMatch) {
537604
if (fence === null) fence = fenceMatch[1]
538-
else if (fenceMatch[1] === fence) fence = null
605+
else if (fenceMatch[1][0] === fence[0] && fenceMatch[1].length >= fence.length) fence = null
539606
return
540607
}
541608
if (fence !== null) return
542609
const line = bodyOffset + index + 1
610+
checkSimCounts(text, line)
543611
if (FAQ_HEADING.test(text.trim())) {
544612
report(
545613
line,
@@ -771,6 +839,7 @@ async function main() {
771839
movedBlogSlugs: LIBRARY_MOVED_BLOG_SLUGS,
772840
customerSlugs: CUSTOMER_STORIES.map((story) => story.slug),
773841
linkedPages: readLinkedPages(path.join(root, 'apps/docs')),
842+
simCatalog: INTEGRATION_TOTALS,
774843
}
775844

776845
const args = process.argv.slice(2)

0 commit comments

Comments
 (0)