diff --git a/justfile b/justfile index 77cc872cf..4f6c3343f 100644 --- a/justfile +++ b/justfile @@ -27,6 +27,26 @@ lint: cargo clippy --locked --all-targets --all-features -- -D warnings cargo clippy --locked --all-targets --no-default-features -- -D warnings +# Score how findable the MCP tools generated from a spec are; with no spec, +# the latest openapi.forge.json release. Takes main.ts options, such as +# --min-top5 0.8 or --json. +mcp-eval spec="" *args: + #!/usr/bin/env bash + set -euo pipefail + spec="{{ spec }}" + mkdir -p target/mcp-eval + if [ -z "$spec" ]; then + spec=target/mcp-eval/openapi.forge.json + gh release download --repo cloudflare/forge --pattern openapi.forge.json --output "$spec" --clobber + fi + # The release has Fern's extensions; forge reads its own x-forge-* ones. + cargo run --release --quiet -- normalize "$spec" > target/mcp-eval/normalized.json + # Inside the package, so the tools' runtime finds the MCP SDK it imports. + rm -rf packages/mcp-search-eval/generated + cargo run --release --quiet -- target/mcp-eval/normalized.json --target mcp --out packages/mcp-search-eval/generated + cd packages/mcp-search-eval + node --import ./src/register.ts src/main.ts generated {{ args }} + # Release builds # # Every way of getting release binaries puts them in target/binaries// diff --git a/packages/mcp-search-eval/.gitignore b/packages/mcp-search-eval/.gitignore new file mode 100644 index 000000000..9ab870da8 --- /dev/null +++ b/packages/mcp-search-eval/.gitignore @@ -0,0 +1 @@ +generated/ diff --git a/packages/mcp-search-eval/README.md b/packages/mcp-search-eval/README.md new file mode 100644 index 000000000..fb717ce40 --- /dev/null +++ b/packages/mcp-search-eval/README.md @@ -0,0 +1,13 @@ +# mcp-search-eval + +How findable are the tools `forge --target mcp` generates? Each task in `tasks/` is something someone wants to do, in their words, and the methods that do it. The eval searches the generated tools for each task with BM25 over what a client sees (name, title and description) and scores where the expected tool ranked: top 1, top 5, and mean reciprocal rank. + +```bash +just mcp-eval # the latest openapi.forge.json release +just mcp-eval path/to/spec.json # a local spec +just mcp-eval "" --min-top5 0.8 # exit 1 below 80% in the top 5, for CI +``` + +Clients search tools in different ways, so this is a proxy: a drop in the score is worth a look, a single rank less so. It catches names that don't carry the words people use, such as a Worker upload named `workers_legacy_scripts_update`. Blind tests with agents (cloudflare/mcp's `bench/`) catch what a search score can't. + +The search and the scoring are generic; only the tasks are Cloudflare's. Expected methods are written `METHOD /path` with the path as forge writes it, so a merged `/{account_or_zone}/{account_or_zone_id}/…` path stays merged. diff --git a/packages/mcp-search-eval/package.json b/packages/mcp-search-eval/package.json new file mode 100644 index 000000000..af75e6379 --- /dev/null +++ b/packages/mcp-search-eval/package.json @@ -0,0 +1,20 @@ +{ + "name": "mcp-search-eval", + "version": "0.1.0", + "private": true, + "description": "Measures how well the tools forge generates for MCP can be found by searching their names and descriptions.", + "type": "module", + "scripts": { + "typecheck": "tsc -p tsconfig.json", + "test": "node --import ./src/register.ts --test \"test/**/*.test.ts\"", + "eval": "node --import ./src/register.ts src/main.ts" + }, + "devDependencies": { + "@modelcontextprotocol/server": "2.3.1", + "@types/node": "26.0.1", + "typescript": "7.0.2" + }, + "engines": { + "node": ">=26" + } +} diff --git a/packages/mcp-search-eval/src/main.ts b/packages/mcp-search-eval/src/main.ts new file mode 100644 index 000000000..f86369751 --- /dev/null +++ b/packages/mcp-search-eval/src/main.ts @@ -0,0 +1,51 @@ +/** + * Scores how findable the tools forge generated are: searches them for each + * task and prints where its tool ranked, as Markdown. + * + * node --import ./src/register.ts src/main.ts [tasks.json] [--json] [--min-top5 ] + * + * The directory is what `forge --target mcp --out ` writes. With + * `--min-top5`, it exits 1 when fewer than that share of tasks rank their + * tool in the top 5, for CI. + */ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { markdown, score, type EvalTool, type Task } from './score.ts'; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { json: { type: 'boolean' }, 'min-top5': { type: 'string' } }, +}); +const [dir, tasksFile = new URL('../tasks/cloudflare.json', import.meta.url).pathname] = positionals; +if (!dir) { + console.error('usage: main.ts [tasks.json] [--json] [--min-top5 ]'); + process.exit(2); +} + +type Generated = { name: string; title?: string; description?: string }; +const generated = (await import(pathToFileURL(resolve(dir, 'index.ts')).href)) as { + tools: readonly Generated[]; + operation: (tool: Generated) => { method: string; path: string }; +}; +// Search what a client sees, and match on the method each tool calls. +const tools: EvalTool[] = generated.tools.map((tool) => { + const { method, path } = generated.operation(tool); + return { + name: tool.name, + ...(tool.title !== undefined && { title: tool.title }), + ...(tool.description !== undefined && { description: tool.description }), + method, + path, + }; +}); +const tasks = JSON.parse(readFileSync(tasksFile, 'utf8')) as Task[]; +const result = score(tools, tasks); + +console.log(values.json ? JSON.stringify(result, null, 2) : markdown(result)); +const min = values['min-top5'] === undefined ? undefined : Number(values['min-top5']); +if (min !== undefined && result.top5 < min * result.tasks) { + console.error(`top 5 hit ${result.top5} of ${result.tasks}, below ${min}`); + process.exit(1); +} diff --git a/packages/mcp-search-eval/src/register.ts b/packages/mcp-search-eval/src/register.ts new file mode 100644 index 000000000..e9543429b --- /dev/null +++ b/packages/mcp-search-eval/src/register.ts @@ -0,0 +1,21 @@ +import { registerHooks } from 'node:module'; + +// The sources import each other as `./name.js`, the way TypeScript emits +// them. Node runs the `.ts` files as they are, so a relative `.js` import +// that does not exist resolves to the `.ts` file next to it. +registerHooks({ + resolve(specifier, context, nextResolve) { + try { + return nextResolve(specifier, context); + } catch (error) { + if ( + (specifier.startsWith('./') || specifier.startsWith('../')) && + specifier.endsWith('.js') && + (error as NodeJS.ErrnoException).code === 'ERR_MODULE_NOT_FOUND' + ) { + return nextResolve(`${specifier.slice(0, -3)}.ts`, context); + } + throw error; + } + }, +}); diff --git a/packages/mcp-search-eval/src/score.ts b/packages/mcp-search-eval/src/score.ts new file mode 100644 index 000000000..f668c3ba3 --- /dev/null +++ b/packages/mcp-search-eval/src/score.ts @@ -0,0 +1,85 @@ +import { index, type Listed } from './search.ts'; + +/** A generated tool: what a client sees, and the method it calls. */ +export interface EvalTool extends Listed { + readonly method: string; + readonly path: string; +} + +/** + * Something someone wants to do, in their words, and the methods that do it, + * as `METHOD /path` with the path as forge writes it (a merged + * `/{account_or_zone}/{account_or_zone_id}/…` path stays merged). Any one of + * them is a hit. + */ +export interface Task { + readonly task: string; + readonly expect: readonly string[]; +} + +/** Where a task's first expected tool ranked. */ +export interface TaskResult { + readonly task: string; + readonly expect: readonly string[]; + /** 1-based, or `undefined` when no expected tool ranked in the top {@link LIMIT}. */ + readonly rank: number | undefined; + /** The tool that ranked first. */ + readonly top: string | undefined; + /** The expected tools, by name; empty when none was generated. */ + readonly expected: readonly string[]; +} + +export interface Score { + readonly tasks: number; + readonly top1: number; + readonly top5: number; + /** Mean reciprocal rank, 0 to 1. */ + readonly mrr: number; + readonly results: readonly TaskResult[]; +} + +/** How far down the ranking a task's tool is looked for. */ +export const LIMIT = 20; + +/** Searches `tools` for each task and scores where its tool ranked. */ +export function score(tools: readonly EvalTool[], tasks: readonly Task[]): Score { + const search = index(tools); + const results = tasks.map(({ task, expect }): TaskResult => { + const wanted = new Set(expect); + const ranked = search(task).slice(0, LIMIT); + const position = ranked.findIndex((tool) => wanted.has(`${tool.method} ${tool.path}`)); + return { + task, + expect, + rank: position < 0 ? undefined : position + 1, + top: ranked[0]?.name, + expected: tools.filter((tool) => wanted.has(`${tool.method} ${tool.path}`)).map((tool) => tool.name), + }; + }); + const hits = (k: number) => results.filter(({ rank }) => rank !== undefined && rank <= k).length; + return { + tasks: results.length, + top1: hits(1), + top5: hits(5), + mrr: results.reduce((sum, { rank }) => sum + (rank ? 1 / rank : 0), 0) / Math.max(results.length, 1), + results, + }; +} + +/** The score as Markdown: the totals, then every task that missed the top 1. */ +export function markdown({ tasks, top1, top5, mrr, results }: Score): string { + const lines = [ + `| Tasks | Top 1 | Top 5 | MRR |`, + `| ----- | ----- | ----- | --- |`, + `| ${tasks} | ${top1} | ${top5} | ${mrr.toFixed(2)} |`, + ]; + const misses = results.filter(({ rank }) => rank !== 1); + if (misses.length > 0) { + lines.push('', '| Task | Rank | Expected | Ranked first |', '| ---- | ---- | -------- | ------------ |'); + for (const { task, rank, expected, top } of misses) { + const want = expected.length > 0 ? expected.map((name) => `\`${name}\``).join(', ') : 'no tool'; + lines.push(`| ${task} | ${rank ?? `>${LIMIT}`} | ${want} | ${top ? `\`${top}\`` : '–'} |`); + } + } + return lines.join('\n'); +} diff --git a/packages/mcp-search-eval/src/search.ts b/packages/mcp-search-eval/src/search.ts new file mode 100644 index 000000000..af9f2dd2b --- /dev/null +++ b/packages/mcp-search-eval/src/search.ts @@ -0,0 +1,78 @@ +/** + * A BM25 index over what a client sees of each tool: its name, title and + * description. It stands in for the search an MCP client runs over a large + * tool list (a tool search tool, or code that greps the list), so a score + * shows whether a tool's name and description carry the words people use for + * what it does. Clients search differently, so it is a proxy: a fall in the + * score is worth a look, a single rank is not. + */ + +/** The parts of a listed tool that a search reads. */ +export interface Listed { + readonly name: string; + readonly title?: string; + readonly description?: string; +} + +/** + * Splits `text` into lower-case words: at anything that isn't a letter or a + * digit, and between the words of a camelCase or snake_case name, so + * `zeroTrust_access_applications_list` is `zero trust access application + * list`. A trailing `s` is dropped, so `bucket` finds `buckets`. + */ +export function words(text: string): string[] { + return text + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((word) => word.length > 1) + .map((word) => (word.length > 3 && word.endsWith('s') && !word.endsWith('ss') ? word.slice(0, -1) : word)); +} + +const K1 = 1.2; +const B = 0.75; + +/** The words a tool is found by. The name counts twice: it is what a client shows and calls. */ +function document(tool: Listed): string[] { + return words(`${tool.name} ${tool.name} ${tool.title ?? ''} ${tool.description ?? ''}`); +} + +/** Ranks `tools` for a query, best first. */ +export function index(tools: readonly T[]): (query: string) => T[] { + const docs = tools.map((tool) => { + const counts = new Map(); + const list = document(tool); + for (const word of list) { + counts.set(word, (counts.get(word) ?? 0) + 1); + } + return { counts, length: list.length }; + }); + const average = docs.reduce((sum, doc) => sum + doc.length, 0) / Math.max(docs.length, 1); + const frequency = new Map(); + for (const doc of docs) { + for (const word of doc.counts.keys()) { + frequency.set(word, (frequency.get(word) ?? 0) + 1); + } + } + const idf = (word: string) => { + const n = frequency.get(word) ?? 0; + return Math.log(1 + (docs.length - n + 0.5) / (n + 0.5)); + }; + return (query) => { + const terms = [...new Set(words(query))]; + return docs + .map((doc, i) => { + let score = 0; + for (const term of terms) { + const f = doc.counts.get(term) ?? 0; + if (f > 0) { + score += (idf(term) * f * (K1 + 1)) / (f + K1 * (1 - B + (B * doc.length) / average)); + } + } + return { score, i }; + }) + .filter(({ score }) => score > 0) + .sort((a, b) => b.score - a.score || a.i - b.i) + .map(({ i }) => tools[i]!); + }; +} diff --git a/packages/mcp-search-eval/tasks/cloudflare.json b/packages/mcp-search-eval/tasks/cloudflare.json new file mode 100644 index 000000000..b7213b69a --- /dev/null +++ b/packages/mcp-search-eval/tasks/cloudflare.json @@ -0,0 +1,92 @@ +[ + { "task": "list my Workers scripts", "expect": ["GET /accounts/{account_id}/workers/scripts"] }, + { "task": "deploy a Worker script", "expect": ["PUT /accounts/{account_id}/workers/scripts/{script_name}"] }, + { "task": "delete a Worker", "expect": ["DELETE /accounts/{account_id}/workers/scripts/{script_name}"] }, + { + "task": "set a secret on a Worker", + "expect": ["PUT /accounts/{account_id}/workers/scripts/{script_name}/secrets"] + }, + { "task": "list the routes of Workers on a zone", "expect": ["GET /zones/{zone_id}/workers/routes"] }, + { "task": "list D1 databases", "expect": ["GET /accounts/{account_id}/d1/database"] }, + { + "task": "run a SQL query against a D1 database", + "expect": [ + "POST /accounts/{account_id}/d1/database/{database_id}/query", + "POST /accounts/{account_id}/d1/database/{database_id}/raw" + ] + }, + { "task": "create a KV namespace", "expect": ["POST /accounts/{account_id}/storage/kv/namespaces"] }, + { + "task": "read a value from KV by key", + "expect": ["GET /accounts/{account_id}/storage/kv/namespaces/{namespace_id}/values/{key_name}"] + }, + { + "task": "write a value to KV", + "expect": [ + "PUT /accounts/{account_id}/storage/kv/namespaces/{namespace_id}/values/{key_name}", + "PUT /accounts/{account_id}/storage/kv/namespaces/{namespace_id}/bulk" + ] + }, + { + "task": "list the keys in a KV namespace", + "expect": ["GET /accounts/{account_id}/storage/kv/namespaces/{namespace_id}/keys"] + }, + { "task": "list R2 buckets", "expect": ["GET /accounts/{account_id}/r2/buckets"] }, + { "task": "create an R2 bucket", "expect": ["POST /accounts/{account_id}/r2/buckets"] }, + { + "task": "upload a file to an R2 bucket", + "expect": ["PUT /accounts/{account_id}/r2/buckets/{bucket_name}/objects/{object_key}"] + }, + { + "task": "list the objects in an R2 bucket", + "expect": ["GET /accounts/{account_id}/r2/buckets/{bucket_name}/objects"] + }, + { "task": "list my zones", "expect": ["GET /zones"] }, + { "task": "add a domain to Cloudflare", "expect": ["POST /zones"] }, + { "task": "list DNS records for a domain", "expect": ["GET /zones/{zone_id}/dns_records"] }, + { "task": "create a DNS record", "expect": ["POST /zones/{zone_id}/dns_records"] }, + { + "task": "update a DNS record", + "expect": ["PATCH /zones/{zone_id}/dns_records/{dns_record_id}", "PUT /zones/{zone_id}/dns_records/{dns_record_id}"] + }, + { "task": "delete a DNS record", "expect": ["DELETE /zones/{zone_id}/dns_records/{dns_record_id}"] }, + { "task": "import DNS records from a BIND zone file", "expect": ["POST /zones/{zone_id}/dns_records/import"] }, + { "task": "purge the cache for a site", "expect": ["POST /zones/{zone_id}/purge_cache"] }, + { + "task": "change a zone setting such as always use HTTPS", + "expect": ["PATCH /zones/{zone_id}/settings/{setting_id}"] + }, + { "task": "list the rulesets of a zone", "expect": ["GET /{account_or_zone}/{account_or_zone_id}/rulesets"] }, + { "task": "list Pages projects", "expect": ["GET /accounts/{account_id}/pages/projects"] }, + { + "task": "list deployments of a Pages project", + "expect": ["GET /accounts/{account_id}/pages/projects/{project_name}/deployments"] + }, + { "task": "list Queues", "expect": ["GET /accounts/{account_id}/queues"] }, + { "task": "send a message to a queue", "expect": ["POST /accounts/{account_id}/queues/{queue_id}/messages"] }, + { "task": "list Access applications", "expect": ["GET /{account_or_zone}/{account_or_zone_id}/access/apps"] }, + { "task": "list members of my account", "expect": ["GET /accounts/{account_id}/members"] }, + { "task": "list my accounts", "expect": ["GET /accounts"] }, + { "task": "verify my API token", "expect": ["GET /user/tokens/verify", "GET /accounts/{account_id}/tokens/verify"] }, + { "task": "create an API token", "expect": ["POST /user/tokens", "POST /accounts/{account_id}/tokens"] }, + { "task": "list Hyperdrive configs", "expect": ["GET /accounts/{account_id}/hyperdrive/configs"] }, + { "task": "create a Vectorize index", "expect": ["POST /accounts/{account_id}/vectorize/v2/indexes"] }, + { + "task": "insert vectors into a Vectorize index", + "expect": ["POST /accounts/{account_id}/vectorize/v2/indexes/{index_name}/insert"] + }, + { "task": "run an AI model with Workers AI", "expect": ["POST /accounts/{account_id}/ai/run/{model_name}"] }, + { "task": "query Workers Analytics Engine with SQL", "expect": ["POST /accounts/{account_id}/analytics_engine/sql"] }, + { "task": "upload an image to Cloudflare Images", "expect": ["POST /accounts/{account_id}/images/v1"] }, + { "task": "list Stream videos", "expect": ["GET /accounts/{account_id}/stream"] }, + { + "task": "list Durable Object namespaces", + "expect": ["GET /accounts/{account_id}/workers/durable_objects/namespaces"] + }, + { "task": "list Turnstile widgets", "expect": ["GET /accounts/{account_id}/challenges/widgets"] }, + { + "task": "list email routing rules for a zone", + "expect": ["GET /{account_or_zone}/{account_or_zone_id}/email/routing/rules"] + }, + { "task": "list Cloudflare Tunnels", "expect": ["GET /accounts/{account_id}/cfd_tunnel"] } +] diff --git a/packages/mcp-search-eval/test/score.test.ts b/packages/mcp-search-eval/test/score.test.ts new file mode 100644 index 000000000..4e6c5d9af --- /dev/null +++ b/packages/mcp-search-eval/test/score.test.ts @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { markdown, score } from '../src/score.ts'; +import { index, words } from '../src/search.ts'; + +const tools = [ + { name: 'zones_list', description: 'Lists zones.', method: 'GET', path: '/zones' }, + { + name: 'dns_records_list', + description: 'Lists the DNS records of a zone.', + method: 'GET', + path: '/zones/{zone_id}/dns_records', + }, + { + name: 'dns_records_create', + description: 'Creates a DNS record.', + method: 'POST', + path: '/zones/{zone_id}/dns_records', + }, + { + name: 'zeroTrust_access_applications_list', + description: 'Lists Access applications.', + method: 'GET', + path: '/{account_or_zone}/{account_or_zone_id}/access/apps', + }, +]; + +test('splits names into words, at camelCase and underscores, without plural s', () => { + assert.deepEqual(words('zeroTrust_access_applications_list'), ['zero', 'trust', 'access', 'application', 'list']); + assert.deepEqual(words('Lists DNS records.'), ['list', 'dns', 'record']); +}); + +test('ranks the tool whose name and description share the most rare words first', () => { + assert.equal(index(tools)('create a DNS record')[0]?.name, 'dns_records_create'); + assert.equal(index(tools)('list access applications')[0]?.name, 'zeroTrust_access_applications_list'); +}); + +test('scores where each task ranked its expected method, by method and path as forge writes them', () => { + const result = score(tools, [ + { task: 'list DNS records', expect: ['GET /zones/{zone_id}/dns_records'] }, + { task: 'list zones', expect: ['GET /zones'] }, + { task: 'purge the cache', expect: ['POST /zones/{zone_id}/purge_cache'] }, + ]); + assert.equal(result.tasks, 3); + assert.equal(result.top1, 2); + assert.deepEqual( + result.results.map(({ rank }) => rank), + [1, 1, undefined], + ); + assert.deepEqual(result.results[2]?.expected, []); + assert.ok(Math.abs(result.mrr - 2 / 3) < 1e-9); + assert.match(markdown(result), /\| purge the cache \| >20 \| no tool \|/); +}); diff --git a/packages/mcp-search-eval/tsconfig.json b/packages/mcp-search-eval/tsconfig.json new file mode 100644 index 000000000..1f52d9db2 --- /dev/null +++ b/packages/mcp-search-eval/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "types": ["node"], + "module": "nodenext", + "moduleResolution": "nodenext", + "noEmit": true, + "strict": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "skipLibCheck": true + }, + "include": ["src", "test"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5c612815..b96830b8d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -301,6 +301,18 @@ importers: specifier: 7.0.2 version: 7.0.2 + packages/mcp-search-eval: + devDependencies: + '@modelcontextprotocol/server': + specifier: 2.3.1 + version: 2.3.1 + '@types/node': + specifier: 26.0.1 + version: 26.0.1 + typescript: + specifier: 7.0.2 + version: 7.0.2 + src/bin/forge/mcp/lib: devDependencies: '@modelcontextprotocol/server':