diff --git a/CHANGELOG.md b/CHANGELOG.md index eecc498..9177e1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,71 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.5.0] - 2026-08-07 + +### Added + +- **`command_search` and `command_run` MCP tools — the catalog bridge.** Every + other MCP tool wraps one wp-ops capability by hand, which caps what a client + can see at whatever has been ported so far. The repo has ~74 commands, so a + client seeing only the 14 wrappers correctly answers "I can't do that" for the + rest — even when the exact script exists. (Observed in practice: a wp-ops-only + session asked for GitHub repo traffic gave a well-reasoned out-of-scope answer + while `scripts/git/gh-traffic.sh` sat two directories away.) These two tools + expose the whole catalog instead of growing the wrapper list one at a time. + + `command_search` reads `go/internal/catalog/catalog.json` — the file the Go CLI + embeds — directly, so it works whether or not the binary has been built, and + gets the full manifest (args, flags, examples, platform) where `list --json` + deliberately exposes a frozen subset. Its matching mirrors `catalog.Search` so + `wp-ops search X` and `command_search(X)` can't disagree. A single match + returns full usage inline. + + `command_run` dispatches through the `wp-ops` binary rather than exec'ing + scripts, reusing the Ansible and WP-CLI executors, the server-side guard, and + `--help` formatting instead of reimplementing them. Read-only commands run + directly; anything that writes, deploys, syncs, or deletes needs + `confirm: true`. `--help` and `--where` are always free, since `executeEntry` + handles both before any executor runs. The read-only allowlist is hardcoded in + `src/tools/catalog.ts` because the manifest has no `@mutates` directive yet; a + startup check warns on stderr if a listed key leaves the catalog. + +- **`gh-traffic.sh` now reports clones and referrers**, not just views. New + `--clones`, `--referrers`, and `--all` flags; views remain the default when no + section flag is given, so existing invocations are unchanged. The script also + accepts multiple `owner/repo` arguments in one pass, and each day-series + section prints a `Total` row alongside a separate `Unique (14d)` row — GitHub's + top-level `uniques` is deduplicated across the whole window, so summing the + daily uniques column would overcount, and folding the two into one "Total" + would be wrong. + + Clone counts are dominated by CI runners, mirrors, and package resolvers rather + than people, which is worth knowing before reading them as interest; `--help` + now says so. + +### Fixed + +- **`gh-traffic.sh --days N` had no effect.** The flag was parsed and validated + (rejecting `> 14`) but never applied to the output, so every invocation showed + the full 14-day window. It now limits the day-series sections to the last N + days with activity. Referrer data has no day series and is always the full + window, which `--help` now states. +- **`gh-traffic.sh` reported success when a repo could not be read.** The traffic + endpoints are maintainer-only, so a 403 is a routine outcome; the script now + explains that specifically, keeps going so one unreadable repo doesn't hide the + others, and exits 1 if any section failed. In `--json` mode a failed section + becomes an explicit `null` — `gh api` writes the API's error body to stdout on + a 4xx, which previously would have corrupted the document. + +### Changed + +- **`gh-traffic.sh --json` now emits a JSON array of per-repo objects** + (`{repo, views?, clones?, referrers?}`) rather than the raw views payload, + since it can now carry three sections for any number of repos. Nothing in the + repo consumed the old shape. +- **`gh-traffic.sh` now requires `jq`**, previously optional and used only for + `--json`. Filtering the three payloads locally needs it. + ## [5.4.0] - 2026-08-06 ### Added diff --git a/go/internal/catalog/catalog.json b/go/internal/catalog/catalog.json index 51ad105..9d5a39f 100644 --- a/go/internal/catalog/catalog.json +++ b/go/internal/catalog/catalog.json @@ -312,12 +312,13 @@ { "category": "scripts", "key": "scripts/git/gh-traffic", - "description": "Fetch and display GitHub repository traffic statistics (14-day window)", + "description": "Fetch and display GitHub repo traffic: views, clones, and referrers (14-day window)", "script_path": "scripts/git/gh-traffic.sh", "runs_on": "local", "runs": "local", "requires": [ - "gh" + "gh", + "jq" ], "args": [ { @@ -325,36 +326,58 @@ "required_raw": "required", "required": true, "default": "imagewize/nynaeve", - "description": "GitHub repository", - "raw": "owner/repo required {imagewize/nynaeve} GitHub repository" + "description": "GitHub repository (repeatable)", + "raw": "owner/repo required {imagewize/nynaeve} GitHub repository (repeatable)" } ], "flags": [ + { + "name": "--clones", + "required_raw": "optional", + "required": false, + "description": "Include clone counts (machine traffic: CI, mirrors, bots)", + "raw": "--clones optional {} Include clone counts (machine traffic: CI, mirrors, bots)" + }, + { + "name": "--referrers", + "required_raw": "optional", + "required": false, + "description": "Include top referring sites", + "raw": "--referrers optional {} Include top referring sites" + }, + { + "name": "--all", + "required_raw": "optional", + "required": false, + "description": "Include every section (views, clones, referrers)", + "raw": "--all optional {} Include every section (views, clones, referrers)" + }, { "name": "--days", "required_raw": "optional", "required": false, "default": "14", - "description": "Number of days to fetch (max: 14)", - "raw": "--days optional {14} Number of days to fetch (max: 14)" + "description": "Limit the day-series sections to the last N days (max: 14)", + "raw": "--days optional {14} Limit the day-series sections to the last N days (max: 14)" }, { "name": "--json", "required_raw": "optional", "required": false, - "description": "Output raw JSON instead of formatted table", - "raw": "--json optional {} Output raw JSON instead of formatted table" + "description": "Output raw JSON instead of formatted tables", + "raw": "--json optional {} Output raw JSON instead of formatted tables" }, { "name": "--quiet", "required_raw": "optional", "required": false, - "description": "Suppress header row in table output", - "raw": "--quiet optional {} Suppress header row in table output" + "description": "Suppress header rows in table output", + "raw": "--quiet optional {} Suppress header rows in table output" } ], "examples": [ - "wp-ops gh-traffic imagewize/nynaeve --quiet" + "wp-ops gh-traffic imagewize/nynaeve --quiet", + "wp-ops gh-traffic imagewize/nynaeve imagewize/wp-ops --all" ], "manifest_category": "git", "platform": "any", diff --git a/mcp-server/README.md b/mcp-server/README.md index 01a07d1..b90f24d 100644 --- a/mcp-server/README.md +++ b/mcp-server/README.md @@ -74,9 +74,48 @@ Scaffold — fourteen tools implemented so far: to have `sshHost`. (`files_push` is deliberately not implemented, for the same production-risk reason as `db_push`.) -More tools (PR creation, releases, image optimization, git/gh helpers) will follow the -same pattern. See the parent repo's `CLAUDE.md` and the relevant README in each -directory for the operations these will eventually wrap. +### The catalog bridge + +Every tool above wraps one wp-ops capability by hand, which caps what a client can +see at whatever someone got round to porting. The repo has ~74 commands; a client +seeing only the wrappers correctly answers "I can't do that" for the rest — even +when the exact script exists. (This is not hypothetical: asking a wp-ops-only +session for GitHub repo traffic got a reasoned "out of scope" while +`scripts/git/gh-traffic.sh` sat two directories away.) + +These two tools expose the whole catalog instead of growing that list one wrapper +at a time: + +- **`command_search`** — searches the full command catalog by name or description, + with optional `platform` (`trellis`/`wordpress`/`any`) and `category` filters. + Mirrors `catalog.Search` in `go/internal/catalog/catalog.go` exactly, so + `wp-ops search X` and `command_search(X)` can't disagree about what exists. A + single match returns full usage — arguments, flags, examples, requirements. + Reads `go/internal/catalog/catalog.json` (the file the Go CLI embeds) directly, + so it works whether or not the binary has been built. +- **`command_run`** — runs a catalog command by key, args as separate argv tokens. + Dispatches through the `wp-ops` binary rather than exec'ing the script, which + reuses the Ansible and WP-CLI executors, the server-side guard, and `--help` + formatting instead of reimplementing them in TypeScript. Resolves the binary + from `WP_OPS_BIN`, then `go/wp-ops`, then `PATH`. + +`command_run` gates on writes: read-only commands (audits, scans, log analysis, +traffic stats) run directly, and anything that writes, deploys, syncs, or deletes +needs `confirm: true`. `--help` and `--where` are always free — `executeEntry` +handles both before any executor runs. The allowlist lives in +`src/tools/catalog.ts` because the manifest has no "does this mutate anything" +directive yet; an `@mutates` field alongside `@runs` and `@platform` would replace +it with catalog data, and until then a startup check warns on stderr when a listed +key no longer exists. + +Note that the gate is a speed bump, not a security boundary — the model can set +`confirm` itself. Its job is to make destructive commands surface to you as a +distinct decision rather than disappearing into a chain of tool calls. Client-side +tool-approval settings are what actually enforce anything. + +A command still deserves its own first-class tool when it needs typed parameters, +site-registry integration, or output shaping that argv and raw stdout can't give +it. The bridge is the floor, not a replacement for that. Two transports are implemented, both verified end-to-end (real MCP `initialize` + `tools/call` round trip against the real scanner): diff --git a/mcp-server/src/server.ts b/mcp-server/src/server.ts index 98b89f1..a5114cd 100644 --- a/mcp-server/src/server.ts +++ b/mcp-server/src/server.ts @@ -15,6 +15,16 @@ import { checkIpReputation, checkDenyList, type IpCheckResult } from "./tools/ip import { runAdminUserCreate } from "./tools/adminUserCreate.js"; import { runDbPull } from "./tools/dbPull.js"; import { runFilesPull } from "./tools/filesPull.js"; +import { + formatRunResult, + formatSearchResults, + isIntrospectionOnly, + isReadOnlyCommand, + loadCatalog, + resolveCommand, + runCatalogCommand, + searchCatalog, +} from "./tools/catalog.js"; // Building z.enum(...) schemas from the registry means a wrong site/env key gets caught // by the client/model before the call is ever made, instead of costing a full round trip @@ -708,5 +718,97 @@ export function createServer(): McpServer { } ); + // The catalog bridge. Every other tool above is a hand-written wrapper around + // one wp-ops capability, which means an MCP client sees only the handful + // someone got round to porting — and correctly answers "I can't do that" for + // the ~74 commands in the repo that have no wrapper, even when the exact + // script exists. These two expose the whole catalog instead of growing that + // list one tool at a time. + server.tool( + "command_search", + "Search the full wp-ops command catalog (~74 commands: backups, monitoring, SEO and security " + + "audits, image processing, releases, GitHub repo traffic, and more) by name or description. " + + "Use this BEFORE concluding that wp-ops cannot do something — most capabilities live here as " + + "scripts rather than as dedicated MCP tools. A single match returns full usage: arguments, " + + "flags, and examples. Run what you find with command_run.", + { + query: z + .string() + .describe('Term matched against command names and descriptions, e.g. "traffic", "backup", "webp". Use "" to list everything.'), + platform: z + .enum(["trellis", "wordpress", "any"]) + .optional() + .describe('Only commands for this stack: "trellis" needs a Trellis project, "wordpress" any WP install, "any" needs neither.'), + category: z + .string() + .optional() + .describe('Only commands in this category, e.g. "monitoring", "backup", "seo", "security", "images", "git".'), + }, + async ({ query, platform, category }) => { + try { + const entries = loadCatalog(); + const matches = searchCatalog(entries, query, { platform, category }); + return { content: [{ type: "text" as const, text: formatSearchResults(matches, query) }] }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { content: [{ type: "text" as const, text: `Error: ${message}` }], isError: true }; + } + } + ); + + server.tool( + "command_run", + "Run a wp-ops catalog command found via command_search, passing args as separate tokens. " + + 'Pass ["--help"] to read a command\'s full usage, or ["--where"] to get its file path — both are ' + + "free of side effects and never need confirmation. Read-only commands (audits, scans, log " + + "analysis) run directly; anything that writes, deploys, syncs, or deletes needs confirm: true, " + + "which you may only set after the user has explicitly approved that specific command.", + { + command: z + .string() + .describe('Command key from command_search, e.g. "scripts/git/gh-traffic". A unique basename like "gh-traffic" also resolves.'), + args: z + .array(z.string()) + .default([]) + .describe('Arguments as separate argv tokens, e.g. ["--all", "imagewize/nynaeve"]. Omit the "wp-ops" prefix and the command name.'), + confirm: z + .boolean() + .default(false) + .describe("Required (true) for any command that isn't read-only. Only set after explicit user approval of this exact command."), + timeoutSeconds: z + .number() + .int() + .positive() + .max(1800) + .default(120) + .describe("Kill the command after this many seconds. Raise it for scanners and full-site backups."), + }, + async ({ command, args, confirm, timeoutSeconds }) => { + try { + const entries = loadCatalog(); + const entry = resolveCommand(entries, command); + + if (!isIntrospectionOnly(args) && !isReadOnlyCommand(entry.key) && !confirm) { + throw new Error( + `"${entry.key}" is not on the read-only allowlist and may change data, files, or remote state. ` + + `Show the user what it does (run it with ["--help"], which needs no confirmation), get their ` + + `explicit approval, then re-run with confirm: true.` + ); + } + + const timeoutMs = timeoutSeconds * 1000; + const result = await runCatalogCommand(entry.key, args, timeoutMs); + const text = formatRunResult(entry.key, result, timeoutMs); + // A nonzero exit is the command's own verdict (a scanner finding + // something, an audit failing), not an MCP-level failure — surface the + // output rather than flagging the call itself as broken. + return { content: [{ type: "text" as const, text }] }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { content: [{ type: "text" as const, text: `Error: ${message}` }], isError: true }; + } + } + ); + return server; } diff --git a/mcp-server/src/tools/catalog.ts b/mcp-server/src/tools/catalog.ts new file mode 100644 index 0000000..f31eecc --- /dev/null +++ b/mcp-server/src/tools/catalog.ts @@ -0,0 +1,327 @@ +import { spawn } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, "../../.."); + +// The same catalog.json the Go CLI embeds at build time (go:generate ./gen). +// Reading the generated file directly rather than shelling out to `wp-ops +// list --json` buys two things: command_search works even when the Go binary +// hasn't been built, and it gets the full manifest — args, flags, examples, +// platform — where `list --json` deliberately exposes only a frozen subset +// (see printJSON's comment in go/cmd/list.go, which calls that output a stable +// contract for external tooling). Regenerated by `go generate ./internal/catalog/`. +const CATALOG_PATH = path.join(REPO_ROOT, "go/internal/catalog/catalog.json"); + +const paramSchema = z + .object({ + name: z.string(), + required: z.boolean().optional(), + choices: z.array(z.string()).optional(), + default: z.string().optional(), + description: z.string().optional(), + }) + .passthrough(); + +const entrySchema = z + .object({ + category: z.string(), + key: z.string(), + description: z.string(), + script_path: z.string(), + runs_on: z.string().optional(), + requires: z.array(z.string()).optional(), + doc: z.string().optional(), + args: z.array(paramSchema).optional(), + flags: z.array(paramSchema).optional(), + examples: z.array(z.string()).optional(), + platform: z.string().optional(), + display_category: z.string().optional(), + }) + .passthrough(); + +export type CatalogEntry = z.infer; + +let cached: CatalogEntry[] | undefined; + +export function loadCatalog(): CatalogEntry[] { + if (cached) return cached; + + if (!existsSync(CATALOG_PATH)) { + throw new Error( + `Command catalog not found at ${CATALOG_PATH}. It is generated from the scripts' ` + + `manifest headers — run \`go generate ./internal/catalog/\` from the repo's go/ directory.` + ); + } + + let raw: unknown; + try { + raw = JSON.parse(readFileSync(CATALOG_PATH, "utf-8")); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Command catalog at ${CATALOG_PATH} is not valid JSON: ${message}`); + } + + const result = z.array(entrySchema).safeParse(raw); + if (!result.success) { + throw new Error(`Command catalog at ${CATALOG_PATH} has an unexpected shape: ${result.error.message}`); + } + + cached = result.data; + warnOnAllowlistDrift(cached); + return cached; +} + +// Commands that only read — audits, scans, log analysis, traffic stats. These +// run without confirm; everything else in the catalog needs an explicit +// confirm: true. +// +// This gate is a speed bump, not a security boundary: the model can always set +// confirm itself. Its real job is to make anything that writes, deploys, or +// deletes surface to the user as a distinct decision instead of disappearing +// into a chain of tool calls. +// +// Hardcoded here because the manifest has no "does this mutate anything" +// directive yet. The data-driven version is an @mutates field alongside @runs +// and @platform, parsed in go/internal/manifest and carried through +// catalog.json — at which case this set goes away and the gate reads the entry. +// Until then warnOnAllowlistDrift catches keys that get renamed out from under it. +const READ_ONLY_COMMANDS = new Set([ + "scripts/git/gh-traffic", + "scripts/git/git-log-oneline", + "scripts/images/openverse_search", + "scripts/misc/post-count", + "scripts/monitoring/404-checker", + "scripts/monitoring/ai-bot-monitor", + "scripts/monitoring/error-monitor", + "scripts/monitoring/monitor", + "scripts/monitoring/redirect-check", + "scripts/monitoring/remote-ttfb-ua", + "scripts/monitoring/security-monitor", + "scripts/monitoring/server-monitor", + "scripts/monitoring/traffic-by-country", + "scripts/monitoring/traffic-monitor", + "scripts/monitoring/ttfb-test", + "trellis/monitoring/quick-status", + "trellis/monitoring/security-scan", + "trellis/monitoring/traffic-report", + "trellis/security/check-deny-ips", + "trellis/security/check-ips", + "wp-cli/diagnostics/diagnostic-transients", + "wp-cli/diagnostics/list-posts-count", + "wp-cli/security/scanner-general", + "wp-cli/security/scanner-targeted", + "wp-cli/security/scanner-wrapper", + "wp-cli/seo/blog-audit", + "wp-cli/seo/orphan-links-audit", + "wp-cli/seo/orphan-pages-audit", + "wp-cli/seo/page-audit", + "wp-cli/seo/redirect-audit", + "wp-cli/seo/schema-audit", +]); + +// A renamed or deleted script would silently drop off the allowlist and start +// demanding confirmation for something read-only — annoying, and easy to +// misread as "this command is dangerous". stderr is safe here: stdio transport +// reserves stdout for the JSON-RPC stream, but stderr is free. +function warnOnAllowlistDrift(entries: CatalogEntry[]): void { + const keys = new Set(entries.map((e) => e.key)); + const stale = [...READ_ONLY_COMMANDS].filter((k) => !keys.has(k)); + if (stale.length > 0) { + console.error( + `wp-ops MCP: READ_ONLY_COMMANDS lists ${stale.length} command(s) not in the catalog ` + + `(renamed or removed?): ${stale.join(", ")}` + ); + } +} + +// Introspection flags never reach the underlying script — go/cmd/dispatch.go's +// executeEntry handles --where and --help before any executor runs — so they +// are always safe regardless of what the command itself does. +const INTROSPECTION_FLAGS = new Set(["--help", "-h", "--where"]); + +export function isIntrospectionOnly(args: string[]): boolean { + return args.length > 0 && INTROSPECTION_FLAGS.has(args[0]); +} + +export function isReadOnlyCommand(key: string): boolean { + return READ_ONLY_COMMANDS.has(key); +} + +// Mirrors catalog.Search in go/internal/catalog/catalog.go: case-insensitive +// substring over key and description, sorted by key. Kept deliberately +// identical so `wp-ops search X` and command_search(X) can't disagree about +// what exists. +export function searchCatalog( + entries: CatalogEntry[], + query: string, + opts: { platform?: string; category?: string } = {} +): CatalogEntry[] { + const term = query.toLowerCase(); + return entries + .filter((e) => { + if (opts.platform && e.platform !== opts.platform) return false; + if (opts.category && e.display_category !== opts.category && e.category !== opts.category) return false; + return e.key.toLowerCase().includes(term) || e.description.toLowerCase().includes(term); + }) + .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); +} + +// Resolves an entry the way the CLI does: exact key first, then a unique +// basename. Ambiguity is reported rather than guessed at — same policy as +// go/cmd/dispatch.go's printAmbiguous. +export function resolveCommand(entries: CatalogEntry[], name: string): CatalogEntry { + const exact = entries.find((e) => e.key === name); + if (exact) return exact; + + const base = (key: string) => key.slice(key.lastIndexOf("/") + 1); + const matches = entries.filter((e) => base(e.key) === name); + + if (matches.length === 1) return matches[0]; + if (matches.length > 1) { + throw new Error( + `"${name}" matches more than one command: ${matches.map((m) => m.key).join(", ")}. ` + + `Use the full key.` + ); + } + throw new Error(`Unknown command "${name}". Use command_search to find the right key.`); +} + +function formatParams(label: string, params: CatalogEntry["args"]): string[] { + if (!params || params.length === 0) return []; + const lines = [`${label}:`]; + for (const p of params) { + const bits: string[] = [p.required ? "required" : "optional"]; + if (p.choices && p.choices.length > 0) bits.push(`one of: ${p.choices.join(" | ")}`); + else if (p.default) bits.push(`default/example: ${p.default}`); + lines.push(` ${p.name} (${bits.join(", ")})${p.description ? ` — ${p.description}` : ""}`); + } + return lines; +} + +export function formatEntryDetail(e: CatalogEntry): string { + const lines = [ + `${e.key}`, + ` ${e.description}`, + "", + `script: ${e.script_path}`, + `platform: ${e.platform ?? "any"} runs on: ${e.runs_on ?? "local"}`, + ]; + if (e.requires && e.requires.length > 0) lines.push(`requires: ${e.requires.join(", ")}`); + lines.push( + `write gate: ${isReadOnlyCommand(e.key) ? "read-only — runs without confirm" : "requires confirm: true"}` + ); + lines.push(""); + lines.push(...formatParams("arguments", e.args)); + lines.push(...formatParams("flags", e.flags)); + if (e.examples && e.examples.length > 0) { + lines.push("examples:"); + for (const ex of e.examples) lines.push(` ${ex}`); + } + if (e.doc) lines.push("", `docs: ${e.doc}`); + return lines.join("\n").replace(/\n{3,}/g, "\n\n"); +} + +export function formatSearchResults(matches: CatalogEntry[], query: string): string { + if (matches.length === 0) { + return `No commands match "${query}". Try a broader term, or search with no filters to list everything.`; + } + + // One hit is almost always the command the caller wanted, and the follow-up + // is always "how do I call it" — so answer that in the same round trip. + if (matches.length === 1) { + return `1 match for "${query}":\n\n${formatEntryDetail(matches[0])}`; + } + + const rows = matches.map((e) => { + const tags = [e.platform ?? "any"]; + if (e.runs_on === "server") tags.push("server"); + if (!isReadOnlyCommand(e.key)) tags.push("needs confirm"); + return ` ${e.key.padEnd(48)} [${tags.join(", ")}] ${e.description}`; + }); + + return ( + `${matches.length} matches for "${query}":\n\n${rows.join("\n")}\n\n` + + `Call command_run with a key and ["--help"] for full usage of any of these.` + ); +} + +// wp-ops resolution, widest-applicability first: an explicit override, then the +// binary built in this checkout, then whatever is installed on PATH. +function resolveWpOpsBin(): string { + if (process.env.WP_OPS_BIN) return process.env.WP_OPS_BIN; + const local = path.join(REPO_ROOT, "go/wp-ops"); + if (existsSync(local)) return local; + return "wp-ops"; +} + +export interface RunResult { + stdout: string; + stderr: string; + code: number; + timedOut: boolean; +} + +export function runCatalogCommand(key: string, args: string[], timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const bin = resolveWpOpsBin(); + // Dispatched by full key, never by basename: the short form can go + // ambiguous the moment a second script shares a basename, and that would + // turn a previously-working call into an error at the worst moment. + const child = spawn(bin, [key, ...args], { cwd: REPO_ROOT }); + + let stdout = ""; + let stderr = ""; + let timedOut = false; + + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + // SIGTERM is enough for a shell script but not for something wedged in + // an uninterruptible read; escalate rather than hang the MCP call. + setTimeout(() => child.kill("SIGKILL"), 5_000).unref(); + }, timeoutMs); + + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + child.on("error", (err) => { + clearTimeout(timer); + const message = (err as NodeJS.ErrnoException).code === "ENOENT" + ? `wp-ops binary not found at "${bin}". Build it with \`go build -o go/wp-ops ./go\`, ` + + `install it on PATH, or set WP_OPS_BIN.` + : err.message; + reject(new Error(message)); + }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ stdout, stderr, code: code ?? 1, timedOut }); + }); + }); +} + +const MAX_OUTPUT_CHARS = 40_000; + +export function formatRunResult(key: string, result: RunResult, timeoutMs: number): string { + const parts: string[] = []; + if (result.stdout.trim()) parts.push(result.stdout.trim()); + if (result.stderr.trim()) parts.push(`STDERR:\n${result.stderr.trim()}`); + + let body = parts.join("\n\n") || "(no output)"; + if (body.length > MAX_OUTPUT_CHARS) { + // Keep the tail: these are scripts, and the summary/verdict is at the end. + body = + `[truncated — showing the last ${MAX_OUTPUT_CHARS} of ${body.length} characters]\n\n` + + body.slice(-MAX_OUTPUT_CHARS); + } + + if (result.timedOut) { + return `${key} timed out after ${timeoutMs}ms and was killed. Partial output:\n\n${body}`; + } + if (result.code !== 0) { + return `${key} exited with code ${result.code}.\n\n${body}`; + } + return body; +} diff --git a/scripts/README.md b/scripts/README.md index 5d8d3e5..4fd2b5b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -27,7 +27,7 @@ scripts/ │ └── site-backup.sh # Trellis: complete site backup (DB + files + config) ├── git/ # Git/GitHub utilities │ ├── create-pr.sh # AI-powered GitHub PR creation -│ ├── gh-traffic.sh # Fetch and display GitHub repository traffic statistics +│ ├── gh-traffic.sh # GitHub repo traffic: views, clones, and referrers │ └── git-log-oneline.sh # Show recent git commits as one-liners ├── images/ # Image resizing and conversion │ ├── batch-resize.sh # Batch resize and center-crop images for featured images @@ -116,9 +116,12 @@ scripts/ # Create GitHub PR with AI description ./scripts/git/create-pr.sh main "Add feature name" -# Show GitHub repository traffic statistics +# Show GitHub repository traffic statistics (views only, by default) ./scripts/git/gh-traffic.sh imagewize/nynaeve +# Views, clones, and referrers for several repos at once +./scripts/git/gh-traffic.sh --all imagewize/nynaeve imagewize/wp-ops + # Show recent git commits as one-liners ./scripts/git/git-log-oneline.sh ./scripts/git/git-log-oneline.sh 25 diff --git a/scripts/git/gh-traffic.sh b/scripts/git/gh-traffic.sh index 55dea5c..3fa81dd 100755 --- a/scripts/git/gh-traffic.sh +++ b/scripts/git/gh-traffic.sh @@ -6,16 +6,20 @@ # Author: wp-ops # Created: 2026-07-09 # -# @desc Fetch and display GitHub repository traffic statistics (14-day window) +# @desc Fetch and display GitHub repo traffic: views, clones, and referrers (14-day window) # @category git # @platform any # @runs local -# @requires gh -# @arg owner/repo required {imagewize/nynaeve} GitHub repository -# @flag --days optional {14} Number of days to fetch (max: 14) -# @flag --json optional {} Output raw JSON instead of formatted table -# @flag --quiet optional {} Suppress header row in table output +# @requires gh jq +# @arg owner/repo required {imagewize/nynaeve} GitHub repository (repeatable) +# @flag --clones optional {} Include clone counts (machine traffic: CI, mirrors, bots) +# @flag --referrers optional {} Include top referring sites +# @flag --all optional {} Include every section (views, clones, referrers) +# @flag --days optional {14} Limit the day-series sections to the last N days (max: 14) +# @flag --json optional {} Output raw JSON instead of formatted tables +# @flag --quiet optional {} Suppress header rows in table output # @example wp-ops gh-traffic imagewize/nynaeve --quiet +# @example wp-ops gh-traffic imagewize/nynaeve imagewize/wp-ops --all set -euo pipefail @@ -23,46 +27,68 @@ set -euo pipefail JSON_OUTPUT=false QUIET=false DAYS=14 +SHOW_VIEWS=false +SHOW_CLONES=false +SHOW_REFERRERS=false +REPOS=() # Help function display_help() { cat <<'EOF' gh-traffic.sh - Fetch and display GitHub repository traffic statistics -Fetches view and unique visitor data for a GitHub repository and displays -it in a formatted table. Uses GitHub's traffic API which retains 14 days of data. +Fetches view, clone, and referrer data for one or more GitHub repositories and +displays each as a formatted table. Uses GitHub's traffic API, which retains 14 +days of data. Usage: - ./gh-traffic.sh [options] owner/repo + ./gh-traffic.sh [options] owner/repo [owner/repo ...] Options: -h, --help Show this help message and exit - -d, --days N Number of days to fetch (default: 14, max: 14) - -j, --json Output raw JSON instead of formatted table - -q, --quiet Suppress header row in table output + -c, --clones Include clone counts alongside views + -r, --referrers Include the top referring sites + -a, --all Include every section (views, clones, referrers) + -d, --days N Limit day-series sections to the last N days (default: 14, max: 14) + -j, --json Output raw JSON instead of formatted tables + -q, --quiet Suppress header rows in table output Arguments: - owner/repo GitHub repository in format owner/repo (required) + owner/repo GitHub repository in format owner/repo (required, repeatable) + +Sections: + With no section flag, only views are shown — the same default this script has + always had. --clones, --referrers, and --all opt into the rest. Examples: - # Show traffic for a specific repository + # Views only (default) ./scripts/git/gh-traffic.sh imagewize/nynaeve - # Show traffic without header row - ./scripts/git/gh-traffic.sh imagewize/nynaeve --quiet + # Everything, for several repos in one pass + ./scripts/git/gh-traffic.sh --all imagewize/nynaeve imagewize/wp-ops + + # Views and clones for the last 7 days + ./scripts/git/gh-traffic.sh --clones --days 7 imagewize/nynaeve - # Output as JSON - ./scripts/git/gh-traffic.sh imagewize/nynaeve --json + # Machine-readable output + ./scripts/git/gh-traffic.sh --all --json imagewize/nynaeve Requirements: - GitHub CLI (gh) installed and authenticated - - jq for JSON processing (if using --json, optional otherwise) + - jq for JSON processing + - Push access to each repository (GitHub restricts traffic data to maintainers) - column for table formatting (if not using --json) +Reading the numbers: + Views count page loads by humans; clones count `git clone`, which is dominated + by CI runners, mirrors, and package resolvers rather than people. A repo with 3 + unique viewers and 200 unique cloners is being fetched by automation, not read. + Note: - GitHub's traffic API only retains 14 days of data, so this will always - show a maximum two-week window. For longer-term tracking, consider running - this script periodically and appending results to a CSV file via cron. + GitHub's traffic API only retains 14 days of data, so this will always show a + maximum two-week window. Referrer data has no day series — it is always the + full 14-day window, so --days does not apply to it. For longer-term tracking, + run this periodically and append results to a CSV via cron. Author: wp-ops Created: 2026-07-09 @@ -76,6 +102,20 @@ while [[ $# -gt 0 ]]; do -h|--help) display_help ;; + -c|--clones) + SHOW_CLONES=true + shift + ;; + -r|--referrers) + SHOW_REFERRERS=true + shift + ;; + -a|--all) + SHOW_VIEWS=true + SHOW_CLONES=true + SHOW_REFERRERS=true + shift + ;; -d|--days) if [[ -n "${2:-}" && "${2:-}" =~ ^[0-9]+$ ]]; then DAYS="$2" @@ -83,6 +123,10 @@ while [[ $# -gt 0 ]]; do echo "Error: Maximum days is 14 (GitHub API limit)" >&2 exit 1 fi + if [[ "$DAYS" -lt 1 ]]; then + echo "Error: --days must be at least 1" >&2 + exit 1 + fi shift 2 else echo "Error: --days requires a numeric argument" >&2 @@ -103,47 +147,196 @@ while [[ $# -gt 0 ]]; do exit 1 ;; *) - REPO="$1" + REPOS+=("$1") shift ;; esac done -# Validate repository argument -if [[ -z "${REPO:-}" ]]; then - echo "Error: Repository argument is required" >&2 - echo "Usage: $0 [options] owner/repo" >&2 - exit 1 +# Views are the default section, but only when nothing else was asked for — +# `--clones` alone means clones alone, the same way `--all` means all three. +if [[ "$SHOW_CLONES" = false && "$SHOW_REFERRERS" = false ]]; then + SHOW_VIEWS=true fi -# Validate repository format (owner/repo) -if [[ ! "$REPO" =~ ^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$ ]]; then - echo "Error: Invalid repository format. Use owner/repo (e.g., imagewize/nynaeve)" >&2 +# Validate repository arguments +if [[ ${#REPOS[@]} -eq 0 ]]; then + echo "Error: At least one repository argument is required" >&2 + echo "Usage: $0 [options] owner/repo [owner/repo ...]" >&2 exit 1 fi +for repo in "${REPOS[@]}"; do + if [[ ! "$repo" =~ ^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$ ]]; then + echo "Error: Invalid repository format '$repo'. Use owner/repo (e.g., imagewize/nynaeve)" >&2 + exit 1 + fi +done + # Check for required commands if ! command -v gh &>/dev/null; then echo "Error: GitHub CLI (gh) is required but not installed" >&2 exit 1 fi +if ! command -v jq &>/dev/null; then + echo "Error: jq is required but not installed" >&2 + exit 1 +fi + if [[ "$JSON_OUTPUT" = false ]] && ! command -v column &>/dev/null; then echo "Error: column command is required for table output but not installed" >&2 exit 1 fi -# Fetch traffic data from GitHub API -if [[ "$JSON_OUTPUT" = true ]]; then - # Raw JSON output - gh api "repos/$REPO/traffic/views" -else - # Formatted table output - if [[ "$QUIET" = false ]]; then - echo -e "Date\tViews\tUnique" +# The traffic endpoints are maintainer-only, so a 403 here means "you don't have +# push access to this repo" far more often than it means anything is broken. +# Reporting that plainly beats letting gh's raw HTTP error surface. +# Tracks whether any repo/section failed, so a partial run still exits nonzero +# — the tables for the repos that did work are worth printing, but a caller +# scripting this needs to know the picture is incomplete. +FAILED=false + +# Captures rather than streams, because `gh api` writes the API's JSON error +# body to stdout on a 4xx as well — passing that straight through would emit a +# second JSON value into --json output and corrupt the document. +fetch_traffic() { + local repo="$1" endpoint="$2" body="" status=0 + body=$(gh api "repos/${repo}/traffic/${endpoint}" 2>/dev/null) || status=$? + if [[ $status -ne 0 ]]; then + echo "Error: could not fetch ${endpoint} for ${repo} — the traffic API needs push access to the repo, and the repo must exist." >&2 + FAILED=true + return 1 + fi + printf '%s' "$body" +} + +# JSON counterpart to fetch_traffic: a section that can't be fetched becomes an +# explicit null rather than an empty slot, which would make the document +# unparseable for every other repo in the same run. +fetch_traffic_json() { + if ! fetch_traffic "$1" "$2"; then + printf 'null' + fi +} + +# Renders one day-series section (views or clones). GitHub's series carries one +# object per day with a zero-filled tail, so `tail -n` over the non-zero rows is +# what --days actually means here. +# +# The totals need care: the API's top-level `uniques` is deduplicated across the +# whole 14-day window, so it is NOT the sum of the daily uniques (one person +# visiting on three days counts once there, three times in the column). Summing +# it would silently overcount, so the two are printed as separate labelled rows +# rather than folded into one "Total". +print_series() { + local endpoint="$1" label="$2" payload="$3" + + local rows + rows=$(printf '%s' "$payload" | jq -r \ + ".${endpoint}[] | select(.count > 0) | [.timestamp[:10], (.count|tostring), (.uniques|tostring)] | @tsv") + # GitHub returns the series oldest-first with a zero-filled tail, so the + # last N non-zero rows are what "--days N" means. + if [[ -n "$rows" ]]; then + rows=$(printf '%s\n' "$rows" | tail -n "$DAYS") fi - - gh api "repos/$REPO/traffic/views" --jq \ - '.views[] | select(.count > 0) | [.timestamp[:10], (.count|tostring), (.uniques|tostring)] | @tsv' \ - | column -t + + local total_count window_uniques + total_count=$(printf '%s\n' "$rows" | awk -F'\t' '{s += $2} END {print s + 0}') + window_uniques=$(printf '%s' "$payload" | jq -r '.uniques') + + echo "${label} (last ${DAYS} day(s) with activity)" + { + if [[ "$QUIET" = false ]]; then + printf 'Date\t%s\tUnique\n' "$label" + fi + if [[ -n "$rows" ]]; then + printf '%s\n' "$rows" + fi + printf 'Total\t%s\t-\n' "$total_count" + printf 'Unique (14d)\t-\t%s\n' "$window_uniques" + } | column -t -s $'\t' + echo +} + +print_referrers() { + local payload="$1" + local rows + rows=$(printf '%s' "$payload" | jq -r '.[] | [.referrer, (.count|tostring), (.uniques|tostring)] | @tsv') + + echo "Referrers (top 10, 14-day window)" + { + if [[ "$QUIET" = false ]]; then + printf 'Source\tViews\tUnique\n' + fi + if [[ -n "$rows" ]]; then + printf '%s\n' "$rows" + else + printf '(none)\t-\t-\n' + fi + } | column -t -s $'\t' + echo +} + +if [[ "$JSON_OUTPUT" = true ]]; then + # One object per repo, carrying only the requested sections. Emitted as a + # JSON array so multi-repo output stays parseable as a single document. + echo "[" + first=true + for repo in "${REPOS[@]}"; do + [[ "$first" = true ]] || echo "," + first=false + echo " {" + printf ' "repo": "%s"' "$repo" + if [[ "$SHOW_VIEWS" = true ]]; then + printf ',\n "views": ' + fetch_traffic_json "$repo" "views" + fi + if [[ "$SHOW_CLONES" = true ]]; then + printf ',\n "clones": ' + fetch_traffic_json "$repo" "clones" + fi + if [[ "$SHOW_REFERRERS" = true ]]; then + printf ',\n "referrers": ' + fetch_traffic_json "$repo" "popular/referrers" + fi + printf '\n }' + done + echo + echo "]" + [[ "$FAILED" = true ]] && exit 1 + exit 0 fi + +for repo in "${REPOS[@]}"; do + echo "=== ${repo} ===" + echo + + # Two things going on here: + # + # - `if payload=$(...)` rather than `payload=$(...) && ...`, because under + # `set -e` the latter aborts the whole run when one repo is unreadable, + # and a 403 on one repo shouldn't stop the other five from reporting. + # - the explicit `else FAILED=true`, because command substitution runs + # fetch_traffic in a subshell, so the FAILED it sets there is discarded. + # (The --json path calls fetch_traffic_json directly, no subshell, so + # the assignment inside fetch_traffic does survive for that one.) + if [[ "$SHOW_VIEWS" = true ]]; then + if payload=$(fetch_traffic "$repo" "views"); then + print_series "views" "Views" "$payload" + else FAILED=true; fi + fi + if [[ "$SHOW_CLONES" = true ]]; then + if payload=$(fetch_traffic "$repo" "clones"); then + print_series "clones" "Clones" "$payload" + else FAILED=true; fi + fi + if [[ "$SHOW_REFERRERS" = true ]]; then + if payload=$(fetch_traffic "$repo" "popular/referrers"); then + print_referrers "$payload" + else FAILED=true; fi + fi +done + +[[ "$FAILED" = true ]] && exit 1 +exit 0