diff --git a/.github/workflows/spec-drift.yml b/.github/workflows/spec-drift.yml new file mode 100644 index 0000000..e1a50b5 --- /dev/null +++ b/.github/workflows/spec-drift.yml @@ -0,0 +1,116 @@ +name: API Spec Drift + +# The `api` skill's references are generated from Deepgram's public OpenAPI and AsyncAPI +# specs, which change without anything in this repo noticing — the checked-in output once +# sat ~5 weeks behind. This job regenerates weekly and reports drift. +# +# REPORT ONLY. It never commits regenerated output: regeneration is a reviewed change +# (`bun run scripts/fetch-specs.ts … && bun run scripts/generate-skills.ts`), and an +# auto-commit would land unreviewed API copy in a skill agents read as reference. +# +# Known limitation: `generate-skills.ts` overwrites the reference files it emits but does +# not prune ones it no longer emits, so a reference file for a *removed* endpoint group +# shows up as no drift here. Additions and edits are detected. This job is deliberately +# tolerant of that — it reports what it can see rather than asserting the tree is clean. + +on: + schedule: + # Mondays, 13:00 UTC. + - cron: "0 13 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check-drift: + name: Check generated api skill against live specs + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - run: bun install --frozen-lockfile + + # Use the .yml URLs. `dpgr.am/openapi.yaml` (.yaml) serves the marketing homepage + # with HTTP 200, and fetch-specs.ts only checks `response.ok`, so a .yaml typo + # writes HTML into specs/ and fails later at parse() with a confusing error. + - name: Fetch live specs + run: bun run scripts/fetch-specs.ts https://dpgr.am/openapi.yml https://dpgr.am/asyncapi.yml + + # Cheap guard against the trap above, so a bad fetch reports as a fetch problem. + - name: Verify specs are YAML, not HTML + run: | + for f in specs/openapi.yml specs/asyncapi.yml; do + if head -c 200 "$f" | grep -qi 'Diff stat" + echo + echo '```' + git diff --cached --stat -- skills/api/references + echo '```' + echo + echo "" + echo + echo "[Workflow run]($RUN_URL)" + } > body.md + + cat body.md >> "$GITHUB_STEP_SUMMARY" + + # One rolling issue rather than a new one every week. + existing=$(gh issue list --state open --label spec-drift --limit 1 --json number --jq '.[0].number') + if [ -n "$existing" ]; then + gh issue comment "$existing" --body-file body.md + echo "Commented on existing issue #$existing" + else + gh label create spec-drift --color FBCA04 --description "Generated api skill is behind the live specs" 2>/dev/null || true + gh issue create --title "API spec drift: skills/api/references is behind the live specs" \ + --label spec-drift --body-file body.md + fi + + # Fail visibly so the scheduled run is red in the Actions tab. + exit 1 diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml new file mode 100644 index 0000000..6abbf57 --- /dev/null +++ b/.github/workflows/validate-skills.yml @@ -0,0 +1,110 @@ +name: Validate Skills + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: validate-skills-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Blocking. Hermetic — no network, so it cannot flake. + validate: + name: Validate skills and manifest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - run: bun install --frozen-lockfile + + # Parses every skills/*/SKILL.md with yaml@2.x (the parser the `skills` installer + # uses) and diffs marketplace.json's local plugin against the filesystem. + - name: Validate skills + run: bun run scripts/validate-skills.ts + + # First-party authoring lint from deepgram/ci-tools (skill_lint.py): description + # quality, inline code, allowed-tools coverage, model/effort/version. Complements the + # job above, which owns parse-correctness and manifest sync. + # + # deepgram/ci-tools is PRIVATE, so the default GITHUB_TOKEN cannot check it out — it + # fails with "Repository not found". Until a read token is provisioned as the + # CI_TOOLS_READ_TOKEN secret, this job reports the gap and passes rather than failing + # every PR on a missing credential. Once the secret exists it lints and blocks, with + # no workflow change needed. + lint-skills: + name: Lint skills (ci-tools) + runs-on: ubuntu-latest + env: + CI_TOOLS_READ_TOKEN: ${{ secrets.CI_TOOLS_READ_TOKEN }} + steps: + - name: Check for a ci-tools read token + id: gate + run: | + if [ -n "$CI_TOOLS_READ_TOKEN" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::warning::Skipping the ci-tools skill lint: deepgram/ci-tools is private and no CI_TOOLS_READ_TOKEN secret is set." + { + echo "## ci-tools skill lint skipped" + echo + echo "\`deepgram/ci-tools\` is a private repository, so the default \`GITHUB_TOKEN\`" + echo "cannot check it out. Add a read-scoped token as the \`CI_TOOLS_READ_TOKEN\`" + echo "repository secret to enable this lint — no workflow change is needed." + echo + echo "The blocking \`Validate skills and manifest\` job is unaffected." + } >> "$GITHUB_STEP_SUMMARY" + fi + + - if: steps.gate.outputs.available == 'true' + uses: actions/checkout@v4 + + - name: Check out ci-tools + if: steps.gate.outputs.available == 'true' + uses: actions/checkout@v4 + with: + repository: deepgram/ci-tools + ref: v3 + path: .ci-tools + token: ${{ secrets.CI_TOOLS_READ_TOKEN }} + + - if: steps.gate.outputs.available == 'true' + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Lint skills + if: steps.gate.outputs.available == 'true' + working-directory: .ci-tools + run: make lint PROJECT_DIR="$GITHUB_WORKSPACE" + + # Non-blocking by design. The SDK plugins' `.agents/skills/*` paths resolve against + # remote github sources, so they cannot be checked from this checkout — this job asks + # the live repos instead. It is `continue-on-error` so a GitHub outage (or an + # in-flight SDK fix) never blocks a PR here; read its log to see what it found. + remote-sdk-skills: + name: Remote SDK plugin skills (report only) + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - run: bun install --frozen-lockfile + + - name: Check remote SDK plugin skill paths + env: + GH_TOKEN: ${{ github.token }} + run: bun run scripts/validate-skills.ts --remote diff --git a/scripts/validate-skills.ts b/scripts/validate-skills.ts new file mode 100644 index 0000000..111e144 --- /dev/null +++ b/scripts/validate-skills.ts @@ -0,0 +1,377 @@ +/** + * Validate this repo's skills and keep .claude-plugin/marketplace.json honest. + * + * Exists because two classes of bug shipped silently: + * + * 1. A `description:` written as an unquoted YAML scalar containing a colon-space + * ("Route honestly: this crate...") is a YAML *parse error*, not a lint nit. The + * `skills` CLI prints one dim "⚠ Skipped ... YAML parse error" line and then exits 0, + * so the SDK installs N-1 skills while the manifest advertises N. To catch that we + * parse frontmatter with the SAME parser the installer uses (yaml@2.x) — a more + * lenient parser would pass files the installer drops. + * + * 2. marketplace.json drifting from what is actually on disk (it once advertised + * `deepgram-{lang}-maintaining-sdk` skills that exist in no repo). So the local + * plugin's skill list is diffed against the filesystem in BOTH directions. + * + * Complements `deepgram/ci-tools` `check-skill` (skill_lint.py), which covers authoring + * quality — inline code, allowed-tools coverage, model/effort/version — but treats + * frontmatter as regex-matched text and never looks at marketplace.json, so it reports + * zero errors for both bugs above. + * + * Usage: + * bun run scripts/validate-skills.ts # local, hermetic, blocking + * bun run scripts/validate-skills.ts --remote # also check SDK plugins via `gh api` + */ + +import { readdirSync, readFileSync, existsSync, statSync, appendFileSync } from "fs"; +import { join, relative } from "path"; +import { parse } from "yaml"; + +const ROOT = join(import.meta.dirname, ".."); +const SKILLS_DIR = join(ROOT, "skills"); +const MANIFEST = join(ROOT, ".claude-plugin", "marketplace.json"); + +/** Plugin in marketplace.json whose `source` is this repo, so its paths are local. */ +const LOCAL_PLUGIN = "deepgram"; + +const failures: string[] = []; + +function fail(message: string): void { + failures.push(message); +} + +function rel(path: string): string { + return relative(ROOT, path) || path; +} + +// --------------------------------------------------------------------------- +// Frontmatter +// --------------------------------------------------------------------------- + +interface Frontmatter { + /** Parsed mapping, or null when the block failed to parse. */ + data: Record | null; + /** Verbatim parser error, for operator-actionable output. */ + error?: string; + /** Line in the file where the frontmatter body starts (for error offsets). */ + bodyStartLine: number; +} + +/** + * Split the `---` delimited block off the top of a SKILL.md and parse it. + * + * The delimiters are stripped before parsing so parser line numbers match what the + * installer reports (it reports "line 2" for the third line of the file). + */ +function readFrontmatter(file: string): Frontmatter { + const text = readFileSync(file, "utf-8"); + const match = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/.exec(text); + + if (!match) { + return { + data: null, + error: "no YAML frontmatter found (expected a leading `---` delimited block)", + bodyStartLine: 2, + }; + } + + try { + const data = parse(match[1]); + if (data === null || typeof data !== "object" || Array.isArray(data)) { + return { + data: null, + error: `frontmatter is not a YAML mapping (parsed as ${ + Array.isArray(data) ? "array" : typeof data + })`, + bodyStartLine: 2, + }; + } + return { data: data as Record, bodyStartLine: 2 }; + } catch (err) { + return { + data: null, + error: err instanceof Error ? err.message : String(err), + bodyStartLine: 2, + }; + } +} + +// --------------------------------------------------------------------------- +// Manifest +// --------------------------------------------------------------------------- + +interface Plugin { + name?: unknown; + source?: unknown; + skills?: unknown; +} + +/** Parse marketplace.json. Returns null (and records a failure) if unusable. */ +function readManifest(): Plugin[] | null { + if (!existsSync(MANIFEST)) { + fail(`${rel(MANIFEST)}: manifest not found`); + return null; + } + + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(MANIFEST, "utf-8")); + } catch (err) { + fail( + `${rel(MANIFEST)}: invalid JSON — ${ + err instanceof Error ? err.message : String(err) + }` + ); + return null; + } + + const plugins = (parsed as { plugins?: unknown })?.plugins; + if (!Array.isArray(plugins)) { + fail(`${rel(MANIFEST)}: missing a top-level "plugins" array`); + return null; + } + return plugins as Plugin[]; +} + +// --------------------------------------------------------------------------- +// Checks +// --------------------------------------------------------------------------- + +/** Every `skills/*` directory on disk, sorted. */ +function skillDirsOnDisk(): string[] { + if (!existsSync(SKILLS_DIR)) { + fail(`${rel(SKILLS_DIR)}: skills directory not found`); + return []; + } + return readdirSync(SKILLS_DIR) + .filter((name) => { + if (name.startsWith(".")) return false; + // throwIfNoEntry:false so a broken symlink is skipped rather than crashing the run. + return statSync(join(SKILLS_DIR, name), { throwIfNoEntry: false })?.isDirectory() ?? false; + }) + .sort(); +} + +/** Parse + validate each skill. Returns how many were fully valid. */ +function checkSkills(dirs: string[]): number { + let valid = 0; + + for (const dir of dirs) { + const file = join(SKILLS_DIR, dir, "SKILL.md"); + + if (!existsSync(file)) { + fail(`${rel(join(SKILLS_DIR, dir))}: directory has no SKILL.md`); + continue; + } + + const { data, error } = readFrontmatter(file); + + if (!data) { + // This is the installer-fatal case: the CLI skips the skill and still exits 0. + fail(`${rel(file)}: YAML frontmatter failed to parse — ${error}`); + continue; + } + + let ok = true; + + for (const field of ["name", "description"] as const) { + const value = data[field]; + if (typeof value !== "string" || value.trim() === "") { + fail(`${rel(file)}: missing or empty required field \`${field}\``); + ok = false; + } + } + + if (typeof data.name === "string" && data.name.trim() !== dir) { + fail( + `${rel(file)}: frontmatter \`name: ${data.name.trim()}\` does not match its directory name \`${dir}\`` + ); + ok = false; + } + + if (ok) valid += 1; + } + + return valid; +} + +/** Diff the local plugin's `skills` list against `skills/*` on disk, both directions. */ +function checkManifestSync(plugins: Plugin[], dirs: string[]): void { + const local = plugins.find((p) => p.name === LOCAL_PLUGIN); + + if (!local) { + fail(`${rel(MANIFEST)}: no plugin named "${LOCAL_PLUGIN}" — cannot check local skill paths`); + return; + } + if (!Array.isArray(local.skills)) { + fail(`${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" has no "skills" array`); + return; + } + + const listed = new Set(); + + for (const entry of local.skills) { + if (typeof entry !== "string") { + fail(`${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" has a non-string skills entry: ${JSON.stringify(entry)}`); + continue; + } + + // Local paths are repo-relative (`./skills/`). Anything else is a remote + // source and does not belong in this plugin. + const path = entry.replace(/^\.\//, ""); + if (!path.startsWith("skills/")) { + fail( + `${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" entry \`${entry}\` is not a local \`./skills/*\` path` + ); + continue; + } + + const name = path.slice("skills/".length); + listed.add(name); + + if (!existsSync(join(ROOT, path, "SKILL.md"))) { + fail( + `${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" lists \`${entry}\` but ${path}/SKILL.md does not exist on disk` + ); + } + } + + for (const dir of dirs) { + if (!listed.has(dir)) { + fail( + `${rel(MANIFEST)}: skills/${dir}/SKILL.md exists on disk but plugin "${LOCAL_PLUGIN}" does not list \`./skills/${dir}\` — it will not install` + ); + } + } +} + +// --------------------------------------------------------------------------- +// Optional remote check (SDK plugins) +// --------------------------------------------------------------------------- + +/** + * The SDK plugins' `.agents/skills/*` paths resolve against github sources, not this + * checkout, so they cannot be verified locally. Checked here against the live repos via + * `gh api`. Deliberately separate and non-blocking: a GitHub outage must not block PRs. + */ +async function checkRemotePlugins(plugins: Plugin[]): Promise { + const remote = plugins.filter( + (p) => typeof p.source === "object" && p.source !== null && "repo" in (p.source as object) + ); + + const problems: string[] = []; + const report = (line: string) => { + problems.push(line); + console.log(` ✗ ${line}`); + }; + + for (const plugin of remote) { + const repo = (plugin.source as { repo?: string }).repo; + const skills = Array.isArray(plugin.skills) ? plugin.skills : []; + if (!repo) continue; + + for (const entry of skills) { + if (typeof entry !== "string") continue; + const path = `${entry.replace(/^\.\//, "")}/SKILL.md`; + const proc = Bun.spawn(["gh", "api", `repos/${repo}/contents/${path}`, "--jq", ".sha"], { + stdout: "pipe", + stderr: "pipe", + }); + const code = await proc.exited; + + if (code !== 0) { + const stderr = (await new Response(proc.stderr).text()).trim().split("\n")[0]; + report(`${repo}: ${path} — ${stderr || `gh api exited ${code}`}`); + continue; + } + + // Present in the repo — now confirm the installer can actually parse it. + const raw = Bun.spawn( + ["gh", "api", `repos/${repo}/contents/${path}`, "-H", "Accept: application/vnd.github.raw"], + { stdout: "pipe", stderr: "pipe" } + ); + if ((await raw.exited) !== 0) continue; + + const text = await new Response(raw.stdout).text(); + const match = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/.exec(text); + if (!match) { + report(`${repo}: ${path} — no YAML frontmatter found`); + continue; + } + try { + parse(match[1]); + } catch (err) { + report( + `${repo}: ${path} — YAML frontmatter failed to parse — ${ + err instanceof Error ? err.message : String(err) + }` + ); + } + } + } + + return problems; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const dirs = skillDirsOnDisk(); + const valid = checkSkills(dirs); + + const plugins = readManifest(); + if (plugins) checkManifestSync(plugins, dirs); + + console.log(`Found ${dirs.length} skills, ${valid} valid`); + + if (failures.length > 0) { + console.log(`\n${failures.length} failure(s):`); + for (const failure of failures) console.log(` ✗ ${failure}`); + } + + if (process.argv.includes("--remote")) { + if (!plugins) { + console.log("\nSkipping remote SDK plugin check — manifest is unusable."); + } else { + console.log("\nChecking remote SDK plugin skill paths via `gh api`..."); + const problems = await checkRemotePlugins(plugins); + console.log( + problems.length === 0 + ? "All remote SDK plugin skill paths exist and parse." + : `${problems.length} remote SDK plugin skill problem(s) — reported, not blocking.` + ); + + // This job is continue-on-error, so it green-checks either way and nobody opens a + // passing job's log. Put findings in the run summary so they are visible without + // digging — the whole point here is not shipping another skippable warning. + const summary = process.env.GITHUB_STEP_SUMMARY; + if (summary && problems.length > 0) { + const lines = [ + `## ${problems.length} remote SDK plugin skill problem(s)`, + "", + "These skills are advertised by `.claude-plugin/marketplace.json` but the live SDK", + "repo either does not have them or ships frontmatter the installer cannot parse.", + "A skill that fails to parse is silently skipped at install time (the CLI still", + "exits 0), so the SDK installs fewer skills than the manifest advertises.", + "", + // Parser errors are multi-line (they include a caret line), so each finding + // gets a fenced block rather than a list item that would break the markdown. + ...problems.flatMap((p) => { + const [location, ...rest] = p.split(" — "); + return [`**${location}**`, "", "```", rest.join(" — ").trimEnd(), "```", ""]; + }), + "Reported, not blocking: these live in other repos and must be fixed there.", + ]; + appendFileSync(summary, `${lines.join("\n")}\n`); + } + } + } + + if (failures.length > 0) process.exit(1); + console.log("\nAll skills valid."); +} + +main();