From 73462f5a51ca5610e3dd2fdf0ff4539b7442d0f2 Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Fri, 18 Sep 2026 08:24:03 -0400 Subject: [PATCH 1/2] ci: validate skills, manifest sync, and api spec drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was no CI validating skills at all, and two classes of bug shipped because of it. A `description:` written as an unquoted YAML scalar containing a colon-space is a YAML parse error, not a lint nit. The `skills` CLI prints one dim "Skipped ... YAML parse error" line and exits 0, so an SDK installs N-1 skills while the manifest advertises N. Separately, marketplace.json drifted from what is actually on disk. Adds three things: - scripts/validate-skills.ts — parses every skills/*/SKILL.md with yaml@2.x (the parser the installer uses, so results match reality), requires name and description, requires name to equal the directory name, and diffs marketplace.json's local plugin against the filesystem in both directions. Non-zero exit on any failure. `--remote` additionally checks the SDK plugins' `.agents/skills/*` paths against the live repos via `gh api`. - .github/workflows/validate-skills.yml — on pull_request and push to main. Two blocking jobs: the validator above, and deepgram/ci-tools `make lint` (skill_lint.py) for authoring quality. The remote SDK check runs as a separate continue-on-error job so a GitHub outage cannot block a PR; its findings go to the run summary rather than log-only. - .github/workflows/spec-drift.yml — weekly. Regenerates the api skill from the live specs and reports drift on a rolling issue. Report only; it never commits regenerated output. ci-tools check-skill covers authoring quality but treats frontmatter as regex-matched text and never reads marketplace.json, so it reports zero errors for both bugs above. check-plugin is not wired: it requires .claude-plugin/plugin.json, and this repo is a marketplace repo. --- .github/workflows/spec-drift.yml | 116 ++++++++ .github/workflows/validate-skills.yml | 78 ++++++ scripts/validate-skills.ts | 377 ++++++++++++++++++++++++++ 3 files changed, 571 insertions(+) create mode 100644 .github/workflows/spec-drift.yml create mode 100644 .github/workflows/validate-skills.yml create mode 100644 scripts/validate-skills.ts diff --git a/.github/workflows/spec-drift.yml b/.github/workflows/spec-drift.yml new file mode 100644 index 0000000..e1a50b5 --- /dev/null +++ b/.github/workflows/spec-drift.yml @@ -0,0 +1,116 @@ +name: API Spec Drift + +# The `api` skill's references are generated from Deepgram's public OpenAPI and AsyncAPI +# specs, which change without anything in this repo noticing — the checked-in output once +# sat ~5 weeks behind. This job regenerates weekly and reports drift. +# +# REPORT ONLY. It never commits regenerated output: regeneration is a reviewed change +# (`bun run scripts/fetch-specs.ts … && bun run scripts/generate-skills.ts`), and an +# auto-commit would land unreviewed API copy in a skill agents read as reference. +# +# Known limitation: `generate-skills.ts` overwrites the reference files it emits but does +# not prune ones it no longer emits, so a reference file for a *removed* endpoint group +# shows up as no drift here. Additions and edits are detected. This job is deliberately +# tolerant of that — it reports what it can see rather than asserting the tree is clean. + +on: + schedule: + # Mondays, 13:00 UTC. + - cron: "0 13 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check-drift: + name: Check generated api skill against live specs + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - run: bun install --frozen-lockfile + + # Use the .yml URLs. `dpgr.am/openapi.yaml` (.yaml) serves the marketing homepage + # with HTTP 200, and fetch-specs.ts only checks `response.ok`, so a .yaml typo + # writes HTML into specs/ and fails later at parse() with a confusing error. + - name: Fetch live specs + run: bun run scripts/fetch-specs.ts https://dpgr.am/openapi.yml https://dpgr.am/asyncapi.yml + + # Cheap guard against the trap above, so a bad fetch reports as a fetch problem. + - name: Verify specs are YAML, not HTML + run: | + for f in specs/openapi.yml specs/asyncapi.yml; do + if head -c 200 "$f" | grep -qi 'Diff stat" + echo + echo '```' + git diff --cached --stat -- skills/api/references + echo '```' + echo + echo "" + echo + echo "[Workflow run]($RUN_URL)" + } > body.md + + cat body.md >> "$GITHUB_STEP_SUMMARY" + + # One rolling issue rather than a new one every week. + existing=$(gh issue list --state open --label spec-drift --limit 1 --json number --jq '.[0].number') + if [ -n "$existing" ]; then + gh issue comment "$existing" --body-file body.md + echo "Commented on existing issue #$existing" + else + gh label create spec-drift --color FBCA04 --description "Generated api skill is behind the live specs" 2>/dev/null || true + gh issue create --title "API spec drift: skills/api/references is behind the live specs" \ + --label spec-drift --body-file body.md + fi + + # Fail visibly so the scheduled run is red in the Actions tab. + exit 1 diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml new file mode 100644 index 0000000..e47ea3c --- /dev/null +++ b/.github/workflows/validate-skills.yml @@ -0,0 +1,78 @@ +name: Validate Skills + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: validate-skills-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Blocking. Hermetic — no network, so it cannot flake. + validate: + name: Validate skills and manifest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - run: bun install --frozen-lockfile + + # Parses every skills/*/SKILL.md with yaml@2.x (the parser the `skills` installer + # uses) and diffs marketplace.json's local plugin against the filesystem. + - name: Validate skills + run: bun run scripts/validate-skills.ts + + # Blocking. First-party authoring lint from deepgram/ci-tools (skill_lint.py): + # description quality, inline code, allowed-tools coverage, model/effort/version. + # Complements the job above, which owns parse-correctness and manifest sync. + lint-skills: + name: Lint skills (ci-tools) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check out ci-tools + uses: actions/checkout@v4 + with: + repository: deepgram/ci-tools + ref: v3 + path: .ci-tools + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Lint skills + working-directory: .ci-tools + run: make lint PROJECT_DIR="$GITHUB_WORKSPACE" + + # Non-blocking by design. The SDK plugins' `.agents/skills/*` paths resolve against + # remote github sources, so they cannot be checked from this checkout — this job asks + # the live repos instead. It is `continue-on-error` so a GitHub outage (or an + # in-flight SDK fix) never blocks a PR here; read its log to see what it found. + remote-sdk-skills: + name: Remote SDK plugin skills (report only) + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - run: bun install --frozen-lockfile + + - name: Check remote SDK plugin skill paths + env: + GH_TOKEN: ${{ github.token }} + run: bun run scripts/validate-skills.ts --remote diff --git a/scripts/validate-skills.ts b/scripts/validate-skills.ts new file mode 100644 index 0000000..111e144 --- /dev/null +++ b/scripts/validate-skills.ts @@ -0,0 +1,377 @@ +/** + * Validate this repo's skills and keep .claude-plugin/marketplace.json honest. + * + * Exists because two classes of bug shipped silently: + * + * 1. A `description:` written as an unquoted YAML scalar containing a colon-space + * ("Route honestly: this crate...") is a YAML *parse error*, not a lint nit. The + * `skills` CLI prints one dim "⚠ Skipped ... YAML parse error" line and then exits 0, + * so the SDK installs N-1 skills while the manifest advertises N. To catch that we + * parse frontmatter with the SAME parser the installer uses (yaml@2.x) — a more + * lenient parser would pass files the installer drops. + * + * 2. marketplace.json drifting from what is actually on disk (it once advertised + * `deepgram-{lang}-maintaining-sdk` skills that exist in no repo). So the local + * plugin's skill list is diffed against the filesystem in BOTH directions. + * + * Complements `deepgram/ci-tools` `check-skill` (skill_lint.py), which covers authoring + * quality — inline code, allowed-tools coverage, model/effort/version — but treats + * frontmatter as regex-matched text and never looks at marketplace.json, so it reports + * zero errors for both bugs above. + * + * Usage: + * bun run scripts/validate-skills.ts # local, hermetic, blocking + * bun run scripts/validate-skills.ts --remote # also check SDK plugins via `gh api` + */ + +import { readdirSync, readFileSync, existsSync, statSync, appendFileSync } from "fs"; +import { join, relative } from "path"; +import { parse } from "yaml"; + +const ROOT = join(import.meta.dirname, ".."); +const SKILLS_DIR = join(ROOT, "skills"); +const MANIFEST = join(ROOT, ".claude-plugin", "marketplace.json"); + +/** Plugin in marketplace.json whose `source` is this repo, so its paths are local. */ +const LOCAL_PLUGIN = "deepgram"; + +const failures: string[] = []; + +function fail(message: string): void { + failures.push(message); +} + +function rel(path: string): string { + return relative(ROOT, path) || path; +} + +// --------------------------------------------------------------------------- +// Frontmatter +// --------------------------------------------------------------------------- + +interface Frontmatter { + /** Parsed mapping, or null when the block failed to parse. */ + data: Record | null; + /** Verbatim parser error, for operator-actionable output. */ + error?: string; + /** Line in the file where the frontmatter body starts (for error offsets). */ + bodyStartLine: number; +} + +/** + * Split the `---` delimited block off the top of a SKILL.md and parse it. + * + * The delimiters are stripped before parsing so parser line numbers match what the + * installer reports (it reports "line 2" for the third line of the file). + */ +function readFrontmatter(file: string): Frontmatter { + const text = readFileSync(file, "utf-8"); + const match = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/.exec(text); + + if (!match) { + return { + data: null, + error: "no YAML frontmatter found (expected a leading `---` delimited block)", + bodyStartLine: 2, + }; + } + + try { + const data = parse(match[1]); + if (data === null || typeof data !== "object" || Array.isArray(data)) { + return { + data: null, + error: `frontmatter is not a YAML mapping (parsed as ${ + Array.isArray(data) ? "array" : typeof data + })`, + bodyStartLine: 2, + }; + } + return { data: data as Record, bodyStartLine: 2 }; + } catch (err) { + return { + data: null, + error: err instanceof Error ? err.message : String(err), + bodyStartLine: 2, + }; + } +} + +// --------------------------------------------------------------------------- +// Manifest +// --------------------------------------------------------------------------- + +interface Plugin { + name?: unknown; + source?: unknown; + skills?: unknown; +} + +/** Parse marketplace.json. Returns null (and records a failure) if unusable. */ +function readManifest(): Plugin[] | null { + if (!existsSync(MANIFEST)) { + fail(`${rel(MANIFEST)}: manifest not found`); + return null; + } + + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(MANIFEST, "utf-8")); + } catch (err) { + fail( + `${rel(MANIFEST)}: invalid JSON — ${ + err instanceof Error ? err.message : String(err) + }` + ); + return null; + } + + const plugins = (parsed as { plugins?: unknown })?.plugins; + if (!Array.isArray(plugins)) { + fail(`${rel(MANIFEST)}: missing a top-level "plugins" array`); + return null; + } + return plugins as Plugin[]; +} + +// --------------------------------------------------------------------------- +// Checks +// --------------------------------------------------------------------------- + +/** Every `skills/*` directory on disk, sorted. */ +function skillDirsOnDisk(): string[] { + if (!existsSync(SKILLS_DIR)) { + fail(`${rel(SKILLS_DIR)}: skills directory not found`); + return []; + } + return readdirSync(SKILLS_DIR) + .filter((name) => { + if (name.startsWith(".")) return false; + // throwIfNoEntry:false so a broken symlink is skipped rather than crashing the run. + return statSync(join(SKILLS_DIR, name), { throwIfNoEntry: false })?.isDirectory() ?? false; + }) + .sort(); +} + +/** Parse + validate each skill. Returns how many were fully valid. */ +function checkSkills(dirs: string[]): number { + let valid = 0; + + for (const dir of dirs) { + const file = join(SKILLS_DIR, dir, "SKILL.md"); + + if (!existsSync(file)) { + fail(`${rel(join(SKILLS_DIR, dir))}: directory has no SKILL.md`); + continue; + } + + const { data, error } = readFrontmatter(file); + + if (!data) { + // This is the installer-fatal case: the CLI skips the skill and still exits 0. + fail(`${rel(file)}: YAML frontmatter failed to parse — ${error}`); + continue; + } + + let ok = true; + + for (const field of ["name", "description"] as const) { + const value = data[field]; + if (typeof value !== "string" || value.trim() === "") { + fail(`${rel(file)}: missing or empty required field \`${field}\``); + ok = false; + } + } + + if (typeof data.name === "string" && data.name.trim() !== dir) { + fail( + `${rel(file)}: frontmatter \`name: ${data.name.trim()}\` does not match its directory name \`${dir}\`` + ); + ok = false; + } + + if (ok) valid += 1; + } + + return valid; +} + +/** Diff the local plugin's `skills` list against `skills/*` on disk, both directions. */ +function checkManifestSync(plugins: Plugin[], dirs: string[]): void { + const local = plugins.find((p) => p.name === LOCAL_PLUGIN); + + if (!local) { + fail(`${rel(MANIFEST)}: no plugin named "${LOCAL_PLUGIN}" — cannot check local skill paths`); + return; + } + if (!Array.isArray(local.skills)) { + fail(`${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" has no "skills" array`); + return; + } + + const listed = new Set(); + + for (const entry of local.skills) { + if (typeof entry !== "string") { + fail(`${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" has a non-string skills entry: ${JSON.stringify(entry)}`); + continue; + } + + // Local paths are repo-relative (`./skills/`). Anything else is a remote + // source and does not belong in this plugin. + const path = entry.replace(/^\.\//, ""); + if (!path.startsWith("skills/")) { + fail( + `${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" entry \`${entry}\` is not a local \`./skills/*\` path` + ); + continue; + } + + const name = path.slice("skills/".length); + listed.add(name); + + if (!existsSync(join(ROOT, path, "SKILL.md"))) { + fail( + `${rel(MANIFEST)}: plugin "${LOCAL_PLUGIN}" lists \`${entry}\` but ${path}/SKILL.md does not exist on disk` + ); + } + } + + for (const dir of dirs) { + if (!listed.has(dir)) { + fail( + `${rel(MANIFEST)}: skills/${dir}/SKILL.md exists on disk but plugin "${LOCAL_PLUGIN}" does not list \`./skills/${dir}\` — it will not install` + ); + } + } +} + +// --------------------------------------------------------------------------- +// Optional remote check (SDK plugins) +// --------------------------------------------------------------------------- + +/** + * The SDK plugins' `.agents/skills/*` paths resolve against github sources, not this + * checkout, so they cannot be verified locally. Checked here against the live repos via + * `gh api`. Deliberately separate and non-blocking: a GitHub outage must not block PRs. + */ +async function checkRemotePlugins(plugins: Plugin[]): Promise { + const remote = plugins.filter( + (p) => typeof p.source === "object" && p.source !== null && "repo" in (p.source as object) + ); + + const problems: string[] = []; + const report = (line: string) => { + problems.push(line); + console.log(` ✗ ${line}`); + }; + + for (const plugin of remote) { + const repo = (plugin.source as { repo?: string }).repo; + const skills = Array.isArray(plugin.skills) ? plugin.skills : []; + if (!repo) continue; + + for (const entry of skills) { + if (typeof entry !== "string") continue; + const path = `${entry.replace(/^\.\//, "")}/SKILL.md`; + const proc = Bun.spawn(["gh", "api", `repos/${repo}/contents/${path}`, "--jq", ".sha"], { + stdout: "pipe", + stderr: "pipe", + }); + const code = await proc.exited; + + if (code !== 0) { + const stderr = (await new Response(proc.stderr).text()).trim().split("\n")[0]; + report(`${repo}: ${path} — ${stderr || `gh api exited ${code}`}`); + continue; + } + + // Present in the repo — now confirm the installer can actually parse it. + const raw = Bun.spawn( + ["gh", "api", `repos/${repo}/contents/${path}`, "-H", "Accept: application/vnd.github.raw"], + { stdout: "pipe", stderr: "pipe" } + ); + if ((await raw.exited) !== 0) continue; + + const text = await new Response(raw.stdout).text(); + const match = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/.exec(text); + if (!match) { + report(`${repo}: ${path} — no YAML frontmatter found`); + continue; + } + try { + parse(match[1]); + } catch (err) { + report( + `${repo}: ${path} — YAML frontmatter failed to parse — ${ + err instanceof Error ? err.message : String(err) + }` + ); + } + } + } + + return problems; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const dirs = skillDirsOnDisk(); + const valid = checkSkills(dirs); + + const plugins = readManifest(); + if (plugins) checkManifestSync(plugins, dirs); + + console.log(`Found ${dirs.length} skills, ${valid} valid`); + + if (failures.length > 0) { + console.log(`\n${failures.length} failure(s):`); + for (const failure of failures) console.log(` ✗ ${failure}`); + } + + if (process.argv.includes("--remote")) { + if (!plugins) { + console.log("\nSkipping remote SDK plugin check — manifest is unusable."); + } else { + console.log("\nChecking remote SDK plugin skill paths via `gh api`..."); + const problems = await checkRemotePlugins(plugins); + console.log( + problems.length === 0 + ? "All remote SDK plugin skill paths exist and parse." + : `${problems.length} remote SDK plugin skill problem(s) — reported, not blocking.` + ); + + // This job is continue-on-error, so it green-checks either way and nobody opens a + // passing job's log. Put findings in the run summary so they are visible without + // digging — the whole point here is not shipping another skippable warning. + const summary = process.env.GITHUB_STEP_SUMMARY; + if (summary && problems.length > 0) { + const lines = [ + `## ${problems.length} remote SDK plugin skill problem(s)`, + "", + "These skills are advertised by `.claude-plugin/marketplace.json` but the live SDK", + "repo either does not have them or ships frontmatter the installer cannot parse.", + "A skill that fails to parse is silently skipped at install time (the CLI still", + "exits 0), so the SDK installs fewer skills than the manifest advertises.", + "", + // Parser errors are multi-line (they include a caret line), so each finding + // gets a fenced block rather than a list item that would break the markdown. + ...problems.flatMap((p) => { + const [location, ...rest] = p.split(" — "); + return [`**${location}**`, "", "```", rest.join(" — ").trimEnd(), "```", ""]; + }), + "Reported, not blocking: these live in other repos and must be fixed there.", + ]; + appendFileSync(summary, `${lines.join("\n")}\n`); + } + } + } + + if (failures.length > 0) process.exit(1); + console.log("\nAll skills valid."); +} + +main(); From 3a53a5cb7f5436f3362bb443085544342fda8cde Mon Sep 17 00:00:00 2001 From: Corey Weathers Date: Fri, 18 Sep 2026 08:27:06 -0400 Subject: [PATCH 2/2] ci: skip the ci-tools skill lint until a read token is provisioned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deepgram/ci-tools is private, so the default GITHUB_TOKEN cannot check it out — actions/checkout fails with "Repository not found" and the job red-walls every PR on a missing credential rather than on a real skill problem. Gate the job on a CI_TOOLS_READ_TOKEN secret. Without it the job reports the gap to the run summary as a warning and passes; with it the job lints and blocks, with no workflow change needed. The blocking validator job is unaffected. --- .github/workflows/validate-skills.yml | 42 +++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml index e47ea3c..6abbf57 100644 --- a/.github/workflows/validate-skills.yml +++ b/.github/workflows/validate-skills.yml @@ -31,27 +31,59 @@ jobs: - name: Validate skills run: bun run scripts/validate-skills.ts - # Blocking. First-party authoring lint from deepgram/ci-tools (skill_lint.py): - # description quality, inline code, allowed-tools coverage, model/effort/version. - # Complements the job above, which owns parse-correctness and manifest sync. + # First-party authoring lint from deepgram/ci-tools (skill_lint.py): description + # quality, inline code, allowed-tools coverage, model/effort/version. Complements the + # job above, which owns parse-correctness and manifest sync. + # + # deepgram/ci-tools is PRIVATE, so the default GITHUB_TOKEN cannot check it out — it + # fails with "Repository not found". Until a read token is provisioned as the + # CI_TOOLS_READ_TOKEN secret, this job reports the gap and passes rather than failing + # every PR on a missing credential. Once the secret exists it lints and blocks, with + # no workflow change needed. lint-skills: name: Lint skills (ci-tools) runs-on: ubuntu-latest + env: + CI_TOOLS_READ_TOKEN: ${{ secrets.CI_TOOLS_READ_TOKEN }} steps: - - uses: actions/checkout@v4 + - name: Check for a ci-tools read token + id: gate + run: | + if [ -n "$CI_TOOLS_READ_TOKEN" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::warning::Skipping the ci-tools skill lint: deepgram/ci-tools is private and no CI_TOOLS_READ_TOKEN secret is set." + { + echo "## ci-tools skill lint skipped" + echo + echo "\`deepgram/ci-tools\` is a private repository, so the default \`GITHUB_TOKEN\`" + echo "cannot check it out. Add a read-scoped token as the \`CI_TOOLS_READ_TOKEN\`" + echo "repository secret to enable this lint — no workflow change is needed." + echo + echo "The blocking \`Validate skills and manifest\` job is unaffected." + } >> "$GITHUB_STEP_SUMMARY" + fi + + - if: steps.gate.outputs.available == 'true' + uses: actions/checkout@v4 - name: Check out ci-tools + if: steps.gate.outputs.available == 'true' uses: actions/checkout@v4 with: repository: deepgram/ci-tools ref: v3 path: .ci-tools + token: ${{ secrets.CI_TOOLS_READ_TOKEN }} - - uses: actions/setup-python@v5 + - if: steps.gate.outputs.available == 'true' + uses: actions/setup-python@v5 with: python-version: "3.12" - name: Lint skills + if: steps.gate.outputs.available == 'true' working-directory: .ci-tools run: make lint PROJECT_DIR="$GITHUB_WORKSPACE"