validate.yml

name: Validate

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]
  workflow_dispatch:

permissions:
  contents: read

jobs:
  validate-json:
    name: Validate JSON files
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
        with:
          persist-credentials: false

      - name: Validate marketplace.json
        run: python3 -m json.tool .claude-plugin/marketplace.json > /dev/null

      - name: Validate every plugin.json
        run: |
          set -e
          shopt -s globstar nullglob
          failed=0
          for f in plugins/*/.claude-plugin/plugin.json; do
            if ! python3 -m json.tool "$f" > /dev/null 2>&1; then
              echo "::error file=$f::invalid JSON"
              failed=1
            fi
          done
          exit $failed

      - name: Validate hooks.json files
        run: |
          set -e
          shopt -s globstar nullglob
          failed=0
          for f in plugins/*/hooks/*.json; do
            if ! python3 -m json.tool "$f" > /dev/null 2>&1; then
              echo "::error file=$f::invalid JSON"
              failed=1
            fi
          done
          exit $failed

      - name: Validate marketplace entries resolve to plugin dirs
        run: |
          python3 - <<'PY'
          import json, os, posixpath, sys
          from urllib.parse import urlparse
          with open('.claude-plugin/marketplace.json') as f:
              mp = json.load(f)
          errors = []
          for p in mp.get('plugins', []):
              src = p.get('source')
              if isinstance(src, str) and src.startswith('./plugins/'):
                  path = src.lstrip('./')
                  if not os.path.isdir(path):
                      errors.append(f"{p['name']}: source {src} does not exist")
                  elif not os.path.isfile(os.path.join(path, '.claude-plugin', 'plugin.json')):
                      errors.append(f"{p['name']}: missing .claude-plugin/plugin.json in {src}")
              elif isinstance(src, dict):
                  if src.get('source') != 'git-subdir':
                      errors.append(f"{p['name']}: unsupported source object {src.get('source')!r}")
                      continue

                  extra_keys = set(src) - {'source', 'url', 'path'}
                  if extra_keys:
                      errors.append(f"{p['name']}: git-subdir entry has unsupported keys: {sorted(extra_keys)}")

                  url = src.get('url')
                  path = src.get('path')

                  if not url:
                      errors.append(f"{p['name']}: git-subdir entry missing url")
                  else:
                      parsed = urlparse(url)
                      if parsed.scheme != 'https' or parsed.netloc != 'github.com' or not parsed.path.endswith('.git'):
                          errors.append(f"{p['name']}: git-subdir url must be an https://github.com/*.git URL")

                  if not path:
                      errors.append(f"{p['name']}: git-subdir entry missing path")
                  elif path != '.':
                      normalized = posixpath.normpath(path)
                      if (
                          path.startswith('/')
                          or '\\' in path
                          or normalized != path
                          or normalized == '..'
                          or normalized.startswith('../')
                      ):
                          errors.append(f"{p['name']}: git-subdir path must be a normalized relative path")
          if errors:
              for e in errors:
                  print(f"::error::{e}")
              sys.exit(1)
          print(f"OK: {len(mp.get('plugins', []))} marketplace entries validated")
          PY

      - name: Check agent name uniqueness
        run: python3 tools/check_agent_name_collisions.py --fail-on-duplicates

  plugin-eval-tests:
    name: plugin-eval pytest
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: plugins/plugin-eval
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install

      - name: Sync dependencies
        run: uv sync --all-extras

      - name: Run tests
        run: uv run pytest

  tools-tests:
    name: tools pytest (adapters + validators + gardener)
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: plugins/plugin-eval
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install

      - name: Sync dependencies
        run: uv sync --all-extras

      - name: Run tools test suite
        run: uv run pytest -q ../../tools/tests/

  multi-harness-generate:
    name: Cross-harness generation + validation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
        with:
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Sync plugin-eval venv (provides pyyaml + adapter imports)
        working-directory: plugins/plugin-eval
        run: uv sync --all-extras

      - name: Generate all harness artifacts
        run: make generate-all

      - name: Verify committed artifacts are in sync with sources
        # Native-install artifacts are committed so the repo installs from a clone.
        # Regeneration must produce no diff — if it does, a source change was committed
        # without running `make generate-all`. Run it locally and commit the result.
        run: |
          if [ -n "$(git status --porcelain)" ]; then
            echo "::error::Generated artifacts drifted from the committed tree. Run 'make generate-all' and commit the result."
            git status --short
            git --no-pager diff --stat
            exit 1
          fi
          echo "OK: committed harness artifacts match a fresh 'make generate-all'."

      - name: Structural validation (strict)
        run: make validate STRICT=1

      - name: Doc-gardener (no errors allowed)
        # Errors fail this step; warnings (e.g. oversized source skills with no
        # references/) are surfaced but don't block. Use STRICT=1 to gate on warnings too.
        run: make garden

      - name: Upload generated artifacts for inspection
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        with:
          name: multi-harness-output
          path: |
            .codex/
            .cursor/
            .cursor-plugin/
            .opencode/
            opencode.json
            commands/
            agents/
            skills/
            AGENTS.md
          retention-days: 7

  cli-smoke-test:
    name: Real-CLI smoke test (OpenCode + Gemini)
    runs-on: ubuntu-latest
    # Real-CLI subprocess tests: invokes the actual harness binaries against our
    # generated artifacts to catch issues that pure-Python parsing misses (CLI
    # version drift, schema-loader surprises, plugin discovery bugs).
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
        with:
          persist-credentials: false

      - name: Set up Node.js (for Gemini CLI)
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4
        with:
          node-version: '20'

      - name: Set up Bun (for OpenCode CLI)
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6  # v2
        with:
          bun-version: latest

      - name: Install OpenCode CLI
        run: |
          curl -fsSL https://opencode.ai/install | bash
          echo "$HOME/.opencode/bin" >> "$GITHUB_PATH"

      - name: Install Gemini CLI
        # Pinned to the released line we developed against; bump alongside any
        # gemini-extension.json schema changes.
        run: npm install -g @google/gemini-cli@latest

      - name: Verify CLI versions
        run: |
          opencode --version
          gemini --version

      - name: Install uv
        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Sync plugin-eval dependencies
        working-directory: plugins/plugin-eval
        run: uv sync --all-extras

      - name: Generate all harness artifacts
        run: make generate-all

      - name: Run real-CLI smoke tests
        working-directory: plugins/plugin-eval
        run: uv run pytest -v ../../tools/tests/test_cli_smoke.py