ci: test install.sh and the skills against the latest release

.github/workflows/ci.yaml

@@ -437,6 +437,117 @@ jobs:
               kill "$(cat /tmp/su-server.pid)" || true
           fi
 
+  # The agent skills against the binary people actually get: the latest
+  # release's. The CLI's source isn't in this repository any more, so there is
+  # no build to check them against. See scripts/check-skills.py.
+  skills:
+    name: skills reference real commands
+    timeout-minutes: 5
+    runs-on: ubuntu-26.04
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - name: Download the latest release's Linux binary
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh release download --repo "${GITHUB_REPOSITORY}" \
+            --pattern 'mergify-*-x86_64-unknown-linux-gnu.tar.gz' --dir release
+          tar -C release -xzf release/mergify-*-x86_64-unknown-linux-gnu.tar.gz
+          release/mergify --version
+      - name: Check the skills
+        run: python3 scripts/check-skills.py release/mergify
+
+  # End to end, the one-liner people run: install.sh against the latest
+  # release's real binaries and SHA256SUMS, served from a local fixture so the
+  # script's own download, checksum and install paths all run. The release is
+  # the only place binaries come from now, so this is also what proves the
+  # current release installs. Plus the two ways a tampered SHA256SUMS must fail
+  # closed. Windows runs the same POSIX script under git-bash.
+  install-script-release:
+    name: install.sh against the latest release (${{ matrix.os }})
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-26.04, macos-15, windows-2025]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - name: Install shellcheck (Linux only)
+        if: matrix.os == 'ubuntu-26.04'
+        run: sudo apt-get install -y --no-install-recommends shellcheck
+      - name: Lint install.sh
+        if: matrix.os == 'ubuntu-26.04'
+        run: shellcheck install.sh
+      # Every binary of the release, so install.sh picks its own target the
+      # way it does against GitHub; latest-release.json stands in for the
+      # releases/latest redirect it resolves the version from.
+      - name: Fetch the latest release as the fixture
+        id: release
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          tag="$(gh release view --repo "${GITHUB_REPOSITORY}" --json tagName --jq .tagName)"
+          mkdir -p fixture/release
+          gh release download "${tag}" --repo "${GITHUB_REPOSITORY}" --dir fixture/release \
+            --pattern 'mergify-*' --pattern SHA256SUMS
+          printf '{"tag_name":"%s"}\n' "${tag}" > fixture/release/latest-release.json
+          ls -la fixture/release
+          echo "tag=${tag}" >> "${GITHUB_OUTPUT}"
+      - name: Serve the fixture
+        shell: bash
+        run: |
+          cd fixture/release
+          # `python3` on the Unix runners, `python` on the Windows one.
+          py=python3; command -v "$py" > /dev/null 2>&1 || py=python
+          "$py" -m http.server 8765 --bind 127.0.0.1 > /tmp/server.log 2>&1 &
+          echo $! > /tmp/server.pid
+          for _ in $(seq 1 50); do
+            curl -fsS http://127.0.0.1:8765/SHA256SUMS > /dev/null && break
+            sleep 0.1
+          done
+      - name: install.sh installs the latest release
+        shell: bash
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+        run: |
+          set -euo pipefail
+          MERGIFY_BASE_URL=http://127.0.0.1:8765 MERGIFY_INSTALL_DIR=/tmp/mergify-install ./install.sh
+          case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) bin=mergify.exe ;; *) bin=mergify ;; esac
+          out="$("/tmp/mergify-install/${bin}" --version)"
+          echo "${out}"
+          [ "${out}" = "mergify ${TAG}" ] || { echo "expected 'mergify ${TAG}', got '${out}'"; exit 1; }
+      # sha256sum -c only warns on a malformed line, so install.sh checks the
+      # line shape itself: both cases must refuse to install.
+      - name: install.sh rejects a tampered SHA256SUMS
+        shell: bash
+        run: |
+          set -euo pipefail
+          cd fixture/release
+          cp SHA256SUMS SHA256SUMS.orig
+          assets=(mergify-*)
+          echo "Case 1: wrong but well-formed hashes"
+          for asset in "${assets[@]}"; do printf '%064d  %s\n' 0 "${asset}"; done > SHA256SUMS
+          if MERGIFY_BASE_URL=http://127.0.0.1:8765 MERGIFY_INSTALL_DIR=/tmp/mergify-bad1 ../../install.sh > /tmp/case1.log 2>&1; then
+            cat /tmp/case1.log; echo "install.sh accepted wrong checksums"; exit 1
+          fi
+          grep -q 'checksum verification failed' /tmp/case1.log
+          echo "Case 2: malformed lines"
+          for asset in "${assets[@]}"; do echo "bogus  ${asset}"; done > SHA256SUMS
+          if MERGIFY_BASE_URL=http://127.0.0.1:8765 MERGIFY_INSTALL_DIR=/tmp/mergify-bad2 ../../install.sh > /tmp/case2.log 2>&1; then
+            cat /tmp/case2.log; echo "install.sh accepted malformed checksums"; exit 1
+          fi
+          grep -q 'malformed checksum entry' /tmp/case2.log
+          mv SHA256SUMS.orig SHA256SUMS
+      - name: Stop the fixture server
+        if: always()
+        shell: bash
+        run: |
+          if [ -f /tmp/server.pid ]; then kill "$(cat /tmp/server.pid)" || true; fi
+
   ci-gate:
     if: ${{ !cancelled() }}
     needs:
@@ -448,6 +559,8 @@ jobs:
       - smoke-test-binary
       - install-script-smoke
       - self-update-smoke
+      - skills
+      - install-script-release
     runs-on: ubuntu-26.04
     steps:
       - name: Verify all jobs succeeded

scripts/check-skills.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Check the agent skills under skills/ against a mergify binary.
+
+Usage: check-skills.py <path-to-mergify-binary>
+
+The skills tell an agent which `mergify` commands to run, so a skill that names
+a command the binary doesn't have sends the agent into an error. The CLI's
+source lives in a private repository now; this runs against a released binary
+(CI uses the latest release's), the one people actually get.
+
+For every skills/<name>/SKILL.md:
+- YAML front matter between `---` lines, with `name: <name>` matching the
+  directory and a `description`, which is what agent skill loaders read;
+- every `mergify <group> <subcommand>` it mentions, for a group the binary
+  has, is a subcommand the binary lists in `--list-native-commands`.
+The merge-queue skill must also keep its required sections.
+"""
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+SKILLS = Path(__file__).resolve().parent.parent / "skills"
+
+REQUIRED_SECTIONS = {
+    "mergify-merge-queue": [
+        "## Commands",
+        "## Checking Queue Status",
+        "## Inspecting a PR",
+        "## Queue States",
+        "## Troubleshooting",
+    ],
+}
+
+FRONT_MATTER = re.compile(r"\A---\n(.+?)\n---\n", re.DOTALL)
+REFERENCE = re.compile(r"\bmergify ([a-z][\w-]*) ([a-z][\w-]*)")
+
+
+def native_commands(binary: str) -> dict[str, set[str]]:
+    output = subprocess.run(
+        [binary, "--list-native-commands"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    commands: dict[str, set[str]] = {}
+    for line in output.splitlines():
+        group, _, sub = line.partition(" ")
+        if sub:
+            commands.setdefault(group, set()).add(sub.strip())
+    return commands
+
+
+def check_skill(path: Path, commands: dict[str, set[str]]) -> list[str]:
+    name = path.parent.name
+    content = path.read_text(encoding="utf-8")
+    problems = []
+
+    match = FRONT_MATTER.match(content)
+    if not match:
+        return [f"{name}: no YAML front matter between --- lines"]
+    keys = {}
+    for line in match.group(1).splitlines():
+        key, sep, value = line.partition(":")
+        if sep and not line.startswith((" ", "\t")):
+            keys[key.strip()] = value.strip()
+    if keys.get("name") != name:
+        problems.append(f"{name}: front matter name is {keys.get('name')!r}, expected {name!r}")
+    if not keys.get("description"):
+        problems.append(f"{name}: front matter has no description")
+
+    for section in REQUIRED_SECTIONS.get(name, []):
+        if section not in content:
+            problems.append(f"{name}: missing required section {section!r}")
+
+    for group, sub in sorted(set(REFERENCE.findall(content))):
+        if group in commands and sub not in commands[group]:
+            problems.append(
+                f"{name}: mentions `mergify {group} {sub}`, which the binary doesn't have"
+                f" (it has: {', '.join(sorted(commands[group]))})"
+            )
+    return problems
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(__doc__.splitlines()[2], file=sys.stderr)
+        return 2
+    commands = native_commands(sys.argv[1])
+    skills = sorted(SKILLS.glob("*/SKILL.md"))
+    if not skills:
+        print(f"::error::no skills found under {SKILLS}")
+        return 1
+    problems = [p for skill in skills for p in check_skill(skill, commands)]
+    for problem in problems:
+        print(f"::error::{problem}")
+    if problems:
+        return 1
+    print(f"ok: {len(skills)} skills checked against {sum(map(len, commands.values()))} commands")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())