diff --git a/.github/workflows/test-verify.yaml b/.github/workflows/test-verify.yaml new file mode 100644 index 0000000..a2dacb2 --- /dev/null +++ b/.github/workflows/test-verify.yaml @@ -0,0 +1,137 @@ +name: test verify +on: + push: + pull_request: + +jobs: + verify_all_green: + runs-on: ubuntu-latest + name: Test verify reports all checks green and exits 0 + # Stub oasdiff (specs resolve) + curl (service returns all checks true) and + # assert the entrypoint renders a full-green checklist and exits 0. + steps: + - uses: actions/checkout@v6 + - name: Stub oasdiff + curl, run verify entrypoint + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + exit 0 + STUB + chmod +x /tmp/stub/oasdiff + + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + cat >/dev/null + printf '{"token_ok":true,"app_installed":true,"specs_found":true,"owner":"acme","repo":"api"}\n200\n' + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REPOSITORY=acme/api + export GITHUB_STEP_SUMMARY=/tmp/run/summary + : > "$GITHUB_STEP_SUMMARY" + export PATH=/tmp/stub:$PATH + + set +e + out=$(./verify/entrypoint.sh 'main:openapi.yaml' 'HEAD:openapi.yaml' 'stub-token' 'http://svc' 'false' 2>&1) + rc=$? + set -e + echo "--- output ---"; echo "$out" + echo "--- summary ---"; cat "$GITHUB_STEP_SUMMARY" + echo "--- exit $rc ---" + + if [ "$rc" -ne 0 ]; then echo "FAIL: expected exit 0, got $rc" >&2; exit 1; fi + grep -q "✅ Connected to oasdiff" "$GITHUB_STEP_SUMMARY" || { echo "FAIL: token check not green" >&2; exit 1; } + grep -q "✅ oasdiff GitHub App installed" "$GITHUB_STEP_SUMMARY" || { echo "FAIL: app check not green" >&2; exit 1; } + grep -q "✅ OpenAPI spec found" "$GITHUB_STEP_SUMMARY" || { echo "FAIL: spec check not green" >&2; exit 1; } + echo "PASS" + + verify_app_not_installed: + runs-on: ubuntu-latest + name: Test verify flags a missing App and exits 1 + # Service reports app_installed=false: the entrypoint must mark that check + # red, emit an annotation, and exit 1 (setup not complete). + steps: + - uses: actions/checkout@v6 + - name: Stub oasdiff + curl (app not installed) + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + exit 0 + STUB + chmod +x /tmp/stub/oasdiff + + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + cat >/dev/null + printf '{"token_ok":true,"app_installed":false,"specs_found":true,"owner":"acme","repo":"api"}\n200\n' + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REPOSITORY=acme/api + export GITHUB_STEP_SUMMARY=/tmp/run/summary + : > "$GITHUB_STEP_SUMMARY" + export PATH=/tmp/stub:$PATH + + set +e + out=$(./verify/entrypoint.sh 'main:openapi.yaml' 'HEAD:openapi.yaml' 'stub-token' 'http://svc' 'false' 2>&1) + rc=$? + set -e + echo "--- output ---"; echo "$out" + echo "--- summary ---"; cat "$GITHUB_STEP_SUMMARY" + echo "--- exit $rc ---" + + if [ "$rc" -ne 1 ]; then echo "FAIL: expected exit 1, got $rc" >&2; exit 1; fi + grep -q "❌ oasdiff GitHub App installed" "$GITHUB_STEP_SUMMARY" || { echo "FAIL: app check not red" >&2; exit 1; } + echo "$out" | grep -q "::error title=oasdiff verify::oasdiff GitHub App is not installed" || { echo "FAIL: missing app annotation" >&2; exit 1; } + echo "PASS" + + verify_external_ref_blocked: + runs-on: ubuntu-latest + name: Test verify distinguishes a blocked external $ref (exit 123) + # oasdiff exits 123 when an external $ref is refused (allow-external-refs + # false). Verify must report that distinctly from "spec not found" — with + # the allow-external-refs hint, not a path hint — and exit 1. + steps: + - uses: actions/checkout@v6 + - name: Stub oasdiff (exit 123) + curl + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + exit 123 + STUB + chmod +x /tmp/stub/oasdiff + + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + cat >/dev/null + printf '{"token_ok":true,"app_installed":true,"specs_found":false,"owner":"acme","repo":"api"}\n200\n' + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REPOSITORY=acme/api + export GITHUB_STEP_SUMMARY=/tmp/run/summary + : > "$GITHUB_STEP_SUMMARY" + export PATH=/tmp/stub:$PATH + + set +e + out=$(./verify/entrypoint.sh 'main:openapi.yaml' 'HEAD:openapi.yaml' 'stub-token' 'http://svc' 'false' 2>&1) + rc=$? + set -e + echo "--- output ---"; echo "$out" + echo "--- summary ---"; cat "$GITHUB_STEP_SUMMARY" + echo "--- exit $rc ---" + + if [ "$rc" -ne 1 ]; then echo "FAIL: expected exit 1, got $rc" >&2; exit 1; fi + grep -q "external \$ref was blocked" "$GITHUB_STEP_SUMMARY" || { echo "FAIL: external-ref case not surfaced distinctly" >&2; exit 1; } + if grep -q "Spec not found" "$GITHUB_STEP_SUMMARY"; then echo "FAIL: mislabeled as spec-not-found" >&2; exit 1; fi + echo "$out" | grep -q "allow-external-refs: true if the spec is trusted" || { echo "FAIL: missing allow-external-refs hint" >&2; exit 1; } + echo "PASS" diff --git a/README.md b/README.md index 16fcc20..2052136 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ GitHub Actions for comparing OpenAPI specs and detecting breaking changes, based - [Configuring with `.oasdiff.yaml`](#configuring-with-oasdiffyaml) - [Spec paths](#spec-paths) - [Pro: Rich PR comment](#pro-rich-pr-comment) +- [Pro: Verify your setup](#pro-verify-your-setup) ## Quick start @@ -35,7 +36,7 @@ jobs: steps: - uses: actions/checkout@v6 - run: git fetch --depth=1 origin ${{ github.base_ref }} - - uses: oasdiff/oasdiff-action/breaking@v0.0.49 + - uses: oasdiff/oasdiff-action/breaking@v0.0.51 with: base: 'origin/${{ github.base_ref }}:openapi.yaml' revision: 'HEAD:openapi.yaml' @@ -65,7 +66,7 @@ jobs: steps: - uses: actions/checkout@v6 - run: git fetch --depth=1 origin ${{ github.base_ref }} - - uses: oasdiff/oasdiff-action/breaking@v0.0.49 + - uses: oasdiff/oasdiff-action/breaking@v0.0.51 with: base: 'origin/${{ github.base_ref }}:openapi.yaml' revision: 'HEAD:openapi.yaml' @@ -105,7 +106,7 @@ jobs: steps: - uses: actions/checkout@v6 - run: git fetch --depth=1 origin ${{ github.base_ref }} - - uses: oasdiff/oasdiff-action/changelog@v0.0.49 + - uses: oasdiff/oasdiff-action/changelog@v0.0.51 with: base: 'origin/${{ github.base_ref }}:openapi.yaml' revision: 'HEAD:openapi.yaml' @@ -144,7 +145,7 @@ jobs: steps: - uses: actions/checkout@v6 - run: git fetch --depth=1 origin ${{ github.base_ref }} - - uses: oasdiff/oasdiff-action/diff@v0.0.49 + - uses: oasdiff/oasdiff-action/diff@v0.0.51 with: base: 'origin/${{ github.base_ref }}:openapi.yaml' revision: 'HEAD:openapi.yaml' @@ -178,7 +179,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: oasdiff/oasdiff-action/validate@v0.0.49 + - uses: oasdiff/oasdiff-action/validate@v0.0.51 with: spec: 'openapi.yaml' ``` @@ -218,7 +219,7 @@ The actions read this file from the runner's `$GITHUB_WORKSPACE` (which `actions **Explicit path**: if your config lives somewhere else, set `OASDIFF_CONFIG` in the workflow `env:` to point at it: ```yaml -- uses: oasdiff/oasdiff-action/breaking@v0.0.49 +- uses: oasdiff/oasdiff-action/breaking@v0.0.51 env: OASDIFF_CONFIG: ./config/oasdiff.yaml with: @@ -270,7 +271,7 @@ jobs: steps: - uses: actions/checkout@v6 - run: git fetch --depth=1 origin ${{ github.base_ref }} - - uses: oasdiff/oasdiff-action/pr-comment@v0.0.49 + - uses: oasdiff/oasdiff-action/pr-comment@v0.0.51 with: base: 'origin/${{ github.base_ref }}:openapi.yaml' revision: 'HEAD:openapi.yaml' @@ -298,3 +299,56 @@ Each **Review** link opens a hosted page with a side-by-side spec diff and **App | `allow-external-refs` | `false` | Resolve external `$ref`s. Defaults to `false` to prevent SSRF on untrusted pull requests. Set `true` if your spec references external URLs or loads split files by file path | `true`, `false` | [Get oasdiff Pro →](https://www.oasdiff.com/pricing) + +## Pro: Verify your setup + +`oasdiff/oasdiff-action/verify` is a read-only check that confirms your setup works end to end. It posts no PR comment and sets no commit status. Run it on demand from the **Actions** tab (the "Run workflow" button). + +Add it to the same workflow as `pr-comment`, guarded by event type, so one file handles both: `pr-comment` on pull requests, and `verify` when you click "Run workflow". + +```yaml +name: oasdiff +on: + pull_request: + branches: [ "main" ] + workflow_dispatch: +jobs: + pr-comment: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - run: git fetch --depth=1 origin ${{ github.base_ref }} + - uses: oasdiff/oasdiff-action/pr-comment@v0.0.52 + with: + base: 'origin/${{ github.base_ref }}:openapi.yaml' + revision: 'HEAD:openapi.yaml' + oasdiff-token: ${{ secrets.OASDIFF_TOKEN }} + verify: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - run: git fetch --depth=1 origin ${{ github.event.repository.default_branch }} + - uses: oasdiff/oasdiff-action/verify@v0.0.52 + with: + base: 'origin/${{ github.event.repository.default_branch }}:openapi.yaml' + revision: 'HEAD:openapi.yaml' + oasdiff-token: ${{ secrets.OASDIFF_TOKEN }} +``` + +The verify run renders a checklist in the workflow **Step Summary**: + +- ✅ GitHub Actions workflow is running +- ✅ Connected to oasdiff (your `OASDIFF_TOKEN` secret) +- ✅ oasdiff GitHub App installed on the repo +- ✅ OpenAPI spec found and compared + +It exits non-zero with a one-line hint for any check that fails, so a verify run is a clear pass/fail. (Reviewer access is checked separately on your setup page.) + +| Input | Default | Description | Accepted values | +|---|---|---|---| +| `base` | — (required) | Path to the base (old) OpenAPI spec | file path, URL, git ref | +| `revision` | — (required) | Path to the revised (new) OpenAPI spec | file path, URL, git ref | +| `oasdiff-token` | — (required) | oasdiff API token, [sign up at oasdiff.com](https://www.oasdiff.com/pricing) | — | +| `allow-external-refs` | `false` | Resolve external `$ref`s. Defaults to `false`; set `true` if your spec references external URLs | `true`, `false` | diff --git a/release.sh b/release.sh index a98b7de..c101c24 100755 --- a/release.sh +++ b/release.sh @@ -11,7 +11,7 @@ set -e REPO_DIR="$(cd "$(dirname "$0")" && pwd)" -DOCKERFILES="breaking/Dockerfile changelog/Dockerfile diff/Dockerfile validate/Dockerfile pr-comment/Dockerfile" +DOCKERFILES="breaking/Dockerfile changelog/Dockerfile diff/Dockerfile validate/Dockerfile pr-comment/Dockerfile verify/Dockerfile" # ── Resolve action version ─────────────────────────────────────────────────── diff --git a/verify/Dockerfile b/verify/Dockerfile new file mode 100644 index 0000000..5ecf585 --- /dev/null +++ b/verify/Dockerfile @@ -0,0 +1,5 @@ +FROM tufin/oasdiff:v1.18.1 +RUN apk add --no-cache curl jq +ENV PLATFORM github-action +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/verify/action.yml b/verify/action.yml new file mode 100644 index 0000000..d081623 --- /dev/null +++ b/verify/action.yml @@ -0,0 +1,29 @@ +name: 'OpenAPI Spec: Verify oasdiff setup' +description: 'Verify your oasdiff Pro setup end to end. Read-only: posts no PR comment and sets no commit status.' +inputs: + base: + description: 'Path of original OpenAPI spec in YAML or JSON format' + required: true + revision: + description: 'Path of revised OpenAPI spec in YAML or JSON format' + required: true + oasdiff-token: + description: 'oasdiff API token — sign up at oasdiff.com/pricing' + required: true + service-url: + description: 'oasdiff service base URL (override for testing)' + required: false + default: 'https://api.oasdiff.com' + allow-external-refs: + description: 'Allow external $refs in the spec; defaults to false (safe for CI on untrusted pull requests).' + required: false + default: 'false' +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.base }} + - ${{ inputs.revision }} + - ${{ inputs.oasdiff-token }} + - ${{ inputs.service-url }} + - ${{ inputs.allow-external-refs }} diff --git a/verify/entrypoint.sh b/verify/entrypoint.sh new file mode 100755 index 0000000..7be21a1 --- /dev/null +++ b/verify/entrypoint.sh @@ -0,0 +1,155 @@ +#!/bin/sh +set -e + +# oasdiff "verify installation" run. +# +# A read-only setup check: it posts NO PR comment and sets NO commit status. +# Meant to be triggered manually (workflow_dispatch) from the GitHub Actions UI. +# It renders a progressive checklist in the workflow Step Summary: +# +# 1. Workflow runs (implicit: this run is executing) +# 2. Connected to oasdiff (the OASDIFF_TOKEN secret authenticated) +# 3. App installed (the oasdiff GitHub App is installed on this repo) +# 4. Spec found (oasdiff resolved base + revision and ran the diff) +# +# Reviewer access (signing in on oasdiff.com) is verified separately on the +# setup page, not here. + +if [ -n "$GITHUB_WORKSPACE" ]; then + git config --global --get-all safe.directory | grep -q "$GITHUB_WORKSPACE" || \ + git config --global --add safe.directory "$GITHUB_WORKSPACE" +fi + +readonly base="$1" +readonly revision="$2" +readonly oasdiff_token="$3" +readonly service_url="${4:-https://api.oasdiff.com}" +readonly allow_external_refs="$5" + +echo "Verifying oasdiff setup — base: $base, revision: $revision (no comment will be posted)" + +flags="" +if [ -n "$allow_external_refs" ]; then + flags="$flags --allow-external-refs=$allow_external_refs" +fi + +# Check 4 (spec found): can oasdiff resolve base + revision (including any +# in-repo / relative $refs, which load via git show when the spec path uses +# the git-revision format ":") and run the diff? We don't need +# the output, only the exit code. Tolerate a non-zero exit (set -e) so we can +# report the failure rather than abort. +# +# Exit 123 is the dedicated "external $ref refused" code (oasdiff v1.18.1): the +# spec loaded but an external $ref was blocked because allow-external-refs is +# false (the safe default). That's a distinct case from "spec not found" — the +# fix is to allow external refs for a trusted spec, not to fix the path. +_err=$(mktemp) +specs_found=false +external_ref_blocked=false +set +e +oasdiff changelog "$base" "$revision" --format json $flags >/dev/null 2>"$_err" +oasdiff_exit=$? +set -e +if [ "$oasdiff_exit" -eq 0 ]; then + specs_found=true +elif [ "$oasdiff_exit" -eq 123 ]; then + external_ref_blocked=true +fi + +owner="${GITHUB_REPOSITORY%%/*}" +repo="${GITHUB_REPOSITORY#*/}" + +if [ -z "$oasdiff_token" ]; then + echo "::error title=oasdiff verify::No oasdiff-token provided. Add your OASDIFF_TOKEN as a repository secret, then re-run." + exit 1 +fi + +# POST the outcome to the service. Reaching a 2xx proves the token authenticated +# (check 2); the response carries the App-installation result (check 3). +payload=$(jq -nc \ + --arg owner "$owner" \ + --arg repo "$repo" \ + --argjson specs_found "$specs_found" \ + --arg base_file "$base" \ + --arg revision_file "$revision" \ + '{owner: $owner, repo: $repo, specs_found: $specs_found, base_file: $base_file, revision_file: $revision_file}') + +response=$(printf '%s' "$payload" | curl -s -w "\n%{http_code}" -X POST \ + "${service_url}/tenants/${oasdiff_token}/verify" \ + -H "Content-Type: application/json" \ + --data-binary @-) +http_code=$(echo "$response" | tail -1) +body=$(echo "$response" | sed '$d') +[ -z "$http_code" ] && http_code=0 + +# Resolve each check from the response. token_ok is true on any 2xx (the request +# authenticated); 401/403 means the secret is wrong/missing or the tenant is +# inactive. app_installed comes from the service probe. +if [ "$http_code" -ge 200 ] 2>/dev/null && [ "$http_code" -lt 300 ] 2>/dev/null; then + token_ok=true + app_installed=$(echo "$body" | jq -r '.app_installed // false' 2>/dev/null) + # specs_found / external_ref_blocked stay as determined locally above; the + # service only echoes specs_found, and it can't see the external-ref case. +elif [ "$http_code" = "401" ] || [ "$http_code" = "403" ]; then + token_ok=false + app_installed=unknown +else + echo "::error title=oasdiff verify::the oasdiff service returned HTTP $http_code" + echo "$body" >&2 + token_ok=unknown + app_installed=unknown +fi + +# Render the progressive checklist. +mark() { # $1 = true|false|unknown, $2 = label + case "$1" in + true) echo "- ✅ $2" ;; + false) echo "- ❌ $2" ;; + *) echo "- ⬜ $2 (could not determine)" ;; + esac +} + +{ + echo "## oasdiff setup verification" + echo "" + mark true "GitHub Actions workflow is running" + mark "$token_ok" "Connected to oasdiff (OASDIFF_TOKEN secret)" + mark "$app_installed" "oasdiff GitHub App installed on ${owner}/${repo}" + if [ "$specs_found" = "true" ]; then + echo "- ✅ OpenAPI spec found and compared" + elif [ "$external_ref_blocked" = "true" ]; then + echo "- ❌ OpenAPI spec found, but an external \$ref was blocked" + else + echo "- ❌ OpenAPI spec found and compared" + fi + echo "" + if [ "$token_ok" = "false" ]; then + echo "> **Connect to oasdiff:** the \`OASDIFF_TOKEN\` repository secret is missing or wrong. Copy it from your oasdiff setup page into repo Settings → Secrets and variables → Actions." + fi + if [ "$app_installed" = "false" ]; then + echo "> **Install the App:** the oasdiff GitHub App is not installed on \`${owner}/${repo}\`. Install it at https://github.com/apps/oasdiff/installations/new (an org owner may need to approve it)." + fi + if [ "$external_ref_blocked" = "true" ]; then + echo "> **External \$ref blocked:** your spec resolves an external \`\$ref\`, disabled by default to prevent SSRF on untrusted pull requests. If the spec is trusted, set \`allow-external-refs: true\` on the action step." + elif [ "$specs_found" = "false" ]; then + echo "> **Spec not found:** oasdiff could not resolve \`$base\` / \`$revision\`. Check the \`base\`/\`revision\` paths (multi-file specs need their referenced files present — use the \`origin/:\` git-revision spec path format so in-repo \$refs resolve via git)." + [ -s "$_err" ] && echo "> \`\`\`" && tr '\n' ' ' < "$_err" | cut -c1-500 && echo "" && echo "> \`\`\`" + fi + echo "" + echo "_Reviewer access (signing in on oasdiff.com) is verified separately on your setup page._" +} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +# Surface a one-line annotation for each red check. +[ "$token_ok" = "false" ] && echo "::error title=oasdiff verify::OASDIFF_TOKEN secret is missing or invalid." +[ "$app_installed" = "false" ] && echo "::error title=oasdiff verify::oasdiff GitHub App is not installed on ${owner}/${repo}." +[ "$external_ref_blocked" = "true" ] && echo "::error title=oasdiff verify::Spec uses an external \$ref, blocked by default. Set allow-external-refs: true if the spec is trusted." +[ "$specs_found" = "false" ] && [ "$external_ref_blocked" = "false" ] && echo "::error title=oasdiff verify::OpenAPI spec not found at the configured base/revision path." + +# Exit non-zero if any bot-chain check is not green, so the verify run is a +# clear red/green signal. token_ok unknown (transient service error) also fails. +if [ "$token_ok" = "true" ] && [ "$app_installed" = "true" ] && [ "$specs_found" = "true" ]; then + echo "✅ oasdiff setup verified — comments will post on every PR." + exit 0 +fi +echo "Setup not complete yet — see the checklist in the run summary." +exit 1