test-github-action.yml

name: Test github-action/ Composite Action

# Runs on every PR touching github-action/ (the cloud-API variant) and on
# the same paths landing on main. The Rafter API is NOT exercised — these
# are pure-bash unit tests of the threshold-eval and PR-comment logic
# plus a drift detector on action.yml's load-bearing defaults.
#
# The poll-path jobs DO drive the action end to end, against a localhost mock
# backend (github-action/tests/mock-rafter-api.py) rather than the real API, so
# they need no API key and spend no credits.
on:
  push:
    branches:
      - main
    paths:
      - 'github-action/**'
      - '.github/workflows/test-github-action.yml'
  pull_request:
    paths:
      - 'github-action/**'
      - '.github/workflows/test-github-action.yml'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  test-threshold-evaluation:
    name: Threshold-evaluation case statement
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run threshold-eval unit test
        run: bash github-action/tests/test-threshold-eval.sh

  test-pr-comment-tip:
    name: PR-comment report-only tip
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run PR-comment-tip unit test
        run: bash github-action/tests/test-pr-comment-tip.sh

  test-action-yml-defaults:
    name: action.yml drift detector
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run action.yml defaults / drift check
        run: bash github-action/tests/test-action-yml-defaults.sh

  # sable-l10k — a paying customer's run died on a single transient 500 during
  # polling ("Failed to fetch report from storage: Object not found"). The
  # report is not durable the instant a scan flips to completed, so that 500 is
  # survivable and must be retried. These two jobs pin both halves of the
  # contract: ride out the transient failure, still fail on a missing report.
  test-poll-transient-500:
    name: "Poll: rides out a transient 500"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (500 on poll #2, then healthy)
        env:
          PORT: '8787'
          FAIL_ON: '2'
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8787/api/static/scan >/dev/null && break
            sleep 1
          done

      - name: Run the action against the mock
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8787'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'

      - name: Assert the scan survived the 500
        run: |
          cat mock.log
          echo "status output: '${{ steps.scan.outputs.status }}'"
          if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then
            echo "FAIL: a single transient 500 during polling killed the run."
            exit 1
          fi
          echo "PASS: the action retried the transient 500 and completed."

  test-poll-report-never-readable:
    name: "Poll: fails clearly when the report is really missing"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (every poll 500s)
        env:
          PORT: '8788'
          FAIL_ON: '2'
          FAIL_FOREVER: '1'
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8788/api/static/scan >/dev/null && break
            sleep 1
          done

      - name: Run the action against the mock
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8788'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'

      - name: Assert it failed, with the right status
        run: |
          cat mock.log
          FAIL=0
          if [ "${{ steps.scan.outputs.status }}" != "unreadable" ]; then
            echo "FAIL: expected status=unreadable, got '${{ steps.scan.outputs.status }}'"
            FAIL=1
          fi
          if [ "${{ steps.scan.outcome }}" != "failure" ]; then
            echo "FAIL: an unreadable report should fail the build."
            FAIL=1
          fi
          # A composite action's log is not capturable from the calling step,
          # so the CONTENT of the give-up message is asserted by the drift
          # detector (github-action/tests/test-action-yml-defaults.sh) instead.
          exit $FAIL

  # The 404-as-transient branch is the subtlest thing in the poll loop: it is
  # correct only because the trigger step has already handed us a scan_id.
  # Nothing else in CI exercises it, so a "simplification" that drops `-eq 404`
  # from the transient condition would otherwise land green.
  test-poll-transient-404:
    name: "Poll: rides out a transient 404"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (404 on poll #2, then healthy)
        env:
          PORT: '8789'
          FAIL_ON: '2'
          FAIL_STATUS: '404'
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8789/api/static/scan >/dev/null && break
            sleep 1
          done
          curl -sf -X POST -d '{}' http://127.0.0.1:8789/api/static/scan >/dev/null || {
            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }

      - name: Run the action against the mock
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8789'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'

      - name: Assert the scan survived the 404
        run: |
          cat mock.log
          if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then
            echo "FAIL: a transient 404 mid-poll killed the run."
            exit 1
          fi
          echo "PASS: the action treated a mid-poll 404 as read-after-write lag."

  # The results fetch runs the instant the scan reports completed — the
  # likeliest moment for the report object to be unreadable. Its retry loop had
  # no coverage at all, and it is where a failed read used to be reported to
  # consumers as status=completed.
  test-results-fetch-transient-500:
    name: "Results fetch: rides out a transient 500"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (poll succeeds, first results fetch 500s)
        env:
          PORT: '8790'
          FAIL_ON: '2'
          FAIL_COUNT: '1'
          COMPLETE_AFTER: '1'
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8790/api/static/scan >/dev/null && break
            sleep 1
          done
          curl -sf -X POST -d '{}' http://127.0.0.1:8790/api/static/scan >/dev/null || {
            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }

      - name: Run the action against the mock
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8790'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'

      - name: Assert the results fetch retried rather than failing the build
        run: |
          cat mock.log
          if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then
            echo "FAIL: a transient 500 on the results fetch killed the run (status='${{ steps.scan.outputs.status }}')."
            exit 1
          fi
          echo "PASS: the results fetch retried and completed."

  # sable-fgk7 — the results step used to coerce EVERY failure to read the
  # report into findings_count=0, which passes every severity threshold and
  # renders ":white_check_mark: No security findings detected". A report the
  # action cannot read is not a clean scan. Three shapes, each of which used
  # to land as a clean green: schema-valid-but-wrong (parses, no key), not
  # JSON at all, and a 200 whose body is an error object.
  test-results-unreadable-is-not-clean:
    name: "Results: an unreadable report is not a clean scan (${{ matrix.shape }})"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        shape: [missing-key, not-json, error-object]
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (poll completes; results body is ${{ matrix.shape }})
        env:
          PORT: '8791'
          FAIL_COUNT: '0'
          COMPLETE_AFTER: '1'
          RESULTS_SHAPE: ${{ matrix.shape }}
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null && break
            sleep 1
          done
          curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null || {
            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }

      - name: Run the action against the mock
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8791'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'

      - name: Assert the build failed and no count was fabricated
        run: |
          cat mock.log
          FAIL=0
          if [ "${{ steps.scan.outcome }}" != "failure" ]; then
            echo "FAIL: an unreadable report must fail the build (outcome='${{ steps.scan.outcome }}')."
            FAIL=1
          fi
          if [ "${{ steps.scan.outputs.status }}" != "unreadable" ]; then
            echo "FAIL: expected status=unreadable, got '${{ steps.scan.outputs.status }}'."
            FAIL=1
          fi
          # The floor: a count that was never computed must be ABSENT, not 0.
          # '0' here is the bug — it is what a consumer gating on the output
          # reads as a clean scan.
          if [ -n "${{ steps.scan.outputs.findings-count }}" ]; then
            echo "FAIL: findings-count was fabricated as '${{ steps.scan.outputs.findings-count }}' from an unreadable report."
            FAIL=1
          fi
          [ "$FAIL" -eq 0 ] && echo "PASS: unreadable report (${{ matrix.shape }}) failed the build with status=unreadable and no counts."
          exit $FAIL

  # The other half of the floor: when the report IS readable the counts must be
  # exactly the report's and the build must pass. Without this, a "validation"
  # that rejected everything would also land green above.
  test-results-counts-exact:
    name: "Results: counts are exactly the report's"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (poll completes; report has 3 findings)
        env:
          PORT: '8792'
          FAIL_COUNT: '0'
          COMPLETE_AFTER: '1'
          RESULTS_SHAPE: 'with-findings'
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null && break
            sleep 1
          done
          curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null || {
            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }

      - name: Run the action against the mock
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8792'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'

      - name: Assert every count is the report's, not a default
        run: |
          cat mock.log
          FAIL=0
          check() {
            if [ "$2" != "$3" ]; then
              echo "FAIL: $1 expected '$3', got '$2'"
              FAIL=1
            fi
          }
          check outcome         "${{ steps.scan.outcome }}"                 "success"
          check status          "${{ steps.scan.outputs.status }}"          "completed"
          check findings-count  "${{ steps.scan.outputs.findings-count }}"  "3"
          check critical-count  "${{ steps.scan.outputs.critical-count }}"  "1"
          check high-count      "${{ steps.scan.outputs.high-count }}"      "1"
          check medium-count    "${{ steps.scan.outputs.medium-count }}"    "0"
          check low-count       "${{ steps.scan.outputs.low-count }}"       "1"
          [ "$FAIL" -eq 0 ] && echo "PASS: counts are exactly the report's (3/1/1/0/1)."
          exit $FAIL

  # sable-1drb — the threshold gate is now unit-tested against the real
  # lib/severity.sh, but a unit test cannot prove action.yml WIRES it: that
  # the counts reach the gate and the gate's verdict reaches the job. One
  # end-to-end run with real findings and a threshold they exceed does.
  test-threshold-gate-end-to-end:
    name: "Threshold gate: real findings above the threshold fail the build"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Start mock backend (report has 1 critical, 1 high, 1 low)
        env:
          PORT: '8793'
          FAIL_COUNT: '0'
          COMPLETE_AFTER: '1'
          RESULTS_SHAPE: 'with-findings'
        run: |
          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
          for _ in $(seq 1 30); do
            curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null && break
            sleep 1
          done
          curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null || {
            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }

      - name: Run the action with severity-threshold high
        id: scan
        continue-on-error: true
        uses: ./github-action
        with:
          api-key: 'not-a-real-key'
          rafter-url: 'http://127.0.0.1:8793'
          timeout-minutes: '2'
          upload-sarif: 'false'
          comment-on-pr: 'false'
          severity-threshold: 'high'

      - name: Assert the gate, not an error, failed the build
        run: |
          cat mock.log
          FAIL=0
          # status=completed AND outcome=failure is the gate's signature: the
          # report was read and counted, then the threshold rejected it.
          if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then
            echo "FAIL: expected status=completed (report read), got '${{ steps.scan.outputs.status }}'"
            FAIL=1
          fi
          if [ "${{ steps.scan.outcome }}" != "failure" ]; then
            echo "FAIL: 1 critical + 1 high with severity-threshold=high must fail the build (outcome='${{ steps.scan.outcome }}')"
            FAIL=1
          fi
          if [ "${{ steps.scan.outputs.findings-count }}" != "3" ]; then
            echo "FAIL: findings-count expected 3, got '${{ steps.scan.outputs.findings-count }}'"
            FAIL=1
          fi
          [ "$FAIL" -eq 0 ] && echo "PASS: counts reached the gate and the gate failed the build."
          exit $FAIL

  test-yaml-validity:
    name: action.yml is valid YAML
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Validate YAML
        run: |
          python -c "import yaml; yaml.safe_load(open('github-action/action.yml'))" \
            && echo "OK: github-action/action.yml parses as valid YAML"