test: run the real code instead of hand-copied mirrors (sable-1drb)

.github/workflows/test-github-action.yml

@@ -227,6 +227,183 @@ jobs:
           fi
           echo "PASS: the results fetch retried and completed."
 
+  # sable-fgk7 — the results step used to coerce EVERY failure to read the
+  # report into findings_count=0, which passes every severity threshold and
+  # renders ":white_check_mark: No security findings detected". A report the
+  # action cannot read is not a clean scan. Three shapes, each of which used
+  # to land as a clean green: schema-valid-but-wrong (parses, no key), not
+  # JSON at all, and a 200 whose body is an error object.
+  test-results-unreadable-is-not-clean:
+    name: "Results: an unreadable report is not a clean scan (${{ matrix.shape }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shape: [missing-key, not-json, error-object]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start mock backend (poll completes; results body is ${{ matrix.shape }})
+        env:
+          PORT: '8791'
+          FAIL_COUNT: '0'
+          COMPLETE_AFTER: '1'
+          RESULTS_SHAPE: ${{ matrix.shape }}
+        run: |
+          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null && break
+            sleep 1
+          done
+          curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null || {
+            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }
+
+      - name: Run the action against the mock
+        id: scan
+        continue-on-error: true
+        uses: ./github-action
+        with:
+          api-key: 'not-a-real-key'
+          rafter-url: 'http://127.0.0.1:8791'
+          timeout-minutes: '2'
+          upload-sarif: 'false'
+          comment-on-pr: 'false'
+
+      - name: Assert the build failed and no count was fabricated
+        run: |
+          cat mock.log
+          FAIL=0
+          if [ "${{ steps.scan.outcome }}" != "failure" ]; then
+            echo "FAIL: an unreadable report must fail the build (outcome='${{ steps.scan.outcome }}')."
+            FAIL=1
+          fi
+          if [ "${{ steps.scan.outputs.status }}" != "unreadable" ]; then
+            echo "FAIL: expected status=unreadable, got '${{ steps.scan.outputs.status }}'."
+            FAIL=1
+          fi
+          # The floor: a count that was never computed must be ABSENT, not 0.
+          # '0' here is the bug — it is what a consumer gating on the output
+          # reads as a clean scan.
+          if [ -n "${{ steps.scan.outputs.findings-count }}" ]; then
+            echo "FAIL: findings-count was fabricated as '${{ steps.scan.outputs.findings-count }}' from an unreadable report."
+            FAIL=1
+          fi
+          [ "$FAIL" -eq 0 ] && echo "PASS: unreadable report (${{ matrix.shape }}) failed the build with status=unreadable and no counts."
+          exit $FAIL
+
+  # The other half of the floor: when the report IS readable the counts must be
+  # exactly the report's and the build must pass. Without this, a "validation"
+  # that rejected everything would also land green above.
+  test-results-counts-exact:
+    name: "Results: counts are exactly the report's"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start mock backend (poll completes; report has 3 findings)
+        env:
+          PORT: '8792'
+          FAIL_COUNT: '0'
+          COMPLETE_AFTER: '1'
+          RESULTS_SHAPE: 'with-findings'
+        run: |
+          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null && break
+            sleep 1
+          done
+          curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null || {
+            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }
+
+      - name: Run the action against the mock
+        id: scan
+        continue-on-error: true
+        uses: ./github-action
+        with:
+          api-key: 'not-a-real-key'
+          rafter-url: 'http://127.0.0.1:8792'
+          timeout-minutes: '2'
+          upload-sarif: 'false'
+          comment-on-pr: 'false'
+
+      - name: Assert every count is the report's, not a default
+        run: |
+          cat mock.log
+          FAIL=0
+          check() {
+            if [ "$2" != "$3" ]; then
+              echo "FAIL: $1 expected '$3', got '$2'"
+              FAIL=1
+            fi
+          }
+          check outcome         "${{ steps.scan.outcome }}"                 "success"
+          check status          "${{ steps.scan.outputs.status }}"          "completed"
+          check findings-count  "${{ steps.scan.outputs.findings-count }}"  "3"
+          check critical-count  "${{ steps.scan.outputs.critical-count }}"  "1"
+          check high-count      "${{ steps.scan.outputs.high-count }}"      "1"
+          check medium-count    "${{ steps.scan.outputs.medium-count }}"    "0"
+          check low-count       "${{ steps.scan.outputs.low-count }}"       "1"
+          [ "$FAIL" -eq 0 ] && echo "PASS: counts are exactly the report's (3/1/1/0/1)."
+          exit $FAIL
+
+  # sable-1drb — the threshold gate is now unit-tested against the real
+  # lib/severity.sh, but a unit test cannot prove action.yml WIRES it: that
+  # the counts reach the gate and the gate's verdict reaches the job. One
+  # end-to-end run with real findings and a threshold they exceed does.
+  test-threshold-gate-end-to-end:
+    name: "Threshold gate: real findings above the threshold fail the build"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start mock backend (report has 1 critical, 1 high, 1 low)
+        env:
+          PORT: '8793'
+          FAIL_COUNT: '0'
+          COMPLETE_AFTER: '1'
+          RESULTS_SHAPE: 'with-findings'
+        run: |
+          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null && break
+            sleep 1
+          done
+          curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null || {
+            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }
+
+      - name: Run the action with severity-threshold high
+        id: scan
+        continue-on-error: true
+        uses: ./github-action
+        with:
+          api-key: 'not-a-real-key'
+          rafter-url: 'http://127.0.0.1:8793'
+          timeout-minutes: '2'
+          upload-sarif: 'false'
+          comment-on-pr: 'false'
+          severity-threshold: 'high'
+
+      - name: Assert the gate, not an error, failed the build
+        run: |
+          cat mock.log
+          FAIL=0
+          # status=completed AND outcome=failure is the gate's signature: the
+          # report was read and counted, then the threshold rejected it.
+          if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then
+            echo "FAIL: expected status=completed (report read), got '${{ steps.scan.outputs.status }}'"
+            FAIL=1
+          fi
+          if [ "${{ steps.scan.outcome }}" != "failure" ]; then
+            echo "FAIL: 1 critical + 1 high with severity-threshold=high must fail the build (outcome='${{ steps.scan.outcome }}')"
+            FAIL=1
+          fi
+          if [ "${{ steps.scan.outputs.findings-count }}" != "3" ]; then
+            echo "FAIL: findings-count expected 3, got '${{ steps.scan.outputs.findings-count }}'"
+            FAIL=1
+          fi
+          [ "$FAIL" -eq 0 ] && echo "PASS: counts reached the gate and the gate failed the build."
+          exit $FAIL
+
   test-yaml-validity:
     name: action.yml is valid YAML
     runs-on: ubuntu-latest

github-action/README.md

@@ -53,12 +53,12 @@ jobs:
 | Output | Description |
 |--------|-------------|
 | `scan-id` | The Rafter scan ID |
-| `findings-count` | Total findings |
+| `findings-count` | Total findings. Empty, never `0`, when the report could not be read (see `status`) |
 | `critical-count` | Critical severity findings |
 | `high-count` | High severity findings |
 | `medium-count` | Medium severity findings |
 | `low-count` | Low severity findings |
-| `status` | Scan status |
+| `status` | `completed`, `failed`, `timeout`, `unreadable` (the scan may have finished but its report could not be read or parsed), or `unreachable` (the API could not be contacted). Count outputs are only written when `completed` |
 
 ## Examples
 

github-action/action.yml

@@ -303,13 +303,28 @@ runs:
         fetch_results "${{ runner.temp }}/rafter-results.md"    "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}&format=md"
         fetch_results "${{ runner.temp }}/rafter-results.sarif" "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}&format=sarif"
 
-        # Extract counts
+        # A report this step cannot read is NOT a clean scan (sable-fgk7).
+        # Every count below used to fall back to 0 on any jq failure, so a
+        # malformed body, a truncated write, or a 200 carrying an error object
+        # rendered as ":white_check_mark: No security findings detected" and
+        # passed every severity threshold. Validate the shape first. Once it
+        # holds, the count expressions cannot fail and need no fallback; if
+        # jq itself is broken the step fails, which is the correct outcome.
         RESULTS="${{ runner.temp }}/rafter-results.json"
-        FINDINGS_COUNT=$(jq '.vulnerabilities | length' "$RESULTS" 2>/dev/null || echo "0")
-        CRITICAL_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' "$RESULTS" 2>/dev/null || echo "0")
-        HIGH_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "error" or .severity == "high")] | length' "$RESULTS" 2>/dev/null || echo "0")
-        MEDIUM_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "warning" or .severity == "medium")] | length' "$RESULTS" 2>/dev/null || echo "0")
-        LOW_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "note" or .severity == "low")] | length' "$RESULTS" 2>/dev/null || echo "0")
+        if ! jq -e 'type == "object" and (.vulnerabilities | type == "array") and all(.vulnerabilities[]; type == "object")' "$RESULTS" >/dev/null 2>&1; then
+          SNIPPET=$(head -c 300 "$RESULTS" 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true)
+          echo "::error::Rafter returned a report for scan ${SCAN_ID} that this action cannot read: no 'vulnerabilities' array."
+          echo "::error::A report that cannot be parsed is not a clean scan, so no counts were produced. Check the scan in your dashboard at ${RAFTER_URL}/dashboard or retry with: rafter get ${SCAN_ID}"
+          echo "::error::Body started with: ${SNIPPET}"
+          echo "status=unreadable" >> "$GITHUB_OUTPUT"
+          exit 1
+        fi
+
+        FINDINGS_COUNT=$(jq '.vulnerabilities | length' "$RESULTS")
+        CRITICAL_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' "$RESULTS")
+        HIGH_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "error" or .severity == "high")] | length' "$RESULTS")
+        MEDIUM_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "warning" or .severity == "medium")] | length' "$RESULTS")
+        LOW_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "note" or .severity == "low")] | length' "$RESULTS")
 
         echo "findings_count=${FINDINGS_COUNT}" >> "$GITHUB_OUTPUT"
         echo "critical_count=${CRITICAL_COUNT}" >> "$GITHUB_OUTPUT"
@@ -341,6 +356,9 @@ runs:
         LOW_COUNT: ${{ steps.results.outputs.low_count }}
         SEVERITY_THRESHOLD: ${{ inputs.severity-threshold }}
       run: |
+        # Shared with the tests under tests/ — see lib/severity.sh (sable-1drb).
+        source "${{ github.action_path }}/lib/severity.sh"
+
         MD_REPORT=$(jq -r '.markdown // empty' "${{ runner.temp }}/rafter-results.md" 2>/dev/null || cat "${{ runner.temp }}/rafter-results.md")
 
         if [ "$FINDINGS_COUNT" -eq 0 ]; then
@@ -379,10 +397,7 @@ runs:
           echo ""
           echo "</details>"
           echo ""
-          if [ "$FINDINGS_COUNT" -gt 0 ] && [ "$SEVERITY_THRESHOLD" = "none" ]; then
-            echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow."
-            echo ""
-          fi
+          rafter_report_only_tip "$FINDINGS_COUNT" "$SEVERITY_THRESHOLD"
           echo "---"
           echo "<sub>Scan ID: ${SCAN_ID} | Powered by [Rafter](https://rafter.so)</sub>"
         } >> "$COMMENT_FILE"
@@ -417,31 +432,11 @@ runs:
         MEDIUM_COUNT: ${{ steps.results.outputs.medium_count }}
         LOW_COUNT: ${{ steps.results.outputs.low_count }}
       run: |
-        FAIL=0
-
-        case "$SEVERITY_THRESHOLD" in
-          critical)
-            [ "$CRITICAL_COUNT" -gt 0 ] && FAIL=1
-            ;;
-          high)
-            [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1
-            ;;
-          medium)
-            [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] && FAIL=1
-            ;;
-          low)
-            [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] || [ "$LOW_COUNT" -gt 0 ] && FAIL=1
-            ;;
-          none)
-            FAIL=0
-            ;;
-          *)
-            echo "::warning::Unknown severity threshold '${SEVERITY_THRESHOLD}', defaulting to 'high'"
-            [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1
-            ;;
-        esac
+        # The case statement lives in lib/severity.sh so the unit tests under
+        # tests/ run the same code, not a transcription of it (sable-1drb).
+        source "${{ github.action_path }}/lib/severity.sh"
 
-        if [ "$FAIL" -eq 1 ]; then
+        if rafter_threshold_fails "$SEVERITY_THRESHOLD" "$CRITICAL_COUNT" "$HIGH_COUNT" "$MEDIUM_COUNT" "$LOW_COUNT"; then
           echo "::error::Security findings exceed severity threshold '${SEVERITY_THRESHOLD}'"
           exit 1
         fi

github-action/lib/severity.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Severity-threshold logic shared by github-action/action.yml and the tests
+# under github-action/tests/. ONE copy, sourced by both, so the tests exercise
+# the code the action runs rather than a transcription of it (sable-1drb).
+#
+# Sourced, never executed: no `set -e`, no side effects at load time. Every
+# function takes explicit arguments so a test can call it without staging
+# environment variables, and prints only what the action wants in its log.
+
+# rafter_threshold_fails THRESHOLD CRITICAL HIGH MEDIUM LOW
+#
+# Returns 0 when the findings exceed THRESHOLD (the build should fail) and 1
+# otherwise. 'none' never fails. An unrecognised threshold behaves like
+# 'high' and says so with a ::warning:: annotation.
+rafter_threshold_fails() {
+  local threshold="$1" critical="$2" high="$3" medium="$4" low="$5"
+  local fail=0
+  case "$threshold" in
+    critical)
+      [ "$critical" -gt 0 ] && fail=1
+      ;;
+    high)
+      [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] && fail=1
+      ;;
+    medium)
+      [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] || [ "$medium" -gt 0 ] && fail=1
+      ;;
+    low)
+      [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] || [ "$medium" -gt 0 ] || [ "$low" -gt 0 ] && fail=1
+      ;;
+    none)
+      fail=0
+      ;;
+    *)
+      echo "::warning::Unknown severity threshold '${threshold}', defaulting to 'high'"
+      [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] && fail=1
+      ;;
+  esac
+  [ "$fail" -eq 1 ]
+}
+
+# rafter_report_only_tip FINDINGS_COUNT THRESHOLD
+#
+# Prints the report-only tip block for the PR comment iff there are findings
+# AND the threshold is 'none' (the default), i.e. the run reported problems
+# but was configured never to fail on them. Prints nothing otherwise.
+rafter_report_only_tip() {
+  local findings="$1" threshold="$2"
+  if [ "$findings" -gt 0 ] && [ "$threshold" = "none" ]; then
+    echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow."
+    echo ""
+  fi
+}