fix: an unreadable scan report is not a clean scan (sable-fgk7)

.github/workflows/test-github-action.yml

@@ -227,6 +227,125 @@ jobs:
           fi
           echo "PASS: the results fetch retried and completed."
 
+  # sable-fgk7 — the results step used to coerce EVERY failure to read the
+  # report into findings_count=0, which passes every severity threshold and
+  # renders ":white_check_mark: No security findings detected". A report the
+  # action cannot read is not a clean scan. Three shapes, each of which used
+  # to land as a clean green: schema-valid-but-wrong (parses, no key), not
+  # JSON at all, and a 200 whose body is an error object.
+  test-results-unreadable-is-not-clean:
+    name: "Results: an unreadable report is not a clean scan (${{ matrix.shape }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shape: [missing-key, not-json, error-object]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start mock backend (poll completes; results body is ${{ matrix.shape }})
+        env:
+          PORT: '8791'
+          FAIL_COUNT: '0'
+          COMPLETE_AFTER: '1'
+          RESULTS_SHAPE: ${{ matrix.shape }}
+        run: |
+          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null && break
+            sleep 1
+          done
+          curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null || {
+            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }
+
+      - name: Run the action against the mock
+        id: scan
+        continue-on-error: true
+        uses: ./github-action
+        with:
+          api-key: 'not-a-real-key'
+          rafter-url: 'http://127.0.0.1:8791'
+          timeout-minutes: '2'
+          upload-sarif: 'false'
+          comment-on-pr: 'false'
+
+      - name: Assert the build failed and no count was fabricated
+        run: |
+          cat mock.log
+          FAIL=0
+          if [ "${{ steps.scan.outcome }}" != "failure" ]; then
+            echo "FAIL: an unreadable report must fail the build (outcome='${{ steps.scan.outcome }}')."
+            FAIL=1
+          fi
+          if [ "${{ steps.scan.outputs.status }}" != "unreadable" ]; then
+            echo "FAIL: expected status=unreadable, got '${{ steps.scan.outputs.status }}'."
+            FAIL=1
+          fi
+          # The floor: a count that was never computed must be ABSENT, not 0.
+          # '0' here is the bug — it is what a consumer gating on the output
+          # reads as a clean scan.
+          if [ -n "${{ steps.scan.outputs.findings-count }}" ]; then
+            echo "FAIL: findings-count was fabricated as '${{ steps.scan.outputs.findings-count }}' from an unreadable report."
+            FAIL=1
+          fi
+          [ "$FAIL" -eq 0 ] && echo "PASS: unreadable report (${{ matrix.shape }}) failed the build with status=unreadable and no counts."
+          exit $FAIL
+
+  # The other half of the floor: when the report IS readable the counts must be
+  # exactly the report's and the build must pass. Without this, a "validation"
+  # that rejected everything would also land green above.
+  test-results-counts-exact:
+    name: "Results: counts are exactly the report's"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start mock backend (poll completes; report has 3 findings)
+        env:
+          PORT: '8792'
+          FAIL_COUNT: '0'
+          COMPLETE_AFTER: '1'
+          RESULTS_SHAPE: 'with-findings'
+        run: |
+          nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null && break
+            sleep 1
+          done
+          curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null || {
+            echo "FAIL: mock backend never started"; cat mock.log; exit 1; }
+
+      - name: Run the action against the mock
+        id: scan
+        continue-on-error: true
+        uses: ./github-action
+        with:
+          api-key: 'not-a-real-key'
+          rafter-url: 'http://127.0.0.1:8792'
+          timeout-minutes: '2'
+          upload-sarif: 'false'
+          comment-on-pr: 'false'
+
+      - name: Assert every count is the report's, not a default
+        run: |
+          cat mock.log
+          FAIL=0
+          check() {
+            if [ "$2" != "$3" ]; then
+              echo "FAIL: $1 expected '$3', got '$2'"
+              FAIL=1
+            fi
+          }
+          check outcome         "${{ steps.scan.outcome }}"                 "success"
+          check status          "${{ steps.scan.outputs.status }}"          "completed"
+          check findings-count  "${{ steps.scan.outputs.findings-count }}"  "3"
+          check critical-count  "${{ steps.scan.outputs.critical-count }}"  "1"
+          check high-count      "${{ steps.scan.outputs.high-count }}"      "1"
+          check medium-count    "${{ steps.scan.outputs.medium-count }}"    "0"
+          check low-count       "${{ steps.scan.outputs.low-count }}"       "1"
+          [ "$FAIL" -eq 0 ] && echo "PASS: counts are exactly the report's (3/1/1/0/1)."
+          exit $FAIL
+
   test-yaml-validity:
     name: action.yml is valid YAML
     runs-on: ubuntu-latest

github-action/README.md

@@ -53,12 +53,12 @@ jobs:
 | Output | Description |
 |--------|-------------|
 | `scan-id` | The Rafter scan ID |
-| `findings-count` | Total findings |
+| `findings-count` | Total findings. Empty, never `0`, when the report could not be read (see `status`) |
 | `critical-count` | Critical severity findings |
 | `high-count` | High severity findings |
 | `medium-count` | Medium severity findings |
 | `low-count` | Low severity findings |
-| `status` | Scan status |
+| `status` | `completed`, `failed`, `timeout`, `unreadable` (the scan may have finished but its report could not be read or parsed), or `unreachable` (the API could not be contacted). Count outputs are only written when `completed` |
 
 ## Examples
 

github-action/action.yml

@@ -303,13 +303,28 @@ runs:
         fetch_results "${{ runner.temp }}/rafter-results.md"    "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}&format=md"
         fetch_results "${{ runner.temp }}/rafter-results.sarif" "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}&format=sarif"
 
-        # Extract counts
+        # A report this step cannot read is NOT a clean scan (sable-fgk7).
+        # Every count below used to fall back to 0 on any jq failure, so a
+        # malformed body, a truncated write, or a 200 carrying an error object
+        # rendered as ":white_check_mark: No security findings detected" and
+        # passed every severity threshold. Validate the shape first. Once it
+        # holds, the count expressions cannot fail and need no fallback; if
+        # jq itself is broken the step fails, which is the correct outcome.
         RESULTS="${{ runner.temp }}/rafter-results.json"
-        FINDINGS_COUNT=$(jq '.vulnerabilities | length' "$RESULTS" 2>/dev/null || echo "0")
-        CRITICAL_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' "$RESULTS" 2>/dev/null || echo "0")
-        HIGH_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "error" or .severity == "high")] | length' "$RESULTS" 2>/dev/null || echo "0")
-        MEDIUM_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "warning" or .severity == "medium")] | length' "$RESULTS" 2>/dev/null || echo "0")
-        LOW_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "note" or .severity == "low")] | length' "$RESULTS" 2>/dev/null || echo "0")
+        if ! jq -e 'type == "object" and (.vulnerabilities | type == "array") and all(.vulnerabilities[]; type == "object")' "$RESULTS" >/dev/null 2>&1; then
+          SNIPPET=$(head -c 300 "$RESULTS" 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true)
+          echo "::error::Rafter returned a report for scan ${SCAN_ID} that this action cannot read: no 'vulnerabilities' array."
+          echo "::error::A report that cannot be parsed is not a clean scan, so no counts were produced. Check the scan in your dashboard at ${RAFTER_URL}/dashboard or retry with: rafter get ${SCAN_ID}"
+          echo "::error::Body started with: ${SNIPPET}"
+          echo "status=unreadable" >> "$GITHUB_OUTPUT"
+          exit 1
+        fi
+
+        FINDINGS_COUNT=$(jq '.vulnerabilities | length' "$RESULTS")
+        CRITICAL_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' "$RESULTS")
+        HIGH_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "error" or .severity == "high")] | length' "$RESULTS")
+        MEDIUM_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "warning" or .severity == "medium")] | length' "$RESULTS")
+        LOW_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "note" or .severity == "low")] | length' "$RESULTS")
 
         echo "findings_count=${FINDINGS_COUNT}" >> "$GITHUB_OUTPUT"
         echo "critical_count=${CRITICAL_COUNT}" >> "$GITHUB_OUTPUT"

github-action/tests/mock-rafter-api.py

@@ -26,6 +26,19 @@
                 i.e. as soon as the injected failures are done). Set it higher
                 than the failure window to make the RESULTS fetch fail rather
                 than the poll.
+  RESULTS_SHAPE  what the completed JSON body looks like (sable-fgk7). Default
+                "ok": {"status":"completed","vulnerabilities":[]}. Others:
+                  with-findings  three findings: one critical, one high, one low
+                  missing-key    {"scan_id":..,"status":"completed"} — parses,
+                                 has no vulnerabilities array at all
+                  not-json       a 200 whose body is an HTML error page
+                  error-object   a 200 whose body is {"error": ...}
+                Each of the last three used to make the action report
+                "No security findings detected" and pass every threshold.
+  SHAPE_FROM     GET index from which the JSON body takes RESULTS_SHAPE
+                (default COMPLETE_AFTER + 1, so the poll loop sees one healthy
+                "completed" and the RESULTS fetch gets the shaped body).
+                md/sarif fetches are never shaped.
 """
 import json
 import os
@@ -38,21 +51,47 @@
 FAIL_FOREVER = os.environ.get("FAIL_FOREVER") == "1"
 FAIL_COUNT = int(os.environ.get("FAIL_COUNT", "1"))
 COMPLETE_AFTER = int(os.environ.get("COMPLETE_AFTER", str(FAIL_ON)))
+RESULTS_SHAPE = os.environ.get("RESULTS_SHAPE", "ok")
+SHAPE_FROM = int(os.environ.get("SHAPE_FROM", str(COMPLETE_AFTER + 1)))
 
 SCAN_ID = "repro-sable-l10k-0001"
 
+# Severities chosen so every count output is pinned to a distinct value:
+# findings=3, critical=1, high=1, medium=0, low=1.
+WITH_FINDINGS = [
+    {"rule_id": "sql-injection", "severity": "critical", "file_path": "db.php", "line_start": 12},
+    {"rule_id": "xss-echo", "severity": "high", "file_path": "view.php", "line_start": 40},
+    {"rule_id": "weak-hash", "severity": "low", "file_path": "auth.php", "line_start": 7},
+]
+
 state = {"polls": 0}
 
 
 class Handler(BaseHTTPRequestHandler):
     def _send(self, code, payload):
-        body = json.dumps(payload).encode()
+        self._send_raw(code, json.dumps(payload).encode(), "application/json")
+
+    def _send_raw(self, code, body, content_type):
         self.send_response(code)
-        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Type", content_type)
         self.send_header("Content-Length", str(len(body)))
         self.end_headers()
         self.wfile.write(body)
 
+    def _send_shaped_results(self):
+        """The completed JSON body under RESULTS_SHAPE (never for md/sarif)."""
+        if RESULTS_SHAPE == "with-findings":
+            return self._send(200, {"scan_id": SCAN_ID, "status": "completed",
+                                    "vulnerabilities": WITH_FINDINGS})
+        if RESULTS_SHAPE == "missing-key":
+            return self._send(200, {"scan_id": SCAN_ID, "status": "completed"})
+        if RESULTS_SHAPE == "not-json":
+            return self._send_raw(200, b"<html><body><h1>502 Bad Gateway</h1></body></html>",
+                                  "text/html")
+        if RESULTS_SHAPE == "error-object":
+            return self._send(200, {"error": "Failed to fetch report from storage: Object not found"})
+        raise SystemExit(f"unknown RESULTS_SHAPE {RESULTS_SHAPE!r}")
+
     def do_POST(self):
         if urlparse(self.path).path != "/api/static/scan":
             return self._send(404, {"error": "not found"})
@@ -84,6 +123,9 @@ def do_GET(self):
         if n < COMPLETE_AFTER:
             return self._send(200, {"scan_id": SCAN_ID, "status": "processing"})
 
+        if fmt == "json" and RESULTS_SHAPE != "ok" and n >= SHAPE_FROM:
+            return self._send_shaped_results()
+
         completed = {"scan_id": SCAN_ID, "status": "completed", "vulnerabilities": []}
         if fmt == "md":
             completed["markdown"] = "# Rafter\n\nNo findings.\n"

github-action/tests/test-action-yml-defaults.sh

@@ -163,6 +163,43 @@ else
   failures=$((failures+1))
 fi
 
+# ── sable-fgk7: an unreadable report is not a clean scan ─────────────────
+# The results step used to coerce every jq failure into findings_count=0,
+# which passed every threshold and rendered "No security findings detected".
+# Reproduced with a 200 whose body was not JSON, a 200 carrying an error
+# object, and a parseable payload with no vulnerabilities key.
+
+# 14. The payload shape must be validated before any count is computed.
+if grep -qF "jq -e 'type == \"object\" and (.vulnerabilities | type == \"array\") and all(.vulnerabilities[]; type == \"object\")'" "$ACTION_YML"; then
+  echo "PASS: results step validates the payload shape before counting"
+else
+  echo "FAIL: results step no longer validates that .vulnerabilities is an array of objects"
+  failures=$((failures+1))
+fi
+
+# 15. No count may fall back to 0 on a jq failure. That fallback IS the bug:
+#     the error path and the clean path produced the same number.
+zero_fallbacks=$(grep -c '|| echo "0"' "$ACTION_YML" || true)
+if [ "$zero_fallbacks" -eq 0 ]; then
+  echo "PASS: no count falls back to 0 on a parse failure"
+else
+  echo "FAIL: ${zero_fallbacks} count(s) still fall back to 0 on a jq failure — an unreadable report would render as clean"
+  failures=$((failures+1))
+fi
+
+# 16. The unreadable-payload path must record status=unreadable and exit 1,
+#     so the declared status output cannot fall back to the poll step's
+#     'completed' for a report that was never read.
+if awk '/no .vulnerabilities. array/,/^          fi$/' "$ACTION_YML" \
+     | grep -q 'status=unreadable' \
+   && awk '/no .vulnerabilities. array/,/^          fi$/' "$ACTION_YML" \
+     | grep -q 'exit 1'; then
+  echo "PASS: unreadable payload records status=unreadable and fails the step"
+else
+  echo "FAIL: unreadable-payload branch no longer records status=unreadable and exits 1"
+  failures=$((failures+1))
+fi
+
 echo ""
 echo "── results ───────────────────────────────────────────────────────────"
 echo "Failures: $failures"

node/src/commands/issues/from-scan.ts

@@ -164,6 +164,36 @@ async function runFromScan(opts: {
   }
 }
 
+/**
+ * The findings list from a scan payload, or an error — never a silent [].
+ *
+ * A payload without a `vulnerabilities` array is not "no findings". It is a
+ * scan that has not completed, a failed scan, or a report this client cannot
+ * read; filing zero issues from it would report a clean codebase for work
+ * that was never done (sable-fgk7). An empty array IS a legitimate clean
+ * result and is returned as such.
+ */
+export function vulnerabilitiesFromPayload(
+  data: unknown,
+  scanId: string
+): BackendVulnerability[] {
+  const payload = data as { vulnerabilities?: unknown; status?: unknown } | null;
+  if (payload && Array.isArray(payload.vulnerabilities)) {
+    return payload.vulnerabilities as BackendVulnerability[];
+  }
+  const status = payload && typeof payload.status === "string" ? payload.status : undefined;
+  if (status && status !== "completed") {
+    throw new Error(
+      `Scan ${scanId} is ${status}, not completed — there are no findings to file yet. ` +
+        `Retry once it completes: rafter get ${scanId}`
+    );
+  }
+  throw new Error(
+    `Scan ${scanId} returned no 'vulnerabilities' array; refusing to treat an unreadable ` +
+      `report as zero findings. Check it with: rafter get ${scanId}`
+  );
+}
+
 async function draftsFromBackendScan(
   scanId: string,
   apiKey?: string
@@ -174,8 +204,7 @@ async function draftsFromBackendScan(
     headers: { "x-api-key": key },
   });
 
-  const vulns: BackendVulnerability[] = data.vulnerabilities || [];
-  return vulns.map(buildFromBackendVulnerability);
+  return vulnerabilitiesFromPayload(data, scanId).map(buildFromBackendVulnerability);
 }
 
 function draftsFromLocalScan(filePath: string): IssueDraft[] {