Skip to content

test: run the real code instead of hand-copied mirrors (sable-1drb) #225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Rome-1 wants to merge 1 commit into from
Closed

test: run the real code instead of hand-copied mirrors (sable-1drb) #225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions .github/workflows/test-github-action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,64 @@ jobs:
[ "$FAIL" -eq 0 ] && echo "PASS: counts are exactly the report's (3/1/1/0/1)."
exit $FAIL

# sable-1drb — the threshold gate is now unit-tested against the real
# lib/severity.sh, but a unit test cannot prove action.yml WIRES it: that
# the counts reach the gate and the gate's verdict reaches the job. One
# end-to-end run with real findings and a threshold they exceed does.
test-threshold-gate-end-to-end:
name: "Threshold gate: real findings above the threshold fail the build"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Start mock backend (report has 1 critical, 1 high, 1 low)
env:
PORT: '8793'
FAIL_COUNT: '0'
COMPLETE_AFTER: '1'
RESULTS_SHAPE: 'with-findings'
run: |
nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 &
for _ in $(seq 1 30); do
curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null && break
sleep 1
done
curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null || {
echo "FAIL: mock backend never started"; cat mock.log; exit 1; }

- name: Run the action with severity-threshold high
id: scan
continue-on-error: true
uses: ./github-action
with:
api-key: 'not-a-real-key'
rafter-url: 'http://127.0.0.1:8793'
timeout-minutes: '2'
upload-sarif: 'false'
comment-on-pr: 'false'
severity-threshold: 'high'

- name: Assert the gate, not an error, failed the build
run: |
cat mock.log
FAIL=0
# status=completed AND outcome=failure is the gate's signature: the
# report was read and counted, then the threshold rejected it.
if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then
echo "FAIL: expected status=completed (report read), got '${{ steps.scan.outputs.status }}'"
FAIL=1
fi
if [ "${{ steps.scan.outcome }}" != "failure" ]; then
echo "FAIL: 1 critical + 1 high with severity-threshold=high must fail the build (outcome='${{ steps.scan.outcome }}')"
FAIL=1
fi
if [ "${{ steps.scan.outputs.findings-count }}" != "3" ]; then
echo "FAIL: findings-count expected 3, got '${{ steps.scan.outputs.findings-count }}'"
FAIL=1
fi
[ "$FAIL" -eq 0 ] && echo "PASS: counts reached the gate and the gate failed the build."
exit $FAIL

test-yaml-validity:
name: action.yml is valid YAML
runs-on: ubuntu-latest
Expand Down
36 changes: 8 additions & 28 deletions github-action/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,9 @@ runs:
LOW_COUNT: ${{ steps.results.outputs.low_count }}
SEVERITY_THRESHOLD: ${{ inputs.severity-threshold }}
run: |
# Shared with the tests under tests/ — see lib/severity.sh (sable-1drb).
source "${{ github.action_path }}/lib/severity.sh"

MD_REPORT=$(jq -r '.markdown // empty' "${{ runner.temp }}/rafter-results.md" 2>/dev/null || cat "${{ runner.temp }}/rafter-results.md")

if [ "$FINDINGS_COUNT" -eq 0 ]; then
Expand Down Expand Up @@ -394,10 +397,7 @@ runs:
echo ""
echo "</details>"
echo ""
if [ "$FINDINGS_COUNT" -gt 0 ] && [ "$SEVERITY_THRESHOLD" = "none" ]; then
echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow."
echo ""
fi
rafter_report_only_tip "$FINDINGS_COUNT" "$SEVERITY_THRESHOLD"
echo "---"
echo "<sub>Scan ID: ${SCAN_ID} | Powered by [Rafter](https://rafter.so)</sub>"
} >> "$COMMENT_FILE"
Expand Down Expand Up @@ -432,31 +432,11 @@ runs:
MEDIUM_COUNT: ${{ steps.results.outputs.medium_count }}
LOW_COUNT: ${{ steps.results.outputs.low_count }}
run: |
FAIL=0

case "$SEVERITY_THRESHOLD" in
critical)
[ "$CRITICAL_COUNT" -gt 0 ] && FAIL=1
;;
high)
[ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1
;;
medium)
[ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] && FAIL=1
;;
low)
[ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] || [ "$LOW_COUNT" -gt 0 ] && FAIL=1
;;
none)
FAIL=0
;;
*)
echo "::warning::Unknown severity threshold '${SEVERITY_THRESHOLD}', defaulting to 'high'"
[ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1
;;
esac
# The case statement lives in lib/severity.sh so the unit tests under
# tests/ run the same code, not a transcription of it (sable-1drb).
source "${{ github.action_path }}/lib/severity.sh"

if [ "$FAIL" -eq 1 ]; then
if rafter_threshold_fails "$SEVERITY_THRESHOLD" "$CRITICAL_COUNT" "$HIGH_COUNT" "$MEDIUM_COUNT" "$LOW_COUNT"; then
echo "::error::Security findings exceed severity threshold '${SEVERITY_THRESHOLD}'"
exit 1
fi
Expand Down
54 changes: 54 additions & 0 deletions github-action/lib/severity.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env bash
#
# Severity-threshold logic shared by github-action/action.yml and the tests
# under github-action/tests/. ONE copy, sourced by both, so the tests exercise
# the code the action runs rather than a transcription of it (sable-1drb).
#
# Sourced, never executed: no `set -e`, no side effects at load time. Every
# function takes explicit arguments so a test can call it without staging
# environment variables, and prints only what the action wants in its log.

# rafter_threshold_fails THRESHOLD CRITICAL HIGH MEDIUM LOW
#
# Returns 0 when the findings exceed THRESHOLD (the build should fail) and 1
# otherwise. 'none' never fails. An unrecognised threshold behaves like
# 'high' and says so with a ::warning:: annotation.
rafter_threshold_fails() {
local threshold="$1" critical="$2" high="$3" medium="$4" low="$5"
local fail=0
case "$threshold" in
critical)
[ "$critical" -gt 0 ] && fail=1
;;
high)
[ "$critical" -gt 0 ] || [ "$high" -gt 0 ] && fail=1
;;
medium)
[ "$critical" -gt 0 ] || [ "$high" -gt 0 ] || [ "$medium" -gt 0 ] && fail=1
;;
low)
[ "$critical" -gt 0 ] || [ "$high" -gt 0 ] || [ "$medium" -gt 0 ] || [ "$low" -gt 0 ] && fail=1
;;
none)
fail=0
;;
*)
echo "::warning::Unknown severity threshold '${threshold}', defaulting to 'high'"
[ "$critical" -gt 0 ] || [ "$high" -gt 0 ] && fail=1
;;
esac
[ "$fail" -eq 1 ]
}

# rafter_report_only_tip FINDINGS_COUNT THRESHOLD
#
# Prints the report-only tip block for the PR comment iff there are findings
# AND the threshold is 'none' (the default), i.e. the run reported problems
# but was configured never to fail on them. Prints nothing otherwise.
rafter_report_only_tip() {
local findings="$1" threshold="$2"
if [ "$findings" -gt 0 ] && [ "$threshold" = "none" ]; then
echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow."
echo ""
fi
}
42 changes: 33 additions & 9 deletions github-action/tests/test-action-yml-defaults.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,25 +45,49 @@ else
failures=$((failures+1))
fi

# 3. The report-only tip block must be present and gated on both conditions.
if grep -qE '\[ "\$FINDINGS_COUNT" -gt 0 \] && \[ "\$SEVERITY_THRESHOLD" = "none" \]' "$ACTION_YML"; then
echo "PASS: report-only tip block gated on (findings > 0) AND (threshold == 'none')"
# The threshold case statement and the report-only tip live in lib/severity.sh
# (sable-1drb), sourced by action.yml AND by the unit tests, so checks 3, 4
# and 17 look there. Check 17 is what stops a "simplification" from inlining
# a copy back into action.yml, which would silently detach the tests again.
SEVERITY_LIB="$(cd "$(dirname "$0")/.." && pwd)/lib/severity.sh"
if [ ! -f "$SEVERITY_LIB" ]; then
echo "FAIL: $SEVERITY_LIB not found"
exit 1
fi

# 3. The report-only tip must be gated on both conditions.
if grep -qE '\[ "\$findings" -gt 0 \] && \[ "\$threshold" = "none" \]' "$SEVERITY_LIB"; then
echo "PASS: report-only tip gated on (findings > 0) AND (threshold == 'none')"
else
echo "FAIL: report-only tip block missing or mis-gated in $ACTION_YML"
echo "FAIL: report-only tip missing or mis-gated in $SEVERITY_LIB"
failures=$((failures+1))
fi

# 4. The threshold-eval step must still handle 'none' as a no-op
# (no FAIL=1 in the none branch).
# 4. The threshold-eval must still handle 'none' as a no-op
# (no fail=1 in the none branch).
if awk '
/none\)/ { in_none=1; next }
in_none && /;;/ { in_none=0; next }
in_none { print }
' "$ACTION_YML" | grep -qE "FAIL *= *1"; then
echo "FAIL: 'none' branch of threshold-eval sets FAIL=1 — that would break the default"
' "$SEVERITY_LIB" | grep -qE "fail *= *1"; then
echo "FAIL: 'none' branch of threshold-eval sets fail=1 — that would break the default"
failures=$((failures+1))
else
echo "PASS: 'none' branch of threshold-eval does not set FAIL=1"
echo "PASS: 'none' branch of threshold-eval does not set fail=1"
fi

# 17. action.yml must SOURCE the library in both steps that use it, and must
# not carry its own copy of the case statement. If either regresses, the
# unit tests go back to testing a transcription.
lib_sources=$(grep -cF 'source "${{ github.action_path }}/lib/severity.sh"' "$ACTION_YML" || true)
inline_cases=$(grep -cE '^\s*(critical|medium|low)\)\s*$' "$ACTION_YML" || true)
if [ "$lib_sources" -ge 2 ] && [ "$inline_cases" -eq 0 ] \
&& grep -q 'rafter_threshold_fails "\$SEVERITY_THRESHOLD"' "$ACTION_YML" \
&& grep -q 'rafter_report_only_tip "\$FINDINGS_COUNT" "\$SEVERITY_THRESHOLD"' "$ACTION_YML"; then
echo "PASS: action.yml sources lib/severity.sh in both steps and carries no inline copy"
else
echo "FAIL: action.yml sources=${lib_sources} (need >=2), inline case branches=${inline_cases} (need 0), or a call site is missing"
failures=$((failures+1))
fi

# ── sable-l10k: poll-path retry contract ─────────────────────────────────
Expand Down
36 changes: 22 additions & 14 deletions github-action/tests/test-pr-comment-tip.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,38 @@
#!/usr/bin/env bash
#
# Unit test for the new PR-comment "report-only tip" block in
# github-action/action.yml. Re-implements the if block verbatim and
# exercises every input combination.
# Unit test for the PR-comment "report-only tip" block in
# github-action/action.yml. Sources github-action/lib/severity.sh — the SAME
# file action.yml sources at run time — and exercises every input
# combination of rafter_report_only_tip.
#
# The tip should appear iff (FINDINGS_COUNT > 0) AND (SEVERITY_THRESHOLD == 'none').
#
# This test used to carry its own copy of the if block (sable-1drb). It now
# runs the code the action runs.

set -u

# shellcheck source=../lib/severity.sh
source "$(cd "$(dirname "$0")/.." && pwd)/lib/severity.sh"

failures=0
total=0

# Mirror of the new block under the "Comment on PR" step's COMMENT_FILE builder.
emit_tip_if_applicable() {
if [ "$FINDINGS_COUNT" -gt 0 ] && [ "$SEVERITY_THRESHOLD" = "none" ]; then
echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow."
echo ""
fi
}

TIP_NEEDLE="report-only"

# assert_tip <name> <expected: yes|no> <findings> <threshold>
assert_tip() {
local name="$1"; local expected="$2"
FINDINGS_COUNT="$3"; SEVERITY_THRESHOLD="$4"
local findings="$3" threshold="$4"
total=$((total+1))

local out
out=$(emit_tip_if_applicable)
out=$(rafter_report_only_tip "$findings" "$threshold")
local has_tip="no"
if echo "$out" | grep -q "$TIP_NEEDLE"; then has_tip="yes"; fi

if [ "$has_tip" != "$expected" ]; then
echo "FAIL: $name — findings=$FINDINGS_COUNT threshold=$SEVERITY_THRESHOLD → expected tip=$expected got $has_tip"
echo "FAIL: $name — findings=$findings threshold=$threshold → expected tip=$expected got $has_tip"
failures=$((failures+1))
else
echo "PASS: $name (tip=$has_tip)"
Expand All @@ -53,6 +52,15 @@ assert_tip "findings + low" no 5 low
assert_tip "no findings + high" no 0 high
assert_tip "no findings + critical" no 0 critical

echo "── the tip must tell the reader what to set ─────────────────────────"
total=$((total+1))
if rafter_report_only_tip 5 none | grep -q 'severity-threshold: high'; then
echo "PASS: tip names the input to set"
else
echo "FAIL: tip no longer names severity-threshold: high"
failures=$((failures+1))
fi

echo ""
echo "── results ───────────────────────────────────────────────────────────"
echo "Total: $total Failures: $failures"
Expand Down
Loading
Loading