From bf78e2c43bb1cb48bf5362501c0f30fa7595730b Mon Sep 17 00:00:00 2001 From: Rome-1 Date: Thu, 27 Aug 2026 05:21:20 +0000 Subject: [PATCH 01/18] fix(scanners): treat betterleaks null report as empty result set (#217) betterleaks >=1.1.2 writes the literal `null`, not `[]`, to its JSON report for a clean scan via the `dir`/`git` subcommands both implementations use. `JSON.parse("null")` is `null` and `Array.isArray(null)` is false, so a clean scan fell into the sable-o4k stale-binary version-mismatch branch and printed a spurious warning for every clean file scanned. Return an empty result set for a null report, before the array check, so the stale-binary guard still fires for genuinely non-array output. Extract the Python parse into `_parse_report` to mirror Node's `parseResults`, so both are directly testable without invoking the binary. Reported by @justJackjon in #217, with an accurate root-cause analysis. Note the older `betterleaks detect --report-format json` form writes `[]`, so the null only appears via the subcommand form we actually invoke. --- node/src/scanners/betterleaks.ts | 7 ++ node/tests/betterleaks-parse-report.test.ts | 77 +++++++++++++++++++ python/rafter_cli/scanners/betterleaks.py | 59 +++++++++----- python/tests/test_betterleaks_parse_report.py | 59 ++++++++++++++ 4 files changed, 181 insertions(+), 21 deletions(-) create mode 100644 node/tests/betterleaks-parse-report.test.ts create mode 100644 python/tests/test_betterleaks_parse_report.py diff --git a/node/src/scanners/betterleaks.ts b/node/src/scanners/betterleaks.ts index 6f4a9178..7ca0d261 100644 --- a/node/src/scanners/betterleaks.ts +++ b/node/src/scanners/betterleaks.ts @@ -206,6 +206,13 @@ export class BetterleaksScanner { return []; } const parsed = JSON.parse(content); + // #217 — betterleaks >=1.1.2 writes the literal `null` (not `[]`) for a + // clean scan via the `dir`/`git` subcommands we invoke. That is a valid + // empty result, not a version mismatch, so it must not reach the warning + // branch below — otherwise every clean file scanned emits warning noise. + if (parsed === null) { + return []; + } if (!Array.isArray(parsed)) { // sable-o4k — a stale/incompatible binary emits a non-array shape and // would otherwise silently yield zero findings. The managed binary is diff --git a/node/tests/betterleaks-parse-report.test.ts b/node/tests/betterleaks-parse-report.test.ts new file mode 100644 index 00000000..ebc9715d --- /dev/null +++ b/node/tests/betterleaks-parse-report.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { BetterleaksScanner } from "../src/scanners/betterleaks.js"; + +// parseResults is private; exercise it directly rather than shelling out to the +// real binary, so these stay hermetic and run without betterleaks installed. +const scanner = new BetterleaksScanner(); +const parseResults = (scanner as any).parseResults.bind(scanner); + +let tmpDir: string; +let stderrSpy: ReturnType; + +function writeReport(content: string): string { + const p = path.join(tmpDir, "report.json"); + fs.writeFileSync(p, content, "utf-8"); + return p; +} + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "bl-parse-test-")); + stderrSpy = vi.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => { + stderrSpy.mockRestore(); + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("BetterleaksScanner.parseResults", () => { + // #217 — betterleaks >=1.1.2 writes the literal `null` for a clean scan via + // the `dir`/`git` subcommands. Regression guard: this is an empty result, not + // a version mismatch, and must not emit warning noise on every clean file. + it("treats a literal `null` report as an empty result set", () => { + expect(parseResults(writeReport("null"))).toEqual([]); + }); + + it("does not warn on a `null` report", () => { + parseResults(writeReport("null")); + expect(stderrSpy).not.toHaveBeenCalled(); + }); + + it("tolerates trailing whitespace around `null`", () => { + expect(parseResults(writeReport("null\n"))).toEqual([]); + expect(stderrSpy).not.toHaveBeenCalled(); + }); + + it("returns an empty result set for an empty report", () => { + expect(parseResults(writeReport(""))).toEqual([]); + expect(stderrSpy).not.toHaveBeenCalled(); + }); + + it("returns findings from a normal array report", () => { + const findings = [{ RuleID: "aws-secret-key", Description: "AWS key", StartLine: 3 }]; + expect(parseResults(writeReport(JSON.stringify(findings)))).toEqual(findings); + expect(stderrSpy).not.toHaveBeenCalled(); + }); + + it("returns an empty result set for an empty array report", () => { + expect(parseResults(writeReport("[]"))).toEqual([]); + expect(stderrSpy).not.toHaveBeenCalled(); + }); + + // sable-o4k — the stale-binary guard must survive the #217 fix. + it("still warns about a non-array object report", () => { + expect(parseResults(writeReport('{"findings": []}'))).toEqual([]); + expect(stderrSpy).toHaveBeenCalledOnce(); + expect(String(stderrSpy.mock.calls[0][0])).toContain("possible version mismatch"); + }); + + it("still warns about malformed JSON", () => { + expect(parseResults(writeReport("{not json"))).toEqual([]); + expect(stderrSpy).toHaveBeenCalledOnce(); + expect(String(stderrSpy.mock.calls[0][0])).toContain("Failed to parse"); + }); +}); diff --git a/python/rafter_cli/scanners/betterleaks.py b/python/rafter_cli/scanners/betterleaks.py index 71b82a8c..a1178e7d 100644 --- a/python/rafter_cli/scanners/betterleaks.py +++ b/python/rafter_cli/scanners/betterleaks.py @@ -164,29 +164,46 @@ def _run_scan(self, target: str, *, use_git: bool = False) -> list[dict]: ) return [] - try: - with open(report_path) as f: - content = f.read().strip() - if not content: - return [] - parsed = json.loads(content) - except json.JSONDecodeError as exc: - print(f"[rafter] Warning: Failed to parse Betterleaks report: {exc}", file=sys.stderr) - return [] + return self._parse_report(report_path) - if not isinstance(parsed, list): - # sable-o4k — a stale/incompatible binary emits a non-array - # shape and would otherwise silently yield zero findings. The - # managed binary is auto-updated upstream; this covers a stale - # binary on PATH, which we can't safely overwrite — so point the - # user at the fix. - print( - "[rafter] Warning: Betterleaks output is not an array — possible version mismatch. " - "Run: rafter agent update-betterleaks", - file=sys.stderr, - ) + @staticmethod + def _parse_report(report_path: str) -> list[dict]: + """Read a betterleaks JSON report into a findings list. + + Mirrors `parseResults` in node/src/scanners/betterleaks.ts — keep the + two in sync. + """ + try: + with open(report_path) as f: + content = f.read().strip() + if not content: return [] - return parsed + parsed = json.loads(content) + except json.JSONDecodeError as exc: + print(f"[rafter] Warning: Failed to parse Betterleaks report: {exc}", file=sys.stderr) + return [] + + # #217 — betterleaks >=1.1.2 writes the literal `null` (not `[]`) + # for a clean scan via the `dir`/`git` subcommands we invoke. That + # is a valid empty result, not a version mismatch, so it must not + # reach the warning branch below — otherwise every clean file + # scanned emits warning noise. + if parsed is None: + return [] + + if not isinstance(parsed, list): + # sable-o4k — a stale/incompatible binary emits a non-array + # shape and would otherwise silently yield zero findings. The + # managed binary is auto-updated upstream; this covers a stale + # binary on PATH, which we can't safely overwrite — so point the + # user at the fix. + print( + "[rafter] Warning: Betterleaks output is not an array — possible version mismatch. " + "Run: rafter agent update-betterleaks", + file=sys.stderr, + ) + return [] + return parsed @staticmethod def _convert(result: dict) -> PatternMatch: diff --git a/python/tests/test_betterleaks_parse_report.py b/python/tests/test_betterleaks_parse_report.py new file mode 100644 index 00000000..dad6dec5 --- /dev/null +++ b/python/tests/test_betterleaks_parse_report.py @@ -0,0 +1,59 @@ +"""Tests for BetterleaksScanner._parse_report. + +Mirrors node/tests/betterleaks-parse-report.test.ts — keep the two in sync. +Exercises the parser directly so the tests stay hermetic and run without the +betterleaks binary installed. +""" +from __future__ import annotations + +import json + +import pytest + +from rafter_cli.scanners.betterleaks import BetterleaksScanner + + +@pytest.fixture +def write_report(tmp_path): + def _write(content: str) -> str: + p = tmp_path / "report.json" + p.write_text(content) + return str(p) + + return _write + + +class TestParseReport: + # #217 — betterleaks >=1.1.2 writes the literal `null` for a clean scan via + # the `dir`/`git` subcommands. Regression guard: this is an empty result, + # not a version mismatch, and must not emit warning noise on every clean + # file scanned. + def test_null_report_is_empty_result(self, write_report, capsys): + assert BetterleaksScanner._parse_report(write_report("null")) == [] + assert capsys.readouterr().err == "" + + def test_null_report_with_trailing_whitespace(self, write_report, capsys): + assert BetterleaksScanner._parse_report(write_report("null\n")) == [] + assert capsys.readouterr().err == "" + + def test_empty_report(self, write_report, capsys): + assert BetterleaksScanner._parse_report(write_report("")) == [] + assert capsys.readouterr().err == "" + + def test_array_report_returns_findings(self, write_report, capsys): + findings = [{"RuleID": "aws-secret-key", "Description": "AWS key", "StartLine": 3}] + assert BetterleaksScanner._parse_report(write_report(json.dumps(findings))) == findings + assert capsys.readouterr().err == "" + + def test_empty_array_report(self, write_report, capsys): + assert BetterleaksScanner._parse_report(write_report("[]")) == [] + assert capsys.readouterr().err == "" + + # sable-o4k — the stale-binary guard must survive the #217 fix. + def test_non_array_object_still_warns(self, write_report, capsys): + assert BetterleaksScanner._parse_report(write_report('{"findings": []}')) == [] + assert "possible version mismatch" in capsys.readouterr().err + + def test_malformed_json_still_warns(self, write_report, capsys): + assert BetterleaksScanner._parse_report(write_report("{not json")) == [] + assert "Failed to parse" in capsys.readouterr().err From 5881204b8569ae27b2d9415555b3d525822cf444 Mon Sep 17 00:00:00 2001 From: Rome-1 Date: Thu, 27 Aug 2026 05:21:28 +0000 Subject: [PATCH 02/18] fix(scan): keep --json stdout parseable on a zero-file diff scan `rafter secrets --diff ` with no changed files printed its "No files changed since " status line to stdout and then called outputScanResults, which printed a second success line. Under --json that human line landed on stdout ahead of the JSON payload, so the documented output contract (stdout = JSON, status = stderr) was violated and the output would not parse. Route the status line to stderr in both implementations and let outputScanResults own the single stdout success line. Python additionally exited early without emitting any result envelope, so a zero-file `--json` scan produced no JSON at all; it now mirrors Node. Also drop a pre-existing unused `execSync` import in scan.ts. --- node/src/commands/agent/scan.ts | 10 ++++++--- python/rafter_cli/commands/agent.py | 35 +++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/node/src/commands/agent/scan.ts b/node/src/commands/agent/scan.ts index 8b6425b6..f18db67e 100644 --- a/node/src/commands/agent/scan.ts +++ b/node/src/commands/agent/scan.ts @@ -16,7 +16,7 @@ import { policyIgnoreToSuppressions, } from "../../core/custom-patterns.js"; import type { ScanIgnoreRule } from "../../core/config-schema.js"; -import { execSync, execFileSync } from "child_process"; +import { execFileSync } from "child_process"; import fs from "fs"; import os from "os"; import path from "path"; @@ -492,7 +492,9 @@ async function runGitAddedLineScan( if (!patch.trim()) { if (!opts.quiet) { - console.log(`\n${fmt.success(emptyMessage)}\n`); + // Status line, so stderr — stdout must stay parseable as JSON under + // --json, and outputScanResults owns the single stdout success line. + console.error(fmt.success(emptyMessage)); } outputScanResults([], opts, contextLabel, true, suppressions); return; @@ -501,7 +503,9 @@ async function runGitAddedLineScan( const addedLines = parseUnifiedDiffAddedLines(patch); if (addedLines.length === 0) { if (!opts.quiet) { - console.log(`\n${fmt.success(emptyMessage)}\n`); + // Status line, so stderr — stdout must stay parseable as JSON under + // --json, and outputScanResults owns the single stdout success line. + console.error(fmt.success(emptyMessage)); } outputScanResults([], opts, contextLabel, true, suppressions); return; diff --git a/python/rafter_cli/commands/agent.py b/python/rafter_cli/commands/agent.py index 541e30f2..f41a6711 100644 --- a/python/rafter_cli/commands/agent.py +++ b/python/rafter_cli/commands/agent.py @@ -1711,6 +1711,31 @@ def run_patterns() -> list[ScanResult]: return run_patterns() +def _output_empty_diff_scan( + empty_message: str, + json_output: bool, + quiet: bool, + context_label: str, + format: str, + suppressions, +) -> None: + """Emit the result of a diff scan that had no files to scan. + + ``empty_message`` ("No files changed since ") is a *status* line, so it + goes to stderr — stdout has to stay parseable as JSON under ``--json``, and + ``_output_scan_results`` owns the single stdout success line. Mirrors the + empty branches of ``runGitAddedLineScan`` in + node/src/commands/agent/scan.ts — keep the two in sync. + """ + if not quiet: + # rprint, not print — fmt.success returns rich markup that plain print + # would emit literally as "[green]...[/green]". + rprint(fmt.success(empty_message), file=sys.stderr) + _output_scan_results( + [], json_output, quiet, context_label, format=format, suppressions=suppressions + ) + + def _run_git_added_line_scan( git_args: list[str], git_cwd: str | None, @@ -1743,14 +1768,16 @@ def _run_git_added_line_scan( raise typer.Exit(code=2) if not patch.strip(): - if not quiet: - rprint(fmt.success(empty_message)) + _output_empty_diff_scan( + empty_message, json_output, quiet, context_label, format, suppressions + ) raise typer.Exit(code=0) added = parse_unified_diff_added_lines(patch) if not added: - if not quiet: - rprint(fmt.success(empty_message)) + _output_empty_diff_scan( + empty_message, json_output, quiet, context_label, format, suppressions + ) raise typer.Exit(code=0) try: From 1b9221f83b831f4413395dc9d2158906716602eb Mon Sep 17 00:00:00 2001 From: Rome-1 Date: Thu, 27 Aug 2026 05:21:34 +0000 Subject: [PATCH 03/18] =?UTF-8?q?docs:=20correct=20stale=20"2=20MCP=20reso?= =?UTF-8?q?urces"=20claim=20=E2=80=94=20rafter://docs=20was=20missing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCP server registers three resources (mcp/server.ts:313-332): rafter://config, rafter://policy, and rafter://docs. rafter://docs was added later and no doc was updated, so four places still advertised two. Surfaced while triaging external PR #215, which mirrored the stale count into recipes/cursor.md by following what was already written. --- .github/copilot-instructions.md | 2 +- CLAUDE.md | 2 +- demo/README.md | 2 +- recipes/hermes.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 0469b168..3f9e41a5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -32,4 +32,4 @@ Rafter is a security CLI for AI coding agents. It ships as two feature-identical - Scanners use dual-engine: Betterleaks binary first, regex fallback. Patterns defined in `secret-patterns.ts` / `secret_patterns.py` - Risk classification: critical > high > medium > low - Audit log: JSONL format, append-only, documented schema in CLI_SPEC.md -- MCP server: 4 tools + 2 resources over stdio transport +- MCP server: 4 tools + 3 resources over stdio transport diff --git a/CLAUDE.md b/CLAUDE.md index d52bd9c4..84b92d69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,7 +60,7 @@ cd python && poetry install && pytest **Secret scanning**: Dual-engine — tries Betterleaks binary first (higher accuracy), falls back to built-in regex patterns (21+ patterns, zero dependencies). Deterministic for a given version. Betterleaks is the gitleaks successor maintained by the original gitleaks authors. Existing installs with a leftover `~/.rafter/bin/gitleaks` are detected by `agent verify`/`status` so users get an upgrade hint, but the legacy CLI flags (`--with-gitleaks`, `--engine gitleaks`, `update-gitleaks`) have been removed. -**MCP server**: `rafter mcp serve` exposes 4 tools (`scan_secrets`, `evaluate_command`, `read_audit_log`, `get_config`) and 2 resources (`rafter://config`, `rafter://policy`) over stdio. +**MCP server**: `rafter mcp serve` exposes 4 tools (`scan_secrets`, `evaluate_command`, `read_audit_log`, `get_config`) and 3 resources (`rafter://config`, `rafter://policy`, `rafter://docs`) over stdio. ## Development diff --git a/demo/README.md b/demo/README.md index 6e564b24..30d0df57 100644 --- a/demo/README.md +++ b/demo/README.md @@ -18,7 +18,7 @@ The `/rafter-showcase` skill walks through all 9 core features with live command 4. Audit logging (JSONL trail) 5. Pre-commit hooks 6. CI/CD integration (GitHub Actions) -7. MCP server (4 tools, 2 resources) +7. MCP server (4 tools, 3 resources) 8. Skill auditing 9. Remote SAST/SCA (requires API key) diff --git a/recipes/hermes.md b/recipes/hermes.md index d7333e78..073e9720 100644 --- a/recipes/hermes.md +++ b/recipes/hermes.md @@ -52,7 +52,7 @@ The Rafter MCP server exposes four tools to Hermes: | `read_audit_log` | Inspect the JSON-lines audit log (`~/.rafter/audit.jsonl`) — every scan, every blocked command, with SHA-256 chain integrity | | `get_config` | Read the active Rafter config (risk-level, custom patterns, audit settings) | -Plus two resources (`rafter://config`, `rafter://policy`) that surface the live config and `.rafter.yml` policy as MCP resources. +Plus three resources (`rafter://config`, `rafter://policy`, `rafter://docs`) that surface the live config, the `.rafter.yml` policy, and the repo-specific security docs declared in `.rafter.yml` (metadata only) as MCP resources. ## Troubleshooting From 41d5b1331cd91ed8d205a1159ab4518b232bd20e Mon Sep 17 00:00:00 2001 From: Rome-1 Date: Fri, 28 Aug 2026 21:40:56 +0000 Subject: [PATCH 04/18] test(node): make the legacy-gitleaks hint test hermetic `agent status` probes `betterleaks` on PATH before falling through to the legacy-gitleaks branch, so the assertion only holds when betterleaks is absent. The test passed on CI and failed on any dev box with betterleaks installed. Run the CLI with PATH pointed at an empty directory so the fallthrough is reached deterministically. --- node/tests/agent-commands.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/node/tests/agent-commands.test.ts b/node/tests/agent-commands.test.ts index 669cd5b9..9f835e48 100644 --- a/node/tests/agent-commands.test.ts +++ b/node/tests/agent-commands.test.ts @@ -707,7 +707,12 @@ describe("agent status", () => { const binDir = path.join(home, ".rafter", "bin"); fs.mkdirSync(binDir, { recursive: true }); fs.writeFileSync(path.join(binDir, "gitleaks"), "#!/bin/sh\necho fake\n", { mode: 0o755 }); - const r = runCli("agent status", home); + // `agent status` probes `betterleaks` on PATH first and only falls through + // to the legacy-gitleaks hint when that fails — so the assertion below is + // only meaningful with an empty PATH. Otherwise this passes on CI and fails + // on any dev box that has betterleaks installed. + const emptyDir = fs.mkdtempSync(path.join(os.tmpdir(), "rafter-empty-path-")); + const r = runCli("agent status", home, { PATH: emptyDir }); expect(r.stdout).toMatch(/legacy gitleaks/i); expect(r.stdout).toMatch(/update-betterleaks/i); }); From e5d486787662fe599352a78dbef9bee261fa8dab Mon Sep 17 00:00:00 2001 From: Alexander Chakmakian <56743714+AlexChakmakian@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:18:37 -0700 Subject: [PATCH 05/18] docs: expand Cursor recipe to match platform recipe standard (#215) * docs: expand Cursor recipe to match platform recipe standard Fill out Prerequisites, MCP tools reference with usage examples, Troubleshooting, and Uninstall so recipes/cursor.md aligns with gemini-cli.md for #33. Co-authored-by: Cursor * Update cursor.md * Update cursor.md docs: address review - add rafter://docs, scan_secrets engines --------- Co-authored-by: Cursor --- recipes/cursor.md | 92 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/recipes/cursor.md b/recipes/cursor.md index 53faf291..ba6e7a60 100644 --- a/recipes/cursor.md +++ b/recipes/cursor.md @@ -1,8 +1,15 @@ # Cursor Setup -Rafter's Cursor integration covers an MCP server + Cursor-native hooks (`preToolUse` / `postToolUse` / `beforeShellExecution`) + per-skill rules + a Cursor sub-agent. See [`shared-docs/PLATFORM_PARITY_AUDIT.md`](../shared-docs/PLATFORM_PARITY_AUDIT.md) for the full surface matrix. +Rafter's Cursor integration covers an MCP server + Cursor-native hooks +(`preToolUse` / `postToolUse` / `beforeShellExecution`) + per-skill rules + a +Cursor sub-agent. See +[`shared-docs/PLATFORM_PARITY_AUDIT.md`](../shared-docs/PLATFORM_PARITY_AUDIT.md) +for the full surface matrix. -## Install the CLI first +## Prerequisites + +- [Cursor](https://cursor.com) installed (creates `~/.cursor` on first launch) +- Rafter CLI on your `PATH`: ```sh npm install -g @rafter-security/cli # Node @@ -10,9 +17,10 @@ npm install -g @rafter-security/cli # Node pip install rafter-cli # Python ``` -> Using `npx`? The canonical form is `npx @rafter-security/cli` — the bare `npx rafter-cli` resolves to an **unrelated** package on npm. +> Using `npx`? The canonical form is `npx @rafter-security/cli` — the bare +> `npx rafter-cli` resolves to an **unrelated** package on npm. -## Automatic setup +## Setup ### Driven by the Cursor agent itself (recommended for first-run) @@ -20,7 +28,9 @@ pip install rafter-cli # Python rafter agent init --local --with-cursor ``` -Writes to `./.rafter/` and `./.cursor/` instead of `$HOME` — sidesteps Cursor's sandbox prompt for writing under your home directory and scopes the install to this project. Run it from inside the repo you're working in. +Writes to `./.rafter/` and `./.cursor/` instead of `$HOME` — sidesteps Cursor's +sandbox prompt for writing under your home directory and scopes the install to +this project. Run it from inside the repo you're working in. ### Global install (one-time, applies to every project) @@ -28,11 +38,10 @@ Writes to `./.rafter/` and `./.cursor/` instead of `$HOME` — sidesteps Cursor' rafter agent init --with-cursor ``` -Auto-detects `~/.cursor` and installs at user scope. Requires elevated permissions if Cursor's sandbox is locked down — the agent will prompt for them. - -## Manual setup +Auto-detects `~/.cursor` and installs at user scope. Requires elevated +permissions if Cursor's sandbox is locked down — the agent will prompt for them. -### 1. MCP server +### Manual setup Add to `~/.cursor/mcp.json` (or `./.cursor/mcp.json` for per-project): @@ -47,7 +56,9 @@ Add to `~/.cursor/mcp.json` (or `./.cursor/mcp.json` for per-project): } ``` -Provides `scan_secrets`, `evaluate_command`, `read_audit_log`, and `get_config` tools. +Restart Cursor afterward so it loads the MCP server. Hooks, rules, and the +sub-agent are installed by `rafter agent init --with-cursor` (local or global); +prefer that over assembling them by hand. ## Verify @@ -55,4 +66,63 @@ Provides `scan_secrets`, `evaluate_command`, `read_audit_log`, and `get_config` rafter agent verify ``` -Confirms MCP server is configured and Cursor is detected. +Confirms MCP server is configured and Cursor is detected. In Cursor, open +**Settings → MCP** and check that the `rafter` server is connected. + +## Available MCP tools + +Once the MCP server is configured, Cursor can call the following tools: + +| Tool | Description | +|------|-------------| +| `scan_secrets` | Scan files or directories for hardcoded secrets and credentials. Supports `auto` (default), `betterleaks`, and `patterns` engines — `auto` runs both engines and unions the results. | +| `evaluate_command` | Check if a shell command is allowed by Rafter security policy. Returns risk level and approval requirement. | +| `read_audit_log` | Query the Rafter audit log with optional filtering by event type, count, or timestamp. | +| `get_config` | Read Rafter configuration — full config or a specific key via dot-path (e.g. `agent.commandPolicy`). | + +Three MCP resources are also exposed: + +| Resource | Description | +|----------|-------------| +| `rafter://config` | Current Rafter configuration as JSON | +| `rafter://policy` | Active security policy (merged `.rafter.yml` + config) | +| `rafter://docs` | Repo-specific security docs declared in `.rafter.yml` (metadata only, no content) | + +### Tool usage examples + +Ask the Cursor agent in natural language — it calls the matching tool: + +- "Scan `src/` for leaked secrets" → `scan_secrets` +- "Is `curl | bash` allowed by our policy?" → `evaluate_command` +- "Show recent blocked commands from the audit log" → `read_audit_log` +- "What's our current command policy?" → `get_config` + +Hooks run automatically on tool/shell use; you do not need to invoke them +manually. + +## Troubleshooting + +- **MCP server not loading**: Restart Cursor after installing. Open + **Settings → MCP** and confirm `rafter` is connected. Check `which rafter`. +- **`rafter` not found**: Ensure `rafter` is on your `PATH`. The MCP entry uses + `"command": "rafter"` — a bare name, not a full path. +- **Sandbox / home-directory write blocked**: Use + `rafter agent init --local --with-cursor` so files land under `./.cursor/`. +- **Cursor not detected**: Launch Cursor once so `~/.cursor` exists, or use + `--local` from inside a project. +- **Existing config preserved**: `rafter agent init --with-cursor` merges into + existing `mcp.json` / `hooks.json` — it won't overwrite other settings. + Re-running is safe and idempotent. + +## Uninstall + +```sh +rafter agent disable cursor.mcp cursor.hooks cursor.instructions +``` + +Removes the Rafter MCP entry, hooks, rules, and sub-agent at user scope while +leaving the rest of your Cursor config intact. + +For a local (`--local`) install, delete the corresponding files under +`./.cursor/` (or remove only the `rafter` key from `mcp.json` if you share that +file with other servers). Restart Cursor afterward. From 09964921a1a5eecc2a73c00e04f7fd8b900965fa Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:20:13 -0700 Subject: [PATCH 06/18] ci: run the test suite on external PRs into main (#219) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite only triggered on `pull_request` into `prod`, so every contributor PR targeting `main` merged with no automated verification — tests ran later, at release time, when main was promoted to prod. #215, #216 and #218 all sat mergeable with an empty status-check rollup. Add `main` to the trigger and gate the jobs: PRs into prod always run (that is the release gate, unchanged), PRs into main run only for outside contributions. Our own work — Rome-1's PRs, or any branch living in the Raftersecurity repo — is reviewed and tested locally before it is pushed, so re-running the full matrix would only burn runner minutes. Uses `pull_request`, not `pull_request_target`: fork PRs run with a read-only token and no secrets. Tests that need RAFTER_API_KEY already skip when it is absent, so a fork PR gets a clean green rather than a spurious failure. --- .github/workflows/test-comprehensive.yml | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/test-comprehensive.yml b/.github/workflows/test-comprehensive.yml index 225aca9e..2813130e 100644 --- a/.github/workflows/test-comprehensive.yml +++ b/.github/workflows/test-comprehensive.yml @@ -4,14 +4,49 @@ on: pull_request: branches: - prod + - main workflow_dispatch: permissions: contents: read jobs: + # ── Who gets the suite ──────────────────────────────────────────── + # PRs into prod are the release gate and always run. + # PRs into main run only for outside contributions: our own work (Rome-1's + # PRs, or any branch living in the Raftersecurity repo) is reviewed and + # tested locally before it is pushed, so running the full matrix again + # would just burn runner minutes. + # + # Note this is `pull_request`, not `pull_request_target` — fork PRs run with + # a read-only token and no access to secrets. Do not "fix" that. + gate: + runs-on: ubuntu-latest + outputs: + run: ${{ steps.decide.outputs.run }} + steps: + - id: decide + # Values go through env rather than direct ${{ }} interpolation into + # the script, so nothing from the PR can be shell-injected. + env: + EVENT: ${{ github.event_name }} + BASE: ${{ github.event.pull_request.base.ref }} + HEAD_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + if [ "$EVENT" != "pull_request" ] || [ "$BASE" != "main" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + elif [ "$HEAD_OWNER" = "Raftersecurity" ] || [ "$AUTHOR" = "Rome-1" ]; then + echo "run=false" >> "$GITHUB_OUTPUT" + echo "Internal PR into main (author=$AUTHOR, head repo owner=$HEAD_OWNER) — suite skipped." >> "$GITHUB_STEP_SUMMARY" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + # ── Unit & integration tests (both languages) ───────────────────── test-node: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest defaults: run: @@ -55,6 +90,8 @@ jobs: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} test-python: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest defaults: run: @@ -78,6 +115,8 @@ jobs: # ── E2E CLI tests ───────────────────────────────────────────────── e2e-node: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest defaults: run: @@ -102,6 +141,8 @@ jobs: # ── Secret detection accuracy ────────────────────────────────────── secret-detection-accuracy: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest defaults: run: @@ -124,6 +165,8 @@ jobs: # ── SARIF output validation ──────────────────────────────────────── sarif-validation: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest defaults: run: @@ -160,6 +203,8 @@ jobs: # ── Remote API integration (only when key available) ─────────────── backend-api: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} @@ -185,6 +230,8 @@ jobs: # ── Package build verification ───────────────────────────────────── package-integrity: + needs: gate + if: needs.gate.outputs.run == 'true' runs-on: ubuntu-latest defaults: run: @@ -231,6 +278,8 @@ jobs: # ── Cross-platform smoke test ────────────────────────────────────── cross-platform: + needs: gate + if: needs.gate.outputs.run == 'true' strategy: fail-fast: false matrix: From 7e58d4c3f29b8bcebdd293cd00f6a6bb481b0e73 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:38:00 -0700 Subject: [PATCH 07/18] fix: retry transient report-read failures during scan polling (sable-l10k) (#220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: retry transient report-read failures during scan polling (sable-l10k) A paying customer's GitHub Actions run died on: ::error::Rafter scan poll failed: HTTP 500 — Failed to fetch report from storage: Object not found A report is not durable the instant a scan flips to completed, so a poll can hit a 5xx on a scan that is perfectly readable seconds later. Every poll path treated any non-2xx as fatal and exited immediately — while the transport-error branch three lines above already retried. That asymmetry was the bug: a curl-level failure was survivable, an HTTP-level one was not. Reproduced in a real GitHub Actions run against a mock backend that injects one transient 500 into an otherwise healthy poll sequence; the run died on a scan that completed on the very next poll. All three surfaces now share one contract (documented in CLI_SPEC.md): - 5xx / 408 / 404 mid-poll and transport errors are transient. Retried up to 5 consecutive times with 2s/4s/8s/16s backoff; the counter resets on any successful poll. - Other 4xx (401/403/429) are not retried. A 404 on the FIRST poll is still a genuinely missing scan, exit 2. - On giving up, the message names the scan id, the `rafter get ` retry, and the dashboard. Storage-layer wording survives as supporting detail rather than as the whole explanation. The Python loop was additionally calling .json() on the 500 body, reading no status, falling out of the loop and writing the error payload out as if it were results — a silent wrong answer rather than a loud failure. Also here, from the security review of this diff: - Strip newlines and cap length on server-controlled `.error` before it reaches `::error::`/`::warning::`. A body containing a newline could forge workflow commands (`::add-mask::`, `::stop-commands::`). Same class as the pre-existing sinks; this diff widened it from 2 to 6. - Add --connect-timeout/--max-time to curl and an axios timeout, so a hung server cannot stall inside a request that the retry loop only checks between attempts. - Source the action's `status` output from the poll step when the results step never runs, so the new `unreadable` status reaches consumers instead of an empty string. Coverage: 8 vitest + 9 pytest cases pinning both halves of the contract, plus two end-to-end CI jobs that drive the composite action against a localhost mock backend — no API key, no credit spend. * test: restore real timers inside each test (Node 18 afterEach hang) Leaving vitest's fake timers installed past the end of the test body hangs the afterEach hook on Node 18 — the cross-platform matrix caught it on both ubuntu and macos. Matches the in-test restore the existing scan-remote tests already use; the afterEach restore stays as a fallback for failed assertions. * fix: address adversarial review of the poll-retry change (sable-l10k) An independent reviewer was asked to argue against merging #220. It found a regression I introduced, a contract my own spec text got wrong, and a test suite that did not test the mechanism it existed to protect. All real. REGRESSION I INTRODUCED — a failed results fetch reported `completed`. Sourcing the action's `status` output from `steps.results.outputs.status || steps.poll.outputs.status` meant that when the results fetch exhausted its retries, the empty results output fell back to the poll step's `completed`. A consumer gating on `status == 'completed'` saw a clean scan, and the artifact upload published the error body as rafter-results.json. Exactly the silent-wrong-answer class this PR set out to remove. Both give-up paths in fetch_results now record `status=unreadable`, and the artifact upload is gated on the results step rather than the poll step. THE TESTS DID NOT TEST THE BACKOFF. Setting BASE_BACKOFF_MS to 0 left all 8 vitest cases green, and the Python fixture patched out time.sleep entirely, making the schedule unobservable by construction. Backoff IS the fix — retrying five times inside a millisecond gives an eventually-consistent store no time to converge. Both suites now pin the exact sequence (10s poll, then 2/4/8/16), and both verify the consecutive counter resets on a successful poll. THE ACTION VIOLATED THE CONTRACT THIS PR WROTE. CLI_SPEC said transport errors were retried on the same budget as 5xx; in the action they retried on a flat 10s and never touched the counter, so an unreachable backend burned the whole timeout and then reported "scan did not complete within N minutes" — a timeout message for a DNS failure. They now share the budget and exit with status=unreachable. The spec is also corrected where the action genuinely cannot match the CLI: it has no first-poll concept, so every 404 there is lag (bounded by the 5-failure budget, not the full timeout). UNBOUNDED CLI LOOP. The consecutive counter was constructed per call, and resets on success, so a backend alternating 200/500 forever never exhausted it — and the CLI has no wall-clock deadline. Added a total budget (20 per invocation) that does not reset, keeping the useful reset-on-success semantics without the hole. THE REMEDY WE RECOMMEND WAS THE ONE PATH NOT FIXED. The give-up message says "retry with rafter get ", which re-enters at the first poll — which had no retry, so it died on the raw storage jargon we had just stopped printing. Worse for 404: the loop retried it five times, then recommended a command that reports "not found" with a different exit code. The first poll now retries transient 5xx while still treating 404 as fatal. PARITY BREAKS (this repo requires strict Node/Python parity): - Python accepted only 200; Node accepts any 2xx. On a 202 they returned opposite outcomes — Python exit 1, Node exit 0 with an empty payload. - Node wrote the retry notice into the ora spinner, which renders nothing on a non-TTY. The diagnostic was invisible in CI, the one place it matters. It goes to stderr now, matching Python. - Node retried ANY error lacking a `.response`, including TypeErrors thrown from our own code. Narrowed to genuine HTTP-layer errors. - Python raised PollGaveUpError for non-transient statuses too, conflating "tried five times" with "did not try". Split out PollFatalError. - Neither runtime truncated server error text; both now cap it like the action does. ALSO: sanitized the three remaining unsanitized `.error`/response echo sites in action.yml, guarded TIMEOUT_MINUTES before bash arithmetic evaluates it, and restored the remaining-budget denominator the poll log line had lost. COVERAGE for the two things a "simplification" would silently break: a CI job injecting a mid-poll 404, a CI job failing the results fetch specifically, and six new assertions in the action.yml drift detector (404 in the transient set, transport errors counted, give-up message actionable, both unreadable writes, artifact gating, exponential backoff). Each was mutation-tested to confirm it fails when the property is removed. CHANGELOG documents the two behavior changes this ships: timeout-minutes is now a real wall-clock deadline rather than a poll count, and Python's non-transient mid-poll failures now exit 1 instead of 0. * fix: nested error body crashed the retry it was supposed to trigger (sable-l10k) A verification pass over the previous fix commit found a defect in code that commit introduced, plus three places where a fix was thinner than it looked. THE BLOCKER — I added a truncate() helper that assumed the server's error field is a string. A backend answering {"error": {"message": "..."}} on a 500 made it call .split() on a dict (Python: AttributeError, uncaught, straight to a traceback) and .replace() on an object (Node: "s.replace is not a function", and crucially NO retry — 2 calls, not 5). So the one shape of error body most likely to appear on a real 500 turned a retryable failure into an immediate hard failure with a nonsense message, inside the very code meant to make transient failures survivable. Both runtimes now coerce before truncating. THE TOTAL-CAP TEST DID NOT TEST THE TOTAL CAP. Deleting the total clause from FailureBudget.exhausted left all 14 Node tests green: the mock queue drained, axios returned undefined, and the resulting TypeError was converted by the loop's own catch into the exact exit code the test asserted. The test now uses an endless flapping mock with a hard ceiling, so a missing cap fails loudly and immediately rather than passing on an unrelated crash. Verified by mutation both ways. (Python's equivalent was already genuine.) THE RECOMMENDED REMEDY STILL DID NOT RETRY. The last commit made the first poll retry, but `rafter get ` WITHOUT --interactive takes a different path entirely — a single un-retried request in both runtimes. So the command the give-up message recommends was still defeated by the failure that produced the message. It now shares the same retry budget. THE MESSAGE LIED ABOUT ITS OWN ATTEMPT COUNT. Both runtimes hardcoded "after 5 attempts" while exhaustion can equally come from the total budget of 20 — a flapping backend produced "after 5 attempts" following 20 failures over four minutes. It now reports the real count. Relatedly, the CLI blamed the report ("could not read the report … retry with rafter get") even when nothing ever reached the server; it now distinguishes unreachable-API from unreadable-report, which the action already did. ALSO: - Narrowed isTransientPollError: Node's own TypeError [ERR_INVALID_CHAR] carries .code, so an API key read from a file with a trailing newline was retried five times and reported as a flaky backend. Dropped .code; real axios timeouts still retry via .request/.isAxiosError. - Validate the server-supplied scan_id before it reaches $GITHUB_OUTPUT. A newline there forges step outputs, including status=completed. The CHANGELOG claimed sanitization was complete "at every site" when this one was open; the claim is now true rather than trimmed. - Three more drift assertions (sanitization present, TIMEOUT_MINUTES guarded, scan_id validated) — the first of which was itself broken on first write and only caught by mutation-testing it. - Tests for the nested-error crash, the truncation cap, the real attempt count, and the unreachable-API message, in both runtimes. --------- Co-authored-by: achebe --- .github/workflows/test-github-action.yml | 183 ++++++++- CHANGELOG.md | 14 + github-action/action.yml | 158 +++++-- github-action/tests/mock-rafter-api.py | 101 +++++ .../tests/test-action-yml-defaults.sh | 97 +++++ node/src/commands/backend/get.ts | 15 +- node/src/commands/backend/scan-status.ts | 298 ++++++++++++-- node/src/utils/api.ts | 6 + node/tests/scan-poll-transient-500.test.ts | 385 ++++++++++++++++++ node/tests/scan-remote.test.ts | 9 +- python/rafter_cli/commands/backend.py | 289 +++++++++++-- python/tests/test_scan_poll_transient_500.py | 303 ++++++++++++++ shared-docs/CLI_SPEC.md | 24 ++ 13 files changed, 1784 insertions(+), 98 deletions(-) create mode 100644 github-action/tests/mock-rafter-api.py create mode 100644 node/tests/scan-poll-transient-500.test.ts create mode 100644 python/tests/test_scan_poll_transient_500.py diff --git a/.github/workflows/test-github-action.yml b/.github/workflows/test-github-action.yml index b7836ee7..38875523 100644 --- a/.github/workflows/test-github-action.yml +++ b/.github/workflows/test-github-action.yml @@ -5,8 +5,9 @@ name: Test github-action/ Composite Action # are pure-bash unit tests of the threshold-eval and PR-comment logic # plus a drift detector on action.yml's load-bearing defaults. # -# An end-to-end test against the real Rafter API is a future addition -# (would require an injectable RAFTER_API_KEY secret and a fixture repo). +# The poll-path jobs DO drive the action end to end, against a localhost mock +# backend (github-action/tests/mock-rafter-api.py) rather than the real API, so +# they need no API key and spend no credits. on: push: branches: @@ -48,6 +49,184 @@ jobs: - name: Run action.yml defaults / drift check run: bash github-action/tests/test-action-yml-defaults.sh + # sable-l10k — a paying customer's run died on a single transient 500 during + # polling ("Failed to fetch report from storage: Object not found"). The + # report is not durable the instant a scan flips to completed, so that 500 is + # survivable and must be retried. These two jobs pin both halves of the + # contract: ride out the transient failure, still fail on a missing report. + test-poll-transient-500: + name: "Poll: rides out a transient 500" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (500 on poll #2, then healthy) + env: + PORT: '8787' + FAIL_ON: '2' + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8787/api/static/scan >/dev/null && break + sleep 1 + done + + - name: Run the action against the mock + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8787' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + + - name: Assert the scan survived the 500 + run: | + cat mock.log + echo "status output: '${{ steps.scan.outputs.status }}'" + if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then + echo "FAIL: a single transient 500 during polling killed the run." + exit 1 + fi + echo "PASS: the action retried the transient 500 and completed." + + test-poll-report-never-readable: + name: "Poll: fails clearly when the report is really missing" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (every poll 500s) + env: + PORT: '8788' + FAIL_ON: '2' + FAIL_FOREVER: '1' + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8788/api/static/scan >/dev/null && break + sleep 1 + done + + - name: Run the action against the mock + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8788' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + + - name: Assert it failed, with the right status + run: | + cat mock.log + FAIL=0 + if [ "${{ steps.scan.outputs.status }}" != "unreadable" ]; then + echo "FAIL: expected status=unreadable, got '${{ steps.scan.outputs.status }}'" + FAIL=1 + fi + if [ "${{ steps.scan.outcome }}" != "failure" ]; then + echo "FAIL: an unreadable report should fail the build." + FAIL=1 + fi + # A composite action's log is not capturable from the calling step, + # so the CONTENT of the give-up message is asserted by the drift + # detector (github-action/tests/test-action-yml-defaults.sh) instead. + exit $FAIL + + # The 404-as-transient branch is the subtlest thing in the poll loop: it is + # correct only because the trigger step has already handed us a scan_id. + # Nothing else in CI exercises it, so a "simplification" that drops `-eq 404` + # from the transient condition would otherwise land green. + test-poll-transient-404: + name: "Poll: rides out a transient 404" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (404 on poll #2, then healthy) + env: + PORT: '8789' + FAIL_ON: '2' + FAIL_STATUS: '404' + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8789/api/static/scan >/dev/null && break + sleep 1 + done + curl -sf -X POST -d '{}' http://127.0.0.1:8789/api/static/scan >/dev/null || { + echo "FAIL: mock backend never started"; cat mock.log; exit 1; } + + - name: Run the action against the mock + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8789' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + + - name: Assert the scan survived the 404 + run: | + cat mock.log + if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then + echo "FAIL: a transient 404 mid-poll killed the run." + exit 1 + fi + echo "PASS: the action treated a mid-poll 404 as read-after-write lag." + + # The results fetch runs the instant the scan reports completed — the + # likeliest moment for the report object to be unreadable. Its retry loop had + # no coverage at all, and it is where a failed read used to be reported to + # consumers as status=completed. + test-results-fetch-transient-500: + name: "Results fetch: rides out a transient 500" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (poll succeeds, first results fetch 500s) + env: + PORT: '8790' + FAIL_ON: '2' + FAIL_COUNT: '1' + COMPLETE_AFTER: '1' + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8790/api/static/scan >/dev/null && break + sleep 1 + done + curl -sf -X POST -d '{}' http://127.0.0.1:8790/api/static/scan >/dev/null || { + echo "FAIL: mock backend never started"; cat mock.log; exit 1; } + + - name: Run the action against the mock + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8790' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + + - name: Assert the results fetch retried rather than failing the build + run: | + cat mock.log + if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then + echo "FAIL: a transient 500 on the results fetch killed the run (status='${{ steps.scan.outputs.status }}')." + exit 1 + fi + echo "PASS: the results fetch retried and completed." + test-yaml-validity: name: action.yml is valid YAML runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5b87fa..6cfa1a95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **A transient 500 during scan polling no longer kills the run** (sable-l10k). An AppSumo customer's GitHub Actions build died on `Rafter scan poll failed: HTTP 500 — Failed to fetch report from storage: Object not found`. A report is not durable the instant a scan flips to `completed`, so a 5xx on that read is survivable — but every poll path treated any non-2xx as fatal, while the transport-error branch three lines above already retried. All three surfaces (the composite action, `rafter run`, `rafter get --interactive`) now retry transient failures with exponential backoff (2s/4s/8s/16s) before giving up, and the give-up message names the scan id, the `rafter get ` retry, and the dashboard instead of leaking storage-layer wording. Full contract in `shared-docs/CLI_SPEC.md`. Both runtimes; end-to-end CI coverage against a mock backend, so no API key or credits are needed to exercise it. +- **Python: a failed poll could be written out as if it were scan results** (sable-l10k). The mid-poll loop called `.json()` on the response without checking the status code, so a 500 carrying a JSON error body parsed cleanly, yielded no `status`, fell out of the loop, and was emitted as the scan payload with exit code `0`. A non-JSON error body raised an unhandled `JSONDecodeError`. Both now fail loudly. **Behavior change:** genuine non-transient mid-poll failures that previously exited `0` with an error payload on stdout now exit `1` — check any pipeline that consumed that output. +- **GitHub Action: a failed results fetch reported the scan as `completed`** (sable-l10k). The declared `status` output read only from the results step, which does not run when the fetch fails. Consumers gating on `status == 'completed'` saw a clean scan, and the artifact upload published the error body as `rafter-results.json`. Both give-up paths in the results fetch now record `status=unreadable`, and the artifact upload is gated on a successful results fetch. +- **GitHub Action: server-controlled error text is sanitized before it reaches workflow commands** (sable-l10k). A response body containing a newline could forge `::error::`, `::add-mask::`, or `::stop-commands::` annotations. Error text from the API is now stripped of newlines and length-capped at every site that echoes it, and the server-supplied `scan_id` is rejected unless it matches `^[A-Za-z0-9_-]+$` before it reaches `$GITHUB_OUTPUT` (where a newline would forge step outputs, including `status=completed`). +- **GitHub Action: an unreachable API is reported as unreachable** (sable-l10k). Transport errors retried on a flat 10s interval without counting toward the failure budget, so a bad `rafter-url` or a down backend burned the whole `timeout-minutes` window and then reported `Scan did not complete within N minutes` — a timeout message for a DNS failure. They now share the same retry budget and exit with `status=unreachable`. + +### Changed + +- **`timeout-minutes` on the GitHub Action is now a wall-clock deadline**, not a poll count. Previously the action ran `timeout-minutes * 6` polls, each costing 10s *plus* API latency, so a slow API pushed real elapsed time past the documented budget. It is now enforced as a real deadline. **This can fail workflows that were relying on the overrun** — if a scan sits near the boundary, raise `timeout-minutes`. +- `rafter get ` (without `--interactive`) now retries transient failures too. It is the command the poll loop's give-up message recommends, so a remedy defeated by the same transient failure it is recommended for was not a remedy. +- HTTP requests on the poll and results paths now carry connect/read timeouts (`--connect-timeout 10 --max-time 60` for curl, 30s for axios), so a hung server cannot stall inside a request that the retry loop only checks between attempts. + ## [0.10.0] - 2026-07-29 ### Added diff --git a/github-action/action.yml b/github-action/action.yml index 32a9b812..e096efad 100644 --- a/github-action/action.yml +++ b/github-action/action.yml @@ -27,7 +27,7 @@ inputs: required: false default: 'true' timeout-minutes: - description: 'Maximum time to wait for scan completion (minutes)' + description: 'Maximum wall-clock time to wait for scan completion (minutes). Enforced as a real deadline: before v0.11 this was a poll COUNT, so a slow API could overrun it.' required: false default: '10' rafter-url: @@ -55,8 +55,8 @@ outputs: description: 'Number of low/note findings' value: ${{ steps.results.outputs.low_count }} status: - description: 'Scan status (completed, failed, timeout)' - value: ${{ steps.results.outputs.status }} + description: 'Scan status: completed, failed, timeout, unreadable (the scan may have finished but its report could not be read), or unreachable (the Rafter API could not be contacted)' + value: ${{ steps.results.outputs.status || steps.poll.outputs.status }} runs: using: 'composite' @@ -73,7 +73,8 @@ runs: # We capture body+status separately so future failures self-explain # (instead of just "curl exit 22"). API key never echoed. BODY_FILE="$(mktemp)" - HTTP_CODE=$(curl -sS -o "$BODY_FILE" -w "%{http_code}" -X POST \ + HTTP_CODE=$(curl -sS --connect-timeout 10 --max-time 60 \ + -o "$BODY_FILE" -w "%{http_code}" -X POST \ -H "Content-Type: application/json" \ -H "x-api-key: ${RAFTER_API_KEY}" \ -d "{ @@ -92,19 +93,28 @@ runs: rm -f "$BODY_FILE" if [ "$HTTP_CODE" -lt 200 ] || [ "$HTTP_CODE" -ge 300 ]; then - ERROR=$(echo "$RESPONSE" | jq -r '.error // empty' 2>/dev/null || true) + ERROR=$(echo "$RESPONSE" | jq -r '.error // empty' 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true) echo "::error::Rafter scan trigger failed: HTTP ${HTTP_CODE}" if [ -n "$ERROR" ]; then echo "::error::Server: ${ERROR}" else - echo "Server response: ${RESPONSE}" + echo "Server response: $(printf '%s' "$RESPONSE" | tr -d '\r\n' | cut -c1-500)" fi exit 1 fi SCAN_ID=$(echo "$RESPONSE" | jq -r '.scan_id // empty') + # $GITHUB_OUTPUT is a key=value file: a newline in a server-controlled + # scan_id forges arbitrary step outputs, including status=completed. + # It also reaches ::error:: annotations and a request URL. + case "$SCAN_ID" in + *[!A-Za-z0-9_-]*) + echo "::error::Rafter returned a malformed scan id; refusing to continue" + exit 1 + ;; + esac if [ -z "$SCAN_ID" ]; then - ERROR=$(echo "$RESPONSE" | jq -r '.error // "Unknown error triggering scan"') + ERROR=$(echo "$RESPONSE" | jq -r '.error // "Unknown error triggering scan"' | tr -d '\r\n' | cut -c1-200) echo "::error::Failed to trigger scan (HTTP ${HTTP_CODE}): ${ERROR}" exit 1 fi @@ -121,38 +131,97 @@ runs: SCAN_ID: ${{ steps.scan.outputs.scan_id }} TIMEOUT_MINUTES: ${{ inputs.timeout-minutes }} run: | - MAX_POLLS=$(( TIMEOUT_MINUTES * 6 )) # Poll every 10s + # sable-l10k — a report is not durable the instant the scan flips to + # completed, so a poll can legitimately hit a 5xx (in practice + # "Failed to fetch report from storage: Object not found") on a scan + # that is perfectly healthy and readable seconds later. Retry those with + # backoff. Only give up once the failures stop looking transient. + # + # 404 counts as transient HERE and only here: the trigger step already + # handed us a scan_id, so a missing scan mid-poll is read-after-write + # lag rather than a wrong id. + case "$TIMEOUT_MINUTES" in + ''|*[!0-9]*) + echo "::error::timeout-minutes must be a whole number of minutes, got '${TIMEOUT_MINUTES}'" + exit 1 + ;; + esac + + MAX_TRANSIENT_FAILURES=5 + TRANSIENT_FAILURES=0 + LAST_ERROR="" + + # Wall-clock deadline so retry backoff cannot quietly stretch the + # documented timeout-minutes budget. + DEADLINE=$(( $(date +%s) + TIMEOUT_MINUTES * 60 )) POLL_COUNT=0 STATUS="pending" - while [ $POLL_COUNT -lt $MAX_POLLS ]; do + while [ "$(date +%s)" -lt "$DEADLINE" ]; do BODY_FILE="$(mktemp)" - HTTP_CODE=$(curl -sS -o "$BODY_FILE" -w "%{http_code}" \ + HTTP_CODE=$(curl -sS --connect-timeout 10 --max-time 60 \ + -o "$BODY_FILE" -w "%{http_code}" \ -H "x-api-key: ${RAFTER_API_KEY}" \ "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}") || { - echo "::warning::curl transport error during poll (will retry)" - cat "$BODY_FILE" || true + # A transport error is exactly as transient as a 5xx, and counts + # the same. Previously it retried on a flat 10s forever, which + # meant an unreachable backend reported "scan did not complete + # within N minutes" — a timeout message for a DNS failure. rm -f "$BODY_FILE" - sleep 10 + LAST_ERROR="curl transport error contacting ${RAFTER_URL}" + TRANSIENT_FAILURES=$((TRANSIENT_FAILURES+1)) + if [ "$TRANSIENT_FAILURES" -ge "$MAX_TRANSIENT_FAILURES" ]; then + echo "::error::Rafter could not reach the API for scan ${SCAN_ID} after ${MAX_TRANSIENT_FAILURES} attempts." + echo "::error::Check that ${RAFTER_URL} is reachable from this runner." + echo "::error::Last error: ${LAST_ERROR}" + echo "status=unreachable" >> "$GITHUB_OUTPUT" + exit 1 + fi + BACKOFF=$(( 2 ** TRANSIENT_FAILURES )) + echo "::warning::${LAST_ERROR}; retrying in ${BACKOFF}s (${TRANSIENT_FAILURES}/${MAX_TRANSIENT_FAILURES})" + sleep "$BACKOFF" POLL_COUNT=$((POLL_COUNT+1)) continue } RESPONSE=$(cat "$BODY_FILE") rm -f "$BODY_FILE" + if [ "$HTTP_CODE" -ge 500 ] || [ "$HTTP_CODE" -eq 408 ] || [ "$HTTP_CODE" -eq 404 ]; then + ERROR=$(echo "$RESPONSE" | jq -r '.error // empty' 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true) + LAST_ERROR="HTTP ${HTTP_CODE}${ERROR:+ — $ERROR}" + TRANSIENT_FAILURES=$((TRANSIENT_FAILURES+1)) + + if [ "$TRANSIENT_FAILURES" -ge "$MAX_TRANSIENT_FAILURES" ]; then + echo "::error::Rafter could not read the report for scan ${SCAN_ID} after ${MAX_TRANSIENT_FAILURES} attempts." + echo "::error::The scan itself may have finished — check it in your dashboard at ${RAFTER_URL}/dashboard" + echo "::error::Last response from the server: ${LAST_ERROR}" + echo "status=unreadable" >> "$GITHUB_OUTPUT" + exit 1 + fi + + BACKOFF=$(( 2 ** TRANSIENT_FAILURES )) # 2s, 4s, 8s, 16s + echo "::warning::Report not readable yet (${LAST_ERROR}); retrying in ${BACKOFF}s (${TRANSIENT_FAILURES}/${MAX_TRANSIENT_FAILURES})" + sleep "$BACKOFF" + POLL_COUNT=$((POLL_COUNT+1)) + continue + fi + if [ "$HTTP_CODE" -lt 200 ] || [ "$HTTP_CODE" -ge 300 ]; then - ERROR=$(echo "$RESPONSE" | jq -r '.error // empty' 2>/dev/null || true) + # 4xx other than 404/408: a bad key or a malformed request. Retrying + # will not help and would only delay a clear answer. + ERROR=$(echo "$RESPONSE" | jq -r '.error // empty' 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true) echo "::error::Rafter scan poll failed: HTTP ${HTTP_CODE}${ERROR:+ — $ERROR}" exit 1 fi + TRANSIENT_FAILURES=0 STATUS=$(echo "$RESPONSE" | jq -r '.status // "unknown"') if [ "$STATUS" = "completed" ] || [ "$STATUS" = "failed" ]; then break fi - echo "Scan status: ${STATUS} (poll $((POLL_COUNT+1))/${MAX_POLLS})" + echo "Scan status: ${STATUS} (poll $((POLL_COUNT+1)), $(( (DEADLINE - $(date +%s)) / 60 ))m of ${TIMEOUT_MINUTES}m budget left)" sleep 10 POLL_COUNT=$((POLL_COUNT+1)) done @@ -180,23 +249,54 @@ runs: run: | # fetch_results : HTTP-status aware GET that surfaces # server error body on non-2xx (avoids silent curl exit 22 failures). + # + # sable-l10k — same read-after-write race as the poll loop, and worse + # here: this runs the instant the scan reports completed, which is the + # likeliest moment for the report object to not be readable yet. Retry + # transient failures with backoff rather than failing the build. fetch_results() { local out="$1" local url="$2" - local code - code=$(curl -sS -o "$out" -w "%{http_code}" \ - -H "x-api-key: ${RAFTER_API_KEY}" "$url") || { - echo "::error::curl transport error fetching ${url}" - cat "$out" || true + local attempt=1 + local max_attempts=5 + local code body err last="" + + while :; do + if code=$(curl -sS --connect-timeout 10 --max-time 60 \ + -o "$out" -w "%{http_code}" \ + -H "x-api-key: ${RAFTER_API_KEY}" "$url"); then + if [ "$code" -ge 200 ] && [ "$code" -lt 300 ]; then + return 0 + fi + body=$(cat "$out" || true) + err=$(echo "$body" | jq -r '.error // empty' 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true) + last="HTTP ${code}${err:+ — $err}" + if [ "$code" -lt 500 ] && [ "$code" -ne 408 ] && [ "$code" -ne 404 ]; then + # Not transient — a bad key or malformed request. Say so now. + echo "::error::Rafter results fetch failed: ${last}" + echo "status=unreadable" >> "$GITHUB_OUTPUT" + return 1 + fi + else + last="curl transport error fetching ${url}" + fi + + if [ "$attempt" -ge "$max_attempts" ]; then + echo "::error::Rafter could not read the report for scan ${SCAN_ID} after ${max_attempts} attempts." + echo "::error::The scan itself finished — check it in your dashboard at ${RAFTER_URL}/dashboard" + echo "::error::Last response from the server: ${last}" + # Without this the declared `status` output falls back to the poll + # step, which already said `completed` — a failed report read would + # be reported to consumers as a clean scan. + echo "status=unreadable" >> "$GITHUB_OUTPUT" return 1 - } - if [ "$code" -lt 200 ] || [ "$code" -ge 300 ]; then - local body err - body=$(cat "$out" || true) - err=$(echo "$body" | jq -r '.error // empty' 2>/dev/null || true) - echo "::error::Rafter results fetch failed: HTTP ${code}${err:+ — $err}" - return 1 - fi + fi + + local backoff=$(( 2 ** attempt )) + echo "::warning::Report not readable yet (${last}); retrying in ${backoff}s (${attempt}/${max_attempts})" + sleep "$backoff" + attempt=$((attempt+1)) + done } fetch_results "${{ runner.temp }}/rafter-results.json" "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}" @@ -297,7 +397,7 @@ runs: gh pr comment "${{ github.event.pull_request.number }}" --body-file "$COMMENT_FILE" - name: Upload artifacts - if: always() && steps.poll.outputs.status == 'completed' + if: steps.results.outputs.status == 'completed' uses: actions/upload-artifact@v4 with: name: rafter-security-results diff --git a/github-action/tests/mock-rafter-api.py b/github-action/tests/mock-rafter-api.py new file mode 100644 index 00000000..2b77ab97 --- /dev/null +++ b/github-action/tests/mock-rafter-api.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Minimal stand-in for the Rafter backend, used to reproduce sable-l10k. + +Serves the two endpoints the GitHub Action talks to and injects exactly one +transient 500 into the poll sequence: + + POST /api/static/scan -> 200 {"scan_id": ...} + GET /api/static/scan?scan_id=.. -> poll 1: 200 {"status": "processing"} + poll 2: 500 {"error": "Failed to fetch + report from storage: Object not found"} + poll 3: 200 {"status": "completed", ...} + +A backend that is eventually consistent about report objects looks exactly like +this from the client's side. The question the repro answers is whether the +action survives it. + +Env: + PORT listen port (default 8787) + FAIL_ON 1-based GET index that starts failing (default 2) + FAIL_STATUS status code to fail with (default 500; 404 exercises the + read-after-write-lag branch) + FAIL_FOREVER if "1", every GET from FAIL_ON onward fails (persistent case) + FAIL_COUNT how many consecutive GETs fail starting at FAIL_ON (default 1; + ignored when FAIL_FOREVER is set) + COMPLETE_AFTER GET index from which status is "completed" (default FAIL_ON, + i.e. as soon as the injected failures are done). Set it higher + than the failure window to make the RESULTS fetch fail rather + than the poll. +""" +import json +import os +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import urlparse, parse_qs + +PORT = int(os.environ.get("PORT", "8787")) +FAIL_ON = int(os.environ.get("FAIL_ON", "2")) +FAIL_STATUS = int(os.environ.get("FAIL_STATUS", "500")) +FAIL_FOREVER = os.environ.get("FAIL_FOREVER") == "1" +FAIL_COUNT = int(os.environ.get("FAIL_COUNT", "1")) +COMPLETE_AFTER = int(os.environ.get("COMPLETE_AFTER", str(FAIL_ON))) + +SCAN_ID = "repro-sable-l10k-0001" + +state = {"polls": 0} + + +class Handler(BaseHTTPRequestHandler): + def _send(self, code, payload): + body = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + if urlparse(self.path).path != "/api/static/scan": + return self._send(404, {"error": "not found"}) + length = int(self.headers.get("Content-Length") or 0) + self.rfile.read(length) + self._send(200, {"scan_id": SCAN_ID}) + + def do_GET(self): + parsed = urlparse(self.path) + if parsed.path != "/api/static/scan": + return self._send(404, {"error": "not found"}) + + qs = parse_qs(parsed.query) + fmt = (qs.get("format") or ["json"])[0] + + state["polls"] += 1 + n = state["polls"] + + failing = (FAIL_FOREVER and n >= FAIL_ON) or ( + FAIL_ON <= n < FAIL_ON + FAIL_COUNT + ) + if failing: + # The verbatim customer-facing body. + return self._send( + FAIL_STATUS, + {"error": "Failed to fetch report from storage: Object not found"}, + ) + + if n < COMPLETE_AFTER: + return self._send(200, {"scan_id": SCAN_ID, "status": "processing"}) + + completed = {"scan_id": SCAN_ID, "status": "completed", "vulnerabilities": []} + if fmt == "md": + completed["markdown"] = "# Rafter\n\nNo findings.\n" + elif fmt == "sarif": + completed = {"version": "2.1.0", "runs": []} + return self._send(200, completed) + + def log_message(self, fmt, *args): + # Keep the runner log readable: one line per request, to stderr. + super().log_message(fmt, *args) + + +if __name__ == "__main__": + print(f"mock rafter api on :{PORT} (500 on poll #{FAIL_ON}, forever={FAIL_FOREVER})", flush=True) + HTTPServer(("127.0.0.1", PORT), Handler).serve_forever() diff --git a/github-action/tests/test-action-yml-defaults.sh b/github-action/tests/test-action-yml-defaults.sh index 084748e6..59444a34 100755 --- a/github-action/tests/test-action-yml-defaults.sh +++ b/github-action/tests/test-action-yml-defaults.sh @@ -66,6 +66,103 @@ else echo "PASS: 'none' branch of threshold-eval does not set FAIL=1" fi +# ── sable-l10k: poll-path retry contract ───────────────────────────────── +# These properties are subtle and cheap to "simplify" away. Each one, if +# dropped, reproduces a bug a paying customer already hit. + +# 5. 404 must be in the poll loop's TRANSIENT condition. It is safe only +# because the trigger step already handed us a scan_id, so a missing scan +# mid-poll is read-after-write lag rather than a wrong id. +if grep -qE '\$HTTP_CODE" -ge 500 \] \|\| \[ "\$HTTP_CODE" -eq 408 \] \|\| \[ "\$HTTP_CODE" -eq 404' "$ACTION_YML"; then + echo "PASS: poll loop treats 5xx/408/404 as transient" +else + echo "FAIL: poll loop's transient condition changed — 404/408/5xx must all retry" + failures=$((failures+1)) +fi + +# 6. A transport error must count toward the SAME failure budget as a 5xx. +# When it did not, an unreachable backend reported "scan did not complete +# within N minutes" — a timeout message for a DNS failure. +if awk '/curl transport error contacting/,/^ \}/' "$ACTION_YML" \ + | grep -q 'TRANSIENT_FAILURES=\$((TRANSIENT_FAILURES+1))'; then + echo "PASS: transport errors count toward the transient-failure budget" +else + echo "FAIL: poll loop's transport-error branch no longer counts toward the budget" + failures=$((failures+1)) +fi + +# 7. The give-up message must be actionable: name the scan, and offer a next +# step. Raw storage wording ("Object not found") alone is not a message a +# customer can act on. +if grep -q 'could not read the report for scan \${SCAN_ID}' "$ACTION_YML" \ + && grep -q 'check it in your dashboard at' "$ACTION_YML"; then + echo "PASS: give-up message names the scan and offers a next step" +else + echo "FAIL: give-up message no longer names the scan id or a next step" + failures=$((failures+1)) +fi + +# 8. Both give-up paths in the results fetch must record status=unreadable. +# Without it the declared `status` output falls back to the poll step's +# `completed`, and a failed report read is reported as a clean scan. +unreadable_writes=$(grep -c 'status=unreadable' "$ACTION_YML" || true) +if [ "$unreadable_writes" -ge 3 ]; then + echo "PASS: poll and both results-fetch give-up paths record status=unreadable" +else + echo "FAIL: expected >=3 status=unreadable writes, found ${unreadable_writes}" + failures=$((failures+1)) +fi + +# 9. The artifact upload must be gated on the RESULTS step, not the poll step. +# Gated on the poll step it published the error body as rafter-results.json. +if grep -qE "if: steps\.results\.outputs\.status == 'completed'" "$ACTION_YML"; then + echo "PASS: artifact upload gated on a successful results fetch" +else + echo "FAIL: artifact upload is not gated on steps.results.outputs.status" + failures=$((failures+1)) +fi + +# 10. Backoff must be exponential. A flat or zeroed backoff gives an +# eventually-consistent object store no time to converge. +if grep -q 'BACKOFF=\$(( 2 \*\* TRANSIENT_FAILURES ))' "$ACTION_YML" \ + && grep -q 'backoff=\$(( 2 \*\* attempt ))' "$ACTION_YML"; then + echo "PASS: both retry loops back off exponentially" +else + echo "FAIL: a retry loop's backoff is no longer exponential" + failures=$((failures+1)) +fi + + +# 11. Server-controlled error text must be newline-stripped and length-capped +# before it reaches a workflow command. A newline forges ::add-mask:: / +# ::stop-commands:: / fabricated ::error:: annotations. +sanitized=$(grep -cF 'cut -c1-' "$ACTION_YML" || true) +stripped=$(grep -cF "tr -d " "$ACTION_YML" || true) +if [ "$sanitized" -ge 5 ] && [ "$stripped" -ge 5 ]; then + echo "PASS: server-controlled text newline-stripped and capped at ${sanitized} sites" +else + echo "FAIL: expected >=5 sanitized sites, found cut=${sanitized} tr=${stripped}" + failures=$((failures+1)) +fi + +# 12. TIMEOUT_MINUTES is evaluated inside bash arithmetic, where a value like +# 'x[$(cmd)]' executes. It must be validated first. +if grep -q 'case "\$TIMEOUT_MINUTES" in' "$ACTION_YML"; then + echo "PASS: timeout-minutes validated before arithmetic evaluation" +else + echo "FAIL: timeout-minutes is no longer validated before arithmetic use" + failures=$((failures+1)) +fi + +# 13. The server-controlled scan id must be validated before it reaches +# \$GITHUB_OUTPUT, where a newline forges step outputs. +if grep -q 'case "\$SCAN_ID" in' "$ACTION_YML"; then + echo "PASS: scan id validated before it reaches \$GITHUB_OUTPUT" +else + echo "FAIL: scan id is no longer validated" + failures=$((failures+1)) +fi + echo "" echo "── results ───────────────────────────────────────────────────────────" echo "Failures: $failures" diff --git a/node/src/commands/backend/get.ts b/node/src/commands/backend/get.ts index 76019d12..66f47469 100644 --- a/node/src/commands/backend/get.ts +++ b/node/src/commands/backend/get.ts @@ -7,7 +7,7 @@ import { EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND } from "../../utils/api.js"; -import { handleScanStatus } from "./scan-status.js"; +import { handleScanStatus, fetchScanWithRetry, PollGaveUpError } from "./scan-status.js"; export function createGetCommand(): Command { return new Command("get") @@ -20,9 +20,14 @@ export function createGetCommand(): Command { const key = resolveKey(opts.apiKey); if (!opts.interactive) { try { - const { data } = await axios.get( - `${API}/static/scan`, - { params: { scan_id, format: opts.format }, headers: { "x-api-key": key } } + // sable-l10k — retried, because this is the command the poll loop's + // give-up message recommends. A remedy defeated by the same transient + // failure it is recommended for is not a remedy. + const { data } = await fetchScanWithRetry( + scan_id, + { "x-api-key": key }, + opts.format, + opts.quiet ); const exitCode = writePayload(data, opts.format, opts.quiet); process.exit(exitCode); @@ -30,6 +35,8 @@ export function createGetCommand(): Command { if (e.response?.status === 404) { console.error(`Scan '${scan_id}' not found`); process.exit(EXIT_SCAN_NOT_FOUND); + } else if (e instanceof PollGaveUpError) { + console.error(e.message); } else if (e.response?.data) { console.error(e.response.data); } else if (e instanceof Error) { diff --git a/node/src/commands/backend/scan-status.ts b/node/src/commands/backend/scan-status.ts index ea551f20..2aa1bb85 100644 --- a/node/src/commands/backend/scan-status.ts +++ b/node/src/commands/backend/scan-status.ts @@ -2,63 +2,299 @@ import axios from "axios"; import ora from "ora"; import { API, + API_TIMEOUT_SHORT_MS, writePayload, EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND } from "../../utils/api.js"; import { fmt as output } from "../../utils/formatter.js"; +/** + * sable-l10k — the report a scan writes is not durable the instant the scan + * flips to completed, so a poll can legitimately hit a 5xx (commonly + * "Failed to fetch report from storage: Object not found") on an otherwise + * healthy scan. Retry those instead of failing the whole run; a scan that + * would have succeeded 10 seconds later must not die on one bad read. + * + * 404 is transient only AFTER the scan is known to exist: once the first poll + * has succeeded, a missing scan is read-after-write lag rather than a wrong id. + * On the first poll a 404 is still fatal. + */ +export const MAX_TRANSIENT_POLL_FAILURES = 5; + +/** + * Total transient failures tolerated across one `handleScanStatus` call. + * + * The consecutive counter resets on every success, which is what we want — a + * twenty-minute scan with one blip at minute 2 and another at minute 18 should + * not die. But reset-on-success alone means a backend alternating 200/500 + * forever never exhausts the budget, and the CLI has no wall-clock deadline to + * stop it. This is the backstop for that. + */ +export const MAX_TOTAL_TRANSIENT_POLL_FAILURES = 20; + +/** Longest single error detail we will echo back. Servers can be verbose. */ +const MAX_ERROR_DETAIL_CHARS = 200; + +/** + * Transient = the request never got an answer, or got one the server itself + * describes as temporary. + * + * `scanExists` gates 404: before the first successful poll a 404 means the + * scan id is wrong, and retrying it just delays a clear answer. + */ +function isTransientPollError(e: any, scanExists: boolean): boolean { + const status = e?.response?.status; + if (status === undefined) { + // Retry only errors that came from the HTTP layer. A TypeError thrown from + // our own code also has no `response`, and must not be mistaken for a flaky + // backend and retried five times. + return Boolean(e?.isAxiosError || e?.request); + } + if (status === 404) return scanExists; + return status >= 500 || status === 408; +} + +function truncate(value: unknown): string { + // A server is free to answer {"error": {"message": "..."}}. Coerce before + // touching string methods — this used to throw, which turned a retryable + // failure into an immediate crash with a nonsense message. + const s = typeof value === "string" ? value : JSON.stringify(value) ?? String(value); + const flat = s.replace(/[\r\n]+/g, " ").trim(); + return flat.length > MAX_ERROR_DETAIL_CHARS + ? `${flat.slice(0, MAX_ERROR_DETAIL_CHARS)}…` + : flat; +} + +function describeHttpError(e: any): string { + const status = e?.response?.status; + const data = e?.response?.data; + let detail: unknown = ""; + if (typeof data === "string") { + detail = data; + } else if (data && typeof data === "object") { + detail = (data as any).error ?? data; + } else if (e instanceof Error) { + detail = e.message; + } + const detailText = truncate(detail); + return status + ? `HTTP ${status}${detailText ? ` — ${detailText}` : ""}` + : detailText || String(e); +} + +/** + * The message a customer actually sees when the report never becomes readable. + * Storage-layer wording ("Object not found") is kept as supporting detail, not + * as the whole explanation, and the next action is spelled out. + */ +export function unreadableReportMessage( + scan_id: string, + lastError: string, + attempts: number = MAX_TRANSIENT_POLL_FAILURES, + reachedServer: boolean = true +): string { + if (!reachedServer) { + return ( + `Rafter could not reach the API after ${attempts} attempts.\n` + + `Check your network and that https://rafter.so is reachable from here.\n` + + `Your scan id is ${scan_id} — the scan may still be running.\n` + + `Last error: ${lastError}` + ); + } + return ( + `Rafter could not read the report for scan ${scan_id} after ` + + `${attempts} attempts.\n` + + `The scan itself may have finished — retry with: rafter get ${scan_id}\n` + + `or open the scan in your dashboard at https://rafter.so/dashboard\n` + + `Last response from the server: ${lastError}` + ); +} + +export const BASE_BACKOFF_MS = 2000; + +/** 2s, 4s, 8s, 16s — the 5th failure gives up rather than sleeping again. */ +export function backoffMs(consecutiveFailures: number): number { + return BASE_BACKOFF_MS * 2 ** (consecutiveFailures - 1); +} + +/** + * Thrown when polling gives up after repeated transient failures. Carries the + * customer-facing message so callers do not have to rebuild it. + */ +export class PollGaveUpError extends Error {} + +/** + * A failure budget shared across every poll in one `handleScanStatus` call. + * + * Counting per-request would let a backend that alternates 200/500 forever + * reset the counter on each success and never exhaust it — the CLI has no + * wall-clock deadline, so that loop would never end. + */ +class FailureBudget { + consecutive = 0; + total = 0; + last = ""; + /** False once any failure carried no HTTP response at all. */ + lastReachedServer = true; + + record(detail: string, reachedServer: boolean): number { + this.consecutive += 1; + this.total += 1; + this.last = detail; + this.lastReachedServer = reachedServer; + return this.consecutive; + } + + /** A success clears the consecutive run, but never refunds the total. */ + reset(): void { + this.consecutive = 0; + } + + get exhausted(): boolean { + return ( + this.consecutive >= MAX_TRANSIENT_POLL_FAILURES || + this.total >= MAX_TOTAL_TRANSIENT_POLL_FAILURES + ); + } +} + +type RetryNotice = (attempt: number, waitMs: number, detail: string) => void; + +/** + * One poll, with retry/backoff over transient failures. + * Non-transient errors are rethrown for the caller to classify. + */ +async function pollUntilReadable( + scan_id: string, + headers: any, + fmt: string, + budget: FailureBudget, + scanExists: boolean, + onRetry?: RetryNotice +): Promise { + for (;;) { + try { + const res = await axios.get(`${API}/static/scan`, { + params: { scan_id, format: fmt }, + headers, + // Without this a hung server stalls inside a single request, and the + // retry loop can only notice between attempts. + timeout: API_TIMEOUT_SHORT_MS, + }); + budget.reset(); + return res; + } catch (e: any) { + if (!isTransientPollError(e, scanExists)) throw e; + + const attempt = budget.record( + describeHttpError(e), + e?.response?.status !== undefined + ); + if (budget.exhausted) { + throw new PollGaveUpError( + unreadableReportMessage( + scan_id, + budget.last, + budget.total, + budget.lastReachedServer + ) + ); + } + + const waitMs = backoffMs(attempt); + onRetry?.(attempt, waitMs, budget.last); + await new Promise((r) => setTimeout(r, waitMs)); + } + } +} + +/** + * A single scan fetch with the same retry budget the poll loop uses. + * + * `rafter get ` is what the give-up message tells customers to run, so it + * must not be defeated by exactly the transient failure that produced the + * message. A 404 here is still fatal — that is a wrong id, not lag. + */ +export async function fetchScanWithRetry( + scan_id: string, + headers: any, + fmt: string, + quiet?: boolean +): Promise { + const budget = new FailureBudget(); + const onRetry: RetryNotice | undefined = quiet + ? undefined + : (attempt, waitMs, detail) => { + console.error( + `Report not readable yet (${detail}); retrying in ${Math.round(waitMs / 1000)}s ` + + `(${attempt}/${MAX_TRANSIENT_POLL_FAILURES})` + ); + }; + return pollUntilReadable(scan_id, headers, fmt, budget, false, onRetry); +} + +const IN_PROGRESS = ["queued", "pending", "processing"]; + export async function handleScanStatus(scan_id: string, headers: any, fmt: string, quiet?: boolean): Promise { - // First poll + const budget = new FailureBudget(); + + // Retries are printed to stderr, not just into the spinner: ora renders + // nothing on a non-TTY, and CI is exactly where this diagnostic matters. + const onRetry: RetryNotice | undefined = quiet + ? undefined + : (attempt, waitMs, detail) => { + console.error( + `Report not readable yet (${detail}); retrying in ${Math.round(waitMs / 1000)}s ` + + `(${attempt}/${MAX_TRANSIENT_POLL_FAILURES})` + ); + }; + + // First poll. A 404 here really does mean "no such scan" — do not retry it. + // Transient 5xx IS retried, so that the `rafter get ` this command + // recommends on failure is not itself defeated by one bad read. let poll; try { - poll = await axios.get( - `${API}/static/scan`, - { params: { scan_id, format: fmt }, headers } - ); + poll = await pollUntilReadable(scan_id, headers, fmt, budget, false, onRetry); } catch (e: any) { - if (e.response?.status === 404) { + if (e?.response?.status === 404) { console.error(output.error(`Scan '${scan_id}' not found`)); return EXIT_SCAN_NOT_FOUND; } - console.error(output.error(`${e.response?.data || e.message}`)); + console.error( + output.error(e instanceof PollGaveUpError ? e.message : describeHttpError(e)) + ); return EXIT_GENERAL_ERROR; } let status = poll.data.status; - if (["queued", "pending", "processing"].includes(status)) { - if (!quiet) { - const spinner = ora("Waiting for scan to complete... (this could take several minutes)").start(); - while (["queued", "pending", "processing"].includes(status)) { + if (IN_PROGRESS.includes(status)) { + const spinner = quiet + ? undefined + : ora("Waiting for scan to complete... (this could take several minutes)").start(); + + try { + while (IN_PROGRESS.includes(status)) { await new Promise((r) => setTimeout(r, 10000)); - poll = await axios.get( - `${API}/static/scan`, - { params: { scan_id, format: fmt }, headers } - ); + poll = await pollUntilReadable(scan_id, headers, fmt, budget, true, onRetry); status = poll.data.status; if (status === "completed") { - spinner.succeed("Scan completed"); + spinner?.succeed("Scan completed"); return writePayload(poll.data, fmt, quiet); } else if (status === "failed") { - spinner.fail("Scan failed"); + spinner?.fail("Scan failed"); return EXIT_GENERAL_ERROR; } } + } catch (e: any) { + spinner?.fail("Could not retrieve scan report"); + console.error( + output.error(e instanceof PollGaveUpError ? e.message : describeHttpError(e)) + ); + return EXIT_GENERAL_ERROR; + } + if (!quiet) { console.error(`Scan status: ${status}`); - } else { - while (["queued", "pending", "processing"].includes(status)) { - await new Promise((r) => setTimeout(r, 10000)); - poll = await axios.get( - `${API}/static/scan`, - { params: { scan_id, format: fmt }, headers } - ); - status = poll.data.status; - if (status === "completed") { - return writePayload(poll.data, fmt, quiet); - } else if (status === "failed") { - return EXIT_GENERAL_ERROR; - } - } } } else if (status === "completed") { if (!quiet) { diff --git a/node/src/utils/api.ts b/node/src/utils/api.ts index fef9426a..740431bd 100644 --- a/node/src/utils/api.ts +++ b/node/src/utils/api.ts @@ -8,6 +8,12 @@ export function apiUrl(path: string): string { } // Exit codes +/** + * Read timeout for short-lived API calls (status polls and the like), in ms. + * Mirrors the read half of Python's `API_TIMEOUT_SHORT`. + */ +export const API_TIMEOUT_SHORT_MS = 30_000; + export const EXIT_SUCCESS = 0; export const EXIT_GENERAL_ERROR = 1; export const EXIT_SCAN_NOT_FOUND = 2; diff --git a/node/tests/scan-poll-transient-500.test.ts b/node/tests/scan-poll-transient-500.test.ts new file mode 100644 index 00000000..b4bf1ba3 --- /dev/null +++ b/node/tests/scan-poll-transient-500.test.ts @@ -0,0 +1,385 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +/** + * sable-l10k — a paying customer's GitHub Actions run died on + * "Rafter scan poll failed: HTTP 500 — Failed to fetch report from storage: Object not found" + * + * A report is not durable the instant a scan flips to completed, so a poll can + * hit a 5xx on a scan that is perfectly healthy seconds later. These tests pin + * the contract: transient read failures are retried, genuinely-missing reports + * still fail, and the failure message is one a customer can act on. + */ + +vi.mock("axios"); +vi.mock("ora", () => ({ + default: () => ({ + start: vi.fn().mockReturnThis(), + succeed: vi.fn().mockReturnThis(), + fail: vi.fn().mockReturnThis(), + stop: vi.fn().mockReturnThis(), + text: "", + }), +})); + +import axios from "axios"; +import { + handleScanStatus, + unreadableReportMessage, + backoffMs, + BASE_BACKOFF_MS, + MAX_TRANSIENT_POLL_FAILURES, + MAX_TOTAL_TRANSIENT_POLL_FAILURES, +} from "../src/commands/backend/scan-status.js"; +import { EXIT_SUCCESS, EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND } from "../src/utils/api.js"; + +const mockedAxios = vi.mocked(axios, true); + +/** The verbatim body the customer saw. */ +const OBJECT_NOT_FOUND = { + response: { status: 500, data: { error: "Failed to fetch report from storage: Object not found" } }, +}; + +function httpError(status: number, error?: string) { + return { response: { status, data: error ? { error } : undefined } }; +} + +describe("handleScanStatus — transient poll failures (sable-l10k)", () => { + const headers = { "x-api-key": "test-key" }; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + }); + + afterEach(() => { + // Belt and braces: every test restores real timers itself (see above), but + // a failing assertion can skip that line. + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("rides out a single 500 mid-poll and completes", async () => { + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md"); + await vi.advanceTimersByTimeAsync(10000); // poll interval + await vi.advanceTimersByTimeAsync(2000); // first backoff + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_SUCCESS); + expect(mockedAxios.get).toHaveBeenCalledTimes(3); + }); + + it("rides out several consecutive 500s, backing off between them", async () => { + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md"); + await vi.advanceTimersByTimeAsync(10000); + await vi.advanceTimersByTimeAsync(2000 + 4000 + 8000); // 3 backoffs + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_SUCCESS); + expect(mockedAxios.get).toHaveBeenCalledTimes(5); + }); + + it("treats a mid-poll 404 as read-after-write lag, not a missing scan", async () => { + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(httpError(404)) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md"); + await vi.advanceTimersByTimeAsync(10000); + await vi.advanceTimersByTimeAsync(2000); + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_SUCCESS); + }); + + it("retries a transient 500 on the FIRST poll", async () => { + // The give-up message tells the user to run `rafter get `, which + // re-enters at the first poll. If that path did not retry, the remedy we + // recommend would be defeated by one bad read. + mockedAxios.get + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md", true); + await vi.advanceTimersByTimeAsync(2000); + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_SUCCESS); + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it("still reports 'not found' when the FIRST poll 404s", async () => { + mockedAxios.get.mockRejectedValueOnce(httpError(404)); + + const code = await handleScanStatus("nope", headers, "md"); + + expect(code).toBe(EXIT_SCAN_NOT_FOUND); + expect(mockedAxios.get).toHaveBeenCalledTimes(1); + }); + + it("gives up when the report never becomes readable", async () => { + mockedAxios.get.mockResolvedValueOnce({ data: { status: "processing" } }); + for (let i = 0; i < MAX_TRANSIENT_POLL_FAILURES; i++) { + mockedAxios.get.mockRejectedValueOnce(OBJECT_NOT_FOUND); + } + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md"); + await vi.advanceTimersByTimeAsync(10000); + await vi.advanceTimersByTimeAsync(2000 + 4000 + 8000 + 16000); + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_GENERAL_ERROR); + // One in-progress poll plus exactly the allowed number of retries. + expect(mockedAxios.get).toHaveBeenCalledTimes(1 + MAX_TRANSIENT_POLL_FAILURES); + }); + + it("does not retry a non-transient error (403)", async () => { + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(httpError(403, "Invalid API key")); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md"); + await vi.advanceTimersByTimeAsync(10000); + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_GENERAL_ERROR); + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it("does NOT retry a plain programming error", async () => { + // A TypeError thrown from our own code also has no `response`. Retrying it + // five times would report a local bug as a flaky backend. + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(new TypeError("x is not a function")); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md", true); + await vi.advanceTimersByTimeAsync(10000); + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_GENERAL_ERROR); + expect(mockedAxios.get).toHaveBeenCalledTimes(2); + }); + + it("retries transport errors that carry no HTTP response", async () => { + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce( + Object.assign(new Error("read ECONNRESET"), { + isAxiosError: true, + code: "ECONNRESET", + request: {}, + }) + ) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + vi.useFakeTimers(); + const promise = handleScanStatus("s1", headers, "md"); + await vi.advanceTimersByTimeAsync(10000); + await vi.advanceTimersByTimeAsync(2000); + const code = await promise; + vi.useRealTimers(); + + expect(code).toBe(EXIT_SUCCESS); + }); +}); + +describe("backoff schedule (sable-l10k)", () => { + const headers = { "x-api-key": "test-key" }; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stdout, "write").mockImplementation(() => true); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("is exponential, not flat — 2s, 4s, 8s, 16s", () => { + expect(BASE_BACKOFF_MS).toBeGreaterThan(0); + expect([1, 2, 3, 4].map(backoffMs)).toEqual([2000, 4000, 8000, 16000]); + }); + + /** + * Record every delay the code asks for and fire the callback immediately. + * This pins the SCHEDULE rather than an upper bound: with BASE_BACKOFF_MS + * mutated to 0, or `2 ** (n-1)` mistyped as `2 * (n-1)`, the recorded + * sequence changes and the test fails. A call-count assertion would not. + */ + function recordDelays(): number[] { + const delays: number[] = []; + const real = globalThis.setTimeout; + vi.stubGlobal("setTimeout", ((fn: any, ms?: number) => { + delays.push(ms ?? 0); + return real(fn, 0); + }) as any); + return delays; + } + + it("actually SLEEPS the 2/4/8/16 schedule between retries", async () => { + mockedAxios.get.mockResolvedValueOnce({ data: { status: "processing" } }); + for (let i = 0; i < MAX_TRANSIENT_POLL_FAILURES; i++) { + mockedAxios.get.mockRejectedValueOnce(OBJECT_NOT_FOUND); + } + + const delays = recordDelays(); + const code = await handleScanStatus("s1", headers, "md", true); + + expect(code).toBe(EXIT_GENERAL_ERROR); + // 10s poll interval, then the four backoffs preceding the fifth failure. + expect(delays).toEqual([10000, 2000, 4000, 8000, 16000]); + }); + + it("restarts the backoff after a successful poll clears the run", async () => { + // Two blips far apart must NOT add up to a give-up. + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockRejectedValueOnce(OBJECT_NOT_FOUND) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + const delays = recordDelays(); + const code = await handleScanStatus("s1", headers, "md", true); + + expect(code).toBe(EXIT_SUCCESS); + expect(delays).toEqual([10000, 2000, 4000, 10000, 2000, 4000]); + }); + + it("caps TOTAL transient failures, so a flapping server cannot loop forever", async () => { + // Alternating success/failure resets the consecutive counter every time. + // Without a total cap, and with no wall-clock deadline in the CLI, that + // loop never terminates. + // Endless flapping: the mock never drains, so the ONLY thing that can stop + // this loop is the total cap. (With the cap deleted the test hangs rather + // than passing on a drained-queue TypeError, which is what it used to do.) + // Hard stop well past the cap, so a missing cap fails loudly here instead + // of hanging the suite. + const ceiling = MAX_TOTAL_TRANSIENT_POLL_FAILURES * 4; + let call = 0; + mockedAxios.get.mockImplementation(async () => { + call += 1; + if (call > ceiling) { + throw new Error(`total transient-failure cap not enforced (${call} calls)`); + } + if (call === 1) return { data: { status: "processing" } }; + if (call % 2 === 0) throw OBJECT_NOT_FOUND; + return { data: { status: "processing" } }; + }); + + recordDelays(); + const code = await handleScanStatus("s1", headers, "md", true); + + expect(code).toBe(EXIT_GENERAL_ERROR); + expect(call).toBeLessThanOrEqual(ceiling); + // Bounded by the TOTAL budget: one success per failure, plus the opener. + expect(call).toBe(MAX_TOTAL_TRANSIENT_POLL_FAILURES * 2); + }); + + it("reports the real attempt count, not the consecutive cap", async () => { + let call = 0; + mockedAxios.get.mockImplementation(async () => { + call += 1; + if (call === 1) return { data: { status: "processing" } }; + if (call % 2 === 0) throw OBJECT_NOT_FOUND; + return { data: { status: "processing" } }; + }); + const errors: string[] = []; + vi.spyOn(console, "error").mockImplementation((m: any) => { + errors.push(String(m)); + }); + + recordDelays(); + await handleScanStatus("s1", headers, "md", true); + + // 20 failures happened; claiming "after 5 attempts" would be a lie. + expect(errors.join("\n")).toContain(`after ${MAX_TOTAL_TRANSIENT_POLL_FAILURES} attempts`); + }); + + it("survives a nested JSON error object instead of crashing", async () => { + // A server may answer {"error": {"message": "..."}}. Calling string + // methods on that object used to throw, defeating the retry entirely. + const nested = { response: { status: 500, data: { error: { message: "nested" } } } }; + mockedAxios.get + .mockResolvedValueOnce({ data: { status: "processing" } }) + .mockRejectedValueOnce(nested) + .mockResolvedValueOnce({ data: { status: "completed", markdown: "# Done" } }); + + recordDelays(); + const code = await handleScanStatus("s1", headers, "md", true); + + expect(code).toBe(EXIT_SUCCESS); + expect(mockedAxios.get).toHaveBeenCalledTimes(3); + }); + + it("truncates a very long server error instead of echoing it whole", async () => { + const huge = "x".repeat(5000); + mockedAxios.get.mockRejectedValueOnce({ + response: { status: 500, data: { error: huge } }, + }); + for (let i = 0; i < MAX_TRANSIENT_POLL_FAILURES; i++) { + mockedAxios.get.mockRejectedValueOnce({ + response: { status: 500, data: { error: huge } }, + }); + } + const errors: string[] = []; + vi.spyOn(console, "error").mockImplementation((m: any) => { + errors.push(String(m)); + }); + + recordDelays(); + await handleScanStatus("s1", headers, "md", true); + + expect(errors.join("\n")).not.toContain(huge); + expect(errors.join("\n")).toContain("…"); + }); +}); + +describe("unreadableReportMessage", () => { + it("gives the customer the scan id and a next step, not just storage jargon", () => { + const msg = unreadableReportMessage( + "scan-abc", + "HTTP 500 — Failed to fetch report from storage: Object not found" + ); + + expect(msg).toContain("scan-abc"); + expect(msg).toContain("rafter get scan-abc"); + expect(msg).toContain("dashboard"); + // The raw server wording survives as supporting detail... + expect(msg).toContain("Object not found"); + // ...but is not the whole message. + expect(msg.split("\n").length).toBeGreaterThan(1); + }); +}); diff --git a/node/tests/scan-remote.test.ts b/node/tests/scan-remote.test.ts index 709ff2b4..c2eefbac 100644 --- a/node/tests/scan-remote.test.ts +++ b/node/tests/scan-remote.test.ts @@ -91,14 +91,19 @@ describe("handleScanStatus", () => { expect(code).toBe(EXIT_GENERAL_ERROR); }); - it("returns EXIT_GENERAL_ERROR for non-404 network error", async () => { + // sable-l10k changed this: a 500 on the first poll is now RETRIED, because + // the give-up message tells the user to run `rafter get `, which + // re-enters here. A non-transient status is what still fails immediately. + // The retry/exhaustion paths are covered in scan-poll-transient-500.test.ts. + it("returns EXIT_GENERAL_ERROR for a non-transient error", async () => { mockedAxios.get.mockRejectedValueOnce({ - response: { status: 500, data: "Internal server error" }, + response: { status: 403, data: "Forbidden" }, message: "Request failed", }); const code = await handleScanStatus("s1", headers, "md"); expect(code).toBe(EXIT_GENERAL_ERROR); + expect(mockedAxios.get).toHaveBeenCalledTimes(1); }); it("polls when status is queued, then returns on completed", async () => { diff --git a/python/rafter_cli/commands/backend.py b/python/rafter_cli/commands/backend.py index ea988cae..5e7314b1 100644 --- a/python/rafter_cli/commands/backend.py +++ b/python/rafter_cli/commands/backend.py @@ -88,40 +88,267 @@ def _confirm_plus_scan(mode: str, yes: bool) -> None: raise typer.Exit(code=EXIT_CONFIRMATION_REQUIRED) +# sable-l10k — the report a scan writes is not durable the instant the scan +# flips to completed, so a poll can legitimately hit a 5xx (commonly +# "Failed to fetch report from storage: Object not found") on an otherwise +# healthy scan. Retry those instead of failing the whole run; a scan that would +# have succeeded 10 seconds later must not die on one bad read. +MAX_TRANSIENT_POLL_FAILURES = 5 + +#: Total transient failures tolerated across one interactive call. +#: +#: The consecutive counter resets on every success, which is what we want — a +#: twenty-minute scan with one blip at minute 2 and another at minute 18 should +#: not die. But reset-on-success alone means a backend alternating 200/500 +#: forever never exhausts the budget, and the CLI has no wall-clock deadline to +#: stop it. This is the backstop for that. +MAX_TOTAL_TRANSIENT_POLL_FAILURES = 20 + +BASE_BACKOFF_SECONDS = 2 + +#: Longest single error detail we will echo back. Servers can be verbose. +MAX_ERROR_DETAIL_CHARS = 200 + +IN_PROGRESS = ("queued", "pending", "processing") + + +class PollGaveUpError(RuntimeError): + """Polling gave up after exhausting its retry budget. + + Carries the customer-facing message so callers need not rebuild it. + """ + + +class PollFatalError(RuntimeError): + """A poll failed in a way retrying cannot fix (bad key, bad request). + + Distinct from :class:`PollGaveUpError` so that "we tried five times" is + never confused with "we did not try at all". + """ + + def __init__(self, message: str, status_code: "int | None" = None): + super().__init__(message) + self.status_code = status_code + + +def backoff_seconds(consecutive_failures: int) -> int: + """2s, 4s, 8s, 16s — the 5th failure gives up rather than sleeping again.""" + return BASE_BACKOFF_SECONDS * 2 ** (consecutive_failures - 1) + + +def _is_transient_poll_status(status_code: int, scan_exists: bool) -> bool: + """Transient = the server itself describes the condition as temporary. + + ``scan_exists`` gates 404: before the first successful poll a 404 means the + scan id is wrong, and retrying it just delays a clear answer. After it, a + missing scan is read-after-write lag. + """ + if status_code == 404: + return scan_exists + return status_code >= 500 or status_code == 408 + + +def _truncate(value) -> str: + """Coerce before truncating. + + A server is free to answer ``{"error": {"message": "..."}}``. This used to + call ``.split()`` on a dict, raising an ``AttributeError`` that no caller + catches — turning a retryable failure into an unhandled traceback. + """ + if value is None: + return "" + text = value if isinstance(value, str) else json.dumps(value, default=str) + flat = " ".join(text.split()) + if len(flat) > MAX_ERROR_DETAIL_CHARS: + return flat[:MAX_ERROR_DETAIL_CHARS] + "\u2026" + return flat + + +def _describe_http_error(status_code: int, body: str) -> str: + detail = body + try: + parsed = json.loads(body) + if isinstance(parsed, dict): + detail = parsed.get("error") or body + except (ValueError, TypeError): + pass + detail = _truncate(detail) + return f"HTTP {status_code}" + (f" \u2014 {detail}" if detail else "") + + +def unreadable_report_message( + scan_id: str, + last_error: str, + attempts: int = MAX_TRANSIENT_POLL_FAILURES, + reached_server: bool = True, +) -> str: + """The message a customer actually sees when the report never becomes readable. + + Storage-layer wording ("Object not found") is kept as supporting detail, not + as the whole explanation, and the next action is spelled out. ``attempts`` is + the real count, which is not always ``MAX_TRANSIENT_POLL_FAILURES`` — the + total budget can trip first. + """ + if not reached_server: + return ( + f"Rafter could not reach the API after {attempts} attempts.\n" + "Check your network and that https://rafter.so is reachable from here.\n" + f"Your scan id is {scan_id} \u2014 the scan may still be running.\n" + f"Last error: {last_error}" + ) + return ( + f"Rafter could not read the report for scan {scan_id} after " + f"{attempts} attempts.\n" + f"The scan itself may have finished \u2014 retry with: rafter get {scan_id}\n" + f"or open the scan in your dashboard at https://rafter.so/dashboard\n" + f"Last response from the server: {last_error}" + ) + + +class _FailureBudget: + """A failure budget shared across every poll in one interactive call. + + Counting per-request would let a backend that alternates 200/500 forever + reset the counter on each success and never exhaust it — the CLI has no + wall-clock deadline, so that loop would never end. + """ + + def __init__(self) -> None: + self.consecutive = 0 + self.total = 0 + self.last = "" + #: False once any failure carried no HTTP response at all. + self.last_reached_server = True + + def record(self, detail: str, reached_server: bool = True) -> int: + self.consecutive += 1 + self.total += 1 + self.last = detail + self.last_reached_server = reached_server + return self.consecutive + + def reset(self) -> None: + """A success clears the consecutive run, but never refunds the total.""" + self.consecutive = 0 + + @property + def exhausted(self) -> bool: + return ( + self.consecutive >= MAX_TRANSIENT_POLL_FAILURES + or self.total >= MAX_TOTAL_TRANSIENT_POLL_FAILURES + ) + + +def _poll_until_readable( + scan_id: str, + headers: dict, + fmt: str, + quiet: bool, + budget: "_FailureBudget", + scan_exists: bool, +): + """One poll, retrying transient failures with exponential backoff. + + Returns the successful response. Raises ``PollGaveUpError`` once the retry + budget is spent and ``PollFatalError`` for anything retrying cannot fix. + """ + while True: + try: + resp = requests.get( + f"{API_BASE}/static/scan", + headers=headers, + params={"scan_id": scan_id, "format": fmt}, + timeout=API_TIMEOUT_SHORT, + ) + # Any 2xx is a success, matching the Node runtime's axios default. + if 200 <= resp.status_code < 300: + budget.reset() + return resp + if not _is_transient_poll_status(resp.status_code, scan_exists): + raise PollFatalError( + _describe_http_error(resp.status_code, resp.text), + status_code=resp.status_code, + ) + detail = _describe_http_error(resp.status_code, resp.text) + reached_server = True + except requests.RequestException as e: + # Transport error (DNS, reset, timeout) — as retryable as a 5xx. + detail = _truncate(str(e)) + reached_server = False + + attempt = budget.record(detail, reached_server) + if budget.exhausted: + raise PollGaveUpError( + unreadable_report_message( + scan_id, + budget.last, + attempts=budget.total, + reached_server=budget.last_reached_server, + ) + ) + + wait = backoff_seconds(attempt) + if not quiet: + print( + f"Report not readable yet ({budget.last}); retrying in {wait}s " + f"({attempt}/{MAX_TRANSIENT_POLL_FAILURES})", + file=sys.stderr, + ) + time.sleep(wait) + + +def fetch_scan_with_retry(scan_id: str, headers: dict, fmt: str, quiet: bool): + """A single scan fetch with the same retry budget the poll loop uses. + + ``rafter get `` is what the give-up message tells customers to run, so + it must not be defeated by exactly the transient failure that produced the + message. A 404 here is still fatal — that is a wrong id, not lag. + """ + return _poll_until_readable( + scan_id, headers, fmt, quiet, _FailureBudget(), scan_exists=False + ) + + def _handle_scan_status_interactive( scan_id: str, headers: dict, fmt: str, quiet: bool ) -> int: - poll = requests.get( - f"{API_BASE}/static/scan", - headers=headers, - params={"scan_id": scan_id, "format": fmt}, - timeout=API_TIMEOUT_SHORT, - ) + budget = _FailureBudget() - if poll.status_code == 404: - print(f"Scan '{scan_id}' not found", file=sys.stderr) - raise typer.Exit(code=EXIT_SCAN_NOT_FOUND) - elif poll.status_code != 200: - print(f"Error: {poll.text}", file=sys.stderr) + # First poll. A 404 here really does mean "no such scan" — do not retry it. + # Transient 5xx IS retried, so that the `rafter get ` this command + # recommends on failure is not itself defeated by one bad read. + try: + poll = _poll_until_readable( + scan_id, headers, fmt, quiet, budget, scan_exists=False + ) + except PollFatalError as e: + if e.status_code == 404: + print(f"Scan '{scan_id}' not found", file=sys.stderr) + raise typer.Exit(code=EXIT_SCAN_NOT_FOUND) + print(f"Error: {e}", file=sys.stderr) + raise typer.Exit(code=EXIT_GENERAL_ERROR) + except PollGaveUpError as e: + print(f"Error: {e}", file=sys.stderr) raise typer.Exit(code=EXIT_GENERAL_ERROR) data = poll.json() status = data.get("status") - if status in ("queued", "pending", "processing"): + if status in IN_PROGRESS: if not quiet: print( "Waiting for scan to complete... (this could take several minutes)", file=sys.stderr, ) - while status in ("queued", "pending", "processing"): + while status in IN_PROGRESS: time.sleep(10) - poll = requests.get( - f"{API_BASE}/static/scan", - headers=headers, - params={"scan_id": scan_id, "format": fmt}, - timeout=API_TIMEOUT_SHORT, - ) + try: + poll = _poll_until_readable( + scan_id, headers, fmt, quiet, budget, scan_exists=True + ) + except (PollGaveUpError, PollFatalError) as e: + print(f"Error: {e}", file=sys.stderr) + raise typer.Exit(code=EXIT_GENERAL_ERROR) data = poll.json() status = data.get("status") if status == "completed": @@ -255,17 +482,19 @@ def get( headers = {"x-api-key": key} if not interactive: - resp = requests.get( - f"{API_BASE}/static/scan", - headers=headers, - params={"scan_id": scan_id, "format": fmt}, - timeout=API_TIMEOUT, - ) - if resp.status_code == 404: - print(f"Scan '{scan_id}' not found", file=sys.stderr) - raise typer.Exit(code=EXIT_SCAN_NOT_FOUND) - elif resp.status_code != 200: - print(f"Error: {resp.text}", file=sys.stderr) + # sable-l10k — retried, because this is the command the poll loop's + # give-up message recommends. A remedy defeated by the same + # transient failure it is recommended for is not a remedy. + try: + resp = fetch_scan_with_retry(scan_id, headers, fmt, quiet) + except PollFatalError as e: + if e.status_code == 404: + print(f"Scan '{scan_id}' not found", file=sys.stderr) + raise typer.Exit(code=EXIT_SCAN_NOT_FOUND) + print(f"Error: {e}", file=sys.stderr) + raise typer.Exit(code=EXIT_GENERAL_ERROR) + except PollGaveUpError as e: + print(f"Error: {e}", file=sys.stderr) raise typer.Exit(code=EXIT_GENERAL_ERROR) data = resp.json() return write_payload(data, fmt, quiet) diff --git a/python/tests/test_scan_poll_transient_500.py b/python/tests/test_scan_poll_transient_500.py new file mode 100644 index 00000000..3d7299a5 --- /dev/null +++ b/python/tests/test_scan_poll_transient_500.py @@ -0,0 +1,303 @@ +"""sable-l10k — transient poll failures must not kill a healthy scan. + +A paying customer's GitHub Actions run died on: + "Rafter scan poll failed: HTTP 500 — Failed to fetch report from storage: Object not found" + +A report is not durable the instant a scan flips to completed, so a poll can hit +a 5xx on a scan that is perfectly readable seconds later. These tests pin the +contract: transient read failures are retried, genuinely-missing reports still +fail, and the failure message is one a customer can act on. + +Mirrors node/tests/scan-poll-transient-500.test.ts. +""" +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest +import requests +import typer + +from rafter_cli.commands.backend import ( + BASE_BACKOFF_SECONDS, + MAX_TOTAL_TRANSIENT_POLL_FAILURES, + MAX_TRANSIENT_POLL_FAILURES, + _handle_scan_status_interactive, + backoff_seconds, + unreadable_report_message, +) +from rafter_cli.utils.api import EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND, EXIT_SUCCESS + +OBJECT_NOT_FOUND_BODY = json.dumps( + {"error": "Failed to fetch report from storage: Object not found"} +) + +HEADERS = {"x-api-key": "test-key"} + + +def _resp(status_code: int, text: str = "", json_body=None) -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.text = text + resp.json.return_value = json_body if json_body is not None else {} + return resp + + +def _processing() -> MagicMock: + return _resp(200, json_body={"status": "processing"}) + + +def _completed() -> MagicMock: + return _resp(200, json_body={"status": "completed", "markdown": "# Done"}) + + +def _server_500() -> MagicMock: + return _resp(500, text=OBJECT_NOT_FOUND_BODY) + + +@pytest.fixture(autouse=True) +def sleeps(): + """Backoff is real time; tests should not pay for it. + + Yields the mock so tests can assert the SCHEDULE, not just that sleeping + happened. Backoff is the fix — retrying five times inside a millisecond + gives an eventually-consistent store no time to converge. + """ + with patch("rafter_cli.commands.backend.time.sleep") as m: + yield m + + +class TestTransientPollFailures: + def test_backoff_is_exponential_not_flat(self): + assert BASE_BACKOFF_SECONDS > 0 + assert [backoff_seconds(n) for n in (1, 2, 3, 4)] == [2, 4, 8, 16] + + def test_actually_sleeps_the_backoff_schedule(self, sleeps): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing()] + [ + _server_500() for _ in range(MAX_TRANSIENT_POLL_FAILURES) + ] + with pytest.raises(typer.Exit): + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + # 10s poll interval, then 2/4/8/16 between the four retries that + # precede giving up on the fifth failure. + assert [c.args[0] for c in sleeps.call_args_list] == [10, 2, 4, 8, 16] + + def test_caps_total_failures_so_a_flapping_server_cannot_loop_forever(self): + # Alternating success/failure resets the consecutive counter every + # time. Without a total cap, and with no wall-clock deadline in the + # CLI, that loop never terminates. + flapping = [_processing()] + for _ in range(MAX_TOTAL_TRANSIENT_POLL_FAILURES + 5): + flapping += [_server_500(), _processing()] + + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = flapping + with pytest.raises(typer.Exit) as exc: + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert exc.value.exit_code == EXIT_GENERAL_ERROR + + def test_consecutive_counter_resets_on_a_successful_poll(self, sleeps): + # Two blips far apart must NOT add up to a give-up. + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [ + _processing(), + _server_500(), + _server_500(), + _processing(), + _server_500(), + _server_500(), + _completed(), + ] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + # Backoff restarts at 2s after the reset rather than continuing to 8s. + assert [c.args[0] for c in sleeps.call_args_list] == [10, 2, 4, 10, 2, 4] + + def test_survives_a_nested_json_error_object(self): + # A server may answer {"error": {"message": "..."}}. Calling string + # methods on that object raised an AttributeError no caller catches, + # surfacing as a traceback and defeating the retry entirely. + nested = _resp(500, text=json.dumps({"error": {"message": "nested"}})) + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing(), nested, _completed()] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + + def test_truncates_a_very_long_server_error(self, capsys): + huge = "x" * 5000 + big = _resp(500, text=json.dumps({"error": huge})) + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing()] + [ + big for _ in range(MAX_TRANSIENT_POLL_FAILURES) + ] + with pytest.raises(typer.Exit): + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + err = capsys.readouterr().err + assert huge not in err + assert "\u2026" in err + + def test_reports_the_real_attempt_count_not_the_consecutive_cap(self, capsys): + flapping = [_processing()] + for _ in range(MAX_TOTAL_TRANSIENT_POLL_FAILURES + 5): + flapping += [_server_500(), _processing()] + + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = flapping + with pytest.raises(typer.Exit): + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + err = capsys.readouterr().err + # 20 failures happened; claiming "after 5 attempts" would be a lie. + assert f"after {MAX_TOTAL_TRANSIENT_POLL_FAILURES} attempts" in err + + def test_unreachable_api_is_not_blamed_on_the_report(self, capsys): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing()] + [ + requests.ConnectionError("no route to host") + for _ in range(MAX_TRANSIENT_POLL_FAILURES) + ] + with pytest.raises(typer.Exit): + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + err = capsys.readouterr().err + assert "could not reach the API" in err + assert "could not read the report" not in err + + def test_first_poll_retries_a_transient_500(self): + # The give-up message tells the user to run `rafter get `, which + # re-enters at the first poll. If that path did not retry, the remedy + # we recommend would be defeated by one bad read. + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_server_500(), _completed()] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + assert get.call_count == 2 + + def test_any_2xx_counts_as_success(self): + # Node's axios accepts any 2xx; Python must not diverge. + accepted = _resp(202, json_body={"status": "completed", "markdown": "# Done"}) + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [accepted] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + + def test_retry_notice_goes_to_stderr(self, capsys): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing(), _server_500(), _completed()] + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=False) + + err = capsys.readouterr().err + assert "Report not readable yet" in err + assert "retrying in 2s" in err + + def test_rides_out_a_single_500_and_completes(self, capsys): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing(), _server_500(), _completed()] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + assert get.call_count == 3 + + def test_rides_out_several_consecutive_500s(self): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [ + _processing(), + _server_500(), + _server_500(), + _server_500(), + _completed(), + ] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + assert get.call_count == 5 + + def test_midpoll_404_is_lag_not_a_missing_scan(self): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing(), _resp(404, text="{}"), _completed()] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + + def test_first_poll_404_still_reports_not_found(self): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_resp(404, text="{}")] + with pytest.raises(typer.Exit) as exc: + _handle_scan_status_interactive("nope", HEADERS, "md", quiet=True) + + assert exc.value.exit_code == EXIT_SCAN_NOT_FOUND + assert get.call_count == 1 + + def test_gives_up_when_report_never_becomes_readable(self, capsys): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing()] + [ + _server_500() for _ in range(MAX_TRANSIENT_POLL_FAILURES) + ] + with pytest.raises(typer.Exit) as exc: + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert exc.value.exit_code == EXIT_GENERAL_ERROR + assert get.call_count == 1 + MAX_TRANSIENT_POLL_FAILURES + + err = capsys.readouterr().err + assert "rafter get s1" in err + assert "Object not found" in err + + def test_does_not_retry_a_non_transient_error(self): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [ + _processing(), + _resp(403, text=json.dumps({"error": "Invalid API key"})), + ] + with pytest.raises(typer.Exit) as exc: + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert exc.value.exit_code == EXIT_GENERAL_ERROR + assert get.call_count == 2 + + def test_retries_transport_errors(self): + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [ + _processing(), + requests.ConnectionError("ECONNRESET"), + _completed(), + ] + code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + assert code == EXIT_SUCCESS + + def test_a_500_body_is_never_mistaken_for_a_report(self): + """The pre-fix bug: the loop called .json() on the 500 body, got no + status, fell out of the loop and wrote the error payload out as if it + were results.""" + with patch("rafter_cli.commands.backend.requests.get") as get: + get.side_effect = [_processing()] + [ + _server_500() for _ in range(MAX_TRANSIENT_POLL_FAILURES) + ] + with pytest.raises(typer.Exit): + _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) + + +class TestUnreadableReportMessage: + def test_gives_scan_id_and_a_next_step(self): + msg = unreadable_report_message( + "scan-abc", + "HTTP 500 — Failed to fetch report from storage: Object not found", + ) + + assert "scan-abc" in msg + assert "rafter get scan-abc" in msg + assert "dashboard" in msg + # The raw server wording survives as supporting detail... + assert "Object not found" in msg + # ...but is not the whole message. + assert len(msg.splitlines()) > 1 diff --git a/shared-docs/CLI_SPEC.md b/shared-docs/CLI_SPEC.md index fd01fb5a..cd23373d 100644 --- a/shared-docs/CLI_SPEC.md +++ b/shared-docs/CLI_SPEC.md @@ -126,6 +126,30 @@ Retrieve results from a scan. **Vulnerability levels (JSON output):** The `level` field on each vulnerability uses SARIF standard values: `"error"`, `"warning"`, or `"note"`. +#### Poll-loop retry contract + +A report is not necessarily durable the instant a scan flips to `completed`, so a poll can hit a 5xx on a scan that is readable seconds later. Both runtimes retry transient read failures instead of aborting. This applies to `rafter run`, `rafter get --interactive`, and plain `rafter get `: + +| Condition during polling | Behavior | +|--------------------------|----------| +| HTTP 5xx or 408 | Transient. Retried up to **5 consecutive times** with exponential backoff (2s, 4s, 8s, 16s). | +| Transport error (DNS, reset, timeout) | Same as above. | +| HTTP 404, **after** the scan is known to exist | Transient — read-after-write lag, not a wrong id. | +| HTTP 404 on the **first** poll | Not retried. The scan genuinely does not exist. Exit code `2`. | +| Other 4xx (401/403/429 …) | Not retried — reported immediately. | + +Two budgets bound the retries. The **consecutive** counter (5) resets on any successful poll, so a long scan with occasional blips is not killed by unrelated failures minutes apart. A **total** counter (20 per command invocation) does *not* reset, so a backend alternating success and failure cannot keep the loop alive indefinitely — the CLI has no wall-clock deadline of its own. + +After either budget is exhausted the command exits `1`. If the failures reached the server, the message names the scan id, the `rafter get ` retry, and the dashboard, with the raw server response as supporting detail — raw storage-layer wording is never the whole message. If no failure reached the server at all, the message says so and points at connectivity rather than blaming the report. Both report the **real** number of attempts, which is not always 5: the total budget can trip first. + +`rafter get ` carries the same retry budget, so the remedy the give-up message recommends is not defeated by the transient failure that produced it. + +**The composite GitHub Action** (`github-action/action.yml`) implements the same classification in both its poll loop and its results fetch, with these differences forced by the shell: + +- It has no "first poll" distinction: by the time it polls, the trigger step has already returned a `scan_id`, so **every** 404 there is treated as read-after-write lag. A scan id the backend accepted but never persisted therefore fails after the 5-failure budget rather than immediately. +- Its poll loop is additionally bounded by a wall-clock deadline derived from `timeout-minutes`. Before v0.11 that input was a poll *count*, so a slow API could overrun it; it is now a real deadline. +- Its `status` output is `completed`, `failed`, `timeout`, `unreadable` (the scan may have finished but its report could not be read), or `unreachable` (the API could not be contacted). + ### rafter usage [OPTIONS] Check API quota and usage statistics. From dbb4f6019fc181cfb93b5f2e37de4a1d53bcbc5d Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:50:12 -0700 Subject: [PATCH 08/18] ci: close three blind spots found investigating why CI missed sable-l10k (#221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assignment was to explain why 8 of 17 checks skipped on PR #220. The skip pattern is real, but it is not why the bug reached a customer. Testing that claim rather than assuming it is what turned up the rest. WOULD ANY EXISTING CHECK HAVE CAUGHT IT, IF IT HAD RUN? No. Checked out main at 0996492 (the buggy tree) and ran the entire suite against it: 2065 passed, and the only failures were three files that fail for environmental reasons here and are unrelated to polling. tests/scan-remote.test.ts and test_scan_remote.py both cover the poll loop — with the HTTP layer mocked, and neither ever injected a non-2xx mid-poll. They pass against the bug. And nothing executed github-action/action.yml at all, which is the file the customer's error came from. test-action.yml drives the ROOT action.yml, a different action that scans locally. Of the jobs in test-github-action.yml, two run hand-copied reimplementations of the action's bash (their own headers admit the duplication) and one greps the YAML as text. So the workflow that fires on github-action/** changes ran, and still executed none of the code. The gap was COVERAGE, and #220 closed it. The skip pattern would not have mattered. Three things found on the way there do: 1. publish-python had no `needs:`. publish-node has needed the test jobs since it was written; the Python half published to PyPI in parallel with the tests, ungated. A red suite blocked the npm release and shipped the PyPI one anyway — in a dual-implementation product where the two versions must match, that diverges them at the registry, the one place users cannot see it. Now gated. (publish.yaml also runs no pytest anywhere; filed separately.) 2. backend-api rendered identically whether it tested the backend or nothing. Its only real step is gated on RAFTER_API_KEY, which has never been set on this repo, so "backend-api ✓" has always meant "checked out and built". It now says so, loudly, in the log and the step summary. 3. test-node and test-python were skipped on internal PRs into main. On #220 — which changed both clients — neither ran. They now run on every PR. The premise that our own work is tested locally first is also weaker than it looks: this repo has test files that fail locally for environmental reasons, so "green on my machine" is not a signal anyone can act on. Cost is ~4 minutes of wall clock (234s and 100s, in parallel). The expensive part, the 6-way cross-platform grid with 3 macOS runners, stays gated — this reverses part of #219 narrowly and deliberately, not wholesale. Also established, not changed here: main has no branch protection at all. The only ruleset targets refs/heads/prod and contains no required-status-checks rule, so no check is required anywhere and a red PR can merge into main. That is a policy call, not a workflow fix. Co-authored-by: achebe --- .github/workflows/publish.yaml | 6 +++ .github/workflows/test-comprehensive.yml | 49 ++++++++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 176b1899..4984dd70 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -143,6 +143,12 @@ jobs: run: npm publish --access public --provenance publish-python: + # sable-bm5k — publish-node has needed the test jobs since it was written; + # this one never did, so a red suite blocked the npm release and shipped + # the PyPI one anyway. In a dual-implementation product that means the two + # runtimes could diverge at the registry, which is the one place users + # cannot see it. + needs: [test-node, test-package] runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/test-comprehensive.yml b/.github/workflows/test-comprehensive.yml index 2813130e..5fd6c4ba 100644 --- a/.github/workflows/test-comprehensive.yml +++ b/.github/workflows/test-comprehensive.yml @@ -12,11 +12,11 @@ permissions: jobs: # ── Who gets the suite ──────────────────────────────────────────── - # PRs into prod are the release gate and always run. - # PRs into main run only for outside contributions: our own work (Rome-1's - # PRs, or any branch living in the Raftersecurity repo) is reviewed and - # tested locally before it is pushed, so running the full matrix again - # would just burn runner minutes. + # The two unit-test jobs (test-node, test-python) run on EVERY PR — see + # sable-bm5k. The rest of the matrix runs for PRs into prod (the release + # gate) and for outside contributions; for our own PRs into main it is + # skipped, because re-running the 6-way cross-platform grid on work that + # was reviewed before it was pushed mostly burns runner minutes. # # Note this is `pull_request`, not `pull_request_target` — fork PRs run with # a read-only token and no access to secrets. Do not "fix" that. @@ -24,6 +24,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.decide.outputs.run }} + run_core: ${{ steps.decide.outputs.run_core }} steps: - id: decide # Values go through env rather than direct ${{ }} interpolation into @@ -34,11 +35,25 @@ jobs: HEAD_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} AUTHOR: ${{ github.event.pull_request.user.login }} run: | + # `run` — the full matrix, including the 6-way cross-platform grid. + # `run_core` — the two unit-test jobs. These now run on EVERY PR. + # + # sable-bm5k: the original gate skipped everything on internal PRs into + # main, on the premise that our own work is tested locally first. On + # #220 — which changed both the Node and the Python client — that meant + # neither test-node nor test-python ran. The premise is also weaker + # than it looks: this repo has test files that fail locally for + # environmental reasons, so "green on my machine" is not a signal you + # can act on. test-node (234s) and test-python (100s) run in parallel, + # so this costs ~4 minutes of wall clock. The expensive part — the + # cross-platform grid, 6 jobs and 3 of them macOS — stays gated. + echo "run_core=true" >> "$GITHUB_OUTPUT" + if [ "$EVENT" != "pull_request" ] || [ "$BASE" != "main" ]; then echo "run=true" >> "$GITHUB_OUTPUT" elif [ "$HEAD_OWNER" = "Raftersecurity" ] || [ "$AUTHOR" = "Rome-1" ]; then echo "run=false" >> "$GITHUB_OUTPUT" - echo "Internal PR into main (author=$AUTHOR, head repo owner=$HEAD_OWNER) — suite skipped." >> "$GITHUB_STEP_SUMMARY" + echo "Internal PR into main (author=$AUTHOR, head repo owner=$HEAD_OWNER) — unit tests still run; extended matrix skipped." >> "$GITHUB_STEP_SUMMARY" else echo "run=true" >> "$GITHUB_OUTPUT" fi @@ -46,7 +61,7 @@ jobs: # ── Unit & integration tests (both languages) ───────────────────── test-node: needs: gate - if: needs.gate.outputs.run == 'true' + if: needs.gate.outputs.run_core == 'true' runs-on: ubuntu-latest defaults: run: @@ -91,7 +106,7 @@ jobs: test-python: needs: gate - if: needs.gate.outputs.run == 'true' + if: needs.gate.outputs.run_core == 'true' runs-on: ubuntu-latest defaults: run: @@ -228,6 +243,24 @@ jobs: if: ${{ env.RAFTER_API_KEY != '' }} run: pnpm exec vitest run tests/backend-api.test.ts + # sable-bm5k — without this the job renders identically whether it tested + # the backend or tested nothing. RAFTER_API_KEY has never been set on this + # repo, so "backend-api ✓" has always meant "checked out and built". + # A skipped step must not look like a passing one. + - name: Say so when the backend tests did not run + if: ${{ env.RAFTER_API_KEY == '' }} + run: | + echo "::warning::backend-api tested NOTHING — RAFTER_API_KEY is not set, so tests/backend-api.test.ts was skipped." + { + echo "### :warning: backend-api ran no tests" + echo "" + echo "\`RAFTER_API_KEY\` is unset, so \`tests/backend-api.test.ts\` was skipped." + echo "This job checked out and built the package and nothing else." + echo "" + echo "The remote scan path is covered without a key by the mock-backed jobs" + echo "in \`test-github-action.yml\`. See sable-bm5k." + } >> "$GITHUB_STEP_SUMMARY" + # ── Package build verification ───────────────────────────────────── package-integrity: needs: gate From efefff0a5e0ccca4a0a93c7707c20eb0c9039d75 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:50:26 -0700 Subject: [PATCH 09/18] fix: stop sending the API key across redirects (sable-2s6p) (#223) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit requests' SessionRedirectMixin.rebuild_auth strips Authorization on a host change and leaves arbitrary custom headers intact; axios/follow-redirects does the same. So `x-api-key` rides a 302 to whatever host it points at. Nothing in this CLI needs to follow a redirect, so nothing does any more. PRE-EXISTING, not introduced by #220. The retry loop added there raises exposure from one transmission to as many as five, which is why it matters more today than it did last week, but it is not the cause. The action's curl paths were never affected — verified, no -L or --location on any of the eight curl invocations in github-action/action.yml. Every authenticated call site, not just the poll path: 14 in Node behind a shared `apiClient` (axios instance, maxRedirects: 0), 13 in Python behind api_get/api_post (allow_redirects forced False, so no call site can opt back in). Deliberately left alone: update-checker's npm registry call and the Slack/Discord webhook posts, none of which carry the key. WHAT THE SECURITY REVIEW CAUGHT, and it would have shipped a broken CLI: API/API_BASE end in "/" and 11 call sites concatenated "/static/...", building https://rafter.so/api//static/scan. Production answers that with a 308 to the single-slash form. It worked only because the client followed the redirect — so refusing redirects turned every core command into a hard failure. Verified against the live API: the double-slash URL 308s, the single-slash one reaches the endpoint. Both runtimes now build URLs through apiUrl()/api_url(), and a test in each fails on any `${API}/` or `{API_BASE}/` construction. Every other test mocks the transport, which is why nothing caught this. Also from that review: - The message told users to point --rafter-url at the final URL. That flag does not exist in the CLI — it is a GitHub Action input. Removed the instruction rather than shipping advice nobody can follow. - A redirect Location is attacker-controlled if the endpoint is. Header values cannot carry CR/LF but ESC is legal, so the raw value could rewrite the user's terminal. Both runtimes strip non-printables and cap at 200 chars, asserted with an ANSI sequence in the fixture. - The source-scanning guards only caught the most literal bypass. They now also match the .request() form and flag any second axios.create() / requests.Session() built outside the api utils. - The Node test shim made axios and apiClient the same mock, so a regression to bare axios would still have passed. create() now returns a distinct object and the tests watch that instance; mutation-tested by reverting one call site to bare axios, which the guard catches. A refused redirect now explains itself instead of surfacing a bare 302. Co-authored-by: achebe --- node/src/commands/backend/get.ts | 1 - node/src/commands/backend/run.ts | 14 +- node/src/commands/backend/scan-status.ts | 8 +- node/src/commands/backend/usage.ts | 5 +- node/src/commands/issues/from-scan.ts | 5 +- node/src/commands/mcp/server.ts | 11 +- node/src/commands/notify.ts | 5 +- node/src/commands/sites/create.ts | 5 +- node/src/commands/sites/get.ts | 5 +- node/src/commands/sites/list.ts | 5 +- node/src/commands/sites/scan.ts | 5 +- node/src/utils/api.ts | 45 ++++++ node/tests/api-no-redirect.test.ts | 125 +++++++++++++++++ node/tests/mcp-sites.test.ts | 33 ++++- node/tests/plus-scan-approval.test.ts | 33 ++++- node/tests/scan-poll-transient-500.test.ts | 33 ++++- node/tests/scan-remote.test.ts | 33 ++++- node/tests/sites-cli.test.ts | 33 ++++- python/rafter_cli/commands/backend.py | 15 +- .../rafter_cli/commands/issues/issues_app.py | 6 +- python/rafter_cli/commands/mcp_server.py | 10 +- python/rafter_cli/commands/notify.py | 6 +- python/rafter_cli/commands/sites.py | 10 +- python/rafter_cli/utils/api.py | 67 +++++++++ python/tests/test_api_no_redirect.py | 130 ++++++++++++++++++ python/tests/test_api_scope.py | 16 +-- python/tests/test_plus_scan_approval.py | 4 +- python/tests/test_scan_poll_transient_500.py | 36 ++--- python/tests/test_scan_remote.py | 70 +++++----- python/tests/test_sites.py | 46 +++---- 30 files changed, 665 insertions(+), 155 deletions(-) create mode 100644 node/tests/api-no-redirect.test.ts create mode 100644 python/tests/test_api_no_redirect.py diff --git a/node/src/commands/backend/get.ts b/node/src/commands/backend/get.ts index 66f47469..40471d1a 100644 --- a/node/src/commands/backend/get.ts +++ b/node/src/commands/backend/get.ts @@ -1,5 +1,4 @@ import { Command } from "commander"; -import axios from "axios"; import { API, resolveKey, diff --git a/node/src/commands/backend/run.ts b/node/src/commands/backend/run.ts index 6fb73162..db46df9f 100644 --- a/node/src/commands/backend/run.ts +++ b/node/src/commands/backend/run.ts @@ -1,5 +1,4 @@ import { Command } from "commander"; -import axios from "axios"; import ora from "ora"; import { detectRepo } from "../../utils/git.js"; import { @@ -8,8 +7,9 @@ import { EXIT_GENERAL_ERROR, EXIT_QUOTA_EXHAUSTED, EXIT_CONFIRMATION_REQUIRED, - handle403 -} from "../../utils/api.js"; + handle403, + apiClient, + apiUrl} from "../../utils/api.js"; import { ConfigManager } from "../../core/config-manager.js"; import { loadPolicy } from "../../core/policy-loader.js"; import { askYesNo } from "../../utils/prompt.js"; @@ -133,8 +133,8 @@ export async function runRemoteScan(opts: RunOpts): Promise { if (!opts.quiet) { const spinner = ora("Submitting scan").start(); try { - const { data } = await axios.post( - `${API}/static/scan`, + const { data } = await apiClient.post( + apiUrl("static/scan"), body, { headers: { "x-api-key": key } } ); @@ -161,8 +161,8 @@ export async function runRemoteScan(opts: RunOpts): Promise { } } else { try { - const { data } = await axios.post( - `${API}/static/scan`, + const { data } = await apiClient.post( + apiUrl("static/scan"), body, { headers: { "x-api-key": key } } ); diff --git a/node/src/commands/backend/scan-status.ts b/node/src/commands/backend/scan-status.ts index 2aa1bb85..2d7090ac 100644 --- a/node/src/commands/backend/scan-status.ts +++ b/node/src/commands/backend/scan-status.ts @@ -1,12 +1,12 @@ -import axios from "axios"; import ora from "ora"; import { API, API_TIMEOUT_SHORT_MS, writePayload, EXIT_GENERAL_ERROR, - EXIT_SCAN_NOT_FOUND -} from "../../utils/api.js"; + EXIT_SCAN_NOT_FOUND, + apiClient, + apiUrl} from "../../utils/api.js"; import { fmt as output } from "../../utils/formatter.js"; /** @@ -175,7 +175,7 @@ async function pollUntilReadable( ): Promise { for (;;) { try { - const res = await axios.get(`${API}/static/scan`, { + const res = await apiClient.get(apiUrl("static/scan"), { params: { scan_id, format: fmt }, headers, // Without this a hung server stalls inside a single request, and the diff --git a/node/src/commands/backend/usage.ts b/node/src/commands/backend/usage.ts index 5a1c2c0e..3c18412a 100644 --- a/node/src/commands/backend/usage.ts +++ b/node/src/commands/backend/usage.ts @@ -1,6 +1,5 @@ import { Command } from "commander"; -import axios from "axios"; -import { API, resolveKey, EXIT_GENERAL_ERROR } from "../../utils/api.js"; +import { API, resolveKey, EXIT_GENERAL_ERROR, apiClient, apiUrl} from "../../utils/api.js"; export function createUsageCommand(): Command { return new Command("usage") @@ -8,7 +7,7 @@ export function createUsageCommand(): Command { .action(async (opts) => { const key = resolveKey(opts.apiKey); try { - const { data } = await axios.get(`${API}/static/usage`, { headers: { "x-api-key": key } }); + const { data } = await apiClient.get(apiUrl("static/usage"), { headers: { "x-api-key": key } }); console.log(JSON.stringify(data, null, 2)); } catch (e: any) { if (e.response?.data) { diff --git a/node/src/commands/issues/from-scan.ts b/node/src/commands/issues/from-scan.ts index 88d01575..38852bcb 100644 --- a/node/src/commands/issues/from-scan.ts +++ b/node/src/commands/issues/from-scan.ts @@ -7,8 +7,7 @@ */ import { Command } from "commander"; import fs from "fs"; -import axios from "axios"; -import { API, resolveKey, EXIT_GENERAL_ERROR } from "../../utils/api.js"; +import { API, resolveKey, EXIT_GENERAL_ERROR, apiClient, apiUrl} from "../../utils/api.js"; import { detectRepo } from "../../utils/git.js"; import { fmt } from "../../utils/formatter.js"; import { createIssue, listOpenIssues } from "./github-client.js"; @@ -170,7 +169,7 @@ async function draftsFromBackendScan( apiKey?: string ): Promise { const key = resolveKey(apiKey); - const { data } = await axios.get(`${API}/static/scan`, { + const { data } = await apiClient.get(apiUrl("static/scan"), { params: { scan_id: scanId, format: "json" }, headers: { "x-api-key": key }, }); diff --git a/node/src/commands/mcp/server.ts b/node/src/commands/mcp/server.ts index 821f0534..48ffd243 100644 --- a/node/src/commands/mcp/server.ts +++ b/node/src/commands/mcp/server.ts @@ -15,9 +15,8 @@ import { AuditLogger } from "../../core/audit-logger.js"; import { ConfigManager, redactConfigSecrets, isSecretConfigKey, maskSecretValue } from "../../core/config-manager.js"; import { listDocs, resolveDocSelector, fetchDoc } from "../../core/docs-loader.js"; import { writeSuppression } from "../../core/suppression-writer.js"; -import { apiUrl } from "../../utils/api.js"; +import { apiUrl, apiClient} from "../../utils/api.js"; import { describeSitesError, resolveMcpApiKey } from "../sites/errors.js"; -import axios from "axios"; import { createRequire } from "module"; const _require = createRequire(import.meta.url); @@ -361,7 +360,7 @@ export function createServer(): Server { const key = resolveMcpApiKey(); if (!key) return errorResult("No API key configured. Set RAFTER_API_KEY or run 'rafter agent config set backend.apiKey '."); try { - const { data } = await axios.post(apiUrl("static/sites"), { url }, { headers: { "x-api-key": key } }); + const { data } = await apiClient.post(apiUrl("static/sites"), { url }, { headers: { "x-api-key": key } }); return textResult(data); } catch (e: any) { return errorResult(describeSitesError(e).message); @@ -378,7 +377,7 @@ export function createServer(): Server { const body: Record = projectId ? { projectId } : { url }; if (Array.isArray(args?.sections)) body.sections = (args!.sections as unknown[]).map((s) => String(s)); try { - const { data } = await axios.post(apiUrl("static/sites/scan"), body, { headers: { "x-api-key": key } }); + const { data } = await apiClient.post(apiUrl("static/sites/scan"), body, { headers: { "x-api-key": key } }); return textResult(data); } catch (e: any) { return errorResult(describeSitesError(e).message); @@ -393,7 +392,7 @@ export function createServer(): Server { if (args?.offset !== undefined) params.offset = String(args.offset); if (args?.include_archived) params.include_archived = "true"; try { - const { data } = await axios.get(apiUrl("static/sites"), { params, headers: { "x-api-key": key } }); + const { data } = await apiClient.get(apiUrl("static/sites"), { params, headers: { "x-api-key": key } }); return textResult(data); } catch (e: any) { return errorResult(describeSitesError(e).message); @@ -406,7 +405,7 @@ export function createServer(): Server { const key = resolveMcpApiKey(); if (!key) return errorResult("No API key configured. Set RAFTER_API_KEY or run 'rafter agent config set backend.apiKey '."); try { - const { data } = await axios.get(apiUrl(`static/sites/${encodeURIComponent(id)}`), { headers: { "x-api-key": key } }); + const { data } = await apiClient.get(apiUrl(`static/sites/${encodeURIComponent(id)}`), { headers: { "x-api-key": key } }); return textResult(data); } catch (e: any) { return errorResult(describeSitesError(e).message); diff --git a/node/src/commands/notify.ts b/node/src/commands/notify.ts index 459f6597..782632eb 100644 --- a/node/src/commands/notify.ts +++ b/node/src/commands/notify.ts @@ -1,6 +1,5 @@ import { Command } from "commander"; -import axios from "axios"; -import { API, resolveKey, EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND } from "../utils/api.js"; +import { API, resolveKey, EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND, apiClient, apiUrl} from "../utils/api.js"; import { validateWebhookUrl } from "../core/audit-logger.js"; import { ConfigManager } from "../core/config-manager.js"; import { fmt, isAgentMode } from "../utils/formatter.js"; @@ -221,7 +220,7 @@ export function createNotifyCommand(): Command { if (scanId) { const key = resolveKey(opts?.apiKey as string | undefined); try { - const { data } = await axios.get(`${API}/static/scan`, { + const { data } = await apiClient.get(apiUrl("static/scan"), { params: { scan_id: scanId, format: "json" }, headers: { "x-api-key": key }, }); diff --git a/node/src/commands/sites/create.ts b/node/src/commands/sites/create.ts index dc35d4c8..d8b36f38 100644 --- a/node/src/commands/sites/create.ts +++ b/node/src/commands/sites/create.ts @@ -1,6 +1,5 @@ import { Command } from "commander"; -import axios from "axios"; -import { apiUrl, resolveKey, writePayload, EXIT_GENERAL_ERROR } from "../../utils/api.js"; +import { apiUrl, resolveKey, writePayload, EXIT_GENERAL_ERROR, apiClient} from "../../utils/api.js"; import { describeSitesError, rejectUnsupportedFormat } from "./errors.js"; export interface SitesCreateOpts { @@ -14,7 +13,7 @@ export async function runSitesCreate(url: string, opts: SitesCreateOpts): Promis if (rejectUnsupportedFormat(opts.format)) return EXIT_GENERAL_ERROR; const key = resolveKey(opts.apiKey); try { - const { data } = await axios.post( + const { data } = await apiClient.post( apiUrl("static/sites"), { url }, { headers: { "x-api-key": key } } diff --git a/node/src/commands/sites/get.ts b/node/src/commands/sites/get.ts index 1a163417..e002f209 100644 --- a/node/src/commands/sites/get.ts +++ b/node/src/commands/sites/get.ts @@ -1,6 +1,5 @@ import { Command } from "commander"; -import axios from "axios"; -import { apiUrl, resolveKey, writePayload, EXIT_GENERAL_ERROR } from "../../utils/api.js"; +import { apiUrl, resolveKey, writePayload, EXIT_GENERAL_ERROR, apiClient} from "../../utils/api.js"; import { describeSitesError, rejectUnsupportedFormat } from "./errors.js"; export interface SitesGetOpts { @@ -14,7 +13,7 @@ export async function runSitesGet(id: string, opts: SitesGetOpts): Promise { if (opts.includeArchived) params.include_archived = "true"; try { - const { data } = await axios.get( + const { data } = await apiClient.get( apiUrl("static/sites"), { params, headers: { "x-api-key": key } } ); diff --git a/node/src/commands/sites/scan.ts b/node/src/commands/sites/scan.ts index b584585b..b57e8c55 100644 --- a/node/src/commands/sites/scan.ts +++ b/node/src/commands/sites/scan.ts @@ -1,6 +1,5 @@ import { Command } from "commander"; -import axios from "axios"; -import { apiUrl, resolveKey, writePayload, EXIT_GENERAL_ERROR } from "../../utils/api.js"; +import { apiUrl, resolveKey, writePayload, EXIT_GENERAL_ERROR, apiClient} from "../../utils/api.js"; import { describeSitesError, rejectUnsupportedFormat } from "./errors.js"; const VALID_SECTIONS = new Set(["flight", "security", "dns"]); @@ -42,7 +41,7 @@ export async function runSitesScan(projectIdOrUrl: string, opts: SitesScanOpts): } try { - const { data } = await axios.post( + const { data } = await apiClient.post( apiUrl("static/sites/scan"), body, { headers: { "x-api-key": key } } diff --git a/node/src/utils/api.ts b/node/src/utils/api.ts index 740431bd..a95a6b48 100644 --- a/node/src/utils/api.ts +++ b/node/src/utils/api.ts @@ -1,7 +1,52 @@ +import axios from "axios"; import { ConfigManager } from "../core/config-manager.js"; export const API = "https://rafter.so/api/"; +/** + * sable-2s6p — the HTTP client for every authenticated Rafter API call. + * + * `maxRedirects: 0` is the point of it. axios (via follow-redirects) replays + * request headers on a redirect, and unlike `Authorization` the custom + * `x-api-key` header is not stripped when the host changes. Since the API base + * is user-settable (`--rafter-url`, self-hosted installs), a 302 from a + * misconfigured or hostile endpoint would walk the caller's API key to another + * host. Nothing in this CLI needs to follow a redirect, so none of them do. + * + * Use this for anything that sends `x-api-key`. Plain `axios` is fine for + * user-supplied webhooks and other unauthenticated calls. + */ +export const apiClient = axios.create({ + maxRedirects: 0, +}); + +/** + * A redirect target is attacker-controlled if the endpoint is. Header values + * cannot contain CR/LF, but ESC is a legal byte, so an unsanitized Location can + * emit ANSI sequences that rewrite the user's terminal. Strip anything + * non-printable and cap the length. + */ +function safeForTerminal(value: unknown): string { + if (typeof value !== "string") return ""; + // eslint-disable-next-line no-control-regex + const printable = value.replace(/[\u0000-\u001f\u007f-\u009f]/g, ""); + return printable.length > 200 ? `${printable.slice(0, 200)}…` : printable; +} + +// A refused redirect otherwise surfaces as a bare "Request failed with status +// code 302", which tells the user nothing about why. Name the cause. +apiClient.interceptors.response.use(undefined, (error: any) => { + const status = error?.response?.status; + if (status >= 300 && status < 400) { + const target = safeForTerminal(error?.response?.headers?.location) || "another host"; + error.message = + `The Rafter API redirected to ${target}, and Rafter does not follow redirects ` + + `on authenticated requests — your API key would be sent to the redirect target. ` + + `If you are pointing Rafter at a self-hosted instance, use its final URL.`; + } + return Promise.reject(error); +}); + /** Join API with a path segment without producing a double slash, regardless of leading/trailing slashes on either side. */ export function apiUrl(path: string): string { return `${API.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`; diff --git a/node/tests/api-no-redirect.test.ts b/node/tests/api-no-redirect.test.ts new file mode 100644 index 00000000..f26d2ccd --- /dev/null +++ b/node/tests/api-no-redirect.test.ts @@ -0,0 +1,125 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +import { apiClient } from "../src/utils/api.js"; + +/** + * sable-2s6p — a custom `x-api-key` header is NOT stripped across a cross-host + * redirect the way `Authorization` is. axios replays it verbatim, and the API + * base is user-settable (`--rafter-url`, self-hosted installs), so a 302 from a + * misconfigured or hostile endpoint walks the caller's API key to another host. + * + * Nothing in this CLI needs to follow a redirect. These tests pin that, and — + * more importantly — pin that no NEW authenticated call site can reintroduce + * the hole by reaching for bare `axios`. + */ + +describe("apiClient (sable-2s6p)", () => { + it("refuses to follow redirects", () => { + expect(apiClient.defaults.maxRedirects).toBe(0); + }); + + it("explains why, instead of surfacing a bare 302", async () => { + // Drive the interceptor directly: it is the thing that turns an opaque + // status code into something a customer can act on. + const handlers = (apiClient.interceptors.response as any).handlers.filter(Boolean); + expect(handlers.length).toBeGreaterThan(0); + const onRejected = handlers[handlers.length - 1].rejected; + + const err: any = { + response: { + status: 302, + // ANSI escape included on purpose: an attacker-controlled Location must + // not be able to rewrite the user's terminal. + headers: { location: "https://evil.example/collect\u001b[31m" }, + }, + message: "Request failed with status code 302", + }; + + await expect(onRejected(err)).rejects.toBeDefined(); + expect(err.message).toContain("evil.example"); + expect(err.message).toContain("does not follow redirects"); + expect(err.message).not.toContain("\u001b"); + expect(err.message).toContain("self-hosted"); + }); +}); + +/** Every .ts file under a directory, recursively. */ +function sourceFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) out.push(...sourceFiles(full)); + else if (entry.endsWith(".ts")) out.push(full); + } + return out; +} + +describe("no authenticated call bypasses apiClient (sable-2s6p)", () => { + it("creates no second axios instance outside the api utils", () => { + const offenders: string[] = []; + for (const file of sourceFiles("src")) { + if (file.endsWith("utils/api.ts")) continue; + const text = readFileSync(file, "utf8"); + if (/\baxios\.create\(/.test(text)) offenders.push(file); + } + expect( + offenders, + `A second axios instance can be created without maxRedirects: 0. Use ` + + `apiClient from src/utils/api.ts:\n${offenders.join("\n")}` + ).toEqual([]); + }); + + it("has no bare axios verb call that sends x-api-key", () => { + const offenders: string[] = []; + + for (const file of sourceFiles("src")) { + const text = readFileSync(file, "utf8"); + const lines = text.split("\n"); + lines.forEach((line, i) => { + if (!/\baxios(\.(get|post|put|delete|patch|request))?\(/.test(line)) return; + // Look at the call and the few lines after it — the header object is + // usually on a following line. + const window = lines.slice(i, i + 6).join("\n"); + if (window.includes("x-api-key")) { + offenders.push(`${file}:${i + 1}`); + } + }); + } + + expect( + offenders, + `These calls send the API key through bare axios, which follows redirects ` + + `across hosts. Use apiClient from src/utils/api.ts instead:\n${offenders.join("\n")}` + ).toEqual([]); + }); +}); + +describe("API URL construction (sable-2s6p)", () => { + it("builds no double slash after the scheme", () => { + // Not cosmetic. `API` ends in "/", and concatenating "/static/..." produced + // https://rafter.so/api//static/scan, which production answers with a 308. + // That worked only because the client followed redirects — so refusing + // them would have broken every core command. Caught by security review, + // not by any test, because every other test mocks the transport. + const offenders: string[] = []; + for (const file of sourceFiles("src")) { + const text = readFileSync(file, "utf8"); + text.split("\n").forEach((line, i) => { + if (/\$\{API\}\//.test(line)) offenders.push(`${file}:${i + 1}`); + }); + } + expect( + offenders, + `These build a double-slash URL. Use apiUrl() instead:\n${offenders.join("\n")}` + ).toEqual([]); + }); + + it("apiUrl joins cleanly regardless of slashes", async () => { + const { apiUrl, API } = await import("../src/utils/api.js"); + expect(apiUrl("static/scan")).toBe("https://rafter.so/api/static/scan"); + expect(apiUrl("/static/scan")).toBe("https://rafter.so/api/static/scan"); + expect(API.endsWith("/")).toBe(true); // the trap this guards against + }); +}); diff --git a/node/tests/mcp-sites.test.ts b/node/tests/mcp-sites.test.ts index 412359ac..5228d59d 100644 --- a/node/tests/mcp-sites.test.ts +++ b/node/tests/mcp-sites.test.ts @@ -9,7 +9,35 @@ import type { Server } from "@modelcontextprotocol/sdk/server/index.js"; * through an in-memory MCP client/server pair, with axios mocked. */ -vi.mock("axios"); +vi.mock("axios", () => { + // sable-2s6p — the code calls `apiClient`, an axios instance created with + // maxRedirects: 0. `create` must return something, and it returns the same + // object as the default export so `mockedAxios.get` still refers to the + // function under test. + const instance: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + defaults: { maxRedirects: 0 }, + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + }; + // The default export gets its OWN mocks, distinct from the instance's. If + // production code regresses to bare `axios.get`, the assertions below — which + // watch the instance — stop seeing calls, and the test fails. A shim where + // both are the same object would silently accept that regression. + const bare: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + create: () => instance, + }; + return { default: bare }; +}); vi.mock("../src/core/config-manager.js", async (importOriginal) => ({ ...(await importOriginal()), @@ -29,7 +57,8 @@ vi.mock("@modelcontextprotocol/sdk/server/stdio.js", () => ({ import axios from "axios"; import { createServer } from "../src/commands/mcp/server.js"; -const mockedAxios = vi.mocked(axios, true); +// The code calls `apiClient` — the instance `create()` returns. +const mockedAxios = vi.mocked((axios as any).create(), true); let client: Client; let server: Server; diff --git a/node/tests/plus-scan-approval.test.ts b/node/tests/plus-scan-approval.test.ts index ccb4721c..cff2d34f 100644 --- a/node/tests/plus-scan-approval.test.ts +++ b/node/tests/plus-scan-approval.test.ts @@ -40,7 +40,35 @@ vi.mock("../src/utils/prompt.js", () => ({ askYesNo: vi.fn(async () => state.promptAnswer), })); -vi.mock("axios"); +vi.mock("axios", () => { + // sable-2s6p — the code calls `apiClient`, an axios instance created with + // maxRedirects: 0. `create` must return something, and it returns the same + // object as the default export so `mockedAxios.get` still refers to the + // function under test. + const instance: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + defaults: { maxRedirects: 0 }, + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + }; + // The default export gets its OWN mocks, distinct from the instance's. If + // production code regresses to bare `axios.get`, the assertions below — which + // watch the instance — stop seeing calls, and the test fails. A shim where + // both are the same object would silently accept that regression. + const bare: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + create: () => instance, + }; + return { default: bare }; +}); vi.mock("ora", () => ({ default: () => ({ start: vi.fn().mockReturnThis(), @@ -58,7 +86,8 @@ import { } from "../src/commands/backend/run.js"; import { EXIT_CONFIRMATION_REQUIRED } from "../src/utils/api.js"; -const mockedAxios = vi.mocked(axios, true); +// The code calls `apiClient` — the instance `create()` returns. +const mockedAxios = vi.mocked((axios as any).create(), true); function resetState() { state.globalFlag = undefined; diff --git a/node/tests/scan-poll-transient-500.test.ts b/node/tests/scan-poll-transient-500.test.ts index b4bf1ba3..aade35f3 100644 --- a/node/tests/scan-poll-transient-500.test.ts +++ b/node/tests/scan-poll-transient-500.test.ts @@ -10,7 +10,35 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; * still fail, and the failure message is one a customer can act on. */ -vi.mock("axios"); +vi.mock("axios", () => { + // sable-2s6p — the code calls `apiClient`, an axios instance created with + // maxRedirects: 0. `create` must return something, and it returns the same + // object as the default export so `mockedAxios.get` still refers to the + // function under test. + const instance: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + defaults: { maxRedirects: 0 }, + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + }; + // The default export gets its OWN mocks, distinct from the instance's. If + // production code regresses to bare `axios.get`, the assertions below — which + // watch the instance — stop seeing calls, and the test fails. A shim where + // both are the same object would silently accept that regression. + const bare: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + create: () => instance, + }; + return { default: bare }; +}); vi.mock("ora", () => ({ default: () => ({ start: vi.fn().mockReturnThis(), @@ -32,7 +60,8 @@ import { } from "../src/commands/backend/scan-status.js"; import { EXIT_SUCCESS, EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND } from "../src/utils/api.js"; -const mockedAxios = vi.mocked(axios, true); +// The code calls `apiClient` — the instance `create()` returns. +const mockedAxios = vi.mocked((axios as any).create(), true); /** The verbatim body the customer saw. */ const OBJECT_NOT_FOUND = { diff --git a/node/tests/scan-remote.test.ts b/node/tests/scan-remote.test.ts index c2eefbac..7d620a1c 100644 --- a/node/tests/scan-remote.test.ts +++ b/node/tests/scan-remote.test.ts @@ -12,7 +12,35 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; // ── Mocks ────────────────────────────────────────────────────────────── -vi.mock("axios"); +vi.mock("axios", () => { + // sable-2s6p — the code calls `apiClient`, an axios instance created with + // maxRedirects: 0. `create` must return something, and it returns the same + // object as the default export so `mockedAxios.get` still refers to the + // function under test. + const instance: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + defaults: { maxRedirects: 0 }, + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + }; + // The default export gets its OWN mocks, distinct from the instance's. If + // production code regresses to bare `axios.get`, the assertions below — which + // watch the instance — stop seeing calls, and the test fails. A shim where + // both are the same object would silently accept that regression. + const bare: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + create: () => instance, + }; + return { default: bare }; +}); vi.mock("ora", () => ({ default: () => ({ start: vi.fn().mockReturnThis(), @@ -26,7 +54,8 @@ import axios from "axios"; import { handleScanStatus } from "../src/commands/backend/scan-status.js"; import { EXIT_SUCCESS, EXIT_GENERAL_ERROR, EXIT_SCAN_NOT_FOUND } from "../src/utils/api.js"; -const mockedAxios = vi.mocked(axios, true); +// The code calls `apiClient` — the instance `create()` returns. +const mockedAxios = vi.mocked((axios as any).create(), true); // ── handleScanStatus ─────────────────────────────────────────────────── diff --git a/node/tests/sites-cli.test.ts b/node/tests/sites-cli.test.ts index 398bdece..05c26406 100644 --- a/node/tests/sites-cli.test.ts +++ b/node/tests/sites-cli.test.ts @@ -5,7 +5,35 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; * security monitoring). All tests mock axios so no network calls are made. */ -vi.mock("axios"); +vi.mock("axios", () => { + // sable-2s6p — the code calls `apiClient`, an axios instance created with + // maxRedirects: 0. `create` must return something, and it returns the same + // object as the default export so `mockedAxios.get` still refers to the + // function under test. + const instance: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + defaults: { maxRedirects: 0 }, + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + }; + // The default export gets its OWN mocks, distinct from the instance's. If + // production code regresses to bare `axios.get`, the assertions below — which + // watch the instance — stop seeing calls, and the test fails. A shim where + // both are the same object would silently accept that regression. + const bare: any = { + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + patch: vi.fn(), + interceptors: { response: { use: vi.fn() }, request: { use: vi.fn() } }, + create: () => instance, + }; + return { default: bare }; +}); import axios from "axios"; import { runSitesCreate } from "../src/commands/sites/create.js"; @@ -14,7 +42,8 @@ import { runSitesList } from "../src/commands/sites/list.js"; import { runSitesGet } from "../src/commands/sites/get.js"; import { EXIT_SUCCESS, EXIT_GENERAL_ERROR, EXIT_INSUFFICIENT_SCOPE, EXIT_SCAN_NOT_FOUND, EXIT_QUOTA_EXHAUSTED } from "../src/utils/api.js"; -const mockedAxios = vi.mocked(axios, true); +// The code calls `apiClient` — the instance `create()` returns. +const mockedAxios = vi.mocked((axios as any).create(), true); const opts = { apiKey: "test-key", format: "json", quiet: true }; beforeEach(() => { diff --git a/python/rafter_cli/commands/backend.py b/python/rafter_cli/commands/backend.py index 5e7314b1..90ead2f4 100644 --- a/python/rafter_cli/commands/backend.py +++ b/python/rafter_cli/commands/backend.py @@ -9,6 +9,9 @@ import typer from ..utils.api import ( + api_url, + api_get, + api_post, API_BASE, API_TIMEOUT, API_TIMEOUT_SHORT, @@ -254,8 +257,8 @@ def _poll_until_readable( """ while True: try: - resp = requests.get( - f"{API_BASE}/static/scan", + resp = api_get( + api_url("static/scan"), headers=headers, params={"scan_id": scan_id, "format": fmt}, timeout=API_TIMEOUT_SHORT, @@ -422,8 +425,8 @@ def _do_remote_scan( body["provider"] = resolved_provider body["repo_url"] = resolved_repo_url - resp = requests.post( - f"{API_BASE}/static/scan", + resp = api_post( + api_url("static/scan"), headers=headers, json=body, timeout=API_TIMEOUT, @@ -508,8 +511,8 @@ def usage( """Check quota and usage.""" key = resolve_key(api_key) headers = {"x-api-key": key} - resp = requests.get( - f"{API_BASE}/static/usage", headers=headers, timeout=API_TIMEOUT_SHORT + resp = api_get( + api_url("static/usage"), headers=headers, timeout=API_TIMEOUT_SHORT ) if resp.status_code != 200: print(f"Error: {resp.text}", file=sys.stderr) diff --git a/python/rafter_cli/commands/issues/issues_app.py b/python/rafter_cli/commands/issues/issues_app.py index 638a9177..ec198c47 100644 --- a/python/rafter_cli/commands/issues/issues_app.py +++ b/python/rafter_cli/commands/issues/issues_app.py @@ -14,7 +14,7 @@ import requests import typer -from ...utils.api import API_BASE, EXIT_GENERAL_ERROR, EXIT_SUCCESS, resolve_key +from ...utils.api import api_url, API_BASE, api_get, EXIT_GENERAL_ERROR, EXIT_SUCCESS, resolve_key from ...utils.formatter import fmt, print_stderr from ...utils.git import detect_repo from .dedup import find_duplicates @@ -213,8 +213,8 @@ def from_text( def _drafts_from_backend(scan_id: str, api_key: str | None) -> list[IssueDraft]: key = resolve_key(api_key) - resp = requests.get( - f"{API_BASE}/static/scan", + resp = api_get( + api_url("static/scan"), headers={"x-api-key": key}, params={"scan_id": scan_id, "format": "json"}, timeout=(10, 60), diff --git a/python/rafter_cli/commands/mcp_server.py b/python/rafter_cli/commands/mcp_server.py index 0982f4c1..9ba7fb35 100644 --- a/python/rafter_cli/commands/mcp_server.py +++ b/python/rafter_cli/commands/mcp_server.py @@ -22,7 +22,7 @@ from ..scanners.betterleaks import BetterleaksScanner from ..scanners.regex_scanner import RegexScanner, ScanResult from ..scanners.union import union_scan_results -from ..utils.api import API_TIMEOUT +from ..utils.api import API_TIMEOUT, api_get, api_post from .sites import SITES_API_BASE, describe_sites_error, resolve_mcp_api_key mcp_app = typer.Typer( @@ -218,7 +218,7 @@ def _require_mcp_api_key() -> str: def handle_sites_create(url: str) -> dict: """Register a URL as a Rafter Site and kick off its first scan.""" key = _require_mcp_api_key() - resp = requests.post(SITES_API_BASE, headers={"x-api-key": key}, json={"url": url}, timeout=API_TIMEOUT) + resp = api_post(SITES_API_BASE, headers={"x-api-key": key}, json={"url": url}, timeout=API_TIMEOUT) if resp.status_code != 200: message, _ = describe_sites_error(resp) raise RuntimeError(message) @@ -241,7 +241,7 @@ def handle_sites_scan( if sections: body["sections"] = list(sections) - resp = requests.post(f"{SITES_API_BASE}/scan", headers={"x-api-key": key}, json=body, timeout=API_TIMEOUT) + resp = api_post(f"{SITES_API_BASE}/scan", headers={"x-api-key": key}, json=body, timeout=API_TIMEOUT) if resp.status_code != 200: message, _ = describe_sites_error(resp) raise RuntimeError(message) @@ -263,7 +263,7 @@ def handle_sites_list( if include_archived: params["include_archived"] = "true" - resp = requests.get(SITES_API_BASE, headers={"x-api-key": key}, params=params, timeout=API_TIMEOUT) + resp = api_get(SITES_API_BASE, headers={"x-api-key": key}, params=params, timeout=API_TIMEOUT) if resp.status_code != 200: message, _ = describe_sites_error(resp) raise RuntimeError(message) @@ -275,7 +275,7 @@ def handle_sites_get(id: str) -> dict: from urllib.parse import quote key = _require_mcp_api_key() - resp = requests.get(f"{SITES_API_BASE}/{quote(id, safe='')}", headers={"x-api-key": key}, timeout=API_TIMEOUT) + resp = api_get(f"{SITES_API_BASE}/{quote(id, safe='')}", headers={"x-api-key": key}, timeout=API_TIMEOUT) if resp.status_code != 200: message, _ = describe_sites_error(resp) raise RuntimeError(message) diff --git a/python/rafter_cli/commands/notify.py b/python/rafter_cli/commands/notify.py index 6e7e684e..e760279b 100644 --- a/python/rafter_cli/commands/notify.py +++ b/python/rafter_cli/commands/notify.py @@ -10,6 +10,8 @@ import typer from ..utils.api import ( + api_url, + api_get, API_BASE, API_TIMEOUT_SHORT, EXIT_GENERAL_ERROR, @@ -309,8 +311,8 @@ def _fetch_scan(scan_id: str, api_key: str) -> dict: import requests headers = {"x-api-key": api_key} - resp = requests.get( - f"{API_BASE}/static/scan", + resp = api_get( + api_url("static/scan"), headers=headers, params={"scan_id": scan_id, "format": "json"}, timeout=API_TIMEOUT_SHORT, diff --git a/python/rafter_cli/commands/sites.py b/python/rafter_cli/commands/sites.py index 5ecef8d4..f36e90b3 100644 --- a/python/rafter_cli/commands/sites.py +++ b/python/rafter_cli/commands/sites.py @@ -18,6 +18,8 @@ import typer from ..utils.api import ( + api_get, + api_post, API_BASE, API_TIMEOUT, EXIT_GENERAL_ERROR, @@ -135,7 +137,7 @@ def sites_create( if reject_unsupported_format(fmt): raise typer.Exit(code=EXIT_GENERAL_ERROR) key = resolve_key(api_key) - resp = requests.post( + resp = api_post( SITES_API_BASE, headers={"x-api-key": key}, json={"url": url}, @@ -174,7 +176,7 @@ def sites_scan( body["sections"] = section_list key = resolve_key(api_key) - resp = requests.post( + resp = api_post( f"{SITES_API_BASE}/scan", headers={"x-api-key": key}, json=body, @@ -209,7 +211,7 @@ def sites_list( params["include_archived"] = "true" key = resolve_key(api_key) - resp = requests.get( + resp = api_get( SITES_API_BASE, headers={"x-api-key": key}, params=params, @@ -234,7 +236,7 @@ def sites_get( raise typer.Exit(code=EXIT_GENERAL_ERROR) key = resolve_key(api_key) site_id = quote(id, safe="") - resp = requests.get( + resp = api_get( f"{SITES_API_BASE}/{site_id}", headers={"x-api-key": key}, timeout=API_TIMEOUT, diff --git a/python/rafter_cli/utils/api.py b/python/rafter_cli/utils/api.py index 10d7ce7d..ee4ccb36 100644 --- a/python/rafter_cli/utils/api.py +++ b/python/rafter_cli/utils/api.py @@ -5,6 +5,7 @@ import os import sys +import requests import typer from dotenv import load_dotenv @@ -62,6 +63,72 @@ def handle_scope_error(resp: "requests.Response") -> bool: API_TIMEOUT_SHORT = (10, 30) +def _safe_for_terminal(value: "str | None") -> str: + """Strip non-printable bytes and cap length before echoing untrusted text. + + A redirect target is attacker-controlled if the endpoint is. Header values + cannot contain CR/LF, but ESC is a legal byte, so an unsanitized Location + can emit ANSI sequences that rewrite the user's terminal. + """ + if not isinstance(value, str): + return "" + printable = "".join(c for c in value if c.isprintable()) + return printable[:200] + "\u2026" if len(printable) > 200 else printable + + +def api_url(path: str) -> str: + """Join API_BASE with a path without producing a double slash. + + Mirrors Node's ``apiUrl()``. This is not cosmetic: API_BASE ends in "/" and + call sites used to concatenate "/static/...", producing + ``https://rafter.so/api//static/scan``, which production answers with a 308 + to the single-slash form. That worked only because the client followed the + redirect — so sable-2s6p's fix would have broken every core command. + """ + return f"{API_BASE.rstrip('/')}/{path.lstrip('/')}" + + +def api_request(method: str, url: str, **kwargs) -> "requests.Response": + """sable-2s6p — the HTTP entry point for every authenticated Rafter API call. + + ``allow_redirects=False`` is the point of it. ``requests`` replays headers on + a redirect and, unlike ``Authorization``, a custom ``x-api-key`` header is + NOT stripped when the host changes (``SessionRedirectMixin.rebuild_auth`` + only handles ``Authorization``). Since the API base is user-settable + (``--rafter-url``, self-hosted installs), a 302 from a misconfigured or + hostile endpoint would walk the caller's API key to another host. + + Nothing in this CLI needs to follow a redirect, so none of them do. A + redirect now arrives at the caller as a plain 3xx response, which every + caller already treats as a non-200 error. + + Use this for anything that sends ``x-api-key``. Plain ``requests`` is fine + for user-supplied webhooks and other unauthenticated calls. + """ + kwargs["allow_redirects"] = False + resp = requests.request(method, url, **kwargs) + if 300 <= resp.status_code < 400: + # Otherwise this surfaces as a bare non-200 with an empty body, which + # tells the user nothing about why. + target = _safe_for_terminal(resp.headers.get("location")) or "another host" + print( + f"The Rafter API redirected to {target}, and Rafter does not follow " + "redirects on authenticated requests — your API key would be sent to " + "the redirect target. If you are pointing Rafter at a self-hosted " + "instance, use its final URL.", + file=sys.stderr, + ) + return resp + + +def api_get(url: str, **kwargs) -> "requests.Response": + return api_request("GET", url, **kwargs) + + +def api_post(url: str, **kwargs) -> "requests.Response": + return api_request("POST", url, **kwargs) + + def resolve_key(cli_opt: str | None) -> str: """Resolve API key: --api-key flag > RAFTER_API_KEY env > global config.""" if cli_opt: diff --git a/python/tests/test_api_no_redirect.py b/python/tests/test_api_no_redirect.py new file mode 100644 index 00000000..f8c7a3fe --- /dev/null +++ b/python/tests/test_api_no_redirect.py @@ -0,0 +1,130 @@ +"""sable-2s6p — authenticated Rafter calls must not follow redirects. + +``requests.sessions.SessionRedirectMixin.rebuild_auth`` strips ``Authorization`` +on a host change and leaves arbitrary custom headers intact, so ``x-api-key`` +rides a 302 to whatever host it points at. The API base is user-settable +(``--rafter-url``, self-hosted installs), which makes that a real exfiltration +path rather than a theoretical one. + +Mirrors node/tests/api-no-redirect.test.ts. +""" +from __future__ import annotations + +import pathlib +import re +from unittest.mock import MagicMock, patch + +from rafter_cli.utils.api import api_get, api_post, api_request + +REPO_PY = pathlib.Path(__file__).resolve().parents[1] / "rafter_cli" + + +def _resp(status_code: int = 200, headers=None) -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.headers = headers or {} + return resp + + +class TestRedirectsRefused: + def test_api_request_forces_allow_redirects_false(self): + with patch("rafter_cli.utils.api.requests.request") as req: + req.return_value = _resp() + api_request("GET", "https://rafter.so/api/static/scan") + + assert req.call_args.kwargs["allow_redirects"] is False + + def test_callers_cannot_re_enable_redirects(self): + # Even an explicit allow_redirects=True is overridden — the point is + # that no call site can opt back into leaking the key. + with patch("rafter_cli.utils.api.requests.request") as req: + req.return_value = _resp() + api_request("GET", "https://rafter.so/x", allow_redirects=True) + + assert req.call_args.kwargs["allow_redirects"] is False + + def test_api_get_and_api_post_use_the_right_verbs(self): + with patch("rafter_cli.utils.api.requests.request") as req: + req.return_value = _resp() + api_get("https://rafter.so/x") + api_post("https://rafter.so/y") + + assert [c.args[0] for c in req.call_args_list] == ["GET", "POST"] + assert all(c.kwargs["allow_redirects"] is False for c in req.call_args_list) + + def test_a_refused_redirect_explains_itself(self, capsys): + with patch("rafter_cli.utils.api.requests.request") as req: + # ANSI escape included on purpose: an attacker-controlled Location + # must not be able to rewrite the user's terminal. + req.return_value = _resp( + 302, {"location": "https://evil.example/collect\x1b[31m"} + ) + api_get("https://rafter.so/api/static/scan") + + err = capsys.readouterr().err + assert "evil.example" in err + assert "does not follow redirects" in err + assert "self-hosted" in err + assert "\x1b" not in err + + +class TestNoCallSiteBypassesTheHelper: + """A new authenticated call site must not be able to reintroduce the hole.""" + + def test_no_bare_requests_call_sends_the_api_key(self): + offenders = [] + call = re.compile(r"\brequests\.(get|post|put|delete|patch|request)\(") + + for path in REPO_PY.rglob("*.py"): + if path.name == "api.py" and path.parent.name == "utils": + continue # the helper itself is where requests is allowed + lines = path.read_text().splitlines() + for i, line in enumerate(lines): + if not call.search(line): + continue + window = "\n".join(lines[i : i + 6]) + if "x-api-key" in window or "headers=headers" in window: + offenders.append(f"{path}:{i + 1}") + + assert offenders == [], ( + "These calls send the API key through bare requests, which replays it " + "across a cross-host redirect. Use api_get/api_post from " + f"rafter_cli.utils.api instead:\n" + "\n".join(offenders) + ) + + +class TestNoSecondSession: + def test_no_module_builds_its_own_requests_session(self): + offenders = [ + str(p) + for p in REPO_PY.rglob("*.py") + if not (p.name == "api.py" and p.parent.name == "utils") + and "requests.Session(" in p.read_text() + ] + assert offenders == [], ( + "A bare Session follows redirects by default. Use api_get/api_post " + "from rafter_cli.utils.api:\n" + "\n".join(offenders) + ) + + +class TestApiUrlConstruction: + def test_no_module_builds_a_double_slash_url(self): + # Not cosmetic. API_BASE ends in "/", and f"{API_BASE}/static/..." + # produced https://rafter.so/api//static/scan, which production answers + # with a 308. That worked only because the client followed redirects. + offenders = [] + for path in REPO_PY.rglob("*.py"): + for i, line in enumerate(path.read_text().splitlines()): + if "{API_BASE}/" in line: + offenders.append(f"{path}:{i + 1}") + assert offenders == [], ( + "These build a double-slash URL. Use api_url() instead:\n" + + "\n".join(offenders) + ) + + def test_api_url_joins_cleanly(self): + from rafter_cli.utils.api import API_BASE, api_url + + assert api_url("static/scan") == "https://rafter.so/api/static/scan" + assert api_url("/static/scan") == "https://rafter.so/api/static/scan" + assert API_BASE.endswith("/") # the trap this guards against diff --git a/python/tests/test_api_scope.py b/python/tests/test_api_scope.py index 6089745d..8e143543 100644 --- a/python/tests/test_api_scope.py +++ b/python/tests/test_api_scope.py @@ -101,7 +101,7 @@ def test_no_collisions(self): class TestRemoteScan403: """Verify _do_remote_scan properly handles 403 scope errors.""" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("org/repo", "main", "github", "https://github.com/org/repo")) def test_scope_403_raises_exit_with_code_4(self, _mock_repo, mock_post, capsys): mock_post.return_value = _mock_response( @@ -124,7 +124,7 @@ def test_scope_403_raises_exit_with_code_4(self, _mock_repo, mock_post, capsys): assert "read access" in err assert "https://rfrr.co/account" in err - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("org/repo", "main", "github", "https://github.com/org/repo")) def test_generic_403_raises_exit_with_code_4(self, _mock_repo, mock_post, capsys): mock_post.return_value = _mock_response(403, "forbidden") @@ -141,7 +141,7 @@ def test_generic_403_raises_exit_with_code_4(self, _mock_repo, mock_post, capsys ) assert exc_info.value.exit_code == EXIT_INSUFFICIENT_SCOPE - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("org/repo", "main", "github", "https://github.com/org/repo")) def test_429_still_raises_quota_exhausted(self, _mock_repo, mock_post, capsys): mock_post.return_value = _mock_response(429, "quota exhausted") @@ -158,7 +158,7 @@ def test_429_still_raises_quota_exhausted(self, _mock_repo, mock_post, capsys): ) assert exc_info.value.exit_code == EXIT_QUOTA_EXHAUSTED - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("org/repo", "main", "github", "https://github.com/org/repo")) def test_200_succeeds(self, _mock_repo, mock_post): mock_post.return_value = _mock_response(200, "") @@ -184,7 +184,7 @@ class TestReadOnlyEndpoints: The server returns 200 for valid keys of either scope on GET endpoints. These tests verify the CLI doesn't accidentally scope-check GET calls.""" - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_get_scan_200_with_read_key(self, mock_get): """GET /api/static/scan works fine — no scope check needed.""" mock_get.return_value = _mock_response(200, "") @@ -198,7 +198,7 @@ def test_get_scan_200_with_read_key(self, mock_get): result = _handle_scan_status_interactive("abc", {"x-api-key": "read_key"}, "json", True) assert result == 0 - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_get_usage_200_with_read_key(self, mock_get, capsys): """GET /api/static/usage works with read-only key.""" mock_get.return_value = _mock_response(200, "") @@ -217,7 +217,7 @@ def test_get_usage_200_with_read_key(self, mock_get, capsys): class TestBackwardCompatibility: """Ensure existing 401 and other error paths are unaffected.""" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("org/repo", "main", "github", "https://github.com/org/repo")) def test_401_raises_general_error(self, _mock_repo, mock_post, capsys): mock_post.return_value = _mock_response(401, "invalid api key") @@ -235,7 +235,7 @@ def test_401_raises_general_error(self, _mock_repo, mock_post, capsys): # 401 should NOT hit scope handler, falls through to general error assert exc_info.value.exit_code == EXIT_GENERAL_ERROR - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("org/repo", "main", "github", "https://github.com/org/repo")) def test_500_raises_general_error(self, _mock_repo, mock_post, capsys): mock_post.return_value = _mock_response(500, "internal server error") diff --git a/python/tests/test_plus_scan_approval.py b/python/tests/test_plus_scan_approval.py index 1046aeae..809dd58c 100644 --- a/python/tests/test_plus_scan_approval.py +++ b/python/tests/test_plus_scan_approval.py @@ -149,7 +149,7 @@ def test_refuses_exit_5_when_prompt_answered_no(self, monkeypatch): class TestGateIntegration: - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") def test_refuses_gated_plus_without_calling_backend(self, mock_post, monkeypatch): monkeypatch.delenv("RAFTER_CONFIRM", raising=False) with patch( @@ -168,7 +168,7 @@ def test_refuses_gated_plus_without_calling_backend(self, mock_post, monkeypatch assert exc.value.exit_code == EXIT_CONFIRMATION_REQUIRED mock_post.assert_not_called() - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch( "rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", None), diff --git a/python/tests/test_scan_poll_transient_500.py b/python/tests/test_scan_poll_transient_500.py index 3d7299a5..659466f9 100644 --- a/python/tests/test_scan_poll_transient_500.py +++ b/python/tests/test_scan_poll_transient_500.py @@ -74,7 +74,7 @@ def test_backoff_is_exponential_not_flat(self): assert [backoff_seconds(n) for n in (1, 2, 3, 4)] == [2, 4, 8, 16] def test_actually_sleeps_the_backoff_schedule(self, sleeps): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing()] + [ _server_500() for _ in range(MAX_TRANSIENT_POLL_FAILURES) ] @@ -93,7 +93,7 @@ def test_caps_total_failures_so_a_flapping_server_cannot_loop_forever(self): for _ in range(MAX_TOTAL_TRANSIENT_POLL_FAILURES + 5): flapping += [_server_500(), _processing()] - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = flapping with pytest.raises(typer.Exit) as exc: _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) @@ -102,7 +102,7 @@ def test_caps_total_failures_so_a_flapping_server_cannot_loop_forever(self): def test_consecutive_counter_resets_on_a_successful_poll(self, sleeps): # Two blips far apart must NOT add up to a give-up. - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [ _processing(), _server_500(), @@ -123,7 +123,7 @@ def test_survives_a_nested_json_error_object(self): # methods on that object raised an AttributeError no caller catches, # surfacing as a traceback and defeating the retry entirely. nested = _resp(500, text=json.dumps({"error": {"message": "nested"}})) - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing(), nested, _completed()] code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) @@ -132,7 +132,7 @@ def test_survives_a_nested_json_error_object(self): def test_truncates_a_very_long_server_error(self, capsys): huge = "x" * 5000 big = _resp(500, text=json.dumps({"error": huge})) - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing()] + [ big for _ in range(MAX_TRANSIENT_POLL_FAILURES) ] @@ -148,7 +148,7 @@ def test_reports_the_real_attempt_count_not_the_consecutive_cap(self, capsys): for _ in range(MAX_TOTAL_TRANSIENT_POLL_FAILURES + 5): flapping += [_server_500(), _processing()] - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = flapping with pytest.raises(typer.Exit): _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) @@ -158,7 +158,7 @@ def test_reports_the_real_attempt_count_not_the_consecutive_cap(self, capsys): assert f"after {MAX_TOTAL_TRANSIENT_POLL_FAILURES} attempts" in err def test_unreachable_api_is_not_blamed_on_the_report(self, capsys): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing()] + [ requests.ConnectionError("no route to host") for _ in range(MAX_TRANSIENT_POLL_FAILURES) @@ -174,7 +174,7 @@ def test_first_poll_retries_a_transient_500(self): # The give-up message tells the user to run `rafter get `, which # re-enters at the first poll. If that path did not retry, the remedy # we recommend would be defeated by one bad read. - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_server_500(), _completed()] code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) @@ -184,14 +184,14 @@ def test_first_poll_retries_a_transient_500(self): def test_any_2xx_counts_as_success(self): # Node's axios accepts any 2xx; Python must not diverge. accepted = _resp(202, json_body={"status": "completed", "markdown": "# Done"}) - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [accepted] code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) assert code == EXIT_SUCCESS def test_retry_notice_goes_to_stderr(self, capsys): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing(), _server_500(), _completed()] _handle_scan_status_interactive("s1", HEADERS, "md", quiet=False) @@ -200,7 +200,7 @@ def test_retry_notice_goes_to_stderr(self, capsys): assert "retrying in 2s" in err def test_rides_out_a_single_500_and_completes(self, capsys): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing(), _server_500(), _completed()] code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) @@ -208,7 +208,7 @@ def test_rides_out_a_single_500_and_completes(self, capsys): assert get.call_count == 3 def test_rides_out_several_consecutive_500s(self): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [ _processing(), _server_500(), @@ -222,14 +222,14 @@ def test_rides_out_several_consecutive_500s(self): assert get.call_count == 5 def test_midpoll_404_is_lag_not_a_missing_scan(self): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing(), _resp(404, text="{}"), _completed()] code = _handle_scan_status_interactive("s1", HEADERS, "md", quiet=True) assert code == EXIT_SUCCESS def test_first_poll_404_still_reports_not_found(self): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_resp(404, text="{}")] with pytest.raises(typer.Exit) as exc: _handle_scan_status_interactive("nope", HEADERS, "md", quiet=True) @@ -238,7 +238,7 @@ def test_first_poll_404_still_reports_not_found(self): assert get.call_count == 1 def test_gives_up_when_report_never_becomes_readable(self, capsys): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing()] + [ _server_500() for _ in range(MAX_TRANSIENT_POLL_FAILURES) ] @@ -253,7 +253,7 @@ def test_gives_up_when_report_never_becomes_readable(self, capsys): assert "Object not found" in err def test_does_not_retry_a_non_transient_error(self): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [ _processing(), _resp(403, text=json.dumps({"error": "Invalid API key"})), @@ -265,7 +265,7 @@ def test_does_not_retry_a_non_transient_error(self): assert get.call_count == 2 def test_retries_transport_errors(self): - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [ _processing(), requests.ConnectionError("ECONNRESET"), @@ -279,7 +279,7 @@ def test_a_500_body_is_never_mistaken_for_a_report(self): """The pre-fix bug: the loop called .json() on the 500 body, got no status, fell out of the loop and wrote the error payload out as if it were results.""" - with patch("rafter_cli.commands.backend.requests.get") as get: + with patch("rafter_cli.commands.backend.api_get") as get: get.side_effect = [_processing()] + [ _server_500() for _ in range(MAX_TRANSIENT_POLL_FAILURES) ] diff --git a/python/tests/test_scan_remote.py b/python/tests/test_scan_remote.py index 1c5091fb..4acf0198 100644 --- a/python/tests/test_scan_remote.py +++ b/python/tests/test_scan_remote.py @@ -44,7 +44,7 @@ def _mock_response(status_code: int, text: str = "", json_body=None) -> MagicMoc class TestDoRemoteScan: """Unit tests for the core remote scan trigger function.""" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_success_skip_interactive(self, _mock_repo, mock_post): """200 with skip_interactive returns without polling.""" @@ -60,7 +60,7 @@ def test_success_skip_interactive(self, _mock_repo, mock_post): quiet=True, ) - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_posts_correct_body(self, _mock_repo, mock_post): """Verify POST body contains repository_name, branch_name, scan_mode.""" @@ -83,7 +83,7 @@ def test_posts_correct_body(self, _mock_repo, mock_post): assert body["scan_mode"] == "fast" assert "github_token" not in body - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_github_body_omits_provider_and_repo_url(self, _mock_repo, mock_post): """A github remote produces a body with NO provider/repo_url (byte-identical to today).""" @@ -110,7 +110,7 @@ def test_github_body_omits_provider_and_repo_url(self, _mock_repo, mock_post): assert "provider" not in body assert "repo_url" not in body - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_explicit_provider_github_still_omits(self, _mock_repo, mock_post): """An explicit --provider github still omits provider/repo_url.""" @@ -132,7 +132,7 @@ def test_explicit_provider_github_still_omits(self, _mock_repo, mock_post): assert "provider" not in body assert "repo_url" not in body - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_flag_provider_gitlab_sends_provider_and_repo_url(self, _mock_repo, mock_post): """--provider gitlab + --repo-url are sent for a non-github remote.""" @@ -154,7 +154,7 @@ def test_flag_provider_gitlab_sends_provider_and_repo_url(self, _mock_repo, mock assert body["provider"] == "gitlab" assert body["repo_url"] == "https://gitlab.com/group/project" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_flag_provider_bitbucket_sends_pair(self, _mock_repo, mock_post): """--provider bitbucket + --repo-url are sent together.""" @@ -176,7 +176,7 @@ def test_flag_provider_bitbucket_sends_pair(self, _mock_repo, mock_post): assert body["provider"] == "bitbucket" assert body["repo_url"] == "https://bitbucket.org/team/repo" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("group/project", "main", None, None)) def test_provider_without_any_repo_url_is_omitted(self, _mock_repo, mock_post): """A resolved provider with no repo_url anywhere can't send the pair; stays backward-compatible.""" @@ -200,7 +200,7 @@ def test_provider_without_any_repo_url_is_omitted(self, _mock_repo, mock_post): assert "provider" not in body assert "repo_url" not in body - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch( "rafter_cli.commands.backend.detect_repo", return_value=("group/project", "main", "gitlab", "https://gitlab.com/group/project"), @@ -224,7 +224,7 @@ def test_inferred_gitlab_provider_flows_into_body(self, _mock_repo, mock_post): assert body["provider"] == "gitlab" assert body["repo_url"] == "https://gitlab.com/group/project" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch( "rafter_cli.commands.backend.detect_repo", return_value=("group/project", "main", "gitlab", "https://gitlab.com/group/project"), @@ -249,7 +249,7 @@ def test_explicit_flag_overrides_inferred_provider(self, _mock_repo, mock_post): assert body["provider"] == "bitbucket" assert body["repo_url"] == "https://bitbucket.org/group/project" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_includes_github_token(self, _mock_repo, mock_post): """GitHub token is included in POST body when provided.""" @@ -268,7 +268,7 @@ def test_includes_github_token(self, _mock_repo, mock_post): _, kwargs = mock_post.call_args assert kwargs["json"]["github_token"] == "ghp_test123" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_plus_mode(self, _mock_repo, mock_post): """scan_mode=plus is sent when mode='plus'.""" @@ -287,7 +287,7 @@ def test_plus_mode(self, _mock_repo, mock_post): _, kwargs = mock_post.call_args assert kwargs["json"]["scan_mode"] == "plus" - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_prints_scan_id_when_not_quiet(self, _mock_repo, mock_post, capsys): """Scan ID is printed to stderr when not quiet.""" @@ -305,7 +305,7 @@ def test_prints_scan_id_when_not_quiet(self, _mock_repo, mock_post, capsys): err = capsys.readouterr().err assert "s-xyz" in err - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_auto_detect_message_when_not_explicit(self, _mock_repo, mock_post, capsys): """Auto-detection message prints when repo/branch not explicitly provided.""" @@ -323,7 +323,7 @@ def test_auto_detect_message_when_not_explicit(self, _mock_repo, mock_post, caps err = capsys.readouterr().err assert "auto-detected" in err.lower() - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_429_raises_quota_exhausted(self, _mock_repo, mock_post): """HTTP 429 → exit code 3 (quota exhausted).""" @@ -340,7 +340,7 @@ def test_429_raises_quota_exhausted(self, _mock_repo, mock_post): ) assert exc_info.value.exit_code == EXIT_QUOTA_EXHAUSTED - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_403_scope_raises_insufficient_scope(self, _mock_repo, mock_post, capsys): """HTTP 403 with scope keyword → exit code 4.""" @@ -361,7 +361,7 @@ def test_403_scope_raises_insufficient_scope(self, _mock_repo, mock_post, capsys ) assert exc_info.value.exit_code == EXIT_INSUFFICIENT_SCOPE - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_403_quota_raises_quota_exhausted(self, _mock_repo, mock_post, capsys): """HTTP 403 with scan_mode body → exit code 3 (quota).""" @@ -380,7 +380,7 @@ def test_403_quota_raises_quota_exhausted(self, _mock_repo, mock_post, capsys): ) assert exc_info.value.exit_code == EXIT_QUOTA_EXHAUSTED - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_401_raises_general_error(self, _mock_repo, mock_post): """HTTP 401 → exit code 1 (general error).""" @@ -397,7 +397,7 @@ def test_401_raises_general_error(self, _mock_repo, mock_post): ) assert exc_info.value.exit_code == EXIT_GENERAL_ERROR - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_500_raises_general_error(self, _mock_repo, mock_post): """HTTP 500 → exit code 1 (general error).""" @@ -431,7 +431,7 @@ def test_detect_repo_failure_raises_general_error(self, mock_detect): assert exc_info.value.exit_code == EXIT_GENERAL_ERROR @patch("rafter_cli.commands.backend._handle_scan_status_interactive") - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_calls_status_handler_when_not_skip_interactive( self, _mock_repo, mock_post, mock_status @@ -457,7 +457,7 @@ def test_calls_status_handler_when_not_skip_interactive( ) @patch("rafter_cli.commands.backend._handle_scan_status_interactive") - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_skip_interactive_does_not_call_status_handler( self, _mock_repo, mock_post, mock_status @@ -476,7 +476,7 @@ def test_skip_interactive_does_not_call_status_handler( mock_status.assert_not_called() - @patch("rafter_cli.commands.backend.requests.post") + @patch("rafter_cli.commands.backend.api_post") @patch("rafter_cli.commands.backend.detect_repo", return_value=("owner/repo", "main", "github", "https://github.com/owner/repo")) def test_sends_api_key_header(self, _mock_repo, mock_post): """x-api-key header is set correctly.""" @@ -501,7 +501,7 @@ def test_sends_api_key_header(self, _mock_repo, mock_post): class TestHandleScanStatusInteractive: """Unit tests for the polling/status handler.""" - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_completed_immediately(self, mock_get): """Scan already completed on first poll → return success.""" mock_get.return_value = _mock_response( @@ -514,7 +514,7 @@ def test_completed_immediately(self, mock_get): assert result == EXIT_SUCCESS assert mock_get.call_count == 1 - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_completed_outputs_markdown(self, mock_get, capsys): """Completed scan outputs markdown to stdout.""" mock_get.return_value = _mock_response( @@ -525,7 +525,7 @@ def test_completed_outputs_markdown(self, mock_get, capsys): out = capsys.readouterr().out assert "# Results" in out - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_completed_outputs_json(self, mock_get, capsys): """Completed scan outputs JSON to stdout.""" response_data = {"status": "completed", "findings": []} @@ -535,7 +535,7 @@ def test_completed_outputs_json(self, mock_get, capsys): out = capsys.readouterr().out assert json.loads(out) == response_data - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_404_raises_exit_scan_not_found(self, mock_get): """HTTP 404 → exit code 2 (scan not found).""" mock_get.return_value = _mock_response(404, "not found") @@ -544,7 +544,7 @@ def test_404_raises_exit_scan_not_found(self, mock_get): _handle_scan_status_interactive("bad-id", {"x-api-key": "k"}, "md", True) assert exc_info.value.exit_code == EXIT_SCAN_NOT_FOUND - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_non_200_raises_general_error(self, mock_get): """Non-200, non-404 → exit code 1 (general error).""" mock_get.return_value = _mock_response(500, "server error") @@ -553,7 +553,7 @@ def test_non_200_raises_general_error(self, mock_get): _handle_scan_status_interactive("s1", {"x-api-key": "k"}, "md", True) assert exc_info.value.exit_code == EXIT_GENERAL_ERROR - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_failed_status_raises_general_error(self, mock_get): """Status 'failed' → exit code 1.""" mock_get.return_value = _mock_response( @@ -565,7 +565,7 @@ def test_failed_status_raises_general_error(self, mock_get): assert exc_info.value.exit_code == EXIT_GENERAL_ERROR @patch("rafter_cli.commands.backend.time.sleep") - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_polls_queued_then_completed(self, mock_get, mock_sleep): """Queued → poll → completed.""" mock_get.side_effect = [ @@ -581,7 +581,7 @@ def test_polls_queued_then_completed(self, mock_get, mock_sleep): mock_sleep.assert_called_with(10) @patch("rafter_cli.commands.backend.time.sleep") - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_polls_pending_then_completed(self, mock_get, mock_sleep): """Pending → poll → completed.""" mock_get.side_effect = [ @@ -596,7 +596,7 @@ def test_polls_pending_then_completed(self, mock_get, mock_sleep): assert mock_get.call_count == 2 @patch("rafter_cli.commands.backend.time.sleep") - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_polls_processing_then_failed(self, mock_get, mock_sleep): """Processing → poll → failed.""" mock_get.side_effect = [ @@ -609,7 +609,7 @@ def test_polls_processing_then_failed(self, mock_get, mock_sleep): assert exc_info.value.exit_code == EXIT_GENERAL_ERROR @patch("rafter_cli.commands.backend.time.sleep") - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_multiple_polls_before_completion(self, mock_get, mock_sleep): """Multiple polls before scan completes.""" mock_get.side_effect = [ @@ -626,7 +626,7 @@ def test_multiple_polls_before_completion(self, mock_get, mock_sleep): assert mock_get.call_count == 4 assert mock_sleep.call_count == 3 - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_waiting_message_in_non_quiet_mode(self, mock_get, capsys): """Status messages print to stderr in non-quiet mode.""" mock_get.return_value = _mock_response( @@ -637,7 +637,7 @@ def test_waiting_message_in_non_quiet_mode(self, mock_get, capsys): err = capsys.readouterr().err assert "completed" in err.lower() - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_quiet_mode_suppresses_stderr(self, mock_get, capsys): """Quiet mode suppresses status messages on stderr.""" mock_get.return_value = _mock_response( @@ -649,7 +649,7 @@ def test_quiet_mode_suppresses_stderr(self, mock_get, capsys): # Should NOT print "Scan completed!" in quiet mode assert "completed" not in err.lower() - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_passes_format_param_to_api(self, mock_get): """format param is passed to the API.""" mock_get.return_value = _mock_response( @@ -660,7 +660,7 @@ def test_passes_format_param_to_api(self, mock_get): _, kwargs = mock_get.call_args assert kwargs["params"]["format"] == "json" - @patch("rafter_cli.commands.backend.requests.get") + @patch("rafter_cli.commands.backend.api_get") def test_passes_scan_id_param_to_api(self, mock_get): """scan_id param is passed to the API.""" mock_get.return_value = _mock_response( diff --git a/python/tests/test_sites.py b/python/tests/test_sites.py index a6cd0b1d..bff355d0 100644 --- a/python/tests/test_sites.py +++ b/python/tests/test_sites.py @@ -41,7 +41,7 @@ def _mock_response(status_code: int, json_body: dict | None = None) -> MagicMock class TestSitesCreateCli: - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_success(self, mock_post): mock_post.return_value = _mock_response( 200, {"site": {"id": "p1"}, "run": {"id": "r1"}, "created": True} @@ -56,7 +56,7 @@ def test_success(self, mock_post): assert kwargs["json"] == {"url": "https://example.com"} assert kwargs["headers"] == {"x-api-key": "test-key"} - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_401(self, mock_post): mock_post.return_value = _mock_response(401, {"error": "bad key"}) result = runner.invoke( @@ -64,7 +64,7 @@ def test_401(self, mock_post): ) assert result.exit_code == EXIT_GENERAL_ERROR - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_403_wrong_scope(self, mock_post): mock_post.return_value = _mock_response( 403, {"error": "insufficient scope: requires read-and-scan"} @@ -74,7 +74,7 @@ def test_403_wrong_scope(self, mock_post): ) assert result.exit_code == EXIT_INSUFFICIENT_SCOPE - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_429(self, mock_post): mock_post.return_value = _mock_response(429, {"error": "Rate limit exceeded"}) result = runner.invoke( @@ -95,7 +95,7 @@ def test_rejects_format_md(self): class TestSitesScanCli: - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_sends_project_id_for_bare_id(self, mock_post): mock_post.return_value = _mock_response(200, {"run": {"id": "r1"}}) result = runner.invoke(sites_app, ["scan", "proj-123", "-k", "test-key"]) @@ -103,7 +103,7 @@ def test_sends_project_id_for_bare_id(self, mock_post): _, kwargs = mock_post.call_args assert kwargs["json"] == {"projectId": "proj-123"} - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_sends_url_for_url(self, mock_post): mock_post.return_value = _mock_response(200, {"run": {"id": "r1"}}) result = runner.invoke( @@ -113,7 +113,7 @@ def test_sends_url_for_url(self, mock_post): _, kwargs = mock_post.call_args assert kwargs["json"] == {"url": "https://example.com"} - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_includes_sections(self, mock_post): mock_post.return_value = _mock_response(200, {"run": {"id": "r1"}}) result = runner.invoke( @@ -124,7 +124,7 @@ def test_includes_sections(self, mock_post): _, kwargs = mock_post.call_args assert kwargs["json"] == {"projectId": "proj-123", "sections": ["security", "dns"]} - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_rejects_invalid_section(self, mock_post): result = runner.invoke( sites_app, @@ -133,13 +133,13 @@ def test_rejects_invalid_section(self, mock_post): assert result.exit_code == EXIT_GENERAL_ERROR mock_post.assert_not_called() - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_404_not_owned(self, mock_post): mock_post.return_value = _mock_response(404, {"error": "not found"}) result = runner.invoke(sites_app, ["scan", "proj-123", "-k", "test-key"]) assert result.exit_code == EXIT_SCAN_NOT_FOUND - @patch("rafter_cli.commands.sites.requests.post") + @patch("rafter_cli.commands.sites.api_post") def test_403_run_limit(self, mock_post): mock_post.return_value = _mock_response(403, {"error": "run limit reached"}) result = runner.invoke(sites_app, ["scan", "proj-123", "-k", "test-key"]) @@ -150,7 +150,7 @@ def test_403_run_limit(self, mock_post): class TestSitesListCli: - @patch("rafter_cli.commands.sites.requests.get") + @patch("rafter_cli.commands.sites.api_get") def test_passes_pagination_params(self, mock_get): mock_get.return_value = _mock_response( 200, {"sites": [], "limit": 10, "offset": 5, "has_more": False} @@ -173,7 +173,7 @@ def test_passes_pagination_params(self, mock_get): "include_archived": "true", } - @patch("rafter_cli.commands.sites.requests.get") + @patch("rafter_cli.commands.sites.api_get") def test_401(self, mock_get): mock_get.return_value = _mock_response(401, {"error": "invalid key"}) result = runner.invoke(sites_app, ["list", "-k", "test-key"]) @@ -184,7 +184,7 @@ def test_401(self, mock_get): class TestSitesGetCli: - @patch("rafter_cli.commands.sites.requests.get") + @patch("rafter_cli.commands.sites.api_get") def test_success(self, mock_get): mock_get.return_value = _mock_response( 200, @@ -199,13 +199,13 @@ def test_success(self, mock_get): args, _ = mock_get.call_args assert args[0].endswith("/static/sites/p1") - @patch("rafter_cli.commands.sites.requests.get") + @patch("rafter_cli.commands.sites.api_get") def test_404(self, mock_get): mock_get.return_value = _mock_response(404, {"error": "not found"}) result = runner.invoke(sites_app, ["get", "nonexistent", "-k", "test-key"]) assert result.exit_code == EXIT_SCAN_NOT_FOUND - @patch("rafter_cli.commands.sites.requests.get") + @patch("rafter_cli.commands.sites.api_get") def test_429(self, mock_get): mock_get.return_value = _mock_response( 429, {"error": "Rate limit exceeded", "retryAfter": 30} @@ -218,7 +218,7 @@ def test_429(self, mock_get): class TestMcpSitesCreate: - @patch("rafter_cli.commands.mcp_server.requests.post") + @patch("rafter_cli.commands.mcp_server.api_post") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_success(self, _mock_key, mock_post): mock_post.return_value = _mock_response( @@ -229,7 +229,7 @@ def test_success(self, _mock_key, mock_post): _, kwargs = mock_post.call_args assert kwargs["json"] == {"url": "https://example.com"} - @patch("rafter_cli.commands.mcp_server.requests.post") + @patch("rafter_cli.commands.mcp_server.api_post") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_401_raises(self, _mock_key, mock_post): mock_post.return_value = _mock_response(401, {"error": "invalid key"}) @@ -243,7 +243,7 @@ def test_missing_key_raises_without_crashing(self, _mock_key): class TestMcpSitesScan: - @patch("rafter_cli.commands.mcp_server.requests.post") + @patch("rafter_cli.commands.mcp_server.api_post") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_by_project_id(self, _mock_key, mock_post): mock_post.return_value = _mock_response(200, {"run": {"id": "r1"}}) @@ -262,7 +262,7 @@ def test_rejects_both_project_id_and_url(self): with pytest.raises(RuntimeError, match="not both"): handle_sites_scan(project_id="proj-1", url="https://example.com") - @patch("rafter_cli.commands.mcp_server.requests.post") + @patch("rafter_cli.commands.mcp_server.api_post") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_404_not_owned(self, _mock_key, mock_post): mock_post.return_value = _mock_response(404, {"error": "not found"}) @@ -271,7 +271,7 @@ def test_404_not_owned(self, _mock_key, mock_post): class TestMcpSitesList: - @patch("rafter_cli.commands.mcp_server.requests.get") + @patch("rafter_cli.commands.mcp_server.api_get") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_success_with_params(self, _mock_key, mock_get): mock_get.return_value = _mock_response( @@ -281,7 +281,7 @@ def test_success_with_params(self, _mock_key, mock_get): _, kwargs = mock_get.call_args assert kwargs["params"] == {"limit": "5", "include_archived": "true"} - @patch("rafter_cli.commands.mcp_server.requests.get") + @patch("rafter_cli.commands.mcp_server.api_get") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_429_raises(self, _mock_key, mock_get): mock_get.return_value = _mock_response(429, {"error": "Rate limit exceeded"}) @@ -290,7 +290,7 @@ def test_429_raises(self, _mock_key, mock_get): class TestMcpSitesGet: - @patch("rafter_cli.commands.mcp_server.requests.get") + @patch("rafter_cli.commands.mcp_server.api_get") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_success(self, _mock_key, mock_get): mock_get.return_value = _mock_response( @@ -304,7 +304,7 @@ def test_success(self, _mock_key, mock_get): result = handle_sites_get("p1") assert result["site"]["id"] == "p1" - @patch("rafter_cli.commands.mcp_server.requests.get") + @patch("rafter_cli.commands.mcp_server.api_get") @patch("rafter_cli.commands.mcp_server.resolve_mcp_api_key", return_value="test-key") def test_404_raises(self, _mock_key, mock_get): mock_get.return_value = _mock_response(404, {"error": "not found"}) From 0a2ae5f907f644bcd36e60409ab72a33d2578dcd Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:51:33 -0700 Subject: [PATCH 10/18] ci: run the Python tests on the release path (sable-cazq) (#222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python/tests/ executed in exactly one workflow in this repo — test-comprehensive.yml — and that workflow is not on the release path. publish.yaml had no Python test job at all, and validate-release.yml's test-build ran `python -m build` without pytest, so the pre-release gate asserted the Python package compiles and never that it works. A Python-only regression reached PyPI green, and PyPI is not somewhere you can quietly unship from. - publish.yaml: add test-python, mirroring test-comprehensive.yml's job. It runs in parallel with test-node, so it costs no wall clock the release was not already spending. - publish.yaml: gate BOTH publish jobs on it. The two registries publish from one push and version parity is enforced elsewhere, so a suite that goes red after npm has already published leaves the two runtimes at different versions on the two indexes. The divergence is the failure mode whichever half breaks. - publish.yaml: give test-node a Python toolchain. The node release path looked safe because publish-node has a `needs:` clause, but the job it needed was quietly weaker than the same job in test-comprehensive.yml: tests/cross-runtime-parity.test.ts gates its whole describe block on `python3 -c "import typer"` and describe.skip is silent, so all 40 parity assertions were skipped and the suite reported green. Verified by stubbing python3: "40 skipped", exit 0. Those tests are what enforce the dual-implementation contract — the ones a release least wants to skip. - validate-release.yml: run pytest in test-build. ~80s. Note for PR #221, which touches the same job: it adds `needs: [test-node, test-package]` to publish-python to stop a red NODE suite shipping the PyPI package. That is a different hole. Keep both sets of needs if the two land together. Co-authored-by: mayor --- .github/workflows/publish.yaml | 72 +++++++++++++++++++++++--- .github/workflows/validate-release.yml | 9 +++- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 4984dd70..86e67dfb 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -26,15 +26,64 @@ jobs: with: node-version: "20" + # sable-cazq — tests/cross-runtime-parity.test.ts gates its whole + # describe block on `python3 -c "import typer"` succeeding, and + # describe.skip is silent: with no Python here the release path ran + # `pnpm test`, skipped all 40 parity assertions and reported green. + # Those are the tests that enforce the dual-implementation contract, + # so they are the ones a release least wants to skip. Mirrors the + # setup in test-comprehensive.yml's test-node. + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Enable pnpm run: corepack enable && corepack prepare pnpm@10 --activate - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Install Python dependencies (for cross-runtime parity tests) + working-directory: ./python + run: | + pip install -e ".[dev]" 2>/dev/null || pip install -e . + - name: Run tests run: pnpm test + # sable-cazq — python/tests/ ran in exactly one place in this repo + # (test-comprehensive.yml) and it was not the release path. publish.yaml + # had no Python test job at all and validate-release.yml only built the + # wheel, so a Python-only regression reached PyPI green — and PyPI is not + # somewhere you can quietly unship from. Mirrors test-comprehensive.yml's + # test-python; runs in parallel with test-node, so it costs no wall clock + # the release was not already spending. + test-python: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./python + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # pyproject.toml is Poetry-style: dev deps live under + # [tool.poetry.group.dev.dependencies], which is not a PEP 621 extra, + # so `.[dev]` always falls through to the bare install. The explicit + # pytest line is what actually provides the test deps. + - name: Install dependencies + run: | + pip install -e ".[dev]" 2>/dev/null || pip install -e . + pip install pytest pytest-mock pytest-asyncio + + - name: Run all tests + run: python -m pytest tests/ -v + env: + RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + test-package: runs-on: ubuntu-latest defaults: @@ -82,7 +131,12 @@ jobs: echo "OK: pre-commit hook installed end-to-end" publish-node: - needs: [test-node, test-package] + # test-python is a gate on the npm release too, not only the PyPI one. + # The two registries are published from one push and version parity is + # enforced, so a Python suite that fails after npm has already published + # leaves the two runtimes at different versions on the two indexes — + # the divergence is the failure mode, whichever half breaks. (sable-cazq) + needs: [test-node, test-python, test-package] runs-on: ubuntu-latest # Trusted Publishing requires id-token: write in scope for THIS job (the # top-level grant covers it, but documented here for the job-local audit @@ -143,12 +197,16 @@ jobs: run: npm publish --access public --provenance publish-python: - # sable-bm5k — publish-node has needed the test jobs since it was written; - # this one never did, so a red suite blocked the npm release and shipped - # the PyPI one anyway. In a dual-implementation product that means the two - # runtimes could diverge at the registry, which is the one place users - # cannot see it. - needs: [test-node, test-package] + # needs: is the union of two fixes that landed together. + # sable-bm5k (#221): publish-node has needed the test jobs since it was + # written; this one never did, so a red suite blocked the npm release and + # shipped the PyPI one anyway. In a dual-implementation product that + # diverges the two runtimes at the registry — the one place users cannot + # see it. + # sable-cazq (#222): adds test-python, which is what makes the Python + # tests run on the release path at all. #221 alone only stops a red NODE + # suite shipping Python; it does not make Python tests run. + needs: [test-node, test-python, test-package] runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index a8b459c2..96e40376 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -93,11 +93,18 @@ jobs: pnpm run build pnpm test - - name: Build Python package + # sable-cazq — this job ran `pnpm test` for Node and nothing but a + # wheel build for Python, so the pre-release gate asserted that the + # Python package *compiles*, never that it works. ~80s of pytest is + # the difference between those two claims. + - name: Build and test Python package run: | cd python python -m pip install --upgrade build python -m build + pip install -e . + pip install pytest pytest-mock pytest-asyncio + python -m pytest tests/ -q - name: Verify all artifacts run: | From 3856a9db1f8a2787f3095266a5b744807a3f9175 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:43:06 -0700 Subject: [PATCH 11/18] fix: an unreadable scan report is not a clean scan (sable-fgk7) (#224) The composite action's results step coerced every jq failure into findings_count=0: a body that was not JSON, a 200 carrying an error object, or a parseable payload with no vulnerabilities key passed every severity threshold and rendered "No security findings detected". Validate the shape first; on failure exit 1 with status=unreadable, an actionable message, and no count outputs at all. Once the shape holds the counts cannot fail, so the fallbacks are removed rather than moved. rafter issues create from-scan (Node and Python) had the same shape: data.vulnerabilities || [] turned a processing or failed scan, an error object, or a keyless payload into "No findings to create issues for". Both now refuse with exit 1 and name the scan status. An empty array is still a clean result. Regression coverage: two end-to-end CI jobs drive the real action against the mock backend (a 3-way matrix of unreadable shapes asserting the build fails with status=unreadable and an EMPTY findings-count, plus an exact counts job asserting 3/1/1/0/1); three drift assertions; unit tests on the real functions in both runtimes. Every new guard was mutation-verified by hand: restoring the old behaviour fails the tests that exist to catch it. --- .github/workflows/test-github-action.yml | 119 ++++++++++++++++++ github-action/README.md | 4 +- github-action/action.yml | 27 +++- github-action/tests/mock-rafter-api.py | 46 ++++++- .../tests/test-action-yml-defaults.sh | 37 ++++++ node/src/commands/issues/from-scan.ts | 33 ++++- node/tests/issues.test.ts | 58 +++++++++ .../rafter_cli/commands/issues/issues_app.py | 39 +++++- python/tests/test_issues.py | 119 ++++++++++++++++++ shared-docs/CLI_SPEC.md | 3 + 10 files changed, 470 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-github-action.yml b/.github/workflows/test-github-action.yml index 38875523..1491b3f0 100644 --- a/.github/workflows/test-github-action.yml +++ b/.github/workflows/test-github-action.yml @@ -227,6 +227,125 @@ jobs: fi echo "PASS: the results fetch retried and completed." + # sable-fgk7 — the results step used to coerce EVERY failure to read the + # report into findings_count=0, which passes every severity threshold and + # renders ":white_check_mark: No security findings detected". A report the + # action cannot read is not a clean scan. Three shapes, each of which used + # to land as a clean green: schema-valid-but-wrong (parses, no key), not + # JSON at all, and a 200 whose body is an error object. + test-results-unreadable-is-not-clean: + name: "Results: an unreadable report is not a clean scan (${{ matrix.shape }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shape: [missing-key, not-json, error-object] + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (poll completes; results body is ${{ matrix.shape }}) + env: + PORT: '8791' + FAIL_COUNT: '0' + COMPLETE_AFTER: '1' + RESULTS_SHAPE: ${{ matrix.shape }} + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null && break + sleep 1 + done + curl -sf -X POST -d '{}' http://127.0.0.1:8791/api/static/scan >/dev/null || { + echo "FAIL: mock backend never started"; cat mock.log; exit 1; } + + - name: Run the action against the mock + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8791' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + + - name: Assert the build failed and no count was fabricated + run: | + cat mock.log + FAIL=0 + if [ "${{ steps.scan.outcome }}" != "failure" ]; then + echo "FAIL: an unreadable report must fail the build (outcome='${{ steps.scan.outcome }}')." + FAIL=1 + fi + if [ "${{ steps.scan.outputs.status }}" != "unreadable" ]; then + echo "FAIL: expected status=unreadable, got '${{ steps.scan.outputs.status }}'." + FAIL=1 + fi + # The floor: a count that was never computed must be ABSENT, not 0. + # '0' here is the bug — it is what a consumer gating on the output + # reads as a clean scan. + if [ -n "${{ steps.scan.outputs.findings-count }}" ]; then + echo "FAIL: findings-count was fabricated as '${{ steps.scan.outputs.findings-count }}' from an unreadable report." + FAIL=1 + fi + [ "$FAIL" -eq 0 ] && echo "PASS: unreadable report (${{ matrix.shape }}) failed the build with status=unreadable and no counts." + exit $FAIL + + # The other half of the floor: when the report IS readable the counts must be + # exactly the report's and the build must pass. Without this, a "validation" + # that rejected everything would also land green above. + test-results-counts-exact: + name: "Results: counts are exactly the report's" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (poll completes; report has 3 findings) + env: + PORT: '8792' + FAIL_COUNT: '0' + COMPLETE_AFTER: '1' + RESULTS_SHAPE: 'with-findings' + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null && break + sleep 1 + done + curl -sf -X POST -d '{}' http://127.0.0.1:8792/api/static/scan >/dev/null || { + echo "FAIL: mock backend never started"; cat mock.log; exit 1; } + + - name: Run the action against the mock + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8792' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + + - name: Assert every count is the report's, not a default + run: | + cat mock.log + FAIL=0 + check() { + if [ "$2" != "$3" ]; then + echo "FAIL: $1 expected '$3', got '$2'" + FAIL=1 + fi + } + check outcome "${{ steps.scan.outcome }}" "success" + check status "${{ steps.scan.outputs.status }}" "completed" + check findings-count "${{ steps.scan.outputs.findings-count }}" "3" + check critical-count "${{ steps.scan.outputs.critical-count }}" "1" + check high-count "${{ steps.scan.outputs.high-count }}" "1" + check medium-count "${{ steps.scan.outputs.medium-count }}" "0" + check low-count "${{ steps.scan.outputs.low-count }}" "1" + [ "$FAIL" -eq 0 ] && echo "PASS: counts are exactly the report's (3/1/1/0/1)." + exit $FAIL + test-yaml-validity: name: action.yml is valid YAML runs-on: ubuntu-latest diff --git a/github-action/README.md b/github-action/README.md index e9d94c15..eec9bf02 100644 --- a/github-action/README.md +++ b/github-action/README.md @@ -53,12 +53,12 @@ jobs: | Output | Description | |--------|-------------| | `scan-id` | The Rafter scan ID | -| `findings-count` | Total findings | +| `findings-count` | Total findings. Empty, never `0`, when the report could not be read (see `status`) | | `critical-count` | Critical severity findings | | `high-count` | High severity findings | | `medium-count` | Medium severity findings | | `low-count` | Low severity findings | -| `status` | Scan status | +| `status` | `completed`, `failed`, `timeout`, `unreadable` (the scan may have finished but its report could not be read or parsed), or `unreachable` (the API could not be contacted). Count outputs are only written when `completed` | ## Examples diff --git a/github-action/action.yml b/github-action/action.yml index e096efad..8193cc98 100644 --- a/github-action/action.yml +++ b/github-action/action.yml @@ -303,13 +303,28 @@ runs: fetch_results "${{ runner.temp }}/rafter-results.md" "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}&format=md" fetch_results "${{ runner.temp }}/rafter-results.sarif" "${RAFTER_URL}/api/static/scan?scan_id=${SCAN_ID}&format=sarif" - # Extract counts + # A report this step cannot read is NOT a clean scan (sable-fgk7). + # Every count below used to fall back to 0 on any jq failure, so a + # malformed body, a truncated write, or a 200 carrying an error object + # rendered as ":white_check_mark: No security findings detected" and + # passed every severity threshold. Validate the shape first. Once it + # holds, the count expressions cannot fail and need no fallback; if + # jq itself is broken the step fails, which is the correct outcome. RESULTS="${{ runner.temp }}/rafter-results.json" - FINDINGS_COUNT=$(jq '.vulnerabilities | length' "$RESULTS" 2>/dev/null || echo "0") - CRITICAL_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' "$RESULTS" 2>/dev/null || echo "0") - HIGH_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "error" or .severity == "high")] | length' "$RESULTS" 2>/dev/null || echo "0") - MEDIUM_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "warning" or .severity == "medium")] | length' "$RESULTS" 2>/dev/null || echo "0") - LOW_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "note" or .severity == "low")] | length' "$RESULTS" 2>/dev/null || echo "0") + if ! jq -e 'type == "object" and (.vulnerabilities | type == "array") and all(.vulnerabilities[]; type == "object")' "$RESULTS" >/dev/null 2>&1; then + SNIPPET=$(head -c 300 "$RESULTS" 2>/dev/null | tr -d '\r\n' | cut -c1-200 || true) + echo "::error::Rafter returned a report for scan ${SCAN_ID} that this action cannot read: no 'vulnerabilities' array." + echo "::error::A report that cannot be parsed is not a clean scan, so no counts were produced. Check the scan in your dashboard at ${RAFTER_URL}/dashboard or retry with: rafter get ${SCAN_ID}" + echo "::error::Body started with: ${SNIPPET}" + echo "status=unreadable" >> "$GITHUB_OUTPUT" + exit 1 + fi + + FINDINGS_COUNT=$(jq '.vulnerabilities | length' "$RESULTS") + CRITICAL_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "critical")] | length' "$RESULTS") + HIGH_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "error" or .severity == "high")] | length' "$RESULTS") + MEDIUM_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "warning" or .severity == "medium")] | length' "$RESULTS") + LOW_COUNT=$(jq '[.vulnerabilities[] | select(.severity == "note" or .severity == "low")] | length' "$RESULTS") echo "findings_count=${FINDINGS_COUNT}" >> "$GITHUB_OUTPUT" echo "critical_count=${CRITICAL_COUNT}" >> "$GITHUB_OUTPUT" diff --git a/github-action/tests/mock-rafter-api.py b/github-action/tests/mock-rafter-api.py index 2b77ab97..23af6cc5 100644 --- a/github-action/tests/mock-rafter-api.py +++ b/github-action/tests/mock-rafter-api.py @@ -26,6 +26,19 @@ i.e. as soon as the injected failures are done). Set it higher than the failure window to make the RESULTS fetch fail rather than the poll. + RESULTS_SHAPE what the completed JSON body looks like (sable-fgk7). Default + "ok": {"status":"completed","vulnerabilities":[]}. Others: + with-findings three findings: one critical, one high, one low + missing-key {"scan_id":..,"status":"completed"} — parses, + has no vulnerabilities array at all + not-json a 200 whose body is an HTML error page + error-object a 200 whose body is {"error": ...} + Each of the last three used to make the action report + "No security findings detected" and pass every threshold. + SHAPE_FROM GET index from which the JSON body takes RESULTS_SHAPE + (default COMPLETE_AFTER + 1, so the poll loop sees one healthy + "completed" and the RESULTS fetch gets the shaped body). + md/sarif fetches are never shaped. """ import json import os @@ -38,21 +51,47 @@ FAIL_FOREVER = os.environ.get("FAIL_FOREVER") == "1" FAIL_COUNT = int(os.environ.get("FAIL_COUNT", "1")) COMPLETE_AFTER = int(os.environ.get("COMPLETE_AFTER", str(FAIL_ON))) +RESULTS_SHAPE = os.environ.get("RESULTS_SHAPE", "ok") +SHAPE_FROM = int(os.environ.get("SHAPE_FROM", str(COMPLETE_AFTER + 1))) SCAN_ID = "repro-sable-l10k-0001" +# Severities chosen so every count output is pinned to a distinct value: +# findings=3, critical=1, high=1, medium=0, low=1. +WITH_FINDINGS = [ + {"rule_id": "sql-injection", "severity": "critical", "file_path": "db.php", "line_start": 12}, + {"rule_id": "xss-echo", "severity": "high", "file_path": "view.php", "line_start": 40}, + {"rule_id": "weak-hash", "severity": "low", "file_path": "auth.php", "line_start": 7}, +] + state = {"polls": 0} class Handler(BaseHTTPRequestHandler): def _send(self, code, payload): - body = json.dumps(payload).encode() + self._send_raw(code, json.dumps(payload).encode(), "application/json") + + def _send_raw(self, code, body, content_type): self.send_response(code) - self.send_header("Content-Type", "application/json") + self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) + def _send_shaped_results(self): + """The completed JSON body under RESULTS_SHAPE (never for md/sarif).""" + if RESULTS_SHAPE == "with-findings": + return self._send(200, {"scan_id": SCAN_ID, "status": "completed", + "vulnerabilities": WITH_FINDINGS}) + if RESULTS_SHAPE == "missing-key": + return self._send(200, {"scan_id": SCAN_ID, "status": "completed"}) + if RESULTS_SHAPE == "not-json": + return self._send_raw(200, b"

502 Bad Gateway

", + "text/html") + if RESULTS_SHAPE == "error-object": + return self._send(200, {"error": "Failed to fetch report from storage: Object not found"}) + raise SystemExit(f"unknown RESULTS_SHAPE {RESULTS_SHAPE!r}") + def do_POST(self): if urlparse(self.path).path != "/api/static/scan": return self._send(404, {"error": "not found"}) @@ -84,6 +123,9 @@ def do_GET(self): if n < COMPLETE_AFTER: return self._send(200, {"scan_id": SCAN_ID, "status": "processing"}) + if fmt == "json" and RESULTS_SHAPE != "ok" and n >= SHAPE_FROM: + return self._send_shaped_results() + completed = {"scan_id": SCAN_ID, "status": "completed", "vulnerabilities": []} if fmt == "md": completed["markdown"] = "# Rafter\n\nNo findings.\n" diff --git a/github-action/tests/test-action-yml-defaults.sh b/github-action/tests/test-action-yml-defaults.sh index 59444a34..5bc8740e 100755 --- a/github-action/tests/test-action-yml-defaults.sh +++ b/github-action/tests/test-action-yml-defaults.sh @@ -163,6 +163,43 @@ else failures=$((failures+1)) fi +# ── sable-fgk7: an unreadable report is not a clean scan ───────────────── +# The results step used to coerce every jq failure into findings_count=0, +# which passed every threshold and rendered "No security findings detected". +# Reproduced with a 200 whose body was not JSON, a 200 carrying an error +# object, and a parseable payload with no vulnerabilities key. + +# 14. The payload shape must be validated before any count is computed. +if grep -qF "jq -e 'type == \"object\" and (.vulnerabilities | type == \"array\") and all(.vulnerabilities[]; type == \"object\")'" "$ACTION_YML"; then + echo "PASS: results step validates the payload shape before counting" +else + echo "FAIL: results step no longer validates that .vulnerabilities is an array of objects" + failures=$((failures+1)) +fi + +# 15. No count may fall back to 0 on a jq failure. That fallback IS the bug: +# the error path and the clean path produced the same number. +zero_fallbacks=$(grep -c '|| echo "0"' "$ACTION_YML" || true) +if [ "$zero_fallbacks" -eq 0 ]; then + echo "PASS: no count falls back to 0 on a parse failure" +else + echo "FAIL: ${zero_fallbacks} count(s) still fall back to 0 on a jq failure — an unreadable report would render as clean" + failures=$((failures+1)) +fi + +# 16. The unreadable-payload path must record status=unreadable and exit 1, +# so the declared status output cannot fall back to the poll step's +# 'completed' for a report that was never read. +if awk '/no .vulnerabilities. array/,/^ fi$/' "$ACTION_YML" \ + | grep -q 'status=unreadable' \ + && awk '/no .vulnerabilities. array/,/^ fi$/' "$ACTION_YML" \ + | grep -q 'exit 1'; then + echo "PASS: unreadable payload records status=unreadable and fails the step" +else + echo "FAIL: unreadable-payload branch no longer records status=unreadable and exits 1" + failures=$((failures+1)) +fi + echo "" echo "── results ───────────────────────────────────────────────────────────" echo "Failures: $failures" diff --git a/node/src/commands/issues/from-scan.ts b/node/src/commands/issues/from-scan.ts index 38852bcb..f73f2ba5 100644 --- a/node/src/commands/issues/from-scan.ts +++ b/node/src/commands/issues/from-scan.ts @@ -164,6 +164,36 @@ async function runFromScan(opts: { } } +/** + * The findings list from a scan payload, or an error — never a silent []. + * + * A payload without a `vulnerabilities` array is not "no findings". It is a + * scan that has not completed, a failed scan, or a report this client cannot + * read; filing zero issues from it would report a clean codebase for work + * that was never done (sable-fgk7). An empty array IS a legitimate clean + * result and is returned as such. + */ +export function vulnerabilitiesFromPayload( + data: unknown, + scanId: string +): BackendVulnerability[] { + const payload = data as { vulnerabilities?: unknown; status?: unknown } | null; + if (payload && Array.isArray(payload.vulnerabilities)) { + return payload.vulnerabilities as BackendVulnerability[]; + } + const status = payload && typeof payload.status === "string" ? payload.status : undefined; + if (status && status !== "completed") { + throw new Error( + `Scan ${scanId} is ${status}, not completed — there are no findings to file yet. ` + + `Retry once it completes: rafter get ${scanId}` + ); + } + throw new Error( + `Scan ${scanId} returned no 'vulnerabilities' array; refusing to treat an unreadable ` + + `report as zero findings. Check it with: rafter get ${scanId}` + ); +} + async function draftsFromBackendScan( scanId: string, apiKey?: string @@ -174,8 +204,7 @@ async function draftsFromBackendScan( headers: { "x-api-key": key }, }); - const vulns: BackendVulnerability[] = data.vulnerabilities || []; - return vulns.map(buildFromBackendVulnerability); + return vulnerabilitiesFromPayload(data, scanId).map(buildFromBackendVulnerability); } function draftsFromLocalScan(filePath: string): IssueDraft[] { diff --git a/node/tests/issues.test.ts b/node/tests/issues.test.ts index 4e81f423..aca114fa 100644 --- a/node/tests/issues.test.ts +++ b/node/tests/issues.test.ts @@ -2,6 +2,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import crypto from "crypto"; import fs from "fs"; import { execFileSync } from "child_process"; +// The real function, not a mirror: these tests exist to fail when the source +// changes (sable-fgk7, sable-d2x2). +import { vulnerabilitiesFromPayload } from "../src/commands/issues/from-scan.js"; // ── dedup logic (mirrored from src/commands/issues/dedup.ts) ───────── @@ -657,6 +660,61 @@ describe("from-scan: backend vulnerability drafts", () => { }); }); +// ── from-scan: a payload with no findings list is an error, not zero ── +// +// sable-fgk7. `data.vulnerabilities || []` turned a still-running scan, a +// failed scan, a 200 carrying an error object, and a schema-valid payload with +// no key into "No findings to create issues for". Delete-the-subject test: +// with the fallback restored, every case below except the first two passes +// with [] and this suite fails. + +describe("from-scan: vulnerabilitiesFromPayload (sable-fgk7)", () => { + const SCAN = "scan-abc"; + + it("returns the list when present", () => { + const v = [{ ruleId: "r", level: "error", message: "m", file: "f" }]; + expect(vulnerabilitiesFromPayload({ status: "completed", vulnerabilities: v }, SCAN)).toBe(v); + }); + + it("returns an empty list as a legitimate clean result", () => { + expect(vulnerabilitiesFromPayload({ status: "completed", vulnerabilities: [] }, SCAN)).toEqual([]); + }); + + it("refuses a completed payload with no vulnerabilities key", () => { + expect(() => vulnerabilitiesFromPayload({ scan_id: SCAN, status: "completed" }, SCAN)) + .toThrow(/no 'vulnerabilities' array/); + }); + + it("names the status when the scan has not completed", () => { + expect(() => vulnerabilitiesFromPayload({ status: "processing" }, SCAN)) + .toThrow(/is processing, not completed/); + expect(() => vulnerabilitiesFromPayload({ status: "failed" }, SCAN)) + .toThrow(/is failed, not completed/); + }); + + it("refuses a 200 whose body is an error object", () => { + expect(() => + vulnerabilitiesFromPayload({ error: "Failed to fetch report from storage: Object not found" }, SCAN) + ).toThrow(/no 'vulnerabilities' array/); + }); + + it("refuses a vulnerabilities value that is not a list", () => { + expect(() => vulnerabilitiesFromPayload({ vulnerabilities: "3" }, SCAN)).toThrow(); + expect(() => vulnerabilitiesFromPayload({ vulnerabilities: null }, SCAN)).toThrow(); + }); + + it("refuses non-object payloads", () => { + expect(() => vulnerabilitiesFromPayload(null, SCAN)).toThrow(); + expect(() => vulnerabilitiesFromPayload("502", SCAN)).toThrow(); + }); + + it("points at the scan id in every refusal", () => { + for (const bad of [{ status: "completed" }, { status: "processing" }, {}]) { + expect(() => vulnerabilitiesFromPayload(bad, SCAN)).toThrow(new RegExp(`rafter get ${SCAN}`)); + } + }); +}); + // ── from-scan: --repo flag override ────────────────────────────────── describe("from-scan: repo flag", () => { diff --git a/python/rafter_cli/commands/issues/issues_app.py b/python/rafter_cli/commands/issues/issues_app.py index ec198c47..2e74bc2a 100644 --- a/python/rafter_cli/commands/issues/issues_app.py +++ b/python/rafter_cli/commands/issues/issues_app.py @@ -14,7 +14,7 @@ import requests import typer -from ...utils.api import api_url, API_BASE, api_get, EXIT_GENERAL_ERROR, EXIT_SUCCESS, resolve_key +from ...utils.api import api_url, api_get, EXIT_GENERAL_ERROR, resolve_key from ...utils.formatter import fmt, print_stderr from ...utils.git import detect_repo from .dedup import find_duplicates @@ -74,7 +74,13 @@ def from_scan( # Build drafts if scan_id: - drafts = _drafts_from_backend(scan_id, api_key) + try: + drafts = _drafts_from_backend(scan_id, api_key) + except (UnreadableScanPayload, requests.RequestException, ValueError) as e: + # ValueError covers a non-JSON body from resp.json(). None of these + # is "no findings"; each is a scan whose report could not be read. + print_stderr(fmt.error(str(e))) + raise typer.Exit(code=EXIT_GENERAL_ERROR) else: drafts = _drafts_from_local(from_local) # type: ignore[arg-type] @@ -211,6 +217,33 @@ def from_text( # ── Internal helpers ────────────────────────────────────────────────── +class UnreadableScanPayload(ValueError): + """The scan payload carries no findings list this command can file from.""" + + +def vulnerabilities_from_payload(data: object, scan_id: str) -> list[dict]: + """The findings list from a scan payload, or an error — never a silent []. + + A payload without a ``vulnerabilities`` list is not "no findings". It is a + scan that has not completed, a failed scan, or a report this client cannot + read; filing zero issues from it would report a clean codebase for work + that was never done (sable-fgk7). An empty list IS a legitimate clean + result and is returned as such. + """ + if isinstance(data, dict) and isinstance(data.get("vulnerabilities"), list): + return data["vulnerabilities"] + status = data.get("status") if isinstance(data, dict) else None + if isinstance(status, str) and status != "completed": + raise UnreadableScanPayload( + f"Scan {scan_id} is {status}, not completed — there are no findings to " + f"file yet. Retry once it completes: rafter get {scan_id}" + ) + raise UnreadableScanPayload( + f"Scan {scan_id} returned no 'vulnerabilities' array; refusing to treat an " + f"unreadable report as zero findings. Check it with: rafter get {scan_id}" + ) + + def _drafts_from_backend(scan_id: str, api_key: str | None) -> list[IssueDraft]: key = resolve_key(api_key) resp = api_get( @@ -222,7 +255,7 @@ def _drafts_from_backend(scan_id: str, api_key: str | None) -> list[IssueDraft]: resp.raise_for_status() data = resp.json() - vulns = data.get("vulnerabilities", []) + vulns = vulnerabilities_from_payload(data, scan_id) return [ build_from_backend_vulnerability( BackendVulnerability( diff --git a/python/tests/test_issues.py b/python/tests/test_issues.py index f24bdf2d..5497d0dd 100644 --- a/python/tests/test_issues.py +++ b/python/tests/test_issues.py @@ -683,3 +683,122 @@ def _parse(self, text): def test_labels_are_unique(self): result = self._parse("Critical security vulnerability with credentials and tokens") assert len(result["labels"]) == len(set(result["labels"])) + + +# ── from-scan: a payload with no findings list is an error, not zero ── +# +# sable-fgk7. `data.get("vulnerabilities", [])` turned a still-running scan, a +# failed scan, a 200 carrying an error object, and a schema-valid payload with +# no key into "No findings to create issues for". Delete-the-subject test: +# with the fallback restored, every case below except the first two passes +# with [] and this class fails. Mirrors the Node suite of the same name. + + +class TestVulnerabilitiesFromPayload: + SCAN = "scan-abc" + + def _call(self, data): + from rafter_cli.commands.issues.issues_app import vulnerabilities_from_payload + + return vulnerabilities_from_payload(data, self.SCAN) + + def test_returns_the_list_when_present(self): + v = [{"ruleId": "r", "level": "error", "message": "m", "file": "f"}] + assert self._call({"status": "completed", "vulnerabilities": v}) is v + + def test_empty_list_is_a_legitimate_clean_result(self): + assert self._call({"status": "completed", "vulnerabilities": []}) == [] + + def test_refuses_completed_payload_with_no_key(self): + from rafter_cli.commands.issues.issues_app import UnreadableScanPayload + + with pytest.raises(UnreadableScanPayload, match="no 'vulnerabilities' array"): + self._call({"scan_id": self.SCAN, "status": "completed"}) + + def test_names_the_status_when_not_completed(self): + from rafter_cli.commands.issues.issues_app import UnreadableScanPayload + + with pytest.raises(UnreadableScanPayload, match="is processing, not completed"): + self._call({"status": "processing"}) + with pytest.raises(UnreadableScanPayload, match="is failed, not completed"): + self._call({"status": "failed"}) + + def test_refuses_error_object(self): + from rafter_cli.commands.issues.issues_app import UnreadableScanPayload + + with pytest.raises(UnreadableScanPayload, match="no 'vulnerabilities' array"): + self._call({"error": "Failed to fetch report from storage: Object not found"}) + + def test_refuses_vulnerabilities_that_is_not_a_list(self): + from rafter_cli.commands.issues.issues_app import UnreadableScanPayload + + with pytest.raises(UnreadableScanPayload): + self._call({"vulnerabilities": "3"}) + with pytest.raises(UnreadableScanPayload): + self._call({"vulnerabilities": None}) + + def test_refuses_non_dict_payloads(self): + from rafter_cli.commands.issues.issues_app import UnreadableScanPayload + + with pytest.raises(UnreadableScanPayload): + self._call(None) + with pytest.raises(UnreadableScanPayload): + self._call("502") + + def test_points_at_the_scan_id_in_every_refusal(self): + from rafter_cli.commands.issues.issues_app import UnreadableScanPayload + + for bad in ({"status": "completed"}, {"status": "processing"}, {}): + with pytest.raises(UnreadableScanPayload, match=f"rafter get {self.SCAN}"): + self._call(bad) + + +class TestFromScanCommandRefusesUnreadablePayload: + """The command surface: an unreadable payload exits 1 with the message on + stderr, and never reaches the "No findings to create issues for" path.""" + + def test_unreadable_payload_exits_1(self, monkeypatch, capsys): + from unittest.mock import MagicMock + + import typer + + from rafter_cli.commands.issues import issues_app as mod + + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = {"scan_id": "scan-abc", "status": "completed"} + resp.raise_for_status.return_value = None + monkeypatch.setattr(mod, "api_get", lambda *a, **k: resp) + monkeypatch.setattr(mod, "resolve_key", lambda _k: "key") + + with pytest.raises(typer.Exit) as exc: + mod.from_scan( + scan_id="scan-abc", from_local=None, repo="org/repo", api_key="k", + no_dedup=True, dry_run=True, quiet=False, + ) + assert exc.value.exit_code == 1 + err = capsys.readouterr().err + assert "no 'vulnerabilities' array" in err + assert "No findings to create issues for" not in err + + def test_non_json_body_exits_1(self, monkeypatch, capsys): + from unittest.mock import MagicMock + + import typer + + from rafter_cli.commands.issues import issues_app as mod + + resp = MagicMock() + resp.status_code = 200 + resp.json.side_effect = ValueError("Expecting value: line 1 column 1 (char 0)") + resp.raise_for_status.return_value = None + monkeypatch.setattr(mod, "api_get", lambda *a, **k: resp) + monkeypatch.setattr(mod, "resolve_key", lambda _k: "key") + + with pytest.raises(typer.Exit) as exc: + mod.from_scan( + scan_id="scan-abc", from_local=None, repo="org/repo", api_key="k", + no_dedup=True, dry_run=True, quiet=False, + ) + assert exc.value.exit_code == 1 + assert "No findings to create issues for" not in capsys.readouterr().err diff --git a/shared-docs/CLI_SPEC.md b/shared-docs/CLI_SPEC.md index cd23373d..ba6e0707 100644 --- a/shared-docs/CLI_SPEC.md +++ b/shared-docs/CLI_SPEC.md @@ -149,6 +149,7 @@ After either budget is exhausted the command exits `1`. If the failures reached - It has no "first poll" distinction: by the time it polls, the trigger step has already returned a `scan_id`, so **every** 404 there is treated as read-after-write lag. A scan id the backend accepted but never persisted therefore fails after the 5-failure budget rather than immediately. - Its poll loop is additionally bounded by a wall-clock deadline derived from `timeout-minutes`. Before v0.11 that input was a poll *count*, so a slow API could overrun it; it is now a real deadline. - Its `status` output is `completed`, `failed`, `timeout`, `unreadable` (the scan may have finished but its report could not be read), or `unreachable` (the API could not be contacted). +- Its results step validates the payload **before** counting. A body with no `vulnerabilities` array — not JSON, a `200` carrying an error object, or a parseable payload missing the key — is `status=unreadable`, the job fails, and **no count outputs are written**: a consumer reading `findings-count` sees an empty string, never a fabricated `0`. A report the action cannot read is not a clean scan. An empty array is a clean scan and counts as `0`. ### rafter usage [OPTIONS] @@ -1245,6 +1246,8 @@ Create GitHub issues from scan results. - `--dry-run` — show issues that would be created without actually creating them - `--quiet` — suppress status messages +**A scan payload without a `vulnerabilities` array is an error, not zero findings.** With `--scan-id`, a response that has no `vulnerabilities` list — the scan is still `processing`, it `failed`, the body is not JSON, or it is an error object — exits `1` with a message on stderr that names the scan id and `rafter get `. It never prints "No findings to create issues for". An empty list is a legitimate clean result. Both runtimes. + #### rafter issues create from-text [OPTIONS] Create a GitHub issue from natural language text (stdin, file, or inline). From 3ac4b0e4b6557a71865c80a429e87167af23fcf8 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:27:47 -0700 Subject: [PATCH 12/18] ci: fail when the suite shrinks or a file's tests all skip; name skipped jobs (sable-5d3q, sable-wl3p) (#226) * ci: fail when the suite shrinks or a file's tests all skip; name skipped jobs (sable-5d3q, sable-wl3p) A green run must have run something. sable-cazq was 40 parity tests describe.skip'd on a missing interpreter and the release path exited 0; a total-count floor would not have caught it, a per-file rule does. .github/scripts/test_floor.py reads the vitest JSON or pytest JUnit report and fails the job when executed tests fall below a floor (~95% of today's counts, lowered in the same PR that removes tests) or any file's tests all skipped, and lists every skipped test by name in the step summary. Wired into test-node and test-python on the PR path and the release path. The gate job's skip of the extended matrix on internal PRs now emits a ::warning:: and a summary section naming each job that did not run, so a grey check reads as skipped rather than passed. * ci: derive the skipped-jobs notice from the workflow file instead of a hand-typed list (it omitted e2e-node within the hour) --- .github/scripts/test_floor.py | 170 +++++++++++++++++++++++ .github/workflows/publish.yaml | 14 +- .github/workflows/test-comprehensive.yml | 55 +++++++- 3 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/test_floor.py diff --git a/.github/scripts/test_floor.py b/.github/scripts/test_floor.py new file mode 100644 index 00000000..a148d326 --- /dev/null +++ b/.github/scripts/test_floor.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Fail CI when the test suite quietly shrinks or a file's tests all skip. + +sable-d2x2 detection rule C ("assert non-emptiness"). Two ways a green run can +be vacuous that the runner's own exit code does not catch: + + 1. A whole file's tests are skipped. sable-cazq: 40 parity tests were + `describe.skip`'d because Python was missing, and the release path exited + 0. A total-count floor does NOT catch this (2112 - 40 is still a big + number); a per-file "every test skipped" rule does. + 2. The suite shrinks sharply: a config change, a renamed directory, a broken + glob, and the runner cheerfully runs the 30 tests it found. + +Reads a vitest JSON report (--vitest) or a pytest JUnit XML written with +`-o junit_family=xunit1` (--junit; xunit1 is what carries the per-test `file` +attribute). Exits 1 when: + + * executed tests (passed + failed) < --min-executed, or + * any file has >= 1 test and every one of them was skipped, unless that + file is listed in --allow-all-skipped (a visible, reviewed exception). + +Always writes the executed / skipped counts and every skipped test's name to +$GITHUB_STEP_SUMMARY when set, so a skip is never invisible even when it is +allowed. Stdlib only: this runs before any project dependency is guaranteed. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import xml.etree.ElementTree as ET +from collections import defaultdict + +SKIPPED_STATES = {"skipped", "pending", "todo", "disabled"} + + +def read_vitest(path: str) -> dict[str, dict[str, list[str]]]: + """{file: {"executed": [names], "skipped": [names]}} from a vitest JSON report.""" + with open(path, encoding="utf-8") as fh: + report = json.load(fh) + files: dict[str, dict[str, list[str]]] = {} + cwd = os.getcwd() + os.sep + for result in report.get("testResults", []): + name = result.get("name", "") + rel = name[len(cwd):] if name.startswith(cwd) else name + bucket = files.setdefault(rel, {"executed": [], "skipped": []}) + for case in result.get("assertionResults", []): + title = case.get("fullName") or case.get("title") or "" + state = case.get("status", "") + (bucket["skipped"] if state in SKIPPED_STATES else bucket["executed"]).append(title) + return files + + +def read_junit(path: str) -> dict[str, dict[str, list[str]]]: + """Same shape from a pytest JUnit XML (xunit1 family, which carries `file`).""" + root = ET.parse(path).getroot() + files: dict[str, dict[str, list[str]]] = defaultdict(lambda: {"executed": [], "skipped": []}) + missing_file_attr = 0 + for case in root.iter("testcase"): + file_attr = case.get("file") + if not file_attr: + missing_file_attr += 1 + # xunit2 drops `file`; fall back to the module part of classname so + # the per-file rule still has something to group by. + classname = case.get("classname", "") + parts = [p for p in classname.split(".") if p and not p[:1].isupper()] + file_attr = "/".join(parts) + ".py" if parts else "" + title = f'{case.get("classname", "")}::{case.get("name", "")}' + skipped = case.find("skipped") is not None + (files[file_attr]["skipped"] if skipped else files[file_attr]["executed"]).append(title) + if missing_file_attr: + print( + f"::warning::{missing_file_attr} testcase(s) had no `file` attribute; " + "run pytest with `-o junit_family=xunit1` for exact per-file grouping.", + flush=True, + ) + return dict(files) + + +def summarize(label: str, files: dict[str, dict[str, list[str]]], executed: int, + skipped: int, min_executed: int, all_skipped: list[str], + allowed: set[str]) -> str: + lines = [f"### Test floor — {label}", ""] + lines.append("| Executed | Skipped | Floor | Files |") + lines.append("|---------:|--------:|------:|------:|") + lines.append(f"| {executed} | {skipped} | {min_executed} | {len(files)} |") + lines.append("") + if all_skipped: + lines.append("**Files with every test skipped:**") + for f in all_skipped: + tag = " (allowed by --allow-all-skipped)" if f in allowed else " **← FAIL**" + lines.append(f"- `{f}`{tag}") + lines.append("") + if skipped: + lines.append("
Skipped tests") + lines.append("") + for f, bucket in sorted(files.items()): + for name in bucket["skipped"]: + lines.append(f"- `{f}` — {name}") + lines.append("") + lines.append("
") + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--vitest", help="vitest JSON report (--reporter=json)") + src.add_argument("--junit", help="pytest JUnit XML (-o junit_family=xunit1 --junitxml=...)") + ap.add_argument("--min-executed", type=int, required=True, + help="fail if fewer than this many tests actually ran (passed + failed)") + ap.add_argument("--allow-all-skipped", default="", + help="comma-separated files allowed to have every test skipped") + ap.add_argument("--label", default=None, help="label for the step summary") + args = ap.parse_args(argv) + + if args.vitest: + files = read_vitest(args.vitest) + label = args.label or "vitest" + else: + files = read_junit(args.junit) + label = args.label or "pytest" + + allowed = {f.strip() for f in args.allow_all_skipped.split(",") if f.strip()} + executed = sum(len(b["executed"]) for b in files.values()) + skipped = sum(len(b["skipped"]) for b in files.values()) + all_skipped = sorted(f for f, b in files.items() if b["skipped"] and not b["executed"]) + offending = [f for f in all_skipped if f not in allowed] + + summary = summarize(label, files, executed, skipped, args.min_executed, all_skipped, allowed) + print(summary, flush=True) + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if step_summary: + with open(step_summary, "a", encoding="utf-8") as fh: + fh.write(summary + "\n") + + failed = False + if not files: + print(f"::error::{label}: the report lists no test files at all — nothing ran.", flush=True) + failed = True + if executed < args.min_executed: + print( + f"::error::{label}: only {executed} tests executed, floor is {args.min_executed}. " + "If tests were deliberately removed, lower the floor in the workflow in the same PR " + "so the shrink is a reviewed decision, not a silent one.", + flush=True, + ) + failed = True + for f in offending: + print( + f"::error::{label}: every test in {f} was skipped ({len(files[f]['skipped'])} tests). " + "A file that runs nothing is a broken prerequisite, not a passing file. Fix the " + "prerequisite, or list the file in --allow-all-skipped with a reason in the workflow.", + flush=True, + ) + failed = True + if not failed: + allowed_note = ( + f", {len(all_skipped)} fully-skipped file(s) on the allow-list" if all_skipped + else ", no file fully skipped" + ) + print(f"OK: {label}: {executed} executed (floor {args.min_executed}), " + f"{skipped} skipped{allowed_note}.", flush=True) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 86e67dfb..bd8ed3da 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -49,7 +49,13 @@ jobs: pip install -e ".[dev]" 2>/dev/null || pip install -e . - name: Run tests - run: pnpm test + run: pnpm exec vitest run --reporter=default --reporter=json --outputFile.json=vitest-report.json + + # sable-d2x2 rule C — the release path is exactly where sable-cazq's 40 + # silently-skipped tests reported green. Same floor as + # test-comprehensive.yml; keep the two in step. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "release test-node (vitest)" # sable-cazq — python/tests/ ran in exactly one place in this repo # (test-comprehensive.yml) and it was not the release path. publish.yaml @@ -80,10 +86,14 @@ jobs: pip install pytest pytest-mock pytest-asyncio - name: Run all tests - run: python -m pytest tests/ -v + run: python -m pytest tests/ -v -o junit_family=xunit1 --junitxml=pytest-report.xml env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + # sable-d2x2 rule C — same floor as test-comprehensive.yml; keep in step. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --junit pytest-report.xml --min-executed 1530 --label "release test-python (pytest)" + test-package: runs-on: ubuntu-latest defaults: diff --git a/.github/workflows/test-comprehensive.yml b/.github/workflows/test-comprehensive.yml index 5fd6c4ba..aa30ddac 100644 --- a/.github/workflows/test-comprehensive.yml +++ b/.github/workflows/test-comprehensive.yml @@ -26,6 +26,14 @@ jobs: run: ${{ steps.decide.outputs.run }} run_core: ${{ steps.decide.outputs.run_core }} steps: + # Checked out only so the skip notice below can read the list of gated + # jobs from this file instead of carrying a copy of it. The first + # hand-typed copy omitted e2e-node within the hour it was written. + - uses: actions/checkout@v4 + with: + sparse-checkout: .github/workflows/test-comprehensive.yml + sparse-checkout-cone-mode: false + - id: decide # Values go through env rather than direct ${{ }} interpolation into # the script, so nothing from the PR can be shell-injected. @@ -53,7 +61,31 @@ jobs: echo "run=true" >> "$GITHUB_OUTPUT" elif [ "$HEAD_OWNER" = "Raftersecurity" ] || [ "$AUTHOR" = "Rome-1" ]; then echo "run=false" >> "$GITHUB_OUTPUT" - echo "Internal PR into main (author=$AUTHOR, head repo owner=$HEAD_OWNER) — unit tests still run; extended matrix skipped." >> "$GITHUB_STEP_SUMMARY" + # sable-d2x2 rule B — a skipped job renders as a grey check and + # satisfies a required status check, so the skip has to be said + # out loud, by name, where a reviewer looks: the checks annotation + # and the run summary. The list is READ FROM THIS FILE (every job + # whose `if:` is gated on needs.gate.outputs.run), never typed by + # hand, so it cannot drift from the jobs it describes. + SKIPPED_JOBS=$(awk ' + /^ [A-Za-z0-9_-]+:[[:space:]]*$/ { job=$1; sub(":", "", job) } + /^[[:space:]]*if:.*needs\.gate\.outputs\.run == .true./ { if (job != "" && !seen[job]++) print job } + ' .github/workflows/test-comprehensive.yml) + SKIPPED_CSV=$(printf '%s' "$SKIPPED_JOBS" | paste -sd ',' - | sed 's/,/, /g') + if [ -z "$SKIPPED_JOBS" ]; then + echo "::error::gate: found no jobs gated on needs.gate.outputs.run — the skip notice would be empty. Either the gate is now pointless or this awk no longer matches the file." + exit 1 + fi + echo "::warning::Internal PR into main: extended matrix NOT run — ${SKIPPED_CSV}. Unit tests (test-node, test-python) still run. To run everything, push to a branch and open the PR from a fork, or use workflow_dispatch." + { + echo "### Extended matrix skipped on this run" + echo "" + echo "Internal PR into main (author=\`$AUTHOR\`, head repo owner=\`$HEAD_OWNER\`). These jobs did **not** run and their grey checks mean *skipped*, not *passed*:" + echo "" + printf '%s\n' "$SKIPPED_JOBS" | sed 's/.*/- `&`/' + echo "" + echo "\`test-node\` and \`test-python\` ran. Trigger the full matrix with **workflow_dispatch** if this PR touches packaging, SARIF output, secret patterns, or platform-specific code." + } >> "$GITHUB_STEP_SUMMARY" else echo "run=true" >> "$GITHUB_OUTPUT" fi @@ -100,10 +132,20 @@ jobs: run: node ./dist/index.js --version - name: Run all tests - run: pnpm test + # The JSON report feeds the floor check below; the default reporter + # keeps the log readable. + run: pnpm exec vitest run --reporter=default --reporter=json --outputFile.json=vitest-report.json env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + # sable-d2x2 rule C — a green run must have RUN something. Fails if the + # executed count falls below the floor or any file's tests all skipped + # (sable-cazq: 40 parity tests describe.skip'd, exit 0). The floor is + # ~95% of the count on 2026-09-02 (2099 executed); lower it in the same + # PR that removes tests, so a shrink is a reviewed decision. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "test-node (vitest)" + test-python: needs: gate if: needs.gate.outputs.run_core == 'true' @@ -124,10 +166,17 @@ jobs: pip install pytest pytest-mock pytest-asyncio - name: Run all tests - run: python -m pytest tests/ -v + # xunit1 is the JUnit family that records `file` per test case, which + # the floor check below groups by. + run: python -m pytest tests/ -v -o junit_family=xunit1 --junitxml=pytest-report.xml env: RAFTER_API_KEY: ${{ secrets.RAFTER_API_KEY }} + # sable-d2x2 rule C — see test-node. Floor is ~95% of 1614 executed on + # 2026-09-02. + - name: Assert the suite actually ran + run: python3 ../.github/scripts/test_floor.py --junit pytest-report.xml --min-executed 1530 --label "test-python (pytest)" + # ── E2E CLI tests ───────────────────────────────────────────────── e2e-node: needs: gate From 1ee4d26d56d593c24d3296fb00e21d2d2cc1eb84 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:49:14 -0700 Subject: [PATCH 13/18] test: run the real code instead of hand-copied mirrors (sable-1drb) (#229) The action's threshold-eval and PR-comment-tip tests each carried their own transcription of the bash they tested, so they could pass in full while action.yml was broken. The logic now lives once in github-action/lib/severity.sh, sourced by both action.yml steps and by both tests. Behaviour unchanged. A new end-to-end job runs the real action with findings above the threshold and asserts status=completed AND outcome=failure, which proves the counts reach the gate; drift check 17 asserts action.yml sources the library in both steps and carries no inline copy, so the tests cannot be silently detached again. node/tests/issues.test.ts mirrored four modules (dedup, issue-builder, from-text, from-scan) and tested the mirrors; one had already drifted. The copies are removed and every describe imports the shipped function. Three source functions gained `export` for that. Every guard was mutated by hand: each mutation fails named tests that were untouched by the same mutation before this change. --- .github/workflows/test-github-action.yml | 58 +++++ github-action/action.yml | 36 +-- github-action/lib/severity.sh | 54 +++++ .../tests/test-action-yml-defaults.sh | 42 +++- github-action/tests/test-pr-comment-tip.sh | 36 +-- github-action/tests/test-threshold-eval.sh | 129 +++++----- node/src/commands/issues/from-scan.ts | 2 +- node/src/commands/issues/from-text.ts | 2 +- node/src/commands/issues/issue-builder.ts | 2 +- node/tests/issues.test.ts | 224 ++---------------- 10 files changed, 261 insertions(+), 324 deletions(-) create mode 100644 github-action/lib/severity.sh diff --git a/.github/workflows/test-github-action.yml b/.github/workflows/test-github-action.yml index 1491b3f0..7a7665d8 100644 --- a/.github/workflows/test-github-action.yml +++ b/.github/workflows/test-github-action.yml @@ -346,6 +346,64 @@ jobs: [ "$FAIL" -eq 0 ] && echo "PASS: counts are exactly the report's (3/1/1/0/1)." exit $FAIL + # sable-1drb — the threshold gate is now unit-tested against the real + # lib/severity.sh, but a unit test cannot prove action.yml WIRES it: that + # the counts reach the gate and the gate's verdict reaches the job. One + # end-to-end run with real findings and a threshold they exceed does. + test-threshold-gate-end-to-end: + name: "Threshold gate: real findings above the threshold fail the build" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start mock backend (report has 1 critical, 1 high, 1 low) + env: + PORT: '8793' + FAIL_COUNT: '0' + COMPLETE_AFTER: '1' + RESULTS_SHAPE: 'with-findings' + run: | + nohup python3 github-action/tests/mock-rafter-api.py > mock.log 2>&1 & + for _ in $(seq 1 30); do + curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null && break + sleep 1 + done + curl -sf -X POST -d '{}' http://127.0.0.1:8793/api/static/scan >/dev/null || { + echo "FAIL: mock backend never started"; cat mock.log; exit 1; } + + - name: Run the action with severity-threshold high + id: scan + continue-on-error: true + uses: ./github-action + with: + api-key: 'not-a-real-key' + rafter-url: 'http://127.0.0.1:8793' + timeout-minutes: '2' + upload-sarif: 'false' + comment-on-pr: 'false' + severity-threshold: 'high' + + - name: Assert the gate, not an error, failed the build + run: | + cat mock.log + FAIL=0 + # status=completed AND outcome=failure is the gate's signature: the + # report was read and counted, then the threshold rejected it. + if [ "${{ steps.scan.outputs.status }}" != "completed" ]; then + echo "FAIL: expected status=completed (report read), got '${{ steps.scan.outputs.status }}'" + FAIL=1 + fi + if [ "${{ steps.scan.outcome }}" != "failure" ]; then + echo "FAIL: 1 critical + 1 high with severity-threshold=high must fail the build (outcome='${{ steps.scan.outcome }}')" + FAIL=1 + fi + if [ "${{ steps.scan.outputs.findings-count }}" != "3" ]; then + echo "FAIL: findings-count expected 3, got '${{ steps.scan.outputs.findings-count }}'" + FAIL=1 + fi + [ "$FAIL" -eq 0 ] && echo "PASS: counts reached the gate and the gate failed the build." + exit $FAIL + test-yaml-validity: name: action.yml is valid YAML runs-on: ubuntu-latest diff --git a/github-action/action.yml b/github-action/action.yml index 8193cc98..8fe5ee3c 100644 --- a/github-action/action.yml +++ b/github-action/action.yml @@ -356,6 +356,9 @@ runs: LOW_COUNT: ${{ steps.results.outputs.low_count }} SEVERITY_THRESHOLD: ${{ inputs.severity-threshold }} run: | + # Shared with the tests under tests/ — see lib/severity.sh (sable-1drb). + source "${{ github.action_path }}/lib/severity.sh" + MD_REPORT=$(jq -r '.markdown // empty' "${{ runner.temp }}/rafter-results.md" 2>/dev/null || cat "${{ runner.temp }}/rafter-results.md") if [ "$FINDINGS_COUNT" -eq 0 ]; then @@ -394,10 +397,7 @@ runs: echo "" echo "" echo "" - if [ "$FINDINGS_COUNT" -gt 0 ] && [ "$SEVERITY_THRESHOLD" = "none" ]; then - echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow." - echo "" - fi + rafter_report_only_tip "$FINDINGS_COUNT" "$SEVERITY_THRESHOLD" echo "---" echo "Scan ID: ${SCAN_ID} | Powered by [Rafter](https://rafter.so)" } >> "$COMMENT_FILE" @@ -432,31 +432,11 @@ runs: MEDIUM_COUNT: ${{ steps.results.outputs.medium_count }} LOW_COUNT: ${{ steps.results.outputs.low_count }} run: | - FAIL=0 - - case "$SEVERITY_THRESHOLD" in - critical) - [ "$CRITICAL_COUNT" -gt 0 ] && FAIL=1 - ;; - high) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1 - ;; - medium) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] && FAIL=1 - ;; - low) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] || [ "$LOW_COUNT" -gt 0 ] && FAIL=1 - ;; - none) - FAIL=0 - ;; - *) - echo "::warning::Unknown severity threshold '${SEVERITY_THRESHOLD}', defaulting to 'high'" - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1 - ;; - esac + # The case statement lives in lib/severity.sh so the unit tests under + # tests/ run the same code, not a transcription of it (sable-1drb). + source "${{ github.action_path }}/lib/severity.sh" - if [ "$FAIL" -eq 1 ]; then + if rafter_threshold_fails "$SEVERITY_THRESHOLD" "$CRITICAL_COUNT" "$HIGH_COUNT" "$MEDIUM_COUNT" "$LOW_COUNT"; then echo "::error::Security findings exceed severity threshold '${SEVERITY_THRESHOLD}'" exit 1 fi diff --git a/github-action/lib/severity.sh b/github-action/lib/severity.sh new file mode 100644 index 00000000..ab38d2fb --- /dev/null +++ b/github-action/lib/severity.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Severity-threshold logic shared by github-action/action.yml and the tests +# under github-action/tests/. ONE copy, sourced by both, so the tests exercise +# the code the action runs rather than a transcription of it (sable-1drb). +# +# Sourced, never executed: no `set -e`, no side effects at load time. Every +# function takes explicit arguments so a test can call it without staging +# environment variables, and prints only what the action wants in its log. + +# rafter_threshold_fails THRESHOLD CRITICAL HIGH MEDIUM LOW +# +# Returns 0 when the findings exceed THRESHOLD (the build should fail) and 1 +# otherwise. 'none' never fails. An unrecognised threshold behaves like +# 'high' and says so with a ::warning:: annotation. +rafter_threshold_fails() { + local threshold="$1" critical="$2" high="$3" medium="$4" low="$5" + local fail=0 + case "$threshold" in + critical) + [ "$critical" -gt 0 ] && fail=1 + ;; + high) + [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] && fail=1 + ;; + medium) + [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] || [ "$medium" -gt 0 ] && fail=1 + ;; + low) + [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] || [ "$medium" -gt 0 ] || [ "$low" -gt 0 ] && fail=1 + ;; + none) + fail=0 + ;; + *) + echo "::warning::Unknown severity threshold '${threshold}', defaulting to 'high'" + [ "$critical" -gt 0 ] || [ "$high" -gt 0 ] && fail=1 + ;; + esac + [ "$fail" -eq 1 ] +} + +# rafter_report_only_tip FINDINGS_COUNT THRESHOLD +# +# Prints the report-only tip block for the PR comment iff there are findings +# AND the threshold is 'none' (the default), i.e. the run reported problems +# but was configured never to fail on them. Prints nothing otherwise. +rafter_report_only_tip() { + local findings="$1" threshold="$2" + if [ "$findings" -gt 0 ] && [ "$threshold" = "none" ]; then + echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow." + echo "" + fi +} diff --git a/github-action/tests/test-action-yml-defaults.sh b/github-action/tests/test-action-yml-defaults.sh index 5bc8740e..d68b4462 100755 --- a/github-action/tests/test-action-yml-defaults.sh +++ b/github-action/tests/test-action-yml-defaults.sh @@ -45,25 +45,49 @@ else failures=$((failures+1)) fi -# 3. The report-only tip block must be present and gated on both conditions. -if grep -qE '\[ "\$FINDINGS_COUNT" -gt 0 \] && \[ "\$SEVERITY_THRESHOLD" = "none" \]' "$ACTION_YML"; then - echo "PASS: report-only tip block gated on (findings > 0) AND (threshold == 'none')" +# The threshold case statement and the report-only tip live in lib/severity.sh +# (sable-1drb), sourced by action.yml AND by the unit tests, so checks 3, 4 +# and 17 look there. Check 17 is what stops a "simplification" from inlining +# a copy back into action.yml, which would silently detach the tests again. +SEVERITY_LIB="$(cd "$(dirname "$0")/.." && pwd)/lib/severity.sh" +if [ ! -f "$SEVERITY_LIB" ]; then + echo "FAIL: $SEVERITY_LIB not found" + exit 1 +fi + +# 3. The report-only tip must be gated on both conditions. +if grep -qE '\[ "\$findings" -gt 0 \] && \[ "\$threshold" = "none" \]' "$SEVERITY_LIB"; then + echo "PASS: report-only tip gated on (findings > 0) AND (threshold == 'none')" else - echo "FAIL: report-only tip block missing or mis-gated in $ACTION_YML" + echo "FAIL: report-only tip missing or mis-gated in $SEVERITY_LIB" failures=$((failures+1)) fi -# 4. The threshold-eval step must still handle 'none' as a no-op -# (no FAIL=1 in the none branch). +# 4. The threshold-eval must still handle 'none' as a no-op +# (no fail=1 in the none branch). if awk ' /none\)/ { in_none=1; next } in_none && /;;/ { in_none=0; next } in_none { print } -' "$ACTION_YML" | grep -qE "FAIL *= *1"; then - echo "FAIL: 'none' branch of threshold-eval sets FAIL=1 — that would break the default" +' "$SEVERITY_LIB" | grep -qE "fail *= *1"; then + echo "FAIL: 'none' branch of threshold-eval sets fail=1 — that would break the default" failures=$((failures+1)) else - echo "PASS: 'none' branch of threshold-eval does not set FAIL=1" + echo "PASS: 'none' branch of threshold-eval does not set fail=1" +fi + +# 17. action.yml must SOURCE the library in both steps that use it, and must +# not carry its own copy of the case statement. If either regresses, the +# unit tests go back to testing a transcription. +lib_sources=$(grep -cF 'source "${{ github.action_path }}/lib/severity.sh"' "$ACTION_YML" || true) +inline_cases=$(grep -cE '^\s*(critical|medium|low)\)\s*$' "$ACTION_YML" || true) +if [ "$lib_sources" -ge 2 ] && [ "$inline_cases" -eq 0 ] \ + && grep -q 'rafter_threshold_fails "\$SEVERITY_THRESHOLD"' "$ACTION_YML" \ + && grep -q 'rafter_report_only_tip "\$FINDINGS_COUNT" "\$SEVERITY_THRESHOLD"' "$ACTION_YML"; then + echo "PASS: action.yml sources lib/severity.sh in both steps and carries no inline copy" +else + echo "FAIL: action.yml sources=${lib_sources} (need >=2), inline case branches=${inline_cases} (need 0), or a call site is missing" + failures=$((failures+1)) fi # ── sable-l10k: poll-path retry contract ───────────────────────────────── diff --git a/github-action/tests/test-pr-comment-tip.sh b/github-action/tests/test-pr-comment-tip.sh index 01e8ba68..1ac97a14 100755 --- a/github-action/tests/test-pr-comment-tip.sh +++ b/github-action/tests/test-pr-comment-tip.sh @@ -1,39 +1,38 @@ #!/usr/bin/env bash # -# Unit test for the new PR-comment "report-only tip" block in -# github-action/action.yml. Re-implements the if block verbatim and -# exercises every input combination. +# Unit test for the PR-comment "report-only tip" block in +# github-action/action.yml. Sources github-action/lib/severity.sh — the SAME +# file action.yml sources at run time — and exercises every input +# combination of rafter_report_only_tip. # # The tip should appear iff (FINDINGS_COUNT > 0) AND (SEVERITY_THRESHOLD == 'none'). +# +# This test used to carry its own copy of the if block (sable-1drb). It now +# runs the code the action runs. set -u +# shellcheck source=../lib/severity.sh +source "$(cd "$(dirname "$0")/.." && pwd)/lib/severity.sh" + failures=0 total=0 -# Mirror of the new block under the "Comment on PR" step's COMMENT_FILE builder. -emit_tip_if_applicable() { - if [ "$FINDINGS_COUNT" -gt 0 ] && [ "$SEVERITY_THRESHOLD" = "none" ]; then - echo "> :information_source: This run is report-only. To fail the build on critical/high findings, set \`severity-threshold: high\` in your workflow." - echo "" - fi -} - TIP_NEEDLE="report-only" # assert_tip assert_tip() { local name="$1"; local expected="$2" - FINDINGS_COUNT="$3"; SEVERITY_THRESHOLD="$4" + local findings="$3" threshold="$4" total=$((total+1)) local out - out=$(emit_tip_if_applicable) + out=$(rafter_report_only_tip "$findings" "$threshold") local has_tip="no" if echo "$out" | grep -q "$TIP_NEEDLE"; then has_tip="yes"; fi if [ "$has_tip" != "$expected" ]; then - echo "FAIL: $name — findings=$FINDINGS_COUNT threshold=$SEVERITY_THRESHOLD → expected tip=$expected got $has_tip" + echo "FAIL: $name — findings=$findings threshold=$threshold → expected tip=$expected got $has_tip" failures=$((failures+1)) else echo "PASS: $name (tip=$has_tip)" @@ -53,6 +52,15 @@ assert_tip "findings + low" no 5 low assert_tip "no findings + high" no 0 high assert_tip "no findings + critical" no 0 critical +echo "── the tip must tell the reader what to set ─────────────────────────" +total=$((total+1)) +if rafter_report_only_tip 5 none | grep -q 'severity-threshold: high'; then + echo "PASS: tip names the input to set" +else + echo "FAIL: tip no longer names severity-threshold: high" + failures=$((failures+1)) +fi + echo "" echo "── results ───────────────────────────────────────────────────────────" echo "Total: $total Failures: $failures" diff --git a/github-action/tests/test-threshold-eval.sh b/github-action/tests/test-threshold-eval.sh index 99dbfc30..eddb5c68 100755 --- a/github-action/tests/test-threshold-eval.sh +++ b/github-action/tests/test-threshold-eval.sh @@ -1,108 +1,95 @@ #!/usr/bin/env bash # # Unit test for the "Evaluate severity threshold" step in -# github-action/action.yml. Re-implements the case statement verbatim and -# exercises every branch with deliberate inputs. +# github-action/action.yml. Sources github-action/lib/severity.sh — the SAME +# file action.yml sources at run time — and exercises every branch of +# rafter_threshold_fails with deliberate inputs. # -# If you change the case body in action.yml, you MUST change it here too — -# the test-action-yml-defaults check enforces drift detection on the default -# value, but the case body itself is duplicated by design (sourcing bash out -# of YAML at test time is fragile). +# This test used to carry its own copy of the case statement, so it could +# pass in full while action.yml was broken (sable-1drb). It now runs the code +# the action runs. The drift detector (test-action-yml-defaults.sh) separately +# asserts that action.yml still sources the library rather than inlining a +# copy again. # # Exit 0 = all cases pass. Exit 1 = at least one case failed. set -u +# shellcheck source=../lib/severity.sh +source "$(cd "$(dirname "$0")/.." && pwd)/lib/severity.sh" + failures=0 total=0 -# Mirror of the case body in github-action/action.yml under the -# "Evaluate severity threshold" step. Returns 1 if the threshold would -# fail the build given the current *_COUNT envs, else 0. -evaluate_threshold() { - local FAIL=0 - case "$SEVERITY_THRESHOLD" in - critical) - [ "$CRITICAL_COUNT" -gt 0 ] && FAIL=1 - ;; - high) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1 - ;; - medium) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] && FAIL=1 - ;; - low) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] || [ "$MEDIUM_COUNT" -gt 0 ] || [ "$LOW_COUNT" -gt 0 ] && FAIL=1 - ;; - none) - FAIL=0 - ;; - *) - [ "$CRITICAL_COUNT" -gt 0 ] || [ "$HIGH_COUNT" -gt 0 ] && FAIL=1 - ;; - esac - return $FAIL -} - -# assert_threshold +# assert_threshold assert_threshold() { local name="$1"; local expected="$2" - SEVERITY_THRESHOLD="$3" - CRITICAL_COUNT="$4"; HIGH_COUNT="$5"; MEDIUM_COUNT="$6"; LOW_COUNT="$7" + local threshold="$3" crit="$4" high="$5" med="$6" low="$7" total=$((total+1)) - evaluate_threshold - local actual=$? + local actual="pass" + if rafter_threshold_fails "$threshold" "$crit" "$high" "$med" "$low" >/dev/null; then + actual="fail" + fi if [ "$actual" != "$expected" ]; then - echo "FAIL: $name — threshold=$SEVERITY_THRESHOLD crit=$CRITICAL_COUNT high=$HIGH_COUNT med=$MEDIUM_COUNT low=$LOW_COUNT → expected exit=$expected got $actual" + echo "FAIL: $name — threshold=$threshold crit=$crit high=$high med=$med low=$low → expected build=$expected got $actual" failures=$((failures+1)) else echo "PASS: $name" fi } -echo "── 'none' threshold (the new default) — must never fail ─────────────" -assert_threshold "none + no findings" 0 none 0 0 0 0 -assert_threshold "none + low only" 0 none 0 0 0 7 -assert_threshold "none + medium only" 0 none 0 0 3 0 -assert_threshold "none + high only" 0 none 0 5 0 0 -assert_threshold "none + critical only" 0 none 2 0 0 0 -assert_threshold "none + everything" 0 none 9 9 9 9 +echo "── 'none' threshold (the default) — must never fail ────────────────" +assert_threshold "none + no findings" pass none 0 0 0 0 +assert_threshold "none + low only" pass none 0 0 0 7 +assert_threshold "none + medium only" pass none 0 0 3 0 +assert_threshold "none + high only" pass none 0 5 0 0 +assert_threshold "none + critical only" pass none 2 0 0 0 +assert_threshold "none + everything" pass none 9 9 9 9 echo "── 'critical' threshold — fail only on critical ────────────────────" -assert_threshold "critical + clean" 0 critical 0 0 0 0 -assert_threshold "critical + only high" 0 critical 0 4 0 0 -assert_threshold "critical + only medium" 0 critical 0 0 4 0 -assert_threshold "critical + only low" 0 critical 0 0 0 4 -assert_threshold "critical + critical=1" 1 critical 1 0 0 0 -assert_threshold "critical + critical+high" 1 critical 1 5 0 0 +assert_threshold "critical + clean" pass critical 0 0 0 0 +assert_threshold "critical + only high" pass critical 0 4 0 0 +assert_threshold "critical + only medium" pass critical 0 0 4 0 +assert_threshold "critical + only low" pass critical 0 0 0 4 +assert_threshold "critical + critical=1" fail critical 1 0 0 0 +assert_threshold "critical + critical+high" fail critical 1 5 0 0 echo "── 'high' threshold — fail on critical or high ─────────────────────" -assert_threshold "high + clean" 0 high 0 0 0 0 -assert_threshold "high + only medium" 0 high 0 0 4 0 -assert_threshold "high + only low" 0 high 0 0 0 4 -assert_threshold "high + critical only" 1 high 1 0 0 0 -assert_threshold "high + high only" 1 high 0 1 0 0 -assert_threshold "high + critical+high" 1 high 1 1 0 0 +assert_threshold "high + clean" pass high 0 0 0 0 +assert_threshold "high + only medium" pass high 0 0 4 0 +assert_threshold "high + only low" pass high 0 0 0 4 +assert_threshold "high + critical only" fail high 1 0 0 0 +assert_threshold "high + high only" fail high 0 1 0 0 +assert_threshold "high + critical+high" fail high 1 1 0 0 echo "── 'medium' threshold — fail on crit/high/medium ───────────────────" -assert_threshold "medium + clean" 0 medium 0 0 0 0 -assert_threshold "medium + only low" 0 medium 0 0 0 4 -assert_threshold "medium + critical only" 1 medium 1 0 0 0 -assert_threshold "medium + high only" 1 medium 0 1 0 0 -assert_threshold "medium + medium only" 1 medium 0 0 1 0 +assert_threshold "medium + clean" pass medium 0 0 0 0 +assert_threshold "medium + only low" pass medium 0 0 0 4 +assert_threshold "medium + critical only" fail medium 1 0 0 0 +assert_threshold "medium + high only" fail medium 0 1 0 0 +assert_threshold "medium + medium only" fail medium 0 0 1 0 echo "── 'low' threshold — fail on anything ──────────────────────────────" -assert_threshold "low + clean" 0 low 0 0 0 0 -assert_threshold "low + only low" 1 low 0 0 0 1 -assert_threshold "low + critical only" 1 low 1 0 0 0 +assert_threshold "low + clean" pass low 0 0 0 0 +assert_threshold "low + only low" fail low 0 0 0 1 +assert_threshold "low + critical only" fail low 1 0 0 0 echo "── unknown threshold — falls back to 'high' behavior ───────────────" -assert_threshold "unknown + clean" 0 badvalue 0 0 0 0 -assert_threshold "unknown + critical" 1 badvalue 1 0 0 0 -assert_threshold "unknown + high" 1 badvalue 0 1 0 0 -assert_threshold "unknown + medium only" 0 badvalue 0 0 3 0 +assert_threshold "unknown + clean" pass badvalue 0 0 0 0 +assert_threshold "unknown + critical" fail badvalue 1 0 0 0 +assert_threshold "unknown + high" fail badvalue 0 1 0 0 +assert_threshold "unknown + medium only" pass badvalue 0 0 3 0 + +echo "── unknown threshold — must say so in the log ──────────────────────" +total=$((total+1)) +if rafter_threshold_fails badvalue 0 0 0 0 | grep -q "::warning::Unknown severity threshold 'badvalue'"; then + echo "PASS: unknown threshold emits a ::warning:: naming the value" +else + echo "FAIL: unknown threshold no longer warns" + failures=$((failures+1)) +fi echo "" echo "── results ───────────────────────────────────────────────────────────" diff --git a/node/src/commands/issues/from-scan.ts b/node/src/commands/issues/from-scan.ts index f73f2ba5..aca9f0f8 100644 --- a/node/src/commands/issues/from-scan.ts +++ b/node/src/commands/issues/from-scan.ts @@ -207,7 +207,7 @@ async function draftsFromBackendScan( return vulnerabilitiesFromPayload(data, scanId).map(buildFromBackendVulnerability); } -function draftsFromLocalScan(filePath: string): IssueDraft[] { +export function draftsFromLocalScan(filePath: string): IssueDraft[] { const raw = fs.readFileSync(filePath, "utf-8"); const parsed = JSON.parse(raw); // New shape: { _note, scan_mode, triage_applied, results: [...] } diff --git a/node/src/commands/issues/from-text.ts b/node/src/commands/issues/from-text.ts index 1298263a..947f2e0f 100644 --- a/node/src/commands/issues/from-text.ts +++ b/node/src/commands/issues/from-text.ts @@ -130,7 +130,7 @@ async function readInput(opts: { * - File paths → mentioned in body * - Security keywords → security label */ -function parseNaturalText(text: string): ParsedIssue { +export function parseNaturalText(text: string): ParsedIssue { const lines = text.trim().split("\n"); const labels: string[] = []; diff --git a/node/src/commands/issues/issue-builder.ts b/node/src/commands/issues/issue-builder.ts index 04853ab0..b9209d51 100644 --- a/node/src/commands/issues/issue-builder.ts +++ b/node/src/commands/issues/issue-builder.ts @@ -35,7 +35,7 @@ export interface LocalScanResult { }>; } -function severityLabel(level: string): string { +export function severityLabel(level: string): string { const map: Record = { error: "critical", critical: "critical", diff --git a/node/tests/issues.test.ts b/node/tests/issues.test.ts index aca114fa..77fe6a4e 100644 --- a/node/tests/issues.test.ts +++ b/node/tests/issues.test.ts @@ -1,47 +1,29 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import crypto from "crypto"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; import fs from "fs"; -import { execFileSync } from "child_process"; -// The real function, not a mirror: these tests exist to fail when the source -// changes (sable-fgk7, sable-d2x2). -import { vulnerabilitiesFromPayload } from "../src/commands/issues/from-scan.js"; - -// ── dedup logic (mirrored from src/commands/issues/dedup.ts) ───────── - -const FINGERPRINT_PREFIX = ""; - -function fingerprint(file: string, ruleId: string): string { - return crypto - .createHash("sha256") - .update(`${file}:${ruleId}`) - .digest("hex") - .slice(0, 12); -} - -function embedFingerprint(body: string, fp: string): string { - return `${body}\n\n${FINGERPRINT_PREFIX}${fp}${FINGERPRINT_SUFFIX}`; -} - -function extractFingerprint(body: string): string | null { - const idx = body.indexOf(FINGERPRINT_PREFIX); - if (idx === -1) return null; - const start = idx + FINGERPRINT_PREFIX.length; - const end = body.indexOf(FINGERPRINT_SUFFIX, start); - if (end === -1) return null; - return body.slice(start, end); -} - -type GHIssue = { number: number; title: string; body: string; labels: string[]; html_url: string; state: string }; - -function findDuplicates(existingIssues: GHIssue[], newFingerprints: string[]): Set { - const existingFps = new Set(); - for (const issue of existingIssues) { - const fp = extractFingerprint(issue.body); - if (fp) existingFps.add(fp); - } - return new Set(newFingerprints.filter((fp) => existingFps.has(fp))); -} +// The real functions, not mirrors. This file used to carry hand-copied +// re-implementations of dedup, issue-builder, from-text and from-scan and +// tested those; it could pass in full while the source was broken +// (sable-1drb, specimen 11 of sable-d2x2). Every describe below now +// exercises the code the CLI ships. +import { + fingerprint, + embedFingerprint, + extractFingerprint, + findDuplicates, +} from "../src/commands/issues/dedup.js"; +import type { GitHubIssue as GHIssue } from "../src/commands/issues/github-client.js"; +import { + severityLabel, + buildFromBackendVulnerability, + buildFromLocalMatch, + type IssueDraft, + type BackendVulnerability, + type LocalScanResult, +} from "../src/commands/issues/issue-builder.js"; +import { parseNaturalText } from "../src/commands/issues/from-text.js"; +import { vulnerabilitiesFromPayload, draftsFromLocalScan } from "../src/commands/issues/from-scan.js"; + +type LocalMatch = LocalScanResult["matches"][number]; describe("fingerprint", () => { it("produces deterministic 12-char hex hash", () => { @@ -111,91 +93,6 @@ describe("findDuplicates", () => { }); }); -// ── issue-builder logic (mirrored from src/commands/issues/issue-builder.ts) ── - -function severityLabel(level: string): string { - const map: Record = { - error: "critical", - critical: "critical", - warning: "high", - high: "high", - note: "medium", - medium: "medium", - low: "low", - }; - return map[level.toLowerCase()] || "medium"; -} - -function severityEmoji(level: string): string { - const sev = severityLabel(level); - const emojis: Record = { - critical: "\u{1F534}", - high: "\u{1F7E0}", - medium: "\u{1F7E1}", - low: "\u{1F7E2}", - }; - return emojis[sev] || "\u{1F7E1}"; -} - -interface IssueDraft { - title: string; - body: string; - labels: string[]; - fingerprint: string; -} - -interface BackendVulnerability { - ruleId: string; - level: string; - message: string; - file: string; - line?: number; -} - -function buildFromBackendVulnerability(vuln: BackendVulnerability): IssueDraft { - const sev = severityLabel(vuln.level); - const emoji = severityEmoji(vuln.level); - const fp = fingerprint(vuln.file, vuln.ruleId); - const title = `${emoji} [${sev.toUpperCase()}] ${vuln.ruleId}: ${vuln.message.length > 80 ? vuln.message.slice(0, 77) + "..." : vuln.message}`; - let body = `## Security Finding\n\n`; - body += `**Rule:** \`${vuln.ruleId}\`\n`; - body += `**Severity:** ${sev}\n`; - body += `**File:** \`${vuln.file}\``; - if (vuln.line) body += ` (line ${vuln.line})`; - body += `\n\n`; - body += `### Description\n\n${vuln.message}\n\n`; - body += `### Remediation\n\nReview and fix the finding in \`${vuln.file}\`.\n`; - body += `\n---\n*Created by [Rafter CLI](https://rafter.so) — security for AI builders*\n`; - const labels = ["security", `severity:${sev}`, `rule:${vuln.ruleId}`]; - return { title, body: embedFingerprint(body, fp), labels, fingerprint: fp }; -} - -type LocalMatch = { pattern: { name: string; severity: string; description?: string }; line?: number; column?: number; redacted?: string }; - -function buildFromLocalMatch(file: string, match: LocalMatch): IssueDraft { - const sev = severityLabel(match.pattern.severity); - const emoji = severityEmoji(match.pattern.severity); - const fp = fingerprint(file, match.pattern.name); - const basename = file.split("/").pop() || file; - const title = `${emoji} [${sev.toUpperCase()}] Secret detected: ${match.pattern.name} in ${basename}`; - let body = `## Secret Detection\n\n`; - body += `**Pattern:** \`${match.pattern.name}\`\n`; - body += `**Severity:** ${sev}\n`; - body += `**File:** \`${file}\``; - if (match.line) body += ` (line ${match.line})`; - body += `\n`; - if (match.redacted) body += `**Match:** \`${match.redacted}\`\n`; - body += `\n`; - if (match.pattern.description) body += `### Description\n\n${match.pattern.description}\n\n`; - body += `### Remediation\n\n`; - body += `1. Rotate the exposed credential immediately\n`; - body += `2. Remove the secret from source code\n`; - body += `3. Use environment variables or a secrets manager instead\n`; - body += `\n---\n*Created by [Rafter CLI](https://rafter.so) — security for AI builders*\n`; - const labels = ["security", "secret-detected", `severity:${sev}`]; - return { title, body: embedFingerprint(body, fp), labels, fingerprint: fp }; -} - describe("severityLabel", () => { it("maps error to critical", () => expect(severityLabel("error")).toBe("critical")); it("maps warning to high", () => expect(severityLabel("warning")).toBe("high")); @@ -288,63 +185,6 @@ describe("buildFromLocalMatch", () => { }); }); -// ── from-text parsing logic (mirrored from src/commands/issues/from-text.ts) ── - -interface ParsedIssue { - title: string; - body: string; - labels: string[]; -} - -function parseNaturalText(text: string): ParsedIssue { - const lines = text.trim().split("\n"); - const labels: string[] = []; - let title = ""; - let bodyStart = 0; - for (let i = 0; i < lines.length; i++) { - const line = lines[i].trim(); - if (line) { - title = line.replace(/^#+\s*/, "").trim(); - bodyStart = i + 1; - break; - } - } - if (!title) title = "Security issue reported via Rafter CLI"; - if (title.length > 120) title = title.slice(0, 117) + "..."; - const bodyLines = lines.slice(bodyStart); - let body = bodyLines.join("\n").trim(); - if (!body) body = text.trim(); - const textLower = text.toLowerCase(); - if (textLower.includes("critical") || textLower.includes("p0")) { - labels.push("severity:critical"); - } else if (textLower.includes("high severity") || textLower.includes("high risk") || textLower.includes("p1")) { - labels.push("severity:high"); - } else if (textLower.includes("medium") || textLower.includes("p2")) { - labels.push("severity:medium"); - } else if (textLower.includes("low") || textLower.includes("p3")) { - labels.push("severity:low"); - } - const securityKeywords = [ - "security", "vulnerability", "cve", "cwe", "owasp", "secret", - "credential", "token", "password", "injection", "xss", "csrf", "ssrf", "exploit", - ]; - if (securityKeywords.some((kw) => textLower.includes(kw))) { - labels.push("security"); - } - const fileRefs = text.match(/(?:^|\s)([a-zA-Z0-9_./-]+\.[a-zA-Z]{1,10})(?::(\d+))?/gm); - if (fileRefs && fileRefs.length > 0) { - const files = fileRefs.map((f) => f.trim()).filter((f) => f.includes("/") || f.includes(".")); - if (files.length > 0) { - body += `\n\n### Referenced Files\n\n`; - for (const f of files.slice(0, 10)) { - body += `- \`${f}\`\n`; - } - } - } - body += `\n\n---\n*Created by [Rafter CLI](https://rafter.so) — security for AI builders*\n`; - return { title, body, labels: [...new Set(labels)] }; -} - describe("parseNaturalText", () => { it("extracts first line as title", () => { const result = parseNaturalText("SQL injection in login form\nDetails here"); @@ -416,20 +256,6 @@ describe("parseNaturalText", () => { }); }); -// ── from-scan command logic (mirrored from src/commands/issues/from-scan.ts) ── - -function draftsFromLocalScan(filePath: string): IssueDraft[] { - const raw = fs.readFileSync(filePath, "utf-8"); - const results: Array<{ file: string; matches: LocalMatch[] }> = JSON.parse(raw); - const drafts: IssueDraft[] = []; - for (const result of results) { - for (const match of result.matches) { - drafts.push(buildFromLocalMatch(result.file, match)); - } - } - return drafts; -} - describe("from-scan: draftsFromLocalScan", () => { const sampleScanJson: Array<{ file: string; matches: LocalMatch[] }> = [ { From 6d6aa3334d69e28f0151ec77899915650c9e0029 Mon Sep 17 00:00:00 2001 From: Rome Thorstenson <36779795+Rome-1@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:09:23 -0700 Subject: [PATCH 14/18] =?UTF-8?q?fix(security):=20CLI=20ship=20set=20?= =?UTF-8?q?=E2=80=94=20classifier=20+=20install/verify=20hardening=20(rf-6?= =?UTF-8?q?pqx,=20rf-3rsj,=20sable-c6an,=20rf-7dda,=20rf-er8a,=20rf-fuwy)?= =?UTF-8?q?=20(#235)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(risk-rules): newline is a statement separator + heredoc-aware sanitizer (rf-6pqx, rf-3rsj, #230) A newline was tokenized as whitespace, so a multi-line Bash command was one segment. When the first word was a text-exec (echo/printf/grep), every later line was redacted as its operand, so `echo x\nrm -rf /` classified LOW and bypassed the CRITICAL hard-block, the deny-list, the approval list, and approve-dangerous mode. Reported independently by kerckhoffs (se-ijzs) and the external reporter J/JJB1 (se-odv4 finding 1); confirmed in both runtimes. The one-line "add \n to CHAIN_OPS" is a NO-OP: the tokenizer eats \n in the whitespace branch before the operator branch ever runs. The real fix: - Exclude \n/\r from the whitespace skip so a newline reaches the operator branch and ends the segment like ";" (both runtimes). Matching still runs over the whole reassembled string, so `curl x | bash` is unaffected. - Strip heredoc BODIES that a command consumes as DATA (`cat > doc.md <` line continuations before parsing, so `r\m -rf /` (which bash runs as `rm -rf /`) forms the token `rm` and is caught; preserved inside single quotes (se-y6vo). - Fail closed on an unterminated quote: `$'a\'b'` desyncs the sanitizer's quote state and swallowed a trailing payload; an unterminated parse now matches the raw string instead (se-y6vo). Test: rf-6pqx-newline-heredoc-battery.json (29 cases, both directions) is run byte-for-byte by both runtimes (node/tests/risk-rules-newline-heredoc.test.ts, python/tests/test_risk_rules_newline_heredoc.py) so a clean result in one cannot hide a bug in the other. Existing suites unchanged: 137 node + 43 python pass. Co-designed with kerckhoffs (se-ijzs, shared battery) and achebe (#230 heredoc). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BbKepZbxosd5TEJas9Mmhg * fix(risk-rules): two heredoc cases that were MORE permissive than main Sits directly on fix/rf-6pqx-newline-heredoc @ db9a809. Cherry-pick or merge; it touches only the two functions named below and adds four cases to the shared battery. Measured on that branch against origin/main, both runtimes: case main #234 cat <<< "hello" > a.txt \n rm -rf / critical LOW cat < a.md <<'EOF'` still matches at the right offset with delimiter EOF. 2. A HEREDOC PIPED INTO A SHELL IS EXECUTED. `heredocOwnerExecutes` reads only the text BEFORE the introducer, so it never sees `| bash` and `cat` looks like a plain writer. main got this right by accident — it scanned the body as bare words — so the heredoc pass has to keep it deliberately. Added `heredocOutputPipedToShell`, which walks the pipeline stages after the introducer with the same env-assignment and TAIL_WRAPPERS skipping the owner check already uses, so `| sudo bash` resolves too. Keeping a body is the safe direction: it can only over-block, and only for a shape that should block anyway. Four cases added to rf-6pqx-newline-heredoc-battery.json, which both runtimes read byte-for-byte, so neither can go green while the other is broken. Each mutation-verified: reverting either fix turns exactly two battery cases red. Not addressed here, filed as sable-c6an (P0, pre-existing on main and on this branch alike, so not a blocker for this one): `bash -c "$(echo rm -rf /)"` and `echo "rm -rf /" | bash` both classify low, because a word that is entirely a command substitution leaves the piece's literal text empty and the `-c` branch reads only that text. Same root as (2) — whether this segment's OUTPUT becomes code. A fix is on fix/rf-3rsj-heredoc-tokenizer @ 2e2d53b. Suites on this branch after the change: node 269 + 33 pass, python 304 pass. My own 18-case heredoc battery goes 12/18 -> 14/18 against it; the four still red are all sable-c6an. * fix(risk-rules): fold the sable-c6an bypass fix into the ship lineage Mayor caught an integration gap: sable-c6an's fix existed ONLY on fix/rf-3rsj-heredoc-tokenizer @ 2e2d53b, which is not an ancestor of this branch. The ship lineage was db9a809 plus the two-regression fix, and neither carried it — so merging #234 as it stood would have shipped the heredoc work and left the live P0 behind. This branch is now the assembled artifact: goldwasser's db9a809 + the two permissive regressions + sable-c6an, all three verified together rather than separately. The fix ports the same distinction the tokenizer branch used, onto this branch's structure: codeCarrying — this segment RUNS a command string it was handed executesOutput — this segment's STDOUT becomes code somewhere else (`… | bash`, or a substitution used as a -c script) Three sites follow from it. The `-c` branch now sanitizes the piece's SUBSTITUTIONS as well as its literal text — a word that is entirely a substitution leaves `text` empty, which is how the whole script argument was vanishing. Text-exec operands and quoted prose arguments are code when the segment's output is executed. And `sanitize` now keeps the operator that ends each segment, so a segment can be asked whether the next one is a shell. The two must stay separate or `bash -c "echo 'rm -rf /'"` breaks: that runs echo, and echo prints. It has an existing test asserting low, it is in the battery as an explicit guard row, and it still passes. VERIFIED ON THE ASSEMBLED ARTIFACT, not on the pieces: - all nine cases mayor asked for: six critical (2 regressions + 4 c6an), #230 documentation still low, print-only script still low, rf-6pqx newline still critical - kerckhoffs's differential, BOTH runtimes: CLEAN (70 generated rows, candidate never weaker than main except a pure data heredoc) - kerckhoffs's battery, BOTH runtimes: ALL PASS - shared battery grown 29 -> 37; every added row proven RED on db9a809 and green here, except the print-only guard row which must stay low on both - repo suites: node 320 pass, python 304 pass The differential is the check that would have caught the original regression, and it is now the one I would gate the merge on rather than any battery — a battery only asks the questions someone thought to ask. * test(risk-rules): commit the differential-vs-main gate; stop tracking the node_modules symlink The stop-ship regressions on db9a809 passed the battery because its only here-string case was single-line. A battery only asks the questions someone thought to ask; the differential asks a generated corpus and compares against main, so an unanticipated permissive move is caught structurally. - python/tests/test_rf6pqx_differential.py runs the gate against the origin/main baseline (via git show) and FAILS LOUDLY if the baseline is unobtainable. - rf6pqx_differential.py / rf6pqx-differential.ts are the committed corpus+compare (authored by kerckhoffs), runnable in both runtimes. - Drop the accidentally-tracked node/node_modules symlink. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BbKepZbxosd5TEJas9Mmhg * fix(agent): init writes a resolvable absolute hook command; verify executes the gate (rf-er8a, rf-fuwy) rf-er8a — `agent init` wrote a bare `rafter hook pretool`. Under the documented `npx @rafter-security/cli agent init --all` there is no global `rafter`, so the command exits 127; agents block only on exit 2, so the gate is inert while settings.json looks configured. init now pins BOTH the node interpreter and the resolved CLI entrypoint (dist/index.js carries `#!/usr/bin/env node`, so an absolute entrypoint alone still fails where node is off PATH), across every integration write path, and refuses to print a clean "initialized" when the written hook is in an npx cache or does not execute-and-enforce. rf-fuwy — `agent verify` reported an inert gate as healthy, byte-identical to a working one, because it only checked that the hook was CONFIGURED. It now EXECUTES each configured command via `sh -c` with a benign payload (expect allow) and the canonical CRITICAL payload (expect deny); 127 / non-JSON / wrong decision is a hard FAIL. A hook deliberately disabled (RAFTER_DISABLE_HOOKS / global) is reported as its own state, not a failure. The opt-in --probe no longer spawns verify's own entrypoint (which always resolved) — it runs the configured command. The misleading "resolve like Claude Code" comment is corrected. Tests: node/tests/verify-hook-liveness.test.ts (positive/negative/127 controls for the executor). End-to-end: init writes an absolute command that verify reports "installed and enforcing" even with rafter off PATH — the exact state that previously read as a healthy but inert gate. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BbKepZbxosd5TEJas9Mmhg * fix(hook): a project .env cannot disable the hook (rf-7dda) dotenv.config() at CLI startup loads $CWD/.env — the cloned repo's own file when rafter runs inside an agent hook. dotenv does not override an already-set var but DOES introduce an unset one, so a repo shipping RAFTER_DISABLE_HOOKS=1 (or any RAFTER_DISABLE_* / RAFTER_HOOK_* value) could switch off the victim's command policy and secret scanning — defeating hook-control's contract that the disable signal is honored only from the machine owner's environment. guardSecurityEnvFromDotenv() runs dotenv then drops any RAFTER_DISABLE_* / RAFTER_HOOK_* var that was not already in the real environment. Owner values and legitimate .env keys (RAFTER_API_KEY, …) are untouched. Python does not load a cwd .env on the hook path; a regression test pins that. Verified: battery goes red on the unguarded sequence, green with the guard, in both runtimes (parity). se-80vx / rf-7dda. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WSumMwjhyvqi8qdZkuQrGD * test(risk-rules): make the committed node differential actually run, and floor the battery Four gaps on the ship branch, all of the same shape: a check that is present is not a check that runs. 1. THE NODE DIFFERENTIAL WAS NEVER INVOKED. `node/tests/rf6pqx-differential.ts` is committed next to its Python twin, but vitest collects `*.test.ts` and nothing wrapped it — so the repo had two differentials on disk and one running, which from a file listing is indistinguishable from having both. Wrapped it in a real test that mirrors test_rf6pqx_differential.py, including the rule that an unobtainable baseline FAILS rather than skips. Verified non-vacuous: pointed at db9a809's classifier it reports all 14 permissive regressions and goes red; on this branch it is CLEAN. 2. THE BATTERY HAD DRIFTED INTO TWO LISTS. The repo's JSON and the copy in the rig working dir had both reached 37 cases with 7 DIFFERING — each looked complete. The repo file is now the union at 44 (gaining `eval "$(…)"`, the sudo-sh pipe, the backtick c6an form and both redirect-target guards), and both unit suites read it byte-for-byte. 3. THE BATTERY HAD NO FLOOR. It is the gate that catches a MISSING fix — the differential structurally cannot, because a fix absent on both sides is not a permissive move, which is exactly how a differential ran CLEAN against a branch that had lost sable-c6an. That makes its contents load-bearing, so classifier_battery_floor.py asserts it has not shrunk AND still gates both directions: want-critical rows catch a missing fix, want-low rows stop a fix being bought with an over-block. #230 was an over-block report, so a battery with no low rows would have been happy to ship it. Non-vacuous both ways: truncating the battery or raising either floor fails it. 4. A PARITY GAP I CREATED. `node/tests/risk-rules-heredoc.test.ts` landed here without its Python mirror — swept in by the same `git add -A` that committed the node_modules symlink. Mirror restored, 18 tests each side. Also closes the ignore gap behind that symlink: `.gitignore` had `node_modules/` with a trailing slash, which matches DIRECTORIES ONLY, so a symlink walked straight past it. Both forms listed now, so the accident cannot recur — f1132ad removed the file but left the hole open. * ci: fetch main so the differential gate can actually run #235 is red on exactly the failure the gate was designed to produce: could not obtain main baseline for the differential gate: fatal: invalid object name 'main'. That is the gate working — it FAILS rather than skips when the baseline is missing, because a differential that silently skips is a vacuous check. But actions/checkout fetches only the PR ref, so `origin/main` is never in the clone and the gate could not have passed in CI at all. It was green locally, where main is always present, which is why nobody saw it until the branch ran. A depth-1 fetch is enough — the gate reads one blob, not history — so this costs a second rather than a full-history clone. Reproduced both halves against a shallow clone: `git show origin/main:python/rafter_cli/core/risk_rules.py` exits 128 before the fetch and prints the 506-line baseline after it. Added to test-node and test-python, the two jobs that run the gate. * fix(agent): tests assert rf-er8a's absolute-command contract; verify retries a transient spawn (test-node CI) The rf-er8a change (init writes an absolute ` hook …` command instead of a bare `rafter hook …`) correctly broke five existing tests that asserted the old bare spelling, and my verify-hook-liveness test asserted an exact exit status that flaked under CI process pressure. Both are fixed so test-node goes green — the classifier and gate tests were already passing. - agent-commands / cursor-deep-support / platform-integration: assert the resolvable-absolute contract (command ends with `hook ` and its first token is an absolute path) instead of `=== "rafter hook pretool"`, and match the dedup filters on the `hook ` tail. This tests the property rf-er8a guarantees, not the old string. - runConfiguredHook: retry a transient spawn failure (EAGAIN under load returned a null status in CI, so the deny case got 0 while allow/127 got null). A resource hiccup must not be reported as an inert gate — a false alarm is worse than a slow check. - verify-hook-liveness: assert the DECISION (deny/allow/none), which carries the meaning, rather than the exact exit code. Local: node 210 + the 7 previously-failing tests now pass (agent-commands 81, cursor-deep 13, platform-integration 77, verify-hook-liveness 4); tsc clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BbKepZbxosd5TEJas9Mmhg * fix(agent): retry the hook probe on a signal kill too, and make the liveness tests retry (achebe review) Follow-up to 63aa475, from achebe's measured review: - runConfiguredHook retried only on `result.error`, but a SIGKILL (which the OOM killer sends under the same process pressure that produces the EAGAIN this retry exists for) sets `result.signal` with a null status and no `error`, so it fell straight through and was reported as an inert gate — the exact false alarm, by another route. Retry now fires on `result.error || result.signal`. A persistent failure still returns decision:null, so a genuinely inert hook is still reported inert (no fail-open). - The liveness tests spawn real subprocesses and are inherently sensitive to process pressure; a per-test `retry: 3` keeps a transient spawn hiccup from flaking the security gate's own tests red without masking a real failure (which reproduces on every attempt). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BbKepZbxosd5TEJas9Mmhg * test(agent): drop the liveness test-level retry — the production retry carries it (achebe review) Reverting the `retry: 3` added in ab0f695. My justification ("a retry can't mask a real failure, which reproduces every attempt") only holds for a DETERMINISTIC failure; for an intermittent one — exactly the failure mode of code spawning real subprocesses — a retry turns a 1-in-4 race green ~99% of the time, a vacuous pass on the very test that asserts the security gate is alive. And it is now redundant: runConfiguredHook retries the transient spawn error/signal itself (same fix), so the production code carries the process-pressure case the test-retry was added for. Keeping both leaves only the weaker mechanism, able to hide a future intermittent real failure. If the test flakes after the production retry, that is signal about runConfiguredHook, and we want to hear it. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BbKepZbxosd5TEJas9Mmhg * fix(exec): --force no longer skips approval; approval needs a person at a TTY; add --dry-run (rf-ss67) `rafter agent exec --force ""` ran any HIGH-tier command unprompted: the PreToolUse hook classified the quoted argument as prose (risk-rules "unrecognized evaluator" limitation), and exec then skipped its own approval prompt because of --force. Reported in the secbolt audit (se-ezvc / se-6g2q). The approval model is now: only a person at an interactive terminal can approve, because every other channel — a flag, an env var, a piped "yes" — can be produced by the agent whose command is being gated. - --force is a hidden no-op kept so old invocations parse; it prints a notice and falls into the normal approval flow. - A command that requires approval is prompted only when stdin is a TTY; otherwise it is denied (exit 1) with a message saying to run it at a terminal or have the machine owner adjust commandPolicy in ~/.rafter/config.json. A piped "yes" is therefore not an approval. - --dry-run, advertised in three shipped docs but absent from exec, now exists: prints the verdict, risk tier and reason, runs nothing; exit 0 allowed, 1 blocked, 2 requires approval. - The documented `-- ` form is accepted (variadic argument); words are re-quoted (shellQuote / shlex.join) so the classifier evaluates the string the shell will run. Both runtimes. Docs: CLI_SPEC agent exec section, skill guardrails and cli-reference (the --force "ack flag" advice is gone), CHANGELOG. Tests: the file mode is the witness in every approval test — the command must not have run. Node: forced high-risk command denied and not executed; --force still parses; piped yes is not an approval; dry-run allowed/approval /blocked exit codes with nothing executed; `--` form; joinCommandParts quoting. Python: the same seven. Residual (filed as a follow-up): the hook still cannot see through `rafter agent exec ""` — that lives in the sanitizer #235 rewrites — and a pty wrapper could fake a terminal. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CKZyh4yEZBJnrjmoyPeSpG * test(exec): assert the denial line itself, not the notice that also names a terminal With the TTY gate removed by hand, the forced-approval tests still passed: the --force notice mentions "interactive terminal" too, and an EOFError on input() happened to exit 1. They now assert "Command denied: approval needs an interactive terminal" and a clean stderr, so removing the gate fails them by name in both runtimes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CKZyh4yEZBJnrjmoyPeSpG * chore: remove the inert node/.gitignore decoy It contained `node/node_modules`. Patterns in a nested .gitignore resolve relative to that file's own directory, so it matched node/node/node_modules — a path that does not exist. It never ignored anything. On ccd54a5 it was the replacement for achebe's working root-.gitignore fix and did not work: fresh repo, that OID's ignore files, a real symlink at node/node_modules -> git check-ignore said NOT IGNORED. The root fix (both forms, e90c8c5) is back and check-ignore attributes the match to .gitignore:17 — never to the nested file. Deleting it because an inert guard is worse than no guard: its only remaining function is to look like protection and stop the next person checking whether there is any. * chore(release): bump to v0.10.1 The bump lives in this PR rather than in a separate commit on main so that merging the ship set yields a releasable main in one step. main-into-prod is what fires publish, and npm rejects a republish of an existing version — so a main left at 0.10.0 would disclose the fixes and publish nothing. #233 touches neither version file, so this does not conflict with it in either merge order. --------- Co-authored-by: Claude Opus 4.8 --- .github/scripts/classifier_battery_floor.py | 108 ++++++ .github/workflows/test-comprehensive.yml | 23 ++ .gitignore | 6 + CHANGELOG.md | 2 + node/package.json | 2 +- .../skills/rafter/docs/cli-reference.md | 2 +- .../skills/rafter/docs/guardrails.md | 4 +- node/src/commands/agent/exec.ts | 87 ++++- node/src/commands/agent/init.ts | 106 +++++- node/src/commands/agent/verify.ts | 138 ++++++- node/src/core/risk-rules.ts | 282 ++++++++++++++- node/src/index.ts | 4 +- node/src/utils/env-guard.ts | 35 ++ node/tests/agent-commands.test.ts | 74 +++- node/tests/cursor-deep-support.test.ts | 18 +- node/tests/env-guard.test.ts | 52 +++ node/tests/exec-join.test.ts | 26 ++ node/tests/platform-integration.test.ts | 17 +- node/tests/rf6pqx-differential.test.ts | 70 ++++ node/tests/rf6pqx-differential.ts | 35 ++ node/tests/risk-rules-heredoc.test.ts | 161 +++++++++ node/tests/risk-rules-newline-heredoc.test.ts | 32 ++ node/tests/verify-hook-liveness.test.ts | 56 +++ python/pyproject.toml | 2 +- python/rafter_cli/commands/agent.py | 71 +++- python/rafter_cli/core/risk_rules.py | 337 +++++++++++++++++- python/tests/rf6pqx_differential.py | 53 +++ python/tests/test_e2e_cli.py | 85 +++++ python/tests/test_hook_env_guard.py | 27 ++ python/tests/test_rf6pqx_differential.py | 37 ++ python/tests/test_risk_rules_heredoc.py | 137 +++++++ .../tests/test_risk_rules_newline_heredoc.py | 34 ++ rf-6pqx-newline-heredoc-battery.json | 315 ++++++++++++++++ shared-docs/CLI_SPEC.md | 7 +- 34 files changed, 2339 insertions(+), 106 deletions(-) create mode 100644 .github/scripts/classifier_battery_floor.py create mode 100644 node/src/utils/env-guard.ts create mode 100644 node/tests/env-guard.test.ts create mode 100644 node/tests/exec-join.test.ts create mode 100644 node/tests/rf6pqx-differential.test.ts create mode 100644 node/tests/rf6pqx-differential.ts create mode 100644 node/tests/risk-rules-heredoc.test.ts create mode 100644 node/tests/risk-rules-newline-heredoc.test.ts create mode 100644 node/tests/verify-hook-liveness.test.ts create mode 100644 python/tests/rf6pqx_differential.py create mode 100644 python/tests/test_hook_env_guard.py create mode 100644 python/tests/test_rf6pqx_differential.py create mode 100644 python/tests/test_risk_rules_heredoc.py create mode 100644 python/tests/test_risk_rules_newline_heredoc.py create mode 100644 rf-6pqx-newline-heredoc-battery.json diff --git a/.github/scripts/classifier_battery_floor.py b/.github/scripts/classifier_battery_floor.py new file mode 100644 index 00000000..11f2fbaa --- /dev/null +++ b/.github/scripts/classifier_battery_floor.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Drift check for the command-classifier battery. + +The battery and the differential catch different things, and CI needs both: + + * The DIFFERENTIAL (PR vs origin/main) catches a REGRESSION — this branch + weaker than main. It is what a hand-written battery misses, because a + battery only asks the questions someone thought to ask. + * The BATTERY catches a MISSING fix and an OVER-BLOCK. It is what the + differential misses, because a fix absent on BOTH sides is not a permissive + *move* and shows up as nothing: the differential ran CLEAN against a branch + that had lost a live P0 fix entirely. + +That makes the battery's CONTENTS load-bearing, and a load-bearing list nobody +can see shrink is a list that will shrink. This is sable-d2x2 rule C applied to +the gate itself rather than to the suite it guards. + +Three assertions: + + * the battery has not shrunk below the floor; + * it still has rows gating the UNDER-block direction (want includes + "critical") — the rows that catch a fix going missing; + * it still has rows gating the OVER-block direction (want is exactly + ["low"]) — without them, blocking everything passes the gate. #230 was an + over-block report, so a battery with no low rows would have been happy to + ship it. + +Lower the floor in the same PR that removes cases, so a shrink is a reviewed +decision rather than an accident. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +BATTERY = os.path.join(HERE, "..", "..", "rf-6pqx-newline-heredoc-battery.json") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--min-cases", type=int, required=True) + ap.add_argument("--min-critical", type=int, default=15) + ap.add_argument("--min-low", type=int, default=10) + ap.add_argument("--battery", default=BATTERY) + args = ap.parse_args() + + try: + cases = json.load(open(args.battery)) + except (OSError, ValueError) as e: + print(f"FAIL: battery unreadable at {args.battery}: {e}") + return 1 + + if not isinstance(cases, list) or not cases: + print( + "FAIL: battery is not a non-empty list — a gate with no cases " + "passes everything, which is worse than no gate at all" + ) + return 1 + + critical = [c for c in cases if "critical" in c.get("want", [])] + low = [c for c in cases if c.get("want") == ["low"]] + + failures = [] + if len(cases) < args.min_cases: + failures.append( + f"battery shrank: {len(cases)} cases, floor is {args.min_cases}. " + "If cases were removed on purpose, lower the floor in the same PR." + ) + if len(critical) < args.min_critical: + failures.append( + f"only {len(critical)} rows gate the under-block direction " + f"(want includes 'critical'), floor is {args.min_critical}. Those " + "are the rows that catch a fix going missing." + ) + if len(low) < args.min_low: + failures.append( + f"only {len(low)} rows gate the over-block direction " + f"(want is exactly ['low']), floor is {args.min_low}. Without " + "those, blocking everything passes the gate." + ) + + # A malformed row is a row that cannot fail. Catch it here rather than + # letting a harness quietly skip it. + for i, c in enumerate(cases): + if not isinstance(c.get("cmd"), str) or not c["cmd"]: + failures.append(f"case {i} ({c.get('label', '?')!r}) has no command") + want = c.get("want") + if not isinstance(want, list) or not want: + failures.append(f"case {i} ({c.get('label', '?')!r}) has no expectations") + + if failures: + print("FAIL: command-classifier battery drift") + for f in failures: + print(f" - {f}") + return 1 + + print( + f"PASS: battery has {len(cases)} cases " + f"({len(critical)} gate under-blocking, {len(low)} gate over-blocking)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-comprehensive.yml b/.github/workflows/test-comprehensive.yml index aa30ddac..d9096f43 100644 --- a/.github/workflows/test-comprehensive.yml +++ b/.github/workflows/test-comprehensive.yml @@ -120,6 +120,15 @@ jobs: run: | pip install -e ".[dev]" 2>/dev/null || pip install -e . + # The rf-6pqx differential compares this branch's classifier against + # main's, and it FAILS rather than skips when the baseline is missing — + # correctly, since a differential that silently skips is a vacuous gate. + # actions/checkout fetches only the PR ref, so `origin/main` is not in the + # clone and the gate cannot run at all. A depth-1 fetch is enough: the + # gate reads one blob, not the history. + - name: Fetch main for the differential gate + run: git fetch --depth=1 origin main:refs/remotes/origin/main + - name: Build run: pnpm run build @@ -146,6 +155,14 @@ jobs: - name: Assert the suite actually ran run: python3 ../.github/scripts/test_floor.py --vitest vitest-report.json --min-executed 1990 --label "test-node (vitest)" + # The battery is the gate that catches a MISSING fix; the differential + # cannot, because a fix absent on both sides is not a permissive move. + # That makes the battery's CONTENTS load-bearing, so they get a floor of + # their own — in both directions, since a fix bought with an over-block + # is how #230 happened. + - name: Assert the classifier battery still gates both directions + run: python3 ../.github/scripts/classifier_battery_floor.py --min-cases 44 + test-python: needs: gate if: needs.gate.outputs.run_core == 'true' @@ -165,6 +182,12 @@ jobs: pip install -e ".[dev]" 2>/dev/null || pip install -e . pip install pytest pytest-mock pytest-asyncio + # Same reason as test-node: the differential FAILS rather than skips + # when main's classifier is unobtainable, and actions/checkout fetches + # only the PR ref. One blob is all the gate needs. + - name: Fetch main for the differential gate + run: git fetch --depth=1 origin main:refs/remotes/origin/main + - name: Run all tests # xunit1 is the JUnit family that records `file` per test case, which # the floor check below groups by. diff --git a/.gitignore b/.gitignore index c0769cf7..edb07dea 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,12 @@ venv/ uv.lock # Node +# Both forms on purpose: a trailing slash matches DIRECTORIES ONLY, so a stray +# `node_modules` SYMLINK slips past it and `git add -A` commits it. That +# happened (removed in f1132ad); on checkout it leaves a broken self- +# referential link where the install belongs, and pnpm/tsc/vitest all fail +# there — tsc exits 216 with no output, which is an unhelpful way to find out. +node_modules node_modules/ dist/ *.tgz diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfa1a95..ac5501cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`rafter agent exec --force` no longer skips approval, and approval needs a person at a terminal** (rf-ss67, reported in the secbolt audit se-ezvc). `--force ""` ran any HIGH-tier command unprompted: the PreToolUse hook classified the quoted argument as prose, and `exec` then skipped its own prompt. `--force` is now a hidden no-op kept only so old invocations parse; a command that needs approval is prompted only when stdin is an interactive TTY and is otherwise denied, so a piped `yes` is not an approval either. `--dry-run`, which three shipped docs already advertised, now exists: it prints the verdict and runs nothing (exit 0 allowed, 1 blocked, 2 needs approval). The documented `-- ` form is accepted, with the words re-quoted so the classifier evaluates exactly what the shell would run. Both runtimes. + - **A transient 500 during scan polling no longer kills the run** (sable-l10k). An AppSumo customer's GitHub Actions build died on `Rafter scan poll failed: HTTP 500 — Failed to fetch report from storage: Object not found`. A report is not durable the instant a scan flips to `completed`, so a 5xx on that read is survivable — but every poll path treated any non-2xx as fatal, while the transport-error branch three lines above already retried. All three surfaces (the composite action, `rafter run`, `rafter get --interactive`) now retry transient failures with exponential backoff (2s/4s/8s/16s) before giving up, and the give-up message names the scan id, the `rafter get ` retry, and the dashboard instead of leaking storage-layer wording. Full contract in `shared-docs/CLI_SPEC.md`. Both runtimes; end-to-end CI coverage against a mock backend, so no API key or credits are needed to exercise it. - **Python: a failed poll could be written out as if it were scan results** (sable-l10k). The mid-poll loop called `.json()` on the response without checking the status code, so a 500 carrying a JSON error body parsed cleanly, yielded no `status`, fell out of the loop, and was emitted as the scan payload with exit code `0`. A non-JSON error body raised an unhandled `JSONDecodeError`. Both now fail loudly. **Behavior change:** genuine non-transient mid-poll failures that previously exited `0` with an error payload on stdout now exit `1` — check any pipeline that consumed that output. - **GitHub Action: a failed results fetch reported the scan as `completed`** (sable-l10k). The declared `status` output read only from the results step, which does not run when the fetch fails. Consumers gating on `status == 'completed'` saw a clean scan, and the artifact upload published the error body as `rafter-results.json`. Both give-up paths in the results fetch now record `status=unreadable`, and the artifact upload is gated on a successful results fetch. diff --git a/node/package.json b/node/package.json index 6abfb82e..6da04ccc 100644 --- a/node/package.json +++ b/node/package.json @@ -1,6 +1,6 @@ { "name": "@rafter-security/cli", - "version": "0.10.0", + "version": "0.10.1", "type": "module", "repository": { "type": "git", diff --git a/node/resources/skills/rafter/docs/cli-reference.md b/node/resources/skills/rafter/docs/cli-reference.md index 23d33a68..78d72e35 100644 --- a/node/resources/skills/rafter/docs/cli-reference.md +++ b/node/resources/skills/rafter/docs/cli-reference.md @@ -64,7 +64,7 @@ When: before firing multiple remote scans, or when the user asks about limits. Classify and optionally run a shell command through Rafter's risk tiers (critical / high / medium / low). -When: any time a destructive-looking command is about to be executed by an agent. Use `--dry-run` to classify without running. +When: any time a destructive-looking command is about to be executed by an agent. Use `--dry-run` to classify without running: exit 0 = allowed, 1 = blocked, 2 = needs a person's approval. Without `--dry-run`, a command that needs approval is only ever approved by a person at an interactive terminal; from an agent's shell it is denied. Example: `rafter agent exec --dry-run -- rm -rf $WORK_DIR` diff --git a/node/resources/skills/rafter/docs/guardrails.md b/node/resources/skills/rafter/docs/guardrails.md index 9a604e67..d1e238c4 100644 --- a/node/resources/skills/rafter/docs/guardrails.md +++ b/node/resources/skills/rafter/docs/guardrails.md @@ -10,7 +10,7 @@ Rafter exposes two hook handlers over stdio: - `rafter hook posttool` — read a JSON event after a tool ran; log to audit trail, optionally rescan written files for secrets. For platforms without hooks, the same classifier is reachable as: -- `rafter agent exec --dry-run -- ` (returns risk, exits 0/1) +- `rafter agent exec --dry-run -- ` (prints the risk tier and runs nothing; exits 0 allowed, 1 blocked, 2 needs a person's approval) - `rafter mcp serve` → MCP tool `evaluate_command` ## Risk Tiers @@ -66,7 +66,7 @@ If the block is a false positive **for this specific context**, the right path i allow: - "^terraform destroy -target=module\\.sandbox" ``` -2. Or run once with an explicit ack flag: `rafter agent exec --force -- ` (logged to audit trail; still shows up in `rafter agent audit` history). +2. Or have a person run it: `rafter agent exec -- ` prompts for approval only at an interactive terminal and logs the override to the audit trail. There is no acknowledgement flag — `--force` no longer skips the prompt and a piped `yes` is not an approval — because any flag or input an agent can supply is not a person's decision. 3. Never disable the hook globally to get past one command — that silently drops protection for every future call. ## Audit Trail diff --git a/node/src/commands/agent/exec.ts b/node/src/commands/agent/exec.ts index 3a9441d9..bd9947e4 100644 --- a/node/src/commands/agent/exec.ts +++ b/node/src/commands/agent/exec.ts @@ -1,4 +1,4 @@ -import { Command } from "commander"; +import { Command, Option } from "commander"; import { CommandInterceptor } from "../../core/command-interceptor.js"; import { scanAddedDiffLines } from "../../scanners/git-diff-scan.js"; import { parseUnifiedDiffAddedLines } from "../../utils/git-diff.js"; @@ -6,18 +6,57 @@ import { execSync } from "child_process"; import readline from "readline"; import { fmt } from "../../utils/formatter.js"; +// Approval model (rf-ss67): the only party who can approve a command that the +// policy says needs approval is a person at an interactive terminal. There is +// no flag, env var or stdin trick that stands in for that, because every one +// of those can be produced by the agent whose command is being gated: +// * `--force` used to skip the prompt. Combined with the PreToolUse hook +// treating a quoted argument as prose, `rafter agent exec --force ""` +// ran any HIGH-tier command unprompted with the hook blind. The flag is +// kept only so old invocations parse; it changes nothing. +// * A piped "yes" is not a person. Approval is offered only when stdin is a +// TTY; otherwise the command is denied and says why. +// The machine owner widens policy in ~/.rafter/config.json, not per call. + +const DRY_RUN_EXIT = { allowed: 0, blocked: 1, approval: 2 } as const; + export function createExecCommand(): Command { return new Command("exec") .description("Execute command with security validation") - .argument("", "Command to execute") + .argument("", "Command to execute (quote it, or pass it after --)") .option("--skip-scan", "Skip pre-execution file scanning") - .option("--force", "Skip approval prompts (use with caution)") - .action(async (command, opts) => { + .option( + "--dry-run", + "Classify the command and exit without running it (exit 0 allowed, 1 blocked, 2 needs approval)", + ) + .addOption( + new Option("--force", "Deprecated: no longer skips approval (rf-ss67)").hideHelp(), + ) + .action(async (parts: string[], opts) => { + const command = joinCommandParts(parts); const interceptor = new CommandInterceptor(); // Step 1: Evaluate command const evaluation = interceptor.evaluate(command); + // Step 1b: --dry-run reports the classification and stops. Nothing runs, + // nothing is scanned, nothing is logged as executed. + if (opts.dryRun) { + const blocked = !evaluation.allowed && !evaluation.requiresApproval; + const verdict = blocked ? "BLOCKED" : evaluation.requiresApproval ? "REQUIRES APPROVAL" : "ALLOWED"; + console.log(`Dry run: ${verdict}`); + console.log(`Risk Level: ${evaluation.riskLevel.toUpperCase()}`); + console.log(`Requires approval: ${evaluation.requiresApproval ? "yes" : "no"}`); + if (evaluation.reason) { + console.log(`Reason: ${evaluation.reason}`); + } + console.log(`Command: ${command}`); + console.log("Not executed (--dry-run)."); + process.exit( + blocked ? DRY_RUN_EXIT.blocked : evaluation.requiresApproval ? DRY_RUN_EXIT.approval : DRY_RUN_EXIT.allowed, + ); + } + // Step 2: Handle blocked commands if (!evaluation.allowed && !evaluation.requiresApproval) { console.error(`\n${fmt.error("Command BLOCKED")}\n`); @@ -42,8 +81,13 @@ export function createExecCommand(): Command { } } - // Step 4: Handle approval required - if (evaluation.requiresApproval && !opts.force) { + // Step 4: Handle approval required — only a person at a terminal can. + if (evaluation.requiresApproval) { + if (opts.force) { + console.log( + `\n${fmt.warning("--force no longer skips approval (rf-ss67); approval needs a person at an interactive terminal")}\n`, + ); + } console.log(`\n${fmt.warning("Command requires approval")}\n`); console.log(`Risk Level: ${evaluation.riskLevel.toUpperCase()}`); console.log(`Command: ${command}`); @@ -52,6 +96,16 @@ export function createExecCommand(): Command { } console.log(); + if (!process.stdin.isTTY) { + console.log(`${fmt.error("Command denied: approval needs an interactive terminal, and stdin is not one")}`); + console.log( + "Run the command yourself at a terminal, or have the machine owner adjust " + + "commandPolicy in ~/.rafter/config.json.\n", + ); + interceptor.logEvaluation(evaluation, "blocked"); + process.exit(1); + } + const approved = await promptApproval(); if (!approved) { @@ -62,16 +116,13 @@ export function createExecCommand(): Command { console.log(`\n${fmt.success("Command approved by user")}\n`); interceptor.logEvaluation(evaluation, "overridden"); - } else if (opts.force && evaluation.requiresApproval) { - console.log(`\n${fmt.warning("Forcing execution (--force flag)")}\n`); - interceptor.logEvaluation(evaluation, "overridden"); } else { interceptor.logEvaluation(evaluation, "allowed"); } // Step 5: Execute command try { - const output = execSync(command, { + execSync(command, { stdio: "inherit", encoding: "utf-8" }); @@ -85,6 +136,22 @@ export function createExecCommand(): Command { }); } +/** + * One quoted argument is the command verbatim. Several (the `-- rm -rf x` form + * the docs show) are re-joined with shell quoting, so what the classifier sees + * is what the shell will run — `-- echo "a b"` becomes `echo 'a b'`, not `echo a b`. + */ +export function joinCommandParts(parts: string[]): string { + if (parts.length === 1) return parts[0]; + return parts.map(shellQuote).join(" "); +} + +function shellQuote(token: string): string { + if (token === "") return "''"; + if (/^[A-Za-z0-9_\/:=@.,+%-]+$/.test(token)) return token; + return `'${token.replace(/'/g, `'\\''`)}'`; +} + function isGitCommand(command: string): boolean { return command.trim().startsWith("git commit") || command.trim().startsWith("git push"); diff --git a/node/src/commands/agent/init.ts b/node/src/commands/agent/init.ts index eadb97dc..d19e3414 100644 --- a/node/src/commands/agent/init.ts +++ b/node/src/commands/agent/init.ts @@ -7,7 +7,7 @@ import { SkillManager } from "../../utils/skill-manager.js"; import fs from "fs"; import path from "path"; import os from "os"; -import { execSync } from "child_process"; +import { execSync, spawnSync } from "child_process"; import { fileURLToPath } from "url"; import { createRequire } from "module"; import { askYesNo } from "../../utils/prompt.js"; @@ -18,6 +18,65 @@ import yaml from "js-yaml"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +/** + * Resolve an ABSOLUTE, self-contained hook command (rf-er8a). Agents run a hook + * through `sh -c ""` in a minimal environment that need not have + * `rafter` — or even `node` — on PATH; a bare `rafter hook pretool` then exits + * 127, which those tools treat as a silent allow, so the gate is inert while + * settings.json says it is installed. Pinning BOTH the node interpreter and the + * CLI entrypoint makes the command resolvable regardless of PATH. (dist/index.js + * carries `#!/usr/bin/env node`, so an absolute entrypoint ALONE still fails + * where node is off PATH — the interpreter must be pinned too.) + */ +function hookEntrypoint(): string { + let entrypoint = process.argv[1] || ""; + try { entrypoint = fs.realpathSync(entrypoint); } catch { /* keep as-is */ } + return entrypoint; +} + +export function absoluteHookCommand(args: string): string { + return `${process.execPath} ${hookEntrypoint()} hook ${args}`; +} + +/** + * After writing the hooks, confirm the exact command we wrote actually runs and + * enforces — and refuse to report a clean success if it does not (rf-er8a / + * rf-fuwy). Returns human-readable warnings; empty means the gate is live. + */ +function installedHookWarnings(): string[] { + const warnings: string[] = []; + const entrypoint = hookEntrypoint(); + if (entrypoint.includes("/_npx/") || entrypoint.includes("\\_npx\\")) { + warnings.push( + `The rafter entrypoint is inside an npx cache (${entrypoint}), which is ephemeral: the ` + + `installed hook will stop resolving when the cache is cleared. Install globally ` + + `(npm install -g @rafter-security/cli) and re-run 'rafter agent init'.`, + ); + } + try { + const cmd = absoluteHookCommand("pretool"); + const res = spawnSync("sh", ["-c", cmd], { + input: JSON.stringify({ + hook_event_name: "PreToolUse", + tool_name: "Bash", + tool_input: { command: "rm -rf / --no-preserve-root" }, + permission_mode: "default", + cwd: process.cwd(), + }), + encoding: "utf-8", + timeout: 10_000, + }); + let decision: string | null = null; + try { decision = JSON.parse(res.stdout || "")?.hookSpecificOutput?.permissionDecision ?? null; } catch { decision = null; } + if (res.error || res.status === 127) { + warnings.push(`The installed PreToolUse hook did not execute (exit ${res.status ?? "spawn error"}) — the gate is inert. Command: ${cmd}`); + } else if (decision !== "deny") { + warnings.push(`The installed PreToolUse hook ran but did not block a synthetic critical command (decision=${decision ?? "none"}).`); + } + } catch { /* best-effort confirmation */ } + return warnings; +} + /** * Skills installed by `rafter agent init` for Claude Code / Codex. * @@ -297,20 +356,20 @@ function installClaudeCodeHooks(root: string): void { if (!settings.hooks.PreToolUse) settings.hooks.PreToolUse = []; if (!settings.hooks.PostToolUse) settings.hooks.PostToolUse = []; - const preHook = { type: "command", command: "rafter hook pretool" }; - const postHook = { type: "command", command: "rafter hook posttool" }; + const preHook = { type: "command", command: absoluteHookCommand("pretool") }; + const postHook = { type: "command", command: absoluteHookCommand("posttool") }; // Remove any existing Rafter hooks to avoid duplicates settings.hooks.PreToolUse = settings.hooks.PreToolUse.filter( (entry: any) => { const hooks = entry.hooks || []; - return !hooks.some((h: any) => h.command === "rafter hook pretool"); + return !hooks.some((h: any) => String(h.command ?? "").includes("hook pretool")); } ); settings.hooks.PostToolUse = settings.hooks.PostToolUse.filter( (entry: any) => { const hooks = entry.hooks || []; - return !hooks.some((h: any) => h.command === "rafter hook posttool"); + return !hooks.some((h: any) => String(h.command ?? "").includes("hook posttool")); } ); // Strip legacy SessionStart entry left over from <=0.7.4 installs. @@ -318,7 +377,7 @@ function installClaudeCodeHooks(root: string): void { settings.hooks.SessionStart = settings.hooks.SessionStart.filter( (entry: any) => { const hooks = entry.hooks || []; - return !hooks.some((h: any) => h.command === "rafter hook session-start"); + return !hooks.some((h: any) => String(h.command ?? "").includes("hook session-start")); } ); if (settings.hooks.SessionStart.length === 0) delete settings.hooks.SessionStart; @@ -364,15 +423,15 @@ function installCodexHooks(root: string): void { if (!config.hooks.PostToolUse) config.hooks.PostToolUse = []; // Codex uses the same hookSpecificOutput protocol as Claude Code (format=claude) - const preHook = { type: "command", command: "rafter hook pretool" }; - const postHook = { type: "command", command: "rafter hook posttool" }; + const preHook = { type: "command", command: absoluteHookCommand("pretool") }; + const postHook = { type: "command", command: absoluteHookCommand("posttool") }; // Remove existing rafter hooks config.hooks.PreToolUse = config.hooks.PreToolUse.filter( - (entry: any) => !(entry.hooks || []).some((h: any) => h.command?.startsWith("rafter hook pretool")) + (entry: any) => !(entry.hooks || []).some((h: any) => String(h.command ?? "").includes("hook pretool")) ); config.hooks.PostToolUse = config.hooks.PostToolUse.filter( - (entry: any) => !(entry.hooks || []).some((h: any) => h.command?.startsWith("rafter hook posttool")) + (entry: any) => !(entry.hooks || []).some((h: any) => String(h.command ?? "").includes("hook posttool")) ); // PreToolUse intercepts the tools Codex documents support for: Bash and @@ -428,15 +487,15 @@ function installCursorHooks(root: string): void { if (!config.hooks) config.hooks = {}; const events: { event: string; command: string }[] = [ - { event: "preToolUse", command: "rafter hook pretool --format cursor" }, - { event: "postToolUse", command: "rafter hook posttool --format cursor" }, - { event: "beforeShellExecution", command: "rafter hook pretool --format cursor" }, + { event: "preToolUse", command: absoluteHookCommand("pretool --format cursor") }, + { event: "postToolUse", command: absoluteHookCommand("posttool --format cursor") }, + { event: "beforeShellExecution", command: absoluteHookCommand("pretool --format cursor") }, ]; for (const { event, command } of events) { if (!Array.isArray(config.hooks[event])) config.hooks[event] = []; config.hooks[event] = config.hooks[event].filter( - (entry: any) => !entry?.command?.includes("rafter hook"), + (entry: any) => !(String(entry?.command ?? "").includes("hook pretool") || String(entry?.command ?? "").includes("hook posttool")), ); config.hooks[event].push({ command, type: "command", timeout: 5000 }); } @@ -572,10 +631,10 @@ function installGeminiHooks(root: string): void { // Remove existing rafter hooks settings.hooks.BeforeTool = settings.hooks.BeforeTool.filter( - (entry: any) => !(entry.hooks || []).some((h: any) => h.command?.includes("rafter hook pretool")) + (entry: any) => !(entry.hooks || []).some((h: any) => String(h.command ?? "").includes("hook pretool")) ); settings.hooks.AfterTool = settings.hooks.AfterTool.filter( - (entry: any) => !(entry.hooks || []).some((h: any) => h.command?.includes("rafter hook posttool")) + (entry: any) => !(entry.hooks || []).some((h: any) => String(h.command ?? "").includes("hook posttool")) ); // Gemini matchers are regexes against built-in tool names per @@ -584,11 +643,11 @@ function installGeminiHooks(root: string): void { // verification 2026-05-03 — schema confirmed against current Gemini docs.) settings.hooks.BeforeTool.push({ matcher: "run_shell_command|write_file|replace|edit", - hooks: [{ type: "command", command: "rafter hook pretool --format gemini", timeout: 5000 }], + hooks: [{ type: "command", command: absoluteHookCommand("pretool --format gemini"), timeout: 5000 }], }); settings.hooks.AfterTool.push({ matcher: ".*", - hooks: [{ type: "command", command: "rafter hook posttool --format gemini", timeout: 5000 }], + hooks: [{ type: "command", command: absoluteHookCommand("posttool --format gemini"), timeout: 5000 }], }); fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2), "utf-8"); @@ -1619,7 +1678,16 @@ export function createInitCommand(): Command { }, root, scope); console.log(); - console.log(fmt.success("Agent security initialized!")); + // rf-er8a: do not report a clean success if the hook we just wrote cannot + // execute and enforce. A silent 127 (or an npx-cache entrypoint) means the + // gate is inert even though settings.json looks configured. + const hookWarnings = claudeCodeOk ? installedHookWarnings() : []; + if (hookWarnings.length > 0) { + for (const w of hookWarnings) console.log(fmt.warning(w)); + console.log(fmt.warning("Agent security initialized WITH WARNINGS — the command gate may NOT be active (see above). Run 'rafter agent verify' to confirm.")); + } else { + console.log(fmt.success("Agent security initialized!")); + } console.log(); const anyIntegration = openclawOk || claudeCodeOk || codexOk || geminiOk || cursorOk || windsurfOk || continueOk || aiderOk || hermesOk || openCodeOk; diff --git a/node/src/commands/agent/verify.ts b/node/src/commands/agent/verify.ts index 10fee20f..31b3b7b1 100644 --- a/node/src/commands/agent/verify.ts +++ b/node/src/commands/agent/verify.ts @@ -2,6 +2,7 @@ import { Command } from "commander"; import { ConfigManager } from "../../core/config-manager.js"; import { BinaryManager } from "../../utils/binary-manager.js"; import { SkillManager } from "../../utils/skill-manager.js"; +import { resolveHookControl } from "../../core/hook-control.js"; import { spawnSync } from "child_process"; import fs from "fs"; import path from "path"; @@ -9,6 +10,58 @@ import os from "os"; import yaml from "js-yaml"; import { fmt } from "../../utils/formatter.js"; +/** + * Run a configured PreToolUse hook command EXACTLY as Claude Code would — through + * `sh -c ""`, with a synthetic payload on + * stdin — and return its exit status and the permissionDecision it emitted. + * + * This is the whole point of rf-fuwy: `agent verify` used to confirm only that + * the hook was CONFIGURED (a substring match) and reported a completely inert + * gate as healthy, byte-identical to a working one. A command that does not + * resolve exits 127; Claude Code blocks only on exit 2, so 127 is a silent + * allow. Executing the command is the only check that cannot be fooled by that. + */ +export function runConfiguredHook( + command: string, + toolCommand: string, +): { status: number | null; decision: string | null; error?: string; stdout: string } { + const payload = JSON.stringify({ + session_id: `rafter-verify-${process.pid}-${Date.now()}`, + transcript_path: "", + cwd: process.cwd(), + permission_mode: "default", + hook_event_name: "PreToolUse", + tool_name: "Bash", + tool_input: { command: toolCommand }, + }); + // `sh -c` reproduces exactly how Claude Code invokes a shell-form hook, so a + // command that resolves in this terminal but not in the editor (or vice versa) + // is exercised the same way the editor would exercise it. Retry a transient + // spawn failure (e.g. EAGAIN under process pressure) so a resource hiccup does + // not get reported as an inert gate — a false alarm is worse than a slow check. + let result = spawnSync("sh", ["-c", command], { input: payload, encoding: "utf-8", timeout: 10_000 }); + // Retry a spawn-level error OR a signal kill: under the same process pressure + // that produces EAGAIN, the OOM killer sends SIGKILL, which sets `signal` (not + // `error`) with a null status — the identical false-alarm scenario by another + // route (measured by achebe). A persistent failure still returns decision:null, + // so a genuinely inert hook is still reported inert — no fail-open. + for (let attempt = 0; attempt < 2 && (result.error || result.signal); attempt++) { + result = spawnSync("sh", ["-c", command], { input: payload, encoding: "utf-8", timeout: 10_000 }); + } + if (result.error) { + return { status: null, decision: null, error: result.error.message, stdout: "" }; + } + const stdout = result.stdout ?? ""; + let decision: string | null = null; + try { + const parsed = JSON.parse(stdout); + decision = parsed?.hookSpecificOutput?.permissionDecision ?? null; + } catch { + decision = null; + } + return { status: result.status, decision, stdout }; +} + interface CheckResult { name: string; passed: boolean; @@ -86,13 +139,67 @@ function checkClaudeCode(): CheckResult { // Substring match — Python install writes an absolute path // (/home/foo/bin/rafter hook pretool), Node writes the bare command. const hooks = settings?.hooks?.PreToolUse || []; - const hasRafterHook = hooks.some((entry: any) => - (entry.hooks || []).some((h: any) => String(h?.command ?? "").includes("rafter hook pretool")) - ); - if (!hasRafterHook) { + const commands: string[] = []; + for (const entry of hooks) { + for (const h of entry.hooks || []) { + const cmd = String(h?.command ?? ""); + if (cmd.includes("hook pretool")) commands.push(cmd); + } + } + if (commands.length === 0) { return { name, passed: false, optional: true, detail: "Rafter hooks not installed — run 'rafter agent init --with-claude-code'" }; } - return { name, passed: true, detail: "Hooks installed" }; + + // A configured hook that has been deliberately switched off is a valid + // state, not a failure: expecting a "deny" from a disabled hook would fail a + // machine whose owner turned it off on purpose (rf-fuwy amendment). + const control = resolveHookControl(); + if (!control.commandPolicyEnabled) { + return { + name, + passed: false, + optional: true, + detail: `Hook installed but command interception is DISABLED (${control.source.commandPolicy}) — no command is blocked. Re-enable to enforce.`, + }; + } + + // rf-fuwy: EXECUTE the configured command, do not just read it. A gate that + // cannot run (exit 127) is inert but was reported "installed"; a gate that + // runs but does not block a CRITICAL command is misconfigured. Assert both + // directions with synthetic payloads. + const hookCmd = commands[0]; + const danger = runConfiguredHook(hookCmd, "rm -rf / --no-preserve-root"); + if (danger.error || danger.status === 127) { + return { + name, + passed: false, + detail: + `Hook is CONFIGURED but NOT EXECUTABLE (${danger.error ? danger.error : "exit 127"}): "${hookCmd}" does not resolve, so the gate is INERT ` + + `(Claude Code blocks only on exit 2; anything else is allowed). Reinstall with 'rafter agent init --with-claude-code', which writes an absolute path.`, + }; + } + if (danger.decision !== "deny") { + return { + name, + passed: false, + detail: + `Hook executes (exit ${danger.status}) but did NOT block a synthetic CRITICAL command (decision=${danger.decision ?? "none/unparseable"}). ` + + `The gate is not enforcing.`, + }; + } + const benign = runConfiguredHook(hookCmd, "echo hello"); + if (benign.decision === "deny") { + return { + name, + passed: false, + detail: `Hook blocks even a benign command (echo) — over-blocking; check policy/config.`, + }; + } + // Note the caveat honestly: verify runs in the terminal's environment, which + // may differ from the editor's — a pass proves the command runs and enforces + // HERE, and a failure proves it is broken; it is not proof the editor's PATH + // resolves it too. + return { name, passed: true, detail: `Hooks installed and enforcing (blocked a synthetic 'rm -rf /'; verify runs in this shell's env)` }; } catch (e) { return { name, passed: false, optional: true, detail: `Cannot read settings: ${e}` }; } @@ -401,9 +508,24 @@ function probeClaudeCode(): CheckResult { const auditPath = path.join(home, ".rafter", "audit.jsonl"); const sizeBefore = fs.existsSync(auditPath) ? fs.statSync(auditPath).size : 0; - // Resolve the rafter binary the same way Claude Code would: `rafter hook - // pretool` on PATH. Fall back to argv[0] if PATH lookup fails. - const result = spawnSync(process.execPath, [process.argv[1], "hook", "pretool"], { + // Run the command EXACTLY as configured in settings.json, through `sh -c`, the + // way Claude Code would. (Earlier this spawned verify's own node + argv[1], + // which always resolves and therefore passed even when the CONFIGURED command + // did not exist — the rf-fuwy defect. Reading and running the real command is + // the only faithful probe.) + let configuredCmd = "rafter hook pretool"; + try { + const settings = JSON.parse(fs.readFileSync(settingsPath, "utf-8")); + for (const entry of settings?.hooks?.PreToolUse || []) { + for (const h of entry.hooks || []) { + const cmd = String(h?.command ?? ""); + if (cmd.includes("hook pretool")) { configuredCmd = cmd; break; } + } + } + } catch { + // fall back to the bare command below + } + const result = spawnSync("sh", ["-c", configuredCmd], { input: stdinPayload, encoding: "utf-8", timeout: 10_000, diff --git a/node/src/core/risk-rules.ts b/node/src/core/risk-rules.ts index 491027dd..0d8c947e 100644 --- a/node/src/core/risk-rules.ts +++ b/node/src/core/risk-rules.ts @@ -162,7 +162,9 @@ interface Piece { } function isOpChar(c: string): boolean { - return c === ";" || c === "&" || c === "|" || c === ">" || c === "<"; + // \n and \r are statement separators (rf-6pqx): a newline ends a command + // exactly as ";" does, so a payload on a later line is classified on its own. + return c === ";" || c === "&" || c === "|" || c === ">" || c === "<" || c === "\n" || c === "\r"; } /** Read a `$(…)` substitution starting at `i`; returns its contents and the next index. */ @@ -191,18 +193,32 @@ function readBacktick(s: string, i: number): { inner: string; next: number } { return { inner, next: Math.min(j + 1, s.length) }; } -/** Split a command line into words and operators, respecting quotes and substitutions. */ -function tokenize(s: string): Piece[] { +/** + * Split a command line into words and operators, respecting quotes and substitutions. + * `unterminated` is set when a quote is never closed — the parse is then unreliable + * and the caller must FAIL CLOSED (match the raw string) rather than trust a + * desynchronized sanitization (se-y6vo: `$'a\'b'` swallows a trailing payload). + */ +function tokenize(s: string): { pieces: Piece[]; unterminated: boolean } { const pieces: Piece[] = []; + let unterminated = false; let i = 0; while (i < s.length) { const c = s[i]; - if (/\s/.test(c)) { i++; continue; } + // Whitespace EXCEPT newlines is skipped; a newline falls through to the + // operator branch below so it becomes a statement separator (rf-6pqx). + if (/\s/.test(c) && c !== "\n" && c !== "\r") { i++; continue; } if (isOpChar(c)) { const start = i; + if (c === "\n" || c === "\r") { + // Normalize a line break (incl. CRLF) to a ";" separator piece. + i += 1; + pieces.push({ start, end: i, op: ";", text: ";", quoted: false, substs: [] }); + continue; + } const two = s.slice(i, i + 2); const op = (two === "&&" || two === "||" || two === ">>" || two === "<<") ? two : c; i += op.length; @@ -220,6 +236,12 @@ function tokenize(s: string): Piece[] { if (/\s/.test(ch) || isOpChar(ch)) break; if (ch === "\\") { + // Line continuation: `\` immediately before a newline (incl. CRLF) is + // deleted by the shell — `r\m` is `rm`, so the newline must not be + // absorbed into the word (rf-6pqx/se-y6vo). + const nxt = s[i + 1] ?? ""; + if (nxt === "\n") { i += 2; continue; } + if (nxt === "\r") { i += 2; if (s[i] === "\n") i++; continue; } i++; if (i < s.length) { text += s[i]; i++; } continue; @@ -229,8 +251,12 @@ function tokenize(s: string): Piece[] { if (ch === "'") { i++; quoted = true; - while (i < s.length && s[i] !== "'") { text += s[i]; i++; } - i++; + let closed = false; + while (i < s.length) { + if (s[i] === "'") { closed = true; i++; break; } + text += s[i]; i++; + } + if (!closed) unterminated = true; continue; } @@ -238,8 +264,13 @@ function tokenize(s: string): Piece[] { if (ch === '"') { i++; quoted = true; - while (i < s.length && s[i] !== '"') { + let closed = false; + while (i < s.length) { + if (s[i] === '"') { closed = true; i++; break; } if (s[i] === "\\") { + const nxt = s[i + 1] ?? ""; + if (nxt === "\n") { i += 2; continue; } + if (nxt === "\r") { i += 2; if (s[i] === "\n") i++; continue; } i++; if (i < s.length) { text += s[i]; i++; } continue; @@ -253,7 +284,7 @@ function tokenize(s: string): Piece[] { text += s[i]; i++; } - i++; + if (!closed) unterminated = true; continue; } @@ -271,7 +302,7 @@ function tokenize(s: string): Piece[] { pieces.push({ start, end: i, op: null, text, quoted, substs }); } - return pieces; + return { pieces, unterminated }; } /** `/usr/bin/rm` → `rm`; used to classify the executable of a segment. */ @@ -283,6 +314,16 @@ function execName(text: string): string { const ENV_ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/; const SHELL_C_FLAG = /^-[a-z]*c$/; const LONG_FLAG_WITH_VALUE = /^(--[a-z][a-z-]*)=/; +const NUMERIC_ARG = /^\d+[a-z]*$/i; + +/** + * A heredoc introducer: `<<`, optional `-`/`~` (indented-terminator forms), an + * optional quote around the delimiter, and the delimiter word. Group 1 is the + * dash/tilde, group 3 is the delimiter name. `g` so we can find all on a line. + */ +const HEREDOC_START = /(? `rm`), used to ask + * what the NEXT stage of a pipeline does with this stage's output. + */ +function segmentExec(pieces: Piece[]): string { + const isRedirectTarget = new Array(pieces.length).fill(false); + for (let i = 1; i < pieces.length; i++) { + const prev = pieces[i - 1]; + if (prev.op && REDIRECT_OPS.has(prev.op) && !pieces[i].op) isRedirectTarget[i] = true; + } + for (let i = 0; i < pieces.length; i++) { + const p = pieces[i]; + if (p.op || isRedirectTarget[i]) continue; + if (!p.quoted && ENV_ASSIGNMENT.test(p.text)) continue; + if (!p.quoted && TAIL_WRAPPERS.has(execName(p.text))) { + let j = i + 1; + while (j < pieces.length) { + const q = pieces[j]; + if (q.op || isRedirectTarget[j]) { j++; continue; } + if (q.text.startsWith("-") || /^\d+[a-z]*$/i.test(q.text)) { j++; continue; } + break; + } + i = j - 1; + continue; + } + return execName(p.text); + } + return ""; +} + +function processSegment( + pieces: Piece[], + depth: number, + out: Replacement[], + pipedIntoShell = false, + outputExecuted = false +): void { // A word is a redirect target when the piece before it is `>`/`>>`/`<`. const isRedirectTarget = new Array(pieces.length).fill(false); for (let i = 1; i < pieces.length; i++) { @@ -337,6 +414,15 @@ function processSegment(pieces: Piece[], depth: number, out: Replacement[]): voi } const codeCarrying = hasShellExec || hasEvalFlag || EVAL_EXECS.has(exec); + // sable-c6an. Two questions the code conflated, and conflating them gets one + // of them wrong: + // codeCarrying — this segment RUNS a command string it was handed + // executesOutput — this segment's STDOUT becomes code somewhere else + // (`… | bash`, or a substitution used as a -c script) + // `bash -c "echo 'rm -rf /'"` is the first and not the second, so its operand + // stays data; `bash -c "$(echo rm -rf /)"` is the second, so it is code. + const executesOutput = pipedIntoShell || outputExecuted; + let seenShell = false; let pendingScript = false; let prevTextFlag = false; @@ -352,7 +438,15 @@ function processSegment(pieces: Piece[], depth: number, out: Replacement[]): voi // `bash -c