diff --git a/orchestration/skills/quad-cli-consensus-gate.md b/orchestration/skills/quad-cli-consensus-gate.md index 8faf173..bbbd607 100644 --- a/orchestration/skills/quad-cli-consensus-gate.md +++ b/orchestration/skills/quad-cli-consensus-gate.md @@ -153,6 +153,7 @@ If any cap is exceeded, the gate is **skipped with a warning on stderr** and exi | `--max-prompt-bytes ` | Skip gate above this prompt byte size; default `262144` | | `--advisory-only` | Never fail the gate outright for below-minimum reviewers; still exits `3` on total (zero-reviewer) outage unless `--allow-zero-reviewers` is also set | | `--min-reviewers ` | Minimum valid reviewers required to trust consensus; default `2`. Below this, the report is marked `status: "advisory-degraded"` | +| `--artifact ` | Write the final validated report object to the given JSON file as well as stdout; omitted by default | | `--allow-zero-reviewers` | Allow a run with **zero** valid reviewers to exit `0` (marked `status: "no-reviewers"`) instead of exit `3`. Without this flag, zero reviewers always exits `3`, even under `--advisory-only` | | `--timeout ` | Override every per-runner timeout. Defaults are Claude `120000`, Codex `240000`, Cursor Agent `240000`, and Antigravity `360000`; `QUAD_CLI_TIMEOUT__MS` or `QUAD_CLI_TIMEOUT_MS` can override them without code changes | | `--gate-timeout ` | Whole parallel gate deadline including one transient retry; default `725000`, also configurable with `QUAD_CLI_GATE_TIMEOUT_MS` | @@ -160,6 +161,13 @@ If any cap is exceeded, the gate is **skipped with a warning on stderr** and exi Mock/test execution is controlled solely by the `QUAD_CLI_MOCK_DIR` environment variable (see Usage Examples) — there is no separate test-mode flag to keep in sync with it. +> `--min-reviewers` defaults to `2` and stays there. Four reviewers running successfully does not +> raise the bar, because reviewer *availability* fails for reasons unrelated to review quality — +> expired OAuth, transient network, a launcher that is not on this machine. Measurement supports +> rejecting `4` as a gate threshold; it does not establish that any specific higher number is better. +> What the contract requires instead is that a run records **which reviewers were declared, which +> were effective, and why the rest dropped out**, so a degraded run is never mistaken for a healthy one. + Timeout cleanup targets the runner's process tree (`taskkill /T /F` on Windows and a detached process group on POSIX). This is best-effort: PID reuse and process-snapshot races mean the orchestrator records what it attempted but does not claim that every descendant was certainly terminated. Antigravity's internal `--print-timeout` stays shorter than its outer timeout (five minutes under the default six-minute outer timeout), so diagnostics can distinguish which layer stopped the run. @@ -201,6 +209,8 @@ The top-level report additionally carries, when relevant: - `status: "advisory-degraded" | "no-reviewers"` — present only when reviewer count is below `--min-reviewers` or zero - `reviewers_effective: N` — the number of CLIs that returned schema-valid output this run +- `reviewers` — the declared and effective tool names plus structured stage/cause diagnostics for every dropped reviewer +- `environment` — the orchestrator commit, OS/architecture, Node version, and per-tool CLI versions Normalized matching uses repo-relative paths with forward slashes. Paths are **not blindly lowercased** by default — `rule_id` normalization **is** lowercased (via `schemas/rule-aliases.json` alias resolution), since rule identifiers are conventionally case-insensitive across tools while file paths are not. @@ -215,6 +225,17 @@ Normalized matching uses repo-relative paths with forward slashes. Paths are **n With `--advisory-only`, a nonzero-but-below-minimum reviewer count exits `0` instead of `3` (with `status: "advisory-degraded"` in the report); a **zero**-reviewer outage still exits `3` unless `--allow-zero-reviewers` is also set. +## Live Smoke Preflight + +The smoke scripts run a fixed small diff through the orchestrator. They are local preflight commands and are intentionally not part of `npm test` or CI. + +| Command | Code | Meaning | +|---------|------|---------| +| `npm run smoke:quad` | `0` | All four declared reviewers returned schema-valid output | +| `npm run smoke:quad` | `1` | At least one declared reviewer was unavailable, unauthenticated, or non-compliant | +| `npm run smoke:quad:available` | `0` | All four declared reviewers were available and returned schema-valid output | +| `npm run smoke:quad:available` | `4` | At least one declared reviewer was missing or invalid; the command prints `status="degraded"` and names each dropped tool | + ## Usage Examples ### Base/Head Range diff --git a/package.json b/package.json index 3ea48cd..d84be49 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,9 @@ "build:plugins": "node scripts/build-plugin-bundles.mjs", "check:plugins": "node scripts/build-plugin-bundles.mjs --check", "lint:skills": "node scripts/vally-gate.mjs", - "lint:plugins": "node scripts/vally-gate.mjs --root plugins/copilot-native-workflows/skills --strict-all" + "lint:plugins": "node scripts/vally-gate.mjs --root plugins/copilot-native-workflows/skills --strict-all", + "smoke:quad": "node scripts/quad-cli-smoke.mjs", + "smoke:quad:available": "node scripts/quad-cli-smoke.mjs --available" }, "devDependencies": { "@microsoft/vally": "0.13.0", diff --git a/schemas/quad-cli-merged-report.json b/schemas/quad-cli-merged-report.json index 5b2ed40..b147170 100644 --- a/schemas/quad-cli-merged-report.json +++ b/schemas/quad-cli-merged-report.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Quad CLI Merged Consensus Report", - "description": "Shape of the FINAL orchestrator output (after merging raw per-CLI reports). This is intentionally a separate schema from quad-cli-report.json, which models a single raw reviewer's input and forbids the extra fields (blocking/contributors/families/effective_votes/status/reviewers_effective) that the merge step adds. Validating orchestrator OUTPUT against the raw input schema will always fail once findings.length > 0 because of quad-cli-report.json's additionalProperties:false.", + "description": "Shape of the FINAL orchestrator output (after merging raw per-CLI reports). This is intentionally a separate schema from quad-cli-report.json, which models a single raw reviewer's input and forbids the extra fields (blocking/contributors/families/effective_votes/status/reviewers_effective/reviewers/environment) that the merge step adds. Validating orchestrator OUTPUT against the raw input schema will always fail once findings.length > 0 because of quad-cli-report.json's additionalProperties:false.", "type": "object", "additionalProperties": false, "required": ["schema_version", "generated_by", "findings"], @@ -19,6 +19,64 @@ "type": "integer", "minimum": 0 }, + "reviewers": { + "type": "object", + "additionalProperties": false, + "required": ["declared", "effective", "dropped"], + "properties": { + "declared": { + "type": "array", + "items": { "type": "string" } + }, + "effective": { + "type": "array", + "items": { "type": "string" } + }, + "dropped": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "tool", + "stage", + "primary_cause", + "observed_failure", + "launcher_kind", + "resolved_path", + "exit_code", + "ms" + ], + "properties": { + "tool": { "type": "string" }, + "stage": { "enum": ["resolve", "spawn", "transport", "response", "schema"] }, + "primary_cause": { "type": "string" }, + "observed_failure": { "type": "string" }, + "launcher_kind": { "type": "string" }, + "resolved_path": { "type": "string" }, + "exit_code": { "type": ["integer", "null"] }, + "ms": { "type": "integer" } + } + } + } + } + }, + "environment": { + "type": "object", + "additionalProperties": false, + "required": ["orchestrator_commit", "os", "node", "cli_versions"], + "properties": { + "orchestrator_commit": { "type": "string" }, + "os": { "type": "string" }, + "node": { "type": "string" }, + "cli_versions": { + "type": "object", + "additionalProperties": { + "type": ["string", "null"] + } + } + } + }, "findings": { "type": "array", "items": { diff --git a/scripts/quad-cli-orchestrate.mjs b/scripts/quad-cli-orchestrate.mjs index a3e1c32..65e7b9e 100644 --- a/scripts/quad-cli-orchestrate.mjs +++ b/scripts/quad-cli-orchestrate.mjs @@ -1,13 +1,18 @@ -import { existsSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; import { + buildSpawnSpec, buildInvocation, formatRunnerDiagnostic, parseJsonReport, + resolveLauncher, resolveGateTimeout, resolveToolTimeout, runRunnerWithRetry, + sanitizeReason, } from "./quad-cli-transport.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -23,6 +28,8 @@ const ALLOWED_MERGED_TOP_LEVEL_KEYS = new Set([ "findings", "status", "reviewers_effective", + "reviewers", + "environment", ]); const ALLOWED_MERGED_FINDING_KEYS = new Set([ "path", @@ -60,8 +67,17 @@ export async function main() { } if (!diffResult.body.trim()) { - const emptyReport = createReport([]); - process.stdout.write(`${JSON.stringify(emptyReport, null, 2)}\n`); + const emptyReport = createReport([], { + reviewers: options.artifact ? { declared: [], effective: [], dropped: [] } : undefined, + environment: options.artifact ? buildEnvironment(options.mockDir) : undefined, + }); + if (!validateMergedReport(emptyReport).ok) { + console.error( + "quad-cli-consensus-gate: internal error — merged report failed schemas/quad-cli-merged-report.json validation." + ); + process.exit(2); + } + emitReport(emptyReport, options.artifact); console.error("quad-cli-consensus-gate: no eligible textual diff content to review."); process.exit(0); } @@ -178,6 +194,8 @@ export async function main() { const finalReport = createReport(mergedFindings, { status: status !== "ok" ? status : undefined, reviewersEffective: validReviewerCount, + reviewers: options.artifact ? buildReviewerRoster(parsedResults, validReports) : undefined, + environment: options.artifact ? buildEnvironment(options.mockDir) : undefined, }); // Defensive self-check: the merged/final report has a distinct shape from the raw @@ -194,7 +212,7 @@ export async function main() { process.exit(2); } - process.stdout.write(`${JSON.stringify(finalReport, null, 2)}\n`); + emitReport(finalReport, options.artifact); console.error( `quad-cli-consensus-gate: ${blockingCount} blocking, ${advisoryCount} advisory, ${validReviewerCount} valid reviewers, ${erroredReviewers} errored reviewers.` ); @@ -239,6 +257,7 @@ function parseArgs(argv) { diffFile: null, minReviewers: 2, allowZeroReviewers: false, + artifact: null, mockDir: process.env.QUAD_CLI_MOCK_DIR ? resolve(process.env.QUAD_CLI_MOCK_DIR) : null, }; @@ -290,6 +309,9 @@ function parseArgs(argv) { case "--allow-zero-reviewers": options.allowZeroReviewers = true; break; + case "--artifact": + options.artifact = resolve(requireValue(argv, ++index, "--artifact")); + break; default: throw new Error(`Unknown argument: ${arg}`); } @@ -727,6 +749,14 @@ function validateMergedReport(report) { return { ok: false }; } + if (Object.hasOwn(report, "reviewers") && !validateReviewerRoster(report.reviewers)) { + return { ok: false }; + } + + if (Object.hasOwn(report, "environment") && !validateEnvironment(report.environment)) { + return { ok: false }; + } + if (!Array.isArray(report.findings)) { return { ok: false }; } @@ -937,9 +967,123 @@ function createReport(findings, meta = {}) { report.reviewers_effective = meta.reviewersEffective; } + if (meta.reviewers) { + report.reviewers = meta.reviewers; + } + + if (meta.environment) { + report.environment = meta.environment; + } + return report; } +function emitReport(report, artifactPath) { + const serializedReport = `${JSON.stringify(report, null, 2)}\n`; + if (artifactPath) writeFileSync(artifactPath, serializedReport, "utf8"); + process.stdout.write(serializedReport); +} + +function deriveFailureStage(result) { + if (result.class === "schema-invalid") return "schema"; + if (result.class === "invalid-response") return "response"; + if (result.launcherKind === "not-found") return "resolve"; + if (result.class === "timeout" || result.timedOut) return "transport"; + if (Number.isInteger(result.exitCode)) return "response"; + return "spawn"; +} + +function redactResolvedPath(value, home = homedir()) { + if (!value) return "-"; + const rawPath = String(value); + const normalizedPath = rawPath.replace(/\\/g, "/"); + const normalizedHome = String(home).replace(/\\/g, "/").replace(/\/$/, ""); + const foldedPath = normalizedPath.toLocaleLowerCase("en-US"); + const foldedHome = normalizedHome.toLocaleLowerCase("en-US"); + if (foldedPath === foldedHome || foldedPath.startsWith(`${foldedHome}/`)) { + return `~${normalizedPath.slice(normalizedHome.length)}`; + } + return isAbsolute(rawPath) ? basename(rawPath) : basename(normalizedPath); +} + +function buildReviewerRoster(parsedResults, validReports) { + const effective = validReports.map(({ tool }) => tool); + const effectiveSet = new Set(effective); + return { + declared: parsedResults.map(({ tool }) => tool), + effective, + dropped: parsedResults.filter(({ tool }) => !effectiveSet.has(tool)).map((result) => ({ + tool: result.tool, + stage: deriveFailureStage(result), + primary_cause: result.class ?? "internal", + observed_failure: sanitizeReason(result.reason || result.stderr || result.error || "-") || "-", + launcher_kind: result.launcherKind ?? "-", + resolved_path: redactResolvedPath(result.resolvedPath), + exit_code: Number.isInteger(result.exitCode) ? result.exitCode : null, + ms: Number.isInteger(result.ms) ? result.ms : 0, + })), + }; +} + +function readOrchestratorCommit() { + const result = spawnSync("git", ["rev-parse", "HEAD"], { cwd: ROOT, encoding: "utf8", windowsHide: true }); + return result.status === 0 && result.stdout?.trim() ? result.stdout.trim() : "unknown"; +} + +function probeCliVersion(tool) { + const launcher = resolveLauncher(tool); + if (launcher.kind === "not-found") return null; + const spec = buildSpawnSpec(launcher, ["--version"]); + const result = spawnSync(spec.command, spec.args, { + cwd: ROOT, + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + windowsVerbatimArguments: spec.windowsVerbatimArguments, + }); + if (result.status !== 0) return null; + return sanitizeReason(result.stdout || result.stderr) || null; +} + +function buildEnvironment(mockDir) { + const cliVersions = Object.fromEntries( + TOOL_ORDER.map((tool) => [tool, mockDir ? (existsSync(join(mockDir, `${tool}.json`)) ? "mock" : null) : probeCliVersion(tool)]) + ); + return { + orchestrator_commit: readOrchestratorCommit(), + os: `${process.platform}/${process.arch}`, + node: process.version, + cli_versions: cliVersions, + }; +} + +function validateReviewerRoster(reviewers) { + if (!reviewers || typeof reviewers !== "object" || Array.isArray(reviewers)) return false; + if (!Object.keys(reviewers).every((key) => ["declared", "effective", "dropped"].includes(key))) return false; + if (!Array.isArray(reviewers.declared) || !reviewers.declared.every((tool) => typeof tool === "string")) return false; + if (!Array.isArray(reviewers.effective) || !reviewers.effective.every((tool) => typeof tool === "string")) return false; + if (!Array.isArray(reviewers.dropped)) return false; + const allowedStages = ["resolve", "spawn", "transport", "response", "schema"]; + const droppedKeys = ["tool", "stage", "primary_cause", "observed_failure", "launcher_kind", "resolved_path", "exit_code", "ms"]; + return reviewers.dropped.every((entry) => + entry && typeof entry === "object" && !Array.isArray(entry) && + Object.keys(entry).length === droppedKeys.length && Object.keys(entry).every((key) => droppedKeys.includes(key)) && + typeof entry.tool === "string" && allowedStages.includes(entry.stage) && + typeof entry.primary_cause === "string" && typeof entry.observed_failure === "string" && + typeof entry.launcher_kind === "string" && typeof entry.resolved_path === "string" && + (entry.exit_code === null || Number.isInteger(entry.exit_code)) && Number.isInteger(entry.ms) + ); +} + +function validateEnvironment(environment) { + if (!environment || typeof environment !== "object" || Array.isArray(environment)) return false; + if (!Object.keys(environment).every((key) => ["orchestrator_commit", "os", "node", "cli_versions"].includes(key))) return false; + return typeof environment.orchestrator_commit === "string" && typeof environment.os === "string" && + typeof environment.node === "string" && environment.cli_versions && typeof environment.cli_versions === "object" && + !Array.isArray(environment.cli_versions) && + Object.values(environment.cli_versions).every((version) => version === null || typeof version === "string"); +} + export { parseArgs, loadRuleAliases, @@ -958,6 +1102,9 @@ export { normalizePath, normalizeRuleId, createReport, + deriveFailureStage, + redactResolvedPath, + buildReviewerRoster, }; const isDirectRun = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url); diff --git a/scripts/quad-cli-smoke.mjs b/scripts/quad-cli-smoke.mjs new file mode 100644 index 0000000..edfc8da --- /dev/null +++ b/scripts/quad-cli-smoke.mjs @@ -0,0 +1,86 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, ".."); +const ORCHESTRATOR = join(__dirname, "quad-cli-orchestrate.mjs"); +const DIFF_FIXTURE = join(ROOT, "tests", "fixtures", "quad-cli", "smoke.diff"); +const DECLARED_TOOLS = ["claude", "codex", "cursor-agent", "agy"]; + +export function runSmoke(argv = process.argv.slice(2), runtime = {}) { + const availableMode = parseArgs(argv); + const tempDir = mkdtempSync(join(tmpdir(), "quad-cli-smoke-")); + const artifactPath = join(tempDir, "report.json"); + + try { + const child = (runtime.spawnSync ?? spawnSync)( + process.execPath, + [ORCHESTRATOR, "--diff-file", DIFF_FIXTURE, "--min-reviewers", "4", "--artifact", artifactPath], + { cwd: ROOT, env: process.env, encoding: "utf8", windowsHide: true } + ); + + if (child.stderr) process.stderr.write(child.stderr); + if (!child.stdout?.trim()) { + console.error(`quad-cli-smoke: orchestrator produced no report (exit ${child.status ?? "unknown"}).`); + return 1; + } + + let report; + try { + report = JSON.parse(child.stdout); + } catch { + console.error("quad-cli-smoke: orchestrator stdout was not a JSON report."); + return 1; + } + + const artifact = JSON.parse(readFileSync(artifactPath, "utf8")); + if (JSON.stringify(artifact) !== JSON.stringify(report)) { + console.error("quad-cli-smoke: stdout and artifact report differed."); + return 1; + } + + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + const effective = new Set(report.reviewers?.effective ?? []); + const missing = DECLARED_TOOLS.filter((tool) => !effective.has(tool)); + if (missing.length === 0) { + console.error('quad-cli-smoke: status="ok"; all declared reviewers returned schema-valid output.'); + return 0; + } + + const droppedByTool = new Map((report.reviewers?.dropped ?? []).map((entry) => [entry.tool, entry])); + for (const tool of missing) { + const dropped = droppedByTool.get(tool); + console.error( + `quad-cli-smoke: missing tool=${tool} stage=${dropped?.stage ?? "unknown"} cause=${dropped?.primary_cause ?? "unknown"} reason=${JSON.stringify(dropped?.observed_failure ?? "not reported")}` + ); + } + + if (availableMode) { + console.error('quad-cli-smoke: status="degraded"; one or more declared reviewers were unavailable or invalid.'); + return 4; + } + console.error('quad-cli-smoke: status="failed"; strict mode requires every declared reviewer.'); + return 1; + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +function parseArgs(argv) { + if (argv.length === 0) return false; + if (argv.length === 1 && argv[0] === "--available") return true; + throw new Error(`Unknown argument: ${argv[0]}`); +} + +const isDirectRun = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isDirectRun) { + try { + process.exit(runSmoke()); + } catch (error) { + console.error(`quad-cli-smoke: ${error.message}`); + process.exit(1); + } +} diff --git a/scripts/quad-cli-transport.mjs b/scripts/quad-cli-transport.mjs index 19835ad..35dc896 100644 --- a/scripts/quad-cli-transport.mjs +++ b/scripts/quad-cli-transport.mjs @@ -307,7 +307,7 @@ function strictDecode(chunks) { return new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)); } -function sanitizeReason(value, maxLength = 240) { +export function sanitizeReason(value, maxLength = 240) { const firstLine = (String(value ?? "").split(/\r?\n/, 1)[0] ?? "") .replace(/[\u0000-\u001f\u007f]+/g, " ") .trim(); @@ -614,7 +614,7 @@ export function parseJsonReport(stdout, tool = null) { } export function formatRunnerDiagnostic(result) { - const reason = sanitizeReason(result.reason ?? result.stderr ?? result.error ?? "-"); + const reason = sanitizeReason(result.reason || result.stderr || result.error || "-"); const parts = [ `tool=${result.tool}`, `status=${result.status}`, diff --git a/tests/fixtures/quad-cli/smoke.diff b/tests/fixtures/quad-cli/smoke.diff new file mode 100644 index 0000000..69d08e1 --- /dev/null +++ b/tests/fixtures/quad-cli/smoke.diff @@ -0,0 +1,7 @@ +diff --git a/src/smoke.js b/src/smoke.js +index 257cc56..3bd1f0e 100644 +--- a/src/smoke.js ++++ b/src/smoke.js +@@ -1 +1 @@ +-export const ready = false; ++export const ready = true; diff --git a/tests/quad-cli-orchestrate.test.js b/tests/quad-cli-orchestrate.test.js index 8f3be63..52065b6 100644 --- a/tests/quad-cli-orchestrate.test.js +++ b/tests/quad-cli-orchestrate.test.js @@ -1,8 +1,9 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join, resolve } from "node:path"; import { buildHunkIndex, findHunkId, @@ -14,6 +15,8 @@ import { validateRawReport, validateMergedReport, parseArgs, + deriveFailureStage, + redactResolvedPath, } from "../scripts/quad-cli-orchestrate.mjs"; const ROOT = resolve(import.meta.dirname, ".."); @@ -321,3 +324,85 @@ describe("quad CLI timeout flags", () => { assert.equal(options.gateTimeout, 4444); }); }); + +describe("reviewer roster and environment artifact", () => { + const validResponse = JSON.stringify({ + schema_version: "1", + generated_by: "quad-cli-orchestrate", + findings: [], + }); + + function withTwoValidReviewers(callback) { + const tempDir = mkdtempSync(join(tmpdir(), "quad-cli-roster-")); + const mockDir = join(tempDir, "mocks"); + const artifactPath = join(tempDir, "report.json"); + try { + mkdirSync(mockDir); + writeFileSync(join(mockDir, "claude.json"), validResponse, "utf8"); + writeFileSync(join(mockDir, "codex.json"), validResponse, "utf8"); + writeFileSync( + join(mockDir, "cursor-agent.json"), + JSON.stringify({ schema_version: "1", generated_by: "quad-cli-orchestrate", findings: [{ path: 7 }] }), + "utf8" + ); + const result = runScript( + ["--diff-file", resolve(FIXTURES_DIR, "smoke.diff"), "--artifact", artifactPath], + { QUAD_CLI_MOCK_DIR: mockDir } + ); + callback({ result, report: JSON.parse(result.stdout), artifactPath, tempDir }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + } + + it("records four declared, two effective, and two dropped reviewers", () => { + withTwoValidReviewers(({ result, report }) => { + assert.equal(result.code, 0); + assert.deepEqual(report.reviewers.declared, ["claude", "codex", "cursor-agent", "agy"]); + assert.deepEqual(report.reviewers.effective, ["claude", "codex"]); + assert.deepEqual(report.reviewers.dropped.map(({ tool }) => tool), ["cursor-agent", "agy"]); + }); + }); + + it("records spawn and schema failures as different stages", () => { + withTwoValidReviewers(({ report }) => { + const stages = Object.fromEntries(report.reviewers.dropped.map(({ tool, stage }) => [tool, stage])); + assert.equal(stages["cursor-agent"], "schema"); + assert.equal(stages.agy, "spawn"); + assert.match(report.reviewers.dropped.find(({ tool }) => tool === "agy").observed_failure, /Mock response not found/); + assert.equal(deriveFailureStage({ class: "timeout", timedOut: true }), "transport"); + }); + }); + + it("redacts home paths and never retains the home directory string", () => { + const home = homedir(); + const pathWithDifferentSeparators = `${home.replace(/\\/g, "/").toUpperCase()}/private/tool.cmd`; + const redacted = redactResolvedPath(pathWithDifferentSeparators, home); + assert.match(redacted, /^~\//); + assert.equal(redacted.toLowerCase().includes(home.replace(/\\/g, "/").toLowerCase()), false); + assert.equal(redactResolvedPath(resolve(ROOT, "outside", "tool.cmd"), join(ROOT, "not-home")), "tool.cmd"); + }); + + it("does not create an artifact when --artifact is omitted", () => { + const tempDir = mkdtempSync(join(tmpdir(), "quad-cli-no-artifact-")); + try { + const stdout = execFileSync("node", [SCRIPT, "--diff-file", resolve(FIXTURES_DIR, "smoke.diff")], { + cwd: tempDir, + encoding: "utf8", + env: { ...process.env, QUAD_CLI_MOCK_DIR: MOCK_DIR }, + }); + assert.equal(readdirSync(tempDir).length, 0); + assert.equal("reviewers" in JSON.parse(stdout), false, "default report shape stays unchanged"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("writes the same validated report object to stdout and --artifact", () => { + withTwoValidReviewers(({ report, artifactPath }) => { + const artifact = JSON.parse(readFileSync(artifactPath, "utf8")); + assert.deepEqual(artifact, report); + assert.equal(validateMergedReport(report).ok, true); + }); + }); +}); diff --git a/tests/quad-cli-smoke.test.js b/tests/quad-cli-smoke.test.js new file mode 100644 index 0000000..66f92c6 --- /dev/null +++ b/tests/quad-cli-smoke.test.js @@ -0,0 +1,72 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; + +const ROOT = resolve(import.meta.dirname, ".."); +const SCRIPT = resolve(ROOT, "scripts", "quad-cli-smoke.mjs"); +const TOOLS = ["claude", "codex", "cursor-agent", "agy"]; +const VALID_RESPONSE = JSON.stringify({ + schema_version: "1", + generated_by: "quad-cli-orchestrate", + findings: [], +}); + +function withMockReviewers(missingTool, callback) { + const tempDir = mkdtempSync(join(tmpdir(), "quad-cli-smoke-test-")); + const mockDir = join(tempDir, "mocks"); + mkdirSync(mockDir); + try { + for (const tool of TOOLS) { + if (tool !== missingTool) writeFileSync(join(mockDir, `${tool}.json`), VALID_RESPONSE, "utf8"); + } + callback(mockDir); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +function runSmoke(args, mockDir) { + const result = spawnSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: "utf8", + env: { ...process.env, QUAD_CLI_MOCK_DIR: mockDir }, + }); + return { code: result.status, stdout: result.stdout, stderr: result.stderr }; +} + +describe("quad CLI live smoke wrapper (mock reviewers)", () => { + it("smoke:quad exits 0 when all four reviewers are valid", () => { + withMockReviewers(null, (mockDir) => { + const result = runSmoke([], mockDir); + assert.equal(result.code, 0); + assert.deepEqual(JSON.parse(result.stdout).reviewers.effective, TOOLS); + }); + }); + + it("smoke:quad exits 1 and names a missing reviewer", () => { + withMockReviewers("agy", (mockDir) => { + const result = runSmoke([], mockDir); + assert.equal(result.code, 1); + assert.match(`${result.stdout}\n${result.stderr}`, /agy/); + }); + }); + + it("smoke:quad:available exits 4 with degraded status and a missing-reviewer list", () => { + withMockReviewers("cursor-agent", (mockDir) => { + const result = runSmoke(["--available"], mockDir); + assert.equal(result.code, 4); + assert.match(result.stderr, /status="degraded"/); + assert.match(`${result.stdout}\n${result.stderr}`, /cursor-agent/); + }); + }); + + it("smoke:quad:available exits 0 when all four reviewers are valid", () => { + withMockReviewers(null, (mockDir) => { + const result = runSmoke(["--available"], mockDir); + assert.equal(result.code, 0); + }); + }); +});