From 07d4e03ce08b0e968daffbf6939597c33281b079 Mon Sep 17 00:00:00 2001 From: Andy00L <89641810+Andy00L@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:31:33 -0400 Subject: [PATCH 1/3] feat(test): attach GitHub-native CI output to "test run --all" (--gh-output, --summary-file) --- src/commands/test.ts | 63 +++++++++ src/lib/gh-output.test.ts | 123 ++++++++++++++++++ src/lib/gh-output.ts | 122 +++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 7 + 4 files changed, 315 insertions(+) create mode 100644 src/lib/gh-output.test.ts create mode 100644 src/lib/gh-output.ts diff --git a/src/commands/test.ts b/src/commands/test.ts index ca6801c..5bb7c33 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,9 +1,11 @@ import { + appendFileSync, createWriteStream, existsSync, readFileSync, readdirSync, statSync, + writeFileSync, type WriteStream, } from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; @@ -92,6 +94,7 @@ import { import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; +import { emitGithubOutputs, summarizeAcceptedPayload } from '../lib/gh-output.js'; import { loadConfig } from '../lib/config.js'; import { flakyExitCode, @@ -6363,6 +6366,10 @@ interface RunTestRunAllOptions extends CommonOptions { reportFile?: string; /** --report-suite-name: optional override for the JUnit . */ reportSuiteName?: string; + /** --gh-output: force the GitHub-native output layer even off-Actions (issue #99). */ + ghOutput?: boolean; + /** --summary-file: also write the reduced machine summary JSON to this path. */ + summaryFile?: string; } async function writeBatchJUnitReportIfRequested( @@ -6927,6 +6934,36 @@ export async function runTestRunAll( }; await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); + // CI-native output layer (issue #99): emitted before the gate throws below so + // the artifacts land even when the batch exits non-zero. The summary file is a + // machine artifact written regardless of --output mode; stdout stays owned by + // the envelope above (plus Actions workflow commands, which Actions parses). + { + const env = deps.env ?? process.env; + const ghEnabled = opts.ghOutput === true || env.GITHUB_ACTIONS === 'true'; + if (ghEnabled || opts.summaryFile !== undefined) { + const ciSummary = summarizeAcceptedPayload(JSON.stringify(jsonPayload)); + if (opts.summaryFile !== undefined) { + try { + writeFileSync(opts.summaryFile, `${JSON.stringify(ciSummary, null, 2)}\n`, 'utf8'); + } catch { + stderrFn(`[run] could not write --summary-file ${opts.summaryFile}; continuing`); + } + } + if (ghEnabled) { + emitGithubOutputs( + ciSummary, + env, + { + stdout: deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)), + stderr: stderrFn, + appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'), + }, + { force: opts.ghOutput === true }, + ); + } + } + } // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), // mirroring `test rerun --all`. Checked before the failed-run throw so the @@ -9125,6 +9162,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--report-suite-name ', 'optional JUnit override (default: testsprite:)', ) + .option( + '--gh-output', + 'with --all: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', + ) + .option( + '--summary-file ', + 'with --all: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + @@ -9173,6 +9218,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { wait: cmdOpts.wait === true, batchPath: isAll, }); + // --gh-output / --summary-file reduce the batch envelope; on the single-id + // path they would be silently ignored — reject loudly (same rule as --filter). + if (cmdOpts.ghOutput === true && !isAll) { + throw localValidationError( + 'gh-output', + '--gh-output only applies with --all (it reduces the batch envelope). Remove --gh-output, or add --all.', + ); + } + if (cmdOpts.summaryFile !== undefined && !isAll) { + throw localValidationError( + 'summary-file', + '--summary-file only applies with --all (it reduces the batch envelope). Remove --summary-file, or add --all.', + ); + } if (isAll) { // --all path: wave-ordered fresh batch run. @@ -9207,6 +9266,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { report, reportFile: cmdOpts.reportFile, reportSuiteName: cmdOpts.reportSuiteName, + ghOutput: cmdOpts.ghOutput === true, + summaryFile: cmdOpts.summaryFile, }, deps, ); @@ -9687,6 +9748,8 @@ interface RunFlagOpts { report?: string; reportFile?: string; reportSuiteName?: string; + ghOutput?: boolean; + summaryFile?: string; } interface WaitFlagOpts { diff --git a/src/lib/gh-output.test.ts b/src/lib/gh-output.test.ts new file mode 100644 index 0000000..07ae890 --- /dev/null +++ b/src/lib/gh-output.test.ts @@ -0,0 +1,123 @@ +/** + * Unit tests for the CI-native output layer attached to `test run --all` + * (issue #99, reshaped from the withdrawn top-level `ci` command). The heavy + * lifting (trigger + poll) is the batch command's, already covered by its own + * suites; these tests cover the presentation seams: payload reduction, the + * job-summary Markdown, and the GitHub gating (env-driven and `--gh-output` + * forced). + */ + +import { describe, expect, it } from 'vitest'; +import { + emitGithubOutputs, + renderJobSummaryMarkdown, + summarizeAcceptedPayload, + type CiSummary, +} from './gh-output.js'; + +const PAYLOAD = JSON.stringify({ + accepted: [ + { + testId: 'test_a', + runId: 'run_a', + status: 'passed', + dashboardUrl: 'https://portal.example.com/a', + }, + { + testId: 'test_b', + runId: 'run_b', + status: 'failed', + error: { code: 'INTERNAL', message: 'boom', exitCode: 1 }, + }, + { testId: 'test_c', runId: 'run_c', status: 'timeout' }, + ], + conflicts: [], +}); + +describe('summarizeAcceptedPayload', () => { + it('reduces accepted[] rows into counts and rows', () => { + const summary = summarizeAcceptedPayload(PAYLOAD); + expect(summary).toMatchObject({ total: 3, passed: 1, failed: 1, timedOut: 1 }); + expect(summary.runs[1]).toMatchObject({ testId: 'test_b', status: 'failed', error: 'boom' }); + }); + + it('unparseable or non-batch output reduces to an empty summary (never throws)', () => { + expect(summarizeAcceptedPayload('')).toMatchObject({ total: 0, passed: 0 }); + expect(summarizeAcceptedPayload('{"method":"POST"}')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('not json')).toMatchObject({ total: 0 }); + }); +}); + +describe('renderJobSummaryMarkdown', () => { + it('renders the counts headline and one table row per run', () => { + const md = renderJobSummaryMarkdown(summarizeAcceptedPayload(PAYLOAD)); + expect(md).toContain('**1/3 passed** (1 failed, 1 timed out)'); + expect(md).toContain('| test_a | passed | [dashboard](https://portal.example.com/a) |'); + expect(md).toContain('| test_c | timeout | run_c |'); + }); +}); + +describe('emitGithubOutputs', () => { + const summary: CiSummary = summarizeAcceptedPayload(PAYLOAD); + + function makeSinks() { + const stdout: string[] = []; + const stderr: string[] = []; + const appended: Array<{ path: string; content: string }> = []; + return { + stdout, + stderr, + appended, + sinks: { + stdout: (line: string) => stdout.push(line), + stderr: (line: string) => stderr.push(line), + appendFile: (path: string, content: string) => appended.push({ path, content }), + }, + }; + } + + it('appends the job summary and annotates only non-passed runs under Actions', () => { + const { stdout, appended, sinks } = makeSinks(); + emitGithubOutputs( + summary, + { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + sinks, + ); + expect(appended).toHaveLength(1); + expect(appended[0]!.path).toBe('/gh/summary.md'); + expect(appended[0]!.content).toContain('TestSprite results'); + const annotations = stdout.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(2); + expect(annotations[0]).toContain('test_b'); + expect(annotations[0]).toContain('boom'); + expect(annotations[1]).toContain('test_c'); + }); + + it('emits nothing off-CI, and a broken summary file downgrades to stderr', () => { + const offCi = makeSinks(); + emitGithubOutputs(summary, {}, offCi.sinks); + expect(offCi.stdout).toHaveLength(0); + expect(offCi.appended).toHaveLength(0); + + const broken = makeSinks(); + emitGithubOutputs( + summary, + { GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + { + ...broken.sinks, + appendFile: () => { + throw new Error('EROFS'); + }, + }, + ); + expect(broken.stderr.join('\n')).toContain('could not append'); + }); + + it('force (--gh-output) emits annotations off-Actions; the step summary still needs its env path', () => { + const forced = makeSinks(); + emitGithubOutputs(summary, {}, forced.sinks, { force: true }); + const annotations = forced.stdout.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(2); + expect(forced.appended).toHaveLength(0); + }); +}); diff --git a/src/lib/gh-output.ts b/src/lib/gh-output.ts new file mode 100644 index 0000000..613b695 --- /dev/null +++ b/src/lib/gh-output.ts @@ -0,0 +1,122 @@ +/** + * CI-native output layer for the batch run path (issue #99, reshaped from + * the withdrawn top-level `ci` command per the #264 review). + * + * `test run --all --wait` presents its result in the formats CI consumes: + * (a) a stable machine summary `{total, passed, failed, timedOut, runs[]}` + * written to `--summary-file ` when requested, + * (b) a Markdown results table appended to `$GITHUB_STEP_SUMMARY` when + * running under GitHub Actions, + * (c) one `::error::` workflow-command line per non-passed run so failures + * annotate the PR checks tab. + * Activation: `GITHUB_ACTIONS=true` in the environment, or the explicit + * `--gh-output` flag (which forces the annotations even off-Actions, so the + * behavior is previewable locally). All writes are best-effort: a broken + * summary file must never mask the batch gate's exit code. + */ + +export interface CiRunRow { + testId: string; + runId?: string; + status: string; + dashboardUrl?: string; + error?: string; +} + +export interface CiSummary { + total: number; + passed: number; + failed: number; + timedOut: number; + runs: CiRunRow[]; +} + +/** + * Reduce the batch command's JSON payload into the CI summary. The parse is + * defensive: it reads the same `accepted[]` rows the automation contract + * documents, and anything unparseable (dry-run envelope, partial output + * after a timeout) reduces to an empty run list rather than a crash. + */ +export function summarizeAcceptedPayload(capturedJson: string): CiSummary { + let payload: { accepted?: unknown } = {}; + try { + payload = JSON.parse(capturedJson) as { accepted?: unknown }; + } catch { + // Not a JSON object (dry-run banner path or truncated output): no rows. + } + const rows: CiRunRow[] = Array.isArray(payload.accepted) + ? (payload.accepted as Array>).map(row => { + const errorMessage = + row.error !== null && typeof row.error === 'object' + ? (row.error as { message?: unknown }).message + : undefined; + return { + testId: String(row.testId ?? ''), + ...(typeof row.runId === 'string' ? { runId: row.runId } : {}), + status: String(row.status ?? 'unknown'), + ...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}), + ...(typeof errorMessage === 'string' ? { error: errorMessage } : {}), + }; + }) + : []; + const passed = rows.filter(row => row.status === 'passed').length; + const timedOut = rows.filter(row => row.status === 'timeout').length; + const failed = rows.length - passed - timedOut; + return { total: rows.length, passed, failed, timedOut, runs: rows }; +} + +/** Markdown table for the GitHub job summary. */ +export function renderJobSummaryMarkdown(summary: CiSummary): string { + return [ + '## TestSprite results', + '', + `**${summary.passed}/${summary.total} passed** (${summary.failed} failed, ${summary.timedOut} timed out)`, + '', + '| Test | Status | Run |', + '| --- | --- | --- |', + ...summary.runs.map( + row => + `| ${row.testId} | ${row.status} | ${ + row.dashboardUrl ? `[dashboard](${row.dashboardUrl})` : (row.runId ?? '') + } |`, + ), + '', + ].join('\n'); +} + +/** + * Emit the GitHub-native surfaces. Self-gating on the standard env vars: + * `$GITHUB_STEP_SUMMARY` (a file path Actions provides) receives the Markdown + * table; `GITHUB_ACTIONS=true` enables one `::error::` workflow command per + * non-passed run on stdout (Actions parses workflow commands from stdout). + * `force` (the `--gh-output` flag) emits the annotations even off-Actions; + * the step summary still requires the env-provided file path to exist. + * Both writes are best-effort: a broken summary file must not mask the gate. + */ +export function emitGithubOutputs( + summary: CiSummary, + env: NodeJS.ProcessEnv, + sinks: { + stdout: (line: string) => void; + stderr: (line: string) => void; + appendFile: (path: string, content: string) => void; + }, + opts: { force?: boolean } = {}, +): void { + const summaryPath = env.GITHUB_STEP_SUMMARY; + if (typeof summaryPath === 'string' && summaryPath.length > 0) { + try { + sinks.appendFile(summaryPath, renderJobSummaryMarkdown(summary)); + } catch { + sinks.stderr('[run] could not append to GITHUB_STEP_SUMMARY; continuing'); + } + } + if (env.GITHUB_ACTIONS === 'true' || opts.force === true) { + for (const row of summary.runs) { + if (row.status === 'passed') continue; + const detail = row.error !== undefined ? ` ${row.error}` : ''; + const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : ''; + sinks.stdout(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`); + } + } +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 55a7252..07f3394 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -679,6 +679,13 @@ Options: --report-file output path for --report (atomic write) --report-suite-name optional JUnit override (default: testsprite:) + --gh-output with --all: emit GitHub-native output (::error:: + annotations per non-passed run; job-summary table + when $GITHUB_STEP_SUMMARY is set). Auto-enabled + when GITHUB_ACTIONS=true + --summary-file with --all: also write the reduced machine + summary JSON {total, passed, failed, timedOut, + runs[]} to this file -h, --help display help for command Dependency-aware fresh run (M4): From 887bb4a2c42fa5d9ada7c67ee6067f6530c50bd2 Mon Sep 17 00:00:00 2001 From: Andy00L <89641810+Andy00L@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:56:55 -0400 Subject: [PATCH 2/3] test(run): isolate batch-run specs from the CI runner env (GITHUB_ACTIONS leak) --- src/commands/test.run.spec.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 7eb8eb0..5f04e6d 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -2725,6 +2725,7 @@ describe('runTestRunAll — batch fresh run', () => { fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, sleep: instantSleep, }, ); @@ -3458,6 +3459,7 @@ describe('[B-E2E-01] runTestRunAll --wait: non-passed runs must exit 1 (regressi fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, sleep: instantSleep, }, ); @@ -3866,6 +3868,7 @@ describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out p fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, sleep: instantSleep, }, ).catch(e => e); From 28030c8c0d1daf78182f0221cccf433b542e92c6 Mon Sep 17 00:00:00 2001 From: Andy00L <89641810+Andy00L@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:14:34 -0400 Subject: [PATCH 3/3] fix(run): require --wait for CI-output flags, keep JSON stdout clean, harden the payload reducer --- src/commands/test.run.spec.ts | 124 +++++++++++++++++- src/commands/test.ts | 25 ++-- src/lib/gh-output.test.ts | 23 ++++ src/lib/gh-output.ts | 51 ++++--- test/__snapshots__/help.snapshot.test.ts.snap | 10 +- 5 files changed, 201 insertions(+), 32 deletions(-) diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 5f04e6d..6c8be5f 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3960,3 +3960,125 @@ describe('runTestRun --wait — InterruptError graceful detach (DEV-331)', () => expect(stderrBlock).toContain('testsprite test wait run_abc'); }); }); + +describe('gh-output integration on run --all --wait (issue #99 reshape)', () => { + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T11:00:00.000Z', + startedAt: '2026-06-09T11:00:01.000Z', + finishedAt: '2026-06-09T11:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, + }; + } + + function mixedHarness() { + const { credentialsPath } = makeCreds(); + const mixedBatch: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_p', runId: 'run_p', enqueuedAt: '2026-06-09T11:00:00.000Z' }, + { testId: 'test_f', runId: 'run_f', enqueuedAt: '2026-06-09T11:00:02.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: mixedBatch }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_p') return { body: makeTerminalRun('run_p', 'test_p', 'passed') }; + if (runId === 'run_f') return { body: makeTerminalRun('run_f', 'test_f', 'failed') }; + return errorBody('NOT_FOUND'); + }); + return { credentialsPath, fetchImpl }; + } + + it('under Actions with --output json: stdout stays parseable JSON, ::error:: goes to stderr', async () => { + const { credentialsPath, fetchImpl } = mixedHarness(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const err = await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + env: { GITHUB_ACTIONS: 'true' } as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + // The documented machine envelope must remain parseable as-is. + const payload = JSON.parse(stdoutLines.join('\n')) as { accepted?: unknown[] }; + expect(Array.isArray(payload.accepted)).toBe(true); + expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(false); + const annotations = stderrLines.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(1); + expect(annotations[0]).toContain('test_f'); + }); + + it('--gh-output --summary-file writes the reduced artifact even though the gate exits 1', async () => { + const { credentialsPath, fetchImpl } = mixedHarness(); + const dir = mkdtempSync(join(tmpdir(), 'cli-gh-output-')); + const summaryFile = join(dir, 'summary.json'); + const stdoutLines: string[] = []; + const err = await runTestRunAll( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + ghOutput: true, + summaryFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + const artifact = JSON.parse(readFileSync(summaryFile, 'utf8')) as { + total: number; + passed: number; + failed: number; + runs: unknown[]; + }; + expect(artifact).toMatchObject({ total: 2, passed: 1, failed: 1 }); + // Forced annotations (off-Actions) land on the text stdout, not the file. + expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(true); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 5bb7c33..6cb692b 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6951,13 +6951,18 @@ export async function runTestRunAll( } } if (ghEnabled) { + const stdoutFn = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); emitGithubOutputs( ciSummary, env, { - stdout: deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)), + stdout: stdoutFn, stderr: stderrFn, appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'), + // Under --output json the envelope above owns stdout; workflow + // commands go to stderr instead (the Actions runner parses both + // streams), keeping the documented machine output parseable. + annotations: opts.output === 'json' ? stderrFn : stdoutFn, }, { force: opts.ghOutput === true }, ); @@ -9164,11 +9169,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--gh-output', - 'with --all: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', + 'with --all --wait: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', ) .option( '--summary-file ', - 'with --all: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', + 'with --all --wait: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', ) .addHelpText( 'after', @@ -9218,18 +9223,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { wait: cmdOpts.wait === true, batchPath: isAll, }); - // --gh-output / --summary-file reduce the batch envelope; on the single-id - // path they would be silently ignored — reject loudly (same rule as --filter). - if (cmdOpts.ghOutput === true && !isAll) { + // --gh-output / --summary-file reduce the terminal batch envelope, which + // only exists on the --all --wait path (without --wait the command returns + // after enqueueing). Anywhere else they would silently no-op — reject + // loudly (same rule as --filter and the JUnit report flags). + if (cmdOpts.ghOutput === true && (!isAll || cmdOpts.wait !== true)) { throw localValidationError( 'gh-output', - '--gh-output only applies with --all (it reduces the batch envelope). Remove --gh-output, or add --all.', + '--gh-output only applies with --all --wait (it reduces the terminal batch envelope). Remove --gh-output, or add --all --wait.', ); } - if (cmdOpts.summaryFile !== undefined && !isAll) { + if (cmdOpts.summaryFile !== undefined && (!isAll || cmdOpts.wait !== true)) { throw localValidationError( 'summary-file', - '--summary-file only applies with --all (it reduces the batch envelope). Remove --summary-file, or add --all.', + '--summary-file only applies with --all --wait (it reduces the terminal batch envelope). Remove --summary-file, or add --all --wait.', ); } diff --git a/src/lib/gh-output.test.ts b/src/lib/gh-output.test.ts index 07ae890..759ecaa 100644 --- a/src/lib/gh-output.test.ts +++ b/src/lib/gh-output.test.ts @@ -46,6 +46,17 @@ describe('summarizeAcceptedPayload', () => { expect(summarizeAcceptedPayload('{"method":"POST"}')).toMatchObject({ total: 0 }); expect(summarizeAcceptedPayload('not json')).toMatchObject({ total: 0 }); }); + + it('valid-JSON non-record payloads and null rows are skipped, not crashes', () => { + expect(summarizeAcceptedPayload('null')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('"a string"')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('[1,2]')).toMatchObject({ total: 0 }); + const mixed = summarizeAcceptedPayload( + JSON.stringify({ accepted: [null, 42, { testId: 'test_ok', status: 'passed' }] }), + ); + expect(mixed.total).toBe(1); + expect(mixed.runs[0]).toMatchObject({ testId: 'test_ok', status: 'passed' }); + }); }); describe('renderJobSummaryMarkdown', () => { @@ -120,4 +131,16 @@ describe('emitGithubOutputs', () => { expect(annotations).toHaveLength(2); expect(forced.appended).toHaveLength(0); }); + + it('a dedicated annotations sink diverts workflow commands off the primary stdout', () => { + const { stdout, sinks } = makeSinks(); + const diverted: string[] = []; + emitGithubOutputs( + summary, + { GITHUB_ACTIONS: 'true' }, + { ...sinks, annotations: line => diverted.push(line) }, + ); + expect(stdout).toHaveLength(0); + expect(diverted.filter(line => line.startsWith('::error'))).toHaveLength(2); + }); }); diff --git a/src/lib/gh-output.ts b/src/lib/gh-output.ts index 613b695..ea9b379 100644 --- a/src/lib/gh-output.ts +++ b/src/lib/gh-output.ts @@ -38,26 +38,35 @@ export interface CiSummary { * after a timeout) reduces to an empty run list rather than a crash. */ export function summarizeAcceptedPayload(capturedJson: string): CiSummary { - let payload: { accepted?: unknown } = {}; + let parsed: unknown; try { - payload = JSON.parse(capturedJson) as { accepted?: unknown }; + parsed = JSON.parse(capturedJson); } catch { - // Not a JSON object (dry-run banner path or truncated output): no rows. + // Not JSON at all (dry-run banner path or truncated output): no rows. + parsed = undefined; } + // `JSON.parse('null')` and non-object payloads are valid JSON but carry no + // batch envelope — treat them like unparseable input instead of crashing. + const payload: { accepted?: unknown } = + parsed !== null && typeof parsed === 'object' ? (parsed as { accepted?: unknown }) : {}; const rows: CiRunRow[] = Array.isArray(payload.accepted) - ? (payload.accepted as Array>).map(row => { - const errorMessage = - row.error !== null && typeof row.error === 'object' - ? (row.error as { message?: unknown }).message - : undefined; - return { - testId: String(row.testId ?? ''), - ...(typeof row.runId === 'string' ? { runId: row.runId } : {}), - status: String(row.status ?? 'unknown'), - ...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}), - ...(typeof errorMessage === 'string' ? { error: errorMessage } : {}), - }; - }) + ? payload.accepted + .filter( + (entry): entry is Record => entry !== null && typeof entry === 'object', + ) + .map(row => { + const errorMessage = + row.error !== null && typeof row.error === 'object' + ? (row.error as { message?: unknown }).message + : undefined; + return { + testId: String(row.testId ?? ''), + ...(typeof row.runId === 'string' ? { runId: row.runId } : {}), + status: String(row.status ?? 'unknown'), + ...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}), + ...(typeof errorMessage === 'string' ? { error: errorMessage } : {}), + }; + }) : []; const passed = rows.filter(row => row.status === 'passed').length; const timedOut = rows.filter(row => row.status === 'timeout').length; @@ -100,6 +109,13 @@ export function emitGithubOutputs( stdout: (line: string) => void; stderr: (line: string) => void; appendFile: (path: string, content: string) => void; + /** + * Where `::error::` workflow-command lines go. Defaults to `stdout`; the + * caller passes stderr under `--output json` so the machine envelope on + * stdout stays parseable (the Actions runner processes workflow commands + * on both streams). + */ + annotations?: (line: string) => void; }, opts: { force?: boolean } = {}, ): void { @@ -112,11 +128,12 @@ export function emitGithubOutputs( } } if (env.GITHUB_ACTIONS === 'true' || opts.force === true) { + const annotate = sinks.annotations ?? sinks.stdout; for (const row of summary.runs) { if (row.status === 'passed') continue; const detail = row.error !== undefined ? ` ${row.error}` : ''; const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : ''; - sinks.stdout(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`); + annotate(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`); } } } diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 07f3394..bdee849 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -679,11 +679,11 @@ Options: --report-file output path for --report (atomic write) --report-suite-name optional JUnit override (default: testsprite:) - --gh-output with --all: emit GitHub-native output (::error:: - annotations per non-passed run; job-summary table - when $GITHUB_STEP_SUMMARY is set). Auto-enabled - when GITHUB_ACTIONS=true - --summary-file with --all: also write the reduced machine + --gh-output with --all --wait: emit GitHub-native output + (::error:: annotations per non-passed run; + job-summary table when $GITHUB_STEP_SUMMARY is + set). Auto-enabled when GITHUB_ACTIONS=true + --summary-file with --all --wait: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file -h, --help display help for command