From 9a2335c2bcccf19533f3c1c52edfbeba5bd1d09b Mon Sep 17 00:00:00 2001 From: Mike Pitre <12040919+mikepitre@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:37:20 -0400 Subject: [PATCH 1/5] test(expo): put a run's evidence in the pull request description `attach` posted the video and screenshots of a run as a pull request comment. It now writes them into the description with `gh pr edit --attach`, in one block between two HTML comments that name the platform. A later `attach` replaces the block, so the description holds the latest run. Everything outside the block is passed back as it was read. `attach` reads the description again just before it writes. If the description changed, it rebuilds on the new text once, and fails if it changed again. It refuses a description whose markers are doubled, halved, or out of order. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-clerk-expo/SKILL.md | 6 +- integration/expo-native/src/core/MANIFEST | 8 +- integration/expo-native/src/core/cli.ts | 2 +- integration/expo-native/src/core/publish.ts | 187 ++++++++++--- integration/expo-native/src/core/types.ts | 12 +- integration/expo-native/src/core/verbs.ts | 4 +- integration/expo-native/test/cli.test.ts | 4 +- integration/expo-native/test/evidence.test.ts | 262 +++++++++++++++--- .../expo-native/test/lease-flow.test.ts | 4 +- .../expo-native/test/run-groups.test.ts | 8 +- 10 files changed, 397 insertions(+), 100 deletions(-) diff --git a/.claude/skills/verify-clerk-expo/SKILL.md b/.claude/skills/verify-clerk-expo/SKILL.md index 402301def1b..aef6049365b 100644 --- a/.claude/skills/verify-clerk-expo/SKILL.md +++ b/.claude/skills/verify-clerk-expo/SKILL.md @@ -175,7 +175,11 @@ $ integration/expo-native/bin/control-clerk-expo attach --pr $ integration/expo-native/bin/control-clerk-expo attach --pr --screenshot profile # the video and one screenshot ``` -`attach` posts one comment per run and PR with `gh pr comment --attach`. It needs a `gh` whose `gh pr comment` has that flag, and it fails with a fix when the flag is missing. It refuses a run that is tainted, that has a failing spec or no passing one, or whose `app.log` names a user that the run did not create. +`attach` puts the video and the screenshots in the description of the pull request with `gh pr edit --attach`. It writes one block for the platform of the run: a line that names the run, the device, and the commit, then the files, between the comments `` and ``, or the same two with `android`. The first `attach` of a platform adds its block after the description. A later `attach` of that platform replaces its block, so the description holds the latest run of each platform and the media does not pile up. `attach` changes nothing outside the block. Keep both comments of a block or remove both: `attach` refuses a description that has one without the other, or either one twice. A comment counts only when it is a whole line outside a code fence, so a description can quote one in a sentence or show a whole block as an example. + +`attach` reads the description again just before it writes, and builds on the newer text once if it changed. It cannot see an edit that someone saves while the files upload, and that edit is lost, so do not edit the description while `attach` runs. + +`attach` needs gh 2.99.0 or newer, whose `gh pr edit` has `--attach`, and it fails with a fix when the flag is missing. It uploads a run to a PR once, and a second `attach` of the same run and PR prints `already posted`. It refuses a run that is tainted, that has a failing spec or no passing one, or whose `app.log` names a user that the run did not create. Attach the run of your own change. Run your new or changed spec on its own and attach that run, so the PR video shows only the behavior the change is about. You do not owe a regression run: PR CI (`.github/workflows/verify-e2e.yml`) runs every golden spec on the pull request and reports them there. If you ran other golden specs anyway, cite that run's id in the PR and leave its video in `.verify/runs/`. diff --git a/integration/expo-native/src/core/MANIFEST b/integration/expo-native/src/core/MANIFEST index 08a32d5c0b9..ecf2473820e 100644 --- a/integration/expo-native/src/core/MANIFEST +++ b/integration/expo-native/src/core/MANIFEST @@ -17,7 +17,7 @@ c9b5d5ebda565670f351cf7ed0b8fe3fa589cb1cb40f8b65eca81739b3d76810 specs/support/ 1b144b8f7362be2019f578bbdf94b7e6fe3196440de5c79d4f790fa8394374e3 src/core/broker.ts 7c741449c755524bba80f1ce314554293003a34f7f7c30696015ba1c0c3dc1ab src/core/claims.ts 1fc59d13e7197e6c7099c0467744b978d498d1fe409c88254d44e2f99913a7b5 src/core/clerk.ts -15dbedb7314bf057a92cabd7900ba7533bb8ec16f6e2ede792e8ecd2b922bfa1 src/core/cli.ts +ac38e56fe0390ee55c8af15e85e579b2d8c3b82907ffd10cca22af11fc3b841c src/core/cli.ts c658cd8472299371aee318f152d7a183f10e576dc0fba26904c3352f10cf5c36 src/core/devices.ts 3ae461d6aa75a2cefac0392a9ad124aec4105ac25b9320371743e2ea90fe2202 src/core/driver.ts b59071aaefeec02dea89bd25615f8ada0da0a51e805a2523011b8b591e7435d3 src/core/e2e.ts @@ -34,9 +34,9 @@ cf6aa68be73e4abaf3b63d5d84c2e9eee6197412f607ec53d0f05f7b058db420 src/core/insta 1287fc7e3328699e104c7bc50d3cd9bf9fbfec9efe764592c98e90e56033273d src/core/launch.mjs a0ee9e4bfd34d4659685108e5e54eb1ec2aff6248236c358e6d3b14b2e2ff6a0 src/core/ledgers.ts 8d533755b21beb1e1d74d160e2740c0b1c4c4b4bc0c33f8fac75083ffeb03e75 src/core/manifest.ts -1f38ad65e58f7b4b62bb2c84bd4a5a16f5481cfb6c5f9b4b2ba059677d900bf3 src/core/publish.ts +15ca07dce20c94a37cb6388ccf7ffb124e8a75592ea6b5904909cd4b66210d62 src/core/publish.ts 685eabf010c7f63d076cfd3da1a681b5cc8024720538c1bf3a99792b563d8b60 src/core/slot.ts a8838aaf1fa9e4a6bc4e350ea674079201da76c5962656039b96db2505591aed src/core/state.ts -ce8fad4756866cc6940d1fcc653640cabcc47861c34185d98211e67b9a4a5847 src/core/types.ts -5b137ac609685c490ac00b1d1ebce0ca0bb540316bfc14bb6c7a6e5b49cfa32e src/core/verbs.ts +45ab2e9984266fb21959752606ed0318fe6d2c32e3fd651e852ca02dfb93ec76 src/core/types.ts +0e8bfaca21d31934eecd1f386a5e6fcb375e4096fa4de501d29d27cec4d2ebc3 src/core/verbs.ts 56d2be8ec46aa0e0cf89747933e03f41fa7f3e6bed7e2a56b9d0b6e5e4db03e3 src/core/workspace.ts diff --git a/integration/expo-native/src/core/cli.ts b/integration/expo-native/src/core/cli.ts index 6c7fa99fe26..4e4a2bb46dc 100644 --- a/integration/expo-native/src/core/cli.ts +++ b/integration/expo-native/src/core/cli.ts @@ -228,7 +228,7 @@ function render(value: VerbResult, packageDir: string): string[] { return lines; } case 'attach': - return [`${value.alreadyPosted ? 'already posted' : 'posted'} ${value.posted.map((p) => basename(p)).join(', ')} ${value.commentUrl}`]; + return [`${value.alreadyPosted ? 'already posted' : 'posted'} ${value.posted.map((p) => basename(p)).join(', ')} in the description of ${value.prUrl}`]; case 'down': return [ ...(value.dryRun diff --git a/integration/expo-native/src/core/publish.ts b/integration/expo-native/src/core/publish.ts index 40715230393..fc6588a9b79 100644 --- a/integration/expo-native/src/core/publish.ts +++ b/integration/expo-native/src/core/publish.ts @@ -1,40 +1,126 @@ -import { existsSync, readFileSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, sep } from 'node:path'; import { run as defaultRunner, type Runner } from './exec.ts'; import type { Publishable } from './evidence.ts'; -import { VerifyFailure, type AttachResult, type EvidencePath, type HostAdapter } from './types.ts'; +import { VerifyFailure, type AttachResult, type EvidencePath, type EvidenceSummary, type HostAdapter, type Platform } from './types.ts'; interface Posted { readonly pr: number; - readonly commentUrl: string; + readonly prUrl: string; readonly posted: readonly EvidencePath[]; } -function tally(results: Publishable['results']): string { - const flaky = results.filter((r) => r.status === 'flaky').length; - return `${results.filter((r) => r.status === 'passed').length} of ${results.length} passed${flaky === 0 ? '' : `, ${flaky} flaky (passed only on a retry)`}.`; +export interface EvidenceMedia { + readonly alt: string; + readonly ref: string; } -export function commentBody(evidence: Publishable): string { - const lines = [ - `verify run \`${evidence.run}\` on ${evidence.platform} (${evidence.device}), build \`${evidence.build}\`, ${tally(evidence.results)}`, - '', - ...evidence.results.map((r) => `- ${r.status}: \`${r.spec.path}\` ${r.title}`), - ]; - for (const group of evidence.settings) { - if (group.askedBy === null) continue; - const results = evidence.results.filter((r) => group.specs.includes(r.spec.path)); - lines.push('', `Instance settings \`${group.label}\` (declared by \`${group.askedBy}\`): ${tally(results)}`); +export function summarize(evidence: Publishable): EvidenceSummary { + return { + run: evidence.run, + platform: evidence.platform, + device: evidence.device.replace(/[^A-Za-z0-9 ._()-]/g, '-').replace(/^[^A-Za-z0-9]+/, '').slice(0, 64) || 'device', + commit: evidence.gitHead, + passed: evidence.results.filter((r) => r.status === 'passed').length, + flaky: evidence.results.filter((r) => r.status === 'flaky').length, + total: evidence.results.length, + }; +} + +export const evidenceMarkers = (platform: Platform): { readonly start: string; readonly end: string } => ({ + start: ``, + end: ``, +}); + +const BLOCK_LINE_START = 'verify run `'; + +// GitHub plays a video only from a URL that is alone in its paragraph, and `gh pr edit --attach` rewrites a reference to that URL only there. +export function evidenceBlock(summary: EvidenceSummary, media: readonly EvidenceMedia[]): string { + const { start, end } = evidenceMarkers(summary.platform); + const flaky = summary.flaky === 0 ? '' : `, ${summary.flaky} flaky (passed only on a retry)`; + const line = `${BLOCK_LINE_START}${summary.run}\` on \`${summary.device}\` at \`${summary.commit.slice(0, 12)}\`, ${summary.passed} of ${summary.total} passed${flaky}.`; + return [start, '', line, ...media.flatMap((m) => ['', `![${m.alt}](${m.ref})`]), '', end].join('\n'); +} + +export type BlockEdit = { readonly ok: true; readonly body: string; readonly was: 'added' | 'replaced' } | { readonly ok: false; readonly why: string }; + +function fencedCode(text: string): readonly (readonly [number, number])[] { + const fences: [number, number][] = []; + let open: { readonly char: string; readonly length: number; readonly at: number } | null = null; + for (let at = 0; at <= text.length; ) { + const newline = text.indexOf('\n', at); + const lineEnd = newline === -1 ? text.length : newline; + const fence = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(text.slice(at, lineEnd).replace(/\r$/, '')); + if (fence !== null) { + const ticks = fence[1]!; + const rest = fence[2]!; + if (open === null && !(ticks[0] === '`' && rest.includes('`'))) open = { char: ticks[0]!, length: ticks.length, at }; + else if (open !== null && ticks[0] === open.char && ticks.length >= open.length && rest.trim() === '') { + fences.push([open.at, lineEnd]); + open = null; + } + } + if (newline === -1) break; + at = newline + 1; + } + if (open !== null) fences.push([open.at, text.length]); + return fences; +} + +function linesThatAre(text: string, marker: string): number[] { + const fences = fencedCode(text); + const found: number[] = []; + for (let at = text.indexOf(marker); at !== -1; at = text.indexOf(marker, at + marker.length)) { + const end = at + marker.length; + const startsItsLine = at === 0 || text[at - 1] === '\n'; + const endsItsLine = end === text.length || text[end] === '\n' || text.startsWith('\r\n', end); + if (startsItsLine && endsItsLine && !fences.some(([from, to]) => at > from && at < to)) found.push(at); + } + return found; +} + +export function withEvidenceBlock(body: string, platform: Platform, lines: string): BlockEdit { + const { start, end } = evidenceMarkers(platform); + const starts = linesThatAre(body, start); + const ends = linesThatAre(body, end); + const newline = body.includes('\r\n') ? '\r\n' : '\n'; + const block = lines.replaceAll('\n', newline); + if (starts.length === 0 && ends.length === 0) { + const gap = body === '' ? '' : body.endsWith('\n') ? newline : `${newline}${newline}`; + return { ok: true, body: `${body}${gap}${block}`, was: 'added' }; } - return lines.join('\n'); + if (starts.length !== 1 || ends.length !== 1) return { ok: false, why: `has \`${start}\` ${starts.length} times and \`${end}\` ${ends.length} times` }; + if (ends[0]! < starts[0]!) return { ok: false, why: `has \`${end}\` before \`${start}\`` }; + if (!body.slice(starts[0]! + start.length, ends[0]!).trimStart().startsWith(BLOCK_LINE_START)) return { ok: false, why: `has text between \`${start}\` and \`${end}\` that is not an evidence block` }; + return { ok: true, body: `${body.slice(0, starts[0]!)}${block}${body.slice(ends[0]! + end.length)}`, was: 'replaced' }; } -const ATTACH_FIX = 'install a gh build whose `gh pr comment` has --attach'; +const ATTACH_FIX = 'install gh 2.99.0 or newer, whose `gh pr edit` has --attach'; export async function missingAttach(runner: Runner): Promise<{ readonly why: string; readonly fix: string } | null> { - const help = await runner('gh', ['pr', 'comment', '--help']); + const help = await runner('gh', ['pr', 'edit', '--help']); if (help.code === 0 && help.stdout.includes('--attach')) return null; - return { why: help.code === 0 ? 'this gh has no `gh pr comment --attach`' : 'gh is not installed', fix: ATTACH_FIX }; + return { why: help.code === 0 ? 'this gh has no `gh pr edit --attach`' : 'gh is not installed', fix: ATTACH_FIX }; +} + +export function chooseFiles(evidence: Publishable, screenshots: 'all' | readonly string[]): { readonly videos: readonly EvidencePath[]; readonly shots: Publishable['screenshots'] } { + if (screenshots === 'all') return { videos: evidence.videos, shots: evidence.screenshots }; + const shots = screenshots.map((label) => { + const shot = evidence.screenshots.find((s) => s.label === label); + if (shot === undefined) { + throw new VerifyFailure('USAGE', `run ${evidence.run} has no screenshot labelled ${label}`, `use one of: ${evidence.screenshots.map((s) => s.label).join(', ') || '(none)'}`); + } + return shot; + }); + return { videos: evidence.videos, shots }; +} + +async function readDescription(runner: Runner, repo: string, pr: number): Promise<{ readonly body: string; readonly url: string }> { + const viewed = await runner('gh', ['pr', 'view', String(pr), '--repo', repo, '--json', 'body,url']); + if (viewed.code !== 0) throw new VerifyFailure('NOT_READY', `gh pr view failed: ${viewed.stderr.trim()}`, 'check `gh auth status` and that the PR exists'); + const parsed = JSON.parse(viewed.stdout) as { readonly body?: string | null; readonly url?: string }; + return { body: parsed.body ?? '', url: parsed.url ?? '' }; } export async function postToPullRequest( @@ -48,28 +134,47 @@ export async function postToPullRequest( const postedFile = join(dir, `posted-${pr}.json`); if (existsSync(postedFile)) { const previous = JSON.parse(readFileSync(postedFile, 'utf8')) as Posted; - return { verb: 'attach', commentUrl: previous.commentUrl, posted: previous.posted, alreadyPosted: true }; + return { verb: 'attach', prUrl: previous.prUrl, posted: previous.posted, alreadyPosted: true }; } - const chosen = - screenshots === 'all' - ? evidence.screenshots - : screenshots.map((label) => { - const shot = evidence.screenshots.find((s) => s.label === label); - if (shot === undefined) { - throw new VerifyFailure('USAGE', `run ${evidence.run} has no screenshot labelled ${label}`, `use one of: ${evidence.screenshots.map((s) => s.label).join(', ') || '(none)'}`); - } - return shot; - }); - const files = [...evidence.videos, ...chosen.map((s) => s.path)]; + const { videos, shots } = chooseFiles(evidence, screenshots); + const files = [...videos, ...shots.map((s) => s.path)]; const missing = files.length === 0 ? null : await missingAttach(runner); if (missing !== null) throw new VerifyFailure('NOT_READY', `${missing.why}, so the video and screenshots of run ${evidence.run} cannot be posted`, missing.fix); - const args = ['pr', 'comment', String(pr), '--repo', host.githubRepo, '--body', commentBody(evidence), ...files.flatMap((f) => ['--attach', f])]; - const result = await runner('gh', args); - if (result.code !== 0) { - throw new VerifyFailure('NOT_READY', `gh pr comment failed: ${result.stderr.trim()}`, 'check `gh auth status` and that the PR exists'); + + const ref = (file: EvidencePath): string => `./${relative(dir, file).split(sep).join('/')}`; + const block = evidenceBlock(summarize(evidence), [ + ...videos.map((video) => ({ alt: basename(video), ref: ref(video) })), + ...shots.map((shot) => ({ alt: shot.label, ref: ref(shot.path) })), + ]); + + let current = await readDescription(runner, host.githubRepo, pr); + let next: string | null = null; + for (let attempt = 0; attempt < 2 && next === null; attempt += 1) { + const edit = withEvidenceBlock(current.body, evidence.platform, block); + if (!edit.ok) { + throw new VerifyFailure('NOT_READY', `the description of PR #${pr} ${edit.why}, so the evidence of run ${evidence.run} has no one place to go`, 'leave one pair of those markers in the description, or none, then rerun'); + } + const again = await readDescription(runner, host.githubRepo, pr); + if (again.body === current.body) next = edit.body; + else current = again; + } + if (next === null) { + throw new VerifyFailure('NOT_READY', `the description of PR #${pr} changed twice while the evidence of run ${evidence.run} was being placed, so nothing was written`, 'rerun {cli} attach once nobody is editing the description'); + } + + const scratch = mkdtempSync(join(tmpdir(), 'verify-attach-')); + try { + const bodyFile = join(scratch, 'body.md'); + writeFileSync(bodyFile, next); + const result = await runner('gh', ['pr', 'edit', String(pr), '--repo', host.githubRepo, '--body-file', bodyFile, ...files.flatMap((f) => ['--attach', ref(f)])], { cwd: dir }); + if (result.code !== 0) { + throw new VerifyFailure('NOT_READY', `gh pr edit failed: ${result.stderr.trim()}`, 'check `gh auth status` and that you can edit the PR; a file that did upload is in the description already'); + } + const prUrl = /https:\/\/github\.com\/\S+/.exec(result.stdout)?.[0] ?? current.url; + const posted: Posted = { pr, prUrl, posted: files }; + writeFileSync(postedFile, `${JSON.stringify(posted, null, 2)}\n`); + return { verb: 'attach', prUrl, posted: files, alreadyPosted: false }; + } finally { + rmSync(scratch, { recursive: true, force: true }); } - const commentUrl = /https:\/\/github\.com\/\S+/.exec(result.stdout)?.[0] ?? result.stdout.trim(); - const posted: Posted = { pr, commentUrl, posted: files }; - writeFileSync(postedFile, `${JSON.stringify(posted, null, 2)}\n`); - return { verb: 'attach', commentUrl, posted: files, alreadyPosted: false }; } diff --git a/integration/expo-native/src/core/types.ts b/integration/expo-native/src/core/types.ts index d580634e83c..444be21e92d 100644 --- a/integration/expo-native/src/core/types.ts +++ b/integration/expo-native/src/core/types.ts @@ -177,9 +177,19 @@ export interface ScreenResult { readonly png: ScratchPath | null; } +export interface EvidenceSummary { + readonly run: RunId; + readonly platform: Platform; + readonly device: string; + readonly commit: string; + readonly passed: number; + readonly flaky: number; + readonly total: number; +} + export interface AttachResult { readonly verb: 'attach'; - readonly commentUrl: string; + readonly prUrl: string; readonly posted: readonly EvidencePath[]; readonly alreadyPosted: boolean; } diff --git a/integration/expo-native/src/core/verbs.ts b/integration/expo-native/src/core/verbs.ts index 330b90b3ea4..54ec78bd2de 100644 --- a/integration/expo-native/src/core/verbs.ts +++ b/integration/expo-native/src/core/verbs.ts @@ -137,8 +137,8 @@ export async function doctor(deps: Deps, command: Extract { backend: { ios: 'local' }, checks: [ { id: 'node', ok: true, detail: '24.15.0' }, - { id: 'gh-attach', ok: true, state: 'warning', detail: 'gh pr comment has no --attach', fix: 'install one that has' }, + { id: 'gh-attach', ok: true, state: 'warning', detail: 'gh pr edit has no --attach', fix: 'install one that has' }, { id: 'live-session', ok: true, state: 'not-run', detail: 'not run: needs --live' }, ], }; @@ -176,7 +176,7 @@ describe('doctor output', () => { it('labels a warning and a check that was not run, and neither fails doctor', () => { let out = ''; createOutput(false, '/tmp', 'bin/control-x', { write: (t: string) => (out += t) }, { write: () => true }).result(report); - assert.deepEqual(out.trimEnd().split('\n'), ['ok node 24.15.0', 'warn gh-attach gh pr comment has no --attach', ' fix: install one that has', 'skip live-session not run: needs --live']); + assert.deepEqual(out.trimEnd().split('\n'), ['ok node 24.15.0', 'warn gh-attach gh pr edit has no --attach', ' fix: install one that has', 'skip live-session not run: needs --live']); assert.equal(exitCodeFor(report), 0); }); diff --git a/integration/expo-native/test/evidence.test.ts b/integration/expo-native/test/evidence.test.ts index dc518e413f7..01dbc032ecc 100644 --- a/integration/expo-native/test/evidence.test.ts +++ b/integration/expo-native/test/evidence.test.ts @@ -8,12 +8,12 @@ import { describe, it } from 'node:test'; import { newTestEmail } from '../specs/support/clerk.ts'; import { assertPublishable, loggedUserIds, sealEvidence } from '../src/core/evidence.ts'; import type { Runner } from '../src/core/exec.ts'; -import { commentBody, postToPullRequest } from '../src/core/publish.ts'; +import { evidenceBlock, evidenceMarkers, postToPullRequest, summarize, withEvidenceBlock } from '../src/core/publish.ts'; import { Secret } from '../specs/support/secret.ts'; import { attach } from '../src/core/verbs.ts'; import { newRunId } from '../specs/support/inputs.ts'; import { openWorkspace } from '../src/core/workspace.ts'; -import type { BuildKey, EvidencePath, EvidenceRecord, HostAdapter, RunId } from '../src/core/types.ts'; +import type { BuildKey, EvidencePath, EvidenceRecord, EvidenceSummary, HostAdapter, RunId } from '../src/core/types.ts'; const OWN_USER = 'user_own'; @@ -116,14 +116,135 @@ describe('assertPublishable', () => { }); }); -describe('the comment attach posts', () => { +describe('the evidence block of a pull request description', () => { + const summary: EvidenceSummary = { run: 'r20261008-052713-253e' as RunId, platform: 'ios', device: 'iPhone Air on xcode-27', commit: '0f50b597b1c2d3e4f5a60718293a4b5c6d7e8f90', passed: 3, flaky: 0, total: 3 }; + const media = [{ alt: 'video.mp4', ref: './video.mp4' }, { alt: 'profile', ref: './profile.png' }]; + const block = evidenceBlock(summary, media); + + it('is one line built from the run, then each file alone in its own paragraph, between the markers of its platform', () => { + assert.equal( + block, + [ + '', + '', + 'verify run `r20261008-052713-253e` on `iPhone Air on xcode-27` at `0f50b597b1c2`, 3 of 3 passed.', + '', + '![video.mp4](./video.mp4)', + '', + '![profile](./profile.png)', + '', + '', + ].join('\n'), + ); + assert.match(evidenceBlock({ ...summary, passed: 1, flaky: 1, total: 2 }, []), /, 1 of 2 passed, 1 flaky \(passed only on a retry\)\.$/m); + }); + it('counts a test that passed only on a retry apart from the passed ones', () => { const { dir, run } = runDir(); const base = partialRecord(dir, run); const record = sealEvidence(dir, { ...base, results: [base.results[0]!, { ...base.results[0]!, title: 'b', status: 'flaky', attempts: 2, error: 'tap failed' }] }, []); - const body = commentBody(assertPublishable(record, [OWN_USER])); - assert.match(body, /, 1 of 2 passed, 1 flaky \(passed only on a retry\)\.$/m); - assert.match(body, /^- flaky: `specs\/explored\/a\.e2e\.ts` b$/m); + assert.deepEqual(summarize(assertPublishable(record, [OWN_USER])), { run, platform: 'ios', device: 'verify-ios-1', commit: 'abc', passed: 1, flaky: 1, total: 2 }); + }); + + it('goes after a description that has no block, and leaves that description as it was', () => { + for (const body of ['## Summary\r\n\r\nFixes the button. \r\n', 'no newline at the end', 'ends with one\n']) { + const edit = withEvidenceBlock(body, 'ios', block); + assert.ok(edit.ok && edit.was === 'added'); + assert.ok(edit.body.startsWith(body), JSON.stringify(body)); + assert.match(edit.body.slice(body.length), /^(\n{1,2}|(\r\n){1,2})/); + assert.ok(edit.body.replaceAll('\r\n', '\n').endsWith(block)); + } + assert.deepEqual(withEvidenceBlock('', 'ios', block), { ok: true, body: block, was: 'added' }); + }); + + it('replaces the block that is there and keeps every byte before and after it', () => { + const before = '## Summary\r\n\r\nText above. \n\n'; + const after = '\n\n- [x] a checklist\r\n\ttrailing\n'; + const old = evidenceBlock({ ...summary, run: 'r20261007-010101-aaaa' as RunId }, [{ alt: 'old', ref: 'https://github.com/user-attachments/assets/1' }]); + const edit = withEvidenceBlock(`${before}${old}${after}`, 'ios', block); + assert.deepEqual(edit, { ok: true, body: `${before}${block.replaceAll('\n', '\r\n')}${after}`, was: 'replaced' }); + assert.equal(edit.ok && edit.body.includes('r20261007-010101-aaaa'), false); + }); + + it('keeps the block of the other platform', () => { + const android = evidenceBlock({ ...summary, platform: 'android', device: 'Pixel 9' }, []); + const edit = withEvidenceBlock(`intro\n\n${android}\n`, 'ios', block); + assert.deepEqual(edit, { ok: true, body: `intro\n\n${android}\n\n${block}`, was: 'added' }); + }); + + it('takes a marker only when it is a whole line, so a description may quote one', () => { + const { start, end } = evidenceMarkers('ios'); + const quoting = `The block sits between \`${start}\` and \`${end}\`.\n\n ${start}\n indented, so not a marker\n ${end}\n`; + assert.deepEqual(withEvidenceBlock(quoting, 'ios', block), { ok: true, body: `${quoting}\n${block}`, was: 'added' }); + }); + + const bodyAfter = (body: string, lines: string = block): string => { + const edit = withEvidenceBlock(body, 'ios', lines); + assert.ok(edit.ok, body); + return edit.body; + }; + + it('writes the block with the line endings of the description, added or replaced', () => { + const crlf = block.replaceAll('\n', '\r\n'); + const old = evidenceBlock({ ...summary, run: 'r20261007-010101-aaaa' as RunId }, media); + const oldCrlf = old.replaceAll('\n', '\r\n'); + assert.deepEqual(withEvidenceBlock(`intro\r\n\r\n${oldCrlf}\r\noutro`, 'ios', block), { ok: true, body: `intro\r\n\r\n${crlf}\r\noutro`, was: 'replaced' }); + assert.equal(bodyAfter('intro\r\n'), `intro\r\n\r\n${crlf}`); + assert.equal(bodyAfter('one\r\ntwo'), `one\r\ntwo\r\n\r\n${crlf}`); + assert.equal(bodyAfter(`intro\n\n${old}\n`), `intro\n\n${block}\n`); + for (const body of [`intro\r\n\r\n${oldCrlf}\r\noutro`, 'intro\r\n']) assert.equal(/[^\r]\n/.test(bodyAfter(body)), false, 'no bare line feed'); + }); + + it('replaces the block that the verify-attach workflow writes for a handed-off run', () => { + const published = [ + '', + '', + 'verify run `r20261007-010101-aaaa`, as reported by [the session that ran it](https://github.com/clerk/clerk-ios/actions/runs/37731397352): 3 of 3 passed on `iPhone Air on xcode-27` at `0f50b597b1c2`.', + '', + '![video.mp4](https://github.com/user-attachments/assets/1)', + '', + '', + ].join('\n'); + assert.deepEqual(withEvidenceBlock(`intro\n\n${published}\n`, 'ios', block), { ok: true, body: `intro\n\n${block}\n`, was: 'replaced' }); + }); + + it('refuses to replace text between two markers that is not a block it wrote', () => { + const { start, end } = evidenceMarkers('ios'); + const notes = `${start}\nIMPORTANT reviewer notes that are not evidence\n${end}\n`; + assert.deepEqual(withEvidenceBlock(notes, 'ios', block), { ok: false, why: `has text between \`${start}\` and \`${end}\` that is not an evidence block` }); + }); + + it('takes no marker from inside a code fence, so a description may show a whole example block', () => { + const example = evidenceBlock({ ...summary, run: 'r20261007-010101-aaaa' as RunId }, media); + const later = evidenceBlock({ ...summary, run: 'r20261009-020202-bbbb' as RunId }, media); + const fences = [['```', '```'], ['```markdown', '```'], ['~~~~', '~~~~~'], [' ```', '``` '], ['```\n```not the end, it has text after the ticks', '```'], ['```\n~~~', '```'], ['````\n```', '````']] as const; + for (const [open, close] of fences) { + const documented = `The block looks like this:\n\n${open}\n${example}\n${close}\n`; + const added = withEvidenceBlock(documented, 'ios', block); + assert.deepEqual(added, { ok: true, body: `${documented}\n${block}`, was: 'added' }, open); + const replaced = withEvidenceBlock(bodyAfter(documented), 'ios', later); + assert.deepEqual(replaced, { ok: true, body: `${documented}\n${later}`, was: 'replaced' }, open); + } + }); + + it('still takes a marker after a fence that closed, and one beside a line that only looks like a fence', () => { + const real = evidenceBlock({ ...summary, run: 'r20261007-010101-aaaa' as RunId }, media); + for (const before of ['```\ncode\n```\n\n', 'Use ```three ticks``` inline.\n\n', '```three ticks``` that open a line are inline code too.\n\n', ' ```\n\n', '``\n\n']) { + assert.deepEqual(withEvidenceBlock(`${before}${real}\n`, 'ios', block), { ok: true, body: `${before}${block}\n`, was: 'replaced' }, JSON.stringify(before)); + } + }); + + it('refuses a description whose markers are doubled, halved, or out of order, and says which', () => { + const { start, end } = evidenceMarkers('ios'); + const refused = (body: string): string => { + const edit = withEvidenceBlock(body, 'ios', block); + assert.equal(edit.ok, false, body); + return edit.ok ? '' : edit.why; + }; + assert.equal(refused(`${block}\n\n${block}`), `has \`${start}\` 2 times and \`${end}\` 2 times`); + assert.equal(refused(`text\n${start}\nno end`), `has \`${start}\` 1 times and \`${end}\` 0 times`); + assert.equal(refused(`no start\n${end}\n`), `has \`${start}\` 0 times and \`${end}\` 1 times`); + assert.equal(refused(`${end}\n${start}`), `has \`${end}\` before \`${start}\``); }); }); @@ -140,15 +261,31 @@ describe('assertPublishable and the groups of a run', () => { describe('attach', () => { const host = { repo: 'clerk-ios', githubRepo: 'clerk/clerk-ios' } as HostAdapter; + const PR_URL = 'https://github.com/clerk/clerk-ios/pull/9'; - function recordingRunner(attachFlag = true): { runner: Runner; calls: (readonly string[])[] } { - const calls: (readonly string[])[] = []; - const runner: Runner = async (command, args) => { - if (args.includes('--help')) return { code: 0, stdout: attachFlag ? ' --attach file Attach a file\n' : ' -b, --body text The comment body text\n', stderr: '' }; - calls.push([command, ...args]); - return { code: 0, stdout: 'https://github.com/clerk/clerk-ios/pull/9#issuecomment-1\n', stderr: '' }; + function fakeGh(options: { readonly attachFlag?: boolean; readonly body?: string; readonly onView?: (views: number, pr: { body: string }) => void } = {}) { + const pr = { body: options.body ?? '## Summary\n\nFixes the button.\n' }; + const edits: { readonly args: readonly string[]; readonly cwd: string | undefined; readonly bodyFile: string }[] = []; + let views = 0; + const runner: Runner = async (command, args, runOptions) => { + assert.equal(command, 'gh'); + if (args.includes('--help')) return { code: 0, stdout: options.attachFlag === false ? ' -b, --body text Set the new body.\n' : ' --attach file Attach a file\n', stderr: '' }; + if (args[1] === 'view') { + views += 1; + options.onView?.(views, pr); + return { code: 0, stdout: JSON.stringify({ body: pr.body, url: PR_URL }), stderr: '' }; + } + const bodyFile = readFileSync(args[args.indexOf('--body-file') + 1]!, 'utf8'); + edits.push({ args, cwd: runOptions?.cwd, bodyFile }); + pr.body = bodyFile.replace(/\]\(\.\/[^)]+\)/g, '](https://github.com/user-attachments/assets/uploaded)'); + return { code: 0, stdout: `${PR_URL}\n`, stderr: '' }; }; - return { runner, calls }; + return { runner, pr, edits }; + } + + function publishableRun() { + const { dir, run } = runDir(); + return { dir, run, publishable: assertPublishable(sealEvidence(dir, partialRecord(dir, run), []), [OWN_USER]) }; } it('never calls gh for a run that fails the gate', async () => { @@ -158,43 +295,88 @@ describe('attach', () => { writeFileSync(join(dir, 'video.mp4'), 'x'); writeFileSync(join(dir, 'app.log'), hostLogLine('user_foreign')); sealEvidence(dir, partialRecord(dir, run), []); - const { runner, calls } = recordingRunner(); + const calls: string[] = []; + const runner: Runner = async (command) => (calls.push(command), { code: 0, stdout: '', stderr: '' }); const deps = { host, workspace, runner, env: {}, progress: () => undefined, instances: heldInstances() }; await assert.rejects(attach(deps, { verb: 'attach', run, pr: 9, screenshots: 'all' }), { code: 'EVIDENCE_UNSAFE' }); assert.equal(calls.length, 0); }); - it('posts once with --repo and --attach, then reports alreadyPosted', async () => { - const { dir, run } = runDir(); - const record = sealEvidence(dir, partialRecord(dir, run), []); - const publishable = assertPublishable(record, [OWN_USER]); - const { runner, calls } = recordingRunner(); - const first = await postToPullRequest(publishable, dir, host, 9, 'all', runner); - const second = await postToPullRequest(publishable, dir, host, 9, 'all', runner); - assert.equal(calls.length, 1); - const args = calls[0]!; - assert.deepEqual(args.slice(0, 6), ['gh', 'pr', 'comment', '9', '--repo', 'clerk/clerk-ios']); - assert.deepEqual(args.filter((_, i) => args[i - 1] === '--attach'), [join(dir, 'video.mp4'), join(dir, 'screenshots', 'profile.png')]); - assert.equal(first.alreadyPosted, false); - assert.equal(second.alreadyPosted, true); - assert.equal(second.commentUrl, 'https://github.com/clerk/clerk-ios/pull/9#issuecomment-1'); + it('puts the block after the description with gh pr edit, from the run directory, and uploads a run only once', async () => { + const { dir, run, publishable } = publishableRun(); + const gh = fakeGh(); + const first = await postToPullRequest(publishable, dir, host, 9, 'all', gh.runner); + const second = await postToPullRequest(publishable, dir, host, 9, 'all', gh.runner); + assert.equal(gh.edits.length, 1); + const edit = gh.edits[0]!; + assert.deepEqual(edit.args.slice(0, 5), ['pr', 'edit', '9', '--repo', 'clerk/clerk-ios']); + assert.deepEqual(edit.args.filter((_, i) => edit.args[i - 1] === '--attach'), ['./video.mp4', './screenshots/profile.png']); + assert.equal(edit.cwd, dir); + assert.equal( + edit.bodyFile, + `## Summary\n\nFixes the button.\n\n\n\nverify run \`${run}\` on \`verify-ios-1\` at \`abc\`, 1 of 1 passed.\n\n![video.mp4](./video.mp4)\n\n![profile](./screenshots/profile.png)\n\n`, + ); + assert.equal(existsSync(edit.args[edit.args.indexOf('--body-file') + 1]!), false, 'the copy of the description is not kept'); + assert.deepEqual(first, { verb: 'attach', prUrl: PR_URL, posted: [join(dir, 'video.mp4'), join(dir, 'screenshots', 'profile.png')], alreadyPosted: false }); + assert.deepEqual(second, { ...first, alreadyPosted: true }); assert.ok(existsSync(join(dir, 'posted-9.json'))); - const other = await postToPullRequest(publishable, dir, host, 10, 'all', runner); - assert.equal(other.alreadyPosted, false); - assert.equal(calls.length, 2); - assert.equal(calls[1]![3], '10'); + await postToPullRequest(publishable, dir, host, 10, 'all', gh.runner); + assert.equal(gh.edits[1]!.args[2], '10'); }); - it('posts nothing and names the fix when gh pr comment has no --attach', async () => { - const { dir, run } = runDir(); - const publishable = assertPublishable(sealEvidence(dir, partialRecord(dir, run), []), [OWN_USER]); - const { runner, calls } = recordingRunner(false); - await assert.rejects(postToPullRequest(publishable, dir, host, 9, 'all', runner), { + it('replaces the block of an earlier run, so a second run does not pile up media', async () => { + const earlier = publishableRun(); + const later = publishableRun(); + const gh = fakeGh(); + await postToPullRequest(earlier.publishable, earlier.dir, host, 9, 'all', gh.runner); + gh.pr.body += '\nReviewer note added below the block.\n'; + await postToPullRequest(later.publishable, later.dir, host, 9, ['profile'], gh.runner); + assert.equal(gh.pr.body.split('').length, 2); + assert.equal(gh.pr.body.includes(earlier.run), false); + assert.ok(gh.pr.body.includes(later.run)); + assert.ok(gh.pr.body.startsWith('## Summary\n\nFixes the button.\n\n')); + assert.ok(gh.pr.body.endsWith('\nReviewer note added below the block.\n')); + }); + + it('posts nothing and names the fix when gh pr edit has no --attach', async () => { + const { dir, run, publishable } = publishableRun(); + const gh = fakeGh({ attachFlag: false }); + await assert.rejects(postToPullRequest(publishable, dir, host, 9, 'all', gh.runner), { + code: 'NOT_READY', + message: `this gh has no \`gh pr edit --attach\`, so the video and screenshots of run ${run} cannot be posted`, + fix: 'install gh 2.99.0 or newer, whose `gh pr edit` has --attach', + }); + assert.deepEqual(gh.edits, []); + assert.equal(existsSync(join(dir, 'posted-9.json')), false); + }); + + it('builds on the newer description when someone edits it between the read and the write', async () => { + const { dir, publishable } = publishableRun(); + const gh = fakeGh({ onView: (views, pr) => void (views === 2 && (pr.body = 'Rewritten by a reviewer.\n')) }); + await postToPullRequest(publishable, dir, host, 9, 'all', gh.runner); + assert.equal(gh.edits.length, 1); + assert.ok(gh.edits[0]!.bodyFile.startsWith('Rewritten by a reviewer.\n\n')); + }); + + it('writes nothing when the description changes twice, and says so', async () => { + const { dir, run, publishable } = publishableRun(); + const gh = fakeGh({ onView: (views, pr) => void (pr.body = `edit ${views}\n`) }); + await assert.rejects(postToPullRequest(publishable, dir, host, 9, 'all', gh.runner), { code: 'NOT_READY', - message: `this gh has no \`gh pr comment --attach\`, so the video and screenshots of run ${run} cannot be posted`, - fix: 'install a gh build whose `gh pr comment` has --attach', + message: `the description of PR #9 changed twice while the evidence of run ${run} was being placed, so nothing was written`, }); - assert.deepEqual(calls, []); + assert.deepEqual(gh.edits, []); assert.equal(existsSync(join(dir, 'posted-9.json')), false); }); + + it('writes nothing to a description whose markers are broken, and says which', async () => { + const { dir, run, publishable } = publishableRun(); + const gh = fakeGh({ body: 'text\n\nthe end marker was deleted\n' }); + await assert.rejects(postToPullRequest(publishable, dir, host, 9, 'all', gh.runner), { + code: 'NOT_READY', + message: `the description of PR #9 has \`\` 1 times and \`\` 0 times, so the evidence of run ${run} has no one place to go`, + fix: 'leave one pair of those markers in the description, or none, then rerun', + }); + assert.deepEqual(gh.edits, []); + }); }); diff --git a/integration/expo-native/test/lease-flow.test.ts b/integration/expo-native/test/lease-flow.test.ts index 582f52e2908..616d1f8b863 100644 --- a/integration/expo-native/test/lease-flow.test.ts +++ b/integration/expo-native/test/lease-flow.test.ts @@ -110,8 +110,8 @@ describe('lease flow', () => { id: 'gh-attach', ok: true, state: 'warning', - detail: "this gh has no `gh pr comment --attach`, so `{cli} attach` cannot post a run's video and screenshots from this machine", - fix: 'install a gh build whose `gh pr comment` has --attach', + detail: "this gh has no `gh pr edit --attach`, so `{cli} attach` cannot put a run's video and screenshots in a pull request description from this machine", + fix: 'install gh 2.99.0 or newer, whose `gh pr edit` has --attach', }); }); diff --git a/integration/expo-native/test/run-groups.test.ts b/integration/expo-native/test/run-groups.test.ts index fe6427b2b71..c1df9d4c38a 100644 --- a/integration/expo-native/test/run-groups.test.ts +++ b/integration/expo-native/test/run-groups.test.ts @@ -9,7 +9,7 @@ import { exitCodeFor } from '../src/core/cli.ts'; import { assertPublishable, readRecord } from '../src/core/evidence.ts'; import { createInstances } from '../src/core/instances/instances.ts'; import { SettingsRefused, settingsFileOf } from '../src/core/instances/settings.ts'; -import { commentBody } from '../src/core/publish.ts'; +import { summarize } from '../src/core/publish.ts'; import { runVerb, type Deps } from '../src/core/verbs.ts'; import { openWorkspace } from '../src/core/workspace.ts'; import { VerifyFailure, type BuildKey, type Command, type DeviceBackend, type HostAdapter, type InstanceSettings, type LocalLease, type ScratchPath } from '../src/core/types.ts'; @@ -252,10 +252,7 @@ describe('a run whose spec files declare different settings', () => { assert.match(result.next, /^\{cli\} attach /); assert.equal(exitCodeFor(result), 0); - const comment = commentBody(assertPublishable(record, [])); - assert.ok(comment.includes(`Instance settings \`${ORG_LABEL}\` (declared by \`${CHOOSE_ORG}\`): 1 of 1 passed.`)); - assert.ok(comment.includes(`Instance settings \`${MFA_LABEL}\` (declared by \`${COMPLETE_MFA}\`): 2 of 2 passed.`)); - assert.equal(comment.includes('Instance settings `standard`'), false); + assert.deepEqual(summarize(assertPublishable(record, [])), { run: record.run, platform: 'ios', device: record.device, commit: record.gitHead, passed: 5, flaky: 0, total: 5 }); w.lines.length = 0; await runVerb(w.deps(), RUN_ALL); @@ -270,7 +267,6 @@ describe('a run whose spec files declare different settings', () => { assert.deepEqual(w.invocations().map((invocation) => invocation.output.split('/').at(-1)), ['e2e']); assert.equal(w.lines.some((line) => /groups in this run/.test(line)), false); assert.deepEqual(record.settings.map((group) => [group.label, group.changed, group.e2eReport]), [['standard', false, join(dir, 'e2e', 'report.json')]], 'the application was put on the standard file before the run drove, so the group changed nothing'); - assert.equal(commentBody(assertPublishable(record, [])).includes('Instance settings'), false); }); it('opens the Platform credential before it leases a device when the run will change settings', async () => { From 772dac32294a2b114dbab034a3bc92288d057596 Mon Sep 17 00:00:00 2001 From: Mike Pitre <12040919+mikepitre@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:32:26 -0400 Subject: [PATCH 2/5] test(expo): borrow a device on a CI runner for a remote session Co-Authored-By: Claude Opus 5.5 --- .changeset/expo-verify-borrowed-device.md | 2 + .github/workflows/verify-remote.yml | 228 ++++++++ integration/expo-native/src/core/MANIFEST | 16 +- integration/expo-native/src/core/launch.d.mts | 1 + integration/expo-native/src/core/launch.mjs | 41 ++ integration/expo-native/src/core/manifest.ts | 8 + .../expo-native/src/core/remote/backend.ts | 270 ++++++++++ .../expo-native/src/core/remote/github.ts | 280 ++++++++++ .../expo-native/src/core/remote/handoff.ts | 231 +++++++++ .../expo-native/src/core/remote/preflight.ts | 232 +++++++++ .../expo-native/src/core/remote/protocol.ts | 130 +++++ .../expo-native/src/core/remote/recipe.ts | 20 + .../src/core/remote/session-agent.ts | 488 ++++++++++++++++++ .../expo-native/src/core/remote/session.ts | 106 ++++ .../expo-native/src/core/remote/settings.ts | 83 +++ .../expo-native/src/core/remote/tunnel.ts | 12 + integration/expo-native/src/fixture.ts | 33 +- .../src/platform/android/emulator.ts | 17 + .../src/platform/android/session-lane.ts | 27 + .../src/platform/session-device.ts | 48 ++ 20 files changed, 2266 insertions(+), 7 deletions(-) create mode 100644 .changeset/expo-verify-borrowed-device.md create mode 100644 .github/workflows/verify-remote.yml create mode 100644 integration/expo-native/src/core/remote/backend.ts create mode 100644 integration/expo-native/src/core/remote/github.ts create mode 100644 integration/expo-native/src/core/remote/handoff.ts create mode 100644 integration/expo-native/src/core/remote/preflight.ts create mode 100644 integration/expo-native/src/core/remote/protocol.ts create mode 100644 integration/expo-native/src/core/remote/recipe.ts create mode 100644 integration/expo-native/src/core/remote/session-agent.ts create mode 100644 integration/expo-native/src/core/remote/session.ts create mode 100644 integration/expo-native/src/core/remote/settings.ts create mode 100644 integration/expo-native/src/core/remote/tunnel.ts create mode 100644 integration/expo-native/src/platform/android/session-lane.ts create mode 100644 integration/expo-native/src/platform/session-device.ts diff --git a/.changeset/expo-verify-borrowed-device.md b/.changeset/expo-verify-borrowed-device.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/expo-verify-borrowed-device.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/verify-remote.yml b/.github/workflows/verify-remote.yml new file mode 100644 index 00000000000..92a746d340b --- /dev/null +++ b/.github/workflows/verify-remote.yml @@ -0,0 +1,228 @@ +name: verify-remote +run-name: verify-remote ${{ inputs.owner }}/${{ inputs.session }} + +on: + workflow_dispatch: + inputs: + owner: + description: Random id of the driver's checkout, so it can find and end its own sessions + required: true + session: + description: Session id the driver chose + required: true + request: + description: The session request as one JSON string (src/core/remote/protocol.ts) + required: true + +permissions: + contents: read + +env: + PACKAGE: integration/expo-native + IOS_RUNTIME: com.apple.CoreSimulator.SimRuntime.iOS-26-5 + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + mode: ${{ steps.plan.outputs.mode }} + runner: ${{ steps.plan.outputs.runner }} + device: ${{ steps.plan.outputs.device }} + sha: ${{ steps.plan.outputs.sha }} + timeout: ${{ steps.plan.outputs.timeout }} + request: ${{ steps.plan.outputs.request }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + sparse-checkout: integration/expo-native/src/core + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24.15.0 + - name: Read the request + id: plan + env: + VERIFY_REQUEST: ${{ inputs.request }} + GITHUB_TOKEN: ${{ github.token }} + run: node "$PACKAGE/src/core/remote/session-agent.ts" plan + + session: + needs: plan + if: needs.plan.outputs.mode == 'session' + runs-on: ${{ needs.plan.outputs.runner }} + timeout-minutes: ${{ fromJSON(needs.plan.outputs.timeout) }} + env: + VERIFY_SESSION_REQUEST: ${{ needs.plan.outputs.request }} + PLATFORM: ${{ fromJSON(needs.plan.outputs.request).platform }} + DEVICE: ${{ needs.plan.outputs.device }} + # agent-device reaps its daemon, its XCTest runner, and its device lease after five idle minutes by default. + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0' + AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS: '0' + AGENT_DEVICE_LEASE_TTL_MS: '21600000' + AGENT_DEVICE_LEASE_MAX_TTL_MS: '21600000' + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ needs.plan.outputs.sha || github.sha }} + persist-credentials: false + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + if: needs.plan.outputs.device != '' + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24.15.0 + cache: ${{ needs.plan.outputs.device != '' && (env.PLATFORM == 'android' || startsWith(needs.plan.outputs.runner, 'blacksmith-')) && 'pnpm' || '' }} + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4 + if: needs.plan.outputs.device != '' && env.PLATFORM == 'android' + with: + distribution: temurin + java-version: 21 + - uses: gradle/actions/setup-gradle@4733eaac7c1b0da527e4206b7671e0061de1ce37 # v6.3.0 + if: needs.plan.outputs.device != '' && env.PLATFORM == 'android' + with: + # Sessions never run on main, so the default (write from main only) would never seed the cache. + cache-read-only: false + cache-disabled: ${{ !startsWith(needs.plan.outputs.runner, 'blacksmith-') }} + add-job-summary: never + + # Nothing ends an idle session before the agent starts, so each step up to it has its own limit. + - name: Make the work directory and name the device + id: device + timeout-minutes: 15 + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/verify-remote" + echo "VERIFY_SESSION_WORK=$RUNNER_TEMP/verify-remote" >> "$GITHUB_ENV" + if [ -z "$DEVICE" ]; then exit 0; fi + if [ "$PLATFORM" = ios ]; then + xcodebuild -version + if ! xcrun simctl list runtimes available -j | jq -e --arg id "$IOS_RUNTIME" '.runtimes[] | select(.identifier == $id)' > /dev/null; then + echo "::error::This runner image has no $IOS_RUNTIME. It has: $(xcrun simctl list runtimes available -j | jq -r '[.runtimes[].identifier] | join(", ")'). Change IOS_RUNTIME in this workflow on purpose, or choose another runner." + exit 1 + fi + ID="$(xcrun simctl create "$DEVICE 1" "$DEVICE" "$IOS_RUNTIME")" + xcrun simctl boot "$ID" + echo "$DEVICE 1 on $IOS_RUNTIME: $ID" + echo "id=$ID" >> "$GITHUB_OUTPUT" + else + [ -x "$ANDROID_HOME/platform-tools/adb" ] || "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install platform-tools < /dev/null > /dev/null + echo "$ANDROID_HOME/platform-tools" >> "$GITHUB_PATH" + echo "id=$(node "$PACKAGE/src/platform/android/session-lane.ts" serial)" >> "$GITHUB_OUTPUT" + fi + + - name: Start the session agent and tunnel + id: tunnel + timeout-minutes: 20 + env: + VERIFY_SESSION_DEVICE_ID: ${{ steps.device.outputs.id }} + VERIFY_SESSION_DEVICE_NAME: ${{ needs.plan.outputs.device }} + VERIFY_SESSION_DEVICE_MODULE: integration/expo-native/src/platform/session-device.ts + GITHUB_TOKEN: ${{ github.token }} + CLOUDFLARED_VERSION: '2026.9.3' + run: | + set -euo pipefail + WORK="$VERIFY_SESSION_WORK" + case "$(uname -s)-$(uname -m)" in + Darwin-arm64) ASSET=cloudflared-darwin-arm64.tgz; SUM=587c2cfb1c230fe36c7fa7727da78be459dae028cabe8c001291999350f07095 ;; + Linux-x86_64) ASSET=cloudflared-linux-amd64; SUM=77e26d8d900e0b8469f416239d14b5f296525fdf79fee6f511ef55609e3fbac2 ;; + *) echo "::error::no cloudflared build for $(uname -s)-$(uname -m)"; exit 1 ;; + esac + curl -fsSL --max-time 300 --retry 3 --retry-delay 2 --retry-all-errors -o "$WORK/$ASSET" "https://github.com/cloudflare/cloudflared/releases/download/$CLOUDFLARED_VERSION/$ASSET" + echo "$SUM $WORK/$ASSET" | shasum -a 256 -c - + if [ "${ASSET%.tgz}" != "$ASSET" ]; then tar -xzf "$WORK/$ASSET" -C "$WORK"; else mv "$WORK/$ASSET" "$WORK/cloudflared"; fi + chmod +x "$WORK/cloudflared" + VERIFY_SESSION_CLOUDFLARED="$WORK/cloudflared" \ + nohup node "$PACKAGE/src/core/remote/session-agent.ts" serve > "$WORK/agent.log" 2>&1 & + disown + for _ in $(seq 1 600); do + [ -s "$WORK/tunnel" ] && break + sleep 1 + done + [ -s "$WORK/tunnel" ] || { echo "::error::no tunnel came up"; tail -20 "$WORK/agent.log" "$WORK/tunnel.log" || true; exit 1; } + echo "host=$(cat "$WORK/tunnel")" >> "$GITHUB_OUTPUT" + + # GitHub's jobs API returns a step's name while the job still runs, so the tunnel host travels in this name. + - name: verify-remote tunnel ${{ steps.tunnel.outputs.host }} + run: 'true' + + # The agent is already building the app, so the system image download and the boot overlap that build. + - name: Start the emulator + if: steps.device.outputs.id != '' && env.PLATFORM == 'android' + timeout-minutes: 15 + run: | + set -euo pipefail + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules > /dev/null + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + IMAGE="system-images;android-36;google_apis;x86_64" + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install "$IMAGE" emulator < /dev/null > "$VERIFY_SESSION_WORK/sdkmanager.log" 2>&1 \ + || { echo "::error::sdkmanager could not install $IMAGE"; tail -20 "$VERIFY_SESSION_WORK/sdkmanager.log"; exit 1; } + nohup node "$PACKAGE/src/platform/android/session-lane.ts" boot "$VERIFY_SESSION_WORK" > "$VERIFY_SESSION_WORK/lane.log" 2>&1 & + disown + + - name: Install the package's pinned tools + timeout-minutes: 10 + run: | + set -euo pipefail + npm ci --prefix "$PACKAGE" --no-audit --no-fund + "$PACKAGE/node_modules/.bin/agent-device" --version + + - name: Wait for the simulator + if: steps.device.outputs.id != '' && env.PLATFORM == 'ios' + timeout-minutes: 15 + env: + CLERK_TEST_DEVICES: ${{ steps.device.outputs.id }} + run: | + xcrun simctl bootstatus "$CLERK_TEST_DEVICES" -b + integration/expo-native/bin/boot-ios-simulators.sh wait + + - name: Wait for the emulator + if: steps.device.outputs.id != '' && env.PLATFORM == 'android' + timeout-minutes: 6 + run: | + until [ -f "$VERIFY_SESSION_WORK/lane-ready" ]; do + if [ -f "$VERIFY_SESSION_WORK/lane-failed" ]; then + echo "::error::the emulator did not boot: $(cat "$VERIFY_SESSION_WORK/lane-failed")" + tail -20 "$HOME/.verify/emulators/android-1.log" || true + exit 1 + fi + sleep 2 + done + cat "$VERIFY_SESSION_WORK/lane.log" + + - name: Prepare the XCTest runner + if: steps.device.outputs.id != '' && env.PLATFORM == 'ios' + timeout-minutes: 20 + run: | + "$PACKAGE/node_modules/.bin/agent-device" prepare ios-runner --platform ios --timeout 900000 || echo "::warning::prepare ios-runner failed; the first snapshot pays for it" + + - name: Mark the device ready + run: touch "$VERIFY_SESSION_WORK/device-ready" + + - name: Hold the session + run: | + set -uo pipefail + while [ ! -f "$VERIFY_SESSION_WORK/ended" ]; do + pgrep -f 'session-agent.ts serve' > /dev/null || [ -f "$VERIFY_SESSION_WORK/ended" ] || { echo "::error::the session agent died"; tail -30 "$VERIFY_SESSION_WORK/agent.log"; exit 1; } + sleep 3 + done + echo "session ended: $(cat "$VERIFY_SESSION_WORK/ended")" + + - name: Shut down + if: always() + env: + ID: ${{ steps.device.outputs.id }} + run: | + pkill -f 'session-agent.ts serve' || true + if [ -z "$ID" ]; then exit 0; fi + if [ "$PLATFORM" = ios ]; then xcrun simctl shutdown "$ID" || true; else adb -s "$ID" emu kill || true; fi + + - name: Keep the evidence that attach handed off + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: verify-evidence + path: ${{ runner.temp }}/verify-remote/evidence/ + if-no-files-found: ignore + retention-days: 3 + compression-level: 0 diff --git a/integration/expo-native/src/core/MANIFEST b/integration/expo-native/src/core/MANIFEST index ecf2473820e..6e3bd68d738 100644 --- a/integration/expo-native/src/core/MANIFEST +++ b/integration/expo-native/src/core/MANIFEST @@ -30,11 +30,21 @@ cf6aa68be73e4abaf3b63d5d84c2e9eee6197412f607ec53d0f05f7b058db420 src/core/insta 26c8c052d7a87ff5401cd3dfc61ae7b1acde0966a3675e92567b9afde0ea96c6 src/core/instances/settings.ts 7272293ca6be682c1a049ac7ee9fd54255721231b7da0fe11af5bbeb50e7e712 src/core/instances/throwaway.ts 39a849e9c07ce23e8a1be9f50a136690deba3cd22b153d66cc3f9f2f53d88576 src/core/keys.ts -72e7a64c6974afdc4612084d2e838075a9ce0edb6c676b23c266b8026400badc src/core/launch.d.mts -1287fc7e3328699e104c7bc50d3cd9bf9fbfec9efe764592c98e90e56033273d src/core/launch.mjs +98cca3ad8aaa142687d3f00b7d2787d6334ab3bf0755ebac8087e036f7b16442 src/core/launch.d.mts +a87e700fe75c15c8ae53a5b30fdd100de850d9f8e485b08e4a3b04c623438f74 src/core/launch.mjs a0ee9e4bfd34d4659685108e5e54eb1ec2aff6248236c358e6d3b14b2e2ff6a0 src/core/ledgers.ts -8d533755b21beb1e1d74d160e2740c0b1c4c4b4bc0c33f8fac75083ffeb03e75 src/core/manifest.ts +afab94a970286c40d6857b03ae4fa39ec59786274beacf6f1e77f6530bf46c54 src/core/manifest.ts 15ca07dce20c94a37cb6388ccf7ffb124e8a75592ea6b5904909cd4b66210d62 src/core/publish.ts +f62b6fc3229e40df310bab210b666bd4b234ddb58e4fd71f276d4c42da47cc7a src/core/remote/backend.ts +37fc42707be7d8bd5f0313a664a8465287ebb0970c520c0e0617f6ff6c9a614e src/core/remote/github.ts +2ed83b089a07969efe4369e027b8840f4c4b7b8fe055058fb22f44cf289618d6 src/core/remote/handoff.ts +9142ea638a06e526a0b626ebfb64c96420083194d6332e1e883fa1e7a39e5f3d src/core/remote/preflight.ts +d5017e4088089e941687c3890d740f0e093a8fbcd72f9816b998ae9f6891fe16 src/core/remote/protocol.ts +5a4ee0e7ed20feaeb711b1527083299b432d3fc0b7f428b636678a8a2f57581f src/core/remote/recipe.ts +263c0dc6174a4b39723d3b0dca3b8f76bc7d9d7bf7e512b08f8680b9a00618c6 src/core/remote/session-agent.ts +12df6d34398f8d1dce753839be00a0dc50f2c6de00638e4e4a43f8570912b403 src/core/remote/session.ts +d614ee5c98a85d15037b744a67404496273f62985fd7d6a049cacfd5ab335503 src/core/remote/settings.ts +e734336ea3b345057b69193790e2efce9a662c67ec5b4a426c811f999b45ccdb src/core/remote/tunnel.ts 685eabf010c7f63d076cfd3da1a681b5cc8024720538c1bf3a99792b563d8b60 src/core/slot.ts a8838aaf1fa9e4a6bc4e350ea674079201da76c5962656039b96db2505591aed src/core/state.ts 45ab2e9984266fb21959752606ed0318fe6d2c32e3fd651e852ca02dfb93ec76 src/core/types.ts diff --git a/integration/expo-native/src/core/launch.d.mts b/integration/expo-native/src/core/launch.d.mts index 2b51fed2c67..0675718b23b 100644 --- a/integration/expo-native/src/core/launch.d.mts +++ b/integration/expo-native/src/core/launch.d.mts @@ -3,3 +3,4 @@ export const AGENT_CREDENTIAL_VARIABLES: readonly string[]; export const INSTANCE_SECRET_VARIABLES: readonly string[]; export function supportsNode(version: string): boolean; export function ensureRuntime(): Promise; +export function chooseEgress(observed: { readonly proxyListens: boolean; readonly directConnects: boolean; readonly tokenStatusDirect: number | null }): { readonly proxy: boolean; readonly why: string }; diff --git a/integration/expo-native/src/core/launch.mjs b/integration/expo-native/src/core/launch.mjs index 448f47970c4..2de3c598d87 100644 --- a/integration/expo-native/src/core/launch.mjs +++ b/integration/expo-native/src/core/launch.mjs @@ -1,5 +1,6 @@ import { spawn, spawnSync } from 'node:child_process'; import { existsSync } from 'node:fs'; +import net from 'node:net'; import { delimiter, dirname } from 'node:path'; export const PLATFORM_CREDENTIAL_VARIABLES = ['CLERK_PLATFORM_API_KEY', 'CLERK_PLATFORM_API_KEY_FILE', 'VERIFY_PLATFORM_KEY_REFERENCE']; @@ -20,6 +21,19 @@ function rerun(command, args, env) { }); } +function accepts(host, port) { + return new Promise((resolve) => { + const socket = net.connect({ host, port, timeout: 1500 }); + const done = (ok) => { + socket.destroy(); + resolve(ok); + }; + socket.on('connect', () => done(true)); + socket.on('timeout', () => done(false)); + socket.on('error', () => done(false)); + }); +} + export async function ensureRuntime() { if (!supportsNode(process.versions.node)) { const withoutKey = Object.fromEntries(Object.entries(process.env).filter(([name]) => !PLATFORM_CREDENTIAL_VARIABLES.includes(name) && !AGENT_CREDENTIAL_VARIABLES.includes(name) && !INSTANCE_SECRET_VARIABLES.includes(name))); @@ -34,4 +48,31 @@ export async function ensureRuntime() { else console.error(`FAIL node ${message}\n fix: ${fix}`); process.exit(3); } + + const proxy = process.env.HTTPS_PROXY || process.env.https_proxy; + if (!proxy || process.env.NODE_USE_ENV_PROXY !== undefined) return; + let url; + try { + url = new URL(proxy); + } catch { + return; + } + const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; + const choice = chooseEgress({ + proxyListens: await accepts(url.hostname, Number(url.port || 80)), + directConnects: await accepts('api.github.com', 443), + tokenStatusDirect: token ? await fetch('https://api.github.com/rate_limit', { headers: { Authorization: `Bearer ${token}`, 'User-Agent': 'verify-remote' }, signal: AbortSignal.timeout(5000) }).then((response) => response.status, () => 0) : null, + }); + process.env.VERIFY_EGRESS_WHY = choice.why; + if (!choice.proxy) return; + const local = 'localhost,127.0.0.1,::1'; + const noProxy = process.env.NO_PROXY || process.env.no_proxy; + process.exit(await rerun(process.execPath, process.argv.slice(1), { ...process.env, NODE_USE_ENV_PROXY: '1', NO_PROXY: noProxy ? `${noProxy},${local}` : local })); +} + +export function chooseEgress({ proxyListens, directConnects, tokenStatusDirect }) { + if (!proxyListens) return { proxy: false, why: 'HTTPS_PROXY is set but nothing accepts connections there' }; + if (!directConnects || tokenStatusDirect === 0) return { proxy: true, why: 'a direct connection to api.github.com fails' }; + if (tokenStatusDirect === 401) return { proxy: true, why: "GitHub rejects this machine's token on a direct connection, so the proxy is where the real credential is added" }; + return { proxy: false, why: tokenStatusDirect === null ? 'a direct connection to api.github.com works' : 'a direct connection to api.github.com works and GitHub accepts the token on it' }; } diff --git a/integration/expo-native/src/core/manifest.ts b/integration/expo-native/src/core/manifest.ts index 7ee6f652a28..5f21f564b8a 100644 --- a/integration/expo-native/src/core/manifest.ts +++ b/integration/expo-native/src/core/manifest.ts @@ -37,6 +37,14 @@ export function manifestDrift(): readonly string[] { return [...names].filter((name) => committed.get(name) !== actual.get(name)).sort(); } +export function coreVersion(): string { + try { + return createHash('sha256').update(readFileSync(MANIFEST_FILE)).digest('hex').slice(0, 12); + } catch { + return 'unknown'; + } +} + if (import.meta.main && process.argv.includes('--write')) { writeFileSync(MANIFEST_FILE, computeManifest()); } diff --git a/integration/expo-native/src/core/remote/backend.ts b/integration/expo-native/src/core/remote/backend.ts new file mode 100644 index 00000000000..7e662bddc83 --- /dev/null +++ b/integration/expo-native/src/core/remote/backend.ts @@ -0,0 +1,270 @@ +import { createWriteStream, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { ReadableStream } from 'node:stream/web'; +import { run, sleep } from '../exec.ts'; +import { VerifyFailure, type AcquireRequest, type DeviceBackend, type EvidencePath, type Recording, type RemoteLease } from '../types.ts'; +import { apiMessage, currentBranch, endSession, liveRuns, openGitHub, startRun, viewRun, waitForStep, waitReporter, type GitHub } from './github.ts'; +import { HANDOFF_VERSION, HandoffRefused, describeFile, parseHandoffManifest, pullRequestMismatch, type HandoffManifest } from './handoff.ts'; +import { coreVersion } from '../manifest.ts'; +import { withEvidenceBlock } from '../publish.ts'; +import { remoteDoctorChecks } from './preflight.ts'; +import { STEP, type SessionHealth } from './protocol.ts'; +import { firstHealth, sendEvidence, sessionCall, sessionHealth, tunnelUrl, type SessionRef } from './session.ts'; +import { forgetSession, newSessionRequest, saveToken, savedDriverId, type RemoteDeps, type RemoteSettings } from './settings.ts'; + +const READY_DEADLINE_MS = 40 * 60_000; +const EXPIRING_MS = 2 * 60_000; + +const COMMITS_PER_PAGE = 100; +const COMMIT_PAGES = 3; + +async function pullRequestCommits(hub: GitHub, pr: number): Promise { + const shas: string[] = []; + for (let page = 1; page <= COMMIT_PAGES; page += 1) { + const listed = await hub.api('GET', `/pulls/${pr}/commits?per_page=${COMMITS_PER_PAGE}&page=${page}`); + if (listed.status !== 200 || !Array.isArray(listed.json)) throw new VerifyFailure('NOT_READY', `could not read the commits of PR #${pr}: ${apiMessage(listed)}`, 'retry {cli} attach'); + shas.push(...(listed.json as readonly { readonly sha: string }[]).map((commit) => commit.sha)); + if (listed.json.length < COMMITS_PER_PAGE) break; + } + return shas; +} + +export function remoteBackend(settings: RemoteSettings, deps: RemoteDeps = { env: process.env, runner: run }): DeviceBackend { + const { platform } = settings; + let opened: Promise | undefined; + const github = () => (opened ??= deps.github?.() ?? openGitHub({ repo: settings.repo, workflow: settings.workflow, env: deps.env, runner: deps.runner })); + const runUrl = (runId: string) => `https://github.com/${settings.repo}/actions/runs/${runId}`; + + function assertSameCore(health: SessionHealth): void { + const mine = coreVersion(); + if (health.core === mine) return; + throw new VerifyFailure( + 'NOT_READY', + `the session runs verify core ${health.core} and this checkout has ${mine}, so its agent may not have the routes this CLI calls`, + 'commit and push the changes to the files MANIFEST lists, then {cli} down and {cli} up', + ); + } + + function describeBuild(health: SessionHealth): string { + const build = health.build; + if (build.state === 'none') return 'no build'; + return `build ${build.sha.slice(0, 12)} ${build.state}${build.state === 'building' ? ` ${build.seconds}s` : ''}`; + } + + async function waitForBuild(lease: RemoteLease, sha: string, progress: (line: string) => void): Promise> { + const deadline = Date.now() + READY_DEADLINE_MS; + let last = ''; + for (;;) { + const health = await sessionHealth(lease); + if (health === null) { + const state = await viewRun(await github(), lease.providerRef); + if (state.status === 'completed') throw new VerifyFailure('LEASE_LOST', `the session ended (${state.conclusion ?? 'no conclusion'}) while it was building`, `read ${runUrl(lease.providerRef)} for why, then {cli} up`); + } else { + if (health.build.state === 'failed' && health.build.sha === sha) { + throw new VerifyFailure('BUILD_FAILED', `the session could not build ${sha.slice(0, 12)}:\n${health.build.tail}`, 'fix the build error above, commit, push, then rerun {cli} up'); + } + if (health.build.state === 'built' && health.build.sha === sha && health.device?.ready === true && health.daemon) return health.build; + const line = `wait ${describeBuild(health)}; device ${health.device?.ready === true ? 'ready' : 'booting'}; agent-device ${health.daemon ? 'up' : 'starting'}`; + const coarse = line.replace(/ \d+s/, ''); + if (coarse !== last) progress(line); + last = coarse; + } + if (Date.now() >= deadline) { + await endSession(await github(), lease.providerRef, lease).catch(() => undefined); + throw new VerifyFailure('NOT_READY', `the session was not ready ${READY_DEADLINE_MS / 60_000} minutes after the build was asked for (${health === null ? 'it does not answer' : describeBuild(health)}), so it was ended`, `read ${runUrl(lease.providerRef)}, then {cli} up`); + } + await sleep(8000); + } + } + + const backend: DeviceBackend = { + kind: 'remote', + platform, + requirement: settings.requirement, + availability: () => ({ usable: true, why: `the device runs on a CI runner (${settings.runner} unless --runner names another), started through ${settings.workflow} on ${settings.repo}` }), + + async sourceCommit(input) { + const status = await deps.runner('git', ['status', '--porcelain', '--', ...input.inputs], { cwd: input.worktree }); + const files = status.stdout.split('\n').filter((line) => line.length > 3).map((line) => line.slice(3)); + if (files.length > 0) { + throw new VerifyFailure( + 'BUILD_FAILED', + `a remote session builds a pushed commit, and this tree has uncommitted changes to the app: ${files.slice(0, 5).join(', ')}${files.length > 5 ? `, and ${files.length - 5} more` : ''}`, + 'git commit the changes and git push, then rerun', + ); + } + const sha = (await deps.runner('git', ['rev-parse', 'HEAD'], { cwd: input.worktree })).stdout.trim(); + const found = await (await github()).api('GET', `/commits/${sha}`); + if (found.status !== 200) { + throw new VerifyFailure('BUILD_FAILED', `a remote session builds a pushed commit, and GitHub does not have ${sha.slice(0, 12)} (${found.status})`, 'git push, then rerun'); + } + return sha; + }, + + async acquire(request: AcquireRequest) { + const hub = await github(); + const ref = await currentBranch(deps.runner, request.worktree); + const { request: order, token } = newSessionRequest(settings, deps, { ...(request.runner === undefined ? {} : { runner: request.runner }), device: settings.device, sha: request.app.source === 'local' ? null : request.app.sourceSha }); + const tokenFile = saveToken(settings, order.session, token); + request.progress(`device remote ${platform} starting session ${order.session} on ${order.runner} (idle stop ${order.idleMinutes} min, cap ${order.capMinutes} min)`); + let runId: string | null = null; + let session: SessionRef | null = null; + try { + runId = await startRun(hub, order, { ref }); + request.progress(`device remote ${platform} run ${runId} ${runUrl(runId)}`); + const host = await waitForStep(hub, runId, STEP.tunnelPattern, 20 * 60, waitReporter(request.progress, runId, { plan: settings.plumbingRunner, session: order.runner })); + session = { baseUrl: tunnelUrl(host), tokenFile }; + const health = await firstHealth(session, 180); + if (health === null || health.device === null) throw new VerifyFailure('NOT_READY', `the session's tunnel at ${host} did not answer with a device`, `read ${runUrl(runId)}`); + assertSameCore(health); + request.progress(`device remote ${platform} tunnel up, ${health.device.name} on ${order.runner}`); + return { + backend: 'remote', + provider: 'github-actions', + platform, + session: order.session, + providerRef: runId, + baseUrl: session.baseUrl, + tokenFile, + deviceId: health.device.id, + deviceName: health.device.name, + runner: order.runner, + expiresAt: health.capAt, + builtSha: null, + acquiredAt: new Date().toISOString(), + installedBuild: null, + }; + } catch (error) { + const ended = runId === null ? null : await endSession(hub, runId, session).catch((failure: Error) => ({ problem: failure.message })); + forgetSession(settings, order.session); + if (runId === null || ended === null || ended.problem === null) throw error; + throw new VerifyFailure('NOT_READY', `${(error as Error).message}; and the run it started could not be ended: ${ended.problem}`, `open ${runUrl(runId)} and cancel it`); + } + }, + + async check(lease) { + if (!existsSync(lease.tokenFile)) return 'lost'; + let health = await sessionHealth(lease); + for (let i = 0; i < 3 && health === null; i += 1) { + await sleep(3000); + health = await sessionHealth(lease); + } + if (health === null) { + const state = await viewRun(await github(), lease.providerRef); + if (state.status === 'completed') return 'lost'; + throw new VerifyFailure('NOT_READY', `the session is still running (${runUrl(lease.providerRef)}) but its tunnel does not answer`, 'retry in a minute; if it stays unreachable, {cli} down, then {cli} up'); + } + if (health.ending !== null) return 'lost'; + assertSameCore(health); + return Date.parse(lease.expiresAt) - Date.now() < EXPIRING_MS ? 'expiring' : 'held'; + }, + + async install(lease, app, progress) { + if (app.source === 'local') throw new VerifyFailure('BUILD_FAILED', 'a remote session builds a pushed commit and this build names none', '{cli} down, then {cli} up'); + const sha = app.sourceSha; + const started = await sessionCall(lease, `/__sim/build?sha=${sha}`, { method: 'POST' }); + if (started.status !== 202) throw new VerifyFailure('BUILD_FAILED', `the session refused to build ${sha.slice(0, 12)}: ${started.status} ${await started.text()}`, '{cli} down, then {cli} up'); + const built = await waitForBuild(lease, sha, progress); + progress(`build ${app.key} github-actions ${sha.slice(0, 12)} ${built.incremental ? 'rebuilt' : 'built'} in ${built.seconds}s on ${lease.runner}`); + return { ...lease, builtSha: sha }; + }, + + async release(lease) { + const ended = await endSession(await github(), lease.providerRef, lease.baseUrl === '' ? null : lease); + if (ended.problem !== null) throw new VerifyFailure('NOT_READY', ended.problem, `open ${runUrl(lease.providerRef)} and cancel it, then {cli} down`); + forgetSession(settings, lease.session); + }, + + async reapable(owner) { + const mine = savedDriverId(settings); + if (owner === undefined || mine === null) return []; + const running = await github().then(liveRuns).catch(() => []); + return running + .filter((live) => live.owner === mine && live.session.startsWith(platform)) + .map((live) => ({ + backend: 'remote' as const, + provider: 'github-actions' as const, + platform, + session: live.session, + providerRef: live.runId, + baseUrl: '', + tokenFile: join(settings.sessionsDir, live.session, 'token'), + deviceId: '', + deviceName: settings.device, + runner: '', + expiresAt: live.createdAt, + builtSha: null, + acquiredAt: live.createdAt, + installedBuild: null, + })); + }, + + async startRecording(lease, into) { + const started = await sessionCall(lease, '/__sim/record/start', { method: 'POST' }); + if (started.status !== 200) throw new VerifyFailure('NOT_READY', `the session did not start recording: ${started.status} ${await started.text()}`, 'rerun with --no-video'); + const file = join(into, 'video.mp4') as EvidencePath; + const recording: Recording = { + process: null, + async stop() { + const stopped = await sessionCall(lease, '/__sim/record/stop', { method: 'POST', timeoutMs: 90_000 }); + if (stopped.status !== 200) return file; + const download = await sessionCall(lease, '/__sim/record/file', { timeoutMs: 300_000 }); + if (download.status === 200 && download.body !== null) await pipeline(Readable.fromWeb(download.body as ReadableStream), createWriteStream(file)); + return file; + }, + }; + return recording; + }, + + async logs(lease, since, extraPredicate) { + const params = new URLSearchParams({ since: since.toISOString() }); + if (extraPredicate !== undefined) params.set('predicate', extraPredicate); + const response = await sessionCall(lease, `/__sim/logs?${params}`, { timeoutMs: 120_000 }); + return response.status === 200 ? response.text() : `the session returned no logs (${response.status})`; + }, + + async handOffEvidence(lease, bundle, progress) { + const { run } = bundle.summary; + const wouldNot = (why: string, fix: string): VerifyFailure => new VerifyFailure('NOT_READY', `the verify-attach workflow would not put the evidence of run ${run} on PR #${bundle.pr}: ${why}`, fix); + if (bundle.files.length === 0) throw wouldNot('the run has no video and no screenshot, and the workflow publishes nothing without a file', '{cli} run again without --no-video, then attach'); + const health = await sessionHealth(lease); + if (health === null || health.ending !== null) { + throw new VerifyFailure('LEASE_LOST', `the session (${runUrl(lease.providerRef)}) ${health === null ? 'does not answer' : 'is ending'}, so it cannot take the evidence of run ${run}`, '{cli} up, run again, and attach before {cli} down'); + } + const hub = await github(); + const [session, pull] = await Promise.all([hub.api('GET', `/actions/runs/${lease.providerRef}`), hub.api('GET', `/pulls/${bundle.pr}`)]); + if (session.status !== 200) throw new VerifyFailure('NOT_READY', `could not read run ${lease.providerRef}: ${apiMessage(session)}`, `open ${runUrl(lease.providerRef)}`); + if (pull.status !== 200) throw new VerifyFailure('NOT_READY', `could not read PR #${bundle.pr} of ${settings.repo}: ${apiMessage(pull)}`, 'pass the number of an open pull request of this branch as --pr'); + const started = session.json as { readonly head_branch: string; readonly head_sha: string }; + const pr = pull.json as { readonly state: string; readonly body: string | null; readonly commits: number; readonly head: { readonly ref: string; readonly sha: string; readonly repo: { readonly full_name: string } | null } }; + const mismatch = pullRequestMismatch( + { state: pr.state, headRepo: pr.head.repo?.full_name ?? null, headRef: pr.head.ref, commits: await pullRequestCommits(hub, bundle.pr), commitCount: pr.commits }, + { repo: settings.repo, headBranch: started.head_branch, startedOn: started.head_sha, runCommit: bundle.summary.commit }, + ); + if (mismatch !== null) throw wouldNot(mismatch.why, mismatch.fix); + const place = withEvidenceBlock(pr.body ?? '', bundle.summary.platform, ''); + if (!place.ok) throw wouldNot(`its description ${place.why}, so the evidence has no one place to go`, `${place.fix}, then rerun {cli} attach`); + let manifest: HandoffManifest; + try { + manifest = parseHandoffManifest(JSON.stringify({ v: HANDOFF_VERSION, pr: bundle.pr, ...bundle.summary, files: bundle.files.map((file) => describeFile(file.name, file.path)) })); + } catch (error) { + if (!(error instanceof HandoffRefused)) throw error; + throw new VerifyFailure('NOT_READY', `the evidence of run ${run} cannot be handed to the runner: ${error.message}`, 'attach fewer or smaller files with --screenshot, or rerun with --no-video'); + } + try { + await sendEvidence(lease, manifest, new Map(bundle.files.map((file) => [file.name, file.path])), progress); + } catch (error) { + if (error instanceof VerifyFailure) throw error; + throw new VerifyFailure('NOT_READY', `the hand-off of run ${run} failed: ${(error as Error).message}`, 'run {cli} attach again while the session is up'); + } + return { sessionRun: lease.providerRef, sessionRunUrl: runUrl(lease.providerRef) }; + }, + + describe: (lease) => `${lease.deviceName} on ${lease.runner === '' ? `run ${lease.providerRef}` : lease.runner}`, + + doctorChecks: (options) => remoteDoctorChecks(settings, deps, github, options), + }; + return backend; +} diff --git a/integration/expo-native/src/core/remote/github.ts b/integration/expo-native/src/core/remote/github.ts new file mode 100644 index 00000000000..f37d4658373 --- /dev/null +++ b/integration/expo-native/src/core/remote/github.ts @@ -0,0 +1,280 @@ +import { existsSync } from 'node:fs'; +import type { Runner } from '../exec.ts'; +import { sleep } from '../exec.ts'; +import { Secret } from '../../../specs/support/secret.ts'; +import { VerifyFailure } from '../types.ts'; +import { RUN_TITLE, type SessionRequest } from './protocol.ts'; +import { sessionCall, type SessionRef } from './session.ts'; + +export type TokenSource = 'GH_TOKEN' | 'GITHUB_TOKEN' | 'gh' | 'none'; + +export interface ApiResponse { + readonly status: number; + readonly json: unknown; + readonly headers: Headers; + readonly firstRefusalWithToken?: string; +} + +export interface GitHub { + readonly repo: string; + readonly workflow: string; + readonly tokenSource: TokenSource; + api(method: 'GET' | 'POST', path: string, body?: unknown): Promise; +} + +export interface GitHubOptions { + readonly repo: string; + readonly workflow: string; + readonly env: Readonly>; + readonly runner: Runner; + readonly retryDelayMs?: number; +} + +const TRANSIENT_RETRIES = 3; + +export async function openGitHub(options: GitHubOptions): Promise { + const { env } = options; + let tokenSource: TokenSource = 'none'; + let token: Secret<'github-token'> | null = null; + const fromEnv = (['GH_TOKEN', 'GITHUB_TOKEN'] as const).find((name) => (env[name] ?? '') !== ''); + if (fromEnv !== undefined) { + tokenSource = fromEnv; + token = new Secret('github-token', env[fromEnv]!); + } else { + const gh = await options.runner('gh', ['auth', 'token']); + if (gh.code === 0 && gh.stdout.trim() !== '') { + tokenSource = 'gh'; + token = new Secret('github-token', gh.stdout.trim()); + } + } + const base = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + return { + repo: options.repo, + workflow: options.workflow, + tokenSource, + async api(method, path, body) { + const send = async (authorized: boolean): Promise => { + const headers: Record = { Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28', 'User-Agent': 'verify-remote' }; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + if (authorized) token?.use('github-authorization', (plain) => (headers.Authorization = `Bearer ${plain}`)); + const response = await fetch(`${base}/repos/${options.repo}${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body), redirect: 'manual', signal: AbortSignal.timeout(30_000) }); + const text = await response.text(); + let json: unknown = null; + try { + json = text === '' ? null : JSON.parse(text); + } catch { + json = null; + } + return { status: response.status, json, headers: response.headers }; + }; + let first = await send(true); + for (let attempt = 1; method === 'GET' && first.status >= 500 && attempt <= TRANSIENT_RETRIES; attempt += 1) { + await sleep(attempt * (options.retryDelayMs ?? 1500)); + first = await send(true); + } + if (method !== 'GET' || token === null || (first.status !== 401 && first.status !== 403)) return first; + return { ...(await send(false)), firstRefusalWithToken: message(first) }; + }, + }; +} + +export const CLOUD_GITHUB_ACCESS = + 'A 403 on git push means this machine\'s GitHub credential may not push to the repository. In a cloud sandbox that credential belongs to the sandbox\'s GitHub integration, and someone who administers the organization gives the integration access to the repository. REST calls there can use another credential, so they may work while a push is refused'; + +export function gitFailure(output: string): string { + const lines = output.trim().split('\n').map((line) => line.trim()).filter((line) => line !== ''); + const remote = lines.filter((line) => line.startsWith('remote:')).map((line) => line.replace(/^remote:\s*/, '')); + return [...remote, lines.at(-1) ?? 'no output'].filter((line, i, all) => all.indexOf(line) === i).join(' | '); +} + +const message = (response: ApiResponse): string => `${response.status}${typeof (response.json as { message?: unknown } | null)?.message === 'string' ? ` ${(response.json as { message: string }).message}` : ''}`; + +export interface StartOptions { + readonly ref: string; + readonly readableWithinMs?: number; +} + +async function untilReadable(github: GitHub, runId: string, withinMs: number): Promise { + const deadline = Date.now() + withinMs; + while ((await github.api('GET', `/actions/runs/${runId}`)).status !== 200 && Date.now() < deadline) await sleep(Math.min(1500, withinMs / 4)); +} + +async function findRun(github: GitHub, query: string, matches: (run: { id: number; display_title: string }) => boolean, seconds: number): Promise { + const deadline = Date.now() + seconds * 1000; + for (;;) { + const listed = await github.api('GET', `/actions/workflows/${github.workflow}/runs?per_page=20&${query}`); + const runs = ((listed.json as { workflow_runs?: { id: number; display_title: string }[] } | null)?.workflow_runs ?? []).filter(matches); + if (runs[0] !== undefined) return String(runs[0].id); + if (Date.now() >= deadline) return null; + await sleep(3000); + } +} + +const isRunOf = (request: Pick) => (run: { display_title: string }): boolean => { + const title = RUN_TITLE.exec(run.display_title); + return title !== null && title[1] === request.owner && title[2] === request.session; +}; + +export async function startRun(github: GitHub, request: SessionRequest, options: StartOptions): Promise { + const dispatched = await github.api('POST', `/actions/workflows/${github.workflow}/dispatches`, { + ref: options.ref, + inputs: { owner: request.owner, session: request.session, request: JSON.stringify(request) }, + return_run_details: true, + }); + const byDispatch = async (runId: string): Promise => { + await untilReadable(github, runId, options.readableWithinMs ?? 60_000); + return runId; + }; + const late = `{cli} down --stale ends it once it shows at https://github.com/${github.repo}/actions/workflows/${github.workflow}`; + if (dispatched.status === 200 || dispatched.status === 204) { + const direct = (dispatched.json as { workflow_run_id?: number } | null)?.workflow_run_id; + const runId = direct !== undefined ? String(direct) : await findRun(github, 'event=workflow_dispatch', isRunOf(request), 180); + if (runId === null) throw new VerifyFailure('NOT_READY', `GitHub accepted the dispatch of ${github.workflow} but no run for session ${request.session} appeared in 3 minutes; if it starts later it bills until it idles out`, late); + return byDispatch(runId); + } + const dispatchRefused = message(dispatched); + if (dispatched.status >= 500) { + const runId = await findRun(github, 'event=workflow_dispatch', isRunOf(request), 60); + if (runId === null) throw new VerifyFailure('NOT_READY', `GitHub answered the dispatch of ${github.workflow} with ${dispatchRefused}, and no run for session ${request.session} appeared in a minute`, `run the command again; if a run for this session starts late, ${late}`); + return byDispatch(runId); + } + throw new VerifyFailure( + 'NOT_READY', + `could not start a session: GitHub refused the dispatch of ${github.workflow} (${dispatchRefused})`, + `a session needs permission to dispatch ${github.workflow} on ${github.repo}; push the branch that holds ${github.workflow} first if GitHub has never seen it. ${CLOUD_GITHUB_ACCESS}`, + ); +} + +export interface RunView { + readonly status: string; + readonly conclusion: string | null; + readonly url: string; +} + +export async function viewRun(github: GitHub, runId: string): Promise { + const response = await github.api('GET', `/actions/runs/${runId}`); + if (response.status !== 200) throw new VerifyFailure('NOT_READY', `could not read run ${runId}: ${message(response)}`, `open https://github.com/${github.repo}/actions/runs/${runId}`); + const run = response.json as { status: string; conclusion: string | null; html_url: string }; + return { status: run.status, conclusion: run.conclusion, url: run.html_url }; +} + +export interface JobView { + readonly name: string; + readonly status: string; + readonly steps: readonly { readonly name: string; readonly status: string; readonly conclusion: string | null }[]; +} + +export async function viewJobs(github: GitHub, runId: string): Promise { + const response = await github.api('GET', `/actions/runs/${runId}/jobs?per_page=30`); + if (response.status !== 200) return []; + type Wire = { name: string; status: string; conclusion: string | null; steps?: { name: string; status: string; conclusion: string | null }[] }; + return ((response.json as { jobs?: Wire[] } | null)?.jobs ?? []).map((job) => ({ + name: job.name, + status: job.status, + steps: (job.steps ?? []).map((step) => ({ name: step.name, status: step.status, conclusion: step.conclusion })), + })); +} + +export function publishedStep(jobs: readonly JobView[], pattern: RegExp): string | null { + for (const job of jobs) { + for (const step of job.steps) { + const found = pattern.exec(step.name)?.[1]; + if (found !== undefined && found !== '') return found; + } + } + return null; +} + +export async function waitForStep(github: GitHub, runId: string, pattern: RegExp, seconds: number, onWait?: (run: RunView, jobs: readonly JobView[]) => void): Promise { + const deadline = Date.now() + seconds * 1000; + for (;;) { + const jobs = await viewJobs(github, runId); + const found = publishedStep(jobs, pattern); + if (found !== null) return found; + const run = await viewRun(github, runId); + if (run.status === 'completed') { + const failed = jobs.flatMap((job) => job.steps.filter((step) => step.conclusion === 'failure').map((step) => `${job.name}: ${step.name}`)); + throw new VerifyFailure('NOT_READY', `run ${runId} ended (${run.conclusion ?? 'no conclusion'}) before it published${failed.length > 0 ? `; failed at ${failed.join(', ')}` : ''}`, `read ${run.url}`); + } + if (Date.now() >= deadline) throw new VerifyFailure('NOT_READY', `run ${runId} published nothing within ${seconds}s (it is ${run.status})`, `read ${run.url}; a runner label with no free machine stays queued`); + onWait?.(run, jobs); + await sleep(4000); + } +} + +export function waitReporter(progress: (line: string) => void, runId: string, labels: { readonly plan: string; readonly session: string }): (run: RunView, jobs: readonly JobView[]) => void { + const began = Date.now(); + let last = ''; + let lastAt = 0; + return (_run, jobs) => { + const session = jobs.find((job) => job.name === 'session'); + const plan = jobs.find((job) => job.name === 'plan'); + const what = + session === undefined + ? plan?.status === 'in_progress' + ? `its request to be read on ${labels.plan}` + : `a ${labels.plan} runner to read its request` + : session.status === 'queued' + ? `a ${labels.session} runner` + : `the tunnel on ${labels.session}`; + if (what === last && Date.now() - lastAt < 60_000) return; + progress(`wait run ${runId} has waited ${Math.round((Date.now() - began) / 1000)}s, now for ${what}`); + last = what; + lastAt = Date.now(); + }; +} + +async function waitForRunEnd(github: GitHub, runId: string, seconds: number): Promise { + const deadline = Date.now() + seconds * 1000; + for (;;) { + const run = await viewRun(github, runId); + if (run.status === 'completed' || Date.now() >= deadline) return run; + await sleep(4000); + } +} + +async function cancelRun(github: GitHub, runId: string): Promise { + const response = await github.api('POST', `/actions/runs/${runId}/cancel`); + return response.status === 202 || response.status === 409 ? null : message(response); +} + +export interface LiveRun { + readonly runId: string; + readonly owner: string; + readonly session: string; + readonly createdAt: string; +} + +export async function liveRuns(github: GitHub): Promise { + const out: LiveRun[] = []; + for (const status of ['in_progress', 'queued'] as const) { + const listed = await github.api('GET', `/actions/workflows/${github.workflow}/runs?per_page=50&status=${status}`); + for (const run of (listed.json as { workflow_runs?: { id: number; display_title: string; created_at: string }[] } | null)?.workflow_runs ?? []) { + const title = RUN_TITLE.exec(run.display_title); + if (title !== null) out.push({ runId: String(run.id), owner: title[1]!, session: title[2]!, createdAt: run.created_at }); + } + } + return out; +} + +export async function currentBranch(runner: Runner, worktree: string): Promise { + const branch = (await runner('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { cwd: worktree })).stdout.trim(); + if (branch === '' || branch === 'HEAD') throw new VerifyFailure('NOT_READY', 'HEAD is detached, so there is no branch to start a session from', 'git switch -c , then git push -u origin '); + return branch; +} + +// A cancelled job abandons the step that uploads the evidence, so a session that holds evidence gets longer to end by itself. +const ARTIFACT_UPLOAD_SECONDS = 300; + +export async function endSession(github: GitHub, runId: string, ref: SessionRef | null): Promise<{ readonly conclusion: string | null; readonly cancelled: boolean; readonly problem: string | null }> { + const stopped = ref !== null && existsSync(ref.tokenFile) ? await sessionCall(ref, '/__sim/stop', { method: 'POST', timeoutMs: 15_000 }).catch(() => null) : null; + const holdsEvidence = stopped?.status === 200 && ((await stopped.json().catch(() => null)) as { evidence?: unknown } | null)?.evidence != null; + let run = await waitForRunEnd(github, runId, ref === null ? 0 : holdsEvidence ? ARTIFACT_UPLOAD_SECONDS : 90); + if (run.status === 'completed') return { conclusion: run.conclusion, cancelled: false, problem: null }; + const refused = await cancelRun(github, runId); + run = await waitForRunEnd(github, runId, 60); + if (run.status === 'completed') return { conclusion: run.conclusion, cancelled: true, problem: null }; + return { conclusion: null, cancelled: refused === null, problem: `run ${runId} is still ${run.status}${refused === null ? '' : ` and the cancel was refused (${refused})`}; it ends itself on idle or at its cap` }; +} + +export { message as apiMessage }; diff --git a/integration/expo-native/src/core/remote/handoff.ts b/integration/expo-native/src/core/remote/handoff.ts new file mode 100644 index 00000000000..05bc2d9b31f --- /dev/null +++ b/integration/expo-native/src/core/remote/handoff.ts @@ -0,0 +1,231 @@ +import { createHash } from 'node:crypto'; +import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Platform } from '../types.ts'; + +export const HANDOFF_VERSION = 1 as const; + +export const HANDOFF = { + artifact: 'verify-evidence', + directory: 'evidence', + staging: 'evidence.partial', + manifestFile: 'manifest.json', +} as const; + +const MIB = 1024 * 1024; + +export const HANDOFF_LIMITS = { + manifestBytes: 16 * 1024, + files: 50, + videos: 1, + imageBytes: 10 * MIB, + videoBytes: 100 * MIB, + totalBytes: 150 * MIB, + chunkBytes: MIB, + chunkBodyBytes: 1_500_000, + pr: 9_999_999, + tests: 10_000, +} as const; + +export interface HandoffFile { + readonly name: string; + readonly bytes: number; + readonly sha256: string; +} + +export interface HandoffManifest { + readonly v: typeof HANDOFF_VERSION; + readonly pr: number; + readonly run: string; + readonly platform: Platform; + readonly device: string; + readonly commit: string; + readonly passed: number; + readonly flaky: number; + readonly total: number; + readonly files: readonly HandoffFile[]; +} + +const SHAPE = { + run: /^r\d{8}-\d{6}-[0-9a-f]{4}$/, + device: /^[A-Za-z0-9][A-Za-z0-9 ._()-]{0,63}$/, + commit: /^[0-9a-f]{40}$/, + sha256: /^[0-9a-f]{64}$/, + fileName: /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\.(png|jpg|mp4)$/, +} as const; + +const MANIFEST_KEYS = ['v', 'pr', 'run', 'platform', 'device', 'commit', 'passed', 'flaky', 'total', 'files']; +const FILE_KEYS = ['name', 'bytes', 'sha256']; + +export class HandoffRefused extends Error { + readonly status: 400 | 409 | 413; + readonly have: number | undefined; + constructor(status: 400 | 409 | 413, message: string, have?: number) { + super(message); + this.status = status; + this.have = have; + } +} + +const refuse = (message: string): never => { + throw new HandoffRefused(400, message); +}; + +function record(value: unknown, keys: readonly string[], what: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return refuse(`${what} is not an object`); + const extra = Object.keys(value).filter((key) => !keys.includes(key)); + if (extra.length > 0 || Object.keys(value).length !== keys.length) return refuse(`${what} must have exactly the keys ${keys.join(', ')}`); + return value as Record; +} + +function whole(value: unknown, min: number, max: number, what: string): number { + if (typeof value !== 'number' || !Number.isInteger(value) || value < min || value > max) return refuse(`${what} must be a whole number from ${min} to ${max}`); + return value; +} + +function shaped(value: unknown, pattern: RegExp, what: string): string { + if (typeof value !== 'string' || !pattern.test(value)) return refuse(`${what} is missing or malformed`); + return value; +} + +export const isVideo = (name: string): boolean => name.endsWith('.mp4'); + +export function parseHandoffManifest(text: string): HandoffManifest { + if (Buffer.byteLength(text) > HANDOFF_LIMITS.manifestBytes) refuse(`the manifest is over ${HANDOFF_LIMITS.manifestBytes} bytes`); + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return refuse('the manifest is not JSON'); + } + const raw = record(parsed, MANIFEST_KEYS, 'the manifest'); + if (raw.v !== HANDOFF_VERSION) refuse(`manifest version ${String(raw.v)} is not ${HANDOFF_VERSION}`); + if (raw.platform !== 'ios' && raw.platform !== 'android') refuse('platform must be ios or android'); + if (!Array.isArray(raw.files) || raw.files.length < 1 || raw.files.length > HANDOFF_LIMITS.files) refuse(`files must be a list of 1 to ${HANDOFF_LIMITS.files}`); + const files = (raw.files as unknown[]).map((entry, i): HandoffFile => { + const file = record(entry, FILE_KEYS, `file ${i + 1}`); + const name = shaped(file.name, SHAPE.fileName, `the name of file ${i + 1}`); + return { name, bytes: whole(file.bytes, 1, isVideo(name) ? HANDOFF_LIMITS.videoBytes : HANDOFF_LIMITS.imageBytes, `the size of ${name}`), sha256: shaped(file.sha256, SHAPE.sha256, `the sha256 of ${name}`) }; + }); + if (new Set(files.map((file) => file.name.toLowerCase())).size !== files.length) refuse('two files have the same name'); + if (files.filter((file) => isVideo(file.name)).length > HANDOFF_LIMITS.videos) refuse(`at most ${HANDOFF_LIMITS.videos} video`); + if (files.reduce((sum, file) => sum + file.bytes, 0) > HANDOFF_LIMITS.totalBytes) refuse(`the files are over ${HANDOFF_LIMITS.totalBytes} bytes in all`); + const manifest: HandoffManifest = { + v: HANDOFF_VERSION, + pr: whole(raw.pr, 1, HANDOFF_LIMITS.pr, 'pr'), + run: shaped(raw.run, SHAPE.run, 'run'), + platform: raw.platform as Platform, + device: shaped(raw.device, SHAPE.device, 'device'), + commit: shaped(raw.commit, SHAPE.commit, 'commit'), + passed: whole(raw.passed, 0, HANDOFF_LIMITS.tests, 'passed'), + flaky: whole(raw.flaky, 0, HANDOFF_LIMITS.tests, 'flaky'), + total: whole(raw.total, 1, HANDOFF_LIMITS.tests, 'total'), + files, + }; + if (manifest.passed + manifest.flaky < 1 || manifest.passed + manifest.flaky > manifest.total) refuse('passed and flaky must count at least one test and at most total'); + return manifest; +} + +export const MOST_COMMITS_GITHUB_LISTS = 250; + +export interface PullRequestView { + readonly state: string; + readonly headRepo: string | null; + readonly headRef: string; + readonly commits: readonly string[]; + readonly commitCount: number; +} + +export interface PullRequestMismatch { + readonly why: string; + readonly fix: string; +} + +const ITS_OWN_PULL_REQUEST = + "it accepts only the open pull request of the session's branch, and only when the commit the session started on and the commit of the run are both commits of that pull request; git push, then {cli} run again and attach, or after a rewrite of the branch's history {cli} down and {cli} up first"; +const FEWER_COMMITS = `it can check a commit only against the first ${MOST_COMMITS_GITHUB_LISTS} commits of a pull request; squash or rebase the branch to ${MOST_COMMITS_GITHUB_LISTS} commits or fewer, git push, then {cli} down, {cli} up, run again and attach, or run {cli} attach on a machine whose gh can attach`; + +export function pullRequestMismatch(pull: PullRequestView, session: { readonly repo: string; readonly headBranch: string; readonly startedOn: string; readonly runCommit: string }): PullRequestMismatch | null { + const refused = (why: string): PullRequestMismatch => ({ why, fix: ITS_OWN_PULL_REQUEST }); + if (pull.state !== 'open') return refused(`it is ${pull.state}`); + if (pull.headRepo !== session.repo) return refused(`its branch is in ${pull.headRepo ?? 'a deleted fork'}, not in ${session.repo}`); + if (pull.headRef !== session.headBranch) return refused(`its branch is ${pull.headRef} and the session runs on ${session.headBranch}`); + for (const [what, commit] of [['the session started on', session.startedOn], ['the run was made at', session.runCommit]] as const) { + if (pull.commits.includes(commit)) continue; + if (pull.commitCount > MOST_COMMITS_GITHUB_LISTS) { + return { why: `it has ${pull.commitCount} commits, GitHub lists only the first ${MOST_COMMITS_GITHUB_LISTS}, and ${what} ${commit.slice(0, 12)}, which is not among those`, fix: FEWER_COMMITS }; + } + return refused(`${what} ${commit.slice(0, 12)}, which is not one of its commits`); + } + return null; +} + +const sha256File = (path: string): string => createHash('sha256').update(readFileSync(path)).digest('hex'); + +export function describeFile(name: string, path: string): HandoffFile { + return { name, bytes: statSync(path).size, sha256: sha256File(path) }; +} + +export interface EvidenceReceiver { + begin(body: string | null): { readonly ok: true }; + chunk(body: string | null): { readonly ok: true; readonly bytes: number }; + commit(): { readonly ok: true; readonly run: string; readonly files: number; readonly bytes: number }; + held(): string | null; +} + +export function evidenceReceiver(work: string): EvidenceReceiver { + const staging = join(work, HANDOFF.staging); + const kept = join(work, HANDOFF.directory); + let receiving: { readonly manifest: HandoffManifest; readonly sizes: Map } | null = null; + let heldRun: string | null = null; + + return { + begin(body) { + if (body === null) throw new HandoffRefused(413, `the manifest is over ${HANDOFF_LIMITS.manifestBytes} bytes`); + const manifest = parseHandoffManifest(body); + rmSync(staging, { recursive: true, force: true }); + mkdirSync(staging, { recursive: true }); + writeFileSync(join(staging, HANDOFF.manifestFile), `${JSON.stringify(manifest)}\n`); + receiving = { manifest, sizes: new Map(manifest.files.map((file) => [file.name, 0])) }; + return { ok: true }; + }, + + chunk(body) { + if (receiving === null) throw new HandoffRefused(409, 'no hand-off has begun'); + if (body === null) throw new HandoffRefused(413, `a chunk request is at most ${HANDOFF_LIMITS.chunkBodyBytes} bytes`); + let raw: Record; + try { + raw = record(JSON.parse(body), ['name', 'offset', 'data'], 'the chunk'); + } catch (error) { + throw error instanceof HandoffRefused ? error : new HandoffRefused(400, 'the chunk is not JSON'); + } + const declared = receiving.manifest.files.find((file) => file.name === raw.name); + if (declared === undefined) throw new HandoffRefused(400, 'the manifest names no such file'); + const have = receiving.sizes.get(declared.name)!; + if (raw.offset !== have) throw new HandoffRefused(409, `${declared.name} is at ${have} bytes`, have); + if (typeof raw.data !== 'string' || !/^[A-Za-z0-9+/]+={0,2}$/.test(raw.data)) throw new HandoffRefused(400, 'data must be base64'); + const data = Buffer.from(raw.data, 'base64'); + if (data.length === 0 || data.length > HANDOFF_LIMITS.chunkBytes) throw new HandoffRefused(413, `a chunk holds 1 to ${HANDOFF_LIMITS.chunkBytes} bytes`); + if (have + data.length > declared.bytes) throw new HandoffRefused(413, `${declared.name} would pass the ${declared.bytes} bytes its manifest declares`); + appendFileSync(join(staging, declared.name), data); + receiving.sizes.set(declared.name, have + data.length); + return { ok: true, bytes: have + data.length }; + }, + + commit() { + if (receiving === null) throw new HandoffRefused(409, 'no hand-off has begun'); + const { manifest, sizes } = receiving; + for (const file of manifest.files) { + if (sizes.get(file.name) !== file.bytes) throw new HandoffRefused(409, `${file.name} has ${sizes.get(file.name)} of ${file.bytes} bytes`); + if (sha256File(join(staging, file.name)) !== file.sha256) throw new HandoffRefused(409, `${file.name} does not have the sha256 its manifest declares`); + } + rmSync(kept, { recursive: true, force: true }); + renameSync(staging, kept); + receiving = null; + heldRun = manifest.run; + return { ok: true, run: manifest.run, files: manifest.files.length, bytes: manifest.files.reduce((sum, file) => sum + file.bytes, 0) }; + }, + + held: () => (heldRun !== null && existsSync(kept) ? heldRun : null), + }; +} diff --git a/integration/expo-native/src/core/remote/preflight.ts b/integration/expo-native/src/core/remote/preflight.ts new file mode 100644 index 00000000000..af5fd5fdaf8 --- /dev/null +++ b/integration/expo-native/src/core/remote/preflight.ts @@ -0,0 +1,232 @@ +import net from 'node:net'; +import { arch, platform as osPlatform, release } from 'node:os'; +import type { Runner } from '../exec.ts'; +import { VerifyFailure, type DoctorCheck, type DoctorCheckId, type DoctorOptions } from '../types.ts'; +import { CLOUD_GITHUB_ACCESS, apiMessage, currentBranch, endSession, gitFailure, liveRuns, startRun, viewJobs, waitForStep, waitReporter, type ApiResponse, type GitHub } from './github.ts'; +import { STEP } from './protocol.ts'; +import { daemonHealthy, firstHealth, sessionHealth, tunnelUrl, type SessionRef } from './session.ts'; +import { forgetSession, newSessionRequest, saveToken, savedDriverId, type RemoteDeps, type RemoteSettings } from './settings.ts'; +import { TUNNEL } from './tunnel.ts'; + +const CLOUD_FIX = `add ${TUNNEL.allowedHost} to the allowed domains of the environment this command runs in`; + +function check(id: DoctorCheckId, ok: boolean, detail: string, fix: string): DoctorCheck { + return ok ? { id, ok, detail } : { id, ok, detail, fix }; +} + +const notRun = (id: DoctorCheckId, why: string): DoctorCheck => ({ id, ok: true, state: 'not-run', detail: `not run: ${why}` }); +const blockedBy = (id: DoctorCheckId, failed: DoctorCheckId, why = ''): DoctorCheck => notRun(id, `needs ${failed}${why === '' ? '' : ` (${why})`}`); + +const ONLY_LIVE = 'doctor starts no workflow run without --live'; + +const failureText = (error: unknown): string => { + const cause = (error as { cause?: { code?: string; message?: string } }).cause; + return `${(error as Error).message}${cause === undefined ? '' : ` (${cause.code ?? cause.message ?? ''})`}`; +}; + +function proxyOf(env: RemoteDeps['env']): URL | null { + const raw = env.HTTPS_PROXY ?? env.https_proxy; + if (raw === undefined || raw === '') return null; + try { + return new URL(raw); + } catch { + return null; + } +} + +export function connectThroughProxy(proxy: URL, host: string, timeoutMs = 10_000): Promise<{ readonly status: number; readonly line: string }> { + return new Promise((resolve) => { + const socket = net.connect(Number(proxy.port || (proxy.protocol === 'https:' ? 443 : 80)), proxy.hostname); + const authorization = proxy.username === '' || proxy.password === '' ? '' : `Proxy-Authorization: Basic ${Buffer.from(`${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`).toString('base64')}\r\n`; + let seen = ''; + const finish = (status: number, line: string) => { + socket.destroy(); + resolve({ status, line }); + }; + socket.setTimeout(timeoutMs, () => finish(0, 'the proxy did not answer')); + socket.on('error', (error) => finish(0, `the proxy is unreachable: ${error.message}`)); + socket.on('connect', () => socket.write(`CONNECT ${host}:443 HTTP/1.1\r\nHost: ${host}:443\r\n${authorization}\r\n`)); + socket.on('data', (chunk: Buffer) => { + seen += chunk.toString('latin1'); + const end = seen.indexOf('\r\n'); + if (end < 0) return; + const line = seen.slice(0, end); + finish(Number(/^HTTP\/\d\.\d (\d{3})/.exec(line)?.[1] ?? 0), line); + }); + }); +} + +export async function egressCheck(id: DoctorCheckId, host: string, env: RemoteDeps['env'], fix: string, request: typeof fetch = fetch): Promise { + const proxy = proxyOf(env); + const usesProxy = proxy !== null && env.NODE_USE_ENV_PROXY === '1'; + if (usesProxy) { + const connect = await connectThroughProxy(proxy, host); + if (connect.status !== 200) return check(id, false, `blocked: the proxy at ${proxy.host} answered the CONNECT to ${host} with "${connect.line}"`, fix); + } + const via = usesProxy ? ` through the proxy at ${proxy.host}` : ''; + try { + const response = await request(`https://${host}/`, { redirect: 'manual', signal: AbortSignal.timeout(15_000) }); + const fromCloudflare = response.headers.has('cf-ray') || response.headers.get('server') === 'cloudflare'; + if (fromCloudflare) return check(id, true, `reached ${host}${via}: HTTP ${response.status} from cloudflare`, ''); + return check(id, false, `blocked: ${host} answered HTTP ${response.status}${via} without Cloudflare's headers, so something between this machine and the host answered in its place`, fix); + } catch (error) { + return check(id, false, `no response from ${host}${via}: ${failureText(error)}`, fix); + } +} + +async function environmentCheck(github: GitHub, deps: RemoteDeps, worktree: string): Promise { + const npm = await deps.runner('npm', ['--version']); + const gh = await deps.runner('gh', ['--version']); + const remote = (await deps.runner('git', ['remote', 'get-url', 'origin'], { cwd: worktree })).stdout.trim(); + const remoteKind = /^(git@|ssh:)/.test(remote) ? 'ssh' : /^https?:\/\/(127\.0\.0\.1|localhost)/.test(remote) ? 'a local git proxy' : /^https?:/.test(remote) ? 'https' : 'unknown'; + const proxy = proxyOf(deps.env); + const why = deps.env.VERIFY_EGRESS_WHY === undefined ? '' : ` (${deps.env.VERIFY_EGRESS_WHY})`; + const egress = proxy === null ? 'direct (no HTTPS_PROXY)' : deps.env.NODE_USE_ENV_PROXY === '1' ? `through the proxy at ${proxy.host}${why}` : `direct, not through the proxy at ${proxy.host}${why}`; + const detail = [ + `${osPlatform()} ${arch()} ${release()}`, + `npm ${npm.code === 0 ? npm.stdout.trim() : 'missing'}`, + `gh ${gh.code === 0 ? 'present' : 'absent'}`, + `GitHub token from ${github.tokenSource === 'none' ? 'nowhere' : github.tokenSource}`, + `origin over ${remoteKind}`, + `egress ${egress}`, + ].join('; '); + return check('remote-env', npm.code === 0, detail, 'install npm with Node 24'); +} + +async function gitChecks(settings: RemoteSettings, runner: Runner, worktree: string): Promise { + const git = (args: readonly string[]) => runner('git', args, { cwd: worktree, env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } }); + let branch: string; + try { + branch = await currentBranch(runner, worktree); + } catch (error) { + const failure = error as VerifyFailure; + return [check('git-fetch', false, failure.message, failure.fix), blockedBy('git-push', 'git-fetch')]; + } + const fetched = await git(['fetch', '--dry-run', '--no-tags', 'origin', branch]); + const pushOwn = await git(['push', '--dry-run', 'origin', `HEAD:refs/heads/${branch}`]); + return [ + check('git-fetch', fetched.code === 0, fetched.code === 0 ? `git fetch origin ${branch} works` : `git fetch origin ${branch} failed: ${gitFailure(fetched.stderr)}`, `git push -u origin ${branch}, and check the remote with git remote -v`), + check( + 'git-push', + pushOwn.code === 0, + pushOwn.code === 0 ? `dry-run push of ${branch} is accepted` : `dry-run push of ${branch} failed: ${gitFailure(pushOwn.stderr)}`, + `verifying a commit you make here needs git push access to ${settings.repo}, because a remote session builds a pushed commit; verifying a commit GitHub already has needs only REST. ${CLOUD_GITHUB_ACCESS}`, + ), + ]; +} + +const withAndWithoutToken = (response: ApiResponse): string => `${apiMessage(response)}${response.firstRefusalWithToken === undefined ? '' : ` without the token (with it: ${response.firstRefusalWithToken})`}`; + +async function restCheck(settings: RemoteSettings, github: GitHub): Promise { + const fix = `check network access to api.github.com. ${CLOUD_GITHUB_ACCESS}`; + try { + const repo = await github.api('GET', ''); + const runs = await github.api('GET', '/actions/runs?per_page=1'); + const ok = repo.status === 200 && runs.status === 200; + const left = repo.headers.get('x-ratelimit-remaining'); + const tokenRefused = repo.firstRefusalWithToken !== undefined || runs.firstRefusalWithToken !== undefined; + return check( + 'github-rest', + ok, + `repository ${withAndWithoutToken(repo)}, workflow runs ${withAndWithoutToken(runs)}${left === null ? '' : `, ${left} requests left this hour`}${ok && tokenRefused ? `; reads of ${settings.repo} work, but GitHub refuses this machine's token, so starting a session will fail` : ''}`, + fix, + ); + } catch (error) { + return check('github-rest', false, `api.github.com did not answer: ${failureText(error)}`, fix); + } +} + +async function commitCheck(github: GitHub, runner: Runner, worktree: string): Promise { + const sha = (await runner('git', ['rev-parse', 'HEAD'], { cwd: worktree })).stdout.trim(); + const found = await github.api('GET', `/commits/${sha}`).catch(() => null); + return check('remote-commit', found?.status === 200, found?.status === 200 ? `GitHub has HEAD ${sha.slice(0, 12)}` : `GitHub does not have HEAD ${sha.slice(0, 12)} (${found === null ? 'no answer' : found.status})`, 'git push; a remote session builds the pushed commit'); +} + +const LIVE_CHECKS = ['live-session', 'live-sim-health', 'live-daemon-health', 'live-device', 'live-stop'] as const satisfies readonly DoctorCheckId[]; +const NO_DEVICE = 'this session boots no device; --runner