ci: a failure is suspect until a second independent run confirms it

Failures record their attempt number and the last error line. A single
failure keeps the combination eligible for one more run on another runner;
a second failure confirms it and renders as unsupported. Any success
outranks failures for the cell.

Author: rigor789

.github/workflows/compat-check.yml

@@ -112,7 +112,8 @@ jobs:
           cd compat
           npm install @nativescript/ios@${{ matrix.runtime }} --save-dev
           # One retry before a failure is recorded: a runner hiccup must not become a permanent verdict.
-          ns build ios --disableAnalytics || ns build ios --disableAnalytics
+          set -o pipefail
+          (ns build ios --disableAnalytics || ns build ios --disableAnalytics) 2>&1 | tee ../build.log
       - name: Record result
         # A pinned combination that failed is as final as one that passed: record it so it is never rebuilt.
         if: always() && steps.build.outcome != 'skipped'
@@ -121,7 +122,7 @@ jobs:
             --toolchain xcode=${{ matrix.xcode }} --toolchain cocoapods="$(pod --version)" \
             --with nativescript=${{ matrix.cli }} --with node=${{ matrix.node }} \
             --resolved node="$(node --version | tr -d v)" --resolved xcode="$(xcodebuild -version | head -1 | cut -d' ' -f2)" \
-            --outcome ${{ steps.build.outcome }} \
+            --outcome ${{ steps.build.outcome }} --signature "$(grep -aE 'error|FAILURE|failed' build.log | grep -viE 'warning|deprecat' | tail -1)" \
             --evidence "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
       - uses: actions/upload-artifact@v4
         if: always() && steps.build.outcome != 'skipped'
@@ -160,15 +161,16 @@ jobs:
           ns create compat --tsc --disableAnalytics
           cd compat
           npm install @nativescript/android@${{ matrix.runtime }} --save-dev
-          ns build android --compileSdk ${{ matrix.compileSdk }} --disableAnalytics || ns build android --compileSdk ${{ matrix.compileSdk }} --disableAnalytics
+          set -o pipefail
+          (ns build android --compileSdk ${{ matrix.compileSdk }} --disableAnalytics || ns build android --compileSdk ${{ matrix.compileSdk }} --disableAnalytics) 2>&1 | tee ../build.log
       - name: Record result
         if: always() && steps.build.outcome != 'skipped'
         run: |
           node scripts/report-verification.mjs --package @nativescript/android --version ${{ matrix.runtime }} \
             --toolchain compileSdk=${{ matrix.compileSdk }} --toolchain buildTools=${{ matrix.buildTools }} --toolchain jdk=${{ matrix.jdk }} \
             --with nativescript=${{ matrix.cli }} --with node=${{ matrix.node }} \
             --resolved node="$(node --version | tr -d v)" --resolved jdk="$(java -version 2>&1 | head -1 | cut -d'"' -f2)" \
-            --outcome ${{ steps.build.outcome }} \
+            --outcome ${{ steps.build.outcome }} --signature "$(grep -aE 'error|FAILURE|failed' build.log | grep -viE 'warning|deprecat' | tail -1)" \
             --evidence "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
       - uses: actions/upload-artifact@v4
         if: always() && steps.build.outcome != 'skipped'

README.md

@@ -40,9 +40,9 @@ as a browser-support-style matrix (Vue + Vite) for people.
 2. **Build jobs**: one job per combination installs the pinned CLI and
    Node.js, creates a fresh app, builds it with the pinned runtime and
    toolchain, and writes its result file under `data/verified/` whether it
-   passed or failed. A success proves the runtime's toolchain cells and the
-   CLI's Node.js cell; a failure marks the toolchain cells unsupported. Either
-   way the combination is never rebuilt.
+   passed or failed, with the last error line as a signature. A success is
+   final. A failure is suspect until a later run fails the same combination
+   again on another runner, which confirms it; a success in between clears it.
 3. **Collect**: the artifacts are dropped onto the checkout and committed to
    the default branch. Every file names the run that produced it, so the
    commit is the audit trail; nothing needs a manual merge.
@@ -59,7 +59,9 @@ For a release and a toolchain version, in this order:
 1. **Unsupported / Advisory**: a maintainer advisory matches the release and
    the toolchain version.
 2. **Verified**: a recorded CI build succeeded with that toolchain version.
-   A recorded CI failure marks the cell unsupported instead.
+   Any success outranks failures. A failure only marks the cell unsupported
+   once a second, independent run has failed the same combination; a single
+   failure shows as unverified with a pending second attempt.
 3. **Declared**: inside the range the package published under
    `nativescript.requirements`, or inside a maintainer override for releases
    that predate the block.

schemas/compatibility.json

@@ -101,7 +101,17 @@
 								},
 								"failed": {
 									"type": "object",
-									"description": "Toolchain versions a CI build failed with, per key.",
+									"description": "Toolchain versions whose builds failed in two independent CI runs, per key.",
+									"additionalProperties": {
+										"type": "array",
+										"items": {
+											"type": "string"
+										}
+									}
+								},
+								"suspect": {
+									"type": "object",
+									"description": "Toolchain versions with a single CI failure, awaiting a second attempt, per key.",
 									"additionalProperties": {
 										"type": "array",
 										"items": {

schemas/verified.json

@@ -2,7 +2,7 @@
 	"$schema": "http://json-schema.org/draft-07/schema#",
 	"$id": "https://compatibility.nativescript.org/v1/schemas/verified.json",
 	"title": "CI build result",
-	"description": "One pinned build of a runtime version with its toolchain versions, CLI version and Node.js major, and whether it succeeded. Stored under data/verified/<package>/ as <timestamp>_<version>_<pinned versions>.json; CI only ever adds files, and a combination recorded once, whatever the outcome, is never rebuilt.",
+	"description": "One pinned build of a runtime version with its toolchain versions, CLI version and Node.js major, and its outcome. Stored under data/verified/<package>/ as <timestamp>_<version>_<pinned versions>.json. A success is final. A failure is suspect until a second, independent run fails the same combination (attempts >= 2); a later success outranks any failure.",
 	"type": "object",
 	"required": [
 		"package",
@@ -79,6 +79,15 @@
 			],
 			"default": "success",
 			"description": "Omitted means success."
+		},
+		"attempts": {
+			"type": "integer",
+			"minimum": 1,
+			"description": "For failures: how many independent runs have failed this combination, this one included."
+		},
+		"signature": {
+			"type": "string",
+			"description": "The last recognisable error line of the failed build."
 		}
 	}
 }

scripts/build-matrix.mjs

@@ -13,7 +13,7 @@
 // MANUAL_XCODE or MANUAL_COMPILE_SDK + MANUAL_JDK. MANUAL_FORCE=true rebuilds
 // a combination that is already recorded.
 import { appendFileSync } from "node:fs";
-import { readResults } from "./lib/results.mjs";
+import { readResults, status } from "./lib/results.mjs";
 
 const REGISTRY = "https://registry.npmjs.org/";
 const XCODE_FEED = "https://xcodereleases.com/data.json";
@@ -190,19 +190,12 @@ async function jdkReleases() {
 
 const recorded = readResults();
 
-// A result counts when its pins match and it covers at least the toolchains
-// the matrix controls; jobs also record incidental versions such as CocoaPods.
+// A success is final and a failure confirmed by two independent runs is
+// final; a single failure stays eligible so a later run can retry it on a
+// different runner.
 function isRecorded(candidate) {
-	return recorded.some(
-		(result) =>
-			result.package === candidate.package &&
-			result.version === candidate.version &&
-			result.with.nativescript === candidate.with.nativescript &&
-			result.with.node === candidate.with.node &&
-			Object.entries(candidate.toolchains).every(
-				([key, version]) => result.toolchains[key] !== undefined && sameLine(result.toolchains[key], version),
-			),
-	);
+	const current = status(recorded, candidate);
+	return current === "success" || current === "confirmed";
 }
 
 function emit(matrix) {

scripts/lib/results.mjs

@@ -21,17 +21,44 @@ export function resultPath(result, date = new Date()) {
 	return new URL(`${result.package}/${timestamp(date)}_${result.version}_${resultKey(result.toolchains, result.with)}.json`, ROOT);
 }
 
-export function findRecorded(results, candidate) {
-	return results.find(
-		(result) =>
-			result.package === candidate.package &&
-			result.version === candidate.version &&
-			result.with.nativescript === candidate.with.nativescript &&
-			result.with.node === candidate.with.node &&
-			resultKey(result.toolchains, {}) === resultKey(candidate.toolchains, {}),
+/** Toolchains recorded for the record but chosen by the job, not by the matrix. */
+const INCIDENTAL = new Set(["cocoapods", "buildTools"]);
+
+function controlled(toolchains) {
+	return Object.fromEntries(Object.entries(toolchains).filter(([key]) => !INCIDENTAL.has(key)));
+}
+
+/** Records of the same pinned combination, ignoring incidental toolchains. */
+export function sameCombination(a, b) {
+	return (
+		a.package === b.package &&
+		a.version === b.version &&
+		a.with.nativescript === b.with.nativescript &&
+		a.with.node === b.with.node &&
+		resultKey(controlled(a.toolchains), {}) === resultKey(controlled(b.toolchains), {})
 	);
 }
 
+/**
+ * How a combination stands: "success" if any run built it, "confirmed" after
+ * two independent failures, "suspect" after one, or null if never recorded.
+ */
+export function status(results, candidate) {
+	const matching = results.filter((result) => sameCombination(result, candidate));
+	if (!matching.length) {
+		return null;
+	}
+	if (matching.some((result) => (result.outcome ?? "success") === "success")) {
+		return "success";
+	}
+	const attempts = Math.max(...matching.map((result) => result.attempts ?? 1));
+	return attempts >= 2 ? "confirmed" : "suspect";
+}
+
+export function failedAttempts(results, candidate) {
+	return results.filter((result) => sameCombination(result, candidate) && result.outcome === "failure").length;
+}
+
 export function readResults() {
 	const results = [];
 	const walk = (dir) => {

scripts/report-verification.mjs

@@ -1,20 +1,25 @@
 #!/usr/bin/env node
-// Records a successful pinned build under data/verified/. CI jobs and humans
-// use the same command; an existing file is left untouched.
+// Records a pinned build under data/verified/. CI jobs and humans use the
+// same command. A success is recorded once. A failure is recorded per
+// independent run with its attempt number, so the second one confirms the
+// combination as failing; a later success outranks any failure.
 //
 //   node scripts/report-verification.mjs --package @nativescript/ios --version 9.1.0 \
 //     --toolchain xcode=26.3 --toolchain cocoapods=1.16.2 \
-//     --with nativescript=9.1.1 --with node=22 --resolved node=22.23.2 --evidence <url> [--outcome failure]
+//     --with nativescript=9.1.1 --with node=22 --resolved node=22.23.2 --evidence <url> \
+//     [--outcome failure --signature "<last error line>"]
 import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
-import { findRecorded, readResults, resultPath } from "./lib/results.mjs";
+import { failedAttempts, readResults, resultPath, status } from "./lib/results.mjs";
 
 const args = process.argv.slice(2);
 const options = { toolchains: {}, with: {}, resolved: {} };
 for (let i = 0; i < args.length; i += 2) {
 	const flag = args[i].replace(/^--/, "");
 	const value = args[i + 1];
-	if (["toolchain", "with", "resolved"].includes(flag)) {
+	if (flag === "signature") {
+		options.signature = value;
+	} else if (["toolchain", "with", "resolved"].includes(flag)) {
 		const [key, version] = value.split("=");
 		options[flag === "toolchain" ? "toolchains" : flag][key] = version;
 	} else {
@@ -27,21 +32,24 @@ if (!options.package || !options.version || !Object.keys(options.toolchains).len
 	process.exit(1);
 }
 
+const recorded = readResults();
+const failure = options.outcome === "failure";
+const candidate = { package: options.package, version: options.version, toolchains: options.toolchains, with: options.with };
 const result = {
 	$schema: "https://compatibility.nativescript.org/v1/schemas/verified.json",
-	package: options.package,
-	version: options.version,
-	toolchains: options.toolchains,
-	with: options.with,
+	...candidate,
 	...(Object.keys(options.resolved).length ? { resolved: options.resolved } : {}),
-	...(options.outcome === "failure" ? { outcome: "failure" } : {}),
+	...(failure ? { outcome: "failure", attempts: failedAttempts(recorded, candidate) + 1 } : {}),
+	...(failure && options.signature ? { signature: options.signature.slice(0, 300) } : {}),
 	...(options.evidence ? { evidence: options.evidence } : {}),
 	recordedAt: new Date().toISOString(),
 };
 
-const existing = findRecorded(readResults(), result);
-if (existing) {
-	console.log(`already recorded at ${existing.recordedAt}`);
+const current = status(recorded, candidate);
+if (current === "success" && !failure) {
+	console.log("already recorded as a success");
+} else if (current === "confirmed" && failure) {
+	console.log("already confirmed as failing");
 } else {
 	const file = resultPath(result);
 	mkdirSync(dirname(file.pathname), { recursive: true });

shared/compute.ts

@@ -63,12 +63,15 @@ export function buildDocument(input: {
 				manifest.requirements,
 				input.overrides,
 			);
-			const failed = verifiedFor(spec.name, manifest.version, input.verified, "failure");
+			const verified = verifiedFor(spec.name, manifest.version, input.verified, "success");
+			const failed = verifiedFor(spec.name, manifest.version, input.verified, "confirmed");
+			const suspect = verifiedFor(spec.name, manifest.version, input.verified, "suspect");
 			versions[manifest.version] = {
 				requirements: requirements.ranges,
 				source: requirements.source,
-				verified: verifiedFor(spec.name, manifest.version, input.verified, "success"),
+				verified,
 				...(Object.keys(failed).length ? { failed } : {}),
+				...(Object.keys(suspect).length ? { suspect } : {}),
 				publishedAt: manifest.publishedAt,
 			};
 		}
@@ -116,16 +119,27 @@ export function effectiveRequirements(
 	return { ranges, source };
 }
 
+export type ResultClass = "success" | "confirmed" | "suspect";
+
+/** A failure counts once a second independent run has failed the same combination. */
+export function classify(entry: VerificationResult): ResultClass {
+	if ((entry.outcome ?? "success") === "success") {
+		return "success";
+	}
+	return (entry.attempts ?? 1) >= 2 ? "confirmed" : "suspect";
+}
+
 /**
- * A result proves every toolchain it pinned for the runtime it built, and the
- * Node.js major for the CLI it built with.
+ * A successful result proves every toolchain it pinned for the runtime it
+ * built, and the Node.js major for the CLI it built with. A failure only
+ * speaks about the runtime's toolchains.
  */
 export function verifiedPairs(
 	verified: VerificationResult[],
-	outcome: "success" | "failure" = "success",
+	outcome: ResultClass = "success",
 ): Array<{ package: string; version: string; toolchain: ToolchainKey; toolchainVersion: string }> {
 	return verified
-		.filter((entry) => (entry.outcome ?? "success") === outcome)
+		.filter((entry) => classify(entry) === outcome)
 		.flatMap((entry) => [
 			...Object.entries(entry.toolchains).map(([toolchain, toolchainVersion]) => ({
 				package: entry.package,
@@ -151,7 +165,7 @@ function verifiedFor(
 	packageName: string,
 	version: string,
 	verified: VerificationResult[],
-	outcome: "success" | "failure",
+	outcome: ResultClass,
 ): VersionCompatibility["verified"] {
 	const result: VersionCompatibility["verified"] = {};
 	for (const pair of verifiedPairs(verified, outcome)) {
@@ -221,11 +235,16 @@ export function cellFor(
 		};
 	}
 
+	// Any success outranks failures: a combination that built once builds.
 	if (entry.verified[key]?.some((verified) => sameLine(verified, toolchainVersion))) {
-		return { state: "verified", reason: "verified by CI" };
+		const alsoFailed = entry.suspect?.[key]?.some((failed) => sameLine(failed, toolchainVersion));
+		return { state: "verified", reason: alsoFailed ? "verified by CI (one other attempt failed)" : "verified by CI" };
 	}
 	if (entry.failed?.[key]?.some((failed) => sameLine(failed, toolchainVersion))) {
-		return { state: "unsupported", reason: "build failed in CI" };
+		return { state: "unsupported", reason: "build failed in two independent CI runs" };
+	}
+	if (entry.suspect?.[key]?.some((failed) => sameLine(failed, toolchainVersion))) {
+		return { state: "unverified", reason: "one CI build failed; a second attempt is pending" };
 	}
 
 	const range = entry.requirements[key];

shared/types.ts

@@ -37,8 +37,10 @@ export interface VersionCompatibility {
 	source: RequirementSource;
 	/** Toolchain versions a CI build proved to work, per key. */
 	verified: Partial<Record<ToolchainKey, string[]>>;
-	/** Toolchain versions a CI build failed with, per key. */
+	/** Toolchain versions whose builds failed in two independent CI runs, per key. */
 	failed?: Partial<Record<ToolchainKey, string[]>>;
+	/** Toolchain versions with a single CI failure, awaiting a second attempt, per key. */
+	suspect?: Partial<Record<ToolchainKey, string[]>>;
 	publishedAt?: string;
 }
 
@@ -105,6 +107,10 @@ export interface VerificationResult {
 	with: { nativescript: string; node: string };
 	/** Omitted means success. */
 	outcome?: "success" | "failure";
+	/** For failures: independent runs that failed this combination, this one included. */
+	attempts?: number;
+	/** The last recognisable error line of a failed build. */
+	signature?: string;
 	resolved?: Record<string, string>;
 	/** Workflow run or job URL that proved it. */
 	evidence?: string;