From 0ed615a3398370a2de013e1e4ccaa35394491f84 Mon Sep 17 00:00:00 2001 From: David First Date: Thu, 6 Aug 2026 14:23:05 -0400 Subject: [PATCH 1/4] ci(merge-queue): run the reconcile from a CircleCI schedule too, surviving Actions outages --- .circleci/config.yml | 36 +++++++++++++++++++++++++++++++ .github/scripts/merge-queue.js | 6 +++++- .github/workflows/merge-queue.yml | 5 ++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0a02765da95..9f8d5961b1ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -727,6 +727,22 @@ jobs: name: 'check @teambit/harmony version synchronization' command: './scripts/check-harmony-version-sync.sh' + # Runs the same reconcile script the merge-queue GitHub workflow runs — see the + # merge_queue_reconcile workflow below for why it also lives here. + merge_queue_reconcile: + <<: *defaults + steps: + - checkout + - run: + name: 'reconcile the merge queue' + command: | + if [ -z "$CIRCLE_TOKEN" ]; then + echo "The CIRCLE_TOKEN project env var is not set; the reconcile script requires it" + echo "(a CircleCI API token, read scope — used to detect an active bit_merge on master)." + exit 1 + fi + GITHUB_TOKEN=$GH_RELEASE_GITHUB_API_TOKEN node .github/scripts/merge-queue.js + generate_and_check_types: <<: *defaults steps: @@ -1591,6 +1607,26 @@ workflows: jobs: - update_e2e_timings + # Independent heartbeat for the merge queue. The GitHub Actions side of the queue is + # event-driven, but its only time-based fallback is the Actions cron — which GitHub throttles + # far beyond its 5m spec (20-40m silent gaps), and which vanishes entirely when Actions itself + # is down (2026-08-06 outage: the queue froze and only admins could bypass the gate). CircleCI + # is independent infrastructure that kept working through that outage, so this schedule runs the + # very same reconcile script from here. The script is stateless and idempotent — concurrent runs + # with the Actions-side reconciles are harmless. It can also be run from any laptop as a + # break-glass: GITHUB_TOKEN= CIRCLE_TOKEN= node .github/scripts/merge-queue.js + merge_queue_heartbeat: + triggers: + - schedule: + # CircleCI's config-level cron rejects step syntax (*/10) — list the minutes explicitly + cron: '4,14,24,34,44,54 * * * *' + filters: + branches: + only: + - master + jobs: + - merge_queue_reconcile + # Windows nightly workflow # windows-nightly: # triggers: diff --git a/.github/scripts/merge-queue.js b/.github/scripts/merge-queue.js index 1325385cd763..52d39945444c 100644 --- a/.github/scripts/merge-queue.js +++ b/.github/scripts/merge-queue.js @@ -33,7 +33,11 @@ * "Merge Queue Dashboard" issue (label: merge-queue) is kept up to date. * * The loop is stateless and idempotent: every run re-derives the queue from the GitHub + CircleCI - * APIs, so a skipped or crashed run costs nothing. + * APIs, so a skipped or crashed run costs nothing. It runs from three places for redundancy: + * the GitHub Actions workflow (event-driven + cron), a CircleCI scheduled workflow + * (merge_queue_heartbeat, every 10m — survives Actions outages on independent infrastructure), + * and, as a break-glass, any machine: GITHUB_TOKEN= CIRCLE_TOKEN= node + * .github/scripts/merge-queue.js (add MERGE_QUEUE_DRY_RUN=true to observe without mutating). * * Required env: GITHUB_TOKEN (statuses+issues write), CIRCLE_TOKEN (CircleCI API, read). * Optional env: MERGE_QUEUE_IGNORE_CHECKS — comma-separated check names to ignore when deciding diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml index 6af536e3de48..db6e9d1eb4ce 100644 --- a/.github/workflows/merge-queue.yml +++ b/.github/workflows/merge-queue.yml @@ -29,7 +29,10 @@ # pull_request_review runs the PR merge commit's workflow copy, which must never see # CIRCLE_TOKEN; the secret-less ping workflow absorbs that context, and its completion lands # here on master's trusted copy. -# - schedule: safety net for missed events. +# - schedule: safety net for missed events. Not the only one: a CircleCI scheduled workflow +# (merge_queue_heartbeat in .circleci/config.yml) runs the same script every 10m from +# independent infrastructure — GitHub's cron is heavily throttled, and an Actions outage +# (2026-08-06) froze this workflow entirely, blocking every non-admin merge. name: merge-queue on: From c62ad96ff21b65ee01e303e582273531b38e5200 Mon Sep 17 00:00:00 2001 From: David First Date: Thu, 6 Aug 2026 15:10:27 -0400 Subject: [PATCH 2/4] ci(merge-queue): no CircleCI token needed (public API), ignore the queue's own check runs --- .circleci/config.yml | 12 ++++-------- .github/scripts/merge-queue.js | 22 ++++++++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f8d5961b1ad..ed56993ba653 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -735,13 +735,9 @@ jobs: - checkout - run: name: 'reconcile the merge queue' - command: | - if [ -z "$CIRCLE_TOKEN" ]; then - echo "The CIRCLE_TOKEN project env var is not set; the reconcile script requires it" - echo "(a CircleCI API token, read scope — used to detect an active bit_merge on master)." - exit 1 - fi - GITHUB_TOKEN=$GH_RELEASE_GITHUB_API_TOKEN node .github/scripts/merge-queue.js + # no CIRCLE_TOKEN needed: the project is public, so the script's CircleCI reads work + # unauthenticated (it uses the token only when one is present in the environment) + command: GITHUB_TOKEN=$GH_RELEASE_GITHUB_API_TOKEN node .github/scripts/merge-queue.js generate_and_check_types: <<: *defaults @@ -1614,7 +1610,7 @@ workflows: # is independent infrastructure that kept working through that outage, so this schedule runs the # very same reconcile script from here. The script is stateless and idempotent — concurrent runs # with the Actions-side reconciles are harmless. It can also be run from any laptop as a - # break-glass: GITHUB_TOKEN= CIRCLE_TOKEN= node .github/scripts/merge-queue.js + # break-glass: GITHUB_TOKEN= node .github/scripts/merge-queue.js merge_queue_heartbeat: triggers: - schedule: diff --git a/.github/scripts/merge-queue.js b/.github/scripts/merge-queue.js index 52d39945444c..a0e805a71dbe 100644 --- a/.github/scripts/merge-queue.js +++ b/.github/scripts/merge-queue.js @@ -36,10 +36,13 @@ * APIs, so a skipped or crashed run costs nothing. It runs from three places for redundancy: * the GitHub Actions workflow (event-driven + cron), a CircleCI scheduled workflow * (merge_queue_heartbeat, every 10m — survives Actions outages on independent infrastructure), - * and, as a break-glass, any machine: GITHUB_TOKEN= CIRCLE_TOKEN= node - * .github/scripts/merge-queue.js (add MERGE_QUEUE_DRY_RUN=true to observe without mutating). + * and, as a break-glass, any machine: GITHUB_TOKEN= node .github/scripts/merge-queue.js + * (add MERGE_QUEUE_DRY_RUN=true to observe without mutating). * - * Required env: GITHUB_TOKEN (statuses+issues write), CIRCLE_TOKEN (CircleCI API, read). + * Required env: GITHUB_TOKEN (statuses+issues write). Optional: CIRCLE_TOKEN (CircleCI API, + * read) — the project is public so unauthenticated reads work; set it only to avoid shared-IP + * rate limits (e.g. on hosted runners). Either way a failed CircleCI read fails the run — the + * queue never treats "couldn't check bit_merge" as settled. * Optional env: MERGE_QUEUE_IGNORE_CHECKS — comma-separated check names to ignore when deciding * whether a PR is green. * @@ -121,7 +124,7 @@ async function githubGraphql(query) { async function circleRequest(path) { const response = await fetch(`https://circleci.com/api/v2${path}`, { - headers: { 'Circle-Token': circleToken }, + headers: circleToken ? { 'Circle-Token': circleToken } : {}, }); if (!response.ok) { throw new Error(`CircleCI GET ${path} failed: ${response.status} ${await response.text()}`); @@ -319,8 +322,14 @@ function evaluateChecks(pullRequest) { console.log(` #${pullRequest.number}: more than 100 check contexts, treating as pending (raise the page size)`); return { state: 'pending', failing: [] }; } + // 'reconcile' (this queue's own workflow job) and 'ping' (the review bridge job) appear as + // check runs on PR head commits; their failures or hangs are queue-infrastructure artifacts + // (e.g. the 2026-08-06 Actions outage failed every reconcile run, which then blocked the PRs + // as "checks failing"), not verdicts about the PR — never let them gate the queue + const selfCheckRunNames = ['reconcile', 'ping']; const ignoredContexts = new Set( [GATE_CONTEXT] + .concat(selfCheckRunNames) .concat((process.env.MERGE_QUEUE_IGNORE_CHECKS || '').split(',')) .map((name) => name.trim()) .filter(Boolean) @@ -529,10 +538,7 @@ async function updateDashboard({ masterState, entries, winner, updateCandidate, async function main() { if (!githubToken) throw new Error('GITHUB_TOKEN is required'); if (!circleToken) { - throw new Error( - 'CIRCLE_TOKEN is required — a CircleCI API token, used to detect whether bit_merge is active on master. ' + - 'Without it the queue cannot tell when master is settled, so it refuses to run rather than merge blindly.' - ); + console.log('CIRCLE_TOKEN not set — using unauthenticated CircleCI reads (public project)'); } const masterState = await getMasterState(); From 9d1d1d912a1c3c5b894e15a0826e1c75dc83c575 Mon Sep 17 00:00:00 2001 From: David First Date: Fri, 7 Aug 2026 10:15:00 -0400 Subject: [PATCH 3/4] ci(merge-queue): serialize heartbeat runs, bound hangs, name the missing token --- .circleci/config.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed56993ba653..051c61465f96 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -736,8 +736,17 @@ jobs: - run: name: 'reconcile the merge queue' # no CIRCLE_TOKEN needed: the project is public, so the script's CircleCI reads work - # unauthenticated (it uses the token only when one is present in the environment) - command: GITHUB_TOKEN=$GH_RELEASE_GITHUB_API_TOKEN node .github/scripts/merge-queue.js + # unauthenticated (it uses the token only when one is present in the environment). + # A hung API call would otherwise let a run outlive the 10m schedule interval — the + # tight no_output_timeout bounds it (the script normally finishes in seconds). + no_output_timeout: 5m + command: | + if [ -z "$GH_RELEASE_GITHUB_API_TOKEN" ]; then + echo "The GH_RELEASE_GITHUB_API_TOKEN project env var is not set — the reconcile" + echo "script needs it as GITHUB_TOKEN (statuses+issues write)." + exit 1 + fi + GITHUB_TOKEN=$GH_RELEASE_GITHUB_API_TOKEN node .github/scripts/merge-queue.js generate_and_check_types: <<: *defaults @@ -1621,7 +1630,11 @@ workflows: only: - master jobs: - - merge_queue_reconcile + - merge_queue_reconcile: + # serialize heartbeat runs against each other (same mechanism bit_merge uses) so a + # slow run can't overlap the next tick; overlap with the Actions-side reconciles + # remains possible and harmless — the script is stateless and idempotent + serial-group: 'merge-queue-reconcile' # Windows nightly workflow # windows-nightly: From a4d8170ded18d266697e2233302c1345f174f8fa Mon Sep 17 00:00:00 2001 From: David First Date: Fri, 7 Aug 2026 11:18:28 -0400 Subject: [PATCH 4/4] ci(merge-queue): match self-checks by owning workflow, retry transient CircleCI errors --- .github/scripts/merge-queue.js | 48 +++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/scripts/merge-queue.js b/.github/scripts/merge-queue.js index a0e805a71dbe..5dbad81cfe41 100644 --- a/.github/scripts/merge-queue.js +++ b/.github/scripts/merge-queue.js @@ -82,6 +82,13 @@ const ACTIVE_JOB_STATUSES = new Set(['running', 'queued', 'not_running', 'blocke // workflow statuses under which a contained bit_merge job could still be active; anything // terminal (success/failed/error/canceled/not_run) is skipped without fetching its jobs. const ACTIVE_WORKFLOW_STATUSES = new Set(['created', 'running', 'failing', 'on_hold']); +// check runs created by the queue's own GitHub workflows (the reconcile job and the review-ping +// bridge job) appear on PR head commits; their failures or hangs are queue-infrastructure +// artifacts (e.g. the 2026-08-06 Actions outage failed every reconcile run, which then blocked +// the PRs as "checks failing"), not verdicts about the PR — never let them gate the queue. +// Matched by owning workflow name (not job name) so unrelated checks that happen to share a job +// name are still honored. +const SELF_WORKFLOW_NAMES = new Set(['merge-queue', 'merge-queue-review-ping']); const MAX_STATUS_DESCRIPTION_LENGTH = 140; const NOT_QUEUED_DESCRIPTION = 'not queued — enable auto-merge (squash) to join the merge queue'; @@ -122,14 +129,34 @@ async function githubGraphql(query) { return result.data; } +// transient CircleCI failures get bounded retries: the reconcile is this queue's outage +// fallback, and (especially unauthenticated) reads can hit rate limits or blips — one 429/5xx +// must not abort a whole heartbeat run. Hard failures (401/404/...) still throw immediately, +// and exhausting retries throws too: "couldn't check bit_merge" is never treated as settled. +const RETRYABLE_CIRCLE_STATUSES = new Set([429, 500, 502, 503, 504]); + async function circleRequest(path) { - const response = await fetch(`https://circleci.com/api/v2${path}`, { - headers: circleToken ? { 'Circle-Token': circleToken } : {}, - }); - if (!response.ok) { - throw new Error(`CircleCI GET ${path} failed: ${response.status} ${await response.text()}`); + const maxAttempts = 3; + for (let attempt = 1; ; attempt += 1) { + let response; + try { + response = await fetch(`https://circleci.com/api/v2${path}`, { + headers: circleToken ? { 'Circle-Token': circleToken } : {}, + }); + } catch (error) { + if (attempt === maxAttempts) throw error; + console.log(`CircleCI GET ${path} network error (attempt ${attempt}/${maxAttempts}): ${error.message}`); + await sleep(attempt * 3000); + continue; + } + if (response.ok) return response.json(); + const body = await response.text(); + if (attempt === maxAttempts || !RETRYABLE_CIRCLE_STATUSES.has(response.status)) { + throw new Error(`CircleCI GET ${path} failed: ${response.status} ${body}`); + } + console.log(`CircleCI GET ${path} got ${response.status} (attempt ${attempt}/${maxAttempts}), retrying`); + await sleep(attempt * 3000); } - return response.json(); } async function getMasterPipelinesWithinWindow() { @@ -236,7 +263,7 @@ async function fetchOpenPullRequestsPage(cursor) { nodes { __typename ... on StatusContext { context state description targetUrl createdAt } - ... on CheckRun { name status conclusion } + ... on CheckRun { name status conclusion checkSuite { workflowRun { workflow { name } } } } } } } @@ -322,14 +349,8 @@ function evaluateChecks(pullRequest) { console.log(` #${pullRequest.number}: more than 100 check contexts, treating as pending (raise the page size)`); return { state: 'pending', failing: [] }; } - // 'reconcile' (this queue's own workflow job) and 'ping' (the review bridge job) appear as - // check runs on PR head commits; their failures or hangs are queue-infrastructure artifacts - // (e.g. the 2026-08-06 Actions outage failed every reconcile run, which then blocked the PRs - // as "checks failing"), not verdicts about the PR — never let them gate the queue - const selfCheckRunNames = ['reconcile', 'ping']; const ignoredContexts = new Set( [GATE_CONTEXT] - .concat(selfCheckRunNames) .concat((process.env.MERGE_QUEUE_IGNORE_CHECKS || '').split(',')) .map((name) => name.trim()) .filter(Boolean) @@ -346,6 +367,7 @@ function evaluateChecks(pullRequest) { else pendingCount += 1; // PENDING / EXPECTED } else if (context.__typename === 'CheckRun') { if (ignoredContexts.has(context.name)) continue; + if (SELF_WORKFLOW_NAMES.has(context.checkSuite?.workflowRun?.workflow?.name)) continue; evaluatedCount += 1; if (context.status !== 'COMPLETED') { pendingCount += 1;