ci(merge-queue): CircleCI-scheduled reconcile heartbeat, surviving Actions outages

.circleci/config.yml

@@ -727,6 +727,27 @@ jobs:
           name: 'check @teambit/harmony version synchronization'
           command: './scripts/check-harmony-version-sync.sh'
 
+  # Runs the same reconcile script the merge-queue GitHub workflow runs — see the
+  # merge_queue_reconcile workflow below for why it also lives here.
+  merge_queue_reconcile:
+    <<: *defaults
+    steps:
+      - checkout
+      - run:
+          name: 'reconcile the merge queue'
+          # no CIRCLE_TOKEN needed: the project is public, so the script's CircleCI reads work
+          # unauthenticated (it uses the token only when one is present in the environment).
+          # A hung API call would otherwise let a run outlive the 10m schedule interval — the
+          # tight no_output_timeout bounds it (the script normally finishes in seconds).
+          no_output_timeout: 5m
+          command: |
+            if [ -z "$GH_RELEASE_GITHUB_API_TOKEN" ]; then
+              echo "The GH_RELEASE_GITHUB_API_TOKEN project env var is not set — the reconcile"
+              echo "script needs it as GITHUB_TOKEN (statuses+issues write)."
+              exit 1
+            fi
+            GITHUB_TOKEN=$GH_RELEASE_GITHUB_API_TOKEN node .github/scripts/merge-queue.js
+
   generate_and_check_types:
     <<: *defaults
     steps:
@@ -1591,6 +1612,30 @@ workflows:
     jobs:
       - update_e2e_timings
 
+  # Independent heartbeat for the merge queue. The GitHub Actions side of the queue is
+  # event-driven, but its only time-based fallback is the Actions cron — which GitHub throttles
+  # far beyond its 5m spec (20-40m silent gaps), and which vanishes entirely when Actions itself
+  # is down (2026-08-06 outage: the queue froze and only admins could bypass the gate). CircleCI
+  # is independent infrastructure that kept working through that outage, so this schedule runs the
+  # very same reconcile script from here. The script is stateless and idempotent — concurrent runs
+  # with the Actions-side reconciles are harmless. It can also be run from any laptop as a
+  # break-glass: GITHUB_TOKEN=<pat> node .github/scripts/merge-queue.js
+  merge_queue_heartbeat:
+    triggers:
+      - schedule:
+          # CircleCI's config-level cron rejects step syntax (*/10) — list the minutes explicitly
+          cron: '4,14,24,34,44,54 * * * *'
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - merge_queue_reconcile:
+          # serialize heartbeat runs against each other (same mechanism bit_merge uses) so a
+          # slow run can't overlap the next tick; overlap with the Actions-side reconciles
+          # remains possible and harmless — the script is stateless and idempotent
+          serial-group: 'merge-queue-reconcile'
+
   # Windows nightly workflow
   # windows-nightly:
     # triggers:

.github/scripts/merge-queue.js

@@ -33,9 +33,16 @@
  *   "Merge Queue Dashboard" issue (label: merge-queue) is kept up to date.
  *
  * The loop is stateless and idempotent: every run re-derives the queue from the GitHub + CircleCI
- * APIs, so a skipped or crashed run costs nothing.
+ * APIs, so a skipped or crashed run costs nothing. It runs from three places for redundancy:
+ * the GitHub Actions workflow (event-driven + cron), a CircleCI scheduled workflow
+ * (merge_queue_heartbeat, every 10m — survives Actions outages on independent infrastructure),
+ * and, as a break-glass, any machine: GITHUB_TOKEN=<pat> node .github/scripts/merge-queue.js
+ * (add MERGE_QUEUE_DRY_RUN=true to observe without mutating).
  *
- * Required env: GITHUB_TOKEN (statuses+issues write), CIRCLE_TOKEN (CircleCI API, read).
+ * Required env: GITHUB_TOKEN (statuses+issues write). Optional: CIRCLE_TOKEN (CircleCI API,
+ * read) — the project is public so unauthenticated reads work; set it only to avoid shared-IP
+ * rate limits (e.g. on hosted runners). Either way a failed CircleCI read fails the run — the
+ * queue never treats "couldn't check bit_merge" as settled.
  * Optional env: MERGE_QUEUE_IGNORE_CHECKS — comma-separated check names to ignore when deciding
  * whether a PR is green.
  *
@@ -75,6 +82,13 @@ const ACTIVE_JOB_STATUSES = new Set(['running', 'queued', 'not_running', 'blocke
 // workflow statuses under which a contained bit_merge job could still be active; anything
 // terminal (success/failed/error/canceled/not_run) is skipped without fetching its jobs.
 const ACTIVE_WORKFLOW_STATUSES = new Set(['created', 'running', 'failing', 'on_hold']);
+// check runs created by the queue's own GitHub workflows (the reconcile job and the review-ping
+// bridge job) appear on PR head commits; their failures or hangs are queue-infrastructure
+// artifacts (e.g. the 2026-08-06 Actions outage failed every reconcile run, which then blocked
+// the PRs as "checks failing"), not verdicts about the PR — never let them gate the queue.
+// Matched by owning workflow name (not job name) so unrelated checks that happen to share a job
+// name are still honored.
+const SELF_WORKFLOW_NAMES = new Set(['merge-queue', 'merge-queue-review-ping']);
 const MAX_STATUS_DESCRIPTION_LENGTH = 140;
 const NOT_QUEUED_DESCRIPTION = 'not queued — enable auto-merge (squash) to join the merge queue';
 
@@ -115,14 +129,34 @@ async function githubGraphql(query) {
   return result.data;
 }
 
+// transient CircleCI failures get bounded retries: the reconcile is this queue's outage
+// fallback, and (especially unauthenticated) reads can hit rate limits or blips — one 429/5xx
+// must not abort a whole heartbeat run. Hard failures (401/404/...) still throw immediately,
+// and exhausting retries throws too: "couldn't check bit_merge" is never treated as settled.
+const RETRYABLE_CIRCLE_STATUSES = new Set([429, 500, 502, 503, 504]);
+
 async function circleRequest(path) {
-  const response = await fetch(`https://circleci.com/api/v2${path}`, {
-    headers: { 'Circle-Token': circleToken },
-  });
-  if (!response.ok) {
-    throw new Error(`CircleCI GET ${path} failed: ${response.status} ${await response.text()}`);
+  const maxAttempts = 3;
+  for (let attempt = 1; ; attempt += 1) {
+    let response;
+    try {
+      response = await fetch(`https://circleci.com/api/v2${path}`, {
+        headers: circleToken ? { 'Circle-Token': circleToken } : {},
+      });
+    } catch (error) {
+      if (attempt === maxAttempts) throw error;
+      console.log(`CircleCI GET ${path} network error (attempt ${attempt}/${maxAttempts}): ${error.message}`);
+      await sleep(attempt * 3000);
+      continue;
+    }
+    if (response.ok) return response.json();
+    const body = await response.text();
+    if (attempt === maxAttempts || !RETRYABLE_CIRCLE_STATUSES.has(response.status)) {
+      throw new Error(`CircleCI GET ${path} failed: ${response.status} ${body}`);
+    }
+    console.log(`CircleCI GET ${path} got ${response.status} (attempt ${attempt}/${maxAttempts}), retrying`);
+    await sleep(attempt * 3000);
   }
-  return response.json();
 }
 
 async function getMasterPipelinesWithinWindow() {
@@ -229,7 +263,7 @@ async function fetchOpenPullRequestsPage(cursor) {
                     nodes {
                       __typename
                       ... on StatusContext { context state description targetUrl createdAt }
-                      ... on CheckRun { name status conclusion }
+                      ... on CheckRun { name status conclusion checkSuite { workflowRun { workflow { name } } } }
                     }
                   }
                 }
@@ -333,6 +367,7 @@ function evaluateChecks(pullRequest) {
       else pendingCount += 1; // PENDING / EXPECTED
     } else if (context.__typename === 'CheckRun') {
       if (ignoredContexts.has(context.name)) continue;
+      if (SELF_WORKFLOW_NAMES.has(context.checkSuite?.workflowRun?.workflow?.name)) continue;
       evaluatedCount += 1;
       if (context.status !== 'COMPLETED') {
         pendingCount += 1;
@@ -525,10 +560,7 @@ async function updateDashboard({ masterState, entries, winner, updateCandidate,
 async function main() {
   if (!githubToken) throw new Error('GITHUB_TOKEN is required');
   if (!circleToken) {
-    throw new Error(
-      'CIRCLE_TOKEN is required — a CircleCI API token, used to detect whether bit_merge is active on master. ' +
-        'Without it the queue cannot tell when master is settled, so it refuses to run rather than merge blindly.'
-    );
+    console.log('CIRCLE_TOKEN not set — using unauthenticated CircleCI reads (public project)');
   }
 
   const masterState = await getMasterState();

.github/workflows/merge-queue.yml

@@ -29,7 +29,10 @@
 #   pull_request_review runs the PR merge commit's workflow copy, which must never see
 #   CIRCLE_TOKEN; the secret-less ping workflow absorbs that context, and its completion lands
 #   here on master's trusted copy.
-# - schedule: safety net for missed events.
+# - schedule: safety net for missed events. Not the only one: a CircleCI scheduled workflow
+#   (merge_queue_heartbeat in .circleci/config.yml) runs the same script every 10m from
+#   independent infrastructure — GitHub's cron is heavily throttled, and an Actions outage
+#   (2026-08-06) froze this workflow entirely, blocking every non-admin merge.
 name: merge-queue
 
 on: