test(e2e): determinism cleanup + e2e performance (workers, 8-core, Chromium cache) (#2655)

Combines the E2E/test-infra determinism work with the e2e performance work — they both tune `playwright.config.ts` and the e2e pipeline, so they ship as one PR.

## What changed

**Determinism**
- Removed the three blind `waitForTimeout` sleeps (replaced with awaited conditions) and resolved the permanently-skipped tests (deleted the dead navigation block, fixed + un-skipped the Kubernetes restarts-sort test, deleted `timeQuery.test.tsx` which can't run without a `nuqs` major upgrade; a genuinely-conditional skip in `sources.spec.ts` stays). Deleting `timeQuery.test.tsx` orphaned `src/fixtures.ts`, the `@jedmao/location` dev dep, and the `getLiveTailTimeRange` export — all removed as verified-dead code.
- Added env-overridable retry counts: `E2E_RETRIES` (Playwright) and `JEST_RETRY_TIMES` (API Jest), for the nightly zero-retry flake hunt.

**Performance**
- Playwright `workers` raised to 4 on CI (via `E2E_WORKERS`), and the e2e shards moved to `8-core-playwright` GitHub-managed larger runners so 4 workers have CPU headroom (4 workers on the old 4-vCPU runner over-subscribed and timed out — validated the 8-core run halves shard time).
- The Chromium binary is cached (`~/.cache/ms-playwright`, keyed on the Playwright version) so it isn't re-downloaded each shard.
- e2e-setup uses `yarn install --immutable`.

## Key decisions

- Both retry knobs and `E2E_WORKERS` parse to a validated integer (retries ≥ 0 honouring an explicit 0 for the flake hunt; workers ≥ 1), falling back to the default on blank/negative/non-numeric input rather than silently coercing to 0/NaN.
- Retries stay on for PRs (pragmatic); the nightly zero-retry hunt is what enforces "a flaky test is a defect".

## Impact

- Faster, less-flaky e2e; retry/worker behaviour tunable via env without code changes.
- Test-infra + CI only — no changeset.

<details>
<summary>Implementation detail</summary>

- Kubernetes test made deterministic via the seed (`restarts = podIdx % 7`) + default `restarts` desc sort. Dashboard save-wait uses the same success-toast locator as sibling tests.
- 8-core-playwright confirmed a GitHub-managed larger runner (ephemeral/isolated — no fork-PR self-hosted RCE surface).

</details>

Author: jordan-simonovski

.github/actions/e2e-setup/action.yml

@@ -36,12 +36,32 @@ runs:
 
     - name: Install dependencies
       shell: bash
-      run: yarn install
+      run: yarn install --immutable
 
     - name: Build dependencies
       shell: bash
       run: npx nx run-many -t ci:build
 
+    # Cache the Chromium binary (~120MB) keyed on the resolved Playwright
+    # version, so it isn't re-downloaded every shard/run. Browser versions are
+    # pinned to the Playwright package version, so the version is the correct
+    # cache key. On a hit, `playwright install` finds the browser present and
+    # only installs OS deps.
+    - name: Resolve Playwright version
+      id: playwright-version
+      shell: bash
+      run:
+        echo "version=$(cd packages/app && node -p
+        "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+
+    - name: Cache Playwright browsers
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ms-playwright
+        key:
+          ${{ runner.os }}-playwright-${{
+          steps.playwright-version.outputs.version }}
+
     - name: Install Playwright browsers
       shell: bash
       run: cd packages/app && npx playwright install --with-deps chromium

.github/workflows/e2e-tests.yml

@@ -5,7 +5,10 @@ on:
 jobs:
   e2e-tests:
     name: E2E Tests - Shard ${{ matrix.shard }}
-    runs-on: ubuntu-24.04
+    # 8-core runner so the 4 Playwright workers (see playwright.config.ts) have
+    # CPU headroom alongside the app + ClickHouse + MongoDB — 4 workers
+    # over-subscribed the 4-vCPU ubuntu-24.04 runner and caused timeouts.
+    runs-on: 8-core-playwright
     timeout-minutes: 15
     permissions:
       contents: read

packages/api/jest.setup.ts

@@ -1,4 +1,14 @@
-jest.retryTimes(1, { logErrorsBeforeRetry: true });
+// Overridable via JEST_RETRY_TIMES (e.g. the nightly zero-retry flake hunt).
+// Honours an explicit 0 so retries can be forced off; an unset/blank/negative/
+// non-integer value falls back to the default of 1 rather than coercing to 0.
+const rawJestRetries = process.env.JEST_RETRY_TIMES;
+const parsedJestRetries = rawJestRetries ? Number(rawJestRetries) : NaN;
+jest.retryTimes(
+  Number.isInteger(parsedJestRetries) && parsedJestRetries >= 0
+    ? parsedJestRetries
+    : 1,
+  { logErrorsBeforeRetry: true },
+);
 
 // http-proxy-middleware v4 is ESM-only and Jest's CJS module loader cannot
 // load ESM packages. Auto-mock since no test exercises the proxy directly.

packages/app/package.json

@@ -104,7 +104,6 @@
     "@chromatic-com/storybook": "^4.1.3",
     "@eslint-react/eslint-plugin": "^3.0.0",
     "@eslint/compat": "^2.0.0",
-    "@jedmao/location": "^3.0.0",
     "@next/eslint-plugin-next": "^16.0.10",
     "@playwright/test": "^1.57.0",
     "@storybook/addon-docs": "^10.1.4",

packages/app/playwright.config.ts

@@ -30,10 +30,25 @@ export default defineConfig({
   fullyParallel: true,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
-  /* Retry on CI only */
-  retries: process.env.CI ? 2 : 1,
-  /* Use multiple workers on CI for faster execution */
-  workers: process.env.CI ? 2 : undefined,
+  /* Retry on CI only. Overridable via E2E_RETRIES (e.g. the nightly zero-retry
+   * flake hunt). Honours an explicit 0; an unset/blank/negative/non-integer
+   * value falls back to the default rather than silently coercing to 0. */
+  retries: (() => {
+    const raw = process.env.E2E_RETRIES;
+    const parsed = raw ? Number(raw) : NaN;
+    if (Number.isInteger(parsed) && parsed >= 0) return parsed;
+    return process.env.CI ? 2 : 1;
+  })(),
+  /* Use multiple workers on CI for faster execution. CI e2e shards run on
+   * 8-core runners (see .github/workflows/e2e-tests.yml), so 4 workers leave
+   * headroom for the app + ClickHouse + Mongo. Override with E2E_WORKERS
+   * (a positive integer); anything else falls back to the default. */
+  workers: (() => {
+    const raw = process.env.E2E_WORKERS;
+    const parsed = raw ? Number(raw) : NaN;
+    if (Number.isInteger(parsed) && parsed >= 1) return parsed;
+    return process.env.CI ? 4 : undefined;
+  })(),
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [
     ['html'],