diff --git a/.github/actions/restore-turbo-cache/action.yml b/.github/actions/restore-turbo-cache/action.yml new file mode 100644 index 0000000000..32c446c948 --- /dev/null +++ b/.github/actions/restore-turbo-cache/action.yml @@ -0,0 +1,13 @@ +name: Restore turbo cache +description: Restore-only turbo cache. warm-caches.yml seeds it on main pushes; PR workflows never save, so per-PR entries don't pollute the cache store. + +runs: + using: composite + steps: + - name: Restore turbo cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .turbo/cache + key: turbo-${{ github.sha }} + restore-keys: | + turbo- diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9fb00dacda..38e6391502 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,29 +59,17 @@ jobs: node-version: 22 cache: "pnpm" + - name: Restore turbo cache + uses: ./.github/actions/restore-turbo-cache + - name: Install dependencies run: pnpm install --frozen-lockfile + env: + # The packaged app is never launched, so the Electron-ABI rebuild is unused. + SKIP_ELECTRON_REBUILD: "1" - - name: Build electron-trpc - run: pnpm --filter @posthog/electron-trpc build - - - name: Build platform - run: pnpm --filter @posthog/platform build - - - name: Build shared - run: pnpm --filter @posthog/shared build - - - name: Build git - run: pnpm --filter @posthog/git build - - - name: Build enricher - run: pnpm --filter @posthog/enricher build - - - name: Build harness - run: pnpm --filter @posthog/harness build - - - name: Build agent - run: pnpm --filter agent build + - name: Build packages + run: pnpm run build:deps - name: Build code run: pnpm --filter code build diff --git a/.github/workflows/code-storybook.yml b/.github/workflows/code-storybook.yml index 6b96e92983..92a42fd72b 100644 --- a/.github/workflows/code-storybook.yml +++ b/.github/workflows/code-storybook.yml @@ -115,8 +115,13 @@ jobs: restore-keys: | playwright-chromium-${{ runner.os }}- + - name: Restore turbo cache + uses: ./.github/actions/restore-turbo-cache + - name: Install dependencies run: pnpm install --frozen-lockfile + env: + SKIP_ELECTRON_REBUILD: "1" - name: Install Playwright Chromium if: steps.playwright-cache.outputs.cache-hit != 'true' @@ -130,38 +135,7 @@ jobs: run: pnpm exec playwright install-deps chromium - name: Build workspace packages - # Bare `wait` returns the exit status of only the last job passed to - # it, silently swallowing failures from the others. `wait_all` waits - # on each PID individually and checks its own status so any failed - # build fails the step. - run: | - wait_all() { - status=0 - for pid in "$@"; do - wait "$pid" || status=$? - done - [ "$status" -eq 0 ] - } - - pnpm --filter @posthog/electron-trpc build & - pid1=$! - (pnpm --filter @posthog/shared build && pnpm --filter @posthog/platform build) & - pid2=$! - wait_all "$pid1" "$pid2" - - # @posthog/agent imports the dist of @posthog/git and - # @posthog/harness, so both must finish before the last group starts. - pnpm --filter @posthog/git build & - pid3=$! - pnpm --filter @posthog/harness build & - pid4=$! - wait_all "$pid3" "$pid4" - - pnpm --filter @posthog/enricher build & - pid5=$! - pnpm --filter @posthog/agent build & - pid6=$! - wait_all "$pid5" "$pid6" + run: pnpm run build:deps - name: Build Storybook working-directory: apps/code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 837103044c..39541a37f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,15 +62,19 @@ jobs: node-version: 22 cache: "pnpm" + # Only dependency builds are cached, tests themselves always run. + - name: Restore turbo cache + uses: ./.github/actions/restore-turbo-cache + - name: Install dependencies run: pnpm install --frozen-lockfile + env: + # The Electron-ABI rebuild would be undone by the Node rebuild below. + SKIP_ELECTRON_REBUILD: "1" - name: Rebuild native modules for Node run: node scripts/rebuild-better-sqlite3-node.mjs - - name: Build dependencies - run: pnpm --filter @posthog/electron-trpc build - - name: Run tests run: pnpm test @@ -127,22 +131,16 @@ jobs: restore-keys: | electron-${{ runner.os }}- + # Seeded on Linux; turbo artifacts are plain JS, so cross-OS restore is fine. + - name: Restore turbo cache + uses: ./.github/actions/restore-turbo-cache + + # apps/code's postinstall rebuilds better-sqlite3 for Electron here. - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Rebuild better-sqlite3 for Electron - run: node scripts/rebuild-better-sqlite3-electron.mjs - - name: Build packages - run: | - pnpm --filter @posthog/electron-trpc build & - pnpm --filter @posthog/platform build & - pnpm --filter @posthog/shared build - pnpm --filter @posthog/git build - pnpm --filter @posthog/enricher build - wait - pnpm --filter @posthog/harness build - pnpm --filter agent build + run: pnpm run build:deps - name: Package Electron app run: pnpm --filter code run package @@ -230,22 +228,24 @@ jobs: persist-credentials: false - name: Setup pnpm - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 cache: "pnpm" + - name: Restore turbo cache + uses: ./.github/actions/restore-turbo-cache + - name: Install dependencies run: pnpm install --frozen-lockfile + env: + SKIP_ELECTRON_REBUILD: "1" - name: Build agent dependencies - run: | - pnpm --filter @posthog/shared run build - pnpm --filter @posthog/git run build - pnpm --filter @posthog/enricher run build + run: pnpm exec turbo build --filter='@posthog/agent^...' - name: Download native codex binary # Non-fatal at the STEP so a failure surfaces as the fail-loud binary guard diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index c469e80216..cb17490b83 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -59,8 +59,13 @@ jobs: node-version: 22 cache: "pnpm" + - name: Restore turbo cache + uses: ./.github/actions/restore-turbo-cache + - name: Install dependencies run: pnpm install --frozen-lockfile + env: + SKIP_ELECTRON_REBUILD: "1" - name: Run type check run: pnpm run typecheck diff --git a/.github/workflows/warm-caches.yml b/.github/workflows/warm-caches.yml new file mode 100644 index 0000000000..a88e2e784c --- /dev/null +++ b/.github/workflows/warm-caches.yml @@ -0,0 +1,117 @@ +name: Warm caches + +# GitHub only shares caches saved on main with other branches, and the PR +# workflows never run on main. This job seeds the caches they restore. + +on: + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: warm-caches-${{ github.ref }} + cancel-in-progress: true + +jobs: + linux: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + cache: "pnpm" + + # No restore-keys: a fresh save per commit keeps the cache from growing + # forever, at the cost of a cold build on every main push. + - name: Cache turbo artifacts + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .turbo/cache + key: turbo-${{ github.sha }} + + # Same key as code-storybook.yml's storybook job. + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + + - name: Install dependencies + run: pnpm install --frozen-lockfile + env: + SKIP_ELECTRON_REBUILD: "1" + + - name: Seed turbo cache (build + typecheck) + run: | + pnpm run build:deps + pnpm run typecheck + + - name: Install Playwright Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + working-directory: apps/code + run: pnpm exec playwright install chromium + + macos: + runs-on: macos-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + + - name: Setup Node.js + id: node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + cache: "pnpm" + + # Same keys as test.yml's integration-test job. + - name: Cache Electron binary + id: electron-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/Library/Caches/electron + key: electron-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/Library/Caches/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + + # macOS minutes are 10x; do nothing when every cache is already warm. + - name: Install dependencies + if: >- + steps.node.outputs.cache-hit != 'true' || + steps.electron-cache.outputs.cache-hit != 'true' || + steps.playwright-cache.outputs.cache-hit != 'true' + # Electron's own postinstall fills ~/Library/Caches/electron; the sqlite + # rebuild only touches node_modules, which is not cached, so skip it. + run: pnpm install --frozen-lockfile + env: + SKIP_ELECTRON_REBUILD: "1" + + - name: Install Playwright Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: pnpm --filter code exec playwright install chromium diff --git a/apps/code/scripts/postinstall.sh b/apps/code/scripts/postinstall.sh index 4f12df1a62..7c33b064f8 100755 --- a/apps/code/scripts/postinstall.sh +++ b/apps/code/scripts/postinstall.sh @@ -8,23 +8,28 @@ set -e REPO_ROOT="$(cd ../.. && pwd)" SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -# Self-heal missing Electron binary. -# pnpm skips package-level postinstall scripts when the lockfile is already -# satisfied, so if node_modules/electron/dist gets wiped (interrupted download, -# cache eviction, arch change, manual cleanup), `pnpm install` won't notice — -# and `electron-vite dev` then fails with "Electron failed to install -# correctly, please delete node_modules/electron and try installing again". -# Detect the missing binary and invoke Electron's own install script to fetch it. -ELECTRON_DIST="$REPO_ROOT/node_modules/electron/dist" -if [ ! -d "$ELECTRON_DIST" ] || [ -z "$(ls -A "$ELECTRON_DIST" 2>/dev/null)" ]; then - echo "Electron binary missing at $ELECTRON_DIST — downloading..." - node "$REPO_ROOT/node_modules/electron/install.js" -fi +# CI jobs that never run Electron set this to skip ~60s of node-gyp per install. +if [ "$SKIP_ELECTRON_REBUILD" = "1" ] || [ "$SKIP_ELECTRON_REBUILD" = "true" ]; then + echo "SKIP_ELECTRON_REBUILD=$SKIP_ELECTRON_REBUILD: skipping Electron binary check and native rebuild." +else + # Self-heal missing Electron binary. + # pnpm skips package-level postinstall scripts when the lockfile is already + # satisfied, so if node_modules/electron/dist gets wiped (interrupted download, + # cache eviction, arch change, manual cleanup), `pnpm install` won't notice — + # and `electron-vite dev` then fails with "Electron failed to install + # correctly, please delete node_modules/electron and try installing again". + # Detect the missing binary and invoke Electron's own install script to fetch it. + ELECTRON_DIST="$REPO_ROOT/node_modules/electron/dist" + if [ ! -d "$ELECTRON_DIST" ] || [ -z "$(ls -A "$ELECTRON_DIST" 2>/dev/null)" ]; then + echo "Electron binary missing at $ELECTRON_DIST — downloading..." + node "$REPO_ROOT/node_modules/electron/install.js" + fi -echo "Rebuilding native modules for Electron..." + echo "Rebuilding native modules for Electron..." -cd "$REPO_ROOT" -node scripts/rebuild-better-sqlite3-electron.mjs + cd "$REPO_ROOT" + node scripts/rebuild-better-sqlite3-electron.mjs +fi # Restore the execute bit on node-pty's spawn-helper. pnpm extracts node-pty's # prebuilt binaries without preserving the executable mode, so the helper lands