diff --git a/.github/RUNNERS.md b/.github/RUNNERS.md index fc0d6df..96eeba5 100644 --- a/.github/RUNNERS.md +++ b/.github/RUNNERS.md @@ -1,8 +1,9 @@ -# Self-Hosted Runners +# Runners And Trusted Labs OpenPhone uses GitHub-hosted runners for lightweight repository checks and -self-hosted runners for work that needs a full Android tree or a physical -phone. +GCP lab VMs for trusted Android build and emulator work. Self-hosted runners +are reserved for local hardware that cannot be represented by disposable cloud +VMs, such as a physical phone connected over USB. Do not document private hostnames, SSH key names, public IP addresses, API keys, signing key paths, or device secrets in this file. Keep environment- @@ -12,34 +13,20 @@ specific operator notes outside the public repository. | Label | Used by | Purpose | | --- | --- | --- | -| `openphone-build` | `release.yml` | Builds Pixel/device artifacts on a large Linux Android build host. | | `openphone-emulator` | `emulator.yml` | Builds or reuses an OpenPhone SDK phone image, boots it in the Android emulator, and runs fast runtime smoke checks. | | `openphone-device` | `eval.yml` | Runs evals against an authorized Android device connected over USB. | -## `openphone-build` +## GCP Lab Builds -The build runner needs: +Release builds and trusted PR emulator lab runs are orchestrated from +GitHub-hosted runners with Workload Identity Federation. The workflow creates a +disposable GCP VM, attaches or restores warm Android cache state, runs the +build/test commands inside that VM, copies artifacts back to GitHub Actions, +and deletes the VM unless debugging explicitly keeps it. -- Linux x86_64 host. -- Full Android build dependencies for the selected LineageOS branch. -- Several hundred GB of free disk space. -- Java version required by the Android branch. -- `repo`, `git-lfs`, `python3`, `bash`, and standard Android build tools. -- Repository checkout with `.worktree/android` or `OPENPHONE_ANDROID_DIR` - pointing at the synced Android tree. - -Register the runner with labels similar to: - -```bash -./config.sh \ - --url https://github.com// \ - --token \ - --labels self-hosted,openphone-build \ - --name openphone-build- -``` - -Install it as a service according to GitHub's self-hosted runner instructions -for the host OS. +Do not use a generic self-hosted runner label for official release builds. The +release workflow must show the GCP project, zone, VM name, source SHA, cache +mode, and artifact directory in the workflow summary. ## `openphone-emulator` diff --git a/.github/workflows/gcp-lab.yml b/.github/workflows/gcp-lab.yml index d47caeb..cb71856 100644 --- a/.github/workflows/gcp-lab.yml +++ b/.github/workflows/gcp-lab.yml @@ -94,8 +94,8 @@ concurrency: cancel-in-progress: false jobs: - gcp-lab-smoke: - name: GCP emulator lab smoke + gcp-emulator-lab: + name: GCP emulator lab if: >- ${{ github.event_name == 'workflow_dispatch' || @@ -212,7 +212,7 @@ jobs: echo "keep_vm=$keep_vm" } >> "$GITHUB_OUTPUT" - - name: Run GCP lab smoke + - name: Run GCP emulator lab shell: bash run: | set -euxo pipefail @@ -250,6 +250,30 @@ jobs: done ./scripts/lab/gcp/run-smoke.sh "${args[@]}" + - name: Summarize GCP emulator lab + if: always() + shell: bash + run: | + { + echo "## GCP emulator lab" + echo + echo "- Cloud provider: \`GCP\`" + echo "- Project: \`${OPENPHONE_GCP_PROJECT}\`" + echo "- Zone: \`${OPENPHONE_GCP_ZONE}\`" + echo "- VM: \`${{ steps.target.outputs.name }}\`" + echo "- Source ref: \`${{ steps.target.outputs.ref }}\`" + echo "- Source repo: \`${{ steps.target.outputs.repo_url }}\`" + echo "- Runtime intents: \`${{ steps.target.outputs.runtime }}\`" + echo "- Emulator arch: \`${{ steps.target.outputs.arch }}\`" + echo "- Machine type: \`${{ steps.target.outputs.machine_type }}\`" + echo "- Cache mode: \`${{ steps.target.outputs.cache_mode }}\`" + echo "- Cache disk: \`${{ steps.target.outputs.cache_disk || 'none' }}\`" + echo "- Cache snapshot: \`${{ steps.target.outputs.cache_source_snapshot || 'none' }}\`" + echo "- VM service account: \`disabled\`" + echo "- VM OAuth scopes: \`disabled\`" + echo "- Keep VM: \`${{ steps.target.outputs.keep_vm }}\`" + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload GCP lab artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03f507e..6b4e26b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,14 +43,6 @@ on: options: - "true" - "false" - run_emulator_smoke: - description: "Require GCP lab emulator smoke before publishing" - required: true - default: "true" - type: choice - options: - - "true" - - "false" keep_gcp_lab_vm: description: "Leave GCP lab VM running for debugging" required: true @@ -65,94 +57,9 @@ permissions: id-token: write jobs: - gcp-lab-smoke: - name: Release GCP lab smoke - if: ${{ inputs.run_emulator_smoke == 'true' }} - runs-on: ubuntu-latest - timeout-minutes: 480 - env: - OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }} - OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }} - OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }} - OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }} - OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }} - OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }} - OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT: ${{ vars.OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT || '' }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set up JDK - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "21" - - - name: Set up Android SDK - uses: android-actions/setup-android@v3 - with: - packages: "platforms;android-35" - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: "24" - - - name: Validate repository scaffold - run: | - set -euxo pipefail - ./scripts/check.sh - ./scripts/check-release-notes.sh "${{ inputs.version }}" "${{ inputs.release_notes }}" - git diff --check - test -f "${{ inputs.release_notes }}" - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ vars.OPENPHONE_GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.OPENPHONE_GCP_SERVICE_ACCOUNT }} - - - name: Set up gcloud - uses: google-github-actions/setup-gcloud@v2 - - - name: Run release GCP lab smoke - run: | - set -euxo pipefail - arch="${{ vars.OPENPHONE_EMULATOR_ARCH || 'x86_64' }}" - args=( - --name "openphone-lab-release-${{ inputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}" - --repo-url "https://github.com/secondly-com/OpenPhone.git" - --ref "${GITHUB_SHA}" - --project "${OPENPHONE_GCP_PROJECT}" - --zone "${OPENPHONE_GCP_ZONE}" - --machine-type "${OPENPHONE_GCP_MACHINE_TYPE}" - --boot-disk-size "${OPENPHONE_GCP_BOOT_DISK_SIZE}" - --boot-disk-type "${OPENPHONE_GCP_BOOT_DISK_TYPE}" - --arch "$arch" - --runtime local - ) - if [[ -n "${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}" ]]; then - args+=(--cache-mode snapshot --cache-source-snapshot "${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}") - else - args+=(--cache-mode scratch) - fi - if [[ "${{ inputs.keep_gcp_lab_vm }}" == "true" ]]; then - args+=(--keep-vm) - fi - ./scripts/lab/gcp/run-smoke.sh "${args[@]}" - - - name: Upload GCP lab artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: openphone-release-gcp-lab - path: .worktree/gcp-lab/ - if-no-files-found: warn - release-dry-run: name: Release dry run - needs: gcp-lab-smoke - if: ${{ always() && inputs.dry_run == 'true' && (inputs.run_emulator_smoke != 'true' || needs.gcp-lab-smoke.result == 'success') }} + if: ${{ inputs.dry_run == 'true' }} runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -191,24 +98,27 @@ jobs: echo "- Version: \`${{ inputs.version }}\`" echo "- Device: \`${{ inputs.device }}\`" echo "- Release notes: \`${{ inputs.release_notes }}\`" - echo "- GCP lab smoke required: \`${{ inputs.run_emulator_smoke }}\`" - echo "- OTA build skipped: \`true\`" + echo "- Cloud release build: \`skipped by dry_run\`" + echo "- Emulator gate skipped: \`true\`" echo "- GitHub Release publish skipped: \`true\`" } >> "$GITHUB_STEP_SUMMARY" - build-device-preview: - name: Build device preview - needs: gcp-lab-smoke - if: ${{ always() && inputs.dry_run != 'true' && (inputs.run_emulator_smoke != 'true' || needs.gcp-lab-smoke.result == 'success') }} - runs-on: [self-hosted, openphone-build] - timeout-minutes: 180 + gcp-release-build: + name: GCP release build, emulator gate, and publish + if: ${{ inputs.dry_run != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 720 env: - OPENPHONE_ANDROID_DIR: ${{ vars.OPENPHONE_BUILD_ANDROID_DIR || vars.OPENPHONE_ANDROID_DIR || '/opt/openphone-build/android' }} - OPENPHONE_BUILD_CACHE_DIR: ${{ vars.OPENPHONE_BUILD_CACHE_DIR || '/opt/openphone-build/cache' }} + OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }} + OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }} + OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }} + OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }} + OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }} + OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }} + OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT: ${{ vars.OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT || '' }} OPENPHONE_TEGU_VENDOR_ZIP_URL: ${{ vars.OPENPHONE_TEGU_VENDOR_ZIP_URL || '' }} OPENPHONE_TEGU_VENDOR_ZIP_SHA256: ${{ vars.OPENPHONE_TEGU_VENDOR_ZIP_SHA256 || '' }} - OPENPHONE_RELEASE: bp4a - OPENPHONE_BUILD_GOAL: bacon + OPENPHONE_RELEASE_BUILD_GOAL: ${{ vars.OPENPHONE_RELEASE_BUILD_GOAL || 'target-files-package ota_from_target_files' }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -224,6 +134,11 @@ jobs: with: packages: "platforms;android-35" + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + - name: Validate repository run: | set -euxo pipefail @@ -232,92 +147,88 @@ jobs: git diff --check test -f "${{ inputs.release_notes }}" - - name: Prepare Android tree - run: | - set -euxo pipefail - - candidates=() - add_candidate() { - local path="$1" - [[ -n "$path" ]] || return 0 - candidates+=("$path") - } - - add_candidate "$OPENPHONE_ANDROID_DIR" - add_candidate "${OPENPHONE_BUILD_ANDROID_DIR:-}" - add_candidate "${HOME}/openphone-android" - add_candidate "${RUNNER_WORKSPACE}/OpenPhoneAndroid/android" - - selected="" - for candidate in "${candidates[@]}"; do - if [[ -f "$candidate/build/envsetup.sh" ]]; then - selected="$candidate" - break - fi - done - selected="${selected:-$OPENPHONE_ANDROID_DIR}" - if ! mkdir -p "$selected"; then - selected="${HOME}/openphone-android" - mkdir -p "$selected" - fi - - echo "OPENPHONE_ANDROID_DIR=$selected" >> "$GITHUB_ENV" - export OPENPHONE_ANDROID_DIR="$selected" - - ./scripts/sync.sh -j"$(nproc)" --detach - if [[ -d "$OPENPHONE_ANDROID_DIR/.repo" ]]; then - OPENPHONE_RESET_PATCH_TARGETS=1 ./scripts/apply-patches.sh - else - ./scripts/apply-patches.sh - fi - if [[ "${{ inputs.device }}" == "tegu" ]]; then - ./scripts/prepare-tegu-device-repos.sh - fi + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.OPENPHONE_GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.OPENPHONE_GCP_SERVICE_ACCOUNT }} - - name: Build OTA - run: | - set -euxo pipefail - ./scripts/build.sh openphone_${{ inputs.device }} + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v2 - - name: Stage release artifacts + - name: Run GCP release build and stage artifacts id: stage run: | set -euxo pipefail - staging=".worktree/releases/${{ inputs.version }}" - product_dir="${OPENPHONE_ANDROID_DIR}/out/target/product/${{ inputs.device }}" - dst="${staging}/openphone_${{ inputs.device }}-${{ inputs.version }}-ota.zip" - src="" - - if [[ -f "${product_dir}/openphone_${{ inputs.device }}-ota.zip" ]]; then - src="${product_dir}/openphone_${{ inputs.device }}-ota.zip" + version="${{ inputs.version }}" + safe_version="$( + printf '%s' "$version" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//' + )" + lab_name="openphone-release-${safe_version}-${{ github.run_id }}-${{ github.run_attempt }}" + arch="${{ vars.OPENPHONE_EMULATOR_ARCH || 'x86_64' }}" + cache_mode="scratch" + cleanup_behavior="delete-vm" + args=( + --name "$lab_name" + --repo-url "https://github.com/secondly-com/OpenPhone.git" + --ref "${GITHUB_SHA}" + --version "$version" + --device "${{ inputs.device }}" + --release-notes "${{ inputs.release_notes }}" + --project "${OPENPHONE_GCP_PROJECT}" + --zone "${OPENPHONE_GCP_ZONE}" + --machine-type "${OPENPHONE_GCP_MACHINE_TYPE}" + --boot-disk-size "${OPENPHONE_GCP_BOOT_DISK_SIZE}" + --boot-disk-type "${OPENPHONE_GCP_BOOT_DISK_TYPE}" + --build-goal "${OPENPHONE_RELEASE_BUILD_GOAL}" + --emulator-arch "$arch" + ) + if [[ -n "${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}" ]]; then + cache_mode="snapshot" + args+=(--cache-mode snapshot --cache-source-snapshot "${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}") else - mapfile -t release_zips < <( - find "$product_dir" -maxdepth 1 -type f \ - -name "lineage-*-UNOFFICIAL-${{ inputs.device }}.zip" \ - -printf '%T@ %p\n' \ - | sort -nr \ - | awk '{sub(/^[^ ]+ /, ""); print}' - ) - if [[ ${#release_zips[@]} -gt 0 ]]; then - src="${release_zips[0]}" - fi + args+=(--cache-mode scratch) fi + if [[ "${{ inputs.keep_gcp_lab_vm }}" == "true" ]]; then + cleanup_behavior="keep-vm" + args+=(--keep-vm) + fi + ./scripts/lab/gcp/run-release.sh "${args[@]}" - [[ -n "$src" ]] || { - echo "No OTA zip found in ${product_dir}" >&2 - find "$product_dir" -maxdepth 1 -type f -name "*.zip" -print >&2 - exit 1 - } - - mkdir -p "$staging" - cp "$src" "$dst" - scripts/generate-release-manifest.sh "${{ inputs.version }}" "$staging" "$staging" - scripts/validate-release-artifacts.sh "$staging" + staging=".worktree/gcp-lab/${lab_name}/artifacts/release/${version}" + scripts/validate-release-artifacts.sh "$staging" "$staging" echo "staging=$staging" >> "$GITHUB_OUTPUT" - echo "ota_path=$dst" >> "$GITHUB_OUTPUT" + echo "ota_path=${staging}/openphone_${{ inputs.device }}-${{ inputs.version }}-ota.zip" >> "$GITHUB_OUTPUT" echo "sha_path=${staging}/SHA256SUMS" >> "$GITHUB_OUTPUT" echo "artifacts_path=${staging}/ARTIFACTS.md" >> "$GITHUB_OUTPUT" + { + echo "## GCP release build" + echo + echo "- Cloud provider: \`GCP\`" + echo "- Project: \`${OPENPHONE_GCP_PROJECT}\`" + echo "- Zone: \`${OPENPHONE_GCP_ZONE}\`" + echo "- VM: \`${lab_name}\`" + echo "- Source SHA: \`${GITHUB_SHA}\`" + echo "- Cache mode: \`${cache_mode}\`" + echo "- Cache snapshot: \`${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT:-none}\`" + echo "- Emulator gate: \`required\`" + echo "- Artifact validation: \`passed\`" + echo "- Artifact directory: \`${staging}\`" + echo "- Cleanup: \`${cleanup_behavior}\`" + echo "- VM service account: \`disabled\`" + echo "- VM OAuth scopes: \`disabled\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload GCP release lab artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: openphone-release-gcp-lab + path: .worktree/gcp-lab/ + if-no-files-found: warn - name: Publish GitHub Release uses: softprops/action-gh-release@v2 @@ -328,6 +239,4 @@ jobs: prerelease: ${{ inputs.prerelease == 'true' }} make_latest: ${{ inputs.make_latest }} files: | - ${{ steps.stage.outputs.ota_path }} - ${{ steps.stage.outputs.sha_path }} - ${{ steps.stage.outputs.artifacts_path }} + ${{ steps.stage.outputs.staging }}/* diff --git a/docs/AI_FIRST_ENGINEERING.md b/docs/AI_FIRST_ENGINEERING.md index 4be83cf..61b8fd4 100644 --- a/docs/AI_FIRST_ENGINEERING.md +++ b/docs/AI_FIRST_ENGINEERING.md @@ -118,12 +118,14 @@ Local-only agent notes belong in ignored `docs/local-temp/`, not in public docs. ### Current Baseline - `ci.yml` runs repository checks and whitespace checks on GitHub-hosted Linux. -- `emulator.yml` boots an OpenPhone SDK phone emulator on the - `openphone-emulator` self-hosted runner for PRs that touch runtime, - assistant, overlay, patch, integration, or emulator code. +- `gcp-lab.yml` runs trusted or maintainer-approved emulator lab checks on + disposable GCP VMs with warm Android cache state. +- `emulator.yml` remains available for a local self-hosted emulator runner, but + it is not the release build path. - `eval.yml` runs physical trajectory smokes on the `openphone-device` self-hosted runner. -- `release.yml` runs release work on the `openphone-build` self-hosted runner. +- `release.yml` runs release build, emulator gate, artifact validation, and + GitHub Release publishing through a GCP lab VM. ### Next CI Layers diff --git a/docs/BUILD.md b/docs/BUILD.md index feade70..077351f 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -134,10 +134,10 @@ openphone_sdk_phone_arm64-bp4a-eng ``` Choose the image architecture for the workstation that will run the emulator -UI: `arm64` for Apple Silicon and `x86_64` for Intel/x86_64 workstations. A -Linux Android build host, including EC2, can be used to produce the portable -image zip. The `eng` variant is the default because ADB is enabled without -first completing emulator-side developer settings. +UI: `arm64` for Apple Silicon and `x86_64` for Intel/x86_64 workstations. The +GCP lab workflow can produce portable image zips from warm Android cache state. +The `eng` variant is the default because ADB is enabled without first +completing emulator-side developer settings. Build the ARM64 emulator image: @@ -183,31 +183,30 @@ On LineageOS 21 and newer, `emu_img_zip` writes `sdk-repo-linux-system-images.zip` under `out/target/product//` for use with Android Studio/AVD system image installs. -### EC2 Build Host +### GCP Lab Build Host -For remote builds, launch an Ubuntu x86_64 EC2 host with at least 64 GB RAM and -roughly 700 GB of fast gp3 storage, then bootstrap Android build dependencies: +For remote builds, use the GCP lab scripts or workflow. They create a +disposable VM, attach or restore warm Android cache state, run the build, copy +artifacts back, and delete the VM unless debugging keeps it. ```bash -sudo ./scripts/bootstrap-android-build-host.sh +scripts/lab/gcp/run-smoke.sh \ + --cache-mode snapshot \ + --cache-source-snapshot "$OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT" \ + --arch x86_64 \ + --runtime local ``` -Copy the OpenPhone repo to the host and build the x86_64 emulator target: +To refresh the warm GCP cache intentionally: ```bash -OPENPHONE_SKIP_JAVA_CHECK=1 ./scripts/check.sh -./scripts/sync.sh -j16 -./scripts/apply-patches.sh -./scripts/build-emulator.sh --arch x86_64 +scripts/lab/gcp/prewarm-cache.sh \ + --cache-disk openphone-cache-x86-64-bp4a \ + --arch x86_64 ``` -The pre-sync scaffold check skips the standalone Java check because the full +The pre-sync scaffold check may skip the standalone Java check when the full Android build provides the authoritative compiler/toolchain validation. -The EC2 host is primarily a build host; it may not have the Android SDK -Emulator binary or `/dev/kvm`. Copy -`.worktree/android/out/target/product/emu64x/sdk-repo-linux-system-images.zip` -back to a workstation with Android Studio/SDK Emulator to create and boot the -AVD. Current generic-target status: @@ -229,7 +228,7 @@ Device-specific OpenPhone Pixel 9a build on Linux: ```bash OPENPHONE_ANDROID_DIR=/path/to/android/tree \ OPENPHONE_RELEASE=bp4a \ -OPENPHONE_BUILD_GOAL="droid target-files-package otapackage" \ +OPENPHONE_BUILD_GOAL="target-files-package ota_from_target_files" \ ./scripts/build.sh openphone_tegu ``` diff --git a/docs/EMULATOR.md b/docs/EMULATOR.md index 9469089..2ce4d07 100644 --- a/docs/EMULATOR.md +++ b/docs/EMULATOR.md @@ -26,9 +26,8 @@ Choose the target that matches the workstation that will run the UI: - Intel Mac or x86_64 Linux workstation: `--arch x86_64`, output directory `emu64x`. -The build can run on a Linux Android build host. A build-only EC2 host is -usually useful for producing the image zip, but it is not a good place to view -the UI unless it has a GUI stack and hardware acceleration. +The build can run in the GCP lab. The lab is the preferred remote path because +it uses the same warm Android cache and emulator smoke flow as trusted CI. ## Build The Image diff --git a/docs/RELEASE_PROCESS.md b/docs/RELEASE_PROCESS.md index 43cb94b..d8f3ef8 100644 --- a/docs/RELEASE_PROCESS.md +++ b/docs/RELEASE_PROCESS.md @@ -216,9 +216,10 @@ workflow requires: become the repository's Latest release. The workflow validates the repository, confirms the release-notes file exists, -builds the OTA on the `openphone-build` runner, stages the OTA, generates -`SHA256SUMS` and `ARTIFACTS.md`, validates the staged directory, and publishes -the GitHub Release with those assets. +authenticates to GCP with Workload Identity Federation, creates a disposable GCP +release lab VM, builds and stages the OTA there, runs the required emulator +gate, generates `SHA256SUMS` and `ARTIFACTS.md`, validates the staged directory, +and publishes the GitHub Release with those assets. Before dispatching a release: diff --git a/docs/devices/tegu.md b/docs/devices/tegu.md index 95b1cfd..f314d54 100644 --- a/docs/devices/tegu.md +++ b/docs/devices/tegu.md @@ -75,7 +75,7 @@ OPENPHONE_ANDROID_DIR="$PWD/.worktree/OpenPhoneAndroid/android" \ ./scripts/apply-patches.sh OPENPHONE_ANDROID_DIR="$PWD/.worktree/OpenPhoneAndroid/android" \ -OPENPHONE_BUILD_GOAL=bacon \ +OPENPHONE_BUILD_GOAL="target-files-package ota_from_target_files" \ ./scripts/build.sh openphone_tegu ``` @@ -102,11 +102,11 @@ Expected release artifacts after a successful device build: ```text vendor_boot.img -lineage-*-UNOFFICIAL-tegu.zip +obj/PACKAGING/target_files_intermediates/openphone_tegu-target_files.zip ``` -The zip name may still use Lineage's package naming until OpenPhone release -packaging is customized. +The GitHub release workflow turns the target-files package into an +`openphone_tegu--ota.zip` artifact with `scripts/stage-release-ota.sh`. ## Unlock and Flash @@ -312,7 +312,8 @@ rows out of `pending`. ## Known Issues - Play Integrity behavior is expected to change after bootloader unlock. -- OTA/release signing is not implemented yet. +- Production OTA/release signing is not implemented yet for public preview + artifacts. - Privileged assistant APK push is validated for assistant-only iteration, but framework, sepolicy, Settings/SystemUI, boot-chain, and first-install changes still require full target-files/OTA builds. diff --git a/docs/releases/README.md b/docs/releases/README.md index ff5e79f..60d4929 100644 --- a/docs/releases/README.md +++ b/docs/releases/README.md @@ -24,9 +24,9 @@ The public release pipeline is manual by design: 5. Release validation checks that the release notes file is `docs/releases/.md` and that `docs/releases/CHANGELOG.md` contains an exact heading for that version. -6. The self-hosted Android build runner builds the OTA, stages the artifact, - generates checksums/manifests, validates the release directory, and uploads - the OTA plus metadata to GitHub Releases. +6. The GCP release lab builds the OTA, runs the required emulator gate, stages + the artifact, generates checksums/manifests, validates the release + directory, and uploads the OTA plus metadata to GitHub Releases. 7. Release notes should link to device support, flashing notes, known issues, and validation evidence. diff --git a/scripts/check-assistant-java.sh b/scripts/check-assistant-java.sh index 08c8365..9f7c582 100755 --- a/scripts/check-assistant-java.sh +++ b/scripts/check-assistant-java.sh @@ -8,7 +8,7 @@ # This is a syntax/reference gate, not a real build: it catches unresolved # symbols (e.g. a helper method referenced but never implemented) that the # JSON/config checks in check.sh cannot see. The authoritative build is -# still the Android tree on the EC2 build host. +# still a full Android tree build. # # Set OPENPHONE_SKIP_JAVA_CHECK=1 to skip (e.g. environments without an # Android SDK). diff --git a/scripts/check.sh b/scripts/check.sh index 7bf5d13..a7fe2dd 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -68,6 +68,7 @@ required=( scripts/prepare-tegu-device-repos.sh scripts/prepare-tegu-dtb.sh scripts/generate-release-manifest.sh + scripts/stage-release-ota.sh scripts/check-release-notes.sh scripts/generate-ota-feed.sh scripts/prepare-github-release.sh @@ -159,6 +160,7 @@ required=( scripts/lab/gcp/delete-vm.sh scripts/lab/gcp/bootstrap-vm.sh scripts/lab/gcp/prewarm-cache.sh + scripts/lab/gcp/run-release.sh scripts/lab/gcp/run-smoke.sh scripts/lab/gcp/seed-cache-from-boot-disk.sh overlay/packages/apps/OpenPhoneAssistant/Android.bp diff --git a/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java b/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java index bb9c434..b16b498 100644 --- a/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java +++ b/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java @@ -1,7 +1,7 @@ // Compile-check stub. The real class is added to frameworks/base by the // OpenPhone patch stack and is only available in the full Android tree. // scripts/check-assistant-java.sh compiles assistant sources against this -// stub to catch syntax/reference breaks without an EC2 build. +// stub to catch syntax/reference breaks without a full Android build. package android.openphone; public final class OpenPhoneAgentManager { diff --git a/scripts/javacheck-stubs/android/openphone/OpenPhoneAssistantDataManager.java b/scripts/javacheck-stubs/android/openphone/OpenPhoneAssistantDataManager.java index 59f5f76..7cc580a 100644 --- a/scripts/javacheck-stubs/android/openphone/OpenPhoneAssistantDataManager.java +++ b/scripts/javacheck-stubs/android/openphone/OpenPhoneAssistantDataManager.java @@ -1,7 +1,7 @@ // Compile-check stub. The real class is added to frameworks/base by the // OpenPhone patch stack and is only available in the full Android tree. // scripts/check-assistant-java.sh compiles assistant sources against this -// stub to catch syntax/reference breaks without an EC2 build. +// stub to catch syntax/reference breaks without a full Android build. package android.openphone; public final class OpenPhoneAssistantDataManager { diff --git a/scripts/javacheck-stubs/android/openphone/OpenPhoneContextManager.java b/scripts/javacheck-stubs/android/openphone/OpenPhoneContextManager.java index 7537f34..69ba27a 100644 --- a/scripts/javacheck-stubs/android/openphone/OpenPhoneContextManager.java +++ b/scripts/javacheck-stubs/android/openphone/OpenPhoneContextManager.java @@ -1,7 +1,7 @@ // Compile-check stub. The real class is added to frameworks/base by the // OpenPhone patch stack and is only available in the full Android tree. // scripts/check-assistant-java.sh compiles assistant sources against this -// stub to catch syntax/reference breaks without an EC2 build. +// stub to catch syntax/reference breaks without a full Android build. package android.openphone; public final class OpenPhoneContextManager { diff --git a/scripts/lab/gcp/run-release.sh b/scripts/lab/gcp/run-release.sh new file mode 100755 index 0000000..8281e31 --- /dev/null +++ b/scripts/lab/gcp/run-release.sh @@ -0,0 +1,517 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/run-release.sh --version --device --release-notes [options] + +Creates a disposable GCP lab VM, updates a warm Android tree/cache, builds the +device release artifact, optionally boots the OpenPhone emulator smoke gate, and +copies release artifacts back to .worktree/gcp-lab//artifacts/release/. + +Options: + --name VM name. Default: generated from version/ref. + --repo-url Git repo URL. Default: current origin or GitHub. + --ref Git ref/SHA to build. Default: current HEAD. + --version Release version, for example v0.0.2. + --device Device codename. Default: tegu. + --release-notes Release notes file in the repo. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --machine-type Machine type. Default: c3-standard-22. + --boot-disk-size Boot disk size. Default: 1000GB. + --boot-disk-type Boot disk type. Default: pd-ssd. + --cache-mode scratch, attach-disk, or snapshot. Default: scratch. + --cache-disk Existing disk to attach, or per-run disk name for snapshot mode. + --cache-source-snapshot Snapshot to clone for snapshot mode. + --cache-mount Mount path for cache disk. Default: /mnt/openphone-cache. + --repo-sync-jobs repo sync jobs. Default: nproc on the VM. + --build-goal Android build goals. Default: target-files-package ota_from_target_files. + --emulator-arch arm64|x86_64 + Emulator arch for the release smoke. Default: x86_64. + --emulator-variant Emulator variant. Default: eng. + --emulator-timeout Emulator boot timeout. Default: 900. + --skip-emulator-smoke Build/stage release artifacts without emulator smoke. + --keep-vm Leave VM running for debug. + -h, --help Show this help. +EOF +} + +default_ref="$(git -C "$root" rev-parse HEAD 2>/dev/null || printf 'main')" +default_repo_url="$(git -C "$root" config --get remote.origin.url 2>/dev/null || printf 'https://github.com/secondly-com/OpenPhone.git')" +case "$default_repo_url" in + git@github.com:secondly-com/OpenPhone.git) + default_repo_url="https://github.com/secondly-com/OpenPhone.git" + ;; +esac + +name="" +repo_url="$default_repo_url" +ref="$default_ref" +version="" +device="tegu" +release_notes="" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" +machine_type="$OPENPHONE_GCP_MACHINE_TYPE" +boot_disk_size="$OPENPHONE_GCP_BOOT_DISK_SIZE" +boot_disk_type="$OPENPHONE_GCP_BOOT_DISK_TYPE" +cache_mode="$OPENPHONE_GCP_CACHE_MODE" +cache_disk="${OPENPHONE_GCP_CACHE_DISK:-}" +cache_source_snapshot="$OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT" +cache_mount="$OPENPHONE_GCP_CACHE_MOUNT" +repo_sync_jobs="" +build_goal="${OPENPHONE_RELEASE_BUILD_GOAL:-target-files-package ota_from_target_files}" +emulator_arch="x86_64" +emulator_variant="eng" +emulator_timeout="900" +skip_emulator_smoke=false +keep_vm=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) + [[ $# -ge 2 ]] || die "--name requires a value" + name="$2" + shift 2 + ;; + --repo-url) + [[ $# -ge 2 ]] || die "--repo-url requires a value" + repo_url="$2" + shift 2 + ;; + --ref) + [[ $# -ge 2 ]] || die "--ref requires a value" + ref="$2" + shift 2 + ;; + --version) + [[ $# -ge 2 ]] || die "--version requires a value" + version="$2" + shift 2 + ;; + --device) + [[ $# -ge 2 ]] || die "--device requires a value" + device="$2" + shift 2 + ;; + --release-notes) + [[ $# -ge 2 ]] || die "--release-notes requires a value" + release_notes="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + --machine-type) + [[ $# -ge 2 ]] || die "--machine-type requires a value" + machine_type="$2" + shift 2 + ;; + --boot-disk-size) + [[ $# -ge 2 ]] || die "--boot-disk-size requires a value" + boot_disk_size="$2" + shift 2 + ;; + --boot-disk-type) + [[ $# -ge 2 ]] || die "--boot-disk-type requires a value" + boot_disk_type="$2" + shift 2 + ;; + --cache-mode) + [[ $# -ge 2 ]] || die "--cache-mode requires a value" + cache_mode="$2" + shift 2 + ;; + --cache-disk) + [[ $# -ge 2 ]] || die "--cache-disk requires a value" + cache_disk="$2" + shift 2 + ;; + --cache-source-snapshot) + [[ $# -ge 2 ]] || die "--cache-source-snapshot requires a value" + cache_source_snapshot="$2" + shift 2 + ;; + --cache-mount) + [[ $# -ge 2 ]] || die "--cache-mount requires a value" + cache_mount="$2" + shift 2 + ;; + --repo-sync-jobs) + [[ $# -ge 2 ]] || die "--repo-sync-jobs requires a value" + repo_sync_jobs="$2" + shift 2 + ;; + --build-goal) + [[ $# -ge 2 ]] || die "--build-goal requires a value" + build_goal="$2" + shift 2 + ;; + --emulator-arch) + [[ $# -ge 2 ]] || die "--emulator-arch requires a value" + emulator_arch="$2" + shift 2 + ;; + --emulator-variant) + [[ $# -ge 2 ]] || die "--emulator-variant requires a value" + emulator_variant="$2" + shift 2 + ;; + --emulator-timeout) + [[ $# -ge 2 ]] || die "--emulator-timeout requires a value" + emulator_timeout="$2" + shift 2 + ;; + --skip-emulator-smoke) + skip_emulator_smoke=true + shift + ;; + --keep-vm) + keep_vm=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$version" ]] || die "--version is required" +[[ -n "$release_notes" ]] || die "--release-notes is required" + +case "$device" in + tegu) ;; + *) die "unsupported release device: $device" ;; +esac + +case "$cache_mode" in + scratch|attach-disk|snapshot) ;; + *) die "unsupported cache mode: $cache_mode" ;; +esac + +case "$emulator_arch" in + arm64|x86_64) ;; + *) die "unsupported emulator arch: $emulator_arch" ;; +esac + +case "$emulator_variant" in + eng|userdebug) ;; + *) die "unsupported emulator variant: $emulator_variant" ;; +esac + +if [[ "$cache_mode" == "snapshot" && -z "$cache_source_snapshot" ]]; then + die "--cache-source-snapshot is required when --cache-mode snapshot" +fi + +need_gcloud + +if [[ -z "$name" ]]; then + short_ref="$(printf '%s' "$ref" | cut -c1-12)" + name="openphone-release-${version}-${short_ref}-$(date -u +%H%M%S)" +fi +name="$(sanitize_gcp_name "$name")" +slot="$(printf '%s' "$name" | tr -c 'A-Za-z0-9_.-' '-')" + +artifact_root="$root/.worktree/gcp-lab/$name" +artifact_dir="$artifact_root/artifacts" +release_artifact_parent="$artifact_dir/release" +emulator_artifact_parent="$artifact_dir/emulator" +mkdir -p "$release_artifact_parent" "$emulator_artifact_parent" + +info "GCP release target: name=$name project=$project zone=$zone ref=$ref" +info "GCP release shape: machine=$machine_type disk=$boot_disk_size/$boot_disk_type cache_mode=$cache_mode" +if [[ -n "$cache_source_snapshot" ]]; then + info "GCP release cache snapshot: $cache_source_snapshot" +fi + +vm_created=false +release_copied=false +emulator_copied=false +remote_script="$(mktemp "${TMPDIR:-/tmp}/openphone-gcp-release.XXXXXX")" + +copy_release_artifacts() { + mkdir -p "$release_artifact_parent" + if gcloud compute scp --recurse \ + "$name:~/openphone-src/.worktree/releases/$version" \ + "$release_artifact_parent/" \ + --project "$project" \ + --zone "$zone" >/dev/null; then + release_copied=true + return 0 + fi + return 1 +} + +copy_emulator_artifacts() { + mkdir -p "$emulator_artifact_parent" + if gcloud compute scp --recurse \ + "$name:~/openphone-src/.worktree/lab/$slot/artifacts" \ + "$emulator_artifact_parent/" \ + --project "$project" \ + --zone "$zone" >/dev/null; then + emulator_copied=true + return 0 + fi + return 1 +} + +cleanup() { + local status=$? + set +e + rm -f "$remote_script" + if [[ "$vm_created" == true && "$release_copied" != true ]]; then + copy_release_artifacts >/dev/null 2>&1 || true + fi + if [[ "$vm_created" == true && "$emulator_copied" != true ]]; then + copy_emulator_artifacts >/dev/null 2>&1 || true + fi + if [[ "$vm_created" == true && "$keep_vm" != true ]]; then + "$script_dir/delete-vm.sh" --name "$name" --project "$project" --zone "$zone" || true + fi + exit "$status" +} +trap cleanup EXIT + +create_args=( + --name "$name" + --project "$project" + --zone "$zone" + --machine-type "$machine_type" + --boot-disk-size "$boot_disk_size" + --boot-disk-type "$boot_disk_type" + --cache-mode "$cache_mode" +) +if [[ -n "$cache_disk" ]]; then + create_args+=(--cache-disk "$cache_disk") +fi +if [[ -n "$cache_source_snapshot" ]]; then + create_args+=(--cache-source-snapshot "$cache_source_snapshot") +fi + +"$script_dir/create-vm.sh" "${create_args[@]}" +vm_created=true + +"$script_dir/bootstrap-vm.sh" --name "$name" --project "$project" --zone "$zone" + +cat > "$remote_script" <<'REMOTE' +#!/usr/bin/env bash +set -euo pipefail + +repo_url="${OPENPHONE_REPO_URL:?}" +ref="${OPENPHONE_REF:?}" +version="${OPENPHONE_RELEASE_VERSION:?}" +device="${OPENPHONE_RELEASE_DEVICE:?}" +release_notes="${OPENPHONE_RELEASE_NOTES:?}" +slot="${OPENPHONE_LAB_SLOT:?}" +build_goal="${OPENPHONE_RELEASE_BUILD_GOAL:?}" +repo_sync_jobs="${OPENPHONE_REPO_SYNC_JOBS:-}" +cache_mode="${OPENPHONE_GCP_CACHE_MODE:-scratch}" +cache_mount="${OPENPHONE_GCP_CACHE_MOUNT:-/mnt/openphone-cache}" +skip_emulator_smoke="${OPENPHONE_SKIP_EMULATOR_SMOKE:-0}" +emulator_arch="${OPENPHONE_EMULATOR_ARCH:-x86_64}" +emulator_variant="${OPENPHONE_EMULATOR_VARIANT:-eng}" +emulator_timeout="${OPENPHONE_EMULATOR_TIMEOUT:-900}" + +export OPENPHONE_RELEASE="${OPENPHONE_RELEASE:-bp4a}" + +prepare_android_workspace() { + if [[ "$cache_mode" == "scratch" ]]; then + export OPENPHONE_ANDROID_DIR="${OPENPHONE_ANDROID_DIR:-$HOME/openphone-android}" + mkdir -p "$OPENPHONE_ANDROID_DIR" + export OPENPHONE_BUILD_CACHE_DIR="${OPENPHONE_BUILD_CACHE_DIR:-$HOME/openphone-build-cache}" + mkdir -p "$OPENPHONE_BUILD_CACHE_DIR" + return 0 + fi + + export OPENPHONE_ANDROID_DIR="${OPENPHONE_ANDROID_DIR:-${OPENPHONE_GCP_CACHE_ANDROID_DIR:-$HOME/openphone-android}}" + + local device_path="/dev/disk/by-id/google-openphone-cache" + local deadline=$((SECONDS + 300)) + while [[ ! -e "$device_path" ]]; do + if [[ "$SECONDS" -ge "$deadline" ]]; then + printf 'error: cache disk device did not appear: %s\n' "$device_path" >&2 + exit 1 + fi + sleep 2 + done + + if ! sudo blkid "$device_path" >/dev/null 2>&1; then + sudo mkfs.ext4 -F -L openphone-cache "$device_path" + fi + + sudo mkdir -p "$cache_mount" + if ! findmnt --mountpoint "$cache_mount" >/dev/null 2>&1; then + sudo mount -o defaults,discard "$device_path" "$cache_mount" + fi + sudo chown "$USER:$USER" "$cache_mount" + + local cache_android_dir="$cache_mount/android" + local android_parent + android_parent="$(dirname "$OPENPHONE_ANDROID_DIR")" + sudo mkdir -p "$android_parent" + sudo chown "$USER:$USER" "$android_parent" + mkdir -p "$cache_android_dir" "$OPENPHONE_ANDROID_DIR" + if ! findmnt --mountpoint "$OPENPHONE_ANDROID_DIR" >/dev/null 2>&1; then + sudo mount --bind "$cache_android_dir" "$OPENPHONE_ANDROID_DIR" + fi + sudo chown "$USER:$USER" "$cache_mount" "$cache_android_dir" "$OPENPHONE_ANDROID_DIR" + + export OPENPHONE_BUILD_CACHE_DIR="${OPENPHONE_BUILD_CACHE_DIR:-$cache_mount/cache}" + mkdir -p "$OPENPHONE_BUILD_CACHE_DIR" +} + +prepare_android_workspace + +printf '==> Cloud provider: GCP\n' +printf '==> Android workspace path: %s\n' "$OPENPHONE_ANDROID_DIR" +printf '==> Build cache path: %s\n' "$OPENPHONE_BUILD_CACHE_DIR" + +ensure_android_workspace_writable() { + local probe_dir="$OPENPHONE_ANDROID_DIR" + if [[ -d "$OPENPHONE_ANDROID_DIR/.repo" ]]; then + probe_dir="$OPENPHONE_ANDROID_DIR/.repo" + fi + + if touch "$probe_dir/.openphone-write-probe" >/dev/null 2>&1; then + rm -f "$probe_dir/.openphone-write-probe" + return 0 + fi + + printf '==> Android tree is not writable by %s; normalizing cache ownership under %s\n' "$USER" "$OPENPHONE_ANDROID_DIR" + sudo chown -R "$USER:$USER" "$OPENPHONE_ANDROID_DIR" + touch "$probe_dir/.openphone-write-probe" + rm -f "$probe_dir/.openphone-write-probe" +} + +ensure_android_workspace_writable + +if [[ ! -d "$HOME/openphone-src/.git" ]]; then + rm -rf "$HOME/openphone-src" + git clone "$repo_url" "$HOME/openphone-src" +fi + +cd "$HOME/openphone-src" +git remote set-url origin "$repo_url" +git fetch --tags --prune origin +git fetch origin "$ref" || true +git checkout --force "$ref" || git checkout --force FETCH_HEAD + +./scripts/bootstrap-android-build-host.sh +./scripts/lab/install-android-sdk-tools.sh +export ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:-/opt/android-sdk}" +export ANDROID_HOME="${ANDROID_HOME:-$ANDROID_SDK_ROOT}" +export PATH="$ANDROID_SDK_ROOT/platform-tools:$ANDROID_SDK_ROOT/emulator:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:$PATH" + +OPENPHONE_SKIP_JAVA_CHECK=1 ./scripts/check.sh +./scripts/check-release-notes.sh "$version" "$release_notes" +test -f "$release_notes" + +sync_args=() +if [[ -n "$repo_sync_jobs" ]]; then + sync_args+=(-j"$repo_sync_jobs") +else + sync_args+=(-j"$(nproc)") +fi +sync_args+=(--detach) +if [[ "$cache_mode" == "scratch" ]]; then + sync_args+=(--force-sync --force-checkout) +fi +./scripts/sync.sh "${sync_args[@]}" + +if [[ "$cache_mode" == "scratch" ]]; then + ./scripts/apply-patches.sh +else + OPENPHONE_RESET_PATCH_TARGETS=1 ./scripts/apply-patches.sh +fi +./scripts/check.sh + +if [[ "$device" == "tegu" ]]; then + ./scripts/prepare-tegu-device-repos.sh +fi + +export OPENPHONE_BUILD_GOAL="$build_goal" +./scripts/build.sh "openphone_${device}" + +release_dir="$HOME/openphone-src/.worktree/releases/$version" +rm -rf "$release_dir" +mkdir -p "$release_dir" +./scripts/stage-release-ota.sh \ + --android-dir "$OPENPHONE_ANDROID_DIR" \ + --device "$device" \ + --version "$version" \ + --output-dir "$release_dir" +./scripts/generate-release-manifest.sh "$version" "$release_dir" "$release_dir" +./scripts/validate-release-artifacts.sh "$release_dir" + +if [[ "$skip_emulator_smoke" != "1" ]]; then + ./scripts/lab/smoke.sh \ + --slot "$slot" \ + --arch "$emulator_arch" \ + --variant "$emulator_variant" \ + --timeout "$emulator_timeout" \ + --runtime local +fi +REMOTE + +info "Copying remote release script to $name" +gcloud compute scp "$remote_script" "$name:/tmp/openphone-gcp-run-release.sh" \ + --project "$project" \ + --zone "$zone" >/dev/null + +skip_emulator_value=0 +if [[ "$skip_emulator_smoke" == true ]]; then + skip_emulator_value=1 +fi + +remote_command="OPENPHONE_REPO_URL=$(shell_quote "$repo_url")" +remote_command+=" OPENPHONE_REF=$(shell_quote "$ref")" +remote_command+=" OPENPHONE_RELEASE_VERSION=$(shell_quote "$version")" +remote_command+=" OPENPHONE_RELEASE_DEVICE=$(shell_quote "$device")" +remote_command+=" OPENPHONE_RELEASE_NOTES=$(shell_quote "$release_notes")" +remote_command+=" OPENPHONE_LAB_SLOT=$(shell_quote "$slot")" +remote_command+=" OPENPHONE_RELEASE_BUILD_GOAL=$(shell_quote "$build_goal")" +remote_command+=" OPENPHONE_REPO_SYNC_JOBS=$(shell_quote "$repo_sync_jobs")" +remote_command+=" OPENPHONE_GCP_CACHE_MODE=$(shell_quote "$cache_mode")" +remote_command+=" OPENPHONE_GCP_CACHE_MOUNT=$(shell_quote "$cache_mount")" +remote_command+=" OPENPHONE_SKIP_EMULATOR_SMOKE=$(shell_quote "$skip_emulator_value")" +remote_command+=" OPENPHONE_EMULATOR_ARCH=$(shell_quote "$emulator_arch")" +remote_command+=" OPENPHONE_EMULATOR_VARIANT=$(shell_quote "$emulator_variant")" +remote_command+=" OPENPHONE_EMULATOR_TIMEOUT=$(shell_quote "$emulator_timeout")" +remote_command+=" OPENPHONE_TEGU_VENDOR_ZIP_URL=$(shell_quote "${OPENPHONE_TEGU_VENDOR_ZIP_URL:-}")" +remote_command+=" OPENPHONE_TEGU_VENDOR_ZIP_SHA256=$(shell_quote "${OPENPHONE_TEGU_VENDOR_ZIP_SHA256:-}")" +remote_command+=" OPENPHONE_BUILD_CACHE_DIR=$(shell_quote "${OPENPHONE_BUILD_CACHE_DIR:-}")" +remote_command+=" bash /tmp/openphone-gcp-run-release.sh" + +info "Running GCP release build on $name" +gcloud compute ssh "$name" \ + --project "$project" \ + --zone "$zone" \ + --command "$remote_command" \ + | tee "$artifact_root/gcp-run-release.log" + +copy_release_artifacts +if [[ "$skip_emulator_smoke" != true ]]; then + copy_emulator_artifacts || true +fi + +staging="$release_artifact_parent/$version" +[[ -f "$staging/SHA256SUMS" ]] || die "release artifacts were not copied back: $staging" +info "GCP release artifacts copied to $staging" +printf '%s\n' "$staging" diff --git a/scripts/lab/gcp/run-smoke.sh b/scripts/lab/gcp/run-smoke.sh index fe6a544..bfccb6a 100755 --- a/scripts/lab/gcp/run-smoke.sh +++ b/scripts/lab/gcp/run-smoke.sh @@ -497,7 +497,7 @@ remote_command+=" OPENPHONE_GCP_CACHE_MODE=$(shell_quote "$cache_mode")" remote_command+=" OPENPHONE_GCP_CACHE_MOUNT=$(shell_quote "$cache_mount")" remote_command+=" bash /tmp/openphone-gcp-run-smoke.sh" -info "Running GCP lab smoke on $name" +info "Running GCP emulator lab on $name" gcloud compute ssh "$name" \ --project "$project" \ --zone "$zone" \ @@ -505,4 +505,4 @@ gcloud compute ssh "$name" \ | tee "$artifact_root/gcp-run-smoke.log" copy_remote_artifacts -info "GCP lab smoke passed; artifacts copied to $artifact_dir" +info "GCP emulator lab passed; artifacts copied to $artifact_dir" diff --git a/scripts/stage-release-ota.sh b/scripts/stage-release-ota.sh new file mode 100755 index 0000000..e0e08cb --- /dev/null +++ b/scripts/stage-release-ota.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/common.sh +source "$root/scripts/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/stage-release-ota.sh --android-dir --device --version --output-dir [options] + +Stages a release OTA from an Android target-files package. This avoids relying +on Lineage's bacon wrapper as the release artifact contract. + +Options: + --android-dir Android source/output tree. + --device Device codename, for example tegu. + --product Android product. Default: openphone_. + --version Release version, for example v0.0.2. + --target-files Explicit target-files zip. Default: auto-detect. + --output-dir Staging directory for release artifacts. + --package-key OTA signing key pair. Default: Android testkey. + -h, --help Show this help. +EOF +} + +android_dir="" +device="" +product="" +version="" +target_files="" +output_dir="" +package_key="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --android-dir) + [[ $# -ge 2 ]] || die "--android-dir requires a value" + android_dir="$2" + shift 2 + ;; + --device) + [[ $# -ge 2 ]] || die "--device requires a value" + device="$2" + shift 2 + ;; + --product) + [[ $# -ge 2 ]] || die "--product requires a value" + product="$2" + shift 2 + ;; + --version) + [[ $# -ge 2 ]] || die "--version requires a value" + version="$2" + shift 2 + ;; + --target-files) + [[ $# -ge 2 ]] || die "--target-files requires a value" + target_files="$2" + shift 2 + ;; + --output-dir) + [[ $# -ge 2 ]] || die "--output-dir requires a value" + output_dir="$2" + shift 2 + ;; + --package-key) + [[ $# -ge 2 ]] || die "--package-key requires a value" + package_key="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$android_dir" ]] || die "--android-dir is required" +[[ -n "$device" ]] || die "--device is required" +[[ -n "$version" ]] || die "--version is required" +[[ -n "$output_dir" ]] || die "--output-dir is required" + +android_dir="$(cd "$android_dir" && pwd)" +[[ -f "$android_dir/build/envsetup.sh" ]] || die "missing Android envsetup: $android_dir/build/envsetup.sh" + +product="${product:-openphone_${device}}" +product_dir="$android_dir/out/target/product/$device" +[[ -d "$product_dir" ]] || die "missing product output directory: $product_dir" + +if [[ -z "$target_files" ]]; then + expected="$product_dir/obj/PACKAGING/target_files_intermediates/${product}-target_files.zip" + if [[ -f "$expected" ]]; then + target_files="$expected" + else + mapfile -t candidates < <( + find "$product_dir/obj/PACKAGING/target_files_intermediates" \ + -maxdepth 2 \ + -type f \ + -name '*target_files*.zip' \ + -printf '%T@ %p\n' 2>/dev/null \ + | sort -nr \ + | awk '{sub(/^[^ ]+ /, ""); print}' + ) + if [[ ${#candidates[@]} -gt 0 ]]; then + target_files="${candidates[0]}" + fi + fi +fi + +[[ -f "$target_files" ]] || { + printf 'error: target-files zip not found for %s under %s\n' "$product" "$product_dir" >&2 + find "$product_dir/obj/PACKAGING" -maxdepth 3 -type f -name '*.zip' -print >&2 2>/dev/null || true + exit 1 +} + +ota_tool="$android_dir/out/host/linux-x86/bin/ota_from_target_files" +if [[ -x "$ota_tool" ]]; then + ota_cmd=("$ota_tool") +elif [[ -x "$android_dir/build/make/tools/releasetools/ota_from_target_files" ]]; then + ota_cmd=("$android_dir/build/make/tools/releasetools/ota_from_target_files") +elif [[ -f "$android_dir/build/make/tools/releasetools/ota_from_target_files.py" ]]; then + ota_cmd=(python3 "$android_dir/build/make/tools/releasetools/ota_from_target_files.py") +else + die "missing ota_from_target_files; build the ota_from_target_files host tool first" +fi + +package_key="${package_key:-$android_dir/build/make/target/product/security/testkey}" +[[ -f "$package_key.x509.pem" || -f "$package_key.pk8" || -f "$package_key" ]] || { + die "missing OTA package key pair prefix: $package_key" +} + +mkdir -p "$output_dir" +output_dir="$(cd "$output_dir" && pwd)" +ota_name="openphone_${device}-${version}-ota.zip" +ota_path="$output_dir/$ota_name" +metadata_path="$output_dir/openphone_${device}-${version}-release-metadata.txt" + +info "Target files: $target_files" +info "OTA output: $ota_path" + +PATH="$android_dir/out/host/linux-x86/bin:$android_dir/out/soong/host/linux-x86/bin:$PATH" \ + "${ota_cmd[@]}" \ + -k "$package_key" \ + "$target_files" \ + "$ota_path" + +unzip -tq "$ota_path" >/dev/null +sha256sum "$ota_path" > "$ota_path.sha256" + +{ + printf 'version=%s\n' "$version" + printf 'device=%s\n' "$device" + printf 'product=%s\n' "$product" + printf 'android_dir=%s\n' "$android_dir" + printf 'target_files=%s\n' "$target_files" + printf 'ota=%s\n' "$ota_path" + printf 'ota_sha256=%s\n' "$(file_sha256 "$ota_path")" + printf 'generated_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +} > "$metadata_path" + +printf 'Staged OTA: %s\n' "$ota_path" +printf 'Staged OTA SHA-256: %s\n' "$ota_path.sha256" +printf 'Release metadata: %s\n' "$metadata_path"