diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a0848b6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.patch whitespace=-blank-at-eol diff --git a/.github/RUNNERS.md b/.github/RUNNERS.md index 2da2ae4..fc0d6df 100644 --- a/.github/RUNNERS.md +++ b/.github/RUNNERS.md @@ -13,6 +13,7 @@ specific operator notes outside the public repository. | Label | Used by | Purpose | | --- | --- | --- | | `openphone-build` | `release.yml` | Builds Pixel/device artifacts on a large Linux Android build host. | +| `openphone-emulator` | `emulator.yml` | Builds or reuses an OpenPhone SDK phone image, boots it in the Android emulator, and runs fast runtime smoke checks. | | `openphone-device` | `eval.yml` | Runs evals against an authorized Android device connected over USB. | ## `openphone-build` @@ -40,6 +41,39 @@ Register the runner with labels similar to: Install it as a service according to GitHub's self-hosted runner instructions for the host OS. +## `openphone-emulator` + +The emulator runner needs: + +- Linux x86_64 or macOS host with hardware acceleration suitable for Android + Emulator. +- Full Android build dependencies for the selected LineageOS branch. +- Android SDK Platform Tools and Emulator on `PATH`. +- `adb`, `emulator`, `node`, `python3`, `bash`, `git-lfs`, and `repo`. +- Repository checkout with `.worktree/android` or `OPENPHONE_ANDROID_DIR` + pointing at the synced Android tree. +- Enough disk for the Android tree, incremental build outputs, emulator system + image, and wiped userdata. + +Prefer setting `OPENPHONE_ANDROID_DIR` to a persistent path outside the Actions +checkout. The workflow disables checkout cleaning to avoid deleting +`.worktree/android`, but keeping the Android tree outside the repository +workspace is easier to reason about on long-lived runners. + +Register the runner with labels similar to: + +```bash +./config.sh \ + --url https://github.com// \ + --token \ + --labels self-hosted,openphone-emulator \ + --name openphone-emulator- +``` + +The workflow runs `scripts/run-emulator-smoke.sh`. On pull requests, it only +runs for same-repository branches because self-hosted runners must not execute +untrusted fork code with local Android build state. + ## `openphone-device` The device runner needs: @@ -65,6 +99,8 @@ Register the runner with labels similar to: - Android source checkouts and build output are too large for standard GitHub-hosted runner disk limits. +- Headless emulator boot with OpenPhone system images needs a prepared Android + tree, SDK Emulator, acceleration, and enough local disk. - Physical evals require a real phone connected over USB. The normal CI workflow in `ci.yml` stays on `ubuntu-latest` and only runs diff --git a/.github/workflows/emulator.yml b/.github/workflows/emulator.yml new file mode 100644 index 0000000..f60ed28 --- /dev/null +++ b/.github/workflows/emulator.yml @@ -0,0 +1,97 @@ +name: Emulator + +on: + push: + branches: + - main + paths: + - ".github/workflows/emulator.yml" + - "integrations/adb/**" + - "integrations/cli/**" + - "integrations/mcp-server/**" + - "overlay/**" + - "patches/**" + - "runtime/**" + - "scripts/build.sh" + - "scripts/build-emulator.sh" + - "scripts/lab/**" + - "scripts/run-emulator*.sh" + - "scripts/run-assistant-task.sh" + workflow_dispatch: + inputs: + arch: + description: "Emulator architecture" + required: true + default: "x86_64" + type: choice + options: + - x86_64 + - arm64 + skip_build: + description: "Reuse the runner's existing emulator image" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + assistant_smoke: + description: "Run local assistant task smoke" + required: true + default: "true" + type: choice + options: + - "true" + - "false" + +permissions: + contents: read + +concurrency: + group: emulator-${{ github.ref }} + cancel-in-progress: true + +jobs: + emulator-smoke: + name: Boot emulator and run OpenPhone smoke + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: [self-hosted, openphone-emulator] + timeout-minutes: 180 + env: + OPENPHONE_ANDROID_DIR: ${{ vars.OPENPHONE_ANDROID_DIR || '.worktree/android' }} + OPENPHONE_RELEASE: bp4a + OPENPHONE_EMULATOR_ASSISTANT_SMOKE: ${{ inputs.assistant_smoke == 'false' && '0' || '1' }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + clean: false + + - name: Validate repository scaffold + run: | + set -euxo pipefail + ./scripts/check.sh + git diff --check + + - name: Run emulator smoke + env: + OPENPHONE_EMULATOR_BUILD: ${{ inputs.skip_build == 'true' && '0' || '1' }} + run: | + set -euxo pipefail + arch="${{ inputs.arch || vars.OPENPHONE_EMULATOR_ARCH || 'x86_64' }}" + slot="ci-${{ github.run_id }}-${{ github.run_attempt }}" + ./scripts/lab/allocate-slot.sh --slot "$slot" + # shellcheck disable=SC1090 + source ".worktree/lab/$slot/env" + ./scripts/lab/smoke.sh \ + --slot "$OPENPHONE_LAB_SLOT" \ + --arch "$arch" \ + --ci + + - name: Upload emulator smoke artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: openphone-emulator-smoke + path: .worktree/lab/*/artifacts/emulator-smoke/ + if-no-files-found: warn diff --git a/.github/workflows/gcp-lab.yml b/.github/workflows/gcp-lab.yml new file mode 100644 index 0000000..674792e --- /dev/null +++ b/.github/workflows/gcp-lab.yml @@ -0,0 +1,236 @@ +name: GCP Lab + +on: + workflow_dispatch: + inputs: + ref: + description: "Git ref/SHA to test. Defaults to this workflow SHA." + required: false + default: "" + repo_url: + description: "Git repo URL to clone on the lab VM" + required: true + default: "https://github.com/secondly-com/OpenPhone.git" + pr_number: + description: "Optional PR number for labels/naming" + required: false + default: "" + runtime: + description: "Comma-separated runtime intents" + required: true + default: "local" + arch: + description: "Emulator architecture" + required: true + default: "x86_64" + type: choice + options: + - x86_64 + - arm64 + machine_type: + description: "GCP machine type" + required: true + default: "c3-standard-22" + boot_disk_size: + description: "GCP boot disk size" + required: true + default: "1000GB" + boot_disk_type: + description: "GCP boot disk type" + required: true + default: "pd-ssd" + cache_mode: + description: "Android tree/cache strategy" + required: true + default: "scratch" + type: choice + options: + - scratch + - attach-disk + - snapshot + cache_disk: + description: "Existing cache disk name for attach-disk, or per-run disk name for snapshot" + required: false + default: "" + cache_source_snapshot: + description: "Warm cache snapshot name when cache_mode=snapshot" + required: false + default: "" + keep_vm: + description: "Leave the lab VM running for debugging" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + pull_request_review: + types: + - submitted + +permissions: + contents: read + id-token: write + pull-requests: read + +concurrency: + group: gcp-lab-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} + cancel-in-progress: false + +jobs: + gcp-lab-smoke: + name: GCP emulator lab smoke + if: >- + ${{ + github.event_name == 'workflow_dispatch' || + ( + github.event.review.state == 'approved' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.review.author_association) + ) + }} + runs-on: ubuntu-latest + timeout-minutes: 480 + env: + OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }} + OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }} + OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }} + OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }} + OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }} + OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }} + OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT: ${{ vars.OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT || '' }} + steps: + - name: Check out trusted workflow repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request_review' && github.event.pull_request.base.sha || github.sha }} + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Set up Android SDK + uses: android-actions/setup-android@v3 + with: + packages: "platforms;android-35" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Validate repository scaffold + run: | + set -euxo pipefail + ./scripts/check.sh + git diff --check + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.OPENPHONE_GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.OPENPHONE_GCP_SERVICE_ACCOUNT }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Select target + id: target + shell: bash + run: | + set -euo pipefail + if [[ "${{ github.event_name }}" == "pull_request_review" ]]; then + pr_number="${{ github.event.pull_request.number }}" + ref="${{ github.event.pull_request.head.sha }}" + repo_url="${{ github.event.pull_request.head.repo.clone_url }}" + name="openphone-lab-pr-${pr_number}-${{ github.run_id }}-${{ github.run_attempt }}" + runtime="local" + arch="${{ vars.OPENPHONE_EMULATOR_ARCH || 'x86_64' }}" + machine_type="${OPENPHONE_GCP_MACHINE_TYPE}" + boot_disk_size="${OPENPHONE_GCP_BOOT_DISK_SIZE}" + boot_disk_type="${OPENPHONE_GCP_BOOT_DISK_TYPE}" + cache_source_snapshot="${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}" + if [[ -n "$cache_source_snapshot" ]]; then + cache_mode="snapshot" + else + cache_mode="scratch" + fi + cache_disk="" + keep_vm="false" + else + pr_number="${{ inputs.pr_number }}" + ref="${{ inputs.ref }}" + if [[ -z "$ref" ]]; then + ref="${GITHUB_SHA}" + fi + repo_url="${{ inputs.repo_url }}" + if [[ -z "$repo_url" ]]; then + repo_url="https://github.com/secondly-com/OpenPhone.git" + fi + name="openphone-lab-manual-${pr_number:-${{ github.run_id }}}-${{ github.run_attempt }}" + runtime="${{ inputs.runtime }}" + arch="${{ inputs.arch }}" + machine_type="${{ inputs.machine_type }}" + boot_disk_size="${{ inputs.boot_disk_size }}" + boot_disk_type="${{ inputs.boot_disk_type }}" + cache_mode="${{ inputs.cache_mode }}" + cache_disk="${{ inputs.cache_disk }}" + cache_source_snapshot="${{ inputs.cache_source_snapshot }}" + keep_vm="${{ inputs.keep_vm }}" + fi + + { + echo "name=$name" + echo "ref=$ref" + echo "repo_url=$repo_url" + echo "runtime=$runtime" + echo "arch=$arch" + echo "machine_type=$machine_type" + echo "boot_disk_size=$boot_disk_size" + echo "boot_disk_type=$boot_disk_type" + echo "cache_mode=$cache_mode" + echo "cache_disk=$cache_disk" + echo "cache_source_snapshot=$cache_source_snapshot" + echo "keep_vm=$keep_vm" + } >> "$GITHUB_OUTPUT" + + - name: Run GCP lab smoke + shell: bash + run: | + set -euxo pipefail + args=( + --name "${{ steps.target.outputs.name }}" + --repo-url "${{ steps.target.outputs.repo_url }}" + --ref "${{ steps.target.outputs.ref }}" + --project "${OPENPHONE_GCP_PROJECT}" + --zone "${OPENPHONE_GCP_ZONE}" + --machine-type "${{ steps.target.outputs.machine_type }}" + --boot-disk-size "${{ steps.target.outputs.boot_disk_size }}" + --boot-disk-type "${{ steps.target.outputs.boot_disk_type }}" + --cache-mode "${{ steps.target.outputs.cache_mode }}" + --arch "${{ steps.target.outputs.arch }}" + ) + if [[ -n "${{ steps.target.outputs.cache_disk }}" ]]; then + args+=(--cache-disk "${{ steps.target.outputs.cache_disk }}") + fi + if [[ -n "${{ steps.target.outputs.cache_source_snapshot }}" ]]; then + args+=(--cache-source-snapshot "${{ steps.target.outputs.cache_source_snapshot }}") + fi + if [[ "${{ steps.target.outputs.keep_vm }}" == "true" ]]; then + args+=(--keep-vm) + fi + IFS=',' read -r -a runtimes <<< "${{ steps.target.outputs.runtime }}" + for runtime in "${runtimes[@]}"; do + [[ -n "$runtime" ]] || continue + args+=(--runtime "$runtime") + done + ./scripts/lab/gcp/run-smoke.sh "${args[@]}" + + - name: Upload GCP lab artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: openphone-gcp-lab + path: .worktree/gcp-lab/ + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 02dbb2b..6835381 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,13 +35,115 @@ on: - "false" - "true" - "legacy" + run_emulator_smoke: + description: "Require GCP lab emulator smoke before publishing" + required: true + default: "true" + type: choice + options: + - "true" + - "false" + keep_gcp_lab_vm: + description: "Leave GCP lab VM running for debugging" + required: true + default: "false" + type: choice + options: + - "false" + - "true" permissions: contents: write + id-token: write jobs: + gcp-lab-smoke: + name: Release GCP lab smoke + if: ${{ inputs.run_emulator_smoke == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 480 + env: + OPENPHONE_GCP_PROJECT: ${{ vars.OPENPHONE_GCP_PROJECT || 'openphone-lab' }} + OPENPHONE_GCP_REGION: ${{ vars.OPENPHONE_GCP_REGION || 'us-central1' }} + OPENPHONE_GCP_ZONE: ${{ vars.OPENPHONE_GCP_ZONE || 'us-central1-a' }} + OPENPHONE_GCP_MACHINE_TYPE: ${{ vars.OPENPHONE_GCP_MACHINE_TYPE || 'c3-standard-22' }} + OPENPHONE_GCP_BOOT_DISK_SIZE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_SIZE || '1000GB' }} + OPENPHONE_GCP_BOOT_DISK_TYPE: ${{ vars.OPENPHONE_GCP_BOOT_DISK_TYPE || 'pd-ssd' }} + OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT: ${{ vars.OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT || '' }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Set up Android SDK + uses: android-actions/setup-android@v3 + with: + packages: "platforms;android-35" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Validate repository scaffold + run: | + set -euxo pipefail + ./scripts/check.sh + git diff --check + test -f "${{ inputs.release_notes }}" + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.OPENPHONE_GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.OPENPHONE_GCP_SERVICE_ACCOUNT }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Run release GCP lab smoke + run: | + set -euxo pipefail + arch="${{ vars.OPENPHONE_EMULATOR_ARCH || 'x86_64' }}" + args=( + --name "openphone-lab-release-${{ inputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}" + --repo-url "https://github.com/secondly-com/OpenPhone.git" + --ref "${GITHUB_SHA}" + --project "${OPENPHONE_GCP_PROJECT}" + --zone "${OPENPHONE_GCP_ZONE}" + --machine-type "${OPENPHONE_GCP_MACHINE_TYPE}" + --boot-disk-size "${OPENPHONE_GCP_BOOT_DISK_SIZE}" + --boot-disk-type "${OPENPHONE_GCP_BOOT_DISK_TYPE}" + --arch "$arch" + --runtime local + ) + if [[ -n "${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}" ]]; then + args+=(--cache-mode snapshot --cache-source-snapshot "${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT}") + else + args+=(--cache-mode scratch) + fi + if [[ "${{ inputs.keep_gcp_lab_vm }}" == "true" ]]; then + args+=(--keep-vm) + fi + ./scripts/lab/gcp/run-smoke.sh "${args[@]}" + + - name: Upload GCP lab artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: openphone-release-gcp-lab + path: .worktree/gcp-lab/ + if-no-files-found: warn + build-device-preview: name: Build device preview + needs: gcp-lab-smoke + if: ${{ always() && (inputs.run_emulator_smoke != 'true' || needs.gcp-lab-smoke.result == 'success') }} runs-on: [self-hosted, openphone-build] timeout-minutes: 180 env: diff --git a/docs/AI_FIRST_ENGINEERING.md b/docs/AI_FIRST_ENGINEERING.md index 63064ab..4522a39 100644 --- a/docs/AI_FIRST_ENGINEERING.md +++ b/docs/AI_FIRST_ENGINEERING.md @@ -119,6 +119,9 @@ See `docs/LOCAL_AGENT_NOTES.md`. ### Current Baseline - `ci.yml` runs repository checks and whitespace checks on GitHub-hosted Linux. +- `emulator.yml` boots an OpenPhone SDK phone emulator on the + `openphone-emulator` self-hosted runner for PRs that touch runtime, + assistant, overlay, patch, integration, or emulator code. - `eval.yml` runs physical trajectory smokes on the `openphone-device` self-hosted runner. - `release.yml` runs release work on the `openphone-build` self-hosted runner. @@ -131,10 +134,10 @@ See `docs/LOCAL_AGENT_NOTES.md`. regressions. 2. **Emulator runtime smoke** - - Build or install an emulator image with the OpenPhone assistant. - - Boot it headlessly on a self-hosted runner. - - Run ADB-backed UI/context checks. - - Exercise local runtime actions without depending on provider keys. + - Build or reuse an emulator image with the OpenPhone assistant. + - Boot it headlessly on the `openphone-emulator` runner. + - Verify OpenPhone services, assistant install, ADB runtime status, screen + context, and a local assistant task without provider keys. 3. **Remote runtime smoke** - Start or connect to an OpenClaw gateway. diff --git a/docs/EMULATOR.md b/docs/EMULATOR.md index fd9da18..c869202 100644 --- a/docs/EMULATOR.md +++ b/docs/EMULATOR.md @@ -204,6 +204,36 @@ Expected signs of a working emulator: registered services - `org.openphone.assistant` is installed +## CI Smoke + +The emulator path is also a CI gate. The +[Emulator workflow](../.github/workflows/emulator.yml) runs on pull requests +that touch runtime, assistant, overlay, patch, integration, or emulator +scripts. It uses a self-hosted `openphone-emulator` runner and calls: + +```bash +./scripts/run-emulator-smoke.sh --arch x86_64 +``` + +The smoke harness: + +- builds or reuses the OpenPhone SDK phone image; +- boots a wiped headless emulator; +- waits for `sys.boot_completed=1`; +- verifies OpenPhone framework services are registered; +- verifies `org.openphone.assistant` is installed; +- starts the assistant UI; +- runs `runtime status` through the OpenPhone CLI; +- invokes `openphone.screen.get` through the ADB runtime transport; +- runs one local assistant task without provider API keys; +- uploads logcat, screenshot, UI XML, runtime status, screen result, and + assistant smoke output as workflow artifacts. + +This is the fast merge gate for AI-generated runtime and assistant changes. It +does not replace the nightly physical `openphone-device` eval, but it should +catch broken boot, package, service, ADB transport, CLI, and local assistant +paths before code reaches device-only validation. + ## CLI And MCP On The Emulator The local Runtime CLI and MCP server use ADB, so they can target the emulator diff --git a/patches/vendor_lineage/0002-OpenPhone-keep-repo-manifest-cache-under-out.patch b/patches/vendor_lineage/0002-OpenPhone-keep-repo-manifest-cache-under-out.patch index 8cc9587..2480c1e 100644 --- a/patches/vendor_lineage/0002-OpenPhone-keep-repo-manifest-cache-under-out.patch +++ b/patches/vendor_lineage/0002-OpenPhone-keep-repo-manifest-cache-under-out.patch @@ -12,27 +12,17 @@ blocked as a read-only write. Run the manifest invocation with HOME pointed at a rule-owned directory under PRODUCT_OUT so repo's cache stays in writable build output. --- - build/tasks/build-manifest_xml.mk | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) + build/tasks/build-manifest_xml.mk | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/tasks/build-manifest_xml.mk b/build/tasks/build-manifest_xml.mk -index 54a6f1bb..79471e4e 100644 +index 5f54967..ecf6d83 100644 --- a/build/tasks/build-manifest_xml.mk +++ b/build/tasks/build-manifest_xml.mk -@@ -16,9 +16,12 @@ ifdef MANIFEST_EXCLUDES - MANIFEST_EXCLUDES := |$(MANIFEST_EXCLUDES) - endif -- -+ -+BUILD_MANIFEST_REPO_HOME := $(PRODUCT_OUT)/obj/PACKAGING/build_manifest_intermediates/home -+ - $(INSTALLED_BUILD_MANIFEST_XML_TARGET): -- mkdir -p $(dir $@) -- python3 .repo/repo/repo manifest -o - -r | grep -Ev "proprietary_$(MANIFEST_EXCLUDES)" > $@ -+ mkdir -p $(dir $@) -+ mkdir -p $(BUILD_MANIFEST_REPO_HOME) -+ HOME=$(abspath $(BUILD_MANIFEST_REPO_HOME)) python3 .repo/repo/repo manifest -o - -r | grep -Ev "proprietary_$(MANIFEST_EXCLUDES)" > $@ +@@ -21,3 +21,4 @@ $(INSTALLED_BUILD_MANIFEST_XML_TARGET): +- REPO_TRACE=0 python3 .repo/repo/repo manifest -o - -r | grep -Ev "proprietary_$(MANIFEST_EXCLUDES)" > $@ - ++ mkdir -p $(PRODUCT_OUT)/obj/PACKAGING/build_manifest_intermediates/home ++ HOME=$(abspath $(PRODUCT_OUT)/obj/PACKAGING/build_manifest_intermediates/home) REPO_TRACE=0 python3 .repo/repo/repo manifest -o - -r | grep -Ev "proprietary_$(MANIFEST_EXCLUDES)" > $@ + .PHONY: build-manifest.xml - build-manifest.xml: $(INSTALLED_BUILD_MANIFEST_XML_TARGET) diff --git a/patches/vendor_lineage/0003-OpenPhone-expand-ARM64-emulator-super-partition.patch b/patches/vendor_lineage/0003-OpenPhone-expand-ARM64-emulator-super-partition.patch index 3a0d96f..71dafcf 100644 --- a/patches/vendor_lineage/0003-OpenPhone-expand-ARM64-emulator-super-partition.patch +++ b/patches/vendor_lineage/0003-OpenPhone-expand-ARM64-emulator-super-partition.patch @@ -1,24 +1,27 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From fd469fa99a49abf46be15ce6bcc8323bdd2700b1 Mon Sep 17 00:00:00 2001 From: OpenPhone -Date: Sun, 28 Jun 2026 21:12:00 +0000 +Date: Tue, 30 Jun 2026 14:40:52 -0700 Subject: [PATCH] OpenPhone: expand ARM64 emulator super partition -OpenPhone's privileged app and runtime assets exceed Lineage's default -ARM64 SDK phone 2 GiB dynamic partition budget. Match the x86_64 SDK -phone super partition sizing so openphone_sdk_phone_arm64 can package -the emulator system image zip. +OpenPhone's privileged app and runtime assets exceed Lineage's default ARM64 SDK phone 2 GiB dynamic partition budget. Match the x86_64 SDK phone super partition sizing so openphone_sdk_phone_arm64 can package the emulator system image zip. --- build/target/product/lineage_sdk_phone_arm64_board.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/target/product/lineage_sdk_phone_arm64_board.mk b/build/target/product/lineage_sdk_phone_arm64_board.mk -index 23f9b416..73661ef5 100644 +index 5f2c436..116408f 100644 --- a/build/target/product/lineage_sdk_phone_arm64_board.mk +++ b/build/target/product/lineage_sdk_phone_arm64_board.mk -@@ -15,3 +15,3 @@ +@@ -12,9 +12,9 @@ + # See the License for the specific language governing permissions and + # limitations under the License. + -# 2.0G + 8M -BOARD_SUPER_PARTITION_SIZE := 2155872256 -BOARD_EMULATOR_DYNAMIC_PARTITIONS_SIZE := 2147483648 +# 8.0G + 8M +BOARD_SUPER_PARTITION_SIZE := 8598323200 +BOARD_EMULATOR_DYNAMIC_PARTITIONS_SIZE := 8589934592 + + PRODUCT_SDK_ADDON_COPY_FILES += \ + device/generic/goldfish/data/etc/advancedFeatures.ini.arm:images/arm64-v8a/advancedFeatures.ini \ diff --git a/scripts/apply-patches.sh b/scripts/apply-patches.sh index add3dbb..b65e435 100755 --- a/scripts/apply-patches.sh +++ b/scripts/apply-patches.sh @@ -5,6 +5,9 @@ set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" need_cmd git +if [[ "${OPENPHONE_RESET_PATCH_TARGETS:-0}" == "1" ]]; then + need_cmd repo +fi [[ -d "$OPENPHONE_ANDROID_DIR/.repo" ]] || die "Android tree not initialized: $OPENPHONE_ANDROID_DIR" @@ -29,9 +32,23 @@ for patch_dir in "$OPENPHONE_ROOT"/patches/*; do [[ -d "$target_dir/.git" ]] || die "patch target is not a git repo: $repo_path" + if [[ "${OPENPHONE_RESET_PATCH_TARGETS:-0}" == "1" ]]; then + info "resetting patch target $repo_path to manifest revision" + ( + cd "$OPENPHONE_ANDROID_DIR" + repo forall "$repo_path" -c ' + git am --abort >/dev/null 2>&1 || true + git reset --hard "${REPO_LREV:-HEAD}" >/dev/null + git clean -ffd >/dev/null + ' + ) + fi + info "applying patches for $repo_path" ( cd "$target_dir" + git config user.name "${OPENPHONE_PATCH_GIT_NAME:-OpenPhone}" + git config user.email "${OPENPHONE_PATCH_GIT_EMAIL:-openphone@example.invalid}" applied_patch_ids="$(mktemp)" git log --format=email --patch --no-ext-diff -n 400 \ | git patch-id --stable \ diff --git a/scripts/check.sh b/scripts/check.sh index e98bada..9afd162 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -17,6 +17,7 @@ required=( docs/BUILD.md docs/CAPABILITIES.md docs/DEVICE_SUPPORT.md + docs/EMULATOR.md docs/GMS.md docs/LOCAL_AGENT_NOTES.md docs/LICENSING.md @@ -52,7 +53,9 @@ required=( schemas/screen-context.schema.json schemas/trajectory-event.schema.json .github/workflows/ci.yml + .github/workflows/emulator.yml .github/workflows/eval.yml + .github/workflows/gcp-lab.yml .github/workflows/release.yml .github/RUNNERS.md .github/ISSUE_TEMPLATE/bug_report.md @@ -138,7 +141,24 @@ required=( overlay/vendor/openphone/products/openphone_tegu.mk scripts/bootstrap-android-build-host.sh scripts/build-emulator.sh + scripts/run-emulator-smoke.sh scripts/run-emulator.sh + scripts/lab/allocate-slot.sh + scripts/lab/up.sh + scripts/lab/down.sh + scripts/lab/local-up.sh + scripts/lab/local-down.sh + scripts/lab/smoke.sh + scripts/lab/prepare-local.sh + scripts/lab/install-android-sdk-tools.sh + scripts/lab/gcp/common.sh + scripts/lab/gcp/setup-wif.sh + scripts/lab/gcp/create-vm.sh + scripts/lab/gcp/delete-vm.sh + scripts/lab/gcp/bootstrap-vm.sh + scripts/lab/gcp/prewarm-cache.sh + scripts/lab/gcp/run-smoke.sh + scripts/lab/gcp/seed-cache-from-boot-disk.sh overlay/packages/apps/OpenPhoneAssistant/Android.bp overlay/packages/apps/OpenPhoneAssistant/AndroidManifest.xml overlay/packages/apps/OpenPhoneAssistant/LICENSE @@ -184,7 +204,8 @@ for file in "${required[@]}"; do } done -for script in "$root"/scripts/*.sh; do +for script in "$root"/scripts/*.sh "$root"/scripts/lab/*.sh "$root"/scripts/lab/gcp/*.sh; do + [[ -e "$script" ]] || continue bash -n "$script" done diff --git a/scripts/common.sh b/scripts/common.sh index d24be8c..fb1d441 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -4,6 +4,11 @@ set -euo pipefail export PATH="$HOME/.local/bin:$PATH" export PATH="/opt/homebrew/opt/coreutils/libexec/gnubin:/usr/local/opt/coreutils/libexec/gnubin:$PATH" +if [[ "$(uname -s)" == "Darwin" && -x /opt/homebrew/opt/openjdk@17/bin/java ]] \ + && { [[ -z "${JAVA_HOME:-}" ]] || [[ "${JAVA_HOME:-}" == /opt/homebrew/opt/openjdk/* ]]; }; then + export JAVA_HOME="/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home" + export PATH="/opt/homebrew/opt/openjdk@17/bin:$PATH" +fi OPENPHONE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" OPENPHONE_ANDROID_DIR="${OPENPHONE_ANDROID_DIR:-$OPENPHONE_ROOT/.worktree/android}" diff --git a/scripts/create-macos-build-volume.sh b/scripts/create-macos-build-volume.sh index 4ab26ce..aebb888 100755 --- a/scripts/create-macos-build-volume.sh +++ b/scripts/create-macos-build-volume.sh @@ -14,7 +14,7 @@ on macOS. Environment: OPENPHONE_MACOS_IMAGE Sparsebundle path. OPENPHONE_MACOS_VOLUME_NAME Mounted volume directory name. - OPENPHONE_MACOS_IMAGE_SIZE Sparsebundle size, default 300g. + OPENPHONE_MACOS_IMAGE_SIZE Sparsebundle max size, default 700g. USAGE } @@ -39,7 +39,7 @@ need_cmd hdiutil image_path="${OPENPHONE_MACOS_IMAGE:-$OPENPHONE_ROOT/.worktree/OpenPhoneAndroid.sparsebundle}" volume_name="${OPENPHONE_MACOS_VOLUME_NAME:-OpenPhoneAndroid}" -size="${OPENPHONE_MACOS_IMAGE_SIZE:-300g}" +size="${OPENPHONE_MACOS_IMAGE_SIZE:-700g}" mkdir -p "$(dirname "$image_path")" @@ -55,14 +55,18 @@ else info "sparsebundle already exists: $image_path" fi -info "mounting $image_path" -mount_output="$(hdiutil attach "$image_path" -mountpoint "$OPENPHONE_ROOT/.worktree/$volume_name" -nobrowse 2>&1)" || { - printf '%s\n' "$mount_output" >&2 - die "failed to mount sparsebundle: $image_path" -} -printf '%s\n' "$mount_output" - mount_path="$OPENPHONE_ROOT/.worktree/$volume_name" +if mount | grep -F " on $mount_path (" >/dev/null 2>&1; then + info "sparsebundle already mounted: $mount_path" +else + info "mounting $image_path" + mount_output="$(hdiutil attach "$image_path" -mountpoint "$mount_path" -nobrowse 2>&1)" || { + printf '%s\n' "$mount_output" >&2 + die "failed to mount sparsebundle: $image_path" + } + printf '%s\n' "$mount_output" +fi + mkdir -p "$mount_path/android" cat <] [--index ] [--print] + +Creates a deterministic local OpenPhone lab slot and writes an env file with +isolated emulator/runtime ports and artifact directories. + +Options: + --slot Human-readable slot name. Default: hash of checkout path. + --index Explicit numeric slot index. Default: stable hash modulo 100. + --print Print shell exports after writing the env file. + -h, --help Show this help. + +The slot env file is written to .worktree/lab//env. +EOF +} + +slot="" +index="" +print_env=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --index) + [[ $# -ge 2 ]] || die "--index requires a value" + index="$2" + shift 2 + ;; + --print) + print_env=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +need_cmd python3 + +slot="$( + python3 - <<'PY' "$root" "$slot" +import hashlib +import re +import sys + +root = sys.argv[1] +slot = sys.argv[2].strip() +if not slot: + slot = "checkout-" + hashlib.sha1(root.encode("utf-8")).hexdigest()[:8] +slot = re.sub(r"[^A-Za-z0-9_.-]+", "-", slot).strip(".-") +print(slot or "default") +PY +)" + +if [[ -z "$index" ]]; then + index="$( + python3 - <<'PY' "$slot" +import hashlib +import sys + +slot = sys.argv[1] +print(int(hashlib.sha1(slot.encode("utf-8")).hexdigest()[:8], 16) % 100) +PY + )" +fi + +[[ "$index" =~ ^[0-9]+$ ]] || die "--index must be numeric" + +lab_root="$root/.worktree/lab" +slot_dir="$lab_root/$slot" +mkdir -p "$slot_dir"/{artifacts,emulator-data,logs,run,runtimes,secrets} + +base_emulator_port="${OPENPHONE_LAB_BASE_EMULATOR_PORT:-5584}" +base_openclaw_port="${OPENPHONE_LAB_BASE_OPENCLAW_PORT:-18791}" +base_hermes_port="${OPENPHONE_LAB_BASE_HERMES_PORT:-18891}" +base_model_broker_port="${OPENPHONE_LAB_BASE_MODEL_BROKER_PORT:-18991}" + +emulator_port=$((base_emulator_port + (index * 2))) +openclaw_port=$((base_openclaw_port + index)) +hermes_port=$((base_hermes_port + index)) +model_broker_port=$((base_model_broker_port + index)) + +env_file="$slot_dir/env" +cat > "$env_file" < [--purge] + +Stops emulator/runtime processes for a local OpenPhone lab slot. With --purge, +deletes that slot's ignored .worktree/lab directory. +EOF +} + +slot="" +purge=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --purge) + purge=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$slot" ]] || die "--slot is required" +env_file="$root/.worktree/lab/$slot/env" +[[ -f "$env_file" ]] || die "missing lab env file: $env_file" +# shellcheck disable=SC1090 +source "$env_file" + +if command -v adb >/dev/null 2>&1; then + adb -s "$ANDROID_SERIAL" emu kill >/dev/null 2>&1 || true +fi + +pid_dir="$OPENPHONE_LAB_DIR/run" +if [[ -d "$pid_dir" ]]; then + for pid_file in "$pid_dir"/*.pid; do + [[ -e "$pid_file" ]] || continue + pid="$(cat "$pid_file" 2>/dev/null || true)" + if [[ "$pid" =~ ^[0-9]+$ ]]; then + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + fi + rm -f "$pid_file" + done +fi + +if [[ "$purge" == true ]]; then + rm -rf "$OPENPHONE_LAB_DIR" +fi + +printf 'Stopped lab slot %s (%s)\n' "$slot" "$ANDROID_SERIAL" diff --git a/scripts/lab/gcp/bootstrap-vm.sh b/scripts/lab/gcp/bootstrap-vm.sh new file mode 100755 index 0000000..086b02d --- /dev/null +++ b/scripts/lab/gcp/bootstrap-vm.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/bootstrap-vm.sh --name [options] + +Waits for SSH and installs base packages/KVM access on one OpenPhone GCP lab VM. +The repo-specific Android build bootstrap still runs from the checked-out repo. + +Options: + --name VM name. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --timeout SSH timeout. Default: 600. + -h, --help Show this help. +EOF +} + +name="" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" +timeout_seconds=600 + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) + [[ $# -ge 2 ]] || die "--name requires a value" + name="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + --timeout) + [[ $# -ge 2 ]] || die "--timeout requires a value" + timeout_seconds="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$name" ]] || die "--name is required" +name="$(sanitize_gcp_name "$name")" +[[ "$timeout_seconds" =~ ^[0-9]+$ ]] || die "--timeout must be numeric" +need_gcloud + +tmp_script="$(mktemp "${TMPDIR:-/tmp}/openphone-gcp-bootstrap.XXXXXX")" +cleanup_tmp() { + rm -f "$tmp_script" +} +trap cleanup_tmp EXIT + +cat > "$tmp_script" <<'REMOTE' +#!/usr/bin/env bash +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive + +sudo apt-get update +sudo apt-get install -y \ + acl \ + ca-certificates \ + curl \ + git \ + git-lfs \ + jq \ + kmod \ + lsof \ + qemu-kvm \ + rsync \ + unzip \ + zip + +sudo modprobe kvm || true +sudo modprobe kvm_intel || true +sudo modprobe kvm_amd || true + +if [[ -e /dev/kvm ]]; then + sudo chmod 0666 /dev/kvm || true +fi + +git lfs install --skip-repo +mkdir -p "$HOME/openphone-lab" "$HOME/openphone-android" +printf 'OpenPhone GCP VM bootstrap complete.\n' +REMOTE + +deadline=$((SECONDS + timeout_seconds)) +until gcloud compute ssh "$name" \ + --project "$project" \ + --zone "$zone" \ + --command "true" >/dev/null 2>&1; do + if [[ "$SECONDS" -ge "$deadline" ]]; then + die "SSH was not ready for $name within ${timeout_seconds}s" + fi + sleep 10 +done + +info "Copying bootstrap script to $name" +gcloud compute scp "$tmp_script" "$name:/tmp/openphone-bootstrap-vm.sh" \ + --project "$project" \ + --zone "$zone" >/dev/null + +info "Bootstrapping $name" +gcloud compute ssh "$name" \ + --project "$project" \ + --zone "$zone" \ + --command "bash /tmp/openphone-bootstrap-vm.sh" diff --git a/scripts/lab/gcp/common.sh b/scripts/lab/gcp/common.sh new file mode 100644 index 0000000..3eeacc0 --- /dev/null +++ b/scripts/lab/gcp/common.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +gcp_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$gcp_script_dir/../../.." && pwd)" +# shellcheck source=scripts/common.sh +source "$root/scripts/common.sh" + +OPENPHONE_GCP_PROJECT="${OPENPHONE_GCP_PROJECT:-openphone-lab}" +OPENPHONE_GCP_REGION="${OPENPHONE_GCP_REGION:-us-central1}" +OPENPHONE_GCP_ZONE="${OPENPHONE_GCP_ZONE:-us-central1-a}" +OPENPHONE_GCP_MACHINE_TYPE="${OPENPHONE_GCP_MACHINE_TYPE:-c3-standard-22}" +OPENPHONE_GCP_BOOT_DISK_SIZE="${OPENPHONE_GCP_BOOT_DISK_SIZE:-1000GB}" +OPENPHONE_GCP_BOOT_DISK_TYPE="${OPENPHONE_GCP_BOOT_DISK_TYPE:-pd-ssd}" +OPENPHONE_GCP_IMAGE_FAMILY="${OPENPHONE_GCP_IMAGE_FAMILY:-ubuntu-2404-lts-amd64}" +OPENPHONE_GCP_IMAGE_PROJECT="${OPENPHONE_GCP_IMAGE_PROJECT:-ubuntu-os-cloud}" +OPENPHONE_GCP_NETWORK="${OPENPHONE_GCP_NETWORK:-default}" +OPENPHONE_GCP_CACHE_MODE="${OPENPHONE_GCP_CACHE_MODE:-scratch}" +OPENPHONE_GCP_CACHE_DISK_TYPE="${OPENPHONE_GCP_CACHE_DISK_TYPE:-$OPENPHONE_GCP_BOOT_DISK_TYPE}" +OPENPHONE_GCP_CACHE_DISK_SIZE="${OPENPHONE_GCP_CACHE_DISK_SIZE:-1000GB}" +OPENPHONE_GCP_CACHE_MOUNT="${OPENPHONE_GCP_CACHE_MOUNT:-/mnt/openphone-cache}" +OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT="${OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT:-}" + +need_gcloud() { + need_cmd gcloud +} + +sanitize_gcp_name() { + need_cmd python3 + python3 - "$1" <<'PY' +import re +import sys + +value = sys.argv[1].lower() +value = re.sub(r"[^a-z0-9-]+", "-", value).strip("-") +if not value: + value = "openphone-lab" +if not value[0].isalpha(): + value = "openphone-" + value +value = value[:63].rstrip("-") +print(value) +PY +} + +shell_quote() { + printf '%q' "$1" +} + +gcp_instance_exists() { + local name="$1" project="$2" zone="$3" + gcloud compute instances describe "$name" \ + --project "$project" \ + --zone "$zone" >/dev/null 2>&1 +} diff --git a/scripts/lab/gcp/create-vm.sh b/scripts/lab/gcp/create-vm.sh new file mode 100755 index 0000000..434a01a --- /dev/null +++ b/scripts/lab/gcp/create-vm.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/create-vm.sh [options] + +Creates one disposable OpenPhone lab VM inside the shared GCP lab project. + +Options: + --name VM name. Default: generated. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --machine-type Machine type. Default: c3-standard-22. + --boot-disk-size Boot disk size. Default: 1000GB. + --boot-disk-type Boot disk type. Default: pd-ssd. + --cache-mode scratch, attach-disk, or snapshot. Default: scratch. + --cache-disk Existing disk to attach for attach-disk mode. + Per-run disk name for snapshot mode. + --cache-source-snapshot Snapshot to clone for snapshot mode. + --network Network. Default: default. + --labels Extra comma-separated labels. + -h, --help Show this help. +EOF +} + +name="" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" +machine_type="$OPENPHONE_GCP_MACHINE_TYPE" +boot_disk_size="$OPENPHONE_GCP_BOOT_DISK_SIZE" +boot_disk_type="$OPENPHONE_GCP_BOOT_DISK_TYPE" +cache_mode="$OPENPHONE_GCP_CACHE_MODE" +cache_disk="${OPENPHONE_GCP_CACHE_DISK:-}" +cache_source_snapshot="$OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT" +cache_disk_size="$OPENPHONE_GCP_CACHE_DISK_SIZE" +cache_disk_type="$OPENPHONE_GCP_CACHE_DISK_TYPE" +network="$OPENPHONE_GCP_NETWORK" +extra_labels="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) + [[ $# -ge 2 ]] || die "--name requires a value" + name="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + --machine-type) + [[ $# -ge 2 ]] || die "--machine-type requires a value" + machine_type="$2" + shift 2 + ;; + --boot-disk-size) + [[ $# -ge 2 ]] || die "--boot-disk-size requires a value" + boot_disk_size="$2" + shift 2 + ;; + --boot-disk-type) + [[ $# -ge 2 ]] || die "--boot-disk-type requires a value" + boot_disk_type="$2" + shift 2 + ;; + --cache-mode) + [[ $# -ge 2 ]] || die "--cache-mode requires a value" + cache_mode="$2" + shift 2 + ;; + --cache-disk) + [[ $# -ge 2 ]] || die "--cache-disk requires a value" + cache_disk="$2" + shift 2 + ;; + --cache-source-snapshot) + [[ $# -ge 2 ]] || die "--cache-source-snapshot requires a value" + cache_source_snapshot="$2" + shift 2 + ;; + --network) + [[ $# -ge 2 ]] || die "--network requires a value" + network="$2" + shift 2 + ;; + --labels) + [[ $# -ge 2 ]] || die "--labels requires a value" + extra_labels="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +need_gcloud + +if [[ -z "$name" ]]; then + name="$(sanitize_gcp_name "openphone-lab-$(date -u +%Y%m%d-%H%M%S)")" +else + name="$(sanitize_gcp_name "$name")" +fi + +case "$cache_mode" in + scratch|attach-disk|snapshot) ;; + *) die "unsupported cache mode: $cache_mode" ;; +esac + +if gcp_instance_exists "$name" "$project" "$zone"; then + info "VM already exists: $name" + printf '%s\n' "$name" + exit 0 +fi + +if [[ "$cache_mode" == "attach-disk" ]]; then + [[ -n "$cache_disk" ]] || die "--cache-disk is required when --cache-mode attach-disk" + gcloud compute disks describe "$cache_disk" \ + --project "$project" \ + --zone "$zone" >/dev/null +fi + +if [[ "$cache_mode" == "snapshot" ]]; then + [[ -n "$cache_source_snapshot" ]] || die "--cache-source-snapshot is required when --cache-mode snapshot" + if [[ -z "$cache_disk" ]]; then + cache_disk="$(sanitize_gcp_name "${name}-cache")" + else + cache_disk="$(sanitize_gcp_name "$cache_disk")" + fi + if gcloud compute disks describe "$cache_disk" \ + --project "$project" \ + --zone "$zone" >/dev/null 2>&1; then + die "cache disk already exists for snapshot mode: $cache_disk" + fi +fi + +labels="app=openphone,purpose=lab,managed-by=codex" +if [[ -n "$extra_labels" ]]; then + labels="${labels},${extra_labels}" +fi + +created_cache_disk=false +cleanup_cache_disk() { + local status=$? + if [[ "$created_cache_disk" == true ]]; then + gcloud compute disks delete "$cache_disk" \ + --project "$project" \ + --zone "$zone" \ + --quiet >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup_cache_disk EXIT + +if [[ "$cache_mode" == "snapshot" ]]; then + info "Creating per-run cache disk from snapshot: $cache_disk" + gcloud compute disks create "$cache_disk" \ + --project "$project" \ + --zone "$zone" \ + --type "$cache_disk_type" \ + --size "$cache_disk_size" \ + --source-snapshot "$cache_source_snapshot" \ + --labels "$labels" + created_cache_disk=true +fi + +args=( + compute instances create "$name" + --project "$project" + --zone "$zone" + --machine-type "$machine_type" + --boot-disk-size "$boot_disk_size" + --boot-disk-type "$boot_disk_type" + --image-family "$OPENPHONE_GCP_IMAGE_FAMILY" + --image-project "$OPENPHONE_GCP_IMAGE_PROJECT" + --enable-nested-virtualization + --maintenance-policy TERMINATE + --restart-on-failure + --no-service-account + --no-scopes + --labels "$labels" + --tags openphone-lab + --metadata "openphone-lab=true" +) + +if [[ -n "$network" ]]; then + args+=(--network "$network") +fi + +if [[ "$cache_mode" == "attach-disk" || "$cache_mode" == "snapshot" ]]; then + cache_auto_delete="no" + if [[ "$cache_mode" == "snapshot" ]]; then + cache_auto_delete="yes" + fi + args+=(--disk "name=$cache_disk,device-name=openphone-cache,mode=rw,boot=no,auto-delete=$cache_auto_delete") +fi + +info "Creating GCP lab VM: $name" +gcloud "${args[@]}" +created_cache_disk=false +printf '%s\n' "$name" diff --git a/scripts/lab/gcp/delete-vm.sh b/scripts/lab/gcp/delete-vm.sh new file mode 100755 index 0000000..8cbdfc3 --- /dev/null +++ b/scripts/lab/gcp/delete-vm.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/delete-vm.sh --name [options] + +Deletes one OpenPhone lab VM. Boot disks are auto-deleted by default by +create-vm.sh; named cache disks are retained. + +Options: + --name VM name. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + -h, --help Show this help. +EOF +} + +name="" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) + [[ $# -ge 2 ]] || die "--name requires a value" + name="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$name" ]] || die "--name is required" +name="$(sanitize_gcp_name "$name")" +need_gcloud + +if ! gcp_instance_exists "$name" "$project" "$zone"; then + info "VM does not exist: $name" + exit 0 +fi + +info "Deleting GCP lab VM: $name" +gcloud compute instances delete "$name" \ + --project "$project" \ + --zone "$zone" \ + --quiet diff --git a/scripts/lab/gcp/prewarm-cache.sh b/scripts/lab/gcp/prewarm-cache.sh new file mode 100755 index 0000000..ce065df --- /dev/null +++ b/scripts/lab/gcp/prewarm-cache.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/prewarm-cache.sh [options] + +Creates or refreshes a persistent OpenPhone Android cache disk, runs the GCP lab +smoke against it, and snapshots the disk for parallel warm lab runs. + +Options: + --cache-disk Persistent cache disk to create/reuse. + --snapshot Snapshot name to create after smoke passes. + --source-snapshot Optional snapshot to seed a new cache disk. + --repo-url Git repo URL. Default: current origin or GitHub. + --ref Git ref/SHA to prewarm. Default: current HEAD. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --machine-type Machine type. Default: c3-standard-22. + --boot-disk-size Boot disk size. Default: 1000GB. + --boot-disk-type Boot disk type. Default: pd-ssd. + --cache-disk-size Cache disk size. Default: 1000GB. + --cache-disk-type Cache disk type. Default: boot disk type. + --arch arm64|x86_64 Emulator arch. Default: x86_64. + --variant eng|userdebug Emulator variant. Default: eng. + --timeout Emulator boot timeout. Default: 900. + --repo-sync-jobs repo sync jobs. Default: nproc on the VM. + --keep-vm Leave the prewarm VM running for debug. + -h, --help Show this help. +EOF +} + +default_ref="$(git -C "$root" rev-parse HEAD 2>/dev/null || printf 'main')" +default_repo_url="$(git -C "$root" config --get remote.origin.url 2>/dev/null || printf 'https://github.com/secondly-com/OpenPhone.git')" +case "$default_repo_url" in + git@github.com:secondly-com/OpenPhone.git) + default_repo_url="https://github.com/secondly-com/OpenPhone.git" + ;; +esac + +repo_url="$default_repo_url" +ref="$default_ref" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" +machine_type="$OPENPHONE_GCP_MACHINE_TYPE" +boot_disk_size="$OPENPHONE_GCP_BOOT_DISK_SIZE" +boot_disk_type="$OPENPHONE_GCP_BOOT_DISK_TYPE" +cache_disk="" +cache_disk_size="$OPENPHONE_GCP_CACHE_DISK_SIZE" +cache_disk_type="$OPENPHONE_GCP_CACHE_DISK_TYPE" +source_snapshot="" +snapshot="" +arch="x86_64" +variant="eng" +timeout_seconds=900 +repo_sync_jobs="" +keep_vm=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --cache-disk) + [[ $# -ge 2 ]] || die "--cache-disk requires a value" + cache_disk="$2" + shift 2 + ;; + --snapshot) + [[ $# -ge 2 ]] || die "--snapshot requires a value" + snapshot="$2" + shift 2 + ;; + --source-snapshot) + [[ $# -ge 2 ]] || die "--source-snapshot requires a value" + source_snapshot="$2" + shift 2 + ;; + --repo-url) + [[ $# -ge 2 ]] || die "--repo-url requires a value" + repo_url="$2" + shift 2 + ;; + --ref) + [[ $# -ge 2 ]] || die "--ref requires a value" + ref="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + --machine-type) + [[ $# -ge 2 ]] || die "--machine-type requires a value" + machine_type="$2" + shift 2 + ;; + --boot-disk-size) + [[ $# -ge 2 ]] || die "--boot-disk-size requires a value" + boot_disk_size="$2" + shift 2 + ;; + --boot-disk-type) + [[ $# -ge 2 ]] || die "--boot-disk-type requires a value" + boot_disk_type="$2" + shift 2 + ;; + --cache-disk-size) + [[ $# -ge 2 ]] || die "--cache-disk-size requires a value" + cache_disk_size="$2" + shift 2 + ;; + --cache-disk-type) + [[ $# -ge 2 ]] || die "--cache-disk-type requires a value" + cache_disk_type="$2" + shift 2 + ;; + --arch) + [[ $# -ge 2 ]] || die "--arch requires a value" + arch="$2" + shift 2 + ;; + --variant) + [[ $# -ge 2 ]] || die "--variant requires a value" + variant="$2" + shift 2 + ;; + --timeout) + [[ $# -ge 2 ]] || die "--timeout requires a value" + timeout_seconds="$2" + shift 2 + ;; + --repo-sync-jobs) + [[ $# -ge 2 ]] || die "--repo-sync-jobs requires a value" + repo_sync_jobs="$2" + shift 2 + ;; + --keep-vm) + keep_vm=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +case "$arch" in + arm64|x86_64) ;; + *) die "unsupported emulator arch: $arch" ;; +esac + +case "$variant" in + eng|userdebug) ;; + *) die "unsupported emulator variant: $variant" ;; +esac + +need_gcloud + +if [[ -z "$cache_disk" ]]; then + cache_disk="$(sanitize_gcp_name "openphone-cache-${arch}-${OPENPHONE_RELEASE}")" +else + cache_disk="$(sanitize_gcp_name "$cache_disk")" +fi + +if [[ -z "$snapshot" ]]; then + snapshot="$(sanitize_gcp_name "${cache_disk}-$(date -u +%Y%m%d-%H%M%S)")" +else + snapshot="$(sanitize_gcp_name "$snapshot")" +fi + +if gcloud compute snapshots describe "$snapshot" \ + --project "$project" >/dev/null 2>&1; then + die "snapshot already exists: $snapshot" +fi + +if ! gcloud compute disks describe "$cache_disk" \ + --project "$project" \ + --zone "$zone" >/dev/null 2>&1; then + create_args=( + compute disks create "$cache_disk" + --project "$project" + --zone "$zone" + --type "$cache_disk_type" + --size "$cache_disk_size" + --labels "app=openphone,purpose=lab-cache,managed-by=codex" + ) + if [[ -n "$source_snapshot" ]]; then + create_args+=(--source-snapshot "$source_snapshot") + fi + info "Creating cache disk: $cache_disk" + gcloud "${create_args[@]}" +else + info "Reusing cache disk: $cache_disk" +fi + +short_ref="$(printf '%s' "$ref" | cut -c1-12)" +name="$(sanitize_gcp_name "openphone-lab-prewarm-${short_ref}-$(date -u +%H%M%S)")" +run_args=( + --name "$name" + --repo-url "$repo_url" + --ref "$ref" + --project "$project" + --zone "$zone" + --machine-type "$machine_type" + --boot-disk-size "$boot_disk_size" + --boot-disk-type "$boot_disk_type" + --cache-mode attach-disk + --cache-disk "$cache_disk" + --arch "$arch" + --variant "$variant" + --runtime local + --timeout "$timeout_seconds" +) +if [[ -n "$repo_sync_jobs" ]]; then + run_args+=(--repo-sync-jobs "$repo_sync_jobs") +fi +if [[ "$keep_vm" == true ]]; then + run_args+=(--keep-vm) +fi + +"$script_dir/run-smoke.sh" "${run_args[@]}" + +if [[ "$keep_vm" == true ]]; then + info "--keep-vm left the cache disk attached; skipping snapshot creation" + printf 'Cache disk: %s\n' "$cache_disk" + printf 'Rerun without --keep-vm to publish a warm snapshot.\n' + exit 0 +fi + +info "Creating cache snapshot: $snapshot" +gcloud compute snapshots create "$snapshot" \ + --project "$project" \ + --source-disk "$cache_disk" \ + --source-disk-zone "$zone" \ + --storage-location "$OPENPHONE_GCP_REGION" \ + --labels "app=openphone,purpose=lab-cache,managed-by=codex" + +printf 'Cache disk: %s\n' "$cache_disk" +printf 'Cache snapshot: %s\n' "$snapshot" +printf 'Set this repo variable for warm PR/release labs:\n' +printf ' OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT=%s\n' "$snapshot" diff --git a/scripts/lab/gcp/run-smoke.sh b/scripts/lab/gcp/run-smoke.sh new file mode 100755 index 0000000..d084c77 --- /dev/null +++ b/scripts/lab/gcp/run-smoke.sh @@ -0,0 +1,412 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/run-smoke.sh [options] + +Creates a disposable GCP lab VM, checks out OpenPhone, syncs/builds the Android +tree, runs the same lab smoke used locally, copies artifacts back, and tears the +VM down unless --keep-vm is set. + +Options: + --name VM name. Default: generated from current ref/time. + --repo-url Git repo URL. Default: current origin or GitHub. + --ref Git ref/SHA to test. Default: current HEAD. + --slot Lab slot name on the VM. Default: VM name. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --machine-type Machine type. Default: c3-standard-22. + --boot-disk-size Boot disk size. Default: 1000GB. + --boot-disk-type Boot disk type. Default: pd-ssd. + --cache-mode scratch, attach-disk, or snapshot. Default: scratch. + --cache-disk Existing disk to attach for attach-disk mode. + Per-run disk name for snapshot mode. + --cache-source-snapshot Snapshot to clone for snapshot mode. + --cache-mount Mount path for cache disk. Default: /mnt/openphone-cache. + The Android tree is bind-mounted back to + $HOME/openphone-android for stable build paths. + --arch arm64|x86_64 Emulator arch. Default: x86_64. + --variant eng|userdebug Emulator variant. Default: eng. + --runtime Runtime intent: local, openclaw, or hermes. + May be repeated. Default: local. + --timeout Emulator boot timeout. Default: 900. + --repo-sync-jobs repo sync jobs. Default: nproc. + --keep-vm Leave VM running for debug. + --skip-build Reuse existing Android build outputs on the VM. + -h, --help Show this help. +EOF +} + +default_ref="$(git -C "$root" rev-parse HEAD 2>/dev/null || printf 'main')" +default_repo_url="$(git -C "$root" config --get remote.origin.url 2>/dev/null || printf 'https://github.com/secondly-com/OpenPhone.git')" +case "$default_repo_url" in + git@github.com:secondly-com/OpenPhone.git) + default_repo_url="https://github.com/secondly-com/OpenPhone.git" + ;; +esac + +name="" +repo_url="$default_repo_url" +ref="$default_ref" +slot="" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" +machine_type="$OPENPHONE_GCP_MACHINE_TYPE" +boot_disk_size="$OPENPHONE_GCP_BOOT_DISK_SIZE" +boot_disk_type="$OPENPHONE_GCP_BOOT_DISK_TYPE" +cache_mode="$OPENPHONE_GCP_CACHE_MODE" +cache_disk="${OPENPHONE_GCP_CACHE_DISK:-}" +cache_source_snapshot="$OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT" +cache_mount="$OPENPHONE_GCP_CACHE_MOUNT" +arch="x86_64" +variant="eng" +timeout_seconds=900 +repo_sync_jobs="" +keep_vm=false +skip_build=false +runtimes=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) + [[ $# -ge 2 ]] || die "--name requires a value" + name="$2" + shift 2 + ;; + --repo-url) + [[ $# -ge 2 ]] || die "--repo-url requires a value" + repo_url="$2" + shift 2 + ;; + --ref) + [[ $# -ge 2 ]] || die "--ref requires a value" + ref="$2" + shift 2 + ;; + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + --machine-type) + [[ $# -ge 2 ]] || die "--machine-type requires a value" + machine_type="$2" + shift 2 + ;; + --boot-disk-size) + [[ $# -ge 2 ]] || die "--boot-disk-size requires a value" + boot_disk_size="$2" + shift 2 + ;; + --boot-disk-type) + [[ $# -ge 2 ]] || die "--boot-disk-type requires a value" + boot_disk_type="$2" + shift 2 + ;; + --cache-mode) + [[ $# -ge 2 ]] || die "--cache-mode requires a value" + cache_mode="$2" + shift 2 + ;; + --cache-disk) + [[ $# -ge 2 ]] || die "--cache-disk requires a value" + cache_disk="$2" + shift 2 + ;; + --cache-source-snapshot) + [[ $# -ge 2 ]] || die "--cache-source-snapshot requires a value" + cache_source_snapshot="$2" + shift 2 + ;; + --cache-mount) + [[ $# -ge 2 ]] || die "--cache-mount requires a value" + cache_mount="$2" + shift 2 + ;; + --arch) + [[ $# -ge 2 ]] || die "--arch requires a value" + arch="$2" + shift 2 + ;; + --variant) + [[ $# -ge 2 ]] || die "--variant requires a value" + variant="$2" + shift 2 + ;; + --runtime) + [[ $# -ge 2 ]] || die "--runtime requires a value" + runtimes+=("$2") + shift 2 + ;; + --timeout) + [[ $# -ge 2 ]] || die "--timeout requires a value" + timeout_seconds="$2" + shift 2 + ;; + --repo-sync-jobs) + [[ $# -ge 2 ]] || die "--repo-sync-jobs requires a value" + repo_sync_jobs="$2" + shift 2 + ;; + --keep-vm) + keep_vm=true + shift + ;; + --skip-build) + skip_build=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +need_gcloud + +case "$arch" in + arm64|x86_64) ;; + *) die "unsupported emulator arch: $arch" ;; +esac + +case "$variant" in + eng|userdebug) ;; + *) die "unsupported emulator variant: $variant" ;; +esac + +case "$cache_mode" in + scratch|attach-disk|snapshot) ;; + *) die "unsupported cache mode: $cache_mode" ;; +esac + +if [[ "$cache_mode" == "snapshot" && -z "$cache_source_snapshot" ]]; then + die "--cache-source-snapshot is required when --cache-mode snapshot" +fi + +if [[ ${#runtimes[@]} -eq 0 ]]; then + runtimes=(local) +fi + +if [[ -z "$name" ]]; then + short_ref="$(printf '%s' "$ref" | cut -c1-12)" + name="openphone-lab-${short_ref}-$(date -u +%H%M%S)" +fi +name="$(sanitize_gcp_name "$name")" +slot="${slot:-$name}" +slot="$(printf '%s' "$slot" | tr -c 'A-Za-z0-9_.-' '-')" + +artifact_root="$root/.worktree/gcp-lab/$name" +artifact_dir="$artifact_root/artifacts" +mkdir -p "$artifact_dir" + +vm_created=false +artifacts_copied=false +remote_script="$(mktemp "${TMPDIR:-/tmp}/openphone-gcp-remote.XXXXXX")" +copy_remote_artifacts() { + mkdir -p "$artifact_dir" + if gcloud compute scp --recurse \ + "$name:~/openphone-src/.worktree/lab/$slot/artifacts" \ + "$artifact_dir/" \ + --project "$project" \ + --zone "$zone" >/dev/null; then + artifacts_copied=true + return 0 + fi + return 1 +} + +cleanup() { + local status=$? + set +e + rm -f "$remote_script" + if [[ "$vm_created" == true && "$artifacts_copied" != true ]]; then + copy_remote_artifacts >/dev/null 2>&1 || true + fi + if [[ "$vm_created" == true && "$keep_vm" != true ]]; then + "$script_dir/delete-vm.sh" --name "$name" --project "$project" --zone "$zone" || true + fi + exit "$status" +} +trap cleanup EXIT + +create_args=( + --name "$name" + --project "$project" + --zone "$zone" + --machine-type "$machine_type" + --boot-disk-size "$boot_disk_size" + --boot-disk-type "$boot_disk_type" + --cache-mode "$cache_mode" +) +if [[ -n "$cache_disk" ]]; then + create_args+=(--cache-disk "$cache_disk") +fi +if [[ -n "$cache_source_snapshot" ]]; then + create_args+=(--cache-source-snapshot "$cache_source_snapshot") +fi + +"$script_dir/create-vm.sh" "${create_args[@]}" +vm_created=true + +"$script_dir/bootstrap-vm.sh" --name "$name" --project "$project" --zone "$zone" + +cat > "$remote_script" <<'REMOTE' +#!/usr/bin/env bash +set -euo pipefail + +repo_url="${OPENPHONE_REPO_URL:?}" +ref="${OPENPHONE_REF:?}" +slot="${OPENPHONE_LAB_SLOT:?}" +arch="${OPENPHONE_EMULATOR_ARCH:?}" +variant="${OPENPHONE_EMULATOR_VARIANT:?}" +timeout_seconds="${OPENPHONE_EMULATOR_TIMEOUT:?}" +repo_sync_jobs="${OPENPHONE_REPO_SYNC_JOBS:-}" +skip_build="${OPENPHONE_SKIP_BUILD:-0}" +runtime_csv="${OPENPHONE_LAB_RUNTIMES:-local}" +cache_mode="${OPENPHONE_GCP_CACHE_MODE:-scratch}" +cache_mount="${OPENPHONE_GCP_CACHE_MOUNT:-/mnt/openphone-cache}" + +export OPENPHONE_RELEASE="${OPENPHONE_RELEASE:-bp4a}" + +prepare_android_workspace() { + export OPENPHONE_ANDROID_DIR="${OPENPHONE_ANDROID_DIR:-$HOME/openphone-android}" + + if [[ "$cache_mode" == "scratch" ]]; then + mkdir -p "$OPENPHONE_ANDROID_DIR" + return 0 + fi + + local device="/dev/disk/by-id/google-openphone-cache" + local deadline=$((SECONDS + 300)) + while [[ ! -e "$device" ]]; do + if [[ "$SECONDS" -ge "$deadline" ]]; then + printf 'error: cache disk device did not appear: %s\n' "$device" >&2 + exit 1 + fi + sleep 2 + done + + if ! sudo blkid "$device" >/dev/null 2>&1; then + sudo mkfs.ext4 -F -L openphone-cache "$device" + fi + + sudo mkdir -p "$cache_mount" + if ! findmnt --mountpoint "$cache_mount" >/dev/null 2>&1; then + sudo mount -o defaults,discard "$device" "$cache_mount" + fi + sudo chown "$USER:$USER" "$cache_mount" + + local cache_android_dir="$cache_mount/android" + mkdir -p "$cache_android_dir" + mkdir -p "$OPENPHONE_ANDROID_DIR" + if ! findmnt --mountpoint "$OPENPHONE_ANDROID_DIR" >/dev/null 2>&1; then + sudo mount --bind "$cache_android_dir" "$OPENPHONE_ANDROID_DIR" + fi + sudo chown "$USER:$USER" "$cache_mount" "$cache_android_dir" "$OPENPHONE_ANDROID_DIR" +} + +prepare_android_workspace + +if [[ ! -d "$HOME/openphone-src/.git" ]]; then + rm -rf "$HOME/openphone-src" + git clone "$repo_url" "$HOME/openphone-src" +fi + +cd "$HOME/openphone-src" +git remote set-url origin "$repo_url" +git fetch --tags --prune origin +git fetch origin "$ref" || true +git checkout --force "$ref" || git checkout --force FETCH_HEAD + +./scripts/bootstrap-android-build-host.sh +./scripts/lab/install-android-sdk-tools.sh +export ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:-/opt/android-sdk}" +export ANDROID_HOME="${ANDROID_HOME:-$ANDROID_SDK_ROOT}" +export PATH="$ANDROID_SDK_ROOT/platform-tools:$ANDROID_SDK_ROOT/emulator:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:$PATH" +OPENPHONE_SKIP_JAVA_CHECK=1 ./scripts/check.sh + +if [[ "$skip_build" != "1" ]]; then + sync_args=() + if [[ -n "$repo_sync_jobs" ]]; then + sync_args+=(-j"$repo_sync_jobs") + else + sync_args+=(-j"$(nproc)") + fi + sync_args+=(--detach) + if [[ "$cache_mode" == "scratch" ]]; then + sync_args+=(--force-sync --force-checkout) + fi + ./scripts/sync.sh "${sync_args[@]}" + if [[ "$cache_mode" == "scratch" ]]; then + ./scripts/apply-patches.sh + else + OPENPHONE_RESET_PATCH_TARGETS=1 ./scripts/apply-patches.sh + fi + ./scripts/check.sh +fi + +IFS=',' read -r -a runtimes <<< "$runtime_csv" +smoke_args=(--slot "$slot" --arch "$arch" --variant "$variant" --timeout "$timeout_seconds") +if [[ "$skip_build" == "1" ]]; then + smoke_args+=(--skip-build) +fi +for runtime in "${runtimes[@]}"; do + [[ -n "$runtime" ]] || continue + smoke_args+=(--runtime "$runtime") +done + +./scripts/lab/smoke.sh "${smoke_args[@]}" +REMOTE + +info "Copying remote smoke script to $name" +gcloud compute scp "$remote_script" "$name:/tmp/openphone-gcp-run-smoke.sh" \ + --project "$project" \ + --zone "$zone" >/dev/null + +runtime_csv="$(IFS=,; printf '%s' "${runtimes[*]}")" +skip_build_value=0 +if [[ "$skip_build" == true ]]; then + skip_build_value=1 +fi +remote_command="OPENPHONE_REPO_URL=$(shell_quote "$repo_url")" +remote_command+=" OPENPHONE_REF=$(shell_quote "$ref")" +remote_command+=" OPENPHONE_LAB_SLOT=$(shell_quote "$slot")" +remote_command+=" OPENPHONE_EMULATOR_ARCH=$(shell_quote "$arch")" +remote_command+=" OPENPHONE_EMULATOR_VARIANT=$(shell_quote "$variant")" +remote_command+=" OPENPHONE_EMULATOR_TIMEOUT=$(shell_quote "$timeout_seconds")" +remote_command+=" OPENPHONE_REPO_SYNC_JOBS=$(shell_quote "$repo_sync_jobs")" +remote_command+=" OPENPHONE_SKIP_BUILD=$(shell_quote "$skip_build_value")" +remote_command+=" OPENPHONE_LAB_RUNTIMES=$(shell_quote "$runtime_csv")" +remote_command+=" OPENPHONE_GCP_CACHE_MODE=$(shell_quote "$cache_mode")" +remote_command+=" OPENPHONE_GCP_CACHE_MOUNT=$(shell_quote "$cache_mount")" +remote_command+=" bash /tmp/openphone-gcp-run-smoke.sh" + +info "Running GCP lab smoke on $name" +gcloud compute ssh "$name" \ + --project "$project" \ + --zone "$zone" \ + --command "$remote_command" \ + | tee "$artifact_root/gcp-run-smoke.log" + +copy_remote_artifacts +info "GCP lab smoke passed; artifacts copied to $artifact_dir" diff --git a/scripts/lab/gcp/seed-cache-from-boot-disk.sh b/scripts/lab/gcp/seed-cache-from-boot-disk.sh new file mode 100755 index 0000000..89c41f4 --- /dev/null +++ b/scripts/lab/gcp/seed-cache-from-boot-disk.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/seed-cache-from-boot-disk.sh --source-disk [options] + +Copies a completed scratch lab Android tree from a preserved VM boot disk into a +persistent cache disk, then optionally snapshots that cache disk for warm lab +runs. + +Options: + --source-disk Preserved scratch VM boot disk to read. + --source-android-dir Android tree path on source disk. + Default: auto-detect /home/*/openphone-android. + --cache-disk Cache disk to create/reuse. + --snapshot Optional snapshot name to create after copy. + --project GCP project. Default: OPENPHONE_GCP_PROJECT. + --zone GCP zone. Default: OPENPHONE_GCP_ZONE. + --machine-type Converter VM type. Default: c3-standard-4. + --boot-disk-size Converter boot disk size. Default: 50GB. + --boot-disk-type Converter boot disk type. Default: pd-balanced. + --cache-disk-size Cache disk size. Default: 1000GB. + --cache-disk-type Cache disk type. Default: pd-ssd. + --keep-vm Leave converter VM running for debug. + -h, --help Show this help. +EOF +} + +source_disk="" +source_android_dir="auto" +cache_disk="${OPENPHONE_GCP_CACHE_DISK:-}" +snapshot="" +project="$OPENPHONE_GCP_PROJECT" +zone="$OPENPHONE_GCP_ZONE" +machine_type="${OPENPHONE_GCP_SEED_MACHINE_TYPE:-c3-standard-4}" +boot_disk_size="${OPENPHONE_GCP_SEED_BOOT_DISK_SIZE:-50GB}" +boot_disk_type="${OPENPHONE_GCP_SEED_BOOT_DISK_TYPE:-pd-balanced}" +cache_disk_size="$OPENPHONE_GCP_CACHE_DISK_SIZE" +cache_disk_type="$OPENPHONE_GCP_CACHE_DISK_TYPE" +keep_vm=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --source-disk) + [[ $# -ge 2 ]] || die "--source-disk requires a value" + source_disk="$2" + shift 2 + ;; + --source-android-dir) + [[ $# -ge 2 ]] || die "--source-android-dir requires a value" + source_android_dir="$2" + shift 2 + ;; + --cache-disk) + [[ $# -ge 2 ]] || die "--cache-disk requires a value" + cache_disk="$2" + shift 2 + ;; + --snapshot) + [[ $# -ge 2 ]] || die "--snapshot requires a value" + snapshot="$2" + shift 2 + ;; + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --zone) + [[ $# -ge 2 ]] || die "--zone requires a value" + zone="$2" + shift 2 + ;; + --machine-type) + [[ $# -ge 2 ]] || die "--machine-type requires a value" + machine_type="$2" + shift 2 + ;; + --boot-disk-size) + [[ $# -ge 2 ]] || die "--boot-disk-size requires a value" + boot_disk_size="$2" + shift 2 + ;; + --boot-disk-type) + [[ $# -ge 2 ]] || die "--boot-disk-type requires a value" + boot_disk_type="$2" + shift 2 + ;; + --cache-disk-size) + [[ $# -ge 2 ]] || die "--cache-disk-size requires a value" + cache_disk_size="$2" + shift 2 + ;; + --cache-disk-type) + [[ $# -ge 2 ]] || die "--cache-disk-type requires a value" + cache_disk_type="$2" + shift 2 + ;; + --keep-vm) + keep_vm=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$source_disk" ]] || die "--source-disk is required" +need_gcloud + +source_disk="$(sanitize_gcp_name "$source_disk")" +if [[ -z "$cache_disk" ]]; then + cache_disk="$(sanitize_gcp_name "openphone-cache-x86-64-${OPENPHONE_RELEASE}")" +else + cache_disk="$(sanitize_gcp_name "$cache_disk")" +fi +if [[ -n "$snapshot" ]]; then + snapshot="$(sanitize_gcp_name "$snapshot")" +fi + +gcloud compute disks describe "$source_disk" \ + --project "$project" \ + --zone "$zone" >/dev/null + +if ! gcloud compute disks describe "$cache_disk" \ + --project "$project" \ + --zone "$zone" >/dev/null 2>&1; then + info "Creating cache disk: $cache_disk" + gcloud compute disks create "$cache_disk" \ + --project "$project" \ + --zone "$zone" \ + --type "$cache_disk_type" \ + --size "$cache_disk_size" \ + --labels "app=openphone,purpose=lab-cache,managed-by=codex" +else + info "Reusing cache disk: $cache_disk" +fi + +name="$(sanitize_gcp_name "openphone-cache-seed-$(date -u +%H%M%S)")" +vm_created=false +tmp_script="$(mktemp "${TMPDIR:-/tmp}/openphone-cache-seed.XXXXXX")" +cleanup() { + local status=$? + set +e + rm -f "$tmp_script" + if [[ "$vm_created" == true && "$keep_vm" != true ]]; then + gcloud compute instances delete "$name" \ + --project "$project" \ + --zone "$zone" \ + --quiet >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT + +info "Creating cache seed VM: $name" +gcloud compute instances create "$name" \ + --project "$project" \ + --zone "$zone" \ + --machine-type "$machine_type" \ + --boot-disk-size "$boot_disk_size" \ + --boot-disk-type "$boot_disk_type" \ + --image-family "$OPENPHONE_GCP_IMAGE_FAMILY" \ + --image-project "$OPENPHONE_GCP_IMAGE_PROJECT" \ + --maintenance-policy TERMINATE \ + --restart-on-failure \ + --no-service-account \ + --no-scopes \ + --labels "app=openphone,purpose=lab-cache-seed,managed-by=codex" \ + --disk "name=$source_disk,device-name=openphone-source,mode=ro,boot=no,auto-delete=no" \ + --disk "name=$cache_disk,device-name=openphone-cache,mode=rw,boot=no,auto-delete=no" +vm_created=true + +cat > "$tmp_script" <<'REMOTE' +#!/usr/bin/env bash +set -euo pipefail + +source_android_dir="${OPENPHONE_SOURCE_ANDROID_DIR:?}" +source_root="/mnt/openphone-source" +cache_root="/mnt/openphone-cache" +source_device="/dev/disk/by-id/google-openphone-source" +cache_device="/dev/disk/by-id/google-openphone-cache" + +sudo apt-get update +sudo apt-get install -y rsync + +deadline=$((SECONDS + 300)) +while [[ ! -e "$source_device" || ! -e "$cache_device" ]]; do + if [[ "$SECONDS" -ge "$deadline" ]]; then + printf 'error: expected source/cache disks did not appear\n' >&2 + exit 1 + fi + sleep 2 +done + +source_mount_device="$source_device" +if compgen -G "${source_device}-part*" >/dev/null; then + source_mount_device="$(ls "${source_device}"-part* | sort | head -1)" +fi + +if ! sudo blkid "$cache_device" >/dev/null 2>&1; then + sudo mkfs.ext4 -F -L openphone-cache "$cache_device" +fi + +sudo mkdir -p "$source_root" "$cache_root" +if ! findmnt --mountpoint "$source_root" >/dev/null 2>&1; then + sudo mount -o ro "$source_mount_device" "$source_root" +fi +if ! findmnt --mountpoint "$cache_root" >/dev/null 2>&1; then + sudo mount -o defaults,discard "$cache_device" "$cache_root" +fi + +if [[ "$source_android_dir" == "auto" ]]; then + source_path="" + for candidate in "$source_root"/home/*/openphone-android "$source_root"/root/openphone-android; do + [[ -d "$candidate/.repo" ]] || continue + source_path="$candidate" + break + done + [[ -n "$source_path" ]] || { + printf 'error: could not auto-detect source Android tree under mounted source disk\n' >&2 + exit 1 + } +else + source_path="$source_root${source_android_dir}" +fi +[[ -d "$source_path/.repo" ]] || { + printf 'error: source Android tree not found: %s\n' "$source_path" >&2 + exit 1 +} + +sudo mkdir -p "$cache_root/android" +sudo rsync -aH --delete "$source_path/" "$cache_root/android/" +sudo sync +sudo umount "$cache_root" +sudo umount "$source_root" +REMOTE + +deadline=$((SECONDS + 600)) +until gcloud compute ssh "$name" \ + --project "$project" \ + --zone "$zone" \ + --command "true" >/dev/null 2>&1; do + if [[ "$SECONDS" -ge "$deadline" ]]; then + die "SSH was not ready for $name within 600s" + fi + sleep 10 +done + +info "Copying cache seed script to $name" +gcloud compute scp "$tmp_script" "$name:/tmp/openphone-cache-seed.sh" \ + --project "$project" \ + --zone "$zone" >/dev/null + +info "Copying Android tree from $source_disk to $cache_disk" +gcloud compute ssh "$name" \ + --project "$project" \ + --zone "$zone" \ + --command "OPENPHONE_SOURCE_ANDROID_DIR=$(shell_quote "$source_android_dir") bash /tmp/openphone-cache-seed.sh" + +if [[ "$keep_vm" == true ]]; then + info "--keep-vm left the converter VM running; skipping snapshot creation" + printf 'Cache disk: %s\n' "$cache_disk" + exit 0 +fi + +if [[ -n "$snapshot" ]]; then + info "Creating cache snapshot: $snapshot" + gcloud compute snapshots create "$snapshot" \ + --project "$project" \ + --source-disk "$cache_disk" \ + --source-disk-zone "$zone" \ + --storage-location "$OPENPHONE_GCP_REGION" \ + --labels "app=openphone,purpose=lab-cache,managed-by=codex" + printf 'Cache snapshot: %s\n' "$snapshot" +fi + +printf 'Cache disk: %s\n' "$cache_disk" diff --git a/scripts/lab/gcp/setup-wif.sh b/scripts/lab/gcp/setup-wif.sh new file mode 100755 index 0000000..cab36aa --- /dev/null +++ b/scripts/lab/gcp/setup-wif.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lab/gcp/common.sh +source "$script_dir/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/gcp/setup-wif.sh [options] + +Creates/verifies GitHub Actions Workload Identity Federation for OpenPhone lab +runs. This avoids long-lived JSON service account keys. + +Options: + --project GCP project. Default: OPENPHONE_GCP_PROJECT/openphone-lab. + --repo GitHub repo allowed to federate. Default: secondly-com/OpenPhone. + --pool WIF pool id. Default: github. + --provider WIF provider id. Default: secondly-openphone. + --service-account Service account id. Default: github-openphone-lab. + --set-github-vars Write required GitHub repo variables with gh. + -h, --help Show this help. +EOF +} + +project="$OPENPHONE_GCP_PROJECT" +repo="secondly-com/OpenPhone" +pool_id="github" +provider_id="secondly-openphone" +service_account_id="github-openphone-lab" +set_github_vars=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --project) + [[ $# -ge 2 ]] || die "--project requires a value" + project="$2" + shift 2 + ;; + --repo) + [[ $# -ge 2 ]] || die "--repo requires a value" + repo="$2" + shift 2 + ;; + --pool) + [[ $# -ge 2 ]] || die "--pool requires a value" + pool_id="$2" + shift 2 + ;; + --provider) + [[ $# -ge 2 ]] || die "--provider requires a value" + provider_id="$2" + shift 2 + ;; + --service-account) + [[ $# -ge 2 ]] || die "--service-account requires a value" + service_account_id="$2" + shift 2 + ;; + --set-github-vars) + set_github_vars=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +need_gcloud + +info "Using GCP project: $project" +gcloud services enable \ + compute.googleapis.com \ + iam.googleapis.com \ + iamcredentials.googleapis.com \ + cloudresourcemanager.googleapis.com \ + serviceusage.googleapis.com \ + artifactregistry.googleapis.com \ + logging.googleapis.com \ + monitoring.googleapis.com \ + --project "$project" + +project_number="$(gcloud projects describe "$project" --format='value(projectNumber)')" +sa_email="${service_account_id}@${project}.iam.gserviceaccount.com" + +if ! gcloud iam service-accounts describe "$sa_email" --project "$project" >/dev/null 2>&1; then + info "Creating service account: $sa_email" + gcloud iam service-accounts create "$service_account_id" \ + --project "$project" \ + --display-name "GitHub OpenPhone Lab" +else + info "Service account already exists: $sa_email" +fi + +if ! gcloud iam workload-identity-pools describe "$pool_id" \ + --project "$project" \ + --location global >/dev/null 2>&1; then + info "Creating WIF pool: $pool_id" + gcloud iam workload-identity-pools create "$pool_id" \ + --project "$project" \ + --location global \ + --display-name "GitHub Actions" +else + info "WIF pool already exists: $pool_id" +fi + +if ! gcloud iam workload-identity-pools providers describe "$provider_id" \ + --project "$project" \ + --location global \ + --workload-identity-pool "$pool_id" >/dev/null 2>&1; then + info "Creating WIF provider: $provider_id" + gcloud iam workload-identity-pools providers create-oidc "$provider_id" \ + --project "$project" \ + --location global \ + --workload-identity-pool "$pool_id" \ + --display-name "$repo" \ + --issuer-uri "https://token.actions.githubusercontent.com" \ + --attribute-mapping "google.subject=assertion.sub,attribute.actor=assertion.actor,attribute.repository=assertion.repository,attribute.repository_owner=assertion.repository_owner,attribute.ref=assertion.ref,attribute.workflow=assertion.workflow" \ + --attribute-condition "assertion.repository == '${repo}'" +else + info "WIF provider already exists: $provider_id" +fi + +principal="principalSet://iam.googleapis.com/projects/${project_number}/locations/global/workloadIdentityPools/${pool_id}/attribute.repository/${repo}" +provider_resource="projects/${project_number}/locations/global/workloadIdentityPools/${pool_id}/providers/${provider_id}" + +info "Binding GitHub repo principal to service account" +gcloud iam service-accounts add-iam-policy-binding "$sa_email" \ + --project "$project" \ + --role roles/iam.workloadIdentityUser \ + --member "$principal" >/dev/null + +roles=( + roles/compute.instanceAdmin.v1 + roles/iam.serviceAccountUser + roles/serviceusage.serviceUsageConsumer +) + +for role in "${roles[@]}"; do + info "Ensuring project role $role" + gcloud projects add-iam-policy-binding "$project" \ + --member "serviceAccount:$sa_email" \ + --role "$role" >/dev/null +done + +if [[ "$set_github_vars" == true ]]; then + need_cmd gh + info "Writing GitHub repo variables for $repo" + gh variable set OPENPHONE_GCP_PROJECT --repo "$repo" --body "$project" + gh variable set OPENPHONE_GCP_REGION --repo "$repo" --body "$OPENPHONE_GCP_REGION" + gh variable set OPENPHONE_GCP_ZONE --repo "$repo" --body "$OPENPHONE_GCP_ZONE" + gh variable set OPENPHONE_GCP_WORKLOAD_IDENTITY_PROVIDER --repo "$repo" --body "$provider_resource" + gh variable set OPENPHONE_GCP_SERVICE_ACCOUNT --repo "$repo" --body "$sa_email" + gh variable set OPENPHONE_GCP_MACHINE_TYPE --repo "$repo" --body "$OPENPHONE_GCP_MACHINE_TYPE" + gh variable set OPENPHONE_GCP_BOOT_DISK_SIZE --repo "$repo" --body "$OPENPHONE_GCP_BOOT_DISK_SIZE" + gh variable set OPENPHONE_GCP_BOOT_DISK_TYPE --repo "$repo" --body "$OPENPHONE_GCP_BOOT_DISK_TYPE" +fi + +cat < SDK install root. Default: ANDROID_SDK_ROOT, + ANDROID_HOME, or /opt/android-sdk. + -h, --help Show this help. +EOF +} + +sdk_root="${ANDROID_SDK_ROOT:-${ANDROID_HOME:-/opt/android-sdk}}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --sdk-root) + [[ $# -ge 2 ]] || { + printf 'error: --sdk-root requires a value\n' >&2 + exit 1 + } + sdk_root="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + printf 'error: unknown argument: %s\n' "$1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ "$(id -u)" -ne 0 ]]; then + exec sudo -E bash "$0" --sdk-root "$sdk_root" +fi + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { + printf 'error: missing required command: %s\n' "$1" >&2 + exit 1 + } +} + +host_os() { + case "$(uname -s)" in + Linux) printf 'linux' ;; + Darwin) printf 'macosx' ;; + *) printf 'error: unsupported host OS: %s\n' "$(uname -s)" >&2; exit 1 ;; + esac +} + +need_cmd curl +need_cmd python3 +need_cmd unzip + +sdk_root="${sdk_root%/}" +install -d -m 0755 "$sdk_root/cmdline-tools" + +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/openphone-android-sdk.XXXXXX")" +cleanup() { + rm -rf "$tmp_dir" +} +trap cleanup EXIT + +cmdline_tools_url="$( + python3 - "$(host_os)" <<'PY' +import sys +import urllib.request +import xml.etree.ElementTree as ET + +host_os = sys.argv[1] +base_url = "https://dl.google.com/android/repository/" +with urllib.request.urlopen(base_url + "repository2-1.xml", timeout=60) as response: + root = ET.fromstring(response.read()) + +for package in root.findall("remotePackage"): + if package.attrib.get("path") != "cmdline-tools;latest": + continue + for archive in package.findall("./archives/archive"): + if archive.findtext("host-os") != host_os: + continue + url = archive.findtext("./complete/url") + if not url: + continue + print(base_url + url) + raise SystemExit(0) + +raise SystemExit(f"no cmdline-tools;latest archive found for {host_os}") +PY +)" + +curl -fL --retry 3 --retry-delay 2 \ + "$cmdline_tools_url" \ + -o "$tmp_dir/cmdline-tools.zip" +unzip -q "$tmp_dir/cmdline-tools.zip" -d "$tmp_dir" + +rm -rf "$sdk_root/cmdline-tools/latest" +mv "$tmp_dir/cmdline-tools" "$sdk_root/cmdline-tools/latest" + +sdkmanager="$sdk_root/cmdline-tools/latest/bin/sdkmanager" +{ yes || true; } | "$sdkmanager" --sdk_root="$sdk_root" --licenses >/dev/null +{ yes || true; } | "$sdkmanager" --sdk_root="$sdk_root" --install \ + "platform-tools" \ + "emulator" +{ yes || true; } | "$sdkmanager" --sdk_root="$sdk_root" --licenses >/dev/null + +ln -sf "$sdk_root/platform-tools/adb" /usr/local/bin/adb +ln -sf "$sdk_root/emulator/emulator" /usr/local/bin/emulator + +cat > /etc/profile.d/openphone-android-sdk.sh < Lab slot whose env should remember the Android dir. + --android-dir Android checkout path. Default: OPENPHONE_ANDROID_DIR. + --arch arm64|x86_64 Emulator image architecture for optional build. + Default: host arch. + --variant eng|userdebug Emulator build variant for optional build. Default: eng. + --repo-sync-jobs repo sync jobs. Default: host CPU count. + --skip-sync Do not run scripts/sync.sh. + --skip-patches Do not run scripts/apply-patches.sh. + --reset-patch-targets Reset patched Android repos to manifest revisions + before applying OpenPhone patches. + --from-scratch Force repo sync/checkout when syncing. + --no-clone-bundle Pass repo sync --no-clone-bundle. Useful for + resumable local cold syncs on flaky networks. + --build-emulator Build the emulator image after sync/patch. + --no-macos-volume Do not auto-create/mount the macOS sparsebundle. + -h, --help Show this help. + +Environment: + OPENPHONE_ANDROID_DIR Android checkout path. + OPENPHONE_MACOS_IMAGE macOS sparsebundle path. + OPENPHONE_MACOS_VOLUME_NAME macOS mounted volume name. + OPENPHONE_MACOS_IMAGE_SIZE macOS sparsebundle max size. Default here: 700g. +EOF +} + +detect_emulator_arch() { + case "$(uname -m)" in + arm64|aarch64) printf 'arm64' ;; + x86_64|amd64) printf 'x86_64' ;; + *) die "unsupported host architecture: $(uname -m). Pass --arch arm64 or --arch x86_64." ;; + esac +} + +host_cpu_count() { + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v sysctl >/dev/null 2>&1; then + sysctl -n hw.ncpu + else + printf '4' + fi +} + +slot="" +android_dir="$requested_android_dir" +arch="" +variant="eng" +repo_sync_jobs="" +run_sync=true +run_patches=true +reset_patch_targets=false +from_scratch=false +no_clone_bundle=false +build_emulator=false +auto_macos_volume=true + +while [[ $# -gt 0 ]]; do + case "$1" in + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --android-dir) + [[ $# -ge 2 ]] || die "--android-dir requires a value" + android_dir="$2" + shift 2 + ;; + --arch) + [[ $# -ge 2 ]] || die "--arch requires a value" + arch="$2" + shift 2 + ;; + --variant) + [[ $# -ge 2 ]] || die "--variant requires a value" + variant="$2" + shift 2 + ;; + --repo-sync-jobs) + [[ $# -ge 2 ]] || die "--repo-sync-jobs requires a value" + repo_sync_jobs="$2" + shift 2 + ;; + --skip-sync) + run_sync=false + shift + ;; + --skip-patches) + run_patches=false + shift + ;; + --reset-patch-targets) + reset_patch_targets=true + shift + ;; + --from-scratch) + from_scratch=true + shift + ;; + --no-clone-bundle) + no_clone_bundle=true + shift + ;; + --build-emulator) + build_emulator=true + shift + ;; + --no-macos-volume) + auto_macos_volume=false + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +arch="${arch:-$(detect_emulator_arch)}" +repo_sync_jobs="${repo_sync_jobs:-$(host_cpu_count)}" +android_dir="${android_dir:-$OPENPHONE_ANDROID_DIR}" + +case "$arch" in + arm64|x86_64) ;; + *) die "unsupported emulator arch: $arch" ;; +esac + +case "$variant" in + eng|userdebug) ;; + *) die "unsupported emulator variant: $variant" ;; +esac + +[[ "$repo_sync_jobs" =~ ^[0-9]+$ ]] || die "--repo-sync-jobs must be numeric" + +if [[ "$(uname -s)" == "Darwin" && "$auto_macos_volume" == true ]]; then + mkdir -p "$android_dir" + if ! is_case_sensitive_dir "$android_dir"; then + info "Local Android dir is not case-sensitive; preparing macOS build volume" + export OPENPHONE_MACOS_IMAGE_SIZE="${OPENPHONE_MACOS_IMAGE_SIZE:-700g}" + "$root/scripts/create-macos-build-volume.sh" + android_dir="$root/.worktree/${OPENPHONE_MACOS_VOLUME_NAME:-OpenPhoneAndroid}/android" + fi +fi + +export OPENPHONE_ANDROID_DIR="$android_dir" + +if [[ -n "$slot" ]]; then + "$root/scripts/lab/allocate-slot.sh" --slot "$slot" >/dev/null +fi + +need_cmd git +if ! command -v repo >/dev/null 2>&1; then + info "Installing repo tool into ~/.local/bin" + "$root/scripts/install-repo.sh" +fi + +if [[ "$run_sync" == true ]]; then + sync_args=(-j"$repo_sync_jobs") + if [[ "$from_scratch" == true ]]; then + sync_args+=(--force-sync --force-checkout) + fi + if [[ "$no_clone_bundle" == true ]]; then + sync_args+=(--no-clone-bundle) + fi + "$root/scripts/sync.sh" "${sync_args[@]}" +else + [[ -d "$OPENPHONE_ANDROID_DIR/.repo" ]] \ + || die "Android tree not initialized and --skip-sync was used: $OPENPHONE_ANDROID_DIR" +fi + +if [[ "$run_patches" == true ]]; then + if [[ "$reset_patch_targets" == true ]]; then + OPENPHONE_RESET_PATCH_TARGETS=1 "$root/scripts/apply-patches.sh" + else + "$root/scripts/apply-patches.sh" + fi +fi + +if [[ "$build_emulator" == true ]]; then + "$root/scripts/build-emulator.sh" --arch "$arch" --variant "$variant" +fi + +cat < Lab slot name. Default: checkout hash. + --arch arm64|x86_64 Emulator image architecture. Default: host arch. + --variant eng|userdebug Emulator build variant. Default: eng. + --runtime Runtime intent: local, openclaw, or hermes. + May be repeated. Default: local. + --openclaw Alias for --runtime openclaw. + --skip-build Reuse an already-built emulator image. + --keep-running Leave the emulator running on exit. + --timeout Boot timeout. Default: run-emulator-smoke default. + --ci CI mode alias; currently only makes intent explicit. + -h, --help Show this help. + +Runtime launch hooks: + OPENPHONE_OPENCLAW_UP_CMD Optional command to start OpenClaw for this slot. + OPENPHONE_HERMES_UP_CMD Optional command to start Hermes for this slot. + OPENPHONE_HERMES_HEALTH_CMD Optional command that must pass for Hermes smoke. +EOF +} + +slot="" +arch="" +variant="" +skip_build=false +keep_running=false +timeout_seconds="" +runtimes=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --arch) + [[ $# -ge 2 ]] || die "--arch requires a value" + arch="$2" + shift 2 + ;; + --variant) + [[ $# -ge 2 ]] || die "--variant requires a value" + variant="$2" + shift 2 + ;; + --skip-build) + skip_build=true + shift + ;; + --keep-running) + keep_running=true + shift + ;; + --timeout) + [[ $# -ge 2 ]] || die "--timeout requires a value" + timeout_seconds="$2" + shift 2 + ;; + --runtime) + [[ $# -ge 2 ]] || die "--runtime requires a value" + runtimes+=("$2") + shift 2 + ;; + --openclaw) + runtimes+=(openclaw) + shift + ;; + --ci) + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +if [[ ${#runtimes[@]} -eq 0 ]]; then + runtimes=(local) +fi + +needs_runtime() { + local needle="$1" + local runtime + for runtime in "${runtimes[@]}"; do + [[ "$runtime" == "$needle" ]] && return 0 + done + return 1 +} + +for runtime in "${runtimes[@]}"; do + case "$runtime" in + local|openclaw|hermes) ;; + *) die "unsupported runtime: $runtime" ;; + esac +done + +if [[ -z "$slot" ]]; then + need_cmd python3 + slot="$( + python3 - <<'PY' "$root" +import hashlib +import sys + +root = sys.argv[1] +print("checkout-" + hashlib.sha1(root.encode("utf-8")).hexdigest()[:8]) +PY + )" +fi + +allocate_args=() +allocate_args+=(--slot "$slot") +"$root/scripts/lab/allocate-slot.sh" "${allocate_args[@]}" >/dev/null + +env_file="$root/.worktree/lab/$slot/env" +[[ -f "$env_file" ]] || die "missing lab env file" +# shellcheck disable=SC1090 +source "$env_file" + +runtime_pids=() +cleanup_runtimes() { + set +e + if [[ "$keep_running" == true ]]; then + return 0 + fi + local pid + for pid in "${runtime_pids[@]:-}"; do + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + done +} +trap cleanup_runtimes EXIT + +start_runtime_cmd() { + local name="$1" + local cmd="$2" + local log="$OPENPHONE_LAB_DIR/artifacts/${name}.log" + local runtime_pid + + mkdir -p "$OPENPHONE_LAB_DIR/artifacts" "$OPENPHONE_LAB_DIR/run" + info "Starting $name runtime command" + ( + cd "$root" + OPENPHONE_LAB_SLOT="$OPENPHONE_LAB_SLOT" \ + OPENPHONE_LAB_DIR="$OPENPHONE_LAB_DIR" \ + ANDROID_SERIAL="$ANDROID_SERIAL" \ + OPENPHONE_OPENCLAW_URL="$OPENPHONE_OPENCLAW_URL" \ + OPENPHONE_HERMES_URL="$OPENPHONE_HERMES_URL" \ + OPENPHONE_MODEL_BROKER_URL="$OPENPHONE_MODEL_BROKER_URL" \ + bash -lc "$cmd" + ) > "$log" 2>&1 & + runtime_pid="$!" + runtime_pids+=("$runtime_pid") + printf '%s\n' "$runtime_pid" > "$OPENPHONE_LAB_DIR/run/$name.pid" + sleep "${OPENPHONE_RUNTIME_BOOT_SLEEP:-5}" +} + +if needs_runtime openclaw && [[ -n "${OPENPHONE_OPENCLAW_UP_CMD:-}" ]]; then + start_runtime_cmd openclaw "$OPENPHONE_OPENCLAW_UP_CMD" +fi + +if needs_runtime hermes && [[ -n "${OPENPHONE_HERMES_UP_CMD:-}" ]]; then + start_runtime_cmd hermes "$OPENPHONE_HERMES_UP_CMD" +fi + +args=(--slot "$OPENPHONE_LAB_SLOT" --port "$OPENPHONE_EMULATOR_PORT" --serial "$ANDROID_SERIAL") +if [[ -n "$arch" ]]; then + args+=(--arch "$arch") +fi +if [[ -n "$variant" ]]; then + args+=(--variant "$variant") +fi +if [[ "$skip_build" == true ]]; then + args+=(--skip-build) +fi +if [[ "$keep_running" == true ]] || needs_runtime openclaw || needs_runtime hermes; then + args+=(--keep-running) +fi +if [[ -n "$timeout_seconds" ]]; then + args+=(--timeout "$timeout_seconds") +fi + +"$root/scripts/run-emulator-smoke.sh" "${args[@]}" + +if needs_runtime openclaw; then + [[ -n "${OPENPHONE_OPENCLAW_TOKEN:-${OPENCLAW_GATEWAY_TOKEN:-}}" ]] \ + || die "set OPENPHONE_OPENCLAW_TOKEN or OPENCLAW_GATEWAY_TOKEN before --openclaw" + OPENPHONE_OPENCLAW_URL="$OPENPHONE_OPENCLAW_URL" \ + ANDROID_SERIAL="$ANDROID_SERIAL" \ + "$root/scripts/smoke-test-openclaw-runtime.sh" +fi + +if needs_runtime hermes; then + if [[ -n "${OPENPHONE_HERMES_HEALTH_CMD:-}" ]]; then + info "Running Hermes health command" + OPENPHONE_HERMES_URL="$OPENPHONE_HERMES_URL" \ + ANDROID_SERIAL="$ANDROID_SERIAL" \ + bash -lc "$OPENPHONE_HERMES_HEALTH_CMD" + elif command -v curl >/dev/null 2>&1; then + info "Checking Hermes health endpoint" + curl --fail --silent --show-error "${OPENPHONE_HERMES_HEALTH_URL:-$OPENPHONE_HERMES_URL/health}" \ + > "$OPENPHONE_LAB_DIR/artifacts/hermes-health.json" + else + die "Hermes runtime requested, but no OPENPHONE_HERMES_HEALTH_CMD and curl is unavailable" + fi +fi diff --git a/scripts/lab/up.sh b/scripts/lab/up.sh new file mode 100755 index 0000000..ce9b488 --- /dev/null +++ b/scripts/lab/up.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=scripts/common.sh +source "$root/scripts/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/lab/up.sh [options] + +Allocates a local OpenPhone lab slot, boots the emulator, runs the smoke checks, +and leaves the emulator running for Codex/human iteration. + +Options: + --slot Lab slot name. Default: checkout hash. + --arch arm64|x86_64 Emulator image architecture. Default: host arch. + --variant eng|userdebug Emulator build variant. Default: eng. + --runtime Runtime intent: local, openclaw, or hermes. + May be repeated. Default: local. + --android-dir Android checkout path for this slot. + --prepare Run local Android tree sync/patch prep before smoke. + --no-prepare Do not auto-prepare a missing Android tree. + --repo-sync-jobs repo sync jobs when preparation runs. + --from-scratch Force repo sync/checkout when preparation runs. + --reset-patch-targets Reset patched Android repos before applying patches. + --no-clone-bundle Pass repo sync --no-clone-bundle during preparation. + --skip-build Reuse an already-built emulator image. + --timeout Boot timeout. Default: run-emulator-smoke default. + -h, --help Show this help. +EOF +} + +slot="" +arch="" +variant="" +android_dir="${OPENPHONE_ANDROID_DIR:-}" +prepare_mode="auto" +repo_sync_jobs="" +from_scratch=false +reset_patch_targets=false +no_clone_bundle=false +skip_build=false +timeout_seconds="" +runtimes=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --arch) + [[ $# -ge 2 ]] || die "--arch requires a value" + arch="$2" + shift 2 + ;; + --variant) + [[ $# -ge 2 ]] || die "--variant requires a value" + variant="$2" + shift 2 + ;; + --android-dir) + [[ $# -ge 2 ]] || die "--android-dir requires a value" + android_dir="$2" + shift 2 + ;; + --prepare) + prepare_mode="always" + shift + ;; + --no-prepare) + prepare_mode="never" + shift + ;; + --repo-sync-jobs) + [[ $# -ge 2 ]] || die "--repo-sync-jobs requires a value" + repo_sync_jobs="$2" + shift 2 + ;; + --from-scratch) + from_scratch=true + shift + ;; + --reset-patch-targets) + reset_patch_targets=true + shift + ;; + --no-clone-bundle) + no_clone_bundle=true + shift + ;; + --runtime) + [[ $# -ge 2 ]] || die "--runtime requires a value" + runtimes+=("$2") + shift 2 + ;; + --skip-build) + skip_build=true + shift + ;; + --timeout) + [[ $# -ge 2 ]] || die "--timeout requires a value" + timeout_seconds="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +if [[ -z "$slot" ]]; then + need_cmd python3 + slot="$( + python3 - <<'PY' "$root" +import hashlib +import sys + +root = sys.argv[1] +print("checkout-" + hashlib.sha1(root.encode("utf-8")).hexdigest()[:8]) +PY + )" +fi + +if [[ ${#runtimes[@]} -eq 0 ]]; then + runtimes=(local) +fi + +if [[ -n "$android_dir" ]]; then + export OPENPHONE_ANDROID_DIR="$android_dir" +fi + +"$root/scripts/lab/allocate-slot.sh" --slot "$slot" >/dev/null + +env_file="$root/.worktree/lab/$slot/env" +[[ -f "$env_file" ]] || die "missing lab env file: $env_file" +# shellcheck disable=SC1090 +source "$env_file" + +should_prepare=false +case "$prepare_mode" in + always) + should_prepare=true + ;; + auto) + if [[ "$skip_build" != true && ! -f "$OPENPHONE_ANDROID_DIR/build/envsetup.sh" ]]; then + should_prepare=true + fi + ;; + never) + ;; + *) + die "invalid prepare mode: $prepare_mode" + ;; +esac + +if [[ "$should_prepare" == true ]]; then + prepare_args=(--slot "$OPENPHONE_LAB_SLOT" --android-dir "$OPENPHONE_ANDROID_DIR") + if [[ -n "$arch" ]]; then + prepare_args+=(--arch "$arch") + fi + if [[ -n "$variant" ]]; then + prepare_args+=(--variant "$variant") + fi + if [[ -n "$repo_sync_jobs" ]]; then + prepare_args+=(--repo-sync-jobs "$repo_sync_jobs") + fi + if [[ "$from_scratch" == true ]]; then + prepare_args+=(--from-scratch) + fi + if [[ "$reset_patch_targets" == true ]]; then + prepare_args+=(--reset-patch-targets) + fi + if [[ "$no_clone_bundle" == true ]]; then + prepare_args+=(--no-clone-bundle) + fi + "$root/scripts/lab/prepare-local.sh" "${prepare_args[@]}" + # Refresh the env file because prepare-local may choose/create the macOS + # case-sensitive build volume and persist its Android dir into the slot. + # shellcheck disable=SC1090 + source "$env_file" +fi + +args=(--slot "$OPENPHONE_LAB_SLOT" --keep-running) +if [[ -n "$arch" ]]; then + args+=(--arch "$arch") +fi +if [[ -n "$variant" ]]; then + args+=(--variant "$variant") +fi +if [[ "$skip_build" == true ]]; then + args+=(--skip-build) +fi +if [[ -n "$timeout_seconds" ]]; then + args+=(--timeout "$timeout_seconds") +fi +for runtime in "${runtimes[@]}"; do + args+=(--runtime "$runtime") +done + +"$root/scripts/lab/smoke.sh" "${args[@]}" + +printf '\nLab is up. To use it in another shell:\n' +printf ' source %q\n' "$env_file" +printf ' node integrations/cli/src/index.mjs --serial "$ANDROID_SERIAL" --json runtime status\n' diff --git a/scripts/run-emulator-smoke.sh b/scripts/run-emulator-smoke.sh new file mode 100755 index 0000000..dfcb920 --- /dev/null +++ b/scripts/run-emulator-smoke.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/common.sh +source "$root/scripts/common.sh" + +usage() { + cat <<'EOF' +Usage: scripts/run-emulator-smoke.sh [options] + +Builds or reuses an OpenPhone SDK phone emulator image, boots it headlessly, +and runs the fast OpenPhone runtime smoke checks used by CI. + +Options: + --arch arm64|x86_64 Emulator image architecture. Default: host arch. + --variant eng|userdebug Emulator build variant. Default: eng. + --slot Lab slot name for isolated data/artifacts. + --port Emulator console port. Default: 5584. + --serial ADB serial. Default: emulator-. + --timeout Boot timeout. Default: 600. + --skip-build Reuse an already-built image. + --keep-running Do not stop the emulator on exit. + -h, --help Show this help. + +Environment: + OPENPHONE_ANDROID_DIR Android checkout path. + OPENPHONE_LAB_SLOT Lab slot name. + OPENPHONE_LAB_DIR Lab slot directory. + OPENPHONE_EMULATOR_PORT Emulator console port. + OPENPHONE_EMULATOR_SERIAL Emulator ADB serial. + OPENPHONE_EMULATOR_BUILD Set to 0 to skip build. + OPENPHONE_EMULATOR_ASSISTANT_SMOKE Set to 0 to skip local assistant task. + OPENPHONE_EMULATOR_ARGS Extra emulator arguments. + +Artifacts are written under .worktree/emulator-smoke//. +EOF +} + +detect_emulator_arch() { + case "$(uname -m)" in + arm64|aarch64) printf 'arm64' ;; + x86_64|amd64) printf 'x86_64' ;; + *) die "unsupported host architecture: $(uname -m). Pass --arch arm64 or --arch x86_64." ;; + esac +} + +run_adb_shell_with_timeout() { + local seconds="$1" + local log_file="$2" + shift 2 + + adb -s "$serial" shell "$@" > "$log_file" 2>&1 & + local command_pid="$!" + local command_deadline=$((SECONDS + seconds)) + + while kill -0 "$command_pid" >/dev/null 2>&1; do + if [[ "$SECONDS" -ge "$command_deadline" ]]; then + { + echo + echo "Timed out after ${seconds}s: adb -s $serial shell $*" + } >> "$log_file" + kill "$command_pid" >/dev/null 2>&1 || true + sleep 2 + kill -9 "$command_pid" >/dev/null 2>&1 || true + wait "$command_pid" >/dev/null 2>&1 || true + return 124 + fi + sleep 1 + done + + wait "$command_pid" +} + +arch="" +variant="eng" +slot="${OPENPHONE_LAB_SLOT:-}" +port="${OPENPHONE_EMULATOR_PORT:-5584}" +serial="" +timeout_seconds="600" +build="${OPENPHONE_EMULATOR_BUILD:-1}" +keep_running=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) + [[ $# -ge 2 ]] || die "--arch requires a value" + arch="$2" + shift 2 + ;; + --variant) + [[ $# -ge 2 ]] || die "--variant requires a value" + variant="$2" + shift 2 + ;; + --slot) + [[ $# -ge 2 ]] || die "--slot requires a value" + slot="$2" + shift 2 + ;; + --port) + [[ $# -ge 2 ]] || die "--port requires a value" + port="$2" + shift 2 + ;; + --serial) + [[ $# -ge 2 ]] || die "--serial requires a value" + serial="$2" + shift 2 + ;; + --timeout) + [[ $# -ge 2 ]] || die "--timeout requires a value" + timeout_seconds="$2" + shift 2 + ;; + --skip-build) + build=0 + shift + ;; + --keep-running) + keep_running=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +arch="${arch:-$(detect_emulator_arch)}" +serial="${serial:-${OPENPHONE_EMULATOR_SERIAL:-${ANDROID_SERIAL:-emulator-$port}}}" + +case "$arch" in + arm64|x86_64) ;; + *) die "unsupported emulator arch: $arch" ;; +esac + +case "$variant" in + eng|userdebug) ;; + *) die "unsupported emulator variant: $variant" ;; +esac + +[[ "$port" =~ ^[0-9]+$ ]] || die "--port must be numeric" +[[ "$timeout_seconds" =~ ^[0-9]+$ ]] || die "--timeout must be numeric" + +need_cmd adb +need_cmd node +need_cmd python3 +need_cmd emulator + +stamp="$(date -u +%Y%m%d-%H%M%S)" +if [[ -n "$slot" ]]; then + lab_dir="${OPENPHONE_LAB_DIR:-$root/.worktree/lab/$slot}" + emulator_data_dir="$lab_dir/emulator-data" + out_dir="$lab_dir/artifacts/emulator-smoke/$stamp" +else + lab_dir="" + emulator_data_dir="$root/.worktree/emulator-smoke/$stamp/emulator-data" + out_dir="$root/.worktree/emulator-smoke/$stamp" +fi +mkdir -p "$emulator_data_dir" +mkdir -p "$out_dir" + +export ANDROID_SERIAL="$serial" + +emulator_pid="" +cleanup() { + set +e + adb -s "$serial" logcat -d > "$out_dir/logcat.txt" 2>/dev/null + adb -s "$serial" exec-out screencap -p > "$out_dir/screenshot.png" 2>/dev/null + adb -s "$serial" shell uiautomator dump /sdcard/openphone-emulator-smoke.xml >/dev/null 2>&1 + adb -s "$serial" exec-out cat /sdcard/openphone-emulator-smoke.xml \ + > "$out_dir/window.xml" 2>/dev/null + if [[ "$keep_running" != true ]]; then + adb -s "$serial" emu kill >/dev/null 2>&1 + fi + if [[ -n "$emulator_pid" ]]; then + wait "$emulator_pid" >/dev/null 2>&1 + fi +} +trap cleanup EXIT + +if [[ "$build" != "0" ]]; then + info "Building OpenPhone emulator image for $arch/$variant" + "$root/scripts/build-emulator.sh" --arch "$arch" --variant "$variant" +fi + +info "Stopping any existing emulator on $serial" +adb -s "$serial" emu kill >/dev/null 2>&1 || true +sleep 2 + +extra_emulator_args=() +if [[ -n "${OPENPHONE_EMULATOR_ARGS:-}" ]]; then + read -r -a extra_emulator_args <<< "$OPENPHONE_EMULATOR_ARGS" +fi +emulator_args=( + -port "$port" + -datadir "$emulator_data_dir" + -no-window + -gpu swiftshader_indirect + -no-snapshot + -wipe-data + -no-boot-anim + -no-audio +) +emulator_args+=("${extra_emulator_args[@]}") + +info "Launching emulator $serial" +"$root/scripts/run-emulator.sh" --arch "$arch" --variant "$variant" -- \ + "${emulator_args[@]}" > "$out_dir/emulator.log" 2>&1 & +emulator_pid="$!" + +info "Waiting for ADB device" +deadline=$((SECONDS + timeout_seconds)) +adb_state="" +while [[ "$SECONDS" -lt "$deadline" ]]; do + if ! kill -0 "$emulator_pid" >/dev/null 2>&1; then + die "emulator process exited before ADB became ready; see $out_dir/emulator.log" + fi + adb_state="$(adb -s "$serial" get-state 2>/dev/null || true)" + if [[ "$adb_state" == "device" ]]; then + break + fi + sleep 2 +done +[[ "$adb_state" == "device" ]] || die "ADB device $serial was not ready within ${timeout_seconds}s" + +boot_completed="" +while [[ "$SECONDS" -lt "$deadline" ]]; do + boot_completed="$(adb -s "$serial" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r' || true)" + if [[ "$boot_completed" == "1" ]]; then + break + fi + sleep 5 +done +[[ "$boot_completed" == "1" ]] || die "emulator did not finish booting within ${timeout_seconds}s" + +info "Emulator booted" +adb -s "$serial" shell input keyevent KEYCODE_WAKEUP >/dev/null 2>&1 || true +adb -s "$serial" shell wm dismiss-keyguard >/dev/null 2>&1 || true + +info "Preparing emulator for headless smoke" +{ + echo "Marking setup complete" + adb -s "$serial" shell settings put global device_provisioned 1 || true + adb -s "$serial" shell settings put secure user_setup_complete 1 || true + adb -s "$serial" shell settings put secure tv_user_setup_complete 1 || true + + for package in org.lineageos.setupwizard com.google.android.setupwizard com.android.provision; do + if adb -s "$serial" shell pm path "$package" >/dev/null 2>&1; then + echo "Disabling setup/provisioning package: $package" + adb -s "$serial" shell am force-stop "$package" || true + adb -s "$serial" shell pm disable-user --user 0 "$package" || true + fi + done + + adb -s "$serial" shell input keyevent KEYCODE_HOME || true +} > "$out_dir/headless-provisioning.txt" 2>&1 + +identity="$out_dir/device-identity.txt" +adb -s "$serial" shell 'printf "model="; getprop ro.product.model; printf "device="; getprop ro.product.device; printf "openphone="; getprop ro.openphone.version; printf "lineage="; getprop ro.lineage.version; printf "boot_completed="; getprop sys.boot_completed' \ + > "$identity" +cat "$identity" + +grep -q '^boot_completed=1' "$identity" || die "boot_completed was not recorded as 1" +grep -q '^openphone=.' "$identity" || die "ro.openphone.version was empty" + +info "Checking OpenPhone framework services" +adb -s "$serial" shell 'service check openphone_agent' | tee "$out_dir/service-openphone-agent.txt" +adb -s "$serial" shell 'service check openphone_context' | tee "$out_dir/service-openphone-context.txt" +adb -s "$serial" shell 'service check openphone_assistant_data' | tee "$out_dir/service-openphone-assistant-data.txt" +grep -q 'found' "$out_dir/service-openphone-agent.txt" || die "openphone_agent service not found" +grep -q 'found' "$out_dir/service-openphone-context.txt" || die "openphone_context service not found" +grep -q 'found' "$out_dir/service-openphone-assistant-data.txt" || die "openphone_assistant_data service not found" + +info "Checking assistant package" +adb -s "$serial" shell 'pm path org.openphone.assistant' | tee "$out_dir/assistant-path.txt" +adb -s "$serial" shell 'cmd package list packages --show-versioncode org.openphone.assistant' \ + | tee "$out_dir/assistant-package.txt" +grep -q 'org.openphone.assistant' "$out_dir/assistant-package.txt" \ + || die "OpenPhone Assistant package is not installed" + +info "Starting assistant activity" +if ! run_adb_shell_with_timeout 60 "$out_dir/start-assistant.txt" \ + am start -W -n org.openphone.assistant/.MainActivity; then + adb -s "$serial" shell dumpsys activity activities \ + > "$out_dir/activity-after-start-failure.txt" 2>/dev/null || true + die "assistant activity did not start within 60s; see $out_dir/start-assistant.txt" +fi +grep -q 'Status: ok' "$out_dir/start-assistant.txt" \ + || die "assistant activity start did not report Status: ok" +adb -s "$serial" shell dumpsys activity activities > "$out_dir/activity-after-start.txt" +grep -q 'ResumedActivity: .*org.openphone.assistant/.MainActivity' \ + "$out_dir/activity-after-start.txt" \ + || die "assistant activity was not the resumed activity" + +info "Running runtime CLI status smoke" +node "$root/integrations/cli/src/index.mjs" --serial "$serial" --json runtime status \ + > "$out_dir/runtime-status.json" +python3 - <<'PY' "$out_dir/runtime-status.json" +import json +import sys + +data = json.load(open(sys.argv[1], encoding="utf-8")) +for key in ("chat_runtime", "volume_runtime", "background_runtime"): + if key not in data: + raise SystemExit(f"missing runtime status key: {key}") +PY + +info "Running runtime screen smoke" +node "$root/integrations/cli/src/index.mjs" --serial "$serial" --json \ + tool invoke openphone.screen.get '{"include_screenshot":false}' \ + > "$out_dir/screen-get.json" +python3 - <<'PY' "$out_dir/screen-get.json" +import json +import sys + +data = json.load(open(sys.argv[1], encoding="utf-8")) +if data.get("ok") is not True: + raise SystemExit(f"screen smoke failed: {data}") +if "ui_tree_xml" not in data: + raise SystemExit("screen smoke did not return ui_tree_xml") +PY + +if [[ "${OPENPHONE_EMULATOR_ASSISTANT_SMOKE:-1}" != "0" ]]; then + info "Running local assistant task smoke" + "$root/scripts/run-assistant-task.sh" \ + --goal "Just respond with the word pong." \ + --local \ + --wait 20 > "$out_dir/assistant-task.txt" +fi + +info "Emulator smoke passed; artifacts in $out_dir" diff --git a/scripts/sync.sh b/scripts/sync.sh index 96b396f..73cc2ca 100755 --- a/scripts/sync.sh +++ b/scripts/sync.sh @@ -34,6 +34,12 @@ mkdir -p .repo/local_manifests cp "$OPENPHONE_ROOT/manifests/openphone.xml" .repo/local_manifests/openphone.xml info "syncing upstream sources" -repo sync --current-branch --no-tags --optimized-fetch --prune "$@" +sync_args=(--current-branch --no-tags --optimized-fetch --prune) +case "${OPENPHONE_REPO_SYNC_NO_CLONE_BUNDLE:-0}" in + 1|true|TRUE|yes|YES) + sync_args+=(--no-clone-bundle) + ;; +esac +repo sync "${sync_args[@]}" "$@" info "sync complete"