secondly-com · adamcohenhillel · Jul 15, 2026
diff --git a/.github/workflows/gcp-lab.yml b/.github/workflows/gcp-lab.yml
@@ -28,6 +28,7 @@ on:
           - incremental-emulator
           - smoke-only
           - export-emulator-image
+          - assistant-apk
           - custom
       arch:
         description: "Emulator architecture"
@@ -202,6 +203,7 @@ jobs:
             keep_vm="false"
             lane="incremental-emulator"
             export_emulator_image="false"
+            build_assistant_apk="false"
             run_smoke="true"
             skip_build="false"
           else
@@ -235,21 +237,31 @@ jobs:
             case "$lane" in
               incremental-emulator)
                 export_emulator_image="false"
+                build_assistant_apk="false"
                 run_smoke="true"
                 skip_build="false"
                 ;;
               smoke-only)
                 export_emulator_image="false"
+                build_assistant_apk="false"
                 run_smoke="true"
                 skip_build="true"
                 ;;
               export-emulator-image)
                 export_emulator_image="true"
+                build_assistant_apk="false"
+                run_smoke="false"
+                skip_build="false"
+                ;;
+              assistant-apk)
+                export_emulator_image="false"
+                build_assistant_apk="true"
                 run_smoke="false"
                 skip_build="false"
                 ;;
               custom)
                 export_emulator_image="${{ inputs.export_emulator_image }}"
+                build_assistant_apk="false"
                 run_smoke="${{ inputs.run_smoke }}"
                 skip_build="${{ inputs.skip_build }}"
                 ;;
@@ -275,6 +287,7 @@ jobs:
             echo "cache_source_snapshot=$cache_source_snapshot"
             echo "keep_vm=$keep_vm"
             echo "export_emulator_image=$export_emulator_image"
+            echo "build_assistant_apk=$build_assistant_apk"
             echo "run_smoke=$run_smoke"
             echo "skip_build=$skip_build"
           } >> "$GITHUB_OUTPUT"
@@ -307,6 +320,9 @@ jobs:
           if [[ "${{ steps.target.outputs.export_emulator_image }}" == "true" ]]; then
             args+=(--export-emulator-image)
           fi
+          if [[ "${{ steps.target.outputs.build_assistant_apk }}" == "true" ]]; then
+            args+=(--device tegu --build-assistant-apk)
+          fi
           if [[ "${{ steps.target.outputs.run_smoke }}" != "true" ]]; then
             args+=(--skip-smoke)
           fi
@@ -341,6 +357,7 @@ jobs:
             echo "- Cache disk: \`${{ steps.target.outputs.cache_disk || 'none' }}\`"
             echo "- Cache snapshot: \`${{ steps.target.outputs.cache_source_snapshot || 'none' }}\`"
             echo "- Export emulator image: \`${{ steps.target.outputs.export_emulator_image }}\`"
+            echo "- Build assistant APK: \`${{ steps.target.outputs.build_assistant_apk }}\`"
             echo "- Run smoke: \`${{ steps.target.outputs.run_smoke }}\`"
             echo "- Skip build: \`${{ steps.target.outputs.skip_build }}\`"
             echo "- VM service account: \`disabled\`"

diff --git a/...apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeVoiceSession.java b/...apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeVoiceSession.java
@@ -50,7 +50,7 @@ public interface Callback {
         void onStopped();
     }
 
-    public static final String MODEL = "gpt-realtime-2";
+    public static final String MODEL = "gpt-realtime-2.1";
 
     private static final int SAMPLE_RATE = 24000;
     private static final long CONNECT_TIMEOUT_MS = 20000;
@@ -71,6 +71,12 @@ public interface Callback {
     private static final long ACTION_RESPONSE_STALL_MS = 9000;
     private static final long STALL_CANCEL_GRACE_MS = 1800;
     private static final int ACTION_RESPONSE_STALL_MAX_RECOVERIES = 2;
+    private static final double LOCAL_SPEECH_RMS = 850.0;
+    private static final long LOCAL_SPEECH_MIN_MS = 220;
+    private static final long LOCAL_SILENCE_COMMIT_MS = 850;
+    private static final long LOCAL_COMMIT_COOLDOWN_MS = 1800;
+    private static final long LOCAL_MIN_AUDIO_BEFORE_COMMIT_MS = 500;
+    private static final long MIC_RMS_LOG_INTERVAL_MS = 1000;
     private static final String REALTIME_URL =
             "wss://api.openai.com/v1/realtime?model=" + MODEL;
 
@@ -91,6 +97,14 @@ public interface Callback {
     private volatile NoiseSuppressor mNoiseSuppressor;
     private volatile AutomaticGainControl mAutomaticGainControl;
     private volatile double mRecentMicRms;
+    private volatile long mInputAudioBytesSinceCommit;
+    private volatile boolean mServerVadActive;
+    private volatile boolean mLocalSpeechActive;
+    private volatile boolean mManualCommitPending;
+    private volatile long mLocalSpeechStartedUptimeMillis;
+    private volatile long mLastLocalSpeechUptimeMillis;
+    private volatile long mLastManualCommitUptimeMillis;
+    private volatile long mLastMicLogUptimeMillis;
     private String mPendingAssistantTranscript;
     private volatile boolean mAssistantAudioActive;
     private boolean mPendingToolResponseCreate;
@@ -386,6 +400,7 @@ public void run() {
                         byte[] chunk = new byte[read];
                         System.arraycopy(buffer, 0, chunk, 0, read);
                         sendAudioChunk(socket, chunk);
+                        observeLocalAudio(socket, callback, rms, read);
                         maybeStopPlaybackForLocalBargeIn(socket, callback, rms);
                     } catch (IOException | JSONException e) {
                         if (!mCancelled) {
@@ -401,6 +416,54 @@ public void run() {
         audioThread.start();
     }
 
+    private void observeLocalAudio(RealtimeWebSocket socket, Callback callback, double rms,
+            int bytesRead) throws IOException, JSONException {
+        long now = SystemClock.uptimeMillis();
+        mInputAudioBytesSinceCommit += Math.max(0, bytesRead);
+        if (now - mLastMicLogUptimeMillis >= MIC_RMS_LOG_INTERVAL_MS) {
+            mLastMicLogUptimeMillis = now;
+            Log.i(TAG, "mic rms=" + Math.round(rms)
+                    + " bytes_since_commit=" + mInputAudioBytesSinceCommit
+                    + " server_vad=" + mServerVadActive
+                    + " manual_pending=" + mManualCommitPending);
+        }
+        if (rms >= LOCAL_SPEECH_RMS) {
+            if (!mLocalSpeechActive) {
+                mLocalSpeechActive = true;
+                mLocalSpeechStartedUptimeMillis = now;
+                Log.i(TAG, "local speech started rms=" + Math.round(rms));
+            }
+            mLastLocalSpeechUptimeMillis = now;
+            return;
+        }
+        if (!mLocalSpeechActive || mServerVadActive || mManualCommitPending) {
+            return;
+        }
+        long speechMs = mLastLocalSpeechUptimeMillis - mLocalSpeechStartedUptimeMillis;
+        long silenceMs = now - mLastLocalSpeechUptimeMillis;
+        long audioMs = audioBytesToMillis(mInputAudioBytesSinceCommit);
+        if (speechMs < LOCAL_SPEECH_MIN_MS || silenceMs < LOCAL_SILENCE_COMMIT_MS
+                || audioMs < LOCAL_MIN_AUDIO_BEFORE_COMMIT_MS
+                || now - mLastManualCommitUptimeMillis < LOCAL_COMMIT_COOLDOWN_MS) {
+            return;
+        }
+        socket.send(new JSONObject().put("type", "input_audio_buffer.commit"));
+        mManualCommitPending = true;
+        mLastManualCommitUptimeMillis = now;
+        mLocalSpeechActive = false;
+        callback.onStatus("Observing");
+        Log.w(TAG, "manual input_audio_buffer.commit after local speech speechMs="
+                + speechMs + " silenceMs=" + silenceMs + " audioMs=" + audioMs);
+    }
+
+    private static long audioBytesToMillis(long bytes) {
+        long bytesPerSecond = SAMPLE_RATE * 2L;
+        if (bytes <= 0 || bytesPerSecond <= 0) {
+            return 0;
+        }
+        return (bytes * 1000L) / bytesPerSecond;
+    }
+
     private static AudioRecord createAudioRecord(int bufferSize) throws IOException {
         int[] sources = new int[] {
                 MediaRecorder.AudioSource.VOICE_COMMUNICATION,
@@ -456,6 +519,7 @@ private void handleEvent(RealtimeWebSocket socket, String taskId,
             ModelAdapter.ToolExecutor executor, Callback callback, JSONObject event)
             throws IOException, JSONException {
         String type = event.optString("type");
+        logRealtimeEvent(type);
         markActionResponseActivity(type);
         if ("error".equals(type)) {
             JSONObject error = event.optJSONObject("error");
@@ -472,6 +536,9 @@ private void handleEvent(RealtimeWebSocket socket, String taskId,
             throw new IOException(error == null ? event.toString() : error.toString());
         }
         if ("input_audio_buffer.speech_started".equals(type)) {
+            mServerVadActive = true;
+            mManualCommitPending = false;
+            Log.i(TAG, "server speech_started micRms=" + Math.round(mRecentMicRms));
             if (isPlaybackActiveOrRecentlyActive()) {
                 if (mRecentMicRms >= SERVER_BARGE_IN_RMS) {
                     handleServerSpeechStartedDuringPlayback(socket, callback);
@@ -485,10 +552,16 @@ private void handleEvent(RealtimeWebSocket socket, String taskId,
             return;
         }
         if ("input_audio_buffer.speech_stopped".equals(type)) {
+            mServerVadActive = false;
+            Log.i(TAG, "server speech_stopped micRms=" + Math.round(mRecentMicRms));
             callback.onStatus("Observing");
             return;
         }
         if ("input_audio_buffer.committed".equals(type)) {
+            mServerVadActive = false;
+            mManualCommitPending = false;
+            mLocalSpeechActive = false;
+            mInputAudioBytesSinceCommit = 0;
             callback.onStatus("Observing");
             resetTurnToolState();
             appendAutoScreenContext(socket, executor,
@@ -586,6 +659,19 @@ private void handleEvent(RealtimeWebSocket socket, String taskId,
         }
     }
 
+    private static void logRealtimeEvent(String type) {
+        if (type == null || type.isEmpty()) {
+            Log.i(TAG, "event=<empty>");
+            return;
+        }
+        if (type.startsWith("response.")
+                || type.startsWith("input_audio_buffer.")
+                || type.startsWith("conversation.item.input_audio_transcription")
+                || type.contains("function_call")) {
+            Log.i(TAG, "event=" + type);
+        }
+    }
+
     private void flushAssistantTranscript(Callback callback) {
         String transcript = mPendingAssistantTranscript;
         if (transcript == null || transcript.trim().isEmpty()) {

diff --git a/scripts/lab/gcp/run-smoke.sh b/scripts/lab/gcp/run-smoke.sh
@@ -33,6 +33,7 @@ Options:
                               $HOME/openphone-android for stable build paths.
   --arch arm64|x86_64         Emulator arch. Default: x86_64.
   --variant eng|userdebug     Emulator variant. Default: eng.
+  --device <codename>         Device codename for focused APK builds. Default: tegu.
   --runtime <name>            Runtime intent: local, openclaw, or hermes.
                              May be repeated. Default: local.
   --timeout <seconds>         Emulator boot timeout. Default: 900.
@@ -41,6 +42,8 @@ Options:
   --skip-build                Reuse existing Android build outputs on the VM.
   --export-emulator-image     Copy sdk-repo-linux-system-images.zip into lab
                               artifacts for local Mac/SDK installation.
+  --build-assistant-apk       Build and upload OpenPhoneAssistant.apk for the
+                              selected device without requiring a release OTA.
   --skip-smoke                Build/export artifacts but do not boot the
                               emulator. Intended for cross-arch image exports.
   -h, --help                  Show this help.
@@ -70,11 +73,13 @@ cache_source_snapshot="$OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT"
 cache_mount="$OPENPHONE_GCP_CACHE_MOUNT"
 arch="x86_64"
 variant="eng"
+device="tegu"
 timeout_seconds=900
 repo_sync_jobs=""
 keep_vm=false
 skip_build=false
 export_emulator_image=false
+build_assistant_apk=false
 skip_smoke=false
 runtimes=()
 
@@ -155,6 +160,11 @@ while [[ $# -gt 0 ]]; do
       variant="$2"
       shift 2
       ;;
+    --device)
+      [[ $# -ge 2 ]] || die "--device requires a value"
+      device="$2"
+      shift 2
+      ;;
     --runtime)
       [[ $# -ge 2 ]] || die "--runtime requires a value"
       runtimes+=("$2")
@@ -182,6 +192,10 @@ while [[ $# -gt 0 ]]; do
       export_emulator_image=true
       shift
       ;;
+    --build-assistant-apk)
+      build_assistant_apk=true
+      shift
+      ;;
     --skip-smoke)
       skip_smoke=true
       shift
@@ -208,6 +222,11 @@ case "$variant" in
   *) die "unsupported emulator variant: $variant" ;;
 esac
 
+case "$device" in
+  tegu) ;;
+  *) die "unsupported assistant APK device: $device" ;;
+esac
+
 case "$cache_mode" in
   scratch|attach-disk|snapshot) ;;
   *) die "unsupported cache mode: $cache_mode" ;;
@@ -297,10 +316,12 @@ ref="${OPENPHONE_REF:?}"
 slot="${OPENPHONE_LAB_SLOT:?}"
 arch="${OPENPHONE_EMULATOR_ARCH:?}"
 variant="${OPENPHONE_EMULATOR_VARIANT:?}"
+device="${OPENPHONE_DEVICE:?}"
 timeout_seconds="${OPENPHONE_EMULATOR_TIMEOUT:?}"
 repo_sync_jobs="${OPENPHONE_REPO_SYNC_JOBS:-}"
 skip_build="${OPENPHONE_SKIP_BUILD:-0}"
 export_emulator_image="${OPENPHONE_EXPORT_EMULATOR_IMAGE:-0}"
+build_assistant_apk="${OPENPHONE_BUILD_ASSISTANT_APK:-0}"
 skip_smoke="${OPENPHONE_SKIP_SMOKE:-0}"
 runtime_csv="${OPENPHONE_LAB_RUNTIMES:-local}"
 cache_mode="${OPENPHONE_GCP_CACHE_MODE:-scratch}"
@@ -444,6 +465,36 @@ if [[ "$export_emulator_image" == "1" ]]; then
   )
 fi
 
+if [[ "$build_assistant_apk" == "1" && "$skip_build" != "1" ]]; then
+  OPENPHONE_BUILD_GOAL=OpenPhoneAssistant \
+    ./scripts/build.sh "openphone_${device}-${OPENPHONE_RELEASE:-bp4a}-userdebug"
+fi
+
+if [[ "$build_assistant_apk" == "1" ]]; then
+  product_dir="$OPENPHONE_ANDROID_DIR/out/target/product/$device"
+  apk=""
+  for candidate in \
+    "$product_dir/system_ext/priv-app/OpenPhoneAssistant/OpenPhoneAssistant.apk" \
+    "$product_dir/obj/APPS/OpenPhoneAssistant_intermediates/package.apk"; do
+    if [[ -f "$candidate" ]]; then
+      apk="$candidate"
+      break
+    fi
+  done
+  if [[ -z "$apk" ]]; then
+    printf 'error: OpenPhoneAssistant.apk not found under %s\n' "$product_dir" >&2
+    exit 1
+  fi
+  artifact_apk_dir="$HOME/openphone-src/.worktree/lab/$slot/artifacts/assistant-apk"
+  mkdir -p "$artifact_apk_dir"
+  cp "$apk" "$artifact_apk_dir/OpenPhoneAssistant-${device}.apk"
+  (
+    cd "$artifact_apk_dir"
+    sha256sum "OpenPhoneAssistant-${device}.apk" \
+      > "OpenPhoneAssistant-${device}.apk.sha256"
+  )
+fi
+
 if [[ "$skip_smoke" == "1" ]]; then
   printf '==> Skipping emulator smoke by request after build/export\n'
   exit 0
@@ -484,10 +535,16 @@ remote_command+=" OPENPHONE_REF=$(shell_quote "$ref")"
 remote_command+=" OPENPHONE_LAB_SLOT=$(shell_quote "$slot")"
 remote_command+=" OPENPHONE_EMULATOR_ARCH=$(shell_quote "$arch")"
 remote_command+=" OPENPHONE_EMULATOR_VARIANT=$(shell_quote "$variant")"
+remote_command+=" OPENPHONE_DEVICE=$(shell_quote "$device")"
 remote_command+=" OPENPHONE_EMULATOR_TIMEOUT=$(shell_quote "$timeout_seconds")"
 remote_command+=" OPENPHONE_REPO_SYNC_JOBS=$(shell_quote "$repo_sync_jobs")"
 remote_command+=" OPENPHONE_SKIP_BUILD=$(shell_quote "$skip_build_value")"
 remote_command+=" OPENPHONE_EXPORT_EMULATOR_IMAGE=$(shell_quote "$export_emulator_image_value")"
+build_assistant_apk_value=0
+if [[ "$build_assistant_apk" == true ]]; then
+  build_assistant_apk_value=1
+fi
+remote_command+=" OPENPHONE_BUILD_ASSISTANT_APK=$(shell_quote "$build_assistant_apk_value")"
 remote_command+=" OPENPHONE_SKIP_SMOKE=$(shell_quote "$skip_smoke_value")"
 remote_command+=" OPENPHONE_LAB_RUNTIMES=$(shell_quote "$runtime_csv")"
 remote_command+=" OPENPHONE_GCP_CACHE_MODE=$(shell_quote "$cache_mode")"