From 6baf0b9f55421a9c3309cc80885b41318b0c979a Mon Sep 17 00:00:00 2001 From: Adam Cohen Hillel Date: Tue, 14 Jul 2026 19:31:47 -0700 Subject: [PATCH] Fix realtime voice turn detection --- .github/workflows/gcp-lab.yml | 17 ++++ .../model/OpenAiRealtimeVoiceSession.java | 88 ++++++++++++++++++- scripts/lab/gcp/run-smoke.sh | 57 ++++++++++++ 3 files changed, 161 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gcp-lab.yml b/.github/workflows/gcp-lab.yml index 6c93601..d08ea66 100644 --- a/.github/workflows/gcp-lab.yml +++ b/.github/workflows/gcp-lab.yml @@ -28,6 +28,7 @@ on: - incremental-emulator - smoke-only - export-emulator-image + - assistant-apk - custom arch: description: "Emulator architecture" @@ -202,6 +203,7 @@ jobs: keep_vm="false" lane="incremental-emulator" export_emulator_image="false" + build_assistant_apk="false" run_smoke="true" skip_build="false" else @@ -235,21 +237,31 @@ jobs: case "$lane" in incremental-emulator) export_emulator_image="false" + build_assistant_apk="false" run_smoke="true" skip_build="false" ;; smoke-only) export_emulator_image="false" + build_assistant_apk="false" run_smoke="true" skip_build="true" ;; export-emulator-image) export_emulator_image="true" + build_assistant_apk="false" + run_smoke="false" + skip_build="false" + ;; + assistant-apk) + export_emulator_image="false" + build_assistant_apk="true" run_smoke="false" skip_build="false" ;; custom) export_emulator_image="${{ inputs.export_emulator_image }}" + build_assistant_apk="false" run_smoke="${{ inputs.run_smoke }}" skip_build="${{ inputs.skip_build }}" ;; @@ -275,6 +287,7 @@ jobs: echo "cache_source_snapshot=$cache_source_snapshot" echo "keep_vm=$keep_vm" echo "export_emulator_image=$export_emulator_image" + echo "build_assistant_apk=$build_assistant_apk" echo "run_smoke=$run_smoke" echo "skip_build=$skip_build" } >> "$GITHUB_OUTPUT" @@ -307,6 +320,9 @@ jobs: if [[ "${{ steps.target.outputs.export_emulator_image }}" == "true" ]]; then args+=(--export-emulator-image) fi + if [[ "${{ steps.target.outputs.build_assistant_apk }}" == "true" ]]; then + args+=(--device tegu --build-assistant-apk) + fi if [[ "${{ steps.target.outputs.run_smoke }}" != "true" ]]; then args+=(--skip-smoke) fi @@ -341,6 +357,7 @@ jobs: echo "- Cache disk: \`${{ steps.target.outputs.cache_disk || 'none' }}\`" echo "- Cache snapshot: \`${{ steps.target.outputs.cache_source_snapshot || 'none' }}\`" echo "- Export emulator image: \`${{ steps.target.outputs.export_emulator_image }}\`" + echo "- Build assistant APK: \`${{ steps.target.outputs.build_assistant_apk }}\`" echo "- Run smoke: \`${{ steps.target.outputs.run_smoke }}\`" echo "- Skip build: \`${{ steps.target.outputs.skip_build }}\`" echo "- VM service account: \`disabled\`" diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeVoiceSession.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeVoiceSession.java index 2eec035..de2fd2e 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeVoiceSession.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeVoiceSession.java @@ -50,7 +50,7 @@ public interface Callback { void onStopped(); } - public static final String MODEL = "gpt-realtime-2"; + public static final String MODEL = "gpt-realtime-2.1"; private static final int SAMPLE_RATE = 24000; private static final long CONNECT_TIMEOUT_MS = 20000; @@ -71,6 +71,12 @@ public interface Callback { private static final long ACTION_RESPONSE_STALL_MS = 9000; private static final long STALL_CANCEL_GRACE_MS = 1800; private static final int ACTION_RESPONSE_STALL_MAX_RECOVERIES = 2; + private static final double LOCAL_SPEECH_RMS = 850.0; + private static final long LOCAL_SPEECH_MIN_MS = 220; + private static final long LOCAL_SILENCE_COMMIT_MS = 850; + private static final long LOCAL_COMMIT_COOLDOWN_MS = 1800; + private static final long LOCAL_MIN_AUDIO_BEFORE_COMMIT_MS = 500; + private static final long MIC_RMS_LOG_INTERVAL_MS = 1000; private static final String REALTIME_URL = "wss://api.openai.com/v1/realtime?model=" + MODEL; @@ -91,6 +97,14 @@ public interface Callback { private volatile NoiseSuppressor mNoiseSuppressor; private volatile AutomaticGainControl mAutomaticGainControl; private volatile double mRecentMicRms; + private volatile long mInputAudioBytesSinceCommit; + private volatile boolean mServerVadActive; + private volatile boolean mLocalSpeechActive; + private volatile boolean mManualCommitPending; + private volatile long mLocalSpeechStartedUptimeMillis; + private volatile long mLastLocalSpeechUptimeMillis; + private volatile long mLastManualCommitUptimeMillis; + private volatile long mLastMicLogUptimeMillis; private String mPendingAssistantTranscript; private volatile boolean mAssistantAudioActive; private boolean mPendingToolResponseCreate; @@ -386,6 +400,7 @@ public void run() { byte[] chunk = new byte[read]; System.arraycopy(buffer, 0, chunk, 0, read); sendAudioChunk(socket, chunk); + observeLocalAudio(socket, callback, rms, read); maybeStopPlaybackForLocalBargeIn(socket, callback, rms); } catch (IOException | JSONException e) { if (!mCancelled) { @@ -401,6 +416,54 @@ public void run() { audioThread.start(); } + private void observeLocalAudio(RealtimeWebSocket socket, Callback callback, double rms, + int bytesRead) throws IOException, JSONException { + long now = SystemClock.uptimeMillis(); + mInputAudioBytesSinceCommit += Math.max(0, bytesRead); + if (now - mLastMicLogUptimeMillis >= MIC_RMS_LOG_INTERVAL_MS) { + mLastMicLogUptimeMillis = now; + Log.i(TAG, "mic rms=" + Math.round(rms) + + " bytes_since_commit=" + mInputAudioBytesSinceCommit + + " server_vad=" + mServerVadActive + + " manual_pending=" + mManualCommitPending); + } + if (rms >= LOCAL_SPEECH_RMS) { + if (!mLocalSpeechActive) { + mLocalSpeechActive = true; + mLocalSpeechStartedUptimeMillis = now; + Log.i(TAG, "local speech started rms=" + Math.round(rms)); + } + mLastLocalSpeechUptimeMillis = now; + return; + } + if (!mLocalSpeechActive || mServerVadActive || mManualCommitPending) { + return; + } + long speechMs = mLastLocalSpeechUptimeMillis - mLocalSpeechStartedUptimeMillis; + long silenceMs = now - mLastLocalSpeechUptimeMillis; + long audioMs = audioBytesToMillis(mInputAudioBytesSinceCommit); + if (speechMs < LOCAL_SPEECH_MIN_MS || silenceMs < LOCAL_SILENCE_COMMIT_MS + || audioMs < LOCAL_MIN_AUDIO_BEFORE_COMMIT_MS + || now - mLastManualCommitUptimeMillis < LOCAL_COMMIT_COOLDOWN_MS) { + return; + } + socket.send(new JSONObject().put("type", "input_audio_buffer.commit")); + mManualCommitPending = true; + mLastManualCommitUptimeMillis = now; + mLocalSpeechActive = false; + callback.onStatus("Observing"); + Log.w(TAG, "manual input_audio_buffer.commit after local speech speechMs=" + + speechMs + " silenceMs=" + silenceMs + " audioMs=" + audioMs); + } + + private static long audioBytesToMillis(long bytes) { + long bytesPerSecond = SAMPLE_RATE * 2L; + if (bytes <= 0 || bytesPerSecond <= 0) { + return 0; + } + return (bytes * 1000L) / bytesPerSecond; + } + private static AudioRecord createAudioRecord(int bufferSize) throws IOException { int[] sources = new int[] { MediaRecorder.AudioSource.VOICE_COMMUNICATION, @@ -456,6 +519,7 @@ private void handleEvent(RealtimeWebSocket socket, String taskId, ModelAdapter.ToolExecutor executor, Callback callback, JSONObject event) throws IOException, JSONException { String type = event.optString("type"); + logRealtimeEvent(type); markActionResponseActivity(type); if ("error".equals(type)) { JSONObject error = event.optJSONObject("error"); @@ -472,6 +536,9 @@ private void handleEvent(RealtimeWebSocket socket, String taskId, throw new IOException(error == null ? event.toString() : error.toString()); } if ("input_audio_buffer.speech_started".equals(type)) { + mServerVadActive = true; + mManualCommitPending = false; + Log.i(TAG, "server speech_started micRms=" + Math.round(mRecentMicRms)); if (isPlaybackActiveOrRecentlyActive()) { if (mRecentMicRms >= SERVER_BARGE_IN_RMS) { handleServerSpeechStartedDuringPlayback(socket, callback); @@ -485,10 +552,16 @@ private void handleEvent(RealtimeWebSocket socket, String taskId, return; } if ("input_audio_buffer.speech_stopped".equals(type)) { + mServerVadActive = false; + Log.i(TAG, "server speech_stopped micRms=" + Math.round(mRecentMicRms)); callback.onStatus("Observing"); return; } if ("input_audio_buffer.committed".equals(type)) { + mServerVadActive = false; + mManualCommitPending = false; + mLocalSpeechActive = false; + mInputAudioBytesSinceCommit = 0; callback.onStatus("Observing"); resetTurnToolState(); appendAutoScreenContext(socket, executor, @@ -586,6 +659,19 @@ private void handleEvent(RealtimeWebSocket socket, String taskId, } } + private static void logRealtimeEvent(String type) { + if (type == null || type.isEmpty()) { + Log.i(TAG, "event="); + return; + } + if (type.startsWith("response.") + || type.startsWith("input_audio_buffer.") + || type.startsWith("conversation.item.input_audio_transcription") + || type.contains("function_call")) { + Log.i(TAG, "event=" + type); + } + } + private void flushAssistantTranscript(Callback callback) { String transcript = mPendingAssistantTranscript; if (transcript == null || transcript.trim().isEmpty()) { diff --git a/scripts/lab/gcp/run-smoke.sh b/scripts/lab/gcp/run-smoke.sh index 9112f76..8ad4662 100755 --- a/scripts/lab/gcp/run-smoke.sh +++ b/scripts/lab/gcp/run-smoke.sh @@ -33,6 +33,7 @@ Options: $HOME/openphone-android for stable build paths. --arch arm64|x86_64 Emulator arch. Default: x86_64. --variant eng|userdebug Emulator variant. Default: eng. + --device Device codename for focused APK builds. Default: tegu. --runtime Runtime intent: local, openclaw, or hermes. May be repeated. Default: local. --timeout Emulator boot timeout. Default: 900. @@ -41,6 +42,8 @@ Options: --skip-build Reuse existing Android build outputs on the VM. --export-emulator-image Copy sdk-repo-linux-system-images.zip into lab artifacts for local Mac/SDK installation. + --build-assistant-apk Build and upload OpenPhoneAssistant.apk for the + selected device without requiring a release OTA. --skip-smoke Build/export artifacts but do not boot the emulator. Intended for cross-arch image exports. -h, --help Show this help. @@ -70,11 +73,13 @@ cache_source_snapshot="$OPENPHONE_GCP_CACHE_SOURCE_SNAPSHOT" cache_mount="$OPENPHONE_GCP_CACHE_MOUNT" arch="x86_64" variant="eng" +device="tegu" timeout_seconds=900 repo_sync_jobs="" keep_vm=false skip_build=false export_emulator_image=false +build_assistant_apk=false skip_smoke=false runtimes=() @@ -155,6 +160,11 @@ while [[ $# -gt 0 ]]; do variant="$2" shift 2 ;; + --device) + [[ $# -ge 2 ]] || die "--device requires a value" + device="$2" + shift 2 + ;; --runtime) [[ $# -ge 2 ]] || die "--runtime requires a value" runtimes+=("$2") @@ -182,6 +192,10 @@ while [[ $# -gt 0 ]]; do export_emulator_image=true shift ;; + --build-assistant-apk) + build_assistant_apk=true + shift + ;; --skip-smoke) skip_smoke=true shift @@ -208,6 +222,11 @@ case "$variant" in *) die "unsupported emulator variant: $variant" ;; esac +case "$device" in + tegu) ;; + *) die "unsupported assistant APK device: $device" ;; +esac + case "$cache_mode" in scratch|attach-disk|snapshot) ;; *) die "unsupported cache mode: $cache_mode" ;; @@ -297,10 +316,12 @@ ref="${OPENPHONE_REF:?}" slot="${OPENPHONE_LAB_SLOT:?}" arch="${OPENPHONE_EMULATOR_ARCH:?}" variant="${OPENPHONE_EMULATOR_VARIANT:?}" +device="${OPENPHONE_DEVICE:?}" timeout_seconds="${OPENPHONE_EMULATOR_TIMEOUT:?}" repo_sync_jobs="${OPENPHONE_REPO_SYNC_JOBS:-}" skip_build="${OPENPHONE_SKIP_BUILD:-0}" export_emulator_image="${OPENPHONE_EXPORT_EMULATOR_IMAGE:-0}" +build_assistant_apk="${OPENPHONE_BUILD_ASSISTANT_APK:-0}" skip_smoke="${OPENPHONE_SKIP_SMOKE:-0}" runtime_csv="${OPENPHONE_LAB_RUNTIMES:-local}" cache_mode="${OPENPHONE_GCP_CACHE_MODE:-scratch}" @@ -444,6 +465,36 @@ if [[ "$export_emulator_image" == "1" ]]; then ) fi +if [[ "$build_assistant_apk" == "1" && "$skip_build" != "1" ]]; then + OPENPHONE_BUILD_GOAL=OpenPhoneAssistant \ + ./scripts/build.sh "openphone_${device}-${OPENPHONE_RELEASE:-bp4a}-userdebug" +fi + +if [[ "$build_assistant_apk" == "1" ]]; then + product_dir="$OPENPHONE_ANDROID_DIR/out/target/product/$device" + apk="" + for candidate in \ + "$product_dir/system_ext/priv-app/OpenPhoneAssistant/OpenPhoneAssistant.apk" \ + "$product_dir/obj/APPS/OpenPhoneAssistant_intermediates/package.apk"; do + if [[ -f "$candidate" ]]; then + apk="$candidate" + break + fi + done + if [[ -z "$apk" ]]; then + printf 'error: OpenPhoneAssistant.apk not found under %s\n' "$product_dir" >&2 + exit 1 + fi + artifact_apk_dir="$HOME/openphone-src/.worktree/lab/$slot/artifacts/assistant-apk" + mkdir -p "$artifact_apk_dir" + cp "$apk" "$artifact_apk_dir/OpenPhoneAssistant-${device}.apk" + ( + cd "$artifact_apk_dir" + sha256sum "OpenPhoneAssistant-${device}.apk" \ + > "OpenPhoneAssistant-${device}.apk.sha256" + ) +fi + if [[ "$skip_smoke" == "1" ]]; then printf '==> Skipping emulator smoke by request after build/export\n' exit 0 @@ -484,10 +535,16 @@ remote_command+=" OPENPHONE_REF=$(shell_quote "$ref")" remote_command+=" OPENPHONE_LAB_SLOT=$(shell_quote "$slot")" remote_command+=" OPENPHONE_EMULATOR_ARCH=$(shell_quote "$arch")" remote_command+=" OPENPHONE_EMULATOR_VARIANT=$(shell_quote "$variant")" +remote_command+=" OPENPHONE_DEVICE=$(shell_quote "$device")" remote_command+=" OPENPHONE_EMULATOR_TIMEOUT=$(shell_quote "$timeout_seconds")" remote_command+=" OPENPHONE_REPO_SYNC_JOBS=$(shell_quote "$repo_sync_jobs")" remote_command+=" OPENPHONE_SKIP_BUILD=$(shell_quote "$skip_build_value")" remote_command+=" OPENPHONE_EXPORT_EMULATOR_IMAGE=$(shell_quote "$export_emulator_image_value")" +build_assistant_apk_value=0 +if [[ "$build_assistant_apk" == true ]]; then + build_assistant_apk_value=1 +fi +remote_command+=" OPENPHONE_BUILD_ASSISTANT_APK=$(shell_quote "$build_assistant_apk_value")" remote_command+=" OPENPHONE_SKIP_SMOKE=$(shell_quote "$skip_smoke_value")" remote_command+=" OPENPHONE_LAB_RUNTIMES=$(shell_quote "$runtime_csv")" remote_command+=" OPENPHONE_GCP_CACHE_MODE=$(shell_quote "$cache_mode")"