Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 182 additions & 6 deletions example/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,15 @@ import {
attachKeywordListenerOnce,
AudioPermissionComponent,
captureWakewordDetection,
checkForWakewordModelUpdate,
cleanDetectedWakeWord,
defaultAudioRoutingConfig,
detachKeywordListener,
formatWakeWord,
initializeWakewordBootstrap,
instanceConfigs,
prepareWakewordSpeechSession,
reloadWakewordModel,
resumeWakewordDetection,
shareWakewordRecordings,
startWakewordDetection,
Expand All @@ -112,6 +114,9 @@ const FULL_AI_CHAT_STT_OPTIONS = {
REQUEST_PERMISSIONS_AUTO: true,
};
let calledOnce = false;
// DaVoice demo license for wake word and speech. Kept in one place so the wake word model
// reload after a CDN update can apply it again.
const DAVOICE_DEMO_LICENSE = 'MTc5MzQ4NDAwMDAwMA==-cpeHAmR/9wRKvv9rBJ+36JqMUxmXR3RIpi5lK67VsUQ=';

function App(): React.JSX.Element {
const [isFlashing, setIsFlashing] = useState(false);
Expand Down Expand Up @@ -250,16 +255,23 @@ function App(): React.JSX.Element {
const [isPermissionGranted, setIsPermissionGranted] = useState(false);
useEffect(() => {
const handleAppStateChange = async (nextAppState: string) => {
console.log('[WakewordFlow] app state ->', nextAppState);
if (nextAppState === 'active') {
try {
if (Platform.OS === 'android') {
const granted = await AudioPermissionComponent();
console.log('[WakewordFlow] android mic permission granted ->', !!granted);
setIsPermissionGranted(!!granted);
} else {
if (await hasIOSMicPermissions() != true) {
const hadMic = await hasIOSMicPermissions();
console.log('[WakewordFlow] ios mic permission already granted ->', hadMic);
if (hadMic != true) {
await requestIOSMicPermissions(20000);
console.log('[WakewordFlow] ios mic permission requested');
}
if (await hasIOSSpeechRecognitionPermissions() != true) {
const hadSpeech = await hasIOSSpeechRecognitionPermissions();
console.log('[WakewordFlow] ios speech recognition permission already granted ->', hadSpeech);
if (hadSpeech != true) {
requestIOSSpeechRecognitionPermissions(20000)
}
// Keep iOS behavior unchanged by Android-first permission gating.
Expand Down Expand Up @@ -319,6 +331,8 @@ function App(): React.JSX.Element {
const [androidKeyboardHeight, setAndroidKeyboardHeight] = useState(0);
const [isMenuOpen, setIsMenuOpen] = useState(false);
const [latestWakewordRecordingPaths, setLatestWakewordRecordingPaths] = useState<string[]>([]);
const [isCheckingWakewordUpdate, setIsCheckingWakewordUpdate] = useState(false);
const startupFlowDoneRef = useRef(false);
const lastPartialTimeRef = useRef(0);
const timeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const speechSessionUIAllowedRef = useRef(false);
Expand Down Expand Up @@ -1124,6 +1138,12 @@ function App(): React.JSX.Element {
// --> STARTING POINT - INIT OF KEYWORD DETECTION !!!!
const initializeKeywordDetection = async () => {
let svChoice: SVPromptChoice = 'skip';
const initStarted = Date.now();
console.log('[WakewordFlow] initializeKeywordDetection: begin', {
platform: Platform.OS,
svChoice,
enrollmentJsonPath: enrollmentJsonPathRef.current,
});

try {
if (Platform.OS === 'android') {
Expand All @@ -1143,10 +1163,8 @@ function App(): React.JSX.Element {
keywordCallback,
listenerRef,
myInstanceRef,
keywordLicense:
'MTc5MzQ4NDAwMDAwMA==-cpeHAmR/9wRKvv9rBJ+36JqMUxmXR3RIpi5lK67VsUQ=',
speechLicense:
'MTc5MzQ4NDAwMDAwMA==-cpeHAmR/9wRKvv9rBJ+36JqMUxmXR3RIpi5lK67VsUQ=',
keywordLicense: DAVOICE_DEMO_LICENSE,
speechLicense: DAVOICE_DEMO_LICENSE,
Speech,
svChoice,
enrollmentJsonPath: enrollmentJsonPathRef.current,
Expand All @@ -1156,6 +1174,11 @@ function App(): React.JSX.Element {
suppressAndroidPartialResultsRef,
speechLibraryInitializedRef,
});
console.log('[WakewordFlow] initializeKeywordDetection: bootstrap returned', {
speechInitCompleted,
failureReason,
elapsedMs: Date.now() - initStarted,
});

if (Platform.OS === 'android') {
if (voiceDemoBootstrapTimeoutRef.current) {
Expand All @@ -1166,11 +1189,14 @@ function App(): React.JSX.Element {

if (!speechInitCompleted) {
if (failureReason === 'invalid-license') {
console.log('[WakewordFlow] initializeKeywordDetection: stopping, license invalid');
return;
}
console.log('[WakewordFlow] initializeKeywordDetection: speech init incomplete, wakeword listening anyway');
setMessage(`Say the wake word "${wakeWords}" to continue.`);
return;
}
console.log('[WakewordFlow] initializeKeywordDetection: speech ready, starting narration/onboarding');

const narratorVoice = selectedTTSVoiceRef.current;
const otherVoices = (['Hanna', 'Rich', 'Ariana'] as TTSVoiceChoice[])
Expand Down Expand Up @@ -1278,9 +1304,20 @@ function App(): React.JSX.Element {
setMessage(`Say the wake word "${wakeWords}" to continue.`);
}
console.error('Error during keyword detection initialization:', error);
} finally {
// The wake word update menu only offers a live model reload once startup has settled.
startupFlowDoneRef.current = true;
console.log('[WakewordFlow] initializeKeywordDetection: startup flow done (hot reload of model now allowed)', {
elapsedMs: Date.now() - initStarted,
});
}
};

console.log('[WakewordFlow] init effect', {
initStarted: initStartedRef.current,
isPermissionGranted,
didInitSID,
});
if (initStartedRef.current) return;
if (!isPermissionGranted || !didInitSID) return;

Expand Down Expand Up @@ -1482,6 +1519,136 @@ function App(): React.JSX.Element {
setIsMenuOpen,
});

// Nothing is mid-flight: no prompt, no narration, no STT/TTS session. Only then is it safe to
// hot-swap the wake word model on the live instance instead of asking for an app restart.
// Mirrored into a ref so the async handlers below read the current value, not the one
// captured when they were created.
const isIdleListeningRef = useRef(false);
isIdleListeningRef.current =
!isSpeechSessionActive &&
!isIntroSpeaking &&
!isTTSTestMode &&
!isFullAIChatMode &&
!isSTTOnlyMode &&
!isCombinedMode &&
!showSVPrompt &&
!showSVStatusScreen &&
!showAppModePrompt &&
!showTTSModelPrompt &&
!svRunning;
const canHotReloadWakeword = () =>
startupFlowDoneRef.current && isIdleListeningRef.current && !!myInstanceRef.current;

const RESTART_FOR_WAKEWORD_UPDATE_MESSAGE =
'A new wake word model was downloaded. Close the app completely and open it again to start using it.';

const reloadWakewordModelNow = async (modelPath: string) => {
const instance = myInstanceRef.current;
console.log('[WakewordFlow] reloadWakewordModelNow: requested', {
modelPath,
hasInstance: !!instance,
startupFlowDone: startupFlowDoneRef.current,
isIdleListening: isIdleListeningRef.current,
});
if (!instance || !canHotReloadWakeword()) {
console.log('[WakewordFlow] reloadWakewordModelNow: cannot hot reload now, asking user to restart the app');
Alert.alert('Wake word updated', RESTART_FOR_WAKEWORD_UPDATE_MESSAGE);
return;
}
setMessage('Reloading wake word model...');
try {
await reloadWakewordModel({
instance,
modelPath,
keywordLicense: DAVOICE_DEMO_LICENSE,
svChoice: enrollmentJsonPathRef.current ? 'use_existing' : 'skip',
enrollmentJsonPath: enrollmentJsonPathRef.current,
sleep,
resumeDetection: true,
});
console.log('[WakewordFlow] reloadWakewordModelNow: hot reload succeeded, new model active');
setMessage(`Wake word model updated. Say the wake word "${wakeWords}" to continue.`);
} catch (error) {
console.warn('[WakewordUpdate] live reload failed, restart required:', error);
console.log('[WakewordFlow] reloadWakewordModelNow: hot reload FAILED, model will load on next app launch');
setMessage(`Say the wake word "${wakeWords}" to continue.`);
Alert.alert(
'Could not load the new model',
'The new wake word model was downloaded but this version of the wake word engine could not load it. The app keeps using the built-in model.',
);
}
};

// Manual "Check for wake word update" (top-right menu). The startup check is silent; this one
// is user-initiated, so it reports the outcome.
const checkForWakewordUpdate = async () => {
console.log('[WakewordFlow] manual update check: requested from menu', { alreadyChecking: isCheckingWakewordUpdate });
if (isCheckingWakewordUpdate) return;
setIsCheckingWakewordUpdate(true);
const checkStarted = Date.now();
try {
const result = await checkForWakewordModelUpdate({
fileName: instanceConfigs[0].modelName,
});
console.log('[WakewordFlow] manual update check: result', {
...result,
elapsedMs: Date.now() - checkStarted,
});
setIsMenuOpen(false);
if (result.status === 'up_to_date') {
console.log('[WakewordFlow] manual update check: already up to date');
Alert.alert('Wake word up to date', `"${wakeWords}" is already using the latest model.`);
return;
}
if (result.status !== 'updated') {
console.log('[WakewordFlow] manual update check: CDN unavailable or download failed', result.reason);
Alert.alert(
'Wake word update',
'Could not check for a wake word update right now. Please try again later.',
);
return;
}
if (!canHotReloadWakeword()) {
console.log('[WakewordFlow] manual update check: new model installed but app is busy, restart required', {
startupFlowDone: startupFlowDoneRef.current,
isIdleListening: isIdleListeningRef.current,
hasInstance: !!myInstanceRef.current,
});
Alert.alert('Wake word updated', RESTART_FOR_WAKEWORD_UPDATE_MESSAGE);
return;
}
console.log('[WakewordFlow] manual update check: new model installed, offering hot reload');
Alert.alert(
'Wake word updated',
'A new wake word model was downloaded. Reload it now, or close and reopen the app later.',
[
{
text: 'Later',
style: 'cancel',
onPress: () => console.log('[WakewordFlow] user chose "Later"; new model loads on next app launch'),
},
{
text: 'Reload now',
onPress: () => {
console.log('[WakewordFlow] user chose "Reload now"');
void reloadWakewordModelNow(result.modelPath);
},
},
],
);
} catch (error) {
console.warn('[WakewordUpdate] manual check failed:', error);
console.log('[WakewordFlow] manual update check: threw', error);
setIsMenuOpen(false);
Alert.alert(
'Wake word update',
'Could not check for a wake word update right now. Please try again later.',
);
} finally {
setIsCheckingWakewordUpdate(false);
}
};

const shouldShowFullAIChatScreen =
isFullAIChatMode ||
(
Expand Down Expand Up @@ -2150,6 +2317,15 @@ function App(): React.JSX.Element {
onPress={shareLatestRecordings}>
<Text style={styles.menuItemText}>Share recordings</Text>
</TouchableOpacity>
<TouchableOpacity
style={styles.menuItemButton}
activeOpacity={0.7}
disabled={isCheckingWakewordUpdate}
onPress={checkForWakewordUpdate}>
<Text style={styles.menuItemText}>
{isCheckingWakewordUpdate ? 'Checking for update...' : 'Check for wake word update'}
</Text>
</TouchableOpacity>
</View>
)}
</View>
Expand Down
71 changes: 71 additions & 0 deletions example/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,75 @@ For production apps, a permanent Gemini key should usually not be shipped direct

Make sure your Metro configuration supports DaVoice model assets such as `.onnx` and `.dm` files. See [react-native.config.js](./react-native.config.js) and the project asset setup.

### Wake word model updates from a CDN

The wake word model `hey_coach_model_28_22012026b.dm` ships inside the app (Android assets and the iOS bundle) and can also be refreshed from a CDN at runtime. The code lives in [src/wakeword/modelUpdater.ts](./src/wakeword/modelUpdater.ts).

Host two files at the root of the CDN, keeping the names:

```text
https://<cdn>/hey_coach_model_28_22012026b.dm
https://<cdn>/hey_coach_model_28_22012026b.dm.sha256
```

`WAKEWORD_MODEL_CDN_BASE_URL` in `src/wakeword/modelUpdater.ts` must be the **directory** that holds the files, not the file itself. The updater appends the model name, so a base URL that already ends in `.dm` produces `.../<model>.dm/<model>.dm` and every request returns 404. The example uses a Cloudflare R2 public bucket:

```ts
export const WAKEWORD_MODEL_CDN_BASE_URL = 'https://pub-fa06ba558cd447a38e86d0d4cf3e6786.r2.dev';
```

Generate the `.sha256` sidecar after every model change and upload it together with the model:

```bash
node scripts/wakeword-model-hash.js path/to/hey_coach_model_28_22012026b.dm
# writes path/to/hey_coach_model_28_22012026b.dm.sha256 (use --out <dir> to place it elsewhere)
```

Give the `.sha256` file a short cache TTL so new versions are noticed quickly. The CDN must be served over HTTPS (iOS App Transport Security is left strict in this example).

#### How it behaves

- **App open**: before the wake word instance is created, the app fetches the `.sha256` sidecar (or, if it is missing, the model's `ETag`/`Last-Modified`/`Content-Length` from a `HEAD` request) and compares it with the model in use. Only when it differs is the `.dm` downloaded, hash-verified, and installed. The instance then loads the new file. The check is time-boxed (15 seconds) and silent: if the CDN is unreachable or the download fails, nothing is shown and the current model keeps working.
- **Menu, "Check for wake word update"**: same check, started by the user. It reports "up to date", "could not check", or "updated". After an update, when the app is idle (no prompt, narration, or speech session), it offers "Reload now", which hot-swaps the model on the running instance. Otherwise it asks the user to close the app completely and open it again.
- **CDN serves the bundled model again**: the downloaded copy is dropped and the app returns to the bundled model.

Downloaded models are stored under the app's documents directory in `wakeword_models/<sha256 prefix>/`, each version in its own folder so the native `.dm` unpack cache never serves a stale copy. The manifest next to them records what is installed; deleting the folder falls back to the bundled model.

#### Platform support

| Platform | Loads a downloaded model? | Notes |
|---|---|---|
| Android | Yes | The DaVoice `KeyWordsDetection` library checks `new File(path).exists()` before treating the string as an asset name. |
| iOS | Not yet (react-native-wakeword 1.1.143) | The DaVoice `KeyWordDetection.xcframework` resolves every model string as a main-bundle asset (`copyAssetToDocumentsDirectory`). An absolute path fails with `Failed to copy asset`. |

On iOS the download and install still succeed, but when the native instance rejects the path the app logs a warning and falls back to the bundled model (`addInstanceMulti` and `reloadWakewordModel` in [src/wakeword/index.ts](./src/wakeword/index.ts)). The "Reload now" action shows "Could not load the new model" instead of asking for a restart, since a restart would not help. Once the iOS framework accepts absolute paths (a one-line early return for paths that start with `/` and exist on disk, matching the Android behavior), downloaded models take effect on iOS with no app change.

#### Logging

The whole flow is traced in the console:

- `[WakewordFlow]`: app launch, permissions, init gating, bootstrap, instance creation, license, detection start, manual update check, and hot reload (`App.tsx` and `src/wakeword/index.ts`).
- `[WakewordModelUpdate]`: manifest state, the exact URLs requested, remote and bundled hashes, HEAD fallback metadata, download progress, verification, install path, cleanup, and the final `RESULT` line with the remaining time budget (`src/wakeword/modelUpdater.ts`).

#### Testing an update end to end

1. Keep two different versions of the model with the same file name, for example the bundled one in `assets/models/` and a newer one in `assets/models/new/`.
2. Create the sidecar for the new model:
```bash
node scripts/wakeword-model-hash.js assets/models/new/hey_coach_model_28_22012026b.dm
```
3. Upload `hey_coach_model_28_22012026b.dm` and `hey_coach_model_28_22012026b.dm.sha256` from that folder to the CDN root, replacing the existing files. Verify:
```bash
curl -sS https://<cdn>/hey_coach_model_28_22012026b.dm.sha256
curl -sS https://<cdn>/hey_coach_model_28_22012026b.dm | shasum -a 256
```
Both hashes must match.
4. **Startup path**: delete the app from the device, install, launch, and filter the console on `[WakewordModelUpdate]`. Expect `step 1: remote sha256 =`, `compare remote vs bundled` with two different hashes, `download verified`, `INSTALLING new model`, `RESULT updated`, then `using DOWNLOADED model`. On iOS a `[WakewordFlow]` warning follows and the bundled model is used (see Platform support).
5. **Already up to date**: relaunch without changing the CDN. Expect `remote sha256 matches the installed downloaded model` and `RESULT up_to_date` with no download.
6. **Manual check**: upload another version plus its sidecar, open the top-right menu, tap "Check for wake word update", then "Reload now". Android hot-swaps the model; iOS shows "Could not load the new model" and keeps running.
7. **Rollback**: upload the bundled model and its sidecar again and relaunch. Expect `switching back to the BUNDLED model` and `RESULT up_to_date`; the old version folder is removed.
8. **Offline**: disable networking and launch. Expect `RESULT unavailable` within seconds and a normal start on the active model.

### Native permissions

Microphone permission is required. iOS speech-recognition permissions may also be required depending on the flow you enable.
Expand All @@ -117,6 +186,8 @@ The example can share recorded wake-word audio. On Android that uses a `FileProv
- [src/stt/](./src/stt): STT transcript merge logic and speech callback registration
- [src/tts/](./src/tts): TTS constants, model assets, and intro speech flow
- [src/wakeword/](./src/wakeword): wakeword config, bootstrap, listener, capture, and sharing helpers
- [src/wakeword/modelUpdater.ts](./src/wakeword/modelUpdater.ts): CDN update check and download for the `.dm` wake word model
- [scripts/wakeword-model-hash.js](./scripts/wakeword-model-hash.js): writes the `.sha256` sidecar to upload next to the model
- [src/speaker_verification/](./src/speaker_verification): onboarding and verification helpers
- [src/aichat/](./src/aichat): Gemini request helpers and AI-chat speech/session helpers
- [package.json](./package.json): example dependencies
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1b9f1ffc64cd96dcaa2813feec1b985d0ef521c80129de2a4ba9c962f0a0affa hey_coach_model_28_22012026b.dm
3 changes: 3 additions & 0 deletions example/assets/models/new/hey_coach_model_28_22012026b.dm
Git LFS file not shown
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b018321f937e1ecdc81876aecf58fede02e28e40034ed74b0961b4f41e4d2888 hey_coach_model_28_22012026b.dm
Loading