From 095aacec6c3cfad7f8c01f104264b6b0936a5806 Mon Sep 17 00:00:00 2001 From: Khaled Alam Date: Wed, 15 Jul 2026 18:19:54 +0400 Subject: [PATCH 1/3] feat: add React Native (Expo) support via platform-split renderer and audio clock --- README.md | 43 ++++++++++++ examples/native/App.tsx | 54 ++++++++++++++ examples/native/README.md | 45 ++++++++++++ examples/native/package.json | 29 ++++++++ package.json | 30 +++++++- src/audio/expoAudioClock.ts | 101 +++++++++++++++++++++++++++ src/audio/types.ts | 37 ++++++++++ src/audio/webAudioClock.ts | 32 +++++++++ src/components/Avatoon.tsx | 3 +- src/components/AvatoonModel.tsx | 28 ++++---- src/components/CameraFovAnimator.tsx | 2 +- src/components/LipSyncAvatoon.tsx | 3 +- src/components/SceneEnvironment.tsx | 2 +- src/platform/index.native.tsx | 14 ++++ src/platform/index.ts | 12 ++++ src/types/native-modules.d.ts | 54 ++++++++++++++ test/webAudioClock.spec.ts | 28 ++++++++ vite.native.config.mjs | 40 +++++++++++ 18 files changed, 533 insertions(+), 24 deletions(-) create mode 100644 examples/native/App.tsx create mode 100644 examples/native/README.md create mode 100644 examples/native/package.json create mode 100644 src/audio/expoAudioClock.ts create mode 100644 src/audio/types.ts create mode 100644 src/audio/webAudioClock.ts create mode 100644 src/platform/index.native.tsx create mode 100644 src/platform/index.ts create mode 100644 src/types/native-modules.d.ts create mode 100644 test/webAudioClock.spec.ts create mode 100644 vite.native.config.mjs diff --git a/README.md b/README.md index f064eda..4c79add 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,49 @@ function App() { npm run example ``` +## ๐Ÿ“ฑ React Native (Expo) + +Avatoon runs on React Native too. The same `` component renders through +`@react-three/fiber/native` (via `expo-gl`) and plays audio through `expo-av` โ€” +selected automatically by the package's `react-native` export condition, so you +import from `'avatoon'` exactly as on web. + +```bash +npx expo install expo-gl expo-av expo-asset expo-file-system \ + @react-three/fiber @react-three/drei three +npm install avatoon +``` + +```tsx +import { useRef } from 'react'; +import { View, Pressable, Text } from 'react-native'; +import { Avatoon, type AvatoonHandle } from 'avatoon'; + +export default function App() { + const avatar = useRef(null); + return ( + + + avatar.current?.toggle()}> + Talk + + + ); +} +``` + +**Platform differences** + +| | Web | React Native | +|---|---|---| +| Renderer | `@react-three/fiber` (DOM ``) | `@react-three/fiber/native` (`expo-gl`) | +| Audio | `HTMLAudioElement` | `expo-av` (base64 WAV written to a temp file) | +| Play control | `showPlayVoiceButton` or `ref` | `ref` only (the built-in button is DOM) | + +A full runnable example lives in [`examples/native`](./examples/native). Note: +HDR `environmentPreset` lighting is more limited under `@react-three/drei/native`, +and R3F's native entry is most battle-tested on the v8 line (React 18 / Expo 51). + ## ๐Ÿงฉ API ### `Avatoon(props)` diff --git a/examples/native/App.tsx b/examples/native/App.tsx new file mode 100644 index 0000000..0085d46 --- /dev/null +++ b/examples/native/App.tsx @@ -0,0 +1,54 @@ +import { useRef, useState } from 'react'; +import { View, Pressable, Text, StyleSheet } from 'react-native'; +import { Avatoon, type AvatoonHandle } from 'avatoon'; + +/** + * Minimal Expo example for avatoon on React Native. + * + * The 3D scene renders through `@react-three/fiber/native` (expo-gl) and audio + * plays through `expo-av` โ€” both selected automatically via the package's + * `react-native` export condition. Playback is driven imperatively through the + * `ref`, since the built-in DOM play button is web-only. + */ +export default function App() { + const avatar = useRef(null); + const [playing, setPlaying] = useState(false); + + const toggle = () => { + avatar.current?.toggle(); + setPlaying(p => !p); + }; + + return ( + + + + + + {playing ? 'โน Stop talking' : 'โ–ถ Talk with me'} + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#f0f0f0' }, + button: { + position: 'absolute', + bottom: 40, + alignSelf: 'center', + paddingVertical: 12, + paddingHorizontal: 24, + borderRadius: 25, + backgroundColor: '#3b82f6', + }, + buttonText: { color: '#fff', fontWeight: '700', fontSize: 16 }, +}); diff --git a/examples/native/README.md b/examples/native/README.md new file mode 100644 index 0000000..17fd4c2 --- /dev/null +++ b/examples/native/README.md @@ -0,0 +1,45 @@ +# avatoon โ€” React Native (Expo) example + +Minimal Expo app rendering `` on React Native. The 3D scene runs on +`@react-three/fiber/native` (via `expo-gl`) and audio plays through `expo-av`. +avatoon picks these automatically through its `react-native` package export +condition โ€” you import from `'avatoon'` exactly as on web. + +## Setup + +```bash +npx create-expo-app avatoon-native --template blank-typescript +cd avatoon-native + +# avatoon + the native peers it needs +npx expo install expo-gl expo-av expo-asset expo-file-system \ + @react-three/fiber @react-three/drei three +npm install avatoon +``` + +Then drop [`App.tsx`](./App.tsx) into your project and run: + +```bash +npx expo start +``` + +## How it differs from web + +| | Web | React Native | +|---|---|---| +| Renderer | `@react-three/fiber` (DOM ``) | `@react-three/fiber/native` (`expo-gl`) | +| Audio | `HTMLAudioElement` | `expo-av` (base64 WAV โ†’ temp file via `expo-file-system`) | +| Play control | `showPlayVoiceButton` or `ref` | `ref` only (the built-in button is DOM) | + +## Notes & caveats + +- **Playback is imperative on native.** The DOM play button (`showPlayVoiceButton`) + is web-only; drive playback with the `ref` (`play()` / `stop()` / `toggle()`) + and your own RN `Pressable`, as in `App.tsx`. +- **`environmentPreset` / HDR lighting** relies on `@react-three/drei/native`, + whose `Environment` support is more limited than on web. Prefer simple + lighting (`ambientLight` is added for you) if you hit issues. +- **Models** load over the network via `useGLTF`; make sure the `.glb` URL is + reachable from the device/simulator. +- R3F's native entry is most battle-tested on the v8 line (React 18 / Expo 51), + which is what this example pins. diff --git a/examples/native/package.json b/examples/native/package.json new file mode 100644 index 0000000..7470c27 --- /dev/null +++ b/examples/native/package.json @@ -0,0 +1,29 @@ +{ + "name": "avatoon-native-example", + "version": "1.0.0", + "private": true, + "description": "Minimal Expo example for avatoon on React Native", + "main": "node_modules/expo/AppEntry.js", + "scripts": { + "start": "expo start", + "android": "expo start --android", + "ios": "expo start --ios" + }, + "dependencies": { + "avatoon": "^1.9.0", + "expo": "~51.0.0", + "expo-asset": "~10.0.0", + "expo-av": "~14.0.0", + "expo-file-system": "~17.0.0", + "expo-gl": "~14.0.0", + "@react-three/drei": "^9.0.0", + "@react-three/fiber": "^8.17.0", + "react": "18.2.0", + "react-native": "0.74.0", + "three": "^0.166.0" + }, + "devDependencies": { + "@types/react": "~18.2.0", + "typescript": "~5.3.0" + } +} diff --git a/package.json b/package.json index 257b99a..ac87a6f 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "exports": { ".": { "types": "./lib/index.d.ts", + "react-native": "./lib/native/index.es.js", "require": "./lib/Avatoon.umd.js", "import": "./lib/Avatoon.es.js" } @@ -17,7 +18,8 @@ "LICENSE" ], "scripts": { - "build": "tsc --project tsconfig.build.json && vite build", + "build": "tsc --project tsconfig.build.json && vite build && npm run build:native", + "build:native": "vite build --config vite.native.config.mjs", "dev": "tsc --project tsconfig.build.json && vite build --watch", "clean": "rm -rf ./lib/", "lint": "eslint ./src/ --fix", @@ -53,8 +55,14 @@ "peerDependencies": { "@react-three/drei": ">=9.0.0", "@react-three/fiber": ">=8.0.0", + "expo": "*", + "expo-asset": "*", + "expo-av": "*", + "expo-file-system": "*", + "expo-gl": "*", "react": ">=18", "react-dom": ">=18", + "react-native": "*", "three": ">=0.153.0" }, "peerDependenciesMeta": { @@ -62,7 +70,25 @@ "optional": false }, "react-dom": { - "optional": false + "optional": true + }, + "react-native": { + "optional": true + }, + "expo": { + "optional": true + }, + "expo-asset": { + "optional": true + }, + "expo-av": { + "optional": true + }, + "expo-file-system": { + "optional": true + }, + "expo-gl": { + "optional": true } }, "devDependencies": { diff --git a/src/audio/expoAudioClock.ts b/src/audio/expoAudioClock.ts new file mode 100644 index 0000000..7cc4aad --- /dev/null +++ b/src/audio/expoAudioClock.ts @@ -0,0 +1,101 @@ +import { Audio } from 'expo-av'; +import * as FileSystem from 'expo-file-system'; +import type { AudioClock, AudioSource, CreateAudioClock } from './types'; + +// Unique-per-clip temp filenames without relying on Date/Math at module load. +let fileCounter = 0; + +/** + * React Native {@link AudioClock} backed by `expo-av`. + * + * Two platform differences from the web version are handled here: + * 1. `expo-av` cannot reliably play `data:` URIs, so a base64 WAV is written to + * a temp file via `expo-file-system` first. + * 2. There is no synchronous `currentTime`; position arrives via playback + * status updates. We cache the last reported position and interpolate with + * wall-clock time so `getCurrentTime()` stays smooth every frame. + */ +export const createAudioClock: CreateAudioClock = (source: AudioSource) => { + const inlineUri = source.uri ?? null; + const base64 = source.audioBase64 ?? null; + if (!inlineUri && !base64) return null; + + let sound: Audio.Sound | null = null; + let loading: Promise | null = null; + let disposed = false; + + let positionSec = 0; + let lastSyncAt = 0; // Date.now() when positionSec was last reported + let playing = false; + + const now = () => (typeof Date !== 'undefined' ? Date.now() : 0); + + const resolveUri = async (): Promise => { + if (inlineUri) return inlineUri; + const fileUri = `${FileSystem.cacheDirectory}avatoon-${fileCounter++}.wav`; + await FileSystem.writeAsStringAsync(fileUri, base64 as string, { + encoding: FileSystem.EncodingType.Base64, + }); + return fileUri; + }; + + const ensureLoaded = async (): Promise => { + if (sound || disposed) return; + if (!loading) { + loading = (async () => { + const uri = await resolveUri(); + if (disposed) return; + const created = await Audio.Sound.createAsync( + { uri }, + { shouldPlay: false } + ); + if (disposed) { + await created.sound.unloadAsync(); + return; + } + sound = created.sound; + sound.setOnPlaybackStatusUpdate(status => { + if (!status.isLoaded) return; + positionSec = (status.positionMillis ?? 0) / 1000; + lastSyncAt = now(); + playing = status.isPlaying ?? false; + }); + })(); + } + await loading; + }; + + const clock: AudioClock = { + play: async () => { + await ensureLoaded(); + if (!sound || disposed) return; + playing = true; + lastSyncAt = now(); + await sound.playAsync(); + }, + pause: () => { + playing = false; + void sound?.pauseAsync(); + }, + getCurrentTime: () => { + // Interpolate between status updates so lip-sync doesn't step at the + // status interval (~100ms). + if (playing && lastSyncAt) { + return positionSec + (now() - lastSyncAt) / 1000; + } + return positionSec; + }, + dispose: () => { + disposed = true; + playing = false; + const s = sound; + sound = null; + if (s) { + s.setOnPlaybackStatusUpdate(null); + void s.unloadAsync(); + } + }, + }; + + return clock; +}; diff --git a/src/audio/types.ts b/src/audio/types.ts new file mode 100644 index 0000000..1bc958a --- /dev/null +++ b/src/audio/types.ts @@ -0,0 +1,37 @@ +/** + * Platform-agnostic audio boundary used to drive lip-sync. + * + * The web build backs this with an `HTMLAudioElement`; the React Native build + * backs it with `expo-av`. `AvatoonModel` only ever talks to this interface, so + * the lip-sync clock (`getCurrentTime`) works identically on both platforms. + */ +export interface AudioClock { + /** + * Start (or resume) playback. Resolves once playback has begun; may reject if + * the platform blocks autoplay until a user gesture. + */ + play(): Promise; + /** Pause playback, retaining the current position. */ + pause(): void; + /** + * Current playback position in **seconds**. Read every frame to align visemes + * with the audio. Returns `0` before playback starts. + */ + getCurrentTime(): number; + /** Stop playback and release all underlying resources. */ + dispose(): void; +} + +/** Where an {@link AudioClock} sources its audio from. */ +export interface AudioSource { + /** Base64-encoded WAV payload (no data-URI prefix). */ + audioBase64?: string; + /** A direct audio URL/URI (takes precedence over {@link audioBase64}). */ + uri?: string; +} + +/** + * Factory for an {@link AudioClock}. Returns `null` when audio is unavailable + * (e.g. server-side rendering, or no source provided), so callers can no-op. + */ +export type CreateAudioClock = (source: AudioSource) => AudioClock | null; diff --git a/src/audio/webAudioClock.ts b/src/audio/webAudioClock.ts new file mode 100644 index 0000000..dbc3a83 --- /dev/null +++ b/src/audio/webAudioClock.ts @@ -0,0 +1,32 @@ +import type { AudioClock, CreateAudioClock } from './types'; + +/** + * Web {@link AudioClock} backed by `HTMLAudioElement`. Preserves the original + * `AvatoonModel` behaviour: a base64 WAV is played from a `data:` URI and the + * element's `currentTime` drives the viseme clock. + */ +export const createAudioClock: CreateAudioClock = source => { + // Guard for SSR / non-DOM environments โ€” mirrors the previous inline check. + if (typeof window === 'undefined' || typeof Audio === 'undefined') { + return null; + } + + const uri = + source.uri ?? + (source.audioBase64 ? `data:audio/wav;base64,${source.audioBase64}` : null); + if (!uri) return null; + + const el = new Audio(uri); + + const clock: AudioClock = { + play: () => el.play(), + pause: () => el.pause(), + getCurrentTime: () => el.currentTime, + dispose: () => { + el.pause(); + el.src = ''; + }, + }; + + return clock; +}; diff --git a/src/components/Avatoon.tsx b/src/components/Avatoon.tsx index dede4da..06a2375 100644 --- a/src/components/Avatoon.tsx +++ b/src/components/Avatoon.tsx @@ -5,8 +5,7 @@ import { useImperativeHandle, type CSSProperties, } from 'react'; -import { Canvas } from '@react-three/fiber'; -import { OrbitControls } from '@react-three/drei'; +import { Canvas, OrbitControls } from '../platform'; import { AvatoonModel } from './AvatoonModel'; import { CameraFovAnimator } from './CameraFovAnimator'; import { AvatoonErrorBoundary } from './AvatoonErrorBoundary'; diff --git a/src/components/AvatoonModel.tsx b/src/components/AvatoonModel.tsx index 201aff5..07464a1 100644 --- a/src/components/AvatoonModel.tsx +++ b/src/components/AvatoonModel.tsx @@ -1,6 +1,6 @@ import { useRef, useEffect, useState, useMemo } from 'react'; -import { useFrame, useThree } from '@react-three/fiber'; -import { useGLTF } from '@react-three/drei'; +import { useFrame, useThree, useGLTF, createAudioClock } from '../platform'; +import type { AudioClock } from '../platform'; import * as THREE from 'three'; import { Bone, Group, Object3D } from 'three'; import { GLTF } from 'three/examples/jsm/loaders/GLTFLoader'; @@ -24,7 +24,7 @@ export function AvatoonModel({ const [visemeData, setVisemeData] = useState< Array<{ time: number; viseme: string | null }> >([]); - const audioRef = useRef(null); + const audioRef = useRef(null); const group = useRef(null); const { scene: gltfScene } = useGLTF(url) as unknown as GLTF; @@ -63,16 +63,13 @@ export function AvatoonModel({ setVisemeData(parseVisemes(visemeJson, phonemeToViseme)); - if ( - visemeJson && - visemeJson.audio_base64 && - typeof window !== 'undefined' && - typeof Audio !== 'undefined' - ) { - const audio = new Audio( - `data:audio/wav;base64,${visemeJson.audio_base64}` - ); - audioRef.current = audio; + if (visemeJson && visemeJson.audio_base64) { + // Platform-specific clock (HTMLAudio on web, expo-av on native). + // Returns null when audio is unavailable (e.g. SSR), so we no-op. + const clock = createAudioClock({ + audioBase64: visemeJson.audio_base64, + }); + if (clock) audioRef.current = clock; } }; @@ -83,8 +80,7 @@ export function AvatoonModel({ // clip playing and overlapping with the new one. return () => { if (audioRef.current) { - audioRef.current.pause(); - audioRef.current.src = ''; + audioRef.current.dispose(); audioRef.current = null; } }; @@ -227,7 +223,7 @@ export function AvatoonModel({ if (!isTalking || !audioRef.current || mouthMeshes.current.length === 0) return; - const currentTime = audioRef.current.currentTime; + const currentTime = audioRef.current.getCurrentTime(); // Find the most recent viseme const current = findActiveViseme(visemeData, currentTime); diff --git a/src/components/CameraFovAnimator.tsx b/src/components/CameraFovAnimator.tsx index 2e5a349..1ed2df4 100644 --- a/src/components/CameraFovAnimator.tsx +++ b/src/components/CameraFovAnimator.tsx @@ -1,4 +1,4 @@ -import { useFrame, useThree } from '@react-three/fiber'; +import { useFrame, useThree } from '../platform'; import { useEffect, useState } from 'react'; import * as THREE from 'three'; diff --git a/src/components/LipSyncAvatoon.tsx b/src/components/LipSyncAvatoon.tsx index 13060e9..46fb64a 100644 --- a/src/components/LipSyncAvatoon.tsx +++ b/src/components/LipSyncAvatoon.tsx @@ -1,6 +1,5 @@ import { useState, useRef, useEffect, useMemo, Suspense } from 'react'; -import { Canvas, useFrame } from '@react-three/fiber'; -import { OrbitControls, useGLTF } from '@react-three/drei'; +import { Canvas, useFrame, OrbitControls, useGLTF } from '../platform'; import * as THREE from 'three'; import { clone as cloneSkeleton } from 'three/examples/jsm/utils/SkeletonUtils'; import { AvatoonErrorBoundary } from './AvatoonErrorBoundary'; diff --git a/src/components/SceneEnvironment.tsx b/src/components/SceneEnvironment.tsx index aa4a6cc..b630b8c 100644 --- a/src/components/SceneEnvironment.tsx +++ b/src/components/SceneEnvironment.tsx @@ -1,4 +1,4 @@ -import { Environment } from '@react-three/drei'; +import { Environment } from '../platform'; import type { AvatoonEnvironmentProps } from '../types'; /** diff --git a/src/platform/index.native.tsx b/src/platform/index.native.tsx new file mode 100644 index 0000000..30fae71 --- /dev/null +++ b/src/platform/index.native.tsx @@ -0,0 +1,14 @@ +/** + * React Native platform bindings (resolved by the native build via extension + * priority: `.native.tsx` wins over `.ts`). + * + * These come from the `/native` entry points, which render through `expo-gl` + * instead of a DOM ``. The consuming app must install the native-capable + * peers (see `package.json` `peerDependenciesMeta`): `@react-three/fiber` + * (native entry), `@react-three/drei`, `expo`, `expo-gl`, `expo-av`, + * `expo-asset`, and `expo-file-system`. + */ +export { Canvas, useFrame, useThree } from '@react-three/fiber/native'; +export { OrbitControls, useGLTF, Environment } from '@react-three/drei/native'; +export { createAudioClock } from '../audio/expoAudioClock'; +export type { AudioClock, AudioSource } from '../audio/types'; diff --git a/src/platform/index.ts b/src/platform/index.ts new file mode 100644 index 0000000..98e7523 --- /dev/null +++ b/src/platform/index.ts @@ -0,0 +1,12 @@ +/** + * Web platform bindings. + * + * Every renderer/audio dependency that differs between web and React Native is + * funnelled through this module. The native build swaps in `index.native.tsx` + * (resolved by extension priority), so the component tree itself stays + * platform-agnostic โ€” it only ever imports from `../platform`. + */ +export { Canvas, useFrame, useThree } from '@react-three/fiber'; +export { OrbitControls, useGLTF, Environment } from '@react-three/drei'; +export { createAudioClock } from '../audio/webAudioClock'; +export type { AudioClock, AudioSource } from '../audio/types'; diff --git a/src/types/native-modules.d.ts b/src/types/native-modules.d.ts new file mode 100644 index 0000000..fef8a5b --- /dev/null +++ b/src/types/native-modules.d.ts @@ -0,0 +1,54 @@ +/** + * Ambient shims for the React Native / Expo peers. + * + * These packages are only used by the native build (`*.native.tsx` + + * `expoAudioClock.ts`), where they are kept external and resolved from the + * consuming app's own install. Declaring them here lets `tsc --noEmit` and the + * native bundle build succeed without adding heavy Expo dev dependencies to + * this library. + * + * The `/native` entry points expose the same API surface as their web + * counterparts, so we simply re-export the real (installed) types. + */ +declare module '@react-three/fiber/native' { + export * from '@react-three/fiber'; +} + +declare module '@react-three/drei/native' { + export * from '@react-three/drei'; +} + +declare module 'expo-av' { + export namespace Audio { + interface PlaybackStatus { + isLoaded: boolean; + positionMillis?: number; + isPlaying?: boolean; + } + class Sound { + static createAsync( + source: { uri: string }, + initialStatus?: { shouldPlay?: boolean } + ): Promise<{ sound: Sound }>; + setOnPlaybackStatusUpdate( + cb: ((status: PlaybackStatus) => void) | null + ): void; + playAsync(): Promise; + pauseAsync(): Promise; + unloadAsync(): Promise; + } + } +} + +declare module 'expo-file-system' { + export const cacheDirectory: string | null; + export enum EncodingType { + UTF8 = 'utf8', + Base64 = 'base64', + } + export function writeAsStringAsync( + fileUri: string, + contents: string, + options?: { encoding?: EncodingType } + ): Promise; +} diff --git a/test/webAudioClock.spec.ts b/test/webAudioClock.spec.ts new file mode 100644 index 0000000..01e38b2 --- /dev/null +++ b/test/webAudioClock.spec.ts @@ -0,0 +1,28 @@ +import { createAudioClock } from '../src/audio/webAudioClock'; + +describe('createAudioClock (web)', () => { + it('returns null when no source is provided', () => { + expect(createAudioClock({})).toBeNull(); + }); + + it('builds a clock from a base64 WAV source', () => { + const clock = createAudioClock({ audioBase64: 'UklGRg==' }); + expect(clock).not.toBeNull(); + // Position starts at 0 before playback. + expect(clock!.getCurrentTime()).toBe(0); + }); + + it('builds a clock from a direct uri source', () => { + const clock = createAudioClock({ uri: 'https://example.com/a.wav' }); + expect(clock).not.toBeNull(); + expect(typeof clock!.getCurrentTime()).toBe('number'); + }); + + it('pause and dispose are safe to call without throwing', () => { + const clock = createAudioClock({ audioBase64: 'UklGRg==' }); + expect(() => { + clock!.pause(); + clock!.dispose(); + }).not.toThrow(); + }); +}); diff --git a/vite.native.config.mjs b/vite.native.config.mjs new file mode 100644 index 0000000..4b2b10f --- /dev/null +++ b/vite.native.config.mjs @@ -0,0 +1,40 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +// Native (React Native / Expo) bundle. +// +// Same entry and component tree as the web build, but `resolve.extensions` +// prioritises `.native.*` files so `../platform` resolves to +// `platform/index.native.tsx` (fiber/native + expo-av) instead of the web +// module. Every non-relative import is externalised, so the heavy native peers +// (fiber/native, drei/native, expo-*) are resolved from the consuming app and +// need not be installed here. +export default defineConfig({ + plugins: [react()], + resolve: { + extensions: [ + '.native.tsx', + '.native.ts', + '.native.jsx', + '.native.js', + '.tsx', + '.ts', + '.jsx', + '.js', + '.json', + ], + }, + build: { + outDir: 'lib/native', + emptyOutDir: true, + lib: { + entry: './src/index.ts', + formats: ['es'], + fileName: format => `index.${format}.js`, + }, + rollupOptions: { + // Externalise everything that isn't part of this package's own source. + external: id => !id.startsWith('.') && !id.startsWith('/'), + }, + }, +}); From 824785f105ec36f0e6b2987d3166f3925db865a0 Mon Sep 17 00:00:00 2001 From: Khaled Alam Date: Wed, 15 Jul 2026 18:48:11 +0400 Subject: [PATCH 2/3] =?UTF-8?q?test:=20cover=20native=20audio=20clock,=20r?= =?UTF-8?q?ef/goal=20controls=20and=20frame=20loops=20(80%=E2=86=9296%);?= =?UTF-8?q?=20add=20RN=20goals=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/native/GoalsExample.tsx | 61 +++++++++++++++++++ examples/native/README.md | 8 ++- jest.config.js | 4 ++ test/Avatoon.control.spec.tsx | 71 ++++++++++++++++++++++ test/AvatoonModel.spec.tsx | 49 ++++++++++++++++ test/CameraFovAnimator.spec.tsx | 34 ++++++++--- test/LipSyncAvatoon.spec.tsx | 30 ++++++++++ test/__mocks__/expo-av.ts | 34 +++++++++++ test/__mocks__/expo-file-system.ts | 10 ++++ test/expoAudioClock.spec.ts | 94 ++++++++++++++++++++++++++++++ 10 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 examples/native/GoalsExample.tsx create mode 100644 test/Avatoon.control.spec.tsx create mode 100644 test/__mocks__/expo-av.ts create mode 100644 test/__mocks__/expo-file-system.ts create mode 100644 test/expoAudioClock.spec.ts diff --git a/examples/native/GoalsExample.tsx b/examples/native/GoalsExample.tsx new file mode 100644 index 0000000..22d07ea --- /dev/null +++ b/examples/native/GoalsExample.tsx @@ -0,0 +1,61 @@ +import { useState } from 'react'; +import { View, Pressable, Text, StyleSheet } from 'react-native'; +import { Avatoon, type AvatoonGoal } from 'avatoon'; + +/** + * React Native example showing goal-based gestures (Normal / Muscle / Sleep). + * + * Switching `goal` re-poses the avatar via the same morph/bone logic as web โ€” + * no audio required. Handy for idle states, reactions, or emotes. + */ +const GOALS: AvatoonGoal[] = ['Normal', 'Muscle', 'Sleep']; + +export default function GoalsExample() { + const [goal, setGoal] = useState('Normal'); + + return ( + + + + + {GOALS.map(g => ( + setGoal(g)} + style={[styles.chip, goal === g && styles.chipActive]} + > + + {g} + + + ))} + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#f0f0f0' }, + row: { + position: 'absolute', + bottom: 40, + left: 0, + right: 0, + flexDirection: 'row', + justifyContent: 'center', + gap: 10, + }, + chip: { + paddingVertical: 10, + paddingHorizontal: 18, + borderRadius: 20, + backgroundColor: '#e2e8f0', + }, + chipActive: { backgroundColor: '#3b82f6' }, + chipText: { color: '#334155', fontWeight: '600' }, + chipTextActive: { color: '#fff' }, +}); diff --git a/examples/native/README.md b/examples/native/README.md index 17fd4c2..9e3f661 100644 --- a/examples/native/README.md +++ b/examples/native/README.md @@ -17,12 +17,18 @@ npx expo install expo-gl expo-av expo-asset expo-file-system \ npm install avatoon ``` -Then drop [`App.tsx`](./App.tsx) into your project and run: +Then drop one of the example screens into your project and run: ```bash npx expo start ``` +## Examples + +- [`App.tsx`](./App.tsx) โ€” audio-synced lip-sync driven imperatively via a `ref`. +- [`GoalsExample.tsx`](./GoalsExample.tsx) โ€” goal-based gestures (Normal / Muscle + / Sleep) with no audio, switched from RN buttons. + ## How it differs from web | | Web | React Native | diff --git a/jest.config.js b/jest.config.js index cd08c30..74fa4ae 100644 --- a/jest.config.js +++ b/jest.config.js @@ -17,6 +17,10 @@ module.exports = { '\\.(css|less|scss|sass)$': 'identity-obj-proxy', '^three/examples/jsm/utils/SkeletonUtils$': '/test/__mocks__/skeletonUtils.ts', + // Optional native peers aren't installed; map them to manual mocks so the + // React Native audio clock can be unit-tested in jsdom. + '^expo-av$': '/test/__mocks__/expo-av.ts', + '^expo-file-system$': '/test/__mocks__/expo-file-system.ts', }, setupFilesAfterEnv: ['./jest.setup.ts'], }; diff --git a/test/Avatoon.control.spec.tsx b/test/Avatoon.control.spec.tsx new file mode 100644 index 0000000..3df6723 --- /dev/null +++ b/test/Avatoon.control.spec.tsx @@ -0,0 +1,71 @@ +import { createRef } from 'react'; +import { render, screen, act, fireEvent } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import Avatoon from '../src/components/Avatoon'; +import type { AvatoonHandle } from '../src/types'; + +// Stub the platform seam so the scene renders in jsdom (no real WebGL), and the +// model fires onRenderComplete on mount so the built-in button becomes visible. +jest.mock('../src/platform', () => ({ + Canvas: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + OrbitControls: () => null, + useFrame: () => {}, + useThree: () => ({ camera: {}, gl: {} }), + Environment: () => null, + createAudioClock: () => null, +})); + +jest.mock('../src/components/AvatoonModel', () => ({ + AvatoonModel: ({ onRenderComplete }: { onRenderComplete?: () => void }) => { + const { useEffect } = require('react'); + useEffect(() => onRenderComplete?.(), []); + return null; + }, +})); + +describe('Avatoon imperative control', () => { + it('shows the play button after first render and toggles on click', () => { + render( + + ); + + const button = screen.getByRole('button'); + expect(button).toHaveTextContent('Talk with me now'); + + fireEvent.click(button); + expect(button).toHaveTextContent('Stop talking'); + + fireEvent.click(button); + expect(button).toHaveTextContent('Talk with me now'); + }); + + it('drives playback state through the ref handle', () => { + const ref = createRef(); + render( + + ); + + const button = screen.getByRole('button'); + + act(() => ref.current!.play()); + expect(button).toHaveTextContent('Stop talking'); + + act(() => ref.current!.stop()); + expect(button).toHaveTextContent('Talk with me now'); + + act(() => ref.current!.toggle()); + expect(button).toHaveTextContent('Stop talking'); + }); + + it('hides the button when showPlayVoiceButton is false', () => { + render(); + expect(screen.queryByRole('button')).toBeNull(); + }); +}); diff --git a/test/AvatoonModel.spec.tsx b/test/AvatoonModel.spec.tsx index 31dfc15..ec21e57 100644 --- a/test/AvatoonModel.spec.tsx +++ b/test/AvatoonModel.spec.tsx @@ -121,4 +121,53 @@ describe('AvatoonModel', () => { runFrames({ clock: { getElapsedTime: () => 1 } }) ).not.toThrow(); }); + + it('plays audio and drives mouth visemes when shouldPlay is true', async () => { + await act(async () => { + render( + + ); + }); + + // isTalking is now true โ†’ the viseme-driving frame reads the audio clock + // (stub currentTime = 0.5) and applies morph influences without throwing. + expect(() => + runFrames({ clock: { getElapsedTime: () => 0.5 } }) + ).not.toThrow(); + }); + + it('stops talking when shouldPlay flips to false', async () => { + const props = { + url: 'dummy.glb', + goal: 'Normal' as const, + visemeJson: { + visemes: [{ time: 0, viseme: 'A' }], + audio_base64: 'UklGRg==', + }, + onRenderComplete: jest.fn(), + }; + let rerender: (ui: React.ReactElement) => void = () => {}; + await act(async () => { + ({ rerender } = render()); + }); + await act(async () => { + rerender(); + }); + expect(() => + runFrames({ clock: { getElapsedTime: () => 0.5 } }) + ).not.toThrow(); + }); }); diff --git a/test/CameraFovAnimator.spec.tsx b/test/CameraFovAnimator.spec.tsx index 2f2ba93..9bc64ea 100644 --- a/test/CameraFovAnimator.spec.tsx +++ b/test/CameraFovAnimator.spec.tsx @@ -2,10 +2,8 @@ import { render } from '@testing-library/react'; import { CameraFovAnimator } from '../src/components/CameraFovAnimator'; import * as THREE from 'three'; -let mockCamera = { - fov: 15, - updateProjectionMatrix: jest.fn(), -} as unknown as THREE.PerspectiveCamera; +// A real PerspectiveCamera so the `instanceof` guard in the frame loop passes. +let mockCamera = new THREE.PerspectiveCamera(15, 1, 0.1, 1000); let useFrameCallback: | ((state: { camera: THREE.PerspectiveCamera }) => void) @@ -26,13 +24,31 @@ describe('CameraFovAnimator', () => { useFrameCallback = null; }); - // it('animates FOV to 17 when readyToPlay is true', () => { - // render(); - // expect(mockCamera.fov).toBe(17); - // }); - it('animates FOV to 15 when readyToPlay is false', () => { render(); expect(mockCamera.fov).toBe(15); }); + + it('lerps fov toward the target and updates the projection on each frame', () => { + mockCamera.fov = 15; + render(); + const spy = jest.spyOn(mockCamera, 'updateProjectionMatrix'); + + // readyToPlay=true โ†’ target is baseFov (24); fov should climb from 15. + useFrameCallback!({ camera: mockCamera }); + + expect(mockCamera.fov).toBeGreaterThan(15); + expect(mockCamera.fov).toBeLessThan(24); + expect(spy).toHaveBeenCalled(); + }); + + it('ignores non-perspective cameras', () => { + render(); + const ortho = new THREE.OrthographicCamera(); + expect(() => + useFrameCallback!({ + camera: ortho as unknown as THREE.PerspectiveCamera, + }) + ).not.toThrow(); + }); }); diff --git a/test/LipSyncAvatoon.spec.tsx b/test/LipSyncAvatoon.spec.tsx index efaebfd..ca9790f 100644 --- a/test/LipSyncAvatoon.spec.tsx +++ b/test/LipSyncAvatoon.spec.tsx @@ -83,4 +83,34 @@ describe('LipSyncAvatoon', () => { render(); expect(screen.getByTestId('lipsync-canvas')).toBeInTheDocument(); }); + + it('applies a blink pulse when the blink window opens', () => { + render(); + // First blink fires at ~3s; land just inside the 0.15s blink window so the + // pulse value is non-zero and gets written to the blink morph target. + expect(() => + runFrames({ clock: { getElapsedTime: () => 3.05 } }, 3.05) + ).not.toThrow(); + }); + + it('lerps the mouth closed on idle frames after talking', () => { + render(); + + // Talk to raise mouth influences, then stop and idle so they lerp down. + fireEvent.click(screen.getByRole('button', { name: /start talking/i })); + runFrames({ clock: { getElapsedTime: () => 5 } }, 0.2); + fireEvent.click(screen.getByRole('button', { name: /stop talking/i })); + expect(() => + runFrames({ clock: { getElapsedTime: () => 6 } }, 0.2) + ).not.toThrow(); + }); + + it('handles button hover in/out', () => { + render(); + const button = screen.getByRole('button', { name: /start talking/i }); + expect(() => { + fireEvent.mouseOver(button); + fireEvent.mouseOut(button); + }).not.toThrow(); + }); }); diff --git a/test/__mocks__/expo-av.ts b/test/__mocks__/expo-av.ts new file mode 100644 index 0000000..9ceaf4f --- /dev/null +++ b/test/__mocks__/expo-av.ts @@ -0,0 +1,34 @@ +// Manual mock of `expo-av` for jest (the real package isn't installed โ€” it's an +// optional native peer). Lets us unit-test the React Native audio clock in +// jsdom and drive playback-status updates manually. + +export const __soundInstances: MockSound[] = []; + +export class MockSound { + _cb: ((s: unknown) => void) | null = null; + playAsync = jest.fn(async () => {}); + pauseAsync = jest.fn(async () => {}); + unloadAsync = jest.fn(async () => {}); + setOnPlaybackStatusUpdate = jest.fn((cb: ((s: unknown) => void) | null) => { + this._cb = cb; + }); + + /** Test helper: emit a playback status update to the registered callback. */ + __emit(status: unknown) { + this._cb?.(status); + } + + static createAsync = jest.fn(async () => { + const sound = new MockSound(); + __soundInstances.push(sound); + return { sound }; + }); +} + +export const Audio = { Sound: MockSound }; + +/** Test helper: reset recorded instances and mock call history. */ +export function __reset() { + __soundInstances.length = 0; + MockSound.createAsync.mockClear(); +} diff --git a/test/__mocks__/expo-file-system.ts b/test/__mocks__/expo-file-system.ts new file mode 100644 index 0000000..597d54c --- /dev/null +++ b/test/__mocks__/expo-file-system.ts @@ -0,0 +1,10 @@ +// Manual mock of `expo-file-system` for jest (optional native peer). + +export const cacheDirectory = 'file:///cache/'; + +export enum EncodingType { + UTF8 = 'utf8', + Base64 = 'base64', +} + +export const writeAsStringAsync = jest.fn(async () => {}); diff --git a/test/expoAudioClock.spec.ts b/test/expoAudioClock.spec.ts new file mode 100644 index 0000000..3765e9a --- /dev/null +++ b/test/expoAudioClock.spec.ts @@ -0,0 +1,94 @@ +import { createAudioClock } from '../src/audio/expoAudioClock'; +// Import through the same module mapper the code-under-test uses, so we share +// the one mock singleton (jest.requireMock would hand back a separate copy). +import * as ExpoAv from 'expo-av'; +import * as ExpoFs from 'expo-file-system'; + +/* eslint-disable @typescript-eslint/no-explicit-any */ +const expoAv: any = ExpoAv; +const fs: any = ExpoFs; + +describe('createAudioClock (native / expo-av)', () => { + beforeEach(() => { + expoAv.__reset(); + fs.writeAsStringAsync.mockClear(); + }); + + it('returns null when no source is provided', () => { + expect(createAudioClock({})).toBeNull(); + }); + + it('writes a base64 WAV to a temp file and plays it', async () => { + const clock = createAudioClock({ audioBase64: 'UklGRg==' })!; + expect(clock).not.toBeNull(); + + await clock.play(); + + // base64 payload written to the cache dir with Base64 encoding + expect(fs.writeAsStringAsync).toHaveBeenCalledTimes(1); + const [uri, contents, opts] = fs.writeAsStringAsync.mock.calls[0]; + expect(uri).toContain('file:///cache/'); + expect(contents).toBe('UklGRg=='); + expect(opts.encoding).toBe('base64'); + + // sound created and played + expect(expoAv.Audio.Sound.createAsync).toHaveBeenCalledTimes(1); + expect(expoAv.__soundInstances[0].playAsync).toHaveBeenCalledTimes(1); + }); + + it('plays a direct uri source without writing a temp file', async () => { + const clock = createAudioClock({ uri: 'https://example.com/a.wav' })!; + await clock.play(); + expect(fs.writeAsStringAsync).not.toHaveBeenCalled(); + const [source] = expoAv.Audio.Sound.createAsync.mock.calls[0]; + expect(source.uri).toBe('https://example.com/a.wav'); + }); + + it('reports 0 before any playback status arrives', () => { + const clock = createAudioClock({ uri: 'x' })!; + expect(clock.getCurrentTime()).toBe(0); + }); + + it('tracks and interpolates position from status updates', async () => { + const clock = createAudioClock({ uri: 'x' })!; + await clock.play(); + expoAv.__soundInstances[0].__emit({ + isLoaded: true, + positionMillis: 2000, + isPlaying: true, + }); + // At least the reported 2s, plus any interpolated wall-clock delta. + expect(clock.getCurrentTime()).toBeGreaterThanOrEqual(2); + }); + + it('pauses without interpolating further', async () => { + const clock = createAudioClock({ uri: 'x' })!; + await clock.play(); + expoAv.__soundInstances[0].__emit({ + isLoaded: true, + positionMillis: 1500, + isPlaying: false, + }); + clock.pause(); + expect(expoAv.__soundInstances[0].pauseAsync).toHaveBeenCalled(); + expect(clock.getCurrentTime()).toBeCloseTo(1.5, 5); + }); + + it('ignores unloaded status updates', async () => { + const clock = createAudioClock({ uri: 'x' })!; + await clock.play(); + expoAv.__soundInstances[0].__emit({ isLoaded: false }); + // paused (isPlaying never became true via a loaded status) โ†’ stays 0 + clock.pause(); + expect(clock.getCurrentTime()).toBe(0); + }); + + it('unloads the sound on dispose', async () => { + const clock = createAudioClock({ uri: 'x' })!; + await clock.play(); + const sound = expoAv.__soundInstances[0]; + clock.dispose(); + expect(sound.setOnPlaybackStatusUpdate).toHaveBeenLastCalledWith(null); + expect(sound.unloadAsync).toHaveBeenCalled(); + }); +}); From df4c9fad9e93e1f0ede891902ae49292ff03eda5 Mon Sep 17 00:00:00 2001 From: Khaled Alam Date: Wed, 15 Jul 2026 19:11:45 +0400 Subject: [PATCH 3/3] ci: eject PR workflow to node 18/20 matrix (vite 5 needs node >=18, drop node 16) --- .github/workflows/pr.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5699ad8..420a077 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,9 +1,33 @@ -# Reusable workflow for PRs; to eject, you can replace this file with -# https://github.com/ryansonshine/ryansonshine/blob/main/.github/workflows/pr.yml name: ๐Ÿ”– Pull Request +# Ejected from the ryansonshine reusable workflow: it ran a Node 16 matrix, and +# vite 5 requires Node >= 18 (`crypto.getRandomValues is not a function` on 16), +# so that job could never pass. This mirrors the local `ci.yml` steps across the +# Node versions the package actually supports. on: [pull_request] jobs: build: - uses: ryansonshine/ryansonshine/.github/workflows/pr.yml@main + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: [18.x, 20.x] + steps: + - name: โฌ‡๏ธ Checkout code + uses: actions/checkout@v4 + + - name: ๐ŸŸฆ Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + + - name: ๐Ÿ“ฆ Install dependencies + run: npm ci + + - name: ๐Ÿ”จ Build + run: npm run build --if-present + + - name: ๐Ÿงช Test + run: npm test