diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td index 6781356cf58d..9df6050347e8 100644 --- a/.github/VOUCHED.td +++ b/.github/VOUCHED.td @@ -68,6 +68,7 @@ github:realAhmedRoach github:Rishet11 github:ryanrhughes github:saphid +github:scratchyone github:sethwebster github:shiroyasha9 github:shivamhwp diff --git a/.github/workflows/mobile-eas-preview.yml b/.github/workflows/mobile-eas-preview.yml index d53602f8f5e8..5db69c9befaf 100644 --- a/.github/workflows/mobile-eas-preview.yml +++ b/.github/workflows/mobile-eas-preview.yml @@ -78,7 +78,19 @@ jobs: working-directory: apps/mobile env: EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} - run: eas env:pull preview --non-interactive + run: | + eas env:pull preview --non-interactive + # EAS Update disables dotenv loading. Keep the downloaded native config + # in its process environment so Android build and OTA fingerprints match. + node --input-type=module <<'NODE' + import { appendFileSync, readFileSync } from "node:fs"; + import { parseEnv } from "node:util"; + const env = parseEnv(readFileSync(".env.local", "utf8")); + const googleServicesFile = env.T3CODE_ANDROID_GOOGLE_SERVICES_FILE; + if (googleServicesFile) { + appendFileSync(process.env.GITHUB_ENV, `T3CODE_ANDROID_GOOGLE_SERVICES_FILE=${googleServicesFile}\n`); + } + NODE - name: Deploy with fingerprint check if: steps.expo-token.outputs.present == 'true' diff --git a/.github/workflows/mobile-showcase-screenshots.yml b/.github/workflows/mobile-showcase-screenshots.yml index c64bccacdca8..fc75c447c6db 100644 --- a/.github/workflows/mobile-showcase-screenshots.yml +++ b/.github/workflows/mobile-showcase-screenshots.yml @@ -76,6 +76,17 @@ jobs: echo "$vp_pnpm_bin" >> "$GITHUB_PATH" "$vp_pnpm_bin/pnpm" --version + - name: Install AXe + # Locks the simulator and answers the notification prompt for the + # agent-activity scene. + # Homebrew refuses third-party tap formulae until they are trusted; + # older releases have no trust command and install them as is. + run: | + brew tap cameroncooke/axe + if brew commands | grep -qx trust; then brew trust --formula cameroncooke/axe/axe; fi + brew install cameroncooke/axe/axe + axe --version + - name: Capture iOS showcase run: pnpm screenshots:mobile --platform ios --appearance "${{ inputs.appearance }}" --theme "${{ inputs.theme }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8890de34e721..387330713d05 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -259,13 +259,76 @@ jobs: - name: Typecheck run: vp run typecheck + # Keep tests on their own runners, using the same package split and server + # shards as CI, so the release check job does not spend its budget on tests. + test: + name: Release tests + needs: [preflight] + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.ref }} + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + - uses: ./.github/actions/setup-apt-mirrors - name: Install browser secret helper build libraries - run: sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config + run: | + sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources + sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config build-essential + + - name: Test + run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' test + + test_server: + name: Release server tests ${{ matrix.shard }} + needs: [preflight] + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.ref }} + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + # No Electron setup here: `t3` (apps/server) has no Electron dependency + # and none of its tests touch the runtime. Only the non-server `test` + # job, which covers @t3tools/desktop, needs the download. - name: Test - run: vp run test + run: vp run --filter t3 test --shard ${{ matrix.shard }}/${{ strategy.job-total }} relay_public_config: name: Resolve T3 Connect public config @@ -630,13 +693,15 @@ jobs: preflight, relay_public_config, quality, + test, + test_server, desktop_mac_arm64, desktop_linux_x64, desktop_linux_arm64, desktop_win_x64, desktop_win_arm64, ] - if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.relay_public_config.result == 'success' && needs.quality.result == 'success' && needs.desktop_mac_arm64.result == 'success' && needs.desktop_linux_x64.result == 'success' && needs.desktop_linux_arm64.result == 'success' && needs.desktop_win_x64.result == 'success' && needs.desktop_win_arm64.result == 'success' }} + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.relay_public_config.result == 'success' && needs.quality.result == 'success' && needs.test.result == 'success' && needs.test_server.result == 'success' && needs.desktop_mac_arm64.result == 'success' && needs.desktop_linux_x64.result == 'success' && needs.desktop_linux_arm64.result == 'success' && needs.desktop_win_x64.result == 'success' && needs.desktop_win_arm64.result == 'success' }} runs-on: ubuntu-24.04 # blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 15 permissions: diff --git a/apps/desktop/src/app/DesktopObservability.test.ts b/apps/desktop/src/app/DesktopObservability.test.ts index d7ccfc43b185..215998631e92 100644 --- a/apps/desktop/src/app/DesktopObservability.test.ts +++ b/apps/desktop/src/app/DesktopObservability.test.ts @@ -1,6 +1,7 @@ import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -427,6 +428,36 @@ describe("DesktopObservability", () => { ); }); + it.effect("exports kill switch warnings through the configured logger", () => { + const requests: Array = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-observability-test-", + }); + const environmentLayer = makeEnvironmentLayer(baseDir, true, { + T3CODE_OTLP_LOGS_URL: "https://collector.example.com/v1/logs", + }); + + yield* Effect.scoped( + Effect.void.pipe( + Effect.provide(DesktopObservability.layer.pipe(Layer.provideMerge(environmentLayer))), + ), + ); + + assert.include(requests[0]?.body ?? "", "OTEL_SDK_DISABLED=1 was read as false"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + collectorLayer(requests), + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { OTEL_SDK_DISABLED: "1" } })), + ), + ), + ); + }); + it.effect("reads every signal endpoint from Settings when the environment names none", () => { const requests: Array = []; return Effect.gen(function* () { @@ -491,4 +522,37 @@ describe("DesktopObservability", () => { Effect.provide(Layer.mergeAll(NodeServices.layer, collectorLayer(requests))), ); }); + + it.effect("stops every export when the OpenTelemetry SDK is disabled", () => { + const requests: Array = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-observability-test-", + }); + const environmentLayer = makeEnvironmentLayer(baseDir); + yield* writeObservabilitySettings(environmentLayer, { + otlpTracesUrl: "https://settings.example.com/v1/traces", + otlpLogsUrl: "https://settings.example.com/v1/logs", + }); + + yield* Effect.scoped( + Effect.logInfo("desktop log stays local when disabled").pipe( + Effect.withSpan("desktop-disabled-test"), + Effect.provide(DesktopObservability.layer.pipe(Layer.provideMerge(environmentLayer))), + ), + ); + + assert.lengthOf(requests, 0); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + collectorLayer(requests), + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { OTEL_SDK_DISABLED: "true" } })), + ), + ), + ); + }); }); diff --git a/apps/desktop/src/app/DesktopObservability.ts b/apps/desktop/src/app/DesktopObservability.ts index 96657215748b..f8fd73a636f1 100644 --- a/apps/desktop/src/app/DesktopObservability.ts +++ b/apps/desktop/src/app/DesktopObservability.ts @@ -4,6 +4,7 @@ import { makeTraceSink, otlpSerializationLayer, } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import { parsePersistedServerObservabilitySettings, type PersistedServerObservabilitySettings, @@ -353,12 +354,18 @@ const readPersistedObservabilitySettings: Effect.Effect< * resolve traces against one revision of the file and logs against another. */ const resolveOtlpEndpoints = Effect.gen(function* () { + const otel = yield* OtelEnvironment.load; + if (otel.disabled) { + return { traces: undefined, metrics: undefined, logs: undefined, warnings: otel.warnings }; + } + const environment = yield* DesktopEnvironment.DesktopEnvironment; const persisted = yield* readPersistedObservabilitySettings; return { traces: Option.getOrUndefined(environment.otlpTracesUrl) ?? persisted.otlpTracesUrl, metrics: Option.getOrUndefined(environment.otlpMetricsUrl) ?? persisted.otlpMetricsUrl, logs: Option.getOrUndefined(environment.otlpLogsUrl) ?? persisted.otlpLogsUrl, + warnings: otel.warnings, }; }); @@ -671,7 +678,12 @@ const telemetryLayer = Layer.unwrap( // resource, // }).pipe(Layer.provide(serializationLayer)); - return Layer.mergeAll(loggerLayer, tracerLayer); + // Logged once the loggers above are installed, so the warnings use them. + const otelWarningsLayer = Layer.effectDiscard( + Effect.forEach(endpoints.warnings, (warning) => Effect.logWarning(warning)), + ); + + return otelWarningsLayer.pipe(Layer.provideMerge(Layer.mergeAll(loggerLayer, tracerLayer))); }), ); diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts index 41eeedc668b6..83189e1ee0af 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts @@ -946,6 +946,49 @@ describe("DesktopBackendConfiguration", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("resolveWsl carries the kill switch into the distro", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-backend-config-test-", + }); + + const previousWslEnv = process.env.WSLENV; + const previousDisabled = process.env.OTEL_SDK_DISABLED; + try { + delete process.env.WSLENV; + process.env.OTEL_SDK_DISABLED = "true"; + + yield* Effect.gen(function* () { + const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; + const config = yield* configuration.resolveWsl({ port: 5050, distro: null }); + + assert.equal(config.env.OTEL_SDK_DISABLED, "true"); + assert.include((config.env.WSLENV ?? "").split(":"), "OTEL_SDK_DISABLED"); + }).pipe( + Effect.provide( + DesktopBackendConfiguration.layer.pipe( + Layer.provideMerge(serverExposureLayer), + Layer.provideMerge(DesktopAppSettings.layerTest()), + Layer.provideMerge(DesktopWslServerTree.layerTest()), + Layer.provideMerge( + DesktopWslEnvironment.layerTest({ + isAvailable: true, + windowsToWslPath: () => Option.some("/mnt/c/repo/apps/server/src/index.ts"), + getDistroIp: () => Option.some("172.27.0.99"), + }), + ), + Layer.provideMerge(makeEnvironmentLayer(baseDir, { platform: "win32" })), + ), + ), + ); + } finally { + restoreEnv("WSLENV", previousWslEnv); + restoreEnv("OTEL_SDK_DISABLED", previousDisabled); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("resolveWsl preserves existing WSLENV entries when forwarding backend secrets", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index 9e809667ac42..ff693d1ac9e3 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -96,6 +96,9 @@ const DESKTOP_BACKEND_ENV_NAMES = [ const WSL_FORWARDED_ENV_NAMES = [ "OPENAI_API_KEY", "ANTHROPIC_API_KEY", + // Otherwise the WSL server keeps exporting to endpoints from the bootstrap. + "T3CODE_OTEL_SDK_DISABLED", + "OTEL_SDK_DISABLED", "T3CODE_OTLP_HEADERS", "T3CODE_OTLP_PROTOCOL", ] as const; diff --git a/apps/desktop/src/electron/ElectronProtocol.test.ts b/apps/desktop/src/electron/ElectronProtocol.test.ts index 508a5c296898..0c2cb2bb7d17 100644 --- a/apps/desktop/src/electron/ElectronProtocol.test.ts +++ b/apps/desktop/src/electron/ElectronProtocol.test.ts @@ -105,7 +105,7 @@ describe("ElectronProtocol", () => { ); assert.include( response.headers.get("content-security-policy") ?? "", - "connect-src 'self' http: https: ws: wss:", + "connect-src 'self' blob: http: https: ws: wss:", ); assert.include( response.headers.get("content-security-policy") ?? "", @@ -255,7 +255,14 @@ describe("ElectronProtocol", () => { "https://clerk.t3.codes", "https://challenges.cloudflare.com", ]); - assert.deepEqual(directives["connect-src"], ["'self'", "http:", "https:", "ws:", "wss:"]); + assert.deepEqual(directives["connect-src"], [ + "'self'", + "blob:", + "http:", + "https:", + "ws:", + "wss:", + ]); assert.deepEqual(directives["img-src"], [ "'self'", "t3code:", diff --git a/apps/desktop/src/electron/ElectronProtocol.ts b/apps/desktop/src/electron/ElectronProtocol.ts index 96cd462bd156..c3cecf88b2e4 100644 --- a/apps/desktop/src/electron/ElectronProtocol.ts +++ b/apps/desktop/src/electron/ElectronProtocol.ts @@ -84,7 +84,8 @@ export function makeDesktopContentSecurityPolicy(input: DesktopProtocolRegistrat // the build-configured Clerk, relay, and OTLP endpoints. Those environment // origins are not known when this response policy is created, so restrict // connections by the network schemes the client supports instead of by host. - const connectSources = ["'self'", "http:", "https:", "ws:", "wss:"]; + // GLTFLoader fetches embedded textures through blob URLs after parsing the model. + const connectSources = ["'self'", "blob:", "http:", "https:", "ws:", "wss:"]; return [ "default-src 'self'", diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index f49b6dbd968e..e8a688c189a3 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -11,6 +11,7 @@ export const MENU_ACTION_CHANNEL = "desktop:menu-action"; export const PASTE_AS_TEXT_CHANNEL = "desktop:paste-as-text"; export const SNAP_SHOT_EVENT_CHANNEL = "desktop:snap-shot-event"; export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut"; +export const TRACKPAD_SCROLL_END_CHANNEL = "desktop:trackpad-scroll-end"; export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state"; export const WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:window-fullscreen-state"; export const DESKTOP_APP_ACTIVATION_READY_CHANNEL = "desktop:app-activation-ready"; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 7256f610fbeb..53d8cca7b088 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -79,6 +79,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.on(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); return () => ipcRenderer.removeListener(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); }, + onTrackpadScrollEnd: (listener) => { + const handler = () => listener(); + ipcRenderer.on(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler); + return () => ipcRenderer.removeListener(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler); + }, getSystemLocale: () => { const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL); return typeof result === "string" ? result : null; diff --git a/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts b/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts index 1c638f2444de..ad7b2e03c495 100644 --- a/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts +++ b/apps/desktop/src/snapShot/MacModifierPairShortcutProcess.ts @@ -11,7 +11,11 @@ const MAC_MODIFIER_PAIR_DEVICE_MASKS: Record { }), ); + it.effect("forwards native trackpad release to the renderer", () => + Effect.gen(function* () { + const fakeWindow = makeFakeBrowserWindow(); + const send = vi.spyOn(fakeWindow.window.webContents, "send"); + const createCount = yield* Ref.make(0); + const mainWindow = yield* Ref.make>(Option.none()); + const layer = makeTestLayer({ window: fakeWindow.window, createCount, mainWindow }); + + yield* Effect.gen(function* () { + const desktopWindow = yield* DesktopWindow.DesktopWindow; + yield* desktopWindow.handleBackendReady(new URL("http://127.0.0.1:3773")); + const onInput = fakeWindow.webContentsListeners.get("input-event"); + if (!onInput) return yield* Effect.die("input-event listener was not registered"); + onInput({}, { type: "gestureScrollUpdate" }); + assert.notInclude( + send.mock.calls.map(([channel]) => channel), + TRACKPAD_SCROLL_END_CHANNEL, + ); + onInput({}, { type: "gestureScrollEnd" }); + assert.isTrue(send.mock.calls.some(([channel]) => channel === TRACKPAD_SCROLL_END_CHANNEL)); + }).pipe(Effect.provide(layer)); + }), + ); + // Chromium hands the main window's zoom level down to embedded preview // guests, so every app zoom has to put the preview browser back at its own // zoom or zooming the UI drags the previewed page with it. diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index b3964cb929c7..754de3caa727 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -22,6 +22,7 @@ import { MENU_ACTION_CHANNEL, QUIT_SHORTCUT_CHANNEL, SNAP_SHOT_EVENT_CHANNEL, + TRACKPAD_SCROLL_END_CHANNEL, WINDOW_FULLSCREEN_STATE_CHANNEL, } from "../ipc/channels.ts"; import * as PreviewManager from "../preview/Manager.ts"; @@ -661,6 +662,9 @@ export const make = Effect.gen(function* () { event.preventDefault(); } }); + window.webContents.on("input-event", (_event, input) => { + if (input.type === "gestureScrollEnd") window.webContents.send(TRACKPAD_SCROLL_END_CHANNEL); + }); window.on("page-title-updated", (event) => { event.preventDefault(); diff --git a/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/AgentNotifications.kt b/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/AgentNotifications.kt index 35086bbd75f9..00af98ac4b28 100644 --- a/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/AgentNotifications.kt +++ b/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/AgentNotifications.kt @@ -138,17 +138,7 @@ object AgentNotifications { // Match iOS foreground presentation. Consume suppressed alerts as well, // so a delivery retry cannot surface them after the app backgrounds. if (!ProcessLifecycleOwner.get().lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)) { - val title = data["alert_title"].orEmpty().take(120) - // Grouped alerts list up to five 120-character thread titles. - val body = data["alert_body"].orEmpty().take(608) - val id = alertId.hashCode() - val notification = base(context, ALERT_CHANNEL) - .setContentTitle(title).setContentText(body) - .setStyle(NotificationCompat.BigTextStyle().bigText(body)) - .setAutoCancel(true) - .setContentIntent(contentIntent(context, scheme, data["alert_path"], id)) - .build() - manager(context).notify(ALERT_TAG, id, notification) + postAlert(context, scheme, data, alertId) } prefs.edit().remove("seenAlerts").putString( "seenAlertsOrdered", @@ -157,6 +147,36 @@ object AgentNotifications { } } + private fun postAlert( + context: Context, + scheme: String, + data: Map, + alertId: String + ) { + val title = data["alert_title"].orEmpty().take(120) + // Grouped alerts list up to five 120-character thread titles. + val body = data["alert_body"].orEmpty().take(608) + val id = alertId.hashCode() + val notification = base(context, ALERT_CHANNEL) + .setContentTitle(title).setContentText(body) + .setStyle(NotificationCompat.BigTextStyle().bigText(body)) + .setAutoCancel(true) + .setContentIntent(contentIntent(context, scheme, data["alert_path"], id)) + .build() + manager(context).notify(ALERT_TAG, id, notification) + } + + /** + * Renders a relay-shaped payload without the registration, freshness and + * foreground checks, for the showcase capture's staged notifications. + */ + @Synchronized + fun showcase(context: Context, scheme: String, data: Map) { + channels(context) + data["alert_id"]?.let { postAlert(context, scheme, data, it) } + showActivity(context, scheme, data, data["active"] == "true", RUNNING_LIFETIME_MS) + } + private fun updateActivity( context: Context, prefs: SharedPreferences, diff --git a/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/T3AgentNotificationsModule.kt b/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/T3AgentNotificationsModule.kt index 246db274a195..0a2d2b30589f 100644 --- a/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/T3AgentNotificationsModule.kt +++ b/apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/T3AgentNotificationsModule.kt @@ -26,6 +26,10 @@ class T3AgentNotificationsModule : Module() { appContext.reactContext?.let { AgentNotifications.clear(it) } } + Function("showShowcaseActivity") { scheme: String, data: Map -> + appContext.reactContext?.let { AgentNotifications.showcase(it, scheme, data) } + } + Function("openLiveUpdateSettings") { val context = appContext.reactContext if (context == null || Build.VERSION.SDK_INT < 36) { diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 524eac860716..7f508f486ce3 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -71,6 +71,7 @@ import { SettingsAppearanceRouteScreen } from "./features/settings/SettingsAppea import { SettingsClientStorageRouteScreen } from "./features/settings/SettingsClientStorageRouteScreen"; import { SettingsDiagnosticsRouteScreen } from "./features/diagnostics/SettingsDiagnosticsRouteScreen"; import { SettingsAuthRouteScreen } from "./features/settings/SettingsAuthRouteScreen"; +import { SettingsEnvironmentDetailRouteScreen } from "./features/settings/SettingsEnvironmentDetailRouteScreen"; import { SettingsEnvironmentsRouteScreen } from "./features/settings/SettingsEnvironmentsRouteScreen"; import { SettingsEnvironmentAgentBehaviorRouteScreen, @@ -193,6 +194,11 @@ const SettingsContentStack = createNativeStackNavigator({ title: "Environments", }, }), + SettingsEnvironmentDetail: createNativeStackScreen({ + screen: SettingsEnvironmentDetailRouteScreen, + linking: "environments/:environmentId", + options: { title: "Environment" }, + }), SettingsEnvironmentNewThreads: createNativeStackScreen({ screen: SettingsEnvironmentNewThreadsRouteScreen, linking: "new-threads", diff --git a/apps/mobile/src/features/agent-awareness/androidNotifications.ts b/apps/mobile/src/features/agent-awareness/androidNotifications.ts index 9ffe586ecb81..a2a0945faaf1 100644 --- a/apps/mobile/src/features/agent-awareness/androidNotifications.ts +++ b/apps/mobile/src/features/agent-awareness/androidNotifications.ts @@ -6,6 +6,7 @@ interface AndroidAgentNotifications { configure(deviceId: string, userId: string, scheme: string, ongoingEnabled: boolean): void; clear(): void; openLiveUpdateSettings?(): boolean; + showShowcaseActivity?(scheme: string, data: Record): void; } const native = @@ -17,18 +18,24 @@ export function supportsAndroidAgentNotifications(): boolean { return typeof native?.configure === "function" && typeof native?.clear === "function"; } +function appScheme(): string { + const scheme = Constants.expoConfig?.scheme; + return (Array.isArray(scheme) ? scheme[0] : scheme) ?? "t3code"; +} + export function configureAndroidAgentNotifications( deviceId: string, userId: string, ongoingEnabled: boolean, ): void { - const scheme = Constants.expoConfig?.scheme; - native?.configure?.( - deviceId, - userId, - (Array.isArray(scheme) ? scheme[0] : scheme) ?? "t3code", - ongoingEnabled, - ); + native?.configure?.(deviceId, userId, appScheme(), ongoingEnabled); +} + +/** Posts a staged relay payload for the showcase capture; false when unsupported. */ +export function showAndroidShowcaseAgentActivity(data: Record): boolean { + if (!native?.showShowcaseActivity) return false; + native.showShowcaseActivity(appScheme(), data); + return true; } export function clearAndroidAgentNotifications(): void { diff --git a/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx b/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx index 2828392ba8f3..d362b78a9360 100644 --- a/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx +++ b/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx @@ -35,6 +35,7 @@ import { type RelayEnvironmentView, useConnectionController } from "./useConnect interface CloudEnvironmentRowsProps { readonly connectedCloudEnvironments: ReadonlyArray; + readonly onOpenEnvironment?: (environmentId: EnvironmentId) => void; readonly onSetEnvironmentEnabled: (environmentId: EnvironmentId, enabled: boolean) => void; /** Long-press on a saved row. The callback owns the confirm. */ readonly onRemoveEnvironment: (environmentId: EnvironmentId) => void; @@ -124,26 +125,30 @@ function CloudEnvironmentRowsContent( ) : null} {hasCloudRows ? ( - - {props.connectedCloudEnvironments.map((environment, index) => ( + + {props.connectedCloudEnvironments.map((environment) => ( props.onSetEnvironmentEnabled(environment.environmentId, enabled) } onRemove={() => props.onRemoveEnvironment(environment.environmentId)} + onOpen={ + props.onOpenEnvironment + ? () => props.onOpenEnvironment?.(environment.environmentId) + : undefined + } errorExpanded={expandedErrorId === environment.environmentId} onToggleError={() => handleToggleCloudError(environment.environmentId)} /> ))} - {availableCloudEnvironments.map((environment, index) => ( + {availableCloudEnvironments.map((environment) => ( 0 || index !== 0} + showChevron={props.onOpenEnvironment !== undefined} onConnect={() => handleConnectCloudEnvironment(environment)} errorExpanded={expandedErrorId === environment.environment.environmentId} onToggleError={() => handleToggleCloudError(environment.environment.environmentId)} @@ -151,14 +156,14 @@ function CloudEnvironmentRowsContent( ))} ) : controller.relayDiscovery.isRefreshing ? ( - + Loading linked cloud environments. ) : controller.relayDiscovery.error ? null : ( - + No additional linked cloud environments. @@ -170,7 +175,7 @@ function CloudEnvironmentRowsContent( {discoveryAvailable && controller.relayDiscovery.error && !controller.relayDiscovery.isRefreshing ? ( - + Could not load T3 Connect environments @@ -202,10 +207,10 @@ function ConnectedCloudEnvironmentRow(props: { readonly environment: ConnectedEnvironmentSummary; /** Discovery's view of the server, for the glyph before the first connection. */ readonly descriptor: ExecutionEnvironmentDescriptor | undefined; - readonly borderTop: boolean; readonly errorExpanded: boolean; readonly onSetEnabled: (enabled: boolean) => void; readonly onRemove: () => void; + readonly onOpen?: (() => void) | undefined; readonly onToggleError: () => void; }) { const serverConfig = useAtomValue( @@ -222,10 +227,13 @@ function ConnectedCloudEnvironmentRow(props: { return ( void; readonly onToggleError: () => void; @@ -260,7 +268,7 @@ function CloudEnvironmentRow(props: { return ( + @@ -409,10 +412,16 @@ function CloudEnvironmentRowShell(props: { + {props.opensDetails || props.showChevron ? ( + + + + ) : null} ); } diff --git a/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx b/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx index cf90cd42a0ad..28b34fdec6d3 100644 --- a/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx +++ b/apps/mobile/src/features/connection/ConnectionEnvironmentRow.tsx @@ -35,6 +35,7 @@ function connectionStatusLabel(environment: ConnectedEnvironmentSummary): string export function ConnectionEnvironmentRow(props: { readonly environment: ConnectedEnvironmentSummary; readonly expanded: boolean; + readonly opensDetails?: boolean; readonly onToggle: () => void; readonly onReconnect: (environmentId: EnvironmentId) => void; readonly onRemove: (environmentId: EnvironmentId) => void; @@ -76,19 +77,22 @@ export function ConnectionEnvironmentRow(props: { }, [label, url, props]); return ( - + - - + - - {props.environment.displayUrl} - + {!props.environment.isRelayManaged && props.environment.displayUrl.trim() ? ( + + {props.environment.displayUrl} + + ) : null} {statusLabel ? ( props.onSetEnabled(props.environment.environmentId, next)} value={enabled} /> - {[...catalog.entries.values()].map((entry, index) => { + {[...catalog.entries.values()].map((entry) => { const environmentId = entry.target.environmentId; const selected = gitHubRoutingPermissionFor(entry, permissions); const disabled = !catalog.isReady || saving || gitHubRoutingConnectionKey(entry) === null; return ( - + { setSaving(true); void update({ environmentId, permission: option.value }).then((result) => { diff --git a/apps/mobile/src/features/connection/LocalEnvironmentList.tsx b/apps/mobile/src/features/connection/LocalEnvironmentList.tsx index dca29e092f64..bec038390a9e 100644 --- a/apps/mobile/src/features/connection/LocalEnvironmentList.tsx +++ b/apps/mobile/src/features/connection/LocalEnvironmentList.tsx @@ -4,7 +4,6 @@ import { View } from "react-native"; import { SymbolView } from "../../components/AppSymbol"; import { AppText as Text } from "../../components/AppText"; -import { cn } from "../../lib/cn"; import { ConnectionEnvironmentRow } from "./ConnectionEnvironmentRow"; type EnvironmentRowProps = ComponentProps; @@ -15,14 +14,20 @@ export function LocalEnvironmentList({ expandedId, onToggle, ...rowActions -}: Pick & { +}: Pick< + EnvironmentRowProps, + "onReconnect" | "onRemove" | "onSetEnabled" | "onUpdate" | "opensDetails" +> & { readonly environments: ReadonlyArray; readonly expandedId: EnvironmentId | null; readonly onToggle: (environmentId: EnvironmentId) => void; }) { if (environments.length === 0) { return ( - + - {environments.map((environment, index) => ( - + + {environments.map((environment) => ( + ) { + // Key local request state to the host even when navigation reuses this screen. + return ( + + ); +} + +function EnvironmentDetail({ environmentId }: { readonly environmentId: EnvironmentId }) { + const insets = useSafeAreaInsets(); + const connections = useRemoteConnections(); + const environment = connections.connectedEnvironments.find( + (entry) => entry.environmentId === environmentId, + ); + const config = useAtomValue(serverEnvironment.configValueAtom(environmentId)); + const session = useAtomValue(environmentSession.sessionStateValueAtom(environmentId)); + const sessionResult = useAtomValue(environmentSession.sessionStateAtom(environmentId)); + const updateState = useAtomValue(serverEnvironment.updateStateAtom(environmentId)); + const updateServer = useAtomCommand(serverEnvironment.updateServer); + const updateProvider = useAtomCommand(serverEnvironment.updateProvider); + const refreshProviders = useAtomCommand(serverEnvironment.refreshProviders); + const [connectionExpanded, setConnectionExpanded] = useState(false); + const [pending, setPending] = useState(null); + const pendingRef = useRef(false); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + const [release, setRelease] = useState<{ + fromVersion: string; + targetVersion: string | null; + } | null>(null); + const checkController = useRef(null); + useEffect(() => () => checkController.current?.abort(), []); + + const connected = environment?.isEnabled === true && environment.connectionState === "connected"; + const allowed = + !AsyncResult.isFailure(sessionResult) && canMaintainEnvironment(session, connected); + const running = updateState.status === "running"; + const providerBusy = + config?.providers.some( + (provider) => + provider.updateState?.status === "running" || provider.updateState?.status === "queued", + ) ?? false; + const disabled = !allowed || pending !== null || running || providerBusy; + const version = config?.environment.serverVersion; + const checkedRelease = release?.fromVersion === version ? release : null; + const capabilities = config?.environment.capabilities; + + async function run(label: string, action: () => Promise) { + if (pendingRef.current) return; + pendingRef.current = true; + setPending(label); + setError(null); + setNotice(null); + try { + await action(); + } catch (cause) { + setError( + cause instanceof Error ? cause.message : "The action could not be completed. Try again.", + ); + } finally { + pendingRef.current = false; + setPending(null); + } + } + + function requestServerUpdate() { + const targetVersion = checkedRelease?.targetVersion; + if (disabled || !targetVersion || !capabilities || !supportsEnvironmentUpdate(capabilities)) + return; + Alert.alert( + `Update ${environment?.environmentLabel ?? "environment"}?`, + `Install T3 Code ${targetVersion}. ${capabilities.serverSelfUpdate === "desktop-managed" ? "The desktop app will close and relaunch." : "The server will restart and reconnect."} Running threads may be interrupted.`, + [ + { text: "Cancel", style: "cancel" }, + { + text: "Update", + onPress: () => + void run("server", async () => { + const result = await updateServer({ + environmentId, + input: { + targetVersion, + ...(capabilities.serverUpdateThreadContinuation && + config?.settings.continueThreadsAfterServerUpdate + ? { continueRunningThreads: true } + : {}), + }, + }); + if (AsyncResult.isFailure(result)) throw squashAtomCommandFailure(result); + setRelease(null); + setNotice(`Updated to ${result.value.targetVersion}.`); + }), + }, + ], + ); + } + + function requestProviderUpdate(provider: ServerProvider) { + if (disabled || !canUpdateEnvironmentProvider(provider)) return; + void run(provider.instanceId, async () => { + const result = await updateProvider({ + environmentId, + input: { + provider: provider.driver, + instanceId: provider.instanceId, + }, + }); + if (AsyncResult.isFailure(result)) throw squashAtomCommandFailure(result); + }); + } + + return ( + + + {!environment ? ( + + This environment is no longer saved on this device. + + ) : ( + <> + + setConnectionExpanded((value) => !value)} + onReconnect={connections.onReconnectEnvironment} + onRemove={connections.onRemoveEnvironmentPress} + onSetEnabled={connections.onSetEnvironmentEnabled} + onUpdate={connections.onUpdateEnvironment} + /> + + {!connected ? ( + + Connect this environment to manage it. + + ) : !allowed ? ( + + {AsyncResult.isFailure(sessionResult) + ? "Could not verify your permissions. Reconnect to try again." + : session === null + ? "Checking permissions…" + : "This connection does not have permission to manage the environment."} + + ) : null} + {error ? ( + + {error} + + ) : null} + {notice ? {notice} : null} + {config ? ( + <> + + + Version {version} + {running ? ( + + {updateState.stage === "resuming" + ? "Restarting and reconnecting…" + : "Downloading update…"} + + ) : updateState.status === "failed" ? ( + + {updateState.message} + + ) : null} + {checkedRelease ? ( + + {checkedRelease.targetVersion + ? `Version ${checkedRelease.targetVersion} is available.` + : "You are up to date."} + + ) : null} + {!supportsEnvironmentUpdate(config.environment.capabilities) ? ( + + {capabilities?.serverSelfUpdate === "desktop-managed" + ? "Update the desktop app on this machine." + : "Update and restart T3 Code on this machine."} + + ) : null} + + { + if (disabled) return; + void run("check", async () => { + const controller = new AbortController(); + checkController.current = controller; + const timeout = setTimeout(() => controller.abort(), 20_000); + try { + const targetVersion = await findEnvironmentUpdate( + config.environment.serverVersion, + controller.signal, + ); + setRelease({ + fromVersion: config.environment.serverVersion, + targetVersion, + }); + } finally { + clearTimeout(timeout); + checkController.current = null; + } + }); + }} + /> + {checkedRelease?.targetVersion && + supportsEnvironmentUpdate(config.environment.capabilities) ? ( + + ) : null} + + + { + if (disabled) return; + void run("refresh", async () => { + const result = await refreshProviders({ environmentId, input: {} }); + if (AsyncResult.isFailure(result)) throw squashAtomCommandFailure(result); + setNotice("Provider status refreshed."); + }); + }} + /> + {config.providers + .filter((provider) => provider.enabled) + .map((provider) => ( + + + + + + {provider.displayName ?? provider.driver} + + + + {provider.installed + ? (provider.version ?? "Version unknown") + : "Not installed"} + {provider.versionAdvisory?.latestVersion + ? ` · Latest ${provider.versionAdvisory.latestVersion}` + : ""} + + {provider.updateState && provider.updateState.status !== "idle" ? ( + + {provider.updateState.message ?? + `Update ${provider.updateState.status}`} + + ) : null} + {provider.compatibilityAdvisory?.message ? ( + + {provider.compatibilityAdvisory.message} + + ) : null} + {provider.unavailableReason || provider.message ? ( + + {provider.unavailableReason ?? provider.message} + + ) : null} + {provider.versionAdvisory?.status === "behind_latest" && + !provider.versionAdvisory.canUpdate ? ( + + Update this provider on the environment's machine. + + ) : null} + + {canUpdateEnvironmentProvider(provider) ? ( + requestProviderUpdate(provider)} + /> + ) : null} + + ))} + + + ) : null} + + )} + + + ); +} diff --git a/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx index 7ca791afec6d..003178a03aca 100644 --- a/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsEnvironmentsRouteScreen.tsx @@ -47,7 +47,6 @@ export function SettingsEnvironmentsRouteScreen() { const connectedCloudEnvironments = SHOWCASE_ENABLED ? SHOWCASE_CONNECTED_CLOUD_ENVIRONMENTS : environmentSections.connectedCloudEnvironments; - const [expandedId, setExpandedId] = useState(null); const headerIconColor = useUniwindTheme()["--color-icon"]; const relaySession = useAtomValue(managedRelaySessionAtom); const refreshRelayEnvironments = useAtomCommand( @@ -68,9 +67,15 @@ export function SettingsEnvironmentsRouteScreen() { } } - const handleToggle = useCallback((environmentId: EnvironmentId) => { - setExpandedId((prev) => (prev === environmentId ? null : environmentId)); - }, []); + const openEnvironment = useCallback( + (environmentId: EnvironmentId) => { + navigation.navigate("SettingsSheet", { + screen: "SettingsContent", + params: { screen: "SettingsEnvironmentDetail", params: { environmentId } }, + }); + }, + [navigation], + ); const handleUpdateEnvironment = useCallback( ( environmentId: EnvironmentId, @@ -159,8 +164,9 @@ export function SettingsEnvironmentsRouteScreen() { > group.key === selectedProjectKey); const projectSelected = selectedProjectKey !== null; @@ -397,36 +400,61 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { ) : null} {props.page === "maintenance" ? ( - - write({ enableProviderUpdateChecks: value })} - /> - + <> + {!projectSelected ? ( + + {selectedTargets.map((target) => ( + + navigation.navigate("SettingsSheet", { + screen: "SettingsContent", + params: { + screen: "SettingsEnvironmentDetail", + params: { environmentId: target.environmentId }, + }, + }) + } + /> + ))} + + ) : null} + write({ continueThreadsAfterServerUpdate: value })} + value={uniform("enableProviderUpdateChecks")} + disabled={disabledFor("enableProviderUpdateChecks")} + onValueChange={(value) => write({ enableProviderUpdateChecks: value })} /> - - + + + write({ continueThreadsAfterServerUpdate: value }) + } + /> + + + ) : null} )} diff --git a/apps/mobile/src/features/settings/components/SettingsSwitchRow.tsx b/apps/mobile/src/features/settings/components/SettingsSwitchRow.tsx index 49fb98e49976..85cb4764e25a 100644 --- a/apps/mobile/src/features/settings/components/SettingsSwitchRow.tsx +++ b/apps/mobile/src/features/settings/components/SettingsSwitchRow.tsx @@ -17,6 +17,7 @@ export function SettingsSwitchRow( subtitle={props.subtitle} > vi.unstubAllGlobals()); + +describe("environment maintenance access", () => { + it("requires a connected authenticated session with operate permission", () => { + const session = { + authenticated: true, + auth: { + policy: "remote-reachable" as const, + bootstrapMethods: [], + sessionMethods: [], + sessionCookieName: "session", + }, + scopes: [AuthOrchestrationOperateScope], + }; + expect(canMaintainEnvironment(session, true)).toBe(true); + expect(canMaintainEnvironment(session, false)).toBe(false); + expect(canMaintainEnvironment({ ...session, authenticated: false }, true)).toBe(false); + expect(canMaintainEnvironment({ ...session, scopes: [] }, true)).toBe(false); + const { scopes: _, ...legacy } = session; + expect(canMaintainEnvironment(legacy, true)).toBe(false); + expect(canMaintainEnvironment(null, true)).toBe(false); + }); + + it("requires remote desktop update support for desktop hosts", () => { + expect(supportsEnvironmentUpdate({})).toBe(false); + expect(supportsEnvironmentUpdate({ serverSelfUpdate: "respawn" })).toBe(true); + expect(supportsEnvironmentUpdate({ serverSelfUpdate: "desktop-managed" })).toBe(false); + expect( + supportsEnvironmentUpdate({ serverSelfUpdate: "desktop-managed", desktopAppUpdate: true }), + ).toBe(true); + }); + + it("excludes unavailable, manual, busy, and incompatible provider updates", () => { + expect(canUpdateEnvironmentProvider(provider)).toBe(true); + expect(canUpdateEnvironmentProvider({ ...provider, installed: false })).toBe(false); + expect(canUpdateEnvironmentProvider({ ...provider, availability: "unavailable" })).toBe(false); + expect(canUpdateEnvironmentProvider({ ...provider, versionAdvisory: undefined })).toBe(false); + for (const latestVersionStatus of ["broken", "unsupported"] as const) { + expect( + canUpdateEnvironmentProvider({ + ...provider, + compatibilityAdvisory: { + status: "supported", + latestVersionStatus, + message: null, + recommendedVersion: null, + recommendedRange: null, + }, + }), + ).toBe(false); + } + for (const status of ["queued", "running"] as const) { + expect( + canUpdateEnvironmentProvider({ + ...provider, + updateState: { + status, + startedAt: null, + finishedAt: null, + message: null, + output: null, + }, + }), + ).toBe(false); + } + expect( + canUpdateEnvironmentProvider({ + ...provider, + versionAdvisory: { + ...provider.versionAdvisory!, + canUpdate: false, + }, + }), + ).toBe(false); + }); +}); + +describe("environment release checks", () => { + const signal = new AbortController().signal; + + it("keeps stable hosts on stable releases and ignores drafts", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation(async () => + Response.json([ + { tag_name: "v2.0.0-nightly.20260923.1" }, + { tag_name: "v1.2.0", draft: true }, + { tag_name: "v1.1.0" }, + ]), + ), + ); + expect(await findEnvironmentUpdate("1.0.0", signal)).toBe("1.1.0"); + expect(await findEnvironmentUpdate("1.1.0", signal)).toBeNull(); + expect(await findEnvironmentUpdate("1.3.0", signal)).toBeNull(); + }); + + it("walks release pages to find the host's channel", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + Response.json(Array.from({ length: 100 }, () => ({ tag_name: "v2.0.0" }))), + ) + .mockResolvedValueOnce(Response.json([{ tag_name: "v1.0.0-preview.20260923.2" }])); + vi.stubGlobal("fetch", fetchMock); + expect(await findEnvironmentUpdate("1.0.0-preview.20260923.1", signal)).toBe( + "1.0.0-preview.20260923.2", + ); + expect(fetchMock.mock.calls[1]?.[0]).toContain("page=2"); + expect(fetchMock.mock.calls[1]?.[1]).toEqual({ signal }); + }); + + it("reports failed checks instead of claiming the server is current", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(null, { status: 403 }))); + await expect(findEnvironmentUpdate("1.0.0", signal)).rejects.toThrow("403"); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json([]))); + await expect(findEnvironmentUpdate("1.0.0", signal)).rejects.toThrow("No stable release"); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ error: "invalid" }))); + await expect(findEnvironmentUpdate("1.0.0", signal)).rejects.toThrow(); + }); +}); diff --git a/apps/mobile/src/features/settings/environment-maintenance.ts b/apps/mobile/src/features/settings/environment-maintenance.ts new file mode 100644 index 000000000000..f441066d7174 --- /dev/null +++ b/apps/mobile/src/features/settings/environment-maintenance.ts @@ -0,0 +1,68 @@ +import { + AuthOrchestrationOperateScope, + type AuthSessionState, + type ExecutionEnvironmentCapabilities, + type ServerProvider, +} from "@t3tools/contracts"; +import { + cliReleaseChannelOf, + cliReleaseIndexPageUrl, + newestCliReleaseVersion, +} from "@t3tools/shared/cliRelease"; +import { compareSemverVersions } from "@t3tools/shared/semver"; +import * as Schema from "effect/Schema"; + +export function canMaintainEnvironment(session: AuthSessionState | null, connected: boolean) { + return ( + connected && + session?.authenticated === true && + session.scopes?.includes(AuthOrchestrationOperateScope) === true + ); +} + +export function supportsEnvironmentUpdate( + capabilities: Pick, +) { + return ( + capabilities.serverSelfUpdate !== undefined && + (capabilities.serverSelfUpdate !== "desktop-managed" || capabilities.desktopAppUpdate === true) + ); +} + +export function canUpdateEnvironmentProvider(provider: ServerProvider) { + const compatibility = provider.compatibilityAdvisory?.latestVersionStatus; + return ( + provider.installed && + provider.availability !== "unavailable" && + provider.versionAdvisory?.status === "behind_latest" && + provider.versionAdvisory.canUpdate && + provider.versionAdvisory.latestVersion !== null && + compatibility !== "broken" && + compatibility !== "unsupported" && + provider.updateState?.status !== "running" && + provider.updateState?.status !== "queued" + ); +} + +const Releases = Schema.Array( + Schema.Struct({ + tag_name: Schema.String, + draft: Schema.optionalKey(Schema.Boolean), + }), +); +const decodeReleases = Schema.decodeUnknownSync(Releases); + +/** Preserve the host's release channel and never offer a downgrade. */ +export async function findEnvironmentUpdate(currentVersion: string, signal: AbortSignal) { + const channel = cliReleaseChannelOf(currentVersion); + for (let page = 1; ; page++) { + const response = await fetch(cliReleaseIndexPageUrl(page), { signal }); + if (!response.ok) throw new Error(`Could not check releases (${response.status}). Try again.`); + const releases = decodeReleases(await response.json()); + const version = newestCliReleaseVersion(releases, channel); + if (version !== undefined) { + return compareSemverVersions(version, currentVersion) > 0 ? version : null; + } + if (releases.length < 100) throw new Error(`No ${channel} release was found.`); + } +} diff --git a/apps/mobile/src/features/showcase/ShowcaseCaptureCoordinator.tsx b/apps/mobile/src/features/showcase/ShowcaseCaptureCoordinator.tsx index 557c3b190359..716f4edd4166 100644 --- a/apps/mobile/src/features/showcase/ShowcaseCaptureCoordinator.tsx +++ b/apps/mobile/src/features/showcase/ShowcaseCaptureCoordinator.tsx @@ -29,6 +29,7 @@ import { buildShowcasePendingTasks, SHOWCASE_PENDING_TASK_DEFINITIONS, } from "./showcasePendingTasks"; +import { buildShowcaseAgentActivity } from "./showcaseAgentActivity"; import { retryShowcaseOperation } from "./showcaseRetry"; import { clearShowcaseRenderSignal, @@ -36,6 +37,7 @@ import { isShowcaseNativeContentReady, subscribeToShowcaseRenderSignal, } from "./showcaseRenderSignal"; +import { stageShowcaseAgentActivity } from "./stageShowcaseAgentActivity"; const SHOWCASE_ENABLED = process.env.EXPO_PUBLIC_SHOWCASE === "1"; const SHOWCASE_THREAD_ID = "remote-command-center"; @@ -75,7 +77,14 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) const [themeRequestSettled, setThemeRequestSettled] = useState(false); const [readyScene, setReadyScene] = useState(null); const [orientationSettled, setOrientationSettled] = useState(false); + const [agentActivityStaged, setAgentActivityStaged] = useState(false); const requestedSceneRef = useRef(null); + // Staging reads the latest entities without restarting on every shell + // update, which would re-enter a permission prompt that is still open. + const entitiesRef = useRef({ threads, projects }); + useEffect(() => { + entitiesRef.current = { threads, projects }; + }, [projects, threads]); const renderSignal = useSyncExternalStore( subscribeToShowcaseRenderSignal, getShowcaseRenderSignal, @@ -109,6 +118,7 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) } let cancelled = false; + let lastOutcome: string | null = null; void retryShowcaseOperation(async () => applyNativeShowcaseOrientation(orientation), { isCancelled: () => cancelled, }).then((applied) => { @@ -129,6 +139,7 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) // A native draw belongs only to the scene request that produced it. In // particular, revisiting review must wait for its newly mounted surface. clearShowcaseRenderSignal(); + setAgentActivityStaged(false); setRequestedScene(value); }; readRequestedScene(); @@ -176,7 +187,11 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) }; }, [connectPairingUrl, pairingUrls]); - const scene = sceneFromPathname(props.pathname); + const routeScene = sceneFromPathname(props.pathname); + // Agent activity is captured over the thread list: the runner locks the + // simulator or opens the notification shade on top of it. + const scene = + requestedScene === "agent-activity" && routeScene === "threads" ? "agent-activity" : routeScene; const hasServerFixture = workspace.state.hasReadyEnvironment && workspace.environments.length >= 3 && @@ -228,6 +243,11 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) navigation.dispatch(StackActions.popToTop()); return; } + // Follows the environments scene, whose settings sheet popToTop leaves open. + if (requestedScene === "agent-activity") { + navigation.dispatch(CommonActions.reset({ index: 0, routes: [{ name: "Home" }] })); + return; + } const routes: ShowcaseResetRoute[] = [{ name: "Home" }]; if (requestedScene === "environments") { routes.push({ @@ -264,6 +284,39 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) ); }, [hasFixture, navigation, requestedScene, scene, showcaseThread]); + useEffect(() => { + if (!SHOWCASE_ENABLED || scene !== "agent-activity" || !hasFixture || agentActivityStaged) { + return; + } + let cancelled = false; + let lastOutcome: string | null = null; + void retryShowcaseOperation( + async () => { + const now = Date.now(); + const { threads: latestThreads, projects: latestProjects } = entitiesRef.current; + const activity = buildShowcaseAgentActivity(latestThreads, latestProjects, now); + const outcome = + activity === null + ? "fixture threads not loaded" + : await stageShowcaseAgentActivity(activity, now); + if (outcome === true) return true; + // Surfaces in the runner's Metro output when the scene never turns ready. + if (outcome !== lastOutcome) + console.warn(`[showcase] agent activity not staged: ${outcome}`); + lastOutcome = outcome; + return false; + }, + // The first attempt waits on the notification permission prompt until + // the runner answers it. + { isCancelled: () => cancelled, attemptTimeoutMs: 60_000 }, + ).then((staged) => { + if (!cancelled && staged) setAgentActivityStaged(true); + }); + return () => { + cancelled = true; + }; + }, [agentActivityStaged, hasFixture, scene]); + useEffect(() => { if ( !SHOWCASE_ENABLED || @@ -276,6 +329,7 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) !orientationSettled || // Likewise for the palette: an early screenshot shows the default theme. !themeApplied || + (scene === "agent-activity" && !agentActivityStaged) || !isShowcaseNativeContentReady({ scene, themeId, renderSignal }) ) { setReadyScene(null); @@ -298,7 +352,16 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string }) if (renderFrame !== null) cancelAnimationFrame(renderFrame); if (readyFrame !== null) cancelAnimationFrame(readyFrame); }; - }, [hasFixture, orientationSettled, renderSignal, requestedScene, scene, themeApplied, themeId]); + }, [ + agentActivityStaged, + hasFixture, + orientationSettled, + renderSignal, + requestedScene, + scene, + themeApplied, + themeId, + ]); if (!SHOWCASE_ENABLED || readyScene === null) return null; diff --git a/apps/mobile/src/features/showcase/nativeShowcaseScene.ts b/apps/mobile/src/features/showcase/nativeShowcaseScene.ts index 11618291932d..33839b2fe473 100644 --- a/apps/mobile/src/features/showcase/nativeShowcaseScene.ts +++ b/apps/mobile/src/features/showcase/nativeShowcaseScene.ts @@ -2,7 +2,14 @@ import { requireOptionalNativeModule } from "expo"; import { MOBILE_THEME_IDS, type MobileThemeId } from "../../lib/mobileTheme"; -export const SHOWCASE_SCENES = ["threads", "thread", "terminal", "review", "environments"] as const; +export const SHOWCASE_SCENES = [ + "threads", + "thread", + "terminal", + "review", + "environments", + "agent-activity", +] as const; export type ShowcaseScene = (typeof SHOWCASE_SCENES)[number]; export type ShowcaseOrientation = "portrait" | "landscape"; diff --git a/apps/mobile/src/features/showcase/showcaseAgentActivity.test.ts b/apps/mobile/src/features/showcase/showcaseAgentActivity.test.ts new file mode 100644 index 000000000000..94aaa2fc5e9f --- /dev/null +++ b/apps/mobile/src/features/showcase/showcaseAgentActivity.test.ts @@ -0,0 +1,93 @@ +import type { + EnvironmentProject, + EnvironmentThreadShell, +} from "@t3tools/client-runtime/state/shell"; +import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; +import { assert, it } from "@effect/vitest"; + +import { + buildShowcaseAgentActivity, + SHOWCASE_AGENT_ACTIVITY_ROWS, + showcaseAndroidActivityData, +} from "./showcaseAgentActivity"; + +const NOW = Date.parse("2026-07-16T09:00:00.000Z"); + +const project = (environmentId: string, id: string, title: string) => + ({ + environmentId: EnvironmentId.make(environmentId), + id: ProjectId.make(id), + title, + }) as EnvironmentProject; + +const thread = (environmentId: string, id: string, projectId: string, title: string) => + ({ + environmentId: EnvironmentId.make(environmentId), + id: ThreadId.make(id), + projectId: ProjectId.make(projectId), + title, + }) as EnvironmentThreadShell; + +const projects = [ + project("moonbase-terminal", "t3code", "T3 Code"), + project("suspense-station", "react", "React"), + project("kernel-cabin", "linux", "Linux"), +]; + +const threads = [ + thread("moonbase-terminal", "remote-command-center", "t3code", "Make remote coding feel local"), + thread( + "moonbase-terminal", + "pocket-command-center", + "t3code", + "Put the command center in your pocket", + ), + thread("suspense-station", "buttery-suspense", "react", "Make Suspense transitions buttery"), + thread("kernel-cabin", "beautiful-boot", "linux", "Make boot logs oddly beautiful"), +]; + +it("waits until every staged thread and its project have loaded", () => { + assert.isNull(buildShowcaseAgentActivity(threads.slice(1), projects, NOW)); + assert.isNull(buildShowcaseAgentActivity(threads, projects.slice(0, 2), NOW)); +}); + +it("stages relay-shaped rows against the seeded threads", () => { + const activity = buildShowcaseAgentActivity(threads, projects, NOW); + assert.isNotNull(activity); + if (!activity) return; + + assert.strictEqual(activity.activeCount, 3); + assert.deepStrictEqual( + activity.activities.map((row) => [row.threadId, row.status, row.projectTitle, row.updatedAt]), + [ + ["pocket-command-center", "Approval", "T3 Code", "2026-07-16T08:59:00.000Z"], + ["beautiful-boot", "Input", "Linux", "2026-07-16T08:56:00.000Z"], + ["buttery-suspense", "Working", "React", "2026-07-16T08:58:00.000Z"], + ["remote-command-center", "Done", "T3 Code", "2026-07-16T08:57:00.000Z"], + ], + ); + assert.strictEqual( + activity.activities[0]?.deepLink, + "/threads/moonbase-terminal/pocket-command-center", + ); + assert.strictEqual(activity.activities.length, SHOWCASE_AGENT_ACTIVITY_ROWS.length); +}); + +it("encodes the Android Live Update and its alert like a relay push", () => { + const activity = buildShowcaseAgentActivity(threads, projects, NOW); + assert.isNotNull(activity); + if (!activity) return; + const data = showcaseAndroidActivityData(activity, NOW); + + assert.strictEqual(data.active, "true"); + assert.strictEqual(data.activity_chip, "Review"); + assert.strictEqual(data.activity_title, "3 active agents · 2 need attention"); + assert.strictEqual( + data.activity_line_0, + "Approval\tPut the command center in your pocket\tT3 Code", + ); + assert.strictEqual(data.activity_line_3, "Done\tMake remote coding feel local\tT3 Code"); + assert.strictEqual(data.alert_title, "Put the command center in your pocket"); + assert.strictEqual(data.alert_body, "Approval: T3 Code"); + assert.strictEqual(data.alert_path, "/threads/moonbase-terminal/pocket-command-center"); +}); diff --git a/apps/mobile/src/features/showcase/showcaseAgentActivity.ts b/apps/mobile/src/features/showcase/showcaseAgentActivity.ts new file mode 100644 index 000000000000..736c672dff0c --- /dev/null +++ b/apps/mobile/src/features/showcase/showcaseAgentActivity.ts @@ -0,0 +1,138 @@ +import type { + EnvironmentProject, + EnvironmentThreadShell, +} from "@t3tools/client-runtime/state/shell"; + +import type { + AgentActivityPhase, + AgentActivityProps, + AgentActivityRowProps, +} from "../../widgets/AgentActivity"; + +/** + * Agent work shown by the agent-activity scene. The rows point at + * seeded showcase threads so the copy matches the thread list, but the phases + * are staged: the relay that normally derives them is not part of the capture. + * Rows follow the relay's order: attention first, then running, then finished. + */ +export const SHOWCASE_AGENT_ACTIVITY_ROWS = [ + { threadId: "pocket-command-center", phase: "waiting_for_approval", minutesAgo: 1 }, + { threadId: "beautiful-boot", phase: "waiting_for_input", minutesAgo: 4 }, + { threadId: "buttery-suspense", phase: "running", minutesAgo: 2 }, + { threadId: "remote-command-center", phase: "completed", minutesAgo: 3 }, +] as const satisfies ReadonlyArray<{ + readonly threadId: string; + readonly phase: AgentActivityPhase; + readonly minutesAgo: number; +}>; + +// Matches the relay's row wording (AgentActivityPublisher.statusForPhase). +const STATUS_BY_PHASE: Record = { + starting: "Connecting", + running: "Working", + waiting_for_approval: "Approval", + waiting_for_input: "Input", + stale: "Waiting", + completed: "Done", + failed: "Failed", +}; + +const ACTIVE_PHASES: ReadonlySet = new Set([ + "starting", + "running", + "waiting_for_approval", + "waiting_for_input", +]); + +/** Returns null until every staged thread and its project have loaded. */ +export function buildShowcaseAgentActivity( + threads: ReadonlyArray, + projects: ReadonlyArray, + now: number, +): AgentActivityProps | null { + const rows: AgentActivityRowProps[] = []; + for (const definition of SHOWCASE_AGENT_ACTIVITY_ROWS) { + const thread = threads.find((candidate) => String(candidate.id) === definition.threadId); + const project = thread + ? projects.find( + (candidate) => + candidate.environmentId === thread.environmentId && candidate.id === thread.projectId, + ) + : undefined; + if (!thread || !project) return null; + const environmentId = String(thread.environmentId); + rows.push({ + environmentId, + threadId: definition.threadId, + projectTitle: project.title, + threadTitle: thread.title, + modelTitle: "", + phase: definition.phase, + status: STATUS_BY_PHASE[definition.phase], + updatedAt: new Date(now - definition.minutesAgo * 60_000).toISOString(), + deepLink: `/threads/${encodeURIComponent(environmentId)}/${encodeURIComponent(definition.threadId)}`, + }); + } + return { + title: "T3 Code", + subtitle: "Agent work in progress", + activeCount: rows.filter((row) => ACTIVE_PHASES.has(row.phase)).length, + updatedAt: new Date(now).toISOString(), + activities: rows, + }; +} + +/** The alert the relay sends when the hero row starts waiting on the user. */ +export function showcaseAgentAlert(activity: AgentActivityProps) { + const row = activity.activities[0]; + if (!row) return null; + return { + title: row.threadTitle, + body: `${row.status}: ${row.projectTitle}`, + path: row.deepLink, + environmentId: row.environmentId, + threadId: row.threadId, + }; +} + +/** + * The FCM data map Android's native receiver renders, mirroring the relay's + * androidActivityData so the shade shows exactly what a real push produces. + */ +export function showcaseAndroidActivityData( + activity: AgentActivityProps, + now: number, +): Record { + const attentionCount = activity.activities.filter( + (row) => row.phase === "waiting_for_approval" || row.phase === "waiting_for_input", + ).length; + const hero = activity.activities[0]; + const alert = showcaseAgentAlert(activity); + return { + t3_kind: "agent_activity", + updated_at: String(now), + active: String(activity.activeCount > 0), + activity_chip: attentionCount > 0 ? "Review" : "Active", + activity_title: `${activity.activeCount} active agents · ${attentionCount} need attention`, + activity_phase: hero?.phase ?? "", + activity_active_count: String(activity.activeCount), + activity_attention_count: String(attentionCount), + activity_body: hero ? `${hero.status}: ${hero.threadTitle} · ${hero.projectTitle}` : "", + ...Object.fromEntries( + activity.activities.map((row, index) => [ + `activity_line_${index}`, + [row.status, row.threadTitle, row.projectTitle].join("\t"), + ]), + ), + activity_path: hero?.deepLink ?? "/", + activity_expires_at: String(now + 2 * 60 * 60_000), + ...(alert + ? { + alert_id: "showcase-alert", + alert_title: alert.title, + alert_body: alert.body, + alert_path: alert.path, + } + : {}), + }; +} diff --git a/apps/mobile/src/features/showcase/stageShowcaseAgentActivity.ts b/apps/mobile/src/features/showcase/stageShowcaseAgentActivity.ts new file mode 100644 index 000000000000..6b1f61815b37 --- /dev/null +++ b/apps/mobile/src/features/showcase/stageShowcaseAgentActivity.ts @@ -0,0 +1,42 @@ +import * as Notifications from "expo-notifications"; +import { Platform } from "react-native"; + +import type { AgentActivityProps } from "../../widgets/AgentActivity"; +import { + getAgentLiveActivities, + startAgentLiveActivity, +} from "../agent-awareness/agentLiveActivity"; +import { showAndroidShowcaseAgentActivity } from "../agent-awareness/androidNotifications"; +import { showcaseAndroidActivityData } from "./showcaseAgentActivity"; + +/** + * Puts the staged agent activity on screen for the capture runner, which then + * locks the simulator (iOS) or opens the notification shade (Android). + * Resolves true once shown, otherwise the reason it could not be, so the + * caller can retry and report. + */ +export async function stageShowcaseAgentActivity( + activity: AgentActivityProps, + now: number, +): Promise { + // The runner answers the iOS prompt and pre-grants Android's, so this only + // settles the permission the runner's alert delivery depends on. + const permission = await Notifications.requestPermissionsAsync({ + ios: { allowAlert: true, allowBadge: true, allowSound: true }, + }); + if (!permission.granted) return `notification permission ${permission.status}`; + + if (Platform.OS === "android") { + return ( + showAndroidShowcaseAgentActivity(showcaseAndroidActivityData(activity, now)) || + "native showShowcaseActivity missing" + ); + } + if (Platform.OS !== "ios") return `unsupported platform ${Platform.OS}`; + + // A retried or revisited scene must not stack a second card. + await Promise.all(getAgentLiveActivities().map((existing) => existing.end("immediate"))); + // ActivityKit only starts activities while the app is foreground, which + // holds here: the runner locks the device after the scene reports ready. + return startAgentLiveActivity(activity) !== null || "Live Activity did not start"; +} diff --git a/apps/server/src/bin.test.ts b/apps/server/src/bin.test.ts index 9bc20fff84e1..31a96304c3e5 100644 --- a/apps/server/src/bin.test.ts +++ b/apps/server/src/bin.test.ts @@ -16,6 +16,7 @@ import { import * as NetService from "@t3tools/shared/Net"; import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as DateTime from "effect/DateTime"; @@ -106,6 +107,7 @@ const makeCliTestServerConfig = (baseDir: string) => otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, mode: "web", port: 0, host: "127.0.0.1", diff --git a/apps/server/src/cli/config.test.ts b/apps/server/src/cli/config.test.ts index 34ea7f685364..f9b45caf7442 100644 --- a/apps/server/src/cli/config.test.ts +++ b/apps/server/src/cli/config.test.ts @@ -18,6 +18,7 @@ import { } from "@t3tools/contracts"; import * as NetService from "@t3tools/shared/Net"; import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { deriveServerPaths } from "../config.ts"; import { resolveServerConfig } from "./config.ts"; @@ -56,6 +57,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, devAllowedOrigins: [], } as const; @@ -666,6 +668,109 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { }), ); + it.effect("zeroes an endpoint stored in Settings when the SDK is disabled", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-config-otel-off-" }); + const derivedPaths = yield* deriveExplicitServerPaths(baseDir, undefined); + yield* fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }); + yield* fs.writeFileString( + derivedPaths.settingsPath, + // @effect-diagnostics-next-line preferSchemaOverJson:off + `${JSON.stringify({ + observability: { + otlpTracesUrl: "http://localhost:4318/v1/traces", + otlpMetricsUrl: "http://localhost:4318/v1/metrics", + otlpLogsUrl: "http://localhost:4318/v1/logs", + }, + })}\n`, + ); + + const resolved = yield* resolveServerConfig( + { + mode: Option.some("desktop"), + port: Option.some(4888), + host: Option.none(), + baseDir: Option.some(baseDir), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { OTEL_SDK_DISABLED: "true" } })), + NetService.layer, + ), + ), + ); + + // The switch beats every source, including an endpoint stored in Settings. + expect(resolved.otlpTracesUrl).toBeUndefined(); + expect(resolved.otlpMetricsUrl).toBeUndefined(); + expect(resolved.otlpLogsUrl).toBeUndefined(); + expect(resolved.otelEnvironment.disabled).toBe(true); + }), + ); + + it.effect("lets T3CODE_OTEL_SDK_DISABLED=false override an ambient OTEL_SDK_DISABLED=true", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-config-otel-on-" }); + const derivedPaths = yield* deriveExplicitServerPaths(baseDir, undefined); + yield* fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }); + yield* fs.writeFileString( + derivedPaths.settingsPath, + // @effect-diagnostics-next-line preferSchemaOverJson:off + `${JSON.stringify({ + observability: { + otlpTracesUrl: "http://localhost:4318/v1/traces", + }, + })}\n`, + ); + + const resolved = yield* resolveServerConfig( + { + mode: Option.some("desktop"), + port: Option.some(4888), + host: Option.none(), + baseDir: Option.some(baseDir), + cwd: Option.none(), + devUrl: Option.none(), + noBrowser: Option.none(), + bootstrapFd: Option.none(), + autoBootstrapProjectFromCwd: Option.none(), + logWebSocketEvents: Option.none(), + tailscaleServeEnabled: Option.none(), + tailscaleServePort: Option.none(), + }, + Option.none(), + ).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { T3CODE_OTEL_SDK_DISABLED: "false", OTEL_SDK_DISABLED: "true" }, + }), + ), + NetService.layer, + ), + ), + ); + + expect(resolved.otelEnvironment.disabled).toBe(false); + expect(resolved.otlpTracesUrl).toBe("http://localhost:4318/v1/traces"); + }), + ); + it.effect("forces noBrowser and disables auto-bootstrap for headless startup presentation", () => Effect.gen(function* () { const { join } = yield* Path.Path; diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index 09a30aeb19e7..1b6449433139 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -4,6 +4,7 @@ import { OtlpProtocol, type SignalExport, } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import { parsePersistedServerObservabilitySettings } from "@t3tools/shared/serverSettings"; import { DesktopBackendBootstrap, PortSchema } from "@t3tools/contracts"; import * as Config from "effect/Config"; @@ -386,6 +387,8 @@ export const resolveServerConfig = ( ); const logLevel = Option.getOrElse(cliLogLevel, () => env.logLevel); + const otel = yield* OtelEnvironment.load; + // T3 Code's own OTLP variables name no signal, so the one answer they give // is the answer for all three. const signalExport: SignalExport = { @@ -401,20 +404,24 @@ export const resolveServerConfig = ( traceBatchWindowMs: env.traceBatchWindowMs, traceMaxBytes: env.traceMaxBytes, traceMaxFiles: env.traceMaxFiles, - otlpTracesUrl: - env.otlpTracesUrl ?? - bootstrap?.otlpTracesUrl ?? - persistedObservabilitySettings.otlpTracesUrl, - otlpMetricsUrl: - env.otlpMetricsUrl ?? - bootstrap?.otlpMetricsUrl ?? - persistedObservabilitySettings.otlpMetricsUrl, - otlpLogsUrl: - env.otlpLogsUrl ?? bootstrap?.otlpLogsUrl ?? persistedObservabilitySettings.otlpLogsUrl, + otlpTracesUrl: otel.disabled + ? undefined + : (env.otlpTracesUrl ?? + bootstrap?.otlpTracesUrl ?? + persistedObservabilitySettings.otlpTracesUrl), + otlpMetricsUrl: otel.disabled + ? undefined + : (env.otlpMetricsUrl ?? + bootstrap?.otlpMetricsUrl ?? + persistedObservabilitySettings.otlpMetricsUrl), + otlpLogsUrl: otel.disabled + ? undefined + : (env.otlpLogsUrl ?? bootstrap?.otlpLogsUrl ?? persistedObservabilitySettings.otlpLogsUrl), otlpTracesExport: signalExport, otlpMetricsExport: signalExport, otlpLogsExport: signalExport, otlpServiceName: env.otlpServiceName, + otelEnvironment: otel, mode, port, cwd, diff --git a/apps/server/src/cli/pair.ts b/apps/server/src/cli/pair.ts index 493e6b719416..6b29ef60927f 100644 --- a/apps/server/src/cli/pair.ts +++ b/apps/server/src/cli/pair.ts @@ -16,6 +16,7 @@ import { } from "@t3tools/contracts"; import { resolveWorktreeT3Home } from "@t3tools/shared/devHome"; import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import { buildTailscaleHttpsBaseUrl, DEFAULT_TAILSCALE_SERVE_PORT, @@ -326,6 +327,7 @@ const makePairServerConfig = Effect.fn(function* (input: { otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, mode: "web", port: state.port, host: state.host, diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 5762ccdb6ff3..344619c0eb93 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -18,6 +18,7 @@ import * as Schema from "effect/Schema"; import { sweepStalePendingAttachments } from "./attachmentStore.ts"; import { DEFAULT_SIGNAL_EXPORT, type SignalExport } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; export const DEFAULT_PORT = 3773; @@ -82,6 +83,7 @@ export class ServerConfig extends Context.Service< readonly otlpMetricsExport: SignalExport; readonly otlpLogsExport: SignalExport; readonly otlpServiceName: string; + readonly otelEnvironment: OtelEnvironment.OtelEnvironment; readonly mode: RuntimeMode; readonly port: number; readonly host: string | undefined; @@ -221,6 +223,7 @@ const makeTest = Effect.fn("ServerConfig.makeTest")(function* ( otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, cwd, baseDir, ...derivedPaths, diff --git a/apps/server/src/environment/ServerEnvironment.test.ts b/apps/server/src/environment/ServerEnvironment.test.ts index b4758e980065..6181b9702bb3 100644 --- a/apps/server/src/environment/ServerEnvironment.test.ts +++ b/apps/server/src/environment/ServerEnvironment.test.ts @@ -11,6 +11,7 @@ import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { @@ -60,6 +61,7 @@ const makeServerConfig = Effect.fn(function* (baseDir: string) { otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, cwd: process.cwd(), baseDir, mode: "web", diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 9fc482d366d3..4241232bdf87 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -2,6 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; import { HttpServer } from "effect/unstable/http"; import * as NetAddress from "effect/unstable/net/NetAddress"; @@ -163,3 +164,64 @@ it.effect("does not keep credentials of other threads alive", () => expect(yield* registry.resolve(token)).toBeUndefined(); }), ); + +it.effect("revokes every superseded thread credential except the accepted provider session", () => { + const layer = McpSessionRegistry.layer.pipe( + Layer.provide(Layer.succeed(HttpServer.HttpServer, fakeHttpServer)), + Layer.provide(Layer.succeed(ServerEnvironment.ServerEnvironment, fakeEnvironment)), + Layer.provide(NodeServices.layer), + ); + return Effect.gen(function* () { + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-retain"); + const otherThreadId = ThreadId.make("thread-retain-other"); + const issue = (id: ThreadId, retainExisting: boolean) => + McpSessionRegistry.issueActiveMcpCredential({ + threadId: id, + providerInstanceId: ProviderInstanceId.make("claudeAgent"), + capabilities: new Set(["preview"]), + ...(retainExisting ? { retainExisting: true } : {}), + }); + const bearer = (header: string | undefined) => header?.replace(/^Bearer\s+/, "") ?? ""; + + const first = yield* issue(threadId, false); + const replaced = yield* issue(threadId, false); + const kept = yield* issue(otherThreadId, false); + if (first === undefined || replaced === undefined || kept === undefined) { + return yield* Effect.die("expected MCP credentials"); + } + const firstToken = bearer(first.config.authorizationHeader); + const replacedToken = bearer(replaced.config.authorizationHeader); + expect(yield* registry.resolve(firstToken)).toBeUndefined(); + expect((yield* registry.resolve(replacedToken))?.threadId).toBe(threadId); + + const retained = yield* issue(threadId, true); + const superseded = yield* issue(threadId, true); + if (retained === undefined || superseded === undefined) { + return yield* Effect.die("expected retained MCP credentials"); + } + const retainedToken = bearer(retained.config.authorizationHeader); + const supersededToken = bearer(superseded.config.authorizationHeader); + const keptToken = bearer(kept.config.authorizationHeader); + + expect((yield* registry.resolve(replacedToken))?.threadId).toBe(threadId); + expect((yield* registry.resolve(retainedToken))?.threadId).toBe(threadId); + expect((yield* registry.resolve(supersededToken))?.threadId).toBe(threadId); + expect(yield* registry.hasProviderSession(retained.config.providerSessionId)).toBe(true); + + yield* McpSessionRegistry.revokeActiveMcpProviderSession(retained.config.providerSessionId); + expect(yield* registry.resolve(retainedToken)).toBeUndefined(); + expect((yield* registry.resolve(replacedToken))?.threadId).toBe(threadId); + expect((yield* registry.resolve(supersededToken))?.threadId).toBe(threadId); + + yield* McpSessionRegistry.revokeActiveMcpThreadExcept( + threadId, + superseded.config.providerSessionId, + ); + expect(yield* registry.resolve(replacedToken)).toBeUndefined(); + expect(yield* registry.resolve(retainedToken)).toBeUndefined(); + expect((yield* registry.resolve(supersededToken))?.threadId).toBe(threadId); + expect(yield* registry.hasProviderSession(superseded.config.providerSessionId)).toBe(true); + expect((yield* registry.resolve(keptToken))?.threadId).toBe(otherThreadId); + }).pipe(Effect.provide(layer)); +}); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index fa0a17d826f9..42d08abc4a4b 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -16,6 +16,11 @@ export interface McpCredentialRequest { readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; readonly capabilities: ReadonlySet; + /** + * Keep credentials already issued for this thread. A refused session + * replacement still needs the token the running query baked in. + */ + readonly retainExisting?: boolean; } export interface McpIssuedCredential { @@ -34,6 +39,17 @@ export interface McpSessionRegistryShape { */ readonly touch: (threadId: ThreadId) => Effect.Effect; readonly revokeProviderSession: (providerSessionId: string) => Effect.Effect; + /** + * Drop every credential for a thread except one provider session. + * An accepted replacement uses this so every superseded bearer stops + * authorizing `/mcp`. + */ + readonly revokeThreadExcept: ( + threadId: ThreadId, + keepProviderSessionId: string, + ) => Effect.Effect; + /** Whether a provider session's bearer token is still authorized. */ + readonly hasProviderSession: (providerSessionId: string) => Effect.Effect; readonly revokeThread: (threadId: ThreadId) => Effect.Effect; readonly revokeAll: Effect.Effect; } @@ -196,6 +212,27 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( yield* revokeWhere((record) => record.scope.providerSessionId === providerSessionId); }, ), + revokeThreadExcept: Effect.fn("McpSessionRegistry.revokeThreadExcept")( + function* (threadId, keepProviderSessionId) { + yield* revokeWhere( + (record) => + record.scope.threadId === threadId && + record.scope.providerSessionId !== keepProviderSessionId, + ); + }, + ), + hasProviderSession: Effect.fn("McpSessionRegistry.hasProviderSession")( + function* (providerSessionId) { + const timestamp = yield* currentTimeMillis; + return yield* SynchronizedRef.modify(state, ({ records }) => { + const current = pruneDead(records, timestamp); + const alive = Array.from(current.values()).some( + (record) => record.scope.providerSessionId === providerSessionId, + ); + return [alive, { records: current }] as const; + }); + }, + ), revokeThread: Effect.fn("McpSessionRegistry.revokeThread")(function* (threadId) { yield* revokeWhere((record) => record.scope.threadId === threadId); }), @@ -223,15 +260,46 @@ const make = Effect.acquireRelease( export const layer = Layer.effect(McpSessionRegistry, make); +/** + * Issue the MCP credential a provider session presents to `/mcp`. + * Replaces any credential already issued for the thread unless + * `retainExisting` is set, which keeps the live query's token valid until + * the caller revokes it. + */ export const issueActiveMcpCredential = ( request: McpCredentialRequest, ): Effect.Effect => activeMcpSessionRegistry - ? activeMcpSessionRegistry - .revokeThread(request.threadId) - .pipe(Effect.andThen(activeMcpSessionRegistry.issue(request))) + ? (request.retainExisting === true + ? Effect.void + : activeMcpSessionRegistry.revokeThread(request.threadId) + ).pipe(Effect.andThen(activeMcpSessionRegistry.issue(request))) : Effect.sync((): McpIssuedCredential | undefined => undefined); +/** Revoke one issued MCP credential without touching the thread's other tokens. */ +export const revokeActiveMcpProviderSession = (providerSessionId: string): Effect.Effect => + activeMcpSessionRegistry + ? activeMcpSessionRegistry.revokeProviderSession(providerSessionId) + : Effect.void; + +/** + * Revoke every MCP credential for a thread except the provider session a + * replacement just accepted. + */ +export const revokeActiveMcpThreadExcept = ( + threadId: ThreadId, + keepProviderSessionId: string, +): Effect.Effect => + activeMcpSessionRegistry + ? activeMcpSessionRegistry.revokeThreadExcept(threadId, keepProviderSessionId) + : Effect.void; + +/** Whether a provider session's bearer token is still authorized for `/mcp`. */ +export const hasActiveMcpProviderSession = (providerSessionId: string): Effect.Effect => + activeMcpSessionRegistry + ? activeMcpSessionRegistry.hasProviderSession(providerSessionId) + : Effect.succeed(false); + /** * Refreshes the liveness of a thread's MCP credential. Called on every provider * turn so an active session is never mistaken for an abandoned one. diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts index ef552f9b04d0..7a8ed739d734 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts @@ -685,6 +685,7 @@ it.effect("pins a provider session to its initial host despite later focus chang environmentId: scope.environmentId, connectionId: "connection-stale", focused: true, + liveTabs: [{ threadId: scope.threadId, tabId: PreviewTabId.make("stale-tab") }], }); expect(yield* broker.invoke({ scope, operation: "status", input: {} })).toBe( "second", @@ -725,6 +726,130 @@ it.effect("pins a provider session to its initial host despite later focus chang ), ); +it.effect("prefers the live tab owner for new sessions without moving existing leases", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* makeBroker; + const connections = new Map(); + for (const clientId of ["owner", "other"]) { + const requests = requestsFrom( + yield* broker.connect(makeHost({ clientId })), + (connectionId) => connections.set(clientId, connectionId), + ); + yield* Stream.runForEach(requests, (request) => + broker.respond({ + clientId, + connectionId: request.connectionId, + requestId: request.requestId, + ok: true, + result: clientId, + }), + ).pipe(Effect.forkScoped); + } + yield* Effect.yieldNow; + yield* broker.focusHost({ + clientId: "owner", + environmentId: scope.environmentId, + connectionId: connections.get("owner")!, + focused: false, + liveTabs: [ + { threadId: scope.threadId, tabId: PreviewTabId.make("signed-in"), visible: true }, + ], + }); + yield* broker.focusHost({ + clientId: "other", + environmentId: scope.environmentId, + connectionId: connections.get("other")!, + focused: true, + liveTabs: [ + { threadId: scope.threadId, tabId: PreviewTabId.make("signed-in"), visible: false }, + { + threadId: ThreadId.make("another-thread"), + tabId: PreviewTabId.make("different-tab"), + visible: true, + }, + ], + }); + expect(yield* broker.invoke({ scope, operation: "evaluate", input: {} })).toBe( + "owner", + ); + expect( + yield* broker.invoke({ + scope: { ...scope, providerSessionId: "explicit-owner" }, + tabId: PreviewTabId.make("signed-in"), + operation: "snapshot", + input: {}, + }), + ).toBe("owner"); + expect( + yield* broker.invoke({ + scope: { ...scope, providerSessionId: "other-tab" }, + tabId: PreviewTabId.make("different-tab"), + operation: "evaluate", + input: {}, + }), + ).toBe("other"); + + yield* broker.focusHost({ + clientId: "owner", + environmentId: scope.environmentId, + connectionId: connections.get("owner")!, + focused: false, + liveTabs: [], + }); + expect(yield* broker.invoke({ scope, operation: "evaluate", input: {} })).toBe( + "owner", + ); + expect( + yield* broker.invoke({ + scope: { ...scope, providerSessionId: "after-tab-closed" }, + operation: "evaluate", + input: {}, + }), + ).toBe("other"); + }), + ), +); + +it.effect("prefers a focused host over unrelated extra capabilities for a new session", () => + Effect.scoped( + Effect.gen(function* () { + const broker = yield* makeBroker; + let focusedConnectionId = ""; + for (const [clientId, supportedOperations] of [ + ["focused", ["status"]], + ["background", ["status", "resize"]], + ] as const) { + const requests = requestsFrom( + yield* broker.connect(makeHost({ clientId, supportedOperations })), + (connectionId) => { + if (clientId === "focused") focusedConnectionId = connectionId; + }, + ); + yield* Stream.runForEach(requests, (request) => + broker.respond({ + clientId, + connectionId: request.connectionId, + requestId: request.requestId, + ok: true, + result: clientId, + }), + ).pipe(Effect.forkScoped); + } + yield* Effect.yieldNow; + yield* broker.focusHost({ + clientId: "focused", + environmentId: scope.environmentId, + connectionId: focusedConnectionId, + focused: true, + }); + expect(yield* broker.invoke({ scope, operation: "status", input: {} })).toBe( + "focused", + ); + }), + ), +); + it.effect("does not route new operations to legacy hosts that did not advertise support", () => Effect.scoped( Effect.gen(function* () { @@ -921,6 +1046,7 @@ it.effect("fails over a pinned provider session only after its host disconnects" environmentId: scope.environmentId, connectionId: firstConnectionId, focused: true, + liveTabs: [{ threadId: scope.threadId, tabId: firstTabId }], }); expect(yield* broker.invoke({ scope, operation: "open", input: {} })).toEqual({ host: "first", diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index 527f71f5915d..65e3064f49a5 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -73,6 +73,7 @@ interface ClientConnection { readonly environmentId: PreviewAutomationHost["environmentId"]; readonly supportedOperations: ReadonlySet; readonly focused: boolean; + readonly liveTabs: NonNullable; readonly focusOrder: number; readonly queue: Queue.Queue; } @@ -376,6 +377,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { environmentId: host.environmentId, supportedOperations: new Set(host.supportedOperations ?? PREVIEW_AUTOMATION_V1_OPERATIONS), focused: false, + liveTabs: [], focusOrder: 0, queue, }; @@ -432,6 +434,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { clients.set(host.clientId, { ...currentHost, focused: host.focused, + liveTabs: host.liveTabs ?? currentHost.liveTabs, focusOrder: host.focused ? focusSequence : currentHost.focusOrder, }); return { ...current, clients, focusSequence }; @@ -492,6 +495,13 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { // operation is not silently moved to a newer client: the caller gets a // capability failure and can deliberately start a fresh provider // session. A dead lease is pruned above and may fail over. + const ownsTargetTab = (host: ClientConnection, visibleOnly = false) => + host.liveTabs.some( + (tab) => + tab.threadId === input.scope.threadId && + (!visibleOnly || tab.visible === true) && + (input.tabId === undefined || tab.tabId === input.tabId), + ); const connection = hasLiveAssignment && supportsOperation(assignedConnection, input.operation) ? assignedConnection @@ -505,7 +515,8 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { ) .sort( (left, right) => - right.supportedOperations.size - left.supportedOperations.size || + Number(ownsTargetTab(right, true)) - Number(ownsTargetTab(left, true)) || + Number(ownsTargetTab(right)) - Number(ownsTargetTab(left)) || Number(right.focused) - Number(left.focused) || right.focusOrder - left.focusOrder, )[0]; diff --git a/apps/server/src/observability/Layers/Observability.ts b/apps/server/src/observability/Layers/Observability.ts index 8f7b607745f4..a754933e5d62 100644 --- a/apps/server/src/observability/Layers/Observability.ts +++ b/apps/server/src/observability/Layers/Observability.ts @@ -20,6 +20,7 @@ import * as BrowserTraceCollector from "../BrowserTraceCollector.ts"; export const ObservabilityLive = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; + const traces = config.otlpTracesExport; const metrics = config.otlpMetricsExport; // The trace serializer stays in the returned context because the browser @@ -86,6 +87,15 @@ export const ObservabilityLive = Layer.unwrap( resource, }).pipe(Layer.provide(otlpSerializationLayer(metrics.protocol))); - return Layer.mergeAll(ServerLoggerLive, traceReferencesLayer, tracerLayer, metricsLayer); + // Logged once the server's loggers are installed, so the warnings use them. + const otelWarningsLayer = Layer.effectDiscard( + Effect.forEach(config.otelEnvironment.warnings, (warning) => Effect.logWarning(warning)), + ); + + return otelWarningsLayer.pipe( + Layer.provideMerge( + Layer.mergeAll(ServerLoggerLive, traceReferencesLayer, tracerLayer, metricsLayer), + ), + ); }), ); diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index ca596e6501cf..18990256bdd4 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -46,7 +46,11 @@ import { SYNTHETIC_CLAUDE_STANDARD_MODEL, SYNTHETIC_CLAUDE_THINKING_MODEL, } from "../ClaudeModelCatalog.testFixtures.ts"; -import { ProviderAdapterProcessError, ProviderAdapterValidationError } from "../Errors.ts"; +import { + ProviderAdapterProcessError, + ProviderAdapterRequestError, + ProviderAdapterValidationError, +} from "../Errors.ts"; import type { ClaudeAdapterShape } from "../Services/ClaudeAdapter.ts"; import type { ClaudeScopedLimitNames } from "./claudeUsageLimits.ts"; import { makeClaudeAdapter, type ClaudeAdapterLiveOptions } from "./ClaudeAdapter.ts"; @@ -367,7 +371,167 @@ const sendCompletedClaudeTurn = ( return turn; }); +function makeQueryRecordingClaudeLayer() { + const queries: Array = []; + const layer = Layer.effect( + ClaudeAdapter, + Effect.gen(function* () { + const claudeConfig = decodeClaudeSettings({}); + return yield* makeClaudeAdapter(claudeConfig, { + createQuery: () => { + const query = new FakeClaudeQuery(); + queries.push(query); + return query; + }, + }); + }), + ).pipe( + Layer.provideMerge(ServerConfig.layerTest("/tmp/claude-adapter-test", "/tmp")), + Layer.provideMerge(ServerSettingsService.layerTest()), + Layer.provideMerge(NodeServices.layer), + ); + return { queries, layer }; +} + describe("ClaudeAdapterLive", () => { + it.effect("does not replace a session while live tasks are still running", () => { + const harness = makeQueryRecordingClaudeLayer(); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + const taskStartedFiber = yield* adapter.streamEvents.pipe( + Stream.filter((event) => event.type === "task.started"), + Stream.take(1), + Stream.runCollect, + Effect.forkChild, + ); + const firstSession = yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ + threadId: firstSession.threadId, + input: "spawn agents", + attachments: [], + }); + harness.queries[0]?.emit({ + type: "system", + subtype: "task_started", + task_id: "task-live", + description: "Agent A", + task_type: "local_agent", + uuid: "task-live-uuid", + session_id: "sdk-session", + } as unknown as SDKMessage); + yield* Fiber.join(taskStartedFiber); + + const warningFiber = yield* adapter.streamEvents.pipe( + Stream.filter((event) => event.type === "runtime.warning"), + Stream.take(1), + Stream.runCollect, + Effect.forkChild, + ); + const replaceError = yield* adapter + .startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "approval-required", + resumeCursor: firstSession.resumeCursor, + }) + .pipe(Effect.flip); + const warnings = Array.from(yield* Fiber.join(warningFiber)); + const activeSessions = yield* adapter.listSessions(); + + assert.equal(replaceError._tag, "ProviderAdapterRequestError"); + if (!(replaceError instanceof ProviderAdapterRequestError)) { + return; + } + assert.equal(replaceError.method, "startSession"); + assert.match(replaceError.detail, /live task/); + assert.equal(harness.queries.length, 1); + assert.equal(harness.queries[0]?.closeCalls, 0); + assert.equal(yield* adapter.hasSession(THREAD_ID), true); + assert.equal(activeSessions.length, 1); + assert.equal(activeSessions[0]?.runtimeMode, "full-access"); + assert.equal(warnings.length, 1); + if (warnings[0]?.type === "runtime.warning") { + assert.match(warnings[0].payload.message, /live task/); + } + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + + it.effect("replaces a session after its live tasks have finished", () => { + const harness = makeQueryRecordingClaudeLayer(); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + const taskStartedFiber = yield* adapter.streamEvents.pipe( + Stream.filter((event) => event.type === "task.started"), + Stream.take(1), + Stream.runCollect, + Effect.forkChild, + ); + const firstSession = yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ + threadId: firstSession.threadId, + input: "spawn agents", + attachments: [], + }); + harness.queries[0]?.emit({ + type: "system", + subtype: "task_started", + task_id: "task-done", + description: "Agent A", + task_type: "local_agent", + uuid: "task-done-uuid", + session_id: "sdk-session", + } as unknown as SDKMessage); + yield* Fiber.join(taskStartedFiber); + + const taskCompletedFiber = yield* adapter.streamEvents.pipe( + Stream.filter((event) => event.type === "task.completed"), + Stream.take(1), + Stream.runCollect, + Effect.forkChild, + ); + harness.queries[0]?.emit({ + type: "system", + subtype: "task_notification", + task_id: "task-done", + status: "completed", + summary: "done", + uuid: "task-done-notify", + session_id: "sdk-session", + } as unknown as SDKMessage); + yield* Fiber.join(taskCompletedFiber); + + const secondSession = yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "approval-required", + resumeCursor: firstSession.resumeCursor, + }); + const activeSessions = yield* adapter.listSessions(); + + assert.equal(harness.queries.length, 2); + assert.equal(harness.queries[0]?.closeCalls, 1); + assert.equal(harness.queries[1]?.closeCalls, 0); + assert.equal(yield* adapter.hasSession(THREAD_ID), true); + assert.equal(activeSessions.length, 1); + assert.equal(activeSessions[0]?.runtimeMode, "approval-required"); + assert.deepEqual(activeSessions[0]?.resumeCursor, secondSession.resumeCursor); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }); + it.effect("returns validation error for non-claude provider on startSession", () => { const harness = makeHarness(); return Effect.gen(function* () { diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 13645a33a05b..c6cee758effb 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -2038,6 +2038,17 @@ function describeUnknownSdkMessage(kind: string, message: unknown): string { return preview ? `${kind} — ${preview}` : `${kind} (no displayable text content)`; } +/** + * Replacing the Claude query stops child tasks and records them completed + * without `session.exited`. Refuse that while any child task is still running. + */ +function shouldBlockClaudeSessionReplacementForLiveTasks(existingContext: { + readonly stopped: boolean; + readonly liveTaskIds: ReadonlySet; +}): boolean { + return !existingContext.stopped && existingContext.liveTaskIds.size > 0; +} + function sdkNativeItemId(message: SDKMessage): string | undefined { if (message.type === "assistant") { const maybeId = (message.message as { id?: unknown }).id; @@ -4404,6 +4415,25 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const existingContext = sessions.get(input.threadId); if (existingContext) { + // Fail instead of returning the current session: callers bind the + // requested runtime mode and send the turn on whatever comes back. + // Stopping here would mark live tasks completed without session.exited. + if (shouldBlockClaudeSessionReplacementForLiveTasks(existingContext)) { + const liveTaskCount = existingContext.liveTaskIds.size; + yield* emitRuntimeWarning( + existingContext, + `Claude session replacement blocked: ${liveTaskCount} live task(s) are still running.`, + { + existingSessionStatus: existingContext.session.status, + liveTaskCount, + }, + ); + return yield* new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "startSession", + detail: `Claude session replacement is blocked: ${liveTaskCount} live task(s) are still running. Retry after background work finishes.`, + }); + } yield* Effect.logWarning("claude.session.replacing", { threadId: input.threadId, existingSessionStatus: existingContext.session.status, diff --git a/apps/server/src/provider/Layers/ProviderRegistry.test.ts b/apps/server/src/provider/Layers/ProviderRegistry.test.ts index 354484e6d395..9ddf54deb3af 100644 --- a/apps/server/src/provider/Layers/ProviderRegistry.test.ts +++ b/apps/server/src/provider/Layers/ProviderRegistry.test.ts @@ -38,6 +38,7 @@ import { checkClaudeProviderStatus } from "./ClaudeProvider.ts"; import * as BackgroundPolicy from "../../background/BackgroundPolicy.ts"; import { AntigravityInstallation } from "../AntigravityInstallation.ts"; import * as ModelManifest from "../ModelManifest.ts"; +import { applyProviderCompatibility } from "../providerCompatibility.ts"; import * as CodexResetCredit from "./codexResetCredit.ts"; import * as OpenCodeRuntime from "../opencodeRuntime.ts"; import * as ProviderEventLoggers from "./ProviderEventLoggers.ts"; @@ -76,6 +77,12 @@ process.env.T3CODE_CURSOR_ENABLED = "1"; const encoder = new TextEncoder(); const TEST_EPOCH = DateTime.makeUnsafe("1970-01-01T00:00:00.000Z"); +const withBundledCompatibility = (snapshot: ServerProvider) => + applyProviderCompatibility( + snapshot, + undefined, + ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, + ); // Provider metadata checks use a bundled manifest and stubbed HTTP. const TestHttpClientLive = Layer.succeed( @@ -1754,7 +1761,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te ); assert.deepStrictEqual( recoveredProviders.find((provider) => provider.instanceId === codexInstanceId), - codexProvider, + withBundledCompatibility(codexProvider), ); yield* Ref.set(catalogSnapshot, changedCatalogProvider); @@ -1766,7 +1773,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te ); assert.deepStrictEqual( changedProviders.find((provider) => provider.instanceId === codexInstanceId), - codexProvider, + withBundledCompatibility(codexProvider), ); }).pipe(Effect.provide(runtimeServices)); @@ -1880,10 +1887,13 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te yield* Fiber.join(persisted); const cachedProvider = yield* readProviderStatusCache(filePath); - assert.deepStrictEqual(cachedProvider, { - ...refreshedProvider, - models: [...initialProvider.models], - }); + assert.deepStrictEqual( + cachedProvider, + withBundledCompatibility({ + ...refreshedProvider, + models: [...initialProvider.models], + }), + ); }).pipe(Effect.provide(runtimeServices)); }), ); @@ -2095,10 +2105,14 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te yield* Effect.gen(function* () { const registry = yield* ProviderRegistry.ProviderRegistry; - assert.deepStrictEqual(yield* registry.getProviders, [cachedProvider]); - assert.deepStrictEqual(yield* registry.refresh(codexDriver), [cachedProvider]); + assert.deepStrictEqual(yield* registry.getProviders, [ + withBundledCompatibility(cachedProvider), + ]); + assert.deepStrictEqual(yield* registry.refresh(codexDriver), [ + withBundledCompatibility(cachedProvider), + ]); assert.deepStrictEqual(yield* registry.refreshInstance(codexInstanceId), [ - cachedProvider, + withBundledCompatibility(cachedProvider), ]); }).pipe(Effect.provide(runtimeServices)); }), @@ -2206,7 +2220,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, ServerSettingsModule.layerTest(), Te yield* Effect.gen(function* () { const registry = yield* ProviderRegistry.ProviderRegistry; - assert.deepStrictEqual(yield* registry.getProviders, [codexProvider]); + assert.deepStrictEqual(yield* registry.getProviders, [ + withBundledCompatibility(codexProvider), + ]); yield* Ref.set(failNextList, true); yield* PubSub.publish(changes, undefined); diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index b9997e1df312..93d27b23f114 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -50,6 +50,8 @@ import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; +import { HttpServer } from "effect/unstable/http"; +import * as NetAddress from "effect/unstable/net/NetAddress"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { @@ -74,6 +76,9 @@ import { SqlitePersistenceMemory, } from "../../persistence/Layers/Sqlite.ts"; import * as ServerConfig from "../../config.ts"; +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; +import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; import * as ServerSettings from "../../serverSettings.ts"; import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; import { makeAdapterRegistryMock } from "../testUtils/providerAdapterRegistryMock.ts"; @@ -5150,3 +5155,533 @@ describe("agent browser access", () => { }).pipe(Effect.provide(NodeServices.layer)), ); }); + +interface ClaudeStartHold { + readonly entered: Deferred.Deferred; + readonly release: Deferred.Deferred; +} + +interface ClaudeStartControl { + rejectNextStart: boolean; + stopThenFailProbe: boolean; + failHasSession: boolean; + hold: ClaudeStartHold | null; + /** Stop the session, wait, then let the original start finish. */ + gapHold: ClaudeStartHold | null; + /** Completed if another startSession begins after the gap hold is waiting. */ + extraStart: Deferred.Deferred | null; +} + +/** + * Two Claude instances that share a continuation key. Ids in `missingAdapters` + * still have routing info, but `getByInstance` fails as if that adapter is gone. + */ +function makeSharedClaudeRegistry( + entries: ReadonlyArray]>, + missingAdapters?: ReadonlySet, +): ProviderAdapterRegistry.ProviderAdapterRegistry["Service"] { + const adapters = new Map(entries); + const continuationKey = "claudeAgent:shared-continuation"; + const unsupported = new ProviderUnsupportedError({ provider: CLAUDE_AGENT_DRIVER }); + return { + getByInstance: (instanceId) => { + if (missingAdapters?.has(instanceId)) { + return Effect.fail(unsupported); + } + const adapter = adapters.get(instanceId); + return adapter ? Effect.succeed(adapter) : Effect.fail(unsupported); + }, + getInstanceInfo: (instanceId) => { + const adapter = adapters.get(instanceId); + if (!adapter) { + return Effect.fail(unsupported); + } + return Effect.succeed({ + instanceId, + driverKind: adapter.provider, + displayName: undefined, + enabled: true, + continuationIdentity: { + driverKind: adapter.provider, + continuationKey, + }, + }); + }, + listInstances: () => Effect.succeed(Array.from(adapters.keys())), + subscribeChanges: Effect.flatMap(PubSub.unbounded(), (pubsub) => + PubSub.subscribe(pubsub), + ), + }; +} + +/** + * Record the MCP header a start would bake in. Can refuse the start, stall it, + * or stop the current session and then fail the follow-up probe. + */ +function instrumentClaudeStart( + claude: ReturnType, + seenAuthorizationHeaders: Array, + control: ClaudeStartControl, +) { + const originalStart = claude.startSession.getMockImplementation() as + | ((input: ProviderSessionStartInput) => Effect.Effect) + | undefined; + const originalHasSession = claude.hasSession.getMockImplementation() as + | ((threadId: ThreadId) => Effect.Effect) + | undefined; + + const probeSession = (threadId: ThreadId): Effect.Effect => { + if (control.failHasSession) { + return Effect.fail( + new ProviderAdapterRequestError({ + provider: CLAUDE_AGENT_DRIVER, + method: "hasSession", + detail: "session probe failed after replacement", + }), + ) as unknown as Effect.Effect; + } + if (originalHasSession === undefined) { + return Effect.die("Claude hasSession mock is missing its original implementation"); + } + return originalHasSession(threadId); + }; + + const implementStart = (input: ProviderSessionStartInput) => + Effect.gen(function* () { + if (control.extraStart !== null && control.hold === null && control.gapHold === null) { + yield* Deferred.succeed(control.extraStart, undefined).pipe(Effect.ignore); + } + seenAuthorizationHeaders.push( + McpProviderSession.readMcpProviderSession(input.threadId)?.authorizationHeader, + ); + const hold = control.hold; + if (hold !== null) { + control.hold = null; + yield* Deferred.succeed(hold.entered, undefined); + yield* Deferred.await(hold.release); + return yield* new ProviderAdapterRequestError({ + provider: CLAUDE_AGENT_DRIVER, + method: "startSession", + detail: "Claude session replacement is blocked: 1 live task(s) are still running.", + }); + } + const gapHold = control.gapHold; + if (gapHold !== null) { + control.gapHold = null; + yield* claude.stopSession(input.threadId); + yield* Deferred.succeed(gapHold.entered, undefined); + yield* Deferred.await(gapHold.release); + } + if (control.stopThenFailProbe) { + control.stopThenFailProbe = false; + yield* claude.stopSession(input.threadId); + control.failHasSession = true; + return yield* new ProviderAdapterRequestError({ + provider: CLAUDE_AGENT_DRIVER, + method: "startSession", + detail: "replacement stopped the session and the follow-up probe failed", + }); + } + if (control.rejectNextStart) { + control.rejectNextStart = false; + return yield* new ProviderAdapterRequestError({ + provider: CLAUDE_AGENT_DRIVER, + method: "startSession", + detail: "Claude session replacement is blocked: 1 live task(s) are still running.", + }); + } + if (originalStart === undefined) { + return yield* Effect.die("Claude startSession mock is missing its original implementation"); + } + return yield* originalStart(input); + }); + + ( + claude.hasSession as unknown as { + mockImplementation(fn: typeof probeSession): void; + } + ).mockImplementation(probeSession); + ( + claude.startSession as unknown as { + mockImplementation(fn: typeof implementStart): void; + } + ).mockImplementation(implementStart); +} + +function makeClaudeMcpProviderHarness(options?: { + readonly secondaryInstanceId?: ProviderInstanceId; + readonly missingAdapters?: ReadonlySet; +}) { + const primary = makeFakeCodexAdapter(CLAUDE_AGENT_DRIVER); + const secondary = + options?.secondaryInstanceId === undefined + ? undefined + : makeFakeCodexAdapter(CLAUDE_AGENT_DRIVER); + const seenAuthorizationHeaders: Array = []; + const control: ClaudeStartControl = { + rejectNextStart: false, + stopThenFailProbe: false, + failHasSession: false, + hold: null, + gapHold: null, + extraStart: null, + }; + instrumentClaudeStart(primary, seenAuthorizationHeaders, control); + if (secondary) { + instrumentClaudeStart(secondary, seenAuthorizationHeaders, control); + } + + const httpServer = HttpServer.HttpServer.of({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), + serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"], + }); + const mcpRegistryLayer = McpSessionRegistry.layer.pipe( + Layer.provide(Layer.succeed(HttpServer.HttpServer, httpServer)), + Layer.provide( + Layer.succeed( + ServerEnvironment.ServerEnvironment, + ServerEnvironment.ServerEnvironment.of({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-mcp-retain")), + getDescriptor: Effect.die("unused"), + }), + ), + ), + Layer.provide(NodeServices.layer), + ); + const registry = + secondary && options?.secondaryInstanceId !== undefined + ? makeSharedClaudeRegistry( + [ + [claudeAgentInstanceId, primary.adapter], + [options.secondaryInstanceId, secondary.adapter], + ], + options.missingAdapters, + ) + : makeAdapterRegistryMock({ [CLAUDE_AGENT_DRIVER]: primary.adapter }); + const providerLayer = makeProviderServiceLive().pipe( + Layer.provide(NodeServices.layer), + Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)), + Layer.provide( + ProviderSessionDirectoryLive.pipe( + Layer.provide(ProviderSessionRuntime.layer.pipe(Layer.provide(SqlitePersistenceMemory))), + ), + ), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(serverConfigTestLayer), + Layer.provide(AnalyticsService.layerTest), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ); + + return { + primary, + secondary, + seenAuthorizationHeaders, + control, + layer: Layer.mergeAll(providerLayer, mcpRegistryLayer), + }; +} + +function mcpBearerToken(authorizationHeader: string | undefined): string { + return authorizationHeader?.replace(/^Bearer\s+/, "") ?? ""; +} + +function claudeStartInput( + threadId: ThreadId, + cwd: string, + runtimeMode: "full-access" | "approval-required" | "auto-accept-edits", + providerInstanceId: ProviderInstanceId = claudeAgentInstanceId, +) { + return { + provider: CLAUDE_AGENT_DRIVER, + providerInstanceId, + threadId, + cwd, + runtimeMode, + }; +} + +describe("MCP credential across session replacement", () => { + it.effect("keeps the live query credential when replacement is refused", () => { + const harness = makeClaudeMcpProviderHarness(); + const threadId = asThreadId("thread-mcp-refuse"); + const cwd = fixtureCwd("mcp-refuse"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + const liveHeader = harness.seenAuthorizationHeaders[0]; + const liveToken = mcpBearerToken(liveHeader); + assert.equal(liveToken.length > 20, true); + assert.equal((yield* registry.resolve(liveToken))?.threadId, threadId); + + harness.control.rejectNextStart = true; + const failure = yield* provider + .startSession(threadId, claudeStartInput(threadId, cwd, "approval-required")) + .pipe(Effect.flip); + const replacementToken = mcpBearerToken(harness.seenAuthorizationHeaders[1]); + + assert.equal(failure._tag, "ProviderAdapterRequestError"); + assert.notEqual(harness.seenAuthorizationHeaders[1], liveHeader); + assert.equal( + McpProviderSession.readMcpProviderSession(threadId)?.authorizationHeader, + liveHeader, + ); + assert.equal((yield* registry.resolve(liveToken))?.threadId, threadId); + assert.equal(yield* registry.resolve(replacementToken), undefined); + const sessions = yield* harness.primary.adapter.listSessions(); + assert.equal(sessions.length, 1); + assert.equal(sessions[0]?.runtimeMode, "full-access"); + }).pipe(Effect.provide(harness.layer)); + }); + + it.effect("rotates the MCP credential when replacement is accepted", () => { + const harness = makeClaudeMcpProviderHarness(); + const threadId = asThreadId("thread-mcp-replace"); + const cwd = fixtureCwd("mcp-replace"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + const liveHeader = harness.seenAuthorizationHeaders[0]; + const liveToken = mcpBearerToken(liveHeader); + + const replaced = yield* provider.startSession( + threadId, + claudeStartInput(threadId, cwd, "approval-required"), + ); + const nextHeader = harness.seenAuthorizationHeaders[1]; + const nextToken = mcpBearerToken(nextHeader); + + assert.equal(replaced.runtimeMode, "approval-required"); + assert.notEqual(nextHeader, liveHeader); + assert.equal( + McpProviderSession.readMcpProviderSession(threadId)?.authorizationHeader, + nextHeader, + ); + assert.equal(yield* registry.resolve(liveToken), undefined); + assert.equal((yield* registry.resolve(nextToken))?.threadId, threadId); + }).pipe(Effect.provide(harness.layer)); + }); + + it.effect("clears the MCP credential when the first start fails", () => { + const harness = makeClaudeMcpProviderHarness(); + const threadId = asThreadId("thread-mcp-initial-fail"); + const cwd = fixtureCwd("mcp-initial-fail"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + harness.control.rejectNextStart = true; + const failure = yield* provider + .startSession(threadId, claudeStartInput(threadId, cwd, "full-access")) + .pipe(Effect.flip); + const issuedToken = mcpBearerToken(harness.seenAuthorizationHeaders[0]); + + assert.equal(failure._tag, "ProviderAdapterRequestError"); + assert.equal(yield* registry.resolve(issuedToken), undefined); + assert.equal(McpProviderSession.readMcpProviderSession(threadId), undefined); + assert.equal(yield* harness.primary.adapter.hasSession(threadId), false); + }).pipe(Effect.provide(harness.layer)); + }); + + it.effect( + "revokes the previous credential when the session probe fails after the old session stopped", + () => { + const harness = makeClaudeMcpProviderHarness(); + const threadId = asThreadId("thread-mcp-probe-fail"); + const cwd = fixtureCwd("mcp-probe-fail"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + const liveToken = mcpBearerToken(harness.seenAuthorizationHeaders[0]); + + harness.control.stopThenFailProbe = true; + const failure = yield* provider + .startSession(threadId, claudeStartInput(threadId, cwd, "approval-required")) + .pipe(Effect.flip); + const replacementToken = mcpBearerToken(harness.seenAuthorizationHeaders[1]); + + assert.equal(failure._tag, "ProviderAdapterRequestError"); + assert.equal(yield* registry.resolve(liveToken), undefined); + assert.equal(yield* registry.resolve(replacementToken), undefined); + assert.equal(McpProviderSession.readMcpProviderSession(threadId), undefined); + const sessions = yield* harness.primary.adapter.listSessions(); + assert.equal(sessions.length, 0); + }).pipe(Effect.provide(harness.layer)); + }, + ); + + it.effect("revokes superseded credentials when replacement starts overlap", () => { + const harness = makeClaudeMcpProviderHarness(); + const threadId = asThreadId("thread-mcp-overlap"); + const cwd = fixtureCwd("mcp-overlap"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + const originalToken = mcpBearerToken(harness.seenAuthorizationHeaders[0]); + + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + harness.control.hold = { entered, release }; + const failureFiber = yield* provider + .startSession(threadId, claudeStartInput(threadId, cwd, "auto-accept-edits")) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(entered); + const replacedFiber = yield* provider + .startSession(threadId, claudeStartInput(threadId, cwd, "approval-required")) + .pipe(Effect.forkChild); + yield* Deferred.succeed(release, undefined); + const failure = yield* Fiber.join(failureFiber); + const replaced = yield* Fiber.join(replacedFiber); + const stalledToken = mcpBearerToken(harness.seenAuthorizationHeaders[1]); + const winnerToken = mcpBearerToken(harness.seenAuthorizationHeaders[2]); + + assert.equal(failure._tag, "ProviderAdapterRequestError"); + assert.equal(replaced.runtimeMode, "approval-required"); + assert.equal(yield* registry.resolve(originalToken), undefined); + assert.equal(yield* registry.resolve(stalledToken), undefined); + assert.equal((yield* registry.resolve(winnerToken))?.threadId, threadId); + assert.equal( + McpProviderSession.readMcpProviderSession(threadId)?.authorizationHeader, + harness.seenAuthorizationHeaders[2], + ); + }).pipe(Effect.provide(harness.layer)); + }); + + it.effect("revokes the thread credential when the previous provider adapter is gone", () => { + const secondaryInstanceId = ProviderInstanceId.make("claudeAgent-missing-adapter"); + const missingAdapters = new Set(); + const harness = makeClaudeMcpProviderHarness({ + secondaryInstanceId, + missingAdapters, + }); + const threadId = asThreadId("thread-mcp-missing-adapter"); + const cwd = fixtureCwd("mcp-missing-adapter"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + const liveToken = mcpBearerToken(harness.seenAuthorizationHeaders[0]); + assert.equal((yield* registry.resolve(liveToken))?.threadId, threadId); + + missingAdapters.add(claudeAgentInstanceId); + harness.control.rejectNextStart = true; + const failure = yield* provider + .startSession( + threadId, + claudeStartInput(threadId, cwd, "approval-required", secondaryInstanceId), + ) + .pipe(Effect.flip); + const replacementToken = mcpBearerToken(harness.seenAuthorizationHeaders[1]); + + assert.equal(failure._tag, "ProviderAdapterRequestError"); + assert.equal(yield* registry.resolve(liveToken), undefined); + assert.equal(yield* registry.resolve(replacementToken), undefined); + assert.equal(McpProviderSession.readMcpProviderSession(threadId), undefined); + }).pipe(Effect.provide(harness.layer)); + }); + + it.effect("keeps the previous session when a switch to another instance is rejected", () => { + const secondaryInstanceId = ProviderInstanceId.make("claudeAgent-alt"); + const harness = makeClaudeMcpProviderHarness({ secondaryInstanceId }); + const threadId = asThreadId("thread-mcp-switch"); + const cwd = fixtureCwd("mcp-switch"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const secondary = harness.secondary; + if (secondary === undefined) { + return yield* Effect.die("expected a secondary Claude adapter"); + } + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + const liveHeader = harness.seenAuthorizationHeaders[0]; + const liveToken = mcpBearerToken(liveHeader); + + harness.control.rejectNextStart = true; + const failure = yield* provider + .startSession( + threadId, + claudeStartInput(threadId, cwd, "approval-required", secondaryInstanceId), + ) + .pipe(Effect.flip); + const replacementToken = mcpBearerToken(harness.seenAuthorizationHeaders[1]); + const primarySessions = yield* harness.primary.adapter.listSessions(); + + assert.equal(failure._tag, "ProviderAdapterRequestError"); + assert.equal(harness.primary.stopSession.mock.calls.length, 0); + assert.equal(yield* secondary.adapter.hasSession(threadId), false); + assert.equal(primarySessions.length, 1); + assert.equal(primarySessions[0]?.runtimeMode, "full-access"); + assert.equal( + McpProviderSession.readMcpProviderSession(threadId)?.authorizationHeader, + liveHeader, + ); + assert.equal((yield* registry.resolve(liveToken))?.threadId, threadId); + assert.equal(yield* registry.resolve(replacementToken), undefined); + }).pipe(Effect.provide(harness.layer)); + }); + + it.effect( + "does not let session recovery revoke a credential an in-flight start already issued", + () => { + const harness = makeClaudeMcpProviderHarness(); + const threadId = asThreadId("thread-mcp-recovery-lock"); + const cwd = fixtureCwd("mcp-recovery-lock"); + return Effect.gen(function* () { + const provider = yield* ProviderService.ProviderService; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + yield* provider.startSession(threadId, claudeStartInput(threadId, cwd, "full-access")); + + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const extraStart = yield* Deferred.make(); + harness.control.extraStart = extraStart; + harness.control.gapHold = { entered, release }; + const replacementFiber = yield* provider + .startSession(threadId, claudeStartInput(threadId, cwd, "approval-required")) + .pipe(Effect.forkChild); + yield* Deferred.await(entered); + const inFlightToken = mcpBearerToken(harness.seenAuthorizationHeaders[1]); + assert.equal((yield* registry.resolve(inFlightToken))?.threadId, threadId); + + // The held start has deleted the adapter session, which is the gap + // `resolveRoutableSession({ allowRecovery: true })` would otherwise + // treat as a missing session and resume. + const turnFiber = yield* provider + .sendTurn({ + threadId, + input: "keep going", + attachments: [], + }) + .pipe(Effect.forkChild); + for (let step = 0; step < 8; step += 1) { + yield* Effect.yieldNow; + } + assert.equal(turnFiber.pollUnsafe(), undefined); + assert.equal(yield* Deferred.poll(extraStart), Option.none()); + assert.equal((yield* registry.resolve(inFlightToken))?.threadId, threadId); + assert.equal(harness.seenAuthorizationHeaders.length, 2); + + yield* Deferred.succeed(release, undefined); + const replaced = yield* Fiber.join(replacementFiber); + const turn = yield* Fiber.join(turnFiber); + + assert.equal(replaced.runtimeMode, "approval-required"); + assert.equal(turn.threadId, threadId); + assert.equal(yield* Deferred.poll(extraStart), Option.none()); + assert.equal(harness.seenAuthorizationHeaders.length, 2); + assert.equal((yield* registry.resolve(inFlightToken))?.threadId, threadId); + assert.equal( + McpProviderSession.readMcpProviderSession(threadId)?.authorizationHeader, + harness.seenAuthorizationHeaders[1], + ); + }).pipe(Effect.provide(harness.layer)); + }, + ); +}); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index cdac979c4dfd..9b03aefa86d7 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -51,7 +51,9 @@ import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as SchemaIssue from "effect/SchemaIssue"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; import { appendUserInputAttachmentPaths } from "../userInputAttachments.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; @@ -232,6 +234,14 @@ function compactAccessibilityForPrompt( /** How long a manual context compaction may run before ProviderService gives up on it. */ const COMPACTION_COMPLETION_TIMEOUT = "10 minutes"; +/** Result of asking an adapter whether it still owns a thread. */ +type McpSessionProbe = "active" | "inactive" | "unconfirmed"; + +interface McpSessionOwner { + readonly adapter: ProviderAdapterShape; + readonly located: boolean; +} + interface PendingCompaction { readonly completion: Deferred.Deferred; readonly native: boolean; @@ -503,6 +513,30 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); let turnAnalyticsRequestId = 0; const nowIso = Effect.map(DateTime.now, DateTime.formatIso); + const threadStartLocks = yield* SynchronizedRef.make(new Map()); + + // One permit per thread so a start and a recovery cannot mint or revoke + // MCP credentials for that thread at the same time. + const lockForThreadStart = (threadId: ThreadId) => + SynchronizedRef.modifyEffect(threadStartLocks, (current) => { + const existing = current.get(threadId); + if (existing) { + return Effect.succeed([existing, current] as const); + } + return Semaphore.make(1).pipe( + Effect.map((semaphore) => { + const next = new Map(current); + next.set(threadId, semaphore); + return [semaphore, next] as const; + }), + ); + }); + + const withThreadStartLock = ( + threadId: ThreadId, + effect: Effect.Effect, + ): Effect.Effect => + lockForThreadStart(threadId).pipe(Effect.flatMap((semaphore) => semaphore.withPermit(effect))); const finishTurnAnalytics = ( state: TurnAnalyticsState, @@ -940,10 +974,21 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( } satisfies Record; }); - const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) => + // `retainExisting` leaves the thread's current bearer registered so a + // rejected restart can keep the token the live query already baked in. + const prepareMcpSession = ( + threadId: ThreadId, + providerInstanceId: ProviderInstanceId, + options?: { readonly retainExisting?: boolean }, + ) => Effect.gen(function* () { const capabilities = yield* agentAccessCapabilities(threadId); - const credential = yield* issueMcpCredential({ threadId, providerInstanceId, capabilities }); + const credential = yield* issueMcpCredential({ + threadId, + providerInstanceId, + capabilities, + ...(options?.retainExisting === true ? { retainExisting: true } : {}), + }); if (credential) { const deviceEnvironment = capabilities.has("device") ? yield* agentDeviceEnvironment @@ -962,6 +1007,133 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( Effect.tap(() => Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId))), ); + // A failed probe is `unconfirmed`: callers must not treat it as proof the + // session is still running. + const probeMcpSession = ( + adapter: ProviderAdapterShape, + threadId: ThreadId, + ) => + adapter.hasSession(threadId).pipe( + Effect.map((active): McpSessionProbe => (active ? "active" : "inactive")), + Effect.orElseSucceed((): McpSessionProbe => "unconfirmed"), + ); + + // The previous provider instance owns the credential when it differs from + // the instance being started. `located` is false when that adapter is gone. + const resolveMcpSessionOwner = (input: { + readonly selected: ProviderAdapterShape; + readonly selectedInstanceId: ProviderInstanceId; + readonly previous: McpProviderSession.McpProviderSessionConfig | undefined; + }) => + Effect.gen(function* () { + const previousInstanceId = input.previous?.providerInstanceId; + if (previousInstanceId === undefined || previousInstanceId === input.selectedInstanceId) { + return { adapter: input.selected, located: true } satisfies McpSessionOwner; + } + const previousAdapter = yield* registry + .getByInstance(previousInstanceId) + .pipe( + Effect.catch(() => + Effect.succeed | undefined>(undefined), + ), + ); + if (previousAdapter === undefined) { + return { adapter: input.selected, located: false } satisfies McpSessionOwner; + } + return { adapter: previousAdapter, located: true } satisfies McpSessionOwner; + }); + + // Keep the existing bearer while a replacement is attempted only when a + // located owner still has the session, or the probe itself failed. + const mcpSessionShouldRetainCredential = (input: { + readonly owner: McpSessionOwner; + readonly previous: McpProviderSession.McpProviderSessionConfig | undefined; + readonly threadId: ThreadId; + }) => + Effect.gen(function* () { + if (!input.owner.located) { + return false; + } + const probe = yield* probeMcpSession(input.owner.adapter, input.threadId); + if (probe === "active") { + return true; + } + if (probe === "inactive") { + return false; + } + return input.previous !== undefined; + }); + + // Drop the credential a rejected start just issued and put the live query's + // config back when that token is still authorized. + const keepLiveMcpCredential = ( + threadId: ThreadId, + previous: McpProviderSession.McpProviderSessionConfig | undefined, + issuedProviderSessionId: string | undefined, + ) => + Effect.gen(function* () { + if (issuedProviderSessionId !== undefined) { + yield* McpSessionRegistry.revokeActiveMcpProviderSession(issuedProviderSessionId); + } + const previousStillAuthorized = + previous !== undefined && + (yield* McpSessionRegistry.hasActiveMcpProviderSession(previous.providerSessionId)); + yield* Effect.sync(() => { + if (previous !== undefined && previousStillAuthorized) { + McpProviderSession.setMcpProviderSession(previous); + return; + } + const current = McpProviderSession.readMcpProviderSession(threadId); + if (current === undefined || current.providerSessionId === issuedProviderSessionId) { + McpProviderSession.clearMcpProviderSession(threadId); + } + }); + }); + + // A rejection that leaves the session up restores its credential. A missing + // previous adapter, or a probe that cannot confirm the session, revokes the + // thread instead of restoring a stopped process's bearer. An accepted start + // revokes every credential except the one it just issued. + const settleMcpCredentialAfterStart = (input: { + readonly threadId: ThreadId; + readonly sessionAdapter: ProviderAdapterShape; + readonly ownerLocated: boolean; + readonly hadActiveSession: boolean; + readonly previous: McpProviderSession.McpProviderSessionConfig | undefined; + readonly issuedProviderSessionId: string | undefined; + readonly accepted: boolean; + }) => + Effect.gen(function* () { + if (input.accepted) { + if (input.issuedProviderSessionId !== undefined) { + yield* McpSessionRegistry.revokeActiveMcpThreadExcept( + input.threadId, + input.issuedProviderSessionId, + ); + } + return; + } + + if (!input.ownerLocated) { + yield* clearMcpSession(input.threadId); + return; + } + + const probe = input.hadActiveSession + ? yield* probeMcpSession(input.sessionAdapter, input.threadId) + : "inactive"; + if (probe === "active") { + yield* keepLiveMcpCredential(input.threadId, input.previous, input.issuedProviderSessionId); + return; + } + if (probe === "unconfirmed") { + yield* Effect.logWarning("provider.mcp.session-probe-failed", { + threadId: input.threadId, + }); + } + yield* clearMcpSession(input.threadId); + }); + const publishRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect => Effect.succeed(event).pipe( Effect.tap((canonicalEvent) => @@ -1235,71 +1407,78 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.instance_id": bindingInstanceId, "provider.thread_id": input.binding.threadId, }); - return yield* Effect.gen(function* () { - const adapter = yield* registry.getByInstance(bindingInstanceId); - const hasResumeCursor = - input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined; - const hasActiveSession = yield* adapter.hasSession(input.binding.threadId); - if (hasActiveSession) { - const activeSessions = yield* adapter.listSessions(); - const existing = activeSessions.find( - (session) => session.threadId === input.binding.threadId, - ); - if (existing) { - yield* upsertSessionBinding( - { ...existing, providerInstanceId: bindingInstanceId }, - input.binding.threadId, + // The hasSession check and any resumed start share startSession's lock. + // A replacement can drop the adapter context before the new process is + // registered; recovery must wait and adopt that start instead of minting + // a second bearer. + return yield* withThreadStartLock( + input.binding.threadId, + Effect.gen(function* () { + const adapter = yield* registry.getByInstance(bindingInstanceId); + const hasResumeCursor = + input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined; + const hasActiveSession = yield* adapter.hasSession(input.binding.threadId); + if (hasActiveSession) { + const activeSessions = yield* adapter.listSessions(); + const existing = activeSessions.find( + (session) => session.threadId === input.binding.threadId, ); - yield* analytics.record("provider.session.recovered", { - provider: existing.provider, - strategy: "adopt-existing", - hasResumeCursor: existing.resumeCursor !== undefined, - }); - return { adapter, session: existing } as const; + if (existing) { + yield* upsertSessionBinding( + { ...existing, providerInstanceId: bindingInstanceId }, + input.binding.threadId, + ); + yield* analytics.record("provider.session.recovered", { + provider: existing.provider, + strategy: "adopt-existing", + hasResumeCursor: existing.resumeCursor !== undefined, + }); + return { adapter, session: existing } as const; + } } - } - if (!hasResumeCursor) { - return yield* toValidationError( - input.operation, - `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`, - ); - } + if (!hasResumeCursor) { + return yield* toValidationError( + input.operation, + `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`, + ); + } - const persistedCwd = readPersistedCwd(input.binding.runtimePayload); - const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload); - - yield* prepareMcpSession(input.binding.threadId, bindingInstanceId); - const resumed = yield* adapter - .startSession({ - threadId: input.binding.threadId, - provider: input.binding.provider, - providerInstanceId: bindingInstanceId, - ...(persistedCwd ? { cwd: persistedCwd } : {}), - ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}), - ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}), - runtimeMode: input.binding.runtimeMode ?? "full-access", - }) - .pipe(Effect.onError(() => clearMcpSession(input.binding.threadId))); - if (resumed.provider !== adapter.provider) { - yield* clearMcpSession(input.binding.threadId); - return yield* toValidationError( - input.operation, - `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`, - ); - } + const persistedCwd = readPersistedCwd(input.binding.runtimePayload); + const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload); - yield* upsertSessionBinding( - { ...resumed, providerInstanceId: bindingInstanceId }, - input.binding.threadId, - ); - yield* analytics.record("provider.session.recovered", { - provider: resumed.provider, - strategy: "resume-thread", - hasResumeCursor: resumed.resumeCursor !== undefined, - }); - return { adapter, session: resumed } as const; - }).pipe( + yield* prepareMcpSession(input.binding.threadId, bindingInstanceId); + const resumed = yield* adapter + .startSession({ + threadId: input.binding.threadId, + provider: input.binding.provider, + providerInstanceId: bindingInstanceId, + ...(persistedCwd ? { cwd: persistedCwd } : {}), + ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}), + ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}), + runtimeMode: input.binding.runtimeMode ?? "full-access", + }) + .pipe(Effect.onError(() => clearMcpSession(input.binding.threadId))); + if (resumed.provider !== adapter.provider) { + yield* clearMcpSession(input.binding.threadId); + return yield* toValidationError( + input.operation, + `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`, + ); + } + + yield* upsertSessionBinding( + { ...resumed, providerInstanceId: bindingInstanceId }, + input.binding.threadId, + ); + yield* analytics.record("provider.session.recovered", { + provider: resumed.provider, + strategy: "resume-thread", + hasResumeCursor: resumed.resumeCursor !== undefined, + }); + return { adapter, session: resumed } as const; + }), + ).pipe( withMetrics({ counter: providerSessionsTotal, attributes: providerMetricAttributes(input.binding.provider, { @@ -1500,32 +1679,71 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( } const adapter = yield* registry.getByInstance(resolvedInstanceId); yield* clearTurnAnalyticsSession(resolvedInstanceId, threadId); - yield* prepareMcpSession(threadId, resolvedInstanceId); - const session = yield* adapter - .startSession({ - ...input, - providerInstanceId: resolvedInstanceId, - ...(effectiveCwd !== undefined ? { cwd: effectiveCwd } : {}), - ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}), - }) - .pipe(Effect.onError(() => clearMcpSession(threadId))); - - if (session.provider !== adapter.provider) { - yield* clearMcpSession(threadId); - return yield* toValidationError( - "ProviderService.startSession", - `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`, - ); - } - const sessionWithInstance = { - ...session, - providerInstanceId: resolvedInstanceId, - }; - - yield* stopStaleSessionsForThread({ + const sessionWithInstance = yield* withThreadStartLock( threadId, - currentInstanceId: resolvedInstanceId, - }); + Effect.gen(function* () { + const previousMcpSession = McpProviderSession.readMcpProviderSession(threadId); + const owner = yield* resolveMcpSessionOwner({ + selected: adapter, + selectedInstanceId: resolvedInstanceId, + previous: previousMcpSession, + }); + const hadActiveSession = yield* mcpSessionShouldRetainCredential({ + owner, + previous: previousMcpSession, + threadId, + }); + const issuedCredential = yield* prepareMcpSession(threadId, resolvedInstanceId, { + retainExisting: hadActiveSession, + }); + const issuedProviderSessionId = issuedCredential?.config.providerSessionId; + const settlement = { + threadId, + sessionAdapter: owner.adapter, + ownerLocated: owner.located, + hadActiveSession, + previous: previousMcpSession, + issuedProviderSessionId, + }; + const session = yield* adapter + .startSession({ + ...input, + providerInstanceId: resolvedInstanceId, + ...(effectiveCwd !== undefined ? { cwd: effectiveCwd } : {}), + ...(effectiveResumeCursor !== undefined + ? { resumeCursor: effectiveResumeCursor } + : {}), + }) + .pipe( + Effect.onError(() => + settleMcpCredentialAfterStart({ + ...settlement, + accepted: false, + }), + ), + ); + + if (session.provider !== adapter.provider) { + yield* clearMcpSession(threadId); + return yield* toValidationError( + "ProviderService.startSession", + `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`, + ); + } + yield* settleMcpCredentialAfterStart({ + ...settlement, + accepted: true, + }); + yield* stopStaleSessionsForThread({ + threadId, + currentInstanceId: resolvedInstanceId, + }); + return { + ...session, + providerInstanceId: resolvedInstanceId, + }; + }), + ); yield* upsertSessionBinding(sessionWithInstance, threadId, { modelSelection: input.modelSelection, }); diff --git a/apps/server/src/provider/model-manifest.json b/apps/server/src/provider/model-manifest.json index 32a103be9d16..69e51bcb019a 100644 --- a/apps/server/src/provider/model-manifest.json +++ b/apps/server/src/provider/model-manifest.json @@ -1,12 +1,58 @@ { "version": 1, - "updatedAt": "2026-09-22T21:40:00Z", + "updatedAt": "2026-09-24T00:11:29Z", "compatibility": [ + { + "driver": "codex", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=0.129.0", + "ranges": [ + { "range": ">=0.129.0", "status": "supported" }, + { "range": "<0.129.0", "status": "broken" } + ] + }, + { + "driver": "claudeAgent", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=2.1.280", + "ranges": [ + { "range": ">=2.1.280", "status": "supported" }, + { "range": ">=2.1.111 <2.1.280", "status": "graceful" }, + { "range": "<2.1.111", "status": "unsupported" } + ] + }, + { + "driver": "cursor", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=2026.05.09", + "ranges": [ + { "range": ">=2026.05.09", "status": "supported" }, + { "range": "<2026.05.09", "status": "unknown" } + ] + }, + { + "driver": "grok", + "t3CodeRange": ">=0.0.42", + "recommendedRange": ">=1.0.13", + "ranges": [ + { "range": ">=1.0.13", "status": "supported" }, + { "range": "<1.0.13", "status": "unknown" } + ] + }, { "driver": "opencode", "t3CodeRange": ">=0.0.42", "recommendedRange": ">=1.14.19", - "ranges": [{ "range": "<1.14.19", "status": "broken" }] + "ranges": [ + { "range": ">=1.14.19", "status": "supported" }, + { "range": "<1.14.19", "status": "broken" } + ] + }, + { + "driver": "antigravity", + "t3CodeRange": ">=0.0.42", + "recommendedRange": "=1.1.1", + "ranges": [{ "range": "=1.1.1", "status": "supported" }] } ], "currentModels": { diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index e6b4e1a608b9..3a43356d2caf 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -15,6 +15,7 @@ import { ProviderRegistry } from "./Services/ProviderRegistry.ts"; import { ProviderInstanceRegistry } from "./Services/ProviderInstanceRegistry.ts"; import type { ProviderInstance } from "./ProviderDriver.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "./providerMaintenance.ts"; +import { BUILT_IN_DRIVERS } from "./builtInDrivers.ts"; import * as Schema from "effect/Schema"; import { applyProviderCompatibility, @@ -51,6 +52,68 @@ const provider: ServerProvider = { }; describe("provider compatibility", () => { + it("bundles a compatibility policy for every built-in harness", () => { + for (const builtIn of BUILT_IN_DRIVERS) { + assert.isDefined( + resolveProviderCompatibility( + ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, + builtIn.driverKind, + null, + ), + `Missing bundled compatibility policy for ${builtIn.driverKind}`, + ); + } + }); + + it("compares Cursor build dates without treating semver prereleases as stable", () => { + const cursor = ProviderDriverKind.make("cursor"); + const cursorPolicy: ProviderCompatibilityPolicy = { + driver: cursor, + t3CodeRange: policy.t3CodeRange, + ranges: [ + { range: "<2026.05.09", status: "unsupported" }, + { range: ">=2026.05.09", status: "supported" }, + ], + }; + for (const [version, expected] of [ + ["2026.05.08-a1b2c3d", "unsupported"], + ["2026.05.09-a1b2c3d", "supported"], + ["2026.09.22-f2b0fcd", "supported"], + ["2026.05.09", "supported"], + ["2026.05.09-beta.1", "unknown"], + ] as const) { + assert.strictEqual( + resolveProviderCompatibility([cursorPolicy], cursor, version)?.status, + expected, + ); + } + assert.strictEqual( + resolveProviderCompatibility([policy], driver, "2.0.0-a1b2c3d")?.status, + "unknown", + ); + }); + + it("recognizes Antigravity semver release tags while keeping dated candidates unknown", () => { + const antigravity = ProviderDriverKind.make("antigravity"); + const taggedPolicy = { + ...policy, + driver: antigravity, + ranges: [{ range: "=2.0.0", status: "supported" as const }], + }; + for (const [version, expected] of [ + ["agy_acp_server_2.0.0", "supported"], + ["2.0.0", "supported"], + ["agy_acp_server_2.0.1", "unknown"], + ["agy_acp_server_2.0.0-beta.1", "unknown"], + ["agy_acp_server_20260818_01_RC01", "unknown"], + ] as const) { + assert.strictEqual( + resolveProviderCompatibility([taggedPolicy], antigravity, version)?.status, + expected, + ); + } + }); + it("classifies boundaries and treats unlisted versions and release tags as unknown", () => { for (const [version, expected] of [ ["0.9.9", "broken"], diff --git a/apps/server/src/provider/providerCompatibility.ts b/apps/server/src/provider/providerCompatibility.ts index 11aedf8dca0d..b4ce66f87603 100644 --- a/apps/server/src/provider/providerCompatibility.ts +++ b/apps/server/src/provider/providerCompatibility.ts @@ -10,7 +10,7 @@ import * as Schema from "effect/Schema"; import packageJson from "../../package.json" with { type: "json" }; // Deliberately uses the shared CLI gate syntax: comparator groups joined by ||. -// Release tags and prereleases remain unknown instead of matching stable ranges. +// Prereleases and unrecognized release tags remain unknown. const StableVersion = TrimmedNonEmptyString.pipe( Schema.check(Schema.makeFilter((value) => /^\d+\.\d+\.\d+$/.test(value))), ); @@ -66,7 +66,15 @@ export function resolveProviderCompatibility( (entry) => entry.driver === driver && satisfiesSemverRange(t3CodeVersion, entry.t3CodeRange), ); if (!policy) return undefined; - const stable = version?.replace(/^v/, ""); + const unprefixed = version?.replace(/^v/, ""); + // Cursor appends a build hash to its date; Google's ACP runtime uses a release prefix. + // Strip only these driver-specific forms, keeping semver prereleases unknown. + const stable = + driver === "cursor" + ? unprefixed?.replace(/^(\d{4}\.\d{2}\.\d{2})-[a-f0-9]+$/, "$1") + : driver === "antigravity" + ? unprefixed?.replace(/^agy_acp_server_(\d+\.\d+\.\d+)$/, "$1") + : unprefixed; const status = stable && /^\d+\.\d+\.\d+$/.test(stable) ? (policy.ranges.find((entry) => satisfiesSemverRange(stable, entry.range))?.status ?? diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 2da7c9183365..998c301cb1fb 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -211,7 +211,13 @@ function makeRegistry( const makeTestRunner = ( registry: ProviderRegistryShape, - manifest = ModelManifest.BUNDLED_MODEL_MANIFEST, + // Generic updater fixtures use synthetic versions. Keep their compatibility + // unknown so real harness minimums do not bypass the command under test. + manifest: ModelManifest.ModelManifestData = { + version: 1, + currentModels: {}, + compatibility: [{ driver: CODEX_DRIVER, t3CodeRange: ">=0.0.42", ranges: [] }], + }, ) => Effect.service(ProviderMaintenanceRunner.ProviderMaintenanceRunner).pipe( Effect.provide( diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 1e7cdc3f00e6..32cf68a61e18 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -20,6 +20,7 @@ import { CommandId, ProviderInstanceId } from "@t3tools/contracts"; import { RelayClientTracer } from "@t3tools/shared/relayTracing"; import { RELAY_ACTIVITY_PUBLISH_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt"; import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -325,6 +326,9 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { const projectId = "project-1" as ProjectId; const activeThreadId = "thread-active" as ThreadId; const idleThreadId = "thread-idle" as ThreadId; + const oldCompletedId = "thread-old-completed" as ThreadId; + const newCompletedId = "thread-new-completed" as ThreadId; + const freshMessageId = "thread-fresh-message" as ThreadId; const baseThread = { projectId, @@ -351,6 +355,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { expect( AgentAwarenessRelay.resolveAgentAwarenessRelayActiveThreadIds({ environmentId, + startedAt: Date.parse(now), projects: [ { id: projectId, @@ -374,6 +379,44 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { ...baseThread, id: idleThreadId, }, + { + ...baseThread, + id: oldCompletedId, + latestTurn: { + turnId: "turn-old" as TurnId, + state: "completed", + requestedAt: "2026-05-24T00:00:00.000Z", + startedAt: "2026-05-24T00:00:00.000Z", + completedAt: "2026-05-24T00:01:00.000Z", + assistantMessageId: null, + }, + }, + { + ...baseThread, + id: newCompletedId, + latestTurn: { + turnId: "turn-new" as TurnId, + state: "completed", + requestedAt: "2026-05-25T00:00:01.000Z", + startedAt: "2026-05-25T00:00:01.000Z", + completedAt: "2026-05-25T00:00:02.000Z", + assistantMessageId: null, + }, + }, + { + ...baseThread, + id: freshMessageId, + latestUserMessageAt: "2026-05-25T00:00:01.000Z", + session: { + threadId: freshMessageId, + status: "ready", + providerName: "Codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: "2026-05-25T00:00:02.000Z", + }, + }, { ...baseThread, id: "thread-missing-project" as ThreadId, @@ -389,7 +432,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { }, ], }), - ).toEqual([activeThreadId]); + ).toEqual([activeThreadId, newCompletedId]); }); it.effect("signs the activity publish JWT and rejects tampering", () => @@ -797,4 +840,138 @@ describe.sequential("signRelayAgentActivityPublishProof", () => { }), ), ); + + it.effect("does not alert for historical completions after startup", () => + Effect.scoped( + Effect.gen(function* () { + const secrets = makeMemorySecretStore(); + const now = yield* DateTime.now; + const old = DateTime.formatIso(DateTime.add(now, { days: -7 })); + const threadId = "thread-old" as ThreadId; + const projectId = "project-1" as ProjectId; + const environmentId = "env-1" as EnvironmentId; + const project = { + id: projectId, + title: "T3 Code", + workspaceRoot: "/workspace", + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + createdAt: old, + updatedAt: old, + } satisfies OrchestrationProjectShell; + const completedTurn = { + turnId: "turn-1" as TurnId, + state: "completed", + requestedAt: old, + startedAt: old, + completedAt: old, + assistantMessageId: null, + } as const; + const completedThread = { + id: threadId, + projectId, + title: "Old task", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + pullRequests: [], + latestTurn: completedTurn, + createdAt: old, + updatedAt: old, + archivedAt: null, + settledOverride: null, + settledAt: null, + session: null, + latestUserMessageAt: old, + hasPendingApprovals: false, + hasPendingUserInput: false, + hasActionableProposedPlan: false, + } satisfies OrchestrationThreadShell; + let currentThread: OrchestrationThreadShell | null = completedThread; + let publishes = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { + publishes += 1; + return Promise.resolve(Response.json({ ok: true, deliveries: [] })); + }) as unknown as typeof fetch; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + globalThis.fetch = originalFetch; + }), + ); + yield* secrets.setString(RELAY_URL_SECRET, "https://relay.example.test"); + yield* secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential"); + yield* secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true"); + + const layer = Layer.mergeAll( + Layer.succeed(ServerSecretStore.ServerSecretStore, secrets.store), + Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed(environmentId), + getDescriptor: Effect.die("unused descriptor"), + }), + Layer.succeed(OrchestrationEngineService, {} as OrchestrationEngineShape), + Layer.succeed(ProjectionSnapshotQuery, { + getThreadShellById: () => Effect.sync(() => Option.fromNullishOr(currentThread)), + getProjectShellById: () => Effect.succeed(Option.some(project)), + } as unknown as ProjectionSnapshotQueryShape), + ); + + yield* Effect.gen(function* () { + const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; + yield* relay.publishThread(threadId); + expect(publishes).toBe(0); + + currentThread = { + ...completedThread, + latestTurn: null, + latestUserMessageAt: DateTime.formatIso(DateTime.add(now, { seconds: 1 })), + session: { + threadId, + status: "ready", + providerName: "Codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: DateTime.formatIso(DateTime.add(now, { seconds: 2 })), + }, + }; + expect( + AgentAwarenessRelay.resolveAgentAwarenessRelayPublishSnapshot({ + environmentId, + threadId, + thread: Option.some(currentThread), + project: Option.some(project), + }).state?.phase, + ).toBe("completed"); + yield* relay.publishThread(threadId); + expect(publishes).toBe(0); + + currentThread = { + ...completedThread, + session: { + threadId, + status: "error", + providerName: "Codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: "old failure", + updatedAt: old, + }, + }; + yield* relay.publishThread(threadId); + expect(publishes).toBe(0); + }).pipe( + Effect.provide( + AgentAwarenessRelay.layer.pipe( + Layer.provide(layer), + Layer.provideMerge(NodeServices.layer), + ), + ), + ); + }), + ), + ); }); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 052a67959ad1..29d2abecebb7 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -267,8 +267,13 @@ export function resolveAgentAwarenessRelayPublishSnapshot(input: { }; } +function terminalWorkSinceStart(thread: OrchestrationThreadShell, startedAt: number): boolean { + return Date.parse(thread.latestTurn?.completedAt ?? "") > startedAt; +} + export function resolveAgentAwarenessRelayActiveThreadIds(input: { readonly environmentId: EnvironmentId; + readonly startedAt: number; readonly projects: ReadonlyArray>; readonly threads: ReadonlyArray; }): ReadonlyArray { @@ -279,12 +284,16 @@ export function resolveAgentAwarenessRelayActiveThreadIds(input: { if (!project) { return false; } + const state = projectThreadAwareness({ + environmentId: input.environmentId, + project, + thread, + }); return ( - projectThreadAwareness({ - environmentId: input.environmentId, - project, - thread, - }) !== null + state !== null && + (state.phase !== "completed" && state.phase !== "failed" + ? true + : terminalWorkSinceStart(thread, input.startedAt)) ); }) .map((thread) => thread.id); @@ -298,6 +307,7 @@ export const make = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService; const crypto = yield* Crypto.Crypto; const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets); + const startedAt = (yield* DateTime.now).epochMilliseconds; const activeSnapshotPublishedRef = yield* Ref.make(false); const publishedStateByThreadRef = yield* Ref.make(new Map()); @@ -417,6 +427,14 @@ export const make = Effect.gen(function* () { }); const publishIdentity = agentAwarenessPublishIdentity(snapshot.state); const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef); + if ( + (snapshot.state?.phase === "completed" || snapshot.state?.phase === "failed") && + !publishedStateByThread.has(threadId) + ) { + // Startup has no publish history. Only work from this server process may + // produce an initial terminal alert; historical threads remain quiet. + if (Option.isNone(thread) || !terminalWorkSinceStart(thread.value, startedAt)) return; + } if (publishedStateByThread.get(threadId) === publishIdentity) { // The projection is back at (or never left) the last published state, so // any pending deferred confirmation is moot. Leaving the deadline in @@ -495,7 +513,11 @@ export const make = Effect.gen(function* () { }); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => { const nextPublishedStates = new Map(publishedStates); - nextPublishedStates.set(threadId, publishIdentity); + if (snapshot.state === null) { + nextPublishedStates.delete(threadId); + } else { + nextPublishedStates.set(threadId, publishIdentity); + } return nextPublishedStates; }); }); @@ -529,6 +551,7 @@ export const make = Effect.gen(function* () { const snapshot = yield* snapshotQuery.getShellSnapshot(); const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({ environmentId, + startedAt, projects: snapshot.projects, threads: snapshot.threads, }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 362cc7e3128a..b42d001ca138 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -223,6 +223,7 @@ import { } from "../integration/TransferBudgetReport.integration.ts"; import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; import { DEFAULT_SIGNAL_EXPORT, otlpSerializationLayer } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; const defaultProjectId = ProjectId.make("project-default"); const defaultThreadId = ThreadId.make("thread-default"); @@ -587,6 +588,7 @@ const buildAppUnderTest = (options?: { otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, mode: "desktop", port: 0, host: "127.0.0.1", diff --git a/apps/server/src/serverLogger.test.ts b/apps/server/src/serverLogger.test.ts index cbb5056ed314..a5437582d2e3 100644 --- a/apps/server/src/serverLogger.test.ts +++ b/apps/server/src/serverLogger.test.ts @@ -9,6 +9,7 @@ import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability"; +import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import * as ServerConfig from "./config.ts"; import { ServerLoggerLive } from "./serverLogger.ts"; @@ -57,6 +58,7 @@ const configLayer = (overrides: Partial) = otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, otlpLogsExport: DEFAULT_SIGNAL_EXPORT, otlpServiceName: "t3-server", + otelEnvironment: OtelEnvironment.none, cwd: baseDir, baseDir, ...derivedPaths, diff --git a/apps/web/src/components/BranchToolbar.tsx b/apps/web/src/components/BranchToolbar.tsx index 8e84ec5d1ed9..912ee901eec8 100644 --- a/apps/web/src/components/BranchToolbar.tsx +++ b/apps/web/src/components/BranchToolbar.tsx @@ -5,7 +5,6 @@ import { FolderGit2Icon, FolderGitIcon, FolderIcon, - HistoryIcon, ScaleIcon, } from "lucide-react"; import { @@ -41,6 +40,7 @@ import { } from "./BranchToolbarBranchSelector"; import { BranchToolbarEnvironmentSelector } from "./BranchToolbarEnvironmentSelector"; import { BranchToolbarEnvModeSelector } from "./BranchToolbarEnvModeSelector"; +import { PreviousWorktreeItemContent } from "./PreviousWorktreeItemContent"; import { ComposerControl } from "./chat/ComposerControl"; import { Menu, @@ -105,6 +105,7 @@ interface MobileRunContextSelectorProps { activeWorktreePath: string | null; onEnvModeChange: (mode: EnvMode) => void; previousWorktreeLabel: string | null; + previousWorktreeBranch: string | null; onUsePreviousWorktree: () => void; } @@ -123,6 +124,7 @@ const MobileRunContextSelector = memo(function MobileRunContextSelector({ activeWorktreePath, onEnvModeChange, previousWorktreeLabel, + previousWorktreeBranch, onUsePreviousWorktree, }: MobileRunContextSelectorProps) { const composerFloatingLayerProps = useComposerMenuProps(); @@ -217,7 +219,12 @@ const MobileRunContextSelector = memo(function MobileRunContextSelector({ {triggerContent} - + {showEnvironmentPicker && availableEnvironments && onEnvironmentChange ? ( <> @@ -295,10 +302,7 @@ const MobileRunContextSelector = memo(function MobileRunContextSelector({ {previousWorktreeLabel ? ( - - - - + ) : null} @@ -630,6 +634,7 @@ export const BranchToolbar = memo(function BranchToolbar({ activeWorktreePath={activeWorktreePath} onEnvModeChange={onEnvModeChange} previousWorktreeLabel={previousWorktreeLabel} + previousWorktreeBranch={previousWorktreeSeed?.branch ?? null} onUsePreviousWorktree={onUsePreviousWorktree} /> @@ -669,6 +674,7 @@ export const BranchToolbar = memo(function BranchToolbar({ activeWorktreePath={activeWorktreePath} onEnvModeChange={onEnvModeChange} previousWorktreeLabel={previousWorktreeLabel} + previousWorktreeBranch={previousWorktreeSeed?.branch ?? null} onUsePreviousWorktree={onUsePreviousWorktree} /> ) : null} diff --git a/apps/web/src/components/BranchToolbarEnvModeSelector.tsx b/apps/web/src/components/BranchToolbarEnvModeSelector.tsx index 1b8b81cfcd74..cb1faef9a50d 100644 --- a/apps/web/src/components/BranchToolbarEnvModeSelector.tsx +++ b/apps/web/src/components/BranchToolbarEnvModeSelector.tsx @@ -1,4 +1,4 @@ -import { FolderGit2Icon, FolderGitIcon, FolderIcon, HistoryIcon } from "lucide-react"; +import { FolderGit2Icon, FolderGitIcon, FolderIcon } from "lucide-react"; import { memo, useMemo } from "react"; import { @@ -8,6 +8,7 @@ import { type EnvMode, } from "./BranchToolbar.logic"; import { useComposerMenuProps } from "./chat/composerEventScope"; +import { PreviousWorktreeItemContent } from "./PreviousWorktreeItemContent"; import { Select, SelectGroup, @@ -28,6 +29,7 @@ interface BranchToolbarEnvModeSelectorProps { activeWorktreePath: string | null; onEnvModeChange: (mode: EnvMode) => void; previousWorktreeLabel?: string | null; + previousWorktreeBranch?: string | null; onUsePreviousWorktree?: () => void; } @@ -38,6 +40,7 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe activeWorktreePath, onEnvModeChange, previousWorktreeLabel, + previousWorktreeBranch = null, onUsePreviousWorktree, }: BranchToolbarEnvModeSelectorProps) { const composerFloatingLayerProps = useComposerMenuProps(); @@ -142,7 +145,11 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe : resolveCurrentWorkspaceLabel(activeWorktreePath)} - + Workspace @@ -163,10 +170,7 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe {showPreviousWorktree && previousWorktreeLabel ? ( - - - {previousWorktreeLabel} - + ) : null} diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index 8af2d18b310e..50305ffdbef0 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -718,7 +718,7 @@ function ComposerPromptEditorTiptapInner(props: ComposerPromptEditorProps) { const editorAttributes = useMemo( () => ({ class: cn( - "composer-tiptap block max-h-50 min-h-17.5 w-full overflow-y-auto whitespace-pre-wrap wrap-break-word bg-transparent leading-relaxed text-foreground focus:outline-none", + "composer-tiptap -m-1 block max-h-52 min-h-19.5 overflow-y-auto p-1 whitespace-pre-wrap wrap-break-word bg-transparent leading-relaxed text-foreground focus:outline-none", className, ), "data-testid": "composer-editor", @@ -1242,7 +1242,7 @@ function ComposerPromptEditorTiptapInner(props: ComposerPromptEditorProps) {
diff --git a/apps/web/src/components/PreviousWorktreeItemContent.tsx b/apps/web/src/components/PreviousWorktreeItemContent.tsx new file mode 100644 index 000000000000..cd25c8c497e2 --- /dev/null +++ b/apps/web/src/components/PreviousWorktreeItemContent.tsx @@ -0,0 +1,19 @@ +import { HistoryIcon } from "lucide-react"; + +import { MiddleTruncate } from "./ui/middle-truncate"; + +export function PreviousWorktreeItemContent({ branch }: { branch: string | null }) { + return ( + + + + Previous worktree + {branch ? ( + + + + ) : null} + + + ); +} diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index d88b1309f0ad..33a1c9b05c3a 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -63,6 +63,7 @@ import { ScrollArea } from "~/components/ui/scroll-area"; import { PanelTabCloseButton } from "~/components/ui/panel-tab-close-button"; import { faviconUrlForOrigin } from "~/lib/favicon"; import { useTheme } from "~/hooks/useTheme"; +import { useDeviceState } from "~/state/device"; import { newestPullRequestSummary, pullRequestEnvironment, @@ -1244,7 +1245,17 @@ export function RightPanelTabs(props: RightPanelTabsProps) { } /> - {title} + + {surface.kind === "device" ? ( + + ) : ( + title + )} + )}
@@ -1421,3 +1432,27 @@ export function RightPanelTabs(props: RightPanelTabsProps) { ); } + +function DeviceTabTooltip(props: { + surface: Extract; + environmentId: EnvironmentId | null; + title: string; +}) { + const target = props.surface.target; + const { state } = useDeviceState(target ? props.environmentId : null); + const device = target + ? state.devices.find((entry) => entry.hostId === target.hostId && entry.id === target.deviceId) + : undefined; + const host = target ? state.hosts.find((entry) => entry.id === target.hostId) : undefined; + return ( +
+ {props.title} + {target ? ( + + {host?.label ?? "Device host"} ·{" "} + {device?.version ?? (target.platform === "ios" ? "iOS" : "Android")} + + ) : null} +
+ ); +} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index b377439882d5..15440c577ef3 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -6859,10 +6859,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) skills={selectedProviderSkills} containerClassName={cn(isComposerResting && "min-w-0 flex-1")} className={cn( - showMobilePendingAnswerActions && "max-sm:pb-11", + showMobilePendingAnswerActions && "max-sm:pb-12", isComposerResting && - "max-h-8 min-h-8 overflow-hidden whitespace-pre! leading-8", - isComposerApprovalState && "min-h-8", + "my-0 max-h-8 min-h-8 overflow-hidden py-0 whitespace-pre! leading-8", + isComposerApprovalState && "min-h-10", )} placeholderClassName={cn( isComposerResting && diff --git a/apps/web/src/components/chat/TraitsPicker.tsx b/apps/web/src/components/chat/TraitsPicker.tsx index f527a37366b7..78c70152710d 100644 --- a/apps/web/src/components/chat/TraitsPicker.tsx +++ b/apps/web/src/components/chat/TraitsPicker.tsx @@ -16,7 +16,7 @@ import { normalizeModelSlug, } from "@t3tools/shared/model"; import { memo, useCallback } from "react"; -import { GaugeIcon, ZapIcon } from "lucide-react"; +import { BrainIcon, ZapIcon } from "lucide-react"; import { Menu, MenuGroup, @@ -645,7 +645,7 @@ export const TraitsPicker = memo(function TraitsPicker({ data-composer-control-compact-icon className="pointer-events-none invisible absolute" > - + )} @@ -660,7 +660,7 @@ export const TraitsPicker = memo(function TraitsPicker({ data-composer-control-compact-icon className="pointer-events-none invisible absolute" > - + )} {triggerLabel} diff --git a/apps/web/src/components/device/DeviceControlsRail.tsx b/apps/web/src/components/device/DeviceControlsRail.tsx new file mode 100644 index 000000000000..443f111f01cc --- /dev/null +++ b/apps/web/src/components/device/DeviceControlsRail.tsx @@ -0,0 +1,308 @@ +import type { DevicePlatform } from "@t3tools/contracts"; +import { + Camera, + ChevronLeft, + Home, + Keyboard, + Box, + Maximize, + Moon, + MoreHorizontal, + PictureInPicture2, + Power, + RotateCcw, + SlidersHorizontal, + Smartphone, + Square, + Sun, + Type, + X, +} from "lucide-react"; +import type { ReactNode } from "react"; +import { Button } from "~/components/ui/button"; +import { + Menu, + MenuItem, + MenuPopup, + MenuSeparator, + MenuTrigger, + MenuRadioGroup, + MenuRadioItem, + MenuRadioItemIndicator, +} from "~/components/ui/menu"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; +import type { DeviceStreamHandle, DeviceViewControls } from "./DeviceStreamView"; +import type { DeviceControls } from "./useDeviceControls"; + +/** Stable floating controls for both presentations, regardless of panel width. */ +export function DeviceControlsRail(props: { + platform: DevicePlatform; + handle: DeviceStreamHandle | null; + view: DeviceViewControls; + controls: DeviceControls; + screenshotPending: boolean; + onScreenshot: () => void; + toolsOpen: boolean; + onTools: () => void; + onFloat: () => void; + onClose: () => void; + onPowerOff: () => void; +}) { + const { view, handle, controls } = props; + const popupSide = "left"; + const settings = controls.detail?.settings; + const inputDisabled = !handle?.inputConnected; + const nextAppearance = settings?.appearance === "dark" ? "light" : "dark"; + return ( + + ); +} + +function RailDivider() { + return
; +} + +function RailButton(props: { + tooltipSide: "left" | "bottom"; + label: string; + description?: string | undefined; + disabled?: boolean; + pressed?: boolean; + onClick: () => void; + children: ReactNode; +}) { + const button = ( + + ); + return ( + + {props.disabled && props.description ? ( + + } + > + {button} + + ) : ( + + )} + {props.description ?? props.label} + + ); +} diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index f46a9fbf8ed7..c6fdaad6f78d 100644 --- a/apps/web/src/components/device/DevicePanel.tsx +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -5,17 +5,7 @@ import type { DeviceSummary, ScopedThreadRef, } from "@t3tools/contracts"; -import { - ChevronLeft, - Home, - PictureInPicture2, - Power, - RotateCcw, - SlidersHorizontal, - Smartphone, - Square, - X, -} from "lucide-react"; +import { Smartphone, X } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; @@ -25,16 +15,13 @@ import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list" import { Dialog } from "~/components/ui/dialog"; import { WizardPopup } from "~/components/ui/wizard"; import { Spinner } from "~/components/ui/spinner"; -import { Toggle } from "~/components/ui/toggle"; -import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; import { cn } from "~/lib/utils"; -import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/device"; +import { deviceEnvironment, useDeviceState } from "~/state/device"; import { formatEnvironmentQueryError } from "~/state/query"; import { useAtomCommand } from "~/state/use-atom-command"; -import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView"; import { DeviceLoadingView } from "./DeviceLoadingView"; import { DeviceSetup } from "./DeviceSetup"; -import { DeviceToolsPanel } from "./DeviceToolsPanel"; +import { DeviceWorkspace } from "./DeviceWorkspace"; import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell"; const platformLabel = (platform: DevicePlatform) => @@ -59,10 +46,6 @@ export function DevicePanel(props: { const [operationError, setOperationError] = useState(null); const [pendingDevice, setPendingDevice] = useState(null); const pendingDeviceKey = pendingDevice ? deviceKey(pendingDevice) : null; - const [handle, setHandle] = useState(null); - const [toolsOpen, setToolsOpen] = useState(false); - const [axOverlay, setAxOverlay] = useState(false); - const access = useDeviceHubAccess(environmentId); const hostDisabled = state.hostStatus === "disabled"; @@ -184,69 +167,7 @@ export function DevicePanel(props: { return ( -
- - {props.surface.target - ? `${state.hosts.find((host) => host.id === props.surface.target?.hostId)?.label ?? "Device host"} · ${activeDevice?.version ?? props.surface.target.platform}` - : (pendingDevice?.name ?? "Choose a device")} - - {activeDevice ? ( - <> - handle?.pressButton("home")} - disabled={!handle?.inputConnected} - > - - - {activeDevice.platform === "android" ? ( - <> - handle?.pressButton("back")} - disabled={!handle?.inputConnected} - > - - - handle?.pressButton("recents")} - disabled={!handle?.inputConnected} - > - - - - ) : ( - handle?.rotate()} - disabled={!handle?.inputConnected} - > - - - )} - setToolsOpen(Boolean(pressed))} - > - - - - - - closeActive(true)}> - - - closeActive(false)}> - - - - ) : null} -
- {hostReady && state.hostStatusDetail ? ( + {hostReady && !activeDevice && state.hostStatusDetail ? (
{activeDevice && activeSession ? ( - <> -
- host.id === activeDevice.hostId)?.label ?? "Device host"} · ${activeDevice.version}`} - deviceId={activeDevice.id} - hostId={activeDevice.hostId} - visible={props.visible} - axOverlay={axOverlay} - onHandle={setHandle} - /> -
- {toolsOpen ? ( - setToolsOpen(false)} - className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[560px]:static @[560px]:w-72 @[560px]:shrink-0 @[560px]:shadow-none" - /> - ) : null} - + host.id === activeDevice.hostId)?.label ?? "Device host" + } + hostDiagnostics={state.hostStatusDetail} + visible={props.visible} + onFloat={floatActive} + onClose={() => closeActive(false)} + onPowerOff={() => closeActive(true)} + /> ) : pendingDevice || hostBusy || !loaded ? ( void; - readonly disabled?: boolean; - readonly children: React.ReactNode; -}) { - return ( - - - } - > - {props.children} - - {props.label} - - ); -} - function groupDevices(state: DeviceServiceState) { const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = []; for (const platform of ["ios", "android"] as const) { diff --git a/apps/web/src/components/device/DevicePhoneViewport.tsx b/apps/web/src/components/device/DevicePhoneViewport.tsx new file mode 100644 index 000000000000..66398c02b52b --- /dev/null +++ b/apps/web/src/components/device/DevicePhoneViewport.tsx @@ -0,0 +1,151 @@ +import { useEffect, useRef, type RefObject } from "react"; +import type { PhoneViewer } from "@t3tools/client-runtime/device/phone-viewer"; +import type { + DeviceAccessorySource, + DeviceModelSource, +} from "@t3tools/client-runtime/device/model"; +import type { DeviceShapeProfile } from "@t3tools/client-runtime/device/shape-profile"; +import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction"; +import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream"; +import { bindPhoneTrackpad } from "./phoneTrackpad"; + +const loadPhoneViewer = () => import("@t3tools/client-runtime/device/phone-viewer"); + +/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */ +export function DevicePhoneViewport(props: { + readonly profile: DeviceShapeProfile; + readonly model: DeviceModelSource | null; + readonly accessory: DeviceAccessorySource | null; + readonly source: RefObject; + readonly onFrameListener: (listener: (() => void) | null) => void; + readonly onResetReady: (reset: (() => void) | null) => void; + readonly onInputCancel: (cancel: (() => void) | null) => void; + readonly client: RefObject; + readonly screen: DeviceScreenSize | null; + readonly onUnavailable: () => void; +}) { + const hostRef = useRef(null); + const canvasRef = useRef(null); + const viewerRef = useRef(null); + const interactionRef = useRef | null>(null); + const screenRef = useRef(props.screen); + const profileRef = useRef(props.profile); + const modelRef = useRef(props.model); + const accessoryRef = useRef(props.accessory); + const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props; + + useEffect(() => { + screenRef.current = props.screen; + profileRef.current = props.profile; + modelRef.current = props.model; + accessoryRef.current = props.accessory; + interactionRef.current?.end(); + viewerRef.current?.setModel(props.model); + viewerRef.current?.setAccessory(props.accessory); + viewerRef.current?.setScreen(props.screen, props.profile); + }, [props.screen, props.profile, props.model, props.accessory]); + + useEffect(() => { + const host = hostRef.current; + const canvas = canvasRef.current; + const decoded = source.current; + if (!host || !canvas || !decoded) return; + let disposed = false; + let trackpad: ReturnType | null = null; + let stopTrackpadEnd: (() => void) | undefined; + const resize = () => { + const { width, height } = host.getBoundingClientRect(); + viewerRef.current?.resize(width, height, window.devicePixelRatio); + }; + const observer = new ResizeObserver(resize); + observer.observe(host); + const blur = () => { + interactionRef.current?.end(); + trackpad?.cancel(); + }; + onInputCancel(blur); + window.addEventListener("blur", blur); + void loadPhoneViewer() + .then(({ createPhoneViewer }) => { + if (disposed) return; + const viewer = createPhoneViewer({ + canvas, + source: decoded, + onUnavailable, + onModelError: (cause) => console.warn("Device 3D asset could not load", cause), + profile: profileRef.current, + model: modelRef.current, + accessory: accessoryRef.current, + }); + viewerRef.current = viewer; + onResetReady(viewer.resetPose); + viewer.setScreen(screenRef.current); + onFrameListener(viewer.frameUpdated); + interactionRef.current = createPhoneInteraction({ + screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured), + touch: (phase, point) => client.current?.sendTouch(phase, point.x, point.y), + orbit: viewer.orbit, + zoomBy: () => {}, + onInteractionActive: viewer.setInteractionActive, + }); + trackpad = bindPhoneTrackpad(canvas, interactionRef.current); + stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit()); + resize(); + viewer.frameUpdated(); + }) + .catch(() => { + if (!disposed) onUnavailable(); + }); + return () => { + disposed = true; + stopTrackpadEnd?.(); + trackpad?.dispose(); + interactionRef.current?.end(); + interactionRef.current = null; + onInputCancel(null); + onResetReady(null); + onFrameListener(null); + observer.disconnect(); + window.removeEventListener("blur", blur); + viewerRef.current?.dispose(); + viewerRef.current = null; + }; + }, [onInputCancel, onResetReady, client, onFrameListener, onUnavailable, source]); + + const point = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + return { + x: (event.clientX - rect.left) / rect.width, + y: (event.clientY - rect.top) / rect.height, + }; + }; + return ( +
+
+
+ { + if (event.button !== 0) return; + if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return; + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus(); + }} + onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))} + onPointerUp={(event) => { + interactionRef.current?.move(event.pointerId, point(event)); + interactionRef.current?.end(event.pointerId); + }} + onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)} + onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)} + /> +
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 4d4744a6268f..3e2cd54e3c26 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -1,9 +1,14 @@ import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { cn } from "~/lib/utils"; import { Button } from "~/components/ui/button"; import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device"; +import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame"; +import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile"; +import { deviceKeyboard, deviceModel } from "./deviceModels"; +import { fitDeviceFrame } from "./deviceFrameLayout"; +import { DevicePhoneViewport } from "./DevicePhoneViewport"; import { DeviceLoadingView } from "./DeviceLoadingView"; import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi"; import { @@ -15,6 +20,17 @@ import { } from "@t3tools/client-runtime/device/stream"; const AX_POLL_INTERVAL_MS = 2_000; +const CONTROLS_RAIL_WIDTH = 56; + +export interface DeviceViewControls { + readonly phone: boolean; + readonly streaming: boolean; + readonly phoneUnavailableReason: string | null; + readonly showPhone: () => void; + readonly showFlat: () => void; + readonly resetView: () => void; + readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null; +} export interface DeviceStreamHandle { readonly pressButton: (button: DeviceHardwareButton) => void; @@ -37,11 +53,31 @@ export function DeviceStreamView(props: { readonly deviceDescription?: string; readonly visible: boolean; readonly hostId: string; + /** The full panel opts into the phone spike; compact viewers retain their flat presentation. */ + readonly allowPhoneView?: boolean; + readonly renderControls?: (view: DeviceViewControls) => ReactNode; /** Draw accessibility element frames over the screen. */ readonly axOverlay?: boolean; readonly onHandle?: (handle: DeviceStreamHandle | null) => void; readonly onScreen?: (screen: DeviceScreenSize | null) => void; }) { + const [presentation, setPresentation] = useState<"phone" | "flat">("phone"); + const [keyboardAttached, setKeyboardAttached] = useState(false); + const [phoneUnavailable, setPhoneUnavailable] = useState(false); + const onPhoneUnavailable = useCallback(() => setPhoneUnavailable(true), []); + const cancelPhoneInputRef = useRef<(() => void) | null>(null); + const cancelPhoneInput = useCallback(() => cancelPhoneInputRef.current?.(), []); + const resetViewRef = useRef<(() => void) | null>(null); + const onResetReady = useCallback((reset: (() => void) | null) => { + resetViewRef.current = reset; + }, []); + const frameListenerRef = useRef<(() => void) | null>(null); + const onFrameListener = useCallback((listener: (() => void) | null) => { + frameListenerRef.current = listener; + }, []); + const onInputCancel = useCallback((cancel: (() => void) | null) => { + cancelPhoneInputRef.current = cancel; + }, []); const access = useDeviceHubAccess(props.environmentId, props.hostId); const canvasRef = useRef(null); const clientRef = useRef(null); @@ -67,7 +103,7 @@ export function DeviceStreamView(props: { } const client = createDeviceStreamClient( { platform: props.platform, deviceId: props.deviceId, access }, - canvas, + createCanvasFrameSink(canvas, () => frameListenerRef.current?.()), { onStatus: (next, nextDetail) => { setStatus(next); @@ -101,6 +137,7 @@ export function DeviceStreamView(props: { client.start(); onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false }); return () => { + cancelPhoneInput(); client.stop(); clientRef.current = null; onHandle?.(null); @@ -109,6 +146,7 @@ export function DeviceStreamView(props: { }; }, [ access, + cancelPhoneInput, onHandle, onScreen, props.deviceId, @@ -131,6 +169,16 @@ export function DeviceStreamView(props: { return w / h; }, [props.platform, screen]); + const showPhone = + props.allowPhoneView && + status === "streaming" && + props.visible && + presentation === "phone" && + !phoneUnavailable && + !mjpegUrl && + !props.axOverlay; + const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0; + // The frame is the largest box at `aspect` that fits the container, so a // narrow panel shows a shorter phone rather than a squeezed one. CSS // `aspect-ratio` alone cannot do this: with the height pinned to 100% the @@ -154,12 +202,8 @@ export function DeviceStreamView(props: { return () => observer.disconnect(); }, []); const frame = useMemo(() => { - if (host.width === 0 || host.height === 0) return { width: 0, height: 0 }; - const byHeight = { width: host.height * aspect, height: host.height }; - return byHeight.width <= host.width - ? byHeight - : { width: host.width, height: host.width / aspect }; - }, [aspect, host]); + return fitDeviceFrame(aspect, host.width, host.height, controlsInset); + }, [aspect, controlsInset, host]); // serve-sim streams the raw framebuffer; rotate the display for a device // that reports landscape while its frames stay portrait. @@ -231,123 +275,230 @@ export function DeviceStreamView(props: { return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; }; + const phoneUnavailableReason = phoneUnavailable + ? "3D is unavailable on this browser" + : mjpegUrl + ? "3D requires the H.264 stream" + : props.axOverlay + ? "Turn off accessibility frames to use 3D" + : null; + + const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? ""); + const resetView = useCallback(() => { + const orientation = keyboardAttached ? "landscape_right" : "portrait"; + if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation); + resetViewRef.current?.(); + }, [keyboardAttached, screen?.orientation]); + const profile = resolveDeviceShape({ + platform: props.platform, + name: props.deviceName ?? "", + portraitAspect: Math.min(aspect, 1 / aspect), + }); + return (
{ - if (event.target !== event.currentTarget) return; - if (event.metaKey && !["r", "R"].includes(event.key)) return; - event.preventDefault(); - clientRef.current?.sendKey(event.nativeEvent, "down"); - }} - onKeyUp={(event) => { - if (event.target !== event.currentTarget) return; - clientRef.current?.sendKey(event.nativeEvent, "up"); - }} + className={cn( + "relative flex size-full min-h-0 min-w-0", + props.allowPhoneView ? "bg-background" : "bg-black/90", + )} > + {props.renderControls ? ( + { + cancelPhoneInput(); + if (!keyboardAttached && screen?.orientation !== "landscape_right") + clientRef.current?.setOrientation("landscape_right"); + setKeyboardAttached(!keyboardAttached); + }, + } + : null, + showPhone: () => setPresentation("phone"), + showFlat: () => setPresentation("flat"), + }} + onResetView={resetView} + /> + ) : null}
{ - event.currentTarget.setPointerCapture(event.pointerId); - (event.currentTarget.parentElement as HTMLElement | null)?.focus(); - pointerActive.current = true; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("begin", x, y); + ref={hostRef} + className={cn( + "relative flex min-h-0 min-w-0 flex-1 items-center justify-center overflow-hidden outline-none", + controlsInset && "pr-14", + )} + tabIndex={0} + role="application" + aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`} + onKeyDown={(event) => { + if (event.target !== event.currentTarget) return; + if (event.metaKey && !["r", "R"].includes(event.key)) return; + event.preventDefault(); + clientRef.current?.sendKey(event.nativeEvent, "down"); }} - onPointerMove={(event) => { - if (!pointerActive.current) return; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("move", x, y); - }} - onPointerUp={(event) => { - if (!pointerActive.current) return; - pointerActive.current = false; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("end", x, y); - }} - onPointerCancel={(event) => { - if (!pointerActive.current) return; - pointerActive.current = false; - const { x, y } = normalizedPoint(event); - clientRef.current?.sendTouch("end", x, y); + onKeyUp={(event) => { + if (event.target !== event.currentTarget) return; + clientRef.current?.sendKey(event.nativeEvent, "up"); }} > - - {props.visible && access && mjpegUrl ? ( - { + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.parentElement as HTMLElement | null)?.focus(); + pointerActive.current = true; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("begin", x, y); + }} + onPointerMove={(event) => { + if (!pointerActive.current) return; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("move", x, y); + }} + onPointerUp={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + onPointerCancel={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + > + + {props.visible && access && mjpegUrl ? ( + + ) : null} + {axElements.length > 0 ? ( +
+ {axElements.map((element) => ( +
+ {element.label ? ( + + {element.label} + + ) : null} +
+ ))} +
+ ) : null} +
+ {showPhone ? ( + + ) : null} + {props.allowPhoneView && !props.renderControls && status === "streaming" ? ( +
+ + +
) : null} - {axElements.length > 0 ? ( -
- {axElements.map((element) => ( -
- {element.label ? ( - - {element.label} - - ) : null} -
- ))} + {status === "streaming" && !inputState.connected ? ( +
+ + Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting… + +
+ ) : null} + {status !== "streaming" ? ( +
+ + {status === "error" ? ( + + ) : null} +
) : null}
- {status === "streaming" && !inputState.connected ? ( -
- - Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting… - -
- ) : null} - {status !== "streaming" ? ( -
- - {status === "error" ? ( - - ) : null} - -
- ) : null}
); } + +function DeviceControlsSlot(props: { + renderControls: (view: DeviceViewControls) => ReactNode; + view: Omit; + onResetView: () => void; +}) { + return props.renderControls({ ...props.view, resetView: props.onResetView }); +} diff --git a/apps/web/src/components/device/DeviceToolsPanel.tsx b/apps/web/src/components/device/DeviceToolsPanel.tsx index 39b8123cd03b..3d7bd2ae7f2c 100644 --- a/apps/web/src/components/device/DeviceToolsPanel.tsx +++ b/apps/web/src/components/device/DeviceToolsPanel.tsx @@ -1,14 +1,7 @@ import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; -import type { - DeviceActionInput, - DeviceDetail, - DevicePermission, - DeviceSummary, - DeviceTextSize, - EnvironmentId, -} from "@t3tools/contracts"; +import type { DevicePermission, DeviceSummary, DeviceTextSize } from "@t3tools/contracts"; import { ChevronDown, X } from "lucide-react"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useEffect, useState } from "react"; import { Button } from "~/components/ui/button"; import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible"; @@ -24,21 +17,8 @@ import { Spinner } from "~/components/ui/spinner"; import { Switch } from "~/components/ui/switch"; import { Toggle, ToggleGroup } from "~/components/ui/toggle-group"; import { cn } from "~/lib/utils"; -import { deviceEnvironment } from "~/state/device"; -import { formatEnvironmentQueryError } from "~/state/query"; -import { useAtomCommand } from "~/state/use-atom-command"; -import { - type DeviceEventLogEntry, - type DeviceForegroundInfo, - subscribeDeviceEventLog, - subscribeDeviceForeground, -} from "./deviceHubApi"; - -type ActionBody = DeviceActionInput extends infer A - ? A extends { readonly type: string } - ? Omit - : never - : never; +import type { DeviceControls } from "./useDeviceControls"; +import { type DeviceEventLogEntry, subscribeDeviceEventLog } from "./deviceHubApi"; const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [ { value: "small", label: "Small" }, @@ -102,7 +82,8 @@ const LOCATION_PRESETS = [ * local state so the controls never show a value the device did not confirm. */ export function DeviceToolsPanel(props: { - readonly environmentId: EnvironmentId; + readonly controls: DeviceControls; + readonly hostDiagnostics: string | undefined; readonly device: DeviceSummary; readonly access: DeviceHubAccess | null; readonly axOverlay: boolean; @@ -110,62 +91,10 @@ export function DeviceToolsPanel(props: { readonly onClose: () => void; readonly className?: string; }) { - const { environmentId, device } = props; - const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); - const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); - const [detail, setDetail] = useState(null); - const [pending, setPending] = useState(false); - const [error, setError] = useState(null); - const [foreground, setForeground] = useState(undefined); - const isIos = device.platform === "ios"; - - const target = useMemo( - () => ({ hostId: device.hostId, deviceId: device.id }), - [device.hostId, device.id], - ); - - // The panel is keyed by device, so a mount is always a fresh device. - useEffect(() => { - let cancelled = false; - void readDetail({ environmentId, input: target }).then((result) => { - if (cancelled) return; - if (result._tag === "Success") setDetail(result.value); - else setError(formatEnvironmentQueryError(result.cause)); - }); - return () => { - cancelled = true; - }; - }, [environmentId, readDetail, target]); - - useEffect(() => { - if (!props.access) return; - return subscribeDeviceForeground( - { access: props.access, platform: device.platform, deviceId: device.id }, - setForeground, - ); - }, [device.id, device.platform, props.access]); - - const act = useCallback( - async (body: ActionBody) => { - setPending(true); - setError(null); - try { - const result = await runAction({ - environmentId, - input: { ...target, ...body } as DeviceActionInput, - }); - if (result._tag === "Success") setDetail(result.value); - else setError(formatEnvironmentQueryError(result.cause)); - } finally { - setPending(false); - } - }, - [environmentId, runAction, target], - ); - + const { device, controls } = props; + const { detail, pending, error, foregroundApp, disabled, act } = controls; const settings = detail?.settings; - const foregroundApp = foreground === undefined ? (detail?.foregroundApp ?? null) : foreground; - const disabled = pending || detail === null; + const isIos = device.platform === "ios"; return (
) : null} + {props.hostDiagnostics ? ( +
+

+ {props.hostDiagnostics} +

+
+ ) : null} +
{foregroundApp?.id ?? "—"} @@ -398,9 +335,7 @@ export function DeviceToolsPanel(props: { function Section(props: { readonly title: string; readonly children: React.ReactNode }) { return (
-

- {props.title} -

+

{props.title}

{props.children}
); @@ -698,7 +633,7 @@ function EventLogSection(props: { return ( - + Event log void; + onClose: () => void; + onPowerOff: () => void; +}) { + const [handle, setHandle] = useState(null); + const [toolsOpen, setToolsOpen] = useState(false); + const [axOverlay, setAxOverlay] = useState(false); + const [screenshotPending, setScreenshotPending] = useState(false); + const [screenshotError, setScreenshotError] = useState(null); + const captureRef = useRef(null); + const downloadRef = useRef(null); + const access = useDeviceHubAccess(props.environmentId, props.device.hostId); + const controls = useDeviceControls({ ...props, access }); + useEffect(() => { + if (!access || !props.visible) captureRef.current?.abort(); + return () => { + captureRef.current?.abort(); + if (downloadRef.current) URL.revokeObjectURL(downloadRef.current); + downloadRef.current = null; + }; + }, [access, props.visible]); + const saveScreenshot = () => { + if (!access || captureRef.current || !props.visible) return; + const controller = new AbortController(); + captureRef.current = controller; + setScreenshotPending(true); + setScreenshotError(null); + return captureDeviceScreenshot( + { access, platform: props.device.platform, deviceId: props.device.id }, + controller.signal, + ) + .then((image) => { + if (controller.signal.aborted) return; + if (downloadRef.current) URL.revokeObjectURL(downloadRef.current); + const url = URL.createObjectURL(image); + downloadRef.current = url; + const link = document.createElement("a"); + link.href = url; + link.download = `${props.device.name.replace(/[^a-z0-9-]/gi, "-")}-${Date.now()}.png`; + document.body.append(link); + link.click(); + link.remove(); + }) + .catch((cause: unknown) => { + if (!controller.signal.aborted) { + if (cause instanceof DeviceScreenshotError && cause.status === 401) + refreshDeviceHubAccess(props.environmentId); + setScreenshotError( + cause instanceof Error ? cause.message : "Screenshot capture failed. Try again.", + ); + } + }) + .finally(() => { + if (captureRef.current === controller) { + captureRef.current = null; + setScreenshotPending(false); + } + }); + }; + return ( + <> +
+ {screenshotError || (controls.error && !toolsOpen) ? ( +

+ {screenshotError ?? controls.error} +

+ ) : null} +
+ ( + void saveScreenshot()} + toolsOpen={toolsOpen} + onTools={() => setToolsOpen(!toolsOpen)} + onFloat={props.onFloat} + onClose={props.onClose} + onPowerOff={props.onPowerOff} + /> + )} + /> +
+
+ {toolsOpen ? ( + setToolsOpen(false)} + className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[700px]:static @[700px]:w-72 @[700px]:shrink-0 @[700px]:shadow-none" + /> + ) : null} + + ); +} diff --git a/apps/web/src/components/device/deviceFrameLayout.test.ts b/apps/web/src/components/device/deviceFrameLayout.test.ts new file mode 100644 index 000000000000..c36c8141f89e --- /dev/null +++ b/apps/web/src/components/device/deviceFrameLayout.test.ts @@ -0,0 +1,9 @@ +import { expect, it } from "vite-plus/test"; +import { fitDeviceFrame } from "./deviceFrameLayout"; + +it("fits a flat device beside the fixed controls rail", () => { + const frame = fitDeviceFrame(9 / 19.5, 480, 1_000, 56); + expect(frame.width + 56).toBeLessThanOrEqual(480); + expect(frame.height).toBeLessThanOrEqual(1_000); + expect(frame.width / frame.height).toBeCloseTo(9 / 19.5); +}); diff --git a/apps/web/src/components/device/deviceFrameLayout.ts b/apps/web/src/components/device/deviceFrameLayout.ts new file mode 100644 index 000000000000..1842c3b2784e --- /dev/null +++ b/apps/web/src/components/device/deviceFrameLayout.ts @@ -0,0 +1,8 @@ +export function fitDeviceFrame(aspect: number, width: number, height: number, rightInset = 0) { + const availableWidth = Math.max(0, width - rightInset); + if (availableWidth === 0 || height === 0) return { width: 0, height: 0 }; + const byHeight = { width: height * aspect, height }; + return byHeight.width <= availableWidth + ? byHeight + : { width: availableWidth, height: availableWidth / aspect }; +} diff --git a/apps/web/src/components/device/deviceModels.ts b/apps/web/src/components/device/deviceModels.ts new file mode 100644 index 000000000000..682e0e6de4d5 --- /dev/null +++ b/apps/web/src/components/device/deviceModels.ts @@ -0,0 +1,35 @@ +import { + resolveDeviceModelId, + type DeviceAccessorySource, + type DeviceModelSource, +} from "@t3tools/client-runtime/device/model"; +import type { DevicePlatform } from "@t3tools/contracts"; +import iphone18Pro from "./models/iphone-18-pro.glb?url"; +import iphone18ProMax from "./models/iphone-18-pro-max.glb?url"; +import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url"; +import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url"; + +// Bundled URLs follow the client origin in local, desktop, hosted and remote sessions. +const models: Record = { + "iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro }, + "iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax }, + "ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 }, +}; + +export function deviceModel(platform: DevicePlatform, name: string): DeviceModelSource | null { + const id = resolveDeviceModelId(platform, name); + return id ? models[id] : null; +} + +const keyboard: DeviceAccessorySource = { + id: "ipad-pro-13-m5-magic-keyboard", + modelId: "ipad-pro-13-m5", + url: magicKeyboard, +}; + +export function deviceKeyboard( + platform: DevicePlatform, + name: string, +): DeviceAccessorySource | null { + return resolveDeviceModelId(platform, name) === keyboard.modelId ? keyboard : null; +} diff --git a/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb new file mode 100644 index 000000000000..fd46333b4270 Binary files /dev/null and b/apps/web/src/components/device/models/ipad-pro-13-m5-magic-keyboard.glb differ diff --git a/apps/web/src/components/device/models/ipad-pro-13-m5.glb b/apps/web/src/components/device/models/ipad-pro-13-m5.glb new file mode 100644 index 000000000000..0611ac749607 Binary files /dev/null and b/apps/web/src/components/device/models/ipad-pro-13-m5.glb differ diff --git a/apps/web/src/components/device/models/iphone-18-pro-max.glb b/apps/web/src/components/device/models/iphone-18-pro-max.glb new file mode 100644 index 000000000000..52438e10f8d6 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-18-pro-max.glb differ diff --git a/apps/web/src/components/device/models/iphone-18-pro.glb b/apps/web/src/components/device/models/iphone-18-pro.glb new file mode 100644 index 000000000000..2dae3e9beaa6 Binary files /dev/null and b/apps/web/src/components/device/models/iphone-18-pro.glb differ diff --git a/apps/web/src/components/device/models/sources.json b/apps/web/src/components/device/models/sources.json new file mode 100644 index 000000000000..513a56fc291d --- /dev/null +++ b/apps/web/src/components/device/models/sources.json @@ -0,0 +1,49 @@ +{ + "author": "Apple Inc.", + "assetLicense": "No open-source or redistribution license has been established.", + "conversion": { + "blender": "4.5.9", + "gltfTransform": "4.2.1", + "changes": [ + "Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.", + "Normalized portrait axes, centered the screen and uniformly scaled its height to 2.2 scene units.", + "Replaced the baked marketing screen with a named placeholder for the live framebuffer.", + "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry." + ] + }, + "models": [ + { + "id": "iphone-18-pro", + "file": "iphone-18-pro.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz", + "sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317", + "bodyNode": "UBGArkKGrAMRRnj", + "screenNode": "EHjrGDHNHzgNjtE.001" + }, + { + "id": "iphone-18-pro-max", + "file": "iphone-18-pro-max.glb", + "sourceUrl": "https://www.apple.com/105/media/us/iphone-18-pro/2026/591df885-5ee2-4173-86e6-401021249f7c/ar/iphone-18-pro-e-sim.usdz", + "sourceSha256": "0f2dac354fbc6505e7917983de460f2cc68e253f96aabf52e9f0468a4803f317", + "bodyNode": "wUOhcMgiBmCgGaw", + "screenNode": "EHjrGDHNHzgNjtE" + }, + { + "id": "ipad-pro-13-m5", + "file": "ipad-pro-13-m5.glb", + "sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz", + "sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6", + "bodyNode": "zRrSLDpdYmKeRJQ", + "screenNode": "lsDiIbtoSGSmWWZ" + }, + { + "id": "ipad-pro-13-m5-magic-keyboard", + "file": "ipad-pro-13-m5-magic-keyboard.glb", + "sourceUrl": "https://www.apple.com/105/media/us/ipad-pro/2025/adee90db-c01e-430d-b726-fe64c0063f08/ar/ipad-pro-space-black.usdz", + "sourceSha256": "48098daf7c49117c81a4e4e1808899b93fad081faeba6c8b65124946b83b5ac6", + "bodyNode": "zRrSLDpdYmKeRJQ", + "screenNode": "lsDiIbtoSGSmWWZ", + "accessoryNode": "PoBqSMmyhhcJsBX" + } + ] +} diff --git a/apps/web/src/components/device/phoneTrackpad.test.ts b/apps/web/src/components/device/phoneTrackpad.test.ts new file mode 100644 index 000000000000..b8fb3f334188 --- /dev/null +++ b/apps/web/src/components/device/phoneTrackpad.test.ts @@ -0,0 +1,105 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { bindPhoneTrackpad } from "./phoneTrackpad"; + +afterEach(() => vi.useRealTimers()); + +class Canvas extends EventTarget { + getBoundingClientRect() { + return { width: 400, height: 800 } as DOMRect; + } +} +const wheel = (ctrlKey = false) => + Object.assign(new Event("wheel", { cancelable: true }), { + deltaX: 16, + deltaY: -20, + deltaMode: 0, + ctrlKey, + }); +const gesture = (type: string, scale: number) => + Object.assign(new Event(type, { cancelable: true }), { scale }); + +it("consumes scrolling and page zoom only on the bound canvas and releases listeners on detach", () => { + const canvas = new Canvas(); + const navigate = vi.fn(() => true); + const listeners = vi.spyOn(canvas, "addEventListener"); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }); + const swipe = wheel(); + canvas.dispatchEvent(swipe); + const pinch = wheel(true); + canvas.dispatchEvent(pinch); + expect(swipe.defaultPrevented).toBe(true); + expect(pinch.defaultPrevented).toBe(true); + expect(navigate.mock.calls).toEqual([[{ type: "orbit", x: -0.04, y: 0.025 }]]); + expect( + listeners.mock.calls.every( + ([, , options]) => typeof options === "object" && options.passive === false, + ), + ).toBe(true); + binding.dispose(); + const detached = wheel(true); + canvas.dispatchEvent(detached); + expect(detached.defaultPrevented).toBe(false); + expect(navigate).toHaveBeenCalledOnce(); +}); + +it("consumes Safari pinch without zooming the model and resumes orbit after cancellation", () => { + const canvas = new Canvas(); + const navigate = vi.fn(() => true); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }); + canvas.dispatchEvent(gesture("gesturestart", 1)); + canvas.dispatchEvent(gesture("gesturechange", 1.2)); + canvas.dispatchEvent(wheel(true)); + expect(navigate).not.toHaveBeenCalled(); + binding.cancel(); + canvas.dispatchEvent(wheel()); + expect(navigate).toHaveBeenCalledOnce(); + binding.dispose(); +}); + +it("normalizes Chrome and Safari pinch, expires wheel sequences and cancels on detach", () => { + vi.useFakeTimers(); + const canvas = new Canvas(); + const pinch = { begin: vi.fn(() => true), move: vi.fn(), end: vi.fn() }; + const navigate = vi.fn(() => true); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel: vi.fn() }, pinch); + canvas.dispatchEvent(wheel(true)); + canvas.dispatchEvent(wheel(true)); + expect(pinch.begin).toHaveBeenCalledOnce(); + expect(pinch.move.mock.calls).toEqual([[0.2], [0.2]]); + vi.advanceTimersByTime(180); + expect(pinch.end).toHaveBeenCalledOnce(); + canvas.dispatchEvent(gesture("gesturestart", 1)); + canvas.dispatchEvent(gesture("gesturechange", 1.2)); + expect(pinch.move).toHaveBeenLastCalledWith(Math.log(1.2)); + canvas.dispatchEvent(wheel(true)); + expect(pinch.move).toHaveBeenCalledTimes(3); + binding.dispose(); + const count = pinch.move.mock.calls.length; + vi.advanceTimersByTime(500); + canvas.dispatchEvent(wheel(true)); + expect(pinch.move).toHaveBeenCalledTimes(count); + expect(navigate).not.toHaveBeenCalled(); +}); + +it("keeps a paused wheel orbit active until native release or browser fallback", () => { + vi.useFakeTimers(); + const canvas = new Canvas(); + const navigate = vi.fn(() => true); + const endWheel = vi.fn(); + const binding = bindPhoneTrackpad(canvas, { navigate, endWheel }); + canvas.dispatchEvent(wheel()); + vi.advanceTimersByTime(500); + expect(endWheel).not.toHaveBeenCalled(); + binding.endOrbit(); + expect(endWheel).toHaveBeenCalledOnce(); + canvas.dispatchEvent(wheel()); + vi.advanceTimersByTime(1199); + expect(endWheel).toHaveBeenCalledOnce(); + vi.advanceTimersByTime(1); + expect(endWheel).toHaveBeenCalledTimes(2); + canvas.dispatchEvent(wheel()); + canvas.dispatchEvent(Object.assign(wheel(), { momentum: true })); + expect(endWheel).toHaveBeenCalledTimes(3); + expect(navigate).toHaveBeenCalledTimes(3); + binding.dispose(); +}); diff --git a/apps/web/src/components/device/phoneTrackpad.ts b/apps/web/src/components/device/phoneTrackpad.ts new file mode 100644 index 000000000000..2851013b61df --- /dev/null +++ b/apps/web/src/components/device/phoneTrackpad.ts @@ -0,0 +1,137 @@ +// @effect-diagnostics globalTimers:off - Canvas-local wheel gesture expiry. +import { + phoneWheelNavigation, + type createPhoneInteraction, +} from "@t3tools/client-runtime/device/phone-interaction"; + +/** Canvas-local, non-passive listeners consume browser zoom. Safari reports cumulative pinch scale instead of Ctrl-wheel. */ +export function bindPhoneTrackpad( + canvas: Pick< + HTMLCanvasElement, + "addEventListener" | "removeEventListener" | "getBoundingClientRect" + >, + interaction: Pick, "navigate" | "endWheel">, + pinch?: { + begin: (x: number, y: number) => boolean; + move: (logScale: number) => void; + end: () => void; + }, +) { + let scale: number | null = null; + let wheelActive = false; + let timer: ReturnType | null = null; + let orbitActive = false; + let orbitTimer: ReturnType | null = null; + const endOrbit = () => { + if (orbitTimer) clearTimeout(orbitTimer); + orbitTimer = null; + if (!orbitActive) return; + orbitActive = false; + interaction.endWheel(); + }; + const finish = () => { + if (timer) clearTimeout(timer); + timer = null; + wheelActive = false; + scale = null; + pinch?.end(); + }; + const begin = (event: Event) => { + const rect = canvas.getBoundingClientRect(); + const x = "clientX" in event && typeof event.clientX === "number" ? event.clientX : NaN; + const y = "clientY" in event && typeof event.clientY === "number" ? event.clientY : NaN; + pinch?.begin((x - rect.left) / rect.width, (y - rect.top) / rect.height); + }; + const consume = (event: Event) => { + event.preventDefault(); + event.stopPropagation(); + }; + const wheel = (event: WheelEvent) => { + consume(event); + if (scale !== null) return; + if (event.ctrlKey) { + endOrbit(); + if (!wheelActive) { + begin(event); + wheelActive = true; + } + const unit = + event.deltaMode === 1 + ? 16 + : event.deltaMode === 2 + ? canvas.getBoundingClientRect().height + : 1; + pinch?.move((-event.deltaY * unit) / 100); + if (timer) clearTimeout(timer); + timer = setTimeout(finish, 180); + return; + } + if (wheelActive) finish(); + if ("momentum" in event && event.momentum === true) { + endOrbit(); + return; + } + const rect = canvas.getBoundingClientRect(); + const navigation = phoneWheelNavigation({ + width: rect.width, + height: rect.height, + deltaX: event.deltaX, + deltaY: event.deltaY, + deltaMode: event.deltaMode, + ctrlKey: event.ctrlKey, + }); + if (navigation && interaction.navigate(navigation)) { + orbitActive = true; + if (orbitTimer) clearTimeout(orbitTimer); + // Browsers without a release signal still return to a useful view. + orbitTimer = setTimeout(endOrbit, 1200); + } + }; + const gestureScale = (event: Event) => { + if ( + !("scale" in event) || + typeof event.scale !== "number" || + !Number.isFinite(event.scale) || + event.scale <= 0 + ) + return null; + return event.scale; + }; + const start = (event: Event) => { + consume(event); + endOrbit(); + finish(); + begin(event); + scale = gestureScale(event) ?? 1; + }; + const change = (event: Event) => { + consume(event); + const next = gestureScale(event); + if (scale === null || next === null) return; + pinch?.move(Math.log(next / scale)); + scale = next; + }; + const end = (event: Event) => { + consume(event); + finish(); + }; + canvas.addEventListener("wheel", wheel, { passive: false }); + canvas.addEventListener("gesturestart", start, { passive: false }); + canvas.addEventListener("gesturechange", change, { passive: false }); + canvas.addEventListener("gestureend", end, { passive: false }); + return { + endOrbit, + cancel() { + endOrbit(); + finish(); + }, + dispose() { + canvas.removeEventListener("wheel", wheel); + canvas.removeEventListener("gesturestart", start); + canvas.removeEventListener("gesturechange", change); + canvas.removeEventListener("gestureend", end); + endOrbit(); + finish(); + }, + }; +} diff --git a/apps/web/src/components/device/useDeviceControls.test.tsx b/apps/web/src/components/device/useDeviceControls.test.tsx new file mode 100644 index 000000000000..cadc0c7852ee --- /dev/null +++ b/apps/web/src/components/device/useDeviceControls.test.tsx @@ -0,0 +1,232 @@ +import { EnvironmentId, type DeviceDetail, type DeviceSummary } from "@t3tools/contracts"; +import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access"; +import { act, useEffect } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test"; +import { useDeviceControls, type DeviceControls } from "./useDeviceControls"; + +type Result = { _tag: "Success"; value: DeviceDetail } | { _tag: "Failure"; cause: unknown }; +const { read, action, subscribe } = vi.hoisted(() => ({ + read: vi.fn<() => Promise>(), + action: vi.fn<() => Promise>(), + subscribe: + vi.fn<(_target: unknown, onChange: (app: { id: string } | null) => void) => () => void>(), +})); +vi.mock("~/state/device", () => ({ deviceEnvironment: { detail: "detail", action: "action" } })); +vi.mock("~/state/use-atom-command", () => ({ + useAtomCommand: (command: string) => (command === "detail" ? read : action), +})); +vi.mock("~/state/query", () => ({ formatEnvironmentQueryError: () => "Device action failed" })); +vi.mock("./deviceHubApi", () => ({ subscribeDeviceForeground: subscribe })); + +const device: DeviceSummary = { + hostId: "remote", + id: "phone", + name: "Phone", + platform: "ios", + version: "iOS", + booted: true, + physical: false, +}; +const snapshot = (appearance: "light" | "dark"): Result => ({ + _tag: "Success", + value: { + hostId: "remote", + deviceId: "phone", + settings: { appearance }, + foregroundApp: null, + readAt: "2026-09-20T00:00:00Z", + }, +}); +function deferred() { + let resolve!: (value: Result) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +let renderer: ReactTestRenderer | undefined; +let controls: DeviceControls; +function Probe({ visible, access = null }: { visible: boolean; access?: DeviceHubAccess | null }) { + const next = useDeviceControls({ + environmentId: EnvironmentId.make("environment"), + device, + access, + visible, + }); + useEffect(() => { + controls = next; + }, [next]); + return null; +} +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + read.mockReset(); + action.mockReset(); + subscribe.mockReset(); + subscribe.mockReturnValue(() => {}); + read.mockResolvedValue(snapshot("light")); +}); +afterEach(async () => { + await act(async () => renderer?.unmount()); + vi.unstubAllGlobals(); +}); +async function mount() { + await act(async () => { + renderer = create(); + }); +} + +it("serializes rapid actions and retains confirmed settings while pending or after failure", async () => { + await mount(); + const response = deferred(); + action.mockReturnValueOnce(response.promise); + let pending!: Promise; + await act(async () => { + pending = controls.act({ type: "setAppearance", value: "dark" }); + await controls.act({ type: "setAppearance", value: "light" }); + }); + expect(action).toHaveBeenCalledOnce(); + expect(controls.disabled).toBe(true); + expect(controls.detail?.settings.appearance).toBe("light"); + await act(async () => { + response.resolve(snapshot("dark")); + await pending; + }); + expect(controls.detail?.settings.appearance).toBe("dark"); + action.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" }); + await act(async () => controls.act({ type: "setAppearance", value: "light" })); + expect(controls.detail?.settings.appearance).toBe("dark"); + expect(controls.error).toBe("Device action failed"); + expect(controls.disabled).toBe(false); +}); + +it("serializes host actions across hide/reopen and discards detached results", async () => { + await mount(); + const response = deferred(); + action.mockReturnValueOnce(response.promise); + let pending!: Promise; + await act(async () => { + pending = controls.act({ type: "setAppearance", value: "dark" }); + }); + await act(async () => renderer!.update()); + expect(controls.disabled).toBe(true); + await act(async () => renderer!.update()); + await act(async () => controls.act({ type: "setAppearance", value: "light" })); + expect(action).toHaveBeenCalledOnce(); + expect(controls.disabled).toBe(true); + const confirmed = deferred(); + read.mockReturnValueOnce(confirmed.promise); + await act(async () => { + response.resolve(snapshot("dark")); + await response.promise; + }); + expect(controls.detail?.settings.appearance).toBe("light"); + expect(controls.disabled).toBe(true); + await act(async () => { + confirmed.resolve(snapshot("dark")); + await pending; + }); + expect(controls.detail?.settings.appearance).toBe("dark"); + expect(controls.disabled).toBe(false); + action.mockResolvedValueOnce(snapshot("dark")); + await act(async () => controls.act({ type: "setAppearance", value: "dark" })); + expect(action).toHaveBeenCalledTimes(2); + expect(controls.detail?.settings.appearance).toBe("dark"); +}); + +it("keeps actions disabled when settings cannot be confirmed after a detached command", async () => { + await mount(); + const response = deferred(); + action.mockReturnValueOnce(response.promise); + let pending!: Promise; + await act(async () => { + pending = controls.act({ type: "setAppearance", value: "dark" }); + }); + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + read.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" }); + await act(async () => { + response.resolve(snapshot("dark")); + await pending; + }); + expect(controls.detail).toBeNull(); + expect(controls.disabled).toBe(true); + expect(controls.error).toBe("Device action failed"); + await act(async () => controls.act({ type: "setAppearance", value: "light" })); + expect(action).toHaveBeenCalledOnce(); + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + expect(controls.disabled).toBe(false); + expect(controls.error).toBeNull(); +}); + +it("disables stale settings after a failed reopen read and recovers on a successful read", async () => { + await mount(); + await act(async () => renderer!.update()); + read.mockResolvedValueOnce({ _tag: "Failure", cause: "offline" }); + await act(async () => renderer!.update()); + expect(controls.detail).toBeNull(); + expect(controls.disabled).toBe(true); + expect(controls.error).toBe("Device action failed"); + await act(async () => controls.act({ type: "setAppearance", value: "dark" })); + expect(action).not.toHaveBeenCalled(); + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + expect(controls.detail?.settings.appearance).toBe("light"); + expect(controls.disabled).toBe(false); + expect(controls.error).toBeNull(); +}); + +it("does not let an older refresh overwrite a newer confirmed action", async () => { + await mount(); + await act(async () => renderer!.update()); + const refresh = deferred(); + read.mockReturnValueOnce(refresh.promise); + await act(async () => renderer!.update()); + action.mockResolvedValueOnce(snapshot("dark")); + await act(async () => controls.act({ type: "setAppearance", value: "dark" })); + await act(async () => { + refresh.resolve(snapshot("light")); + await refresh.promise; + }); + expect(controls.detail?.settings.appearance).toBe("dark"); +}); + +it.each(["hidden", "access renewed"] as const)( + "uses the detail snapshot until a new foreground event after the subscription is %s", + async (restart) => { + const access: DeviceHubAccess = { + httpBase: "https://remote.test/api/device-hub", + wsBase: "wss://remote.test/api/device-hub", + query: {}, + credentials: false, + }; + const detail = snapshot("light"); + if (detail._tag !== "Success") throw new Error("Expected a successful snapshot"); + read.mockResolvedValue({ + ...detail, + value: { ...detail.value, foregroundApp: { id: "snapshot.app" } }, + }); + await act(async () => { + renderer = create(); + }); + const oldEvent = subscribe.mock.calls[0]![1]; + await act(async () => oldEvent({ id: "previous.live.app" })); + expect(controls.foregroundApp?.id).toBe("previous.live.app"); + if (restart === "hidden") { + await act(async () => renderer!.update()); + await act(async () => renderer!.update()); + } else { + await act(async () => + renderer!.update(), + ); + } + expect(controls.foregroundApp?.id).toBe("snapshot.app"); + await act(async () => oldEvent({ id: "late.previous.app" })); + expect(controls.foregroundApp?.id).toBe("snapshot.app"); + const newEvent = subscribe.mock.calls[1]![1]; + await act(async () => newEvent({ id: "current.live.app" })); + expect(controls.foregroundApp?.id).toBe("current.live.app"); + }, +); diff --git a/apps/web/src/components/device/useDeviceControls.ts b/apps/web/src/components/device/useDeviceControls.ts new file mode 100644 index 000000000000..1ac8d2862984 --- /dev/null +++ b/apps/web/src/components/device/useDeviceControls.ts @@ -0,0 +1,130 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { + DeviceActionInput, + DeviceDetail, + DeviceSummary, + EnvironmentId, +} from "@t3tools/contracts"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { deviceEnvironment } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { subscribeDeviceForeground, type DeviceForegroundInfo } from "./deviceHubApi"; + +type ActionBody = DeviceActionInput extends infer A + ? A extends { readonly type: string } + ? Omit + : never + : never; + +/** One confirmed settings snapshot and serialized actions for the rail and drawer. Mount keyed by device. */ +export function useDeviceControls(options: { + environmentId: EnvironmentId; + device: DeviceSummary; + access: DeviceHubAccess | null; + visible: boolean; +}) { + const { environmentId, device, access, visible } = options; + const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); + const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); + const [detail, setDetail] = useState(null); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [foreground, setForeground] = useState(); + // Hiding invalidates UI results, but does not cancel host commands. Keep them serialized + // until settlement rather than allowing a reopened panel to race the previous command. + const busy = useRef(false); + const generation = useRef(0); + const visibleRead = useRef[0] | null>(null); + const target = useMemo( + () => ({ hostId: device.hostId, deviceId: device.id }), + [device.hostId, device.id], + ); + + useEffect(() => { + const revision = ++generation.current; + if (!visible) return; + const request = { environmentId, input: target }; + visibleRead.current = request; + void readDetail(request).then((result) => { + if (generation.current !== revision) return; + if (result._tag === "Success") { + setDetail(result.value); + setError(null); + } else { + setDetail(null); + setError(formatEnvironmentQueryError(result.cause)); + } + }); + return () => { + visibleRead.current = null; + generation.current++; + }; + }, [environmentId, readDetail, target, visible]); + + useEffect(() => { + if (!access || !visible) return; + let active = true; + const unsubscribe = subscribeDeviceForeground( + { access, platform: device.platform, deviceId: device.id }, + (app) => { + if (active) setForeground(app); + }, + ); + return () => { + active = false; + unsubscribe(); + setForeground(undefined); + }; + }, [access, device.id, device.platform, visible]); + + const available = detail !== null && visible; + const act = async (body: ActionBody) => { + if (busy.current || !available) return; + busy.current = true; + // A settings read started before this action must not overwrite its confirmed result. + const revision = ++generation.current; + setPending(true); + setError(null); + return runAction({ + environmentId, + input: { ...target, ...body } as DeviceActionInput, + }) + .then((result) => { + if (generation.current !== revision) { + const request = visibleRead.current; + if (!request) return; + // Reopening can read settings before the host command finishes. Confirm them again + // after it settles, keeping actions serialized through this refresh. + const refreshRevision = ++generation.current; + return readDetail(request).then((refreshed) => { + if (generation.current !== refreshRevision) return; + if (refreshed._tag === "Success") { + setDetail(refreshed.value); + setError(null); + } else { + setDetail(null); + setError(formatEnvironmentQueryError(refreshed.cause)); + } + }); + } + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + }) + .finally(() => { + busy.current = false; + setPending(false); + }); + }; + + return { + detail, + pending, + error, + act, + disabled: pending || !detail || !visible, + foregroundApp: foreground === undefined ? (detail?.foregroundApp ?? null) : foreground, + }; +} + +export type DeviceControls = ReturnType; diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx index 75ed20cc4fe7..542b84907b6e 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.test.tsx @@ -8,13 +8,22 @@ import { type PreviewOpenInput, type PreviewSessionSnapshot, } from "@t3tools/contracts"; +import type { AtomCommandResult } from "@t3tools/client-runtime/state/runtime"; +import * as Cause from "effect/Cause"; import { AsyncResult, Atom } from "effect/unstable/reactivity"; import { act } from "react"; import { create, type ReactTestRenderer } from "react-test-renderer"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; import { __resetClientSettingsPersistenceForTests } from "~/hooks/useSettings"; -import { readThreadPreviewState, resetPreviewStateForTests } from "~/previewStateStore"; +import { + applyPreviewServerSnapshot, + readThreadPreviewState, + resetPreviewStateForTests, + applyPreviewDesktopState, +} from "~/previewStateStore"; import { appAtomRegistry, AppAtomRegistryProvider } from "~/rpc/atomRegistry"; import { PreviewAutomationHosts } from "./PreviewAutomationHosts"; @@ -31,7 +40,7 @@ const mocks = vi.hoisted(() => ({ vi.fn< (target: { environmentId: EnvironmentId; input: PreviewAutomationResponse }) => Promise >(), - focus: vi.fn(async () => undefined), + focus: vi.fn<() => Promise>>(), })); vi.mock("~/localApi", () => ({ @@ -110,12 +119,21 @@ beforeEach(async () => { vi.clearAllMocks(); mocks.getClientSettings.mockReset().mockResolvedValue(savedSettings); mocks.respond.mockReset(); + mocks.focus.mockReset().mockResolvedValue(AsyncResult.success(undefined)); __resetClientSettingsPersistenceForTests(); resetPreviewStateForTests(); + useBrowserSurfaceStore.setState({ byTabId: {} }); appAtomRegistry.set(requestsAtom, AsyncResult.initial(false)); vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); vi.stubGlobal("window", { addEventListener: vi.fn(), removeEventListener: vi.fn() }); - vi.stubGlobal("document", { hasFocus: () => false, querySelectorAll: () => [] }); + vi.stubGlobal( + "document", + Object.assign(new EventTarget(), { + hasFocus: () => false, + visibilityState: "visible", + querySelectorAll: () => [], + }), + ); await act(() => { renderer = create( @@ -161,7 +179,11 @@ describe("PreviewAutomationHosts open", () => { input: { threadId, viewport, profileId: "work" }, }); expect(mocks.getClientSettings).toHaveBeenCalledOnce(); - await expect(response.promise).resolves.toMatchObject({ requestId: "open-request", ok: true }); + await expect(response.promise).resolves.toMatchObject({ + requestId: "open-request", + ok: true, + result: { available: false, tabId: snapshot.tabId }, + }); expect(readThreadPreviewState(threadRef).snapshot).toEqual(snapshot); expect(mocks.setClientSettings).not.toHaveBeenCalled(); }); @@ -188,3 +210,154 @@ describe("PreviewAutomationHosts open", () => { expect(mocks.setClientSettings).not.toHaveBeenCalled(); }); }); + +describe("PreviewAutomationHosts ownership", () => { + it("reports only local live tabs and removes ownership when their web contents close", async () => { + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "automation-connection" }), + ); + applyPreviewServerSnapshot(threadRef, snapshot); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ input: expect.objectContaining({ liveTabs: [] }) }), + ); + const overlay = { + hasWebContents: true, + canGoBack: false, + canGoForward: false, + loading: false, + zoomFactor: 1, + pictureInPicture: false, + colorScheme: "system" as const, + audioMuted: false, + audible: false, + controller: "none" as const, + favicon: null, + }; + await act(() => applyPreviewDesktopState(threadRef, snapshot.tabId, overlay)); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: false }], + }), + }), + ); + const reportCount = mocks.focus.mock.calls.length; + await act(() => + applyPreviewDesktopState(threadRef, snapshot.tabId, { ...overlay, loading: true }), + ); + expect(mocks.focus).toHaveBeenCalledTimes(reportCount); + const runtimeTabId = previewRuntimeTabId(threadRef, null, snapshot.tabId); + const owner = Symbol(); + await act(() => { + useBrowserSurfaceStore.getState().claim(runtimeTabId, owner, false); + useBrowserSurfaceStore + .getState() + .present(runtimeTabId, owner, { x: 0, y: 0, width: 800, height: 600 }, true, 0, 1); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: true }], + }), + }), + ); + for (const visibilityState of ["hidden", "visible"]) { + await act(() => { + Object.assign(document, { visibilityState }); + document.dispatchEvent(new Event("visibilitychange")); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + focused: false, + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: visibilityState === "visible" }], + }), + }), + ); + } + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "reconnected" }), + ); + }); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + connectionId: "reconnected", + liveTabs: [{ threadId, tabId: snapshot.tabId, visible: true }], + }), + }), + ); + await act(() => + applyPreviewDesktopState(threadRef, snapshot.tabId, { ...overlay, hasWebContents: false }), + ); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ input: expect.objectContaining({ liveTabs: [] }) }), + ); + }); + + it.each([false, true])( + "retries failed reports without clearing newer connection reports (reconnect: %s)", + async (reconnect) => { + const report = deferred>>(); + mocks.focus.mockReturnValueOnce(report.promise); + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "first" }), + ); + }); + if (reconnect) { + await act(() => { + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ type: "connected", connectionId: "second" }), + ); + }); + } + await act(async () => { + report.resolve(AsyncResult.failure(Cause.fail(new Error("Focus report failed")))); + await report.promise; + }); + expect(mocks.focus).toHaveBeenCalledTimes(reconnect ? 2 : 1); + await act(() => { + document.dispatchEvent(new Event("visibilitychange")); + }); + expect(mocks.focus).toHaveBeenCalledTimes(2); + expect(mocks.focus).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ connectionId: reconnect ? "second" : "first" }), + }), + ); + }, + ); + + it("does not claim an available runtime from a server snapshot alone", async () => { + const response = deferred(); + mocks.respond.mockImplementationOnce(async ({ input }) => response.resolve(input)); + await act(async () => { + applyPreviewServerSnapshot(threadRef, snapshot); + appAtomRegistry.set( + requestsAtom, + AsyncResult.success({ + ...requestEvent, + request: { + ...requestEvent.request, + operation: "status", + tabId: snapshot.tabId, + input: {}, + }, + }), + ); + await response.promise; + }); + await expect(response.promise).resolves.toMatchObject({ + ok: true, + result: { available: false, tabId: snapshot.tabId }, + }); + }); +}); diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index a793e8a2c8c0..173a027e9df2 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -1,6 +1,7 @@ "use client"; import { RegistryContext, useAtomSet, useAtomValue } from "@effect/atom-react"; +import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { FILL_PREVIEW_VIEWPORT, @@ -22,12 +23,14 @@ import { import { resolvePreviewViewport } from "@t3tools/shared/previewViewport"; import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { Atom } from "effect/unstable/reactivity"; +import { useShallow } from "zustand/react/shallow"; import { applyPreviewServerSnapshot, readThreadPreviewState, reconcilePreviewServerSessions, updatePreviewServerSnapshot, + useActivePreviewSessions, } from "~/previewStateStore"; import { browserMiniPlayerSource, @@ -250,7 +253,7 @@ const currentStatus = async ( } const navStatus = snapshot?.navStatus; return { - available: Boolean(previewBridge?.automation), + available: false, visible, tabId, url: navStatus && navStatus._tag !== "Idle" ? navStatus.url : null, @@ -292,6 +295,30 @@ export function PreviewAutomationHosts() { function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) { const { environmentId } = props; + const previewSessions = useActivePreviewSessions(); + const visibleRuntimeTabIds = useBrowserSurfaceStore( + useShallow((state) => + Object.keys(state.byTabId).filter((tabId) => state.byTabId[tabId]?.visible), + ), + ); + const liveTabs = useMemo( + () => + Object.entries(previewSessions).flatMap(([key, state]) => { + const ref = parseScopedThreadKey(key); + if (ref?.environmentId !== environmentId) return []; + return Object.values(state.sessions) + .filter((tab) => state.desktopByTabId[tab.tabId]?.hasWebContents) + .map((tab) => ({ + threadId: ref.threadId, + tabId: tab.tabId, + visible: visibleRuntimeTabIds.includes( + previewRuntimeTabId(ref, state.serverEpoch, tab.tabId), + ), + })); + }), + [environmentId, previewSessions, visibleRuntimeTabIds], + ); + const lastFocusReportRef = useRef(null); const registry = useContext(RegistryContext); const [automationClientId] = useState(createPreviewAutomationClientId); const initialAutomationHost = useMemo( @@ -804,24 +831,35 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) useEffect(() => { const report = () => { if (!automationConnectionId) return; - void focusAutomationHost({ + const input = { + clientId: automationClientId, environmentId, - input: { - clientId: automationClientId, - environmentId, - connectionId: automationConnectionId, - focused: document.hasFocus(), - }, + connectionId: automationConnectionId, + focused: document.hasFocus() && document.visibilityState === "visible", + liveTabs: liveTabs.map((tab) => ({ + ...tab, + visible: tab.visible && document.visibilityState === "visible", + })), + }; + const reportKey = JSON.stringify(input); + if (lastFocusReportRef.current === reportKey) return; + lastFocusReportRef.current = reportKey; + void focusAutomationHost({ environmentId, input }).then((result) => { + if (result._tag === "Failure" && lastFocusReportRef.current === reportKey) { + lastFocusReportRef.current = null; + } }); }; report(); window.addEventListener("focus", report); window.addEventListener("blur", report); + document.addEventListener("visibilitychange", report); return () => { window.removeEventListener("focus", report); window.removeEventListener("blur", report); + document.removeEventListener("visibilitychange", report); }; - }, [automationClientId, automationConnectionId, environmentId, focusAutomationHost]); + }, [automationClientId, automationConnectionId, environmentId, focusAutomationHost, liveTabs]); return null; } diff --git a/apps/web/src/components/settings/SettingsFontPreviews.tsx b/apps/web/src/components/settings/SettingsFontPreviews.tsx index 2d64538d5c46..6f392598b9e7 100644 --- a/apps/web/src/components/settings/SettingsFontPreviews.tsx +++ b/apps/web/src/components/settings/SettingsFontPreviews.tsx @@ -46,7 +46,7 @@ export function PromptFontPreview() { skills={EMPTY_SKILLS} disabled={false} placeholder={DISCONNECTED_COMPOSER_PLACEHOLDER} - className="max-h-40 min-h-12" + className="max-h-42 min-h-14" onChange={onChange} onPaste={noop} /> diff --git a/apps/web/src/components/ui/switch.test.tsx b/apps/web/src/components/ui/switch.test.tsx new file mode 100644 index 000000000000..b63fa79e3849 --- /dev/null +++ b/apps/web/src/components/ui/switch.test.tsx @@ -0,0 +1,20 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vite-plus/test"; + +import { Switch } from "./switch"; + +function renderSwitchRoot(props: Partial[0]>) { + const html = renderToStaticMarkup(); + return html.match(/]*role="switch"[^>]*>/)?.[0] ?? ""; +} + +describe("Switch accessibility", () => { + it("exposes the checked state to assistive tech", () => { + expect(renderSwitchRoot({ checked: true })).toContain('aria-checked="true"'); + expect(renderSwitchRoot({ checked: false })).toContain('aria-checked="false"'); + }); + + it("exposes the mixed state", () => { + expect(renderSwitchRoot({ checked: false, mixed: true })).toContain('aria-checked="mixed"'); + }); +}); diff --git a/apps/web/src/components/ui/switch.tsx b/apps/web/src/components/ui/switch.tsx index 33114c5512be..e6bf20aa64c7 100644 --- a/apps/web/src/components/ui/switch.tsx +++ b/apps/web/src/components/ui/switch.tsx @@ -27,7 +27,10 @@ function Switch({ data-size={size} data-slot="switch" data-mixed={mixed ? "" : undefined} - aria-checked={mixed ? "mixed" : undefined} + // Base UI copies every key we pass, even `undefined`, over its own + // aria-checked. Only pass the attribute when mixed so the real state + // survives for screen readers. + {...(mixed ? { "aria-checked": "mixed" as const } : {})} {...props} > ` (repeatable) or `--theme all` to capture the app's other palettes: `t3-code`, `t3-chat`, `grove`, `ocean`, `ember`, and `iris`. The runner hands the palette to the app as a launch @@ -72,23 +72,32 @@ The default matrix is: | `google-play/tablet-7/dark/t3-code/` | Pixel AVD at 600dp width | 1080×1920 | Google Play 7-inch tablet, portrait 9:16 | | `google-play/tablet-10/dark/t3-code/` | Pixel AVD at 800dp width | 1440×2560 | Google Play 10-inch tablet, portrait 9:16 | -Each target captures thread, terminal, review, thread list, and environments. Each palette folder's -five screenshots satisfy the configured Apple limit of 1–10, Google +Each target captures thread, terminal, review, thread list, and environments, and every target but +the iPad also captures agent activity. Each palette folder's five or six screenshots satisfy the configured Apple limit of 1–10, Google phone requirement of 2–8, and Google tablet recommendation/slot minimum of 4 with a maximum of 8. Every palette gets its own leaf folder so one upload slot never mixes themes and each folder keeps a store-legal screenshot count. +The agent-activity scene shows what a user sees away from the app. The app stages the same Live +Activity (iOS) or ongoing Live Update (Android) the relay would publish for four seeded threads. On +iOS the runner then locks the simulator and pushes the matching approval alert with `simctl push`; +on Android the staged update carries the alert and the runner opens the notification shade. Locking +the simulator and answering the notification permission prompt use +[AXe](https://github.com/cameroncooke/AXe), so install it (`brew tap cameroncooke/axe && brew +install axe`) or set `AXE_PATH` before capturing iOS. The iPad skips the scene because the lock +screen does not follow the app's landscape self-rotation. + The generated tree is deliberately aligned with the store upload fields: artifacts/app-store/screenshots/ ├── apple/ - │ ├── iphone-6.9/dark/t3-code/{thread,terminal,review,threads,environments}.png - │ ├── iphone-6.5/dark/t3-code/{thread,terminal,review,threads,environments}.png + │ ├── iphone-6.9/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png + │ ├── iphone-6.5/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png │ └── ipad-13/dark/t3-code/{thread,terminal,review,threads,environments}.png └── google-play/ - ├── phone/dark/t3-code/{thread,terminal,review,threads,environments}.png - ├── tablet-7/dark/t3-code/{thread,terminal,review,threads,environments}.png - └── tablet-10/dark/t3-code/{thread,terminal,review,threads,environments}.png + ├── phone/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png + ├── tablet-7/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png + └── tablet-10/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png A light-only run writes the same tree under `light/`; `--appearance both` writes both appearance folders, and each requested theme adds a sibling folder next to `t3-code/`. diff --git a/docs/operations/observability.md b/docs/operations/observability.md index adeb983d1bc7..97c6936ba608 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -555,6 +555,18 @@ OTLP export: If the OTLP URLs are unset, local tracing still works, metrics stay in-process only, and logs stay on stdout only. +### The Kill Switch + +`T3CODE_OTEL_SDK_DISABLED` and `OTEL_SDK_DISABLED` turn off every OTLP export in both the server and +the desktop main process, overriding any endpoint from the environment or Settings. Local trace +files and stdout logs are unaffected. + +`T3CODE_OTEL_SDK_DISABLED` wins when set, so `T3CODE_OTEL_SDK_DISABLED=false` re-enables export on a +machine that sets `OTEL_SDK_DISABLED` for everything else. It accepts the usual boolean spellings +(`true`/`false`, `yes`/`no`, `on`/`off`, `1`/`0`, `y`/`n`). `OTEL_SDK_DISABLED` follows the +OpenTelemetry specification and only `true` disables export, so `OTEL_SDK_DISABLED=1` does not. +Values are case-insensitive and trimmed. An unrecognized value is ignored with a startup warning. + ### What Is Instrumented Today Current high-value span and metric boundaries include: diff --git a/docs/user/updating.md b/docs/user/updating.md index 25833d3e79ec..6909c7b6022d 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -58,6 +58,18 @@ update can roll back to the previous version. If the update still fails: ## Mobile updates +To update an environment from your phone, open **Settings → Environments** and +select it. **Check for updates** finds the latest release on that environment's +current release channel. Keep the app open while the environment updates and +reconnects. Hosts that cannot update remotely show instructions for updating on +the machine instead. + +The same page lets you refresh provider status and update supported providers. +These controls require a connected environment and permission to operate it. +Provider update checks and restart continuation preferences are in +**Settings → Maintenance**. If provider update checks are disabled, enable them +there before refreshing to find newer versions. + Install App Store or Google Play releases as usual. The mobile app can also download updates in the background and apply them when you next leave the app. It saves drafts and queued messages before restarting. If you keep the app open diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 63dfc4ab344c..dd6113ad6426 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -266,6 +266,30 @@ "./file-preview": { "types": "./src/filePreview.ts", "default": "./src/filePreview.ts" + }, + "./device/frame": { + "types": "./src/device/frame.ts", + "default": "./src/device/frame.ts" + }, + "./device/phone-viewer": { + "types": "./src/device/phoneViewer.ts", + "default": "./src/device/phoneViewer.ts" + }, + "./device/phone-interaction": { + "types": "./src/device/phoneInteraction.ts", + "default": "./src/device/phoneInteraction.ts" + }, + "./device/shape-profile": { + "types": "./src/device/shapeProfile.ts", + "default": "./src/device/shapeProfile.ts" + }, + "./device/screenshot": { + "types": "./src/device/screenshot.ts", + "default": "./src/device/screenshot.ts" + }, + "./device/model": { + "types": "./src/device/model.ts", + "default": "./src/device/model.ts" } }, "scripts": { @@ -280,10 +304,12 @@ "micromark-extension-directive": "^4.0.0", "micromark-util-character": "^2.1.1", "remark-parse": "^11.0.0", + "three": "0.180.0", "unified": "^11.0.5" }, "devDependencies": { "@effect/vitest": "catalog:", + "@types/three": "0.180.0", "micromark-util-types": "^2.0.2", "vite-plus": "catalog:" } diff --git a/packages/client-runtime/src/device/deviceFraming.test.ts b/packages/client-runtime/src/device/deviceFraming.test.ts new file mode 100644 index 000000000000..151a0882a89d --- /dev/null +++ b/packages/client-runtime/src/device/deviceFraming.test.ts @@ -0,0 +1,53 @@ +import { expect, it } from "vite-plus/test"; +import { Box3, Vector3 } from "three"; +import { createDeviceFraming } from "./deviceFraming.ts"; + +const box = (x: number, width: number) => + new Box3(new Vector3(x - width / 2, -1, -0.2), new Vector3(x + width / 2, 1, 0.2)); +it("fits both axes through a moving pivot, freezes for contact, and stops once framing settles", () => { + const framing = createDeviceFraming(); + framing.setBounds(box(0, 1), Math.PI / 8, 0.7, 0); + const initialDistance = framing.distance(); + const next = box(1.5, 3); + framing.setBounds(next, Math.PI / 8, 0.7, 10); + framing.advance(50); + expect(framing.center.x).toBeGreaterThan(0); + expect(framing.center.x).toBeLessThan(1.5); + for (const x of [next.min.x, next.max.x]) + expect( + Math.abs(x - framing.center.x) / + ((framing.distance() - next.max.z) * Math.tan(Math.PI / 8) * 0.7), + ).toBeLessThan(1); + framing.hold(true, 50); + const center = framing.center.clone(), + distance = framing.distance(); + framing.advance(10000); + expect(framing.center.equals(center)).toBe(true); + expect(framing.distance()).toBe(distance); + expect(framing.needsFrame()).toBe(false); + framing.hold(false, 10000); + framing.advance(10016); + expect(framing.center.x).toBeLessThan(1.5); + framing.advance(12000); + expect(framing.center.x).toBe(1.5); + expect(framing.distance()).toBeGreaterThan(initialDistance); + expect(framing.needsFrame()).toBe(false); +}); + +it("uses elapsed time independently of frame partitions and fits a changed viewport immediately", () => { + const run = (step: number) => { + const framing = createDeviceFraming(); + framing.setBounds(box(0, 1), Math.PI / 8, 1, 0); + framing.setBounds(box(-1, 2), Math.PI / 8, 1, 0); + for (let time = step; time < 400; time += step) framing.advance(time); + framing.advance(400); + return framing; + }; + const fast = run(8), + slow = run(33); + expect(fast.center.distanceTo(slow.center)).toBeLessThan(1e-8); + expect(fast.distance()).toBeCloseTo(slow.distance(), 8); + fast.setBounds(box(-1, 2), Math.PI / 8, 0.4, 400, true); + expect(fast.distance()).toBeGreaterThan(slow.distance()); + expect(fast.needsFrame()).toBe(false); +}); diff --git a/packages/client-runtime/src/device/deviceFraming.ts b/packages/client-runtime/src/device/deviceFraming.ts new file mode 100644 index 000000000000..1d8dddb10eb3 --- /dev/null +++ b/packages/client-runtime/src/device/deviceFraming.ts @@ -0,0 +1,82 @@ +import { Box3, Vector3 } from "three"; + +/** Fit the current assembly, with independent framing dynamics so a leaf pivot does not drag it off-screen. */ +export function createDeviceFraming() { + const center = new Vector3(); + const target = new Vector3(); + const velocity = new Vector3(); + const bounds = new Box3(); + let distance = 1; + let targetDistance = 1; + let distanceVelocity = 0; + let tanX = 1; + let tanY = 1; + let at = 0; + let initialized = false; + let moving = false; + let held = false; + const advance = (now: number, immediate = false) => { + if (held) return false; + const dt = Math.max(0, (now - at) / 1000); + at = now; + if (!moving) return false; + const omega = 14; + const decay = Math.exp(-omega * dt); + const error = center.clone().sub(target); + const coefficient = velocity.clone().addScaledVector(error, omega); + center.copy(error).addScaledVector(coefficient, dt).multiplyScalar(decay).add(target); + velocity.addScaledVector(coefficient, -omega * dt).multiplyScalar(decay); + const distanceError = distance - targetDistance; + const distanceCoefficient = distanceVelocity + omega * distanceError; + distance = targetDistance + (distanceError + distanceCoefficient * dt) * decay; + distanceVelocity = (distanceVelocity - omega * distanceCoefficient * dt) * decay; + moving = + center.distanceTo(target) > 0.0005 || + velocity.length() > 0.005 || + Math.abs(distance - targetDistance) > 0.0005 || + Math.abs(distanceVelocity) > 0.005; + if (immediate || !moving) { + center.copy(target); + distance = targetDistance; + velocity.set(0, 0, 0); + distanceVelocity = 0; + moving = false; + } + return true; + }; + return { + center, + setBounds(next: Box3, vertical: number, aspect: number, now: number, immediate = false) { + if (next.isEmpty()) return; + advance(now); + bounds.copy(next); + tanY = Math.tan(vertical); + tanX = tanY * aspect; + bounds.getCenter(target); + target.z = 0; + const size = bounds.getSize(new Vector3()); + targetDistance = Math.max(1, Math.max(size.x / tanX, size.y / tanY) * 0.565 + bounds.max.z); + moving = true; + advance(now, immediate || !initialized); + initialized = true; + }, + advance, + distance() { + // Spring lag must never clip the fitted assembly. + const clearance = Math.max( + Math.abs(bounds.min.x - center.x) / tanX, + Math.abs(bounds.max.x - center.x) / tanX, + Math.abs(bounds.min.y - center.y) / tanY, + Math.abs(bounds.max.y - center.y) / tanY, + ); + return Math.max(distance, clearance * 1.02 + bounds.max.z, 1); + }, + hold(active: boolean, now: number) { + held = active; + at = now; + }, + needsFrame() { + return moving && !held; + }, + }; +} diff --git a/packages/client-runtime/src/device/deviceMotion.test.ts b/packages/client-runtime/src/device/deviceMotion.test.ts new file mode 100644 index 000000000000..fc777133c351 --- /dev/null +++ b/packages/client-runtime/src/device/deviceMotion.test.ts @@ -0,0 +1,175 @@ +import { expect, it, vi } from "vite-plus/test"; +import { Quaternion, Vector3 } from "three"; +import { createDeviceMotion, rotationVector } from "./deviceMotion.ts"; +import { nearestDeviceView } from "./deviceViewSnap.ts"; + +const snaps = ["portrait", "landscape_left", "portrait_upside_down", "landscape_right"].map( + (orientation, index) => ({ + orientation, + rotation: new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), (-index * Math.PI) / 2), + yawLimit: Math.PI / 3, + }), +); +const rotation = (x: number, y: number, z = 0) => + new Quaternion().setFromAxisAngle(new Vector3(x, y, z).normalize(), Math.hypot(x, y, z)); + +it("selects the nearest quarter turn, retains a nearby yaw, and treats quaternion signs identically", () => { + const side = new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), -Math.PI / 2); + const released = side.clone().premultiply(rotation(0, 0.4)); + const nearest = nearestDeviceView(released, snaps)!; + expect(nearest.orientation).toBe("landscape_left"); + expect(nearest.rotation.angleTo(released)).toBeLessThan(1e-6); + const negative = released.clone().set(-released.x, -released.y, -released.z, -released.w); + expect(nearestDeviceView(negative, snaps)!.rotation.angleTo(nearest.rotation)).toBeLessThan(1e-6); + expect( + nearestDeviceView(rotation(0, 3), snaps)!.rotation.angleTo(new Quaternion()), + ).toBeLessThanOrEqual(Math.PI / 3 + 1e-6); +}); + +it("springs toward a cumulative drag without teleporting and keeps the same target across event partitions", () => { + const run = (parts: number) => { + const choose = vi.fn((q: Quaternion) => q); + const motion = createDeviceMotion({ choose }); + motion.dragActive(true, 0); + for (let part = 0; part < parts; part++) motion.orbit(160 / parts, 80 / parts, 0); + expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6); + for (let time = 8; time <= 400; time += 8) motion.advance(time); + expect(motion.rotation.angleTo(new Quaternion())).toBeGreaterThan(0.5); + motion.dragActive(false, 400); + for (let time = 408; time <= 2400; time += 8) motion.advance(time); + expect(choose).toHaveBeenCalledOnce(); + expect(motion.needsFrame()).toBe(false); + return motion.rotation; + }; + expect(run(1).angleTo(run(20))).toBeLessThan(1e-6); +}); + +it("selects the predicted nearest view once and preserves velocity when interrupted", () => { + const choose = vi.fn((q: Quaternion) => nearestDeviceView(q, snaps)!.rotation); + const motion = createDeviceMotion({ choose }); + motion.dragActive(true, 0); + motion.orbit(0, 240, 0); + motion.orbit(0, 20, 20); + motion.advance(40); + motion.dragActive(false, 40); + const before = motion.rotation.clone(); + motion.advance(80); + expect(motion.rotation.angleTo(before)).toBeGreaterThan(0.01); + const interrupted = motion.rotation.clone(); + motion.dragActive(true, 80); + expect(motion.rotation.angleTo(interrupted)).toBeLessThan(1e-6); + motion.orbit(30, -30, 80); + motion.advance(96); + motion.dragActive(false, 100); + for (let time = 116; time <= 2500; time += 16) motion.advance(time); + expect(choose).toHaveBeenCalledTimes(2); + expect(motion.needsFrame()).toBe(false); +}); + +it("lets a hard flick coast through multiple turns before settling on a visible view", () => { + const choose = vi.fn(() => new Quaternion()); + const motion = createDeviceMotion({ choose }); + motion.dragActive(true, 0); + motion.orbit(0, 80, 16); + motion.advance(16); + motion.orbit(0, 80, 32); + motion.advance(32); + motion.dragActive(false, 32); + + let previous = motion.rotation.clone(); + let travel = 0; + let lateSpeed = 0; + for (let time = 40; time <= 8500; time += 8) { + motion.advance(time); + const distance = motion.rotation.angleTo(previous); + travel += distance; + if (time >= 2500) lateSpeed = Math.max(lateSpeed, distance / 0.008); + previous = motion.rotation.clone(); + } + expect(travel).toBeGreaterThan(4 * Math.PI); + expect(lateSpeed).toBeLessThan(2.5); + expect(choose).toHaveBeenCalledOnce(); + expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6); + expect(motion.needsFrame()).toBe(false); +}); + +it("release uses elapsed time equally at different frame rates and a captured contact freezes projection", () => { + const run = (step: number) => { + const motion = createDeviceMotion({ choose: () => new Quaternion() }); + motion.dragActive(true, 0); + motion.orbit(160, 80, 0); + motion.advance(50); + motion.dragActive(false, 50); + for (let time = 50; time < 450; time += step) motion.advance(time); + motion.advance(450); + return motion; + }; + const fast = run(8), + slow = run(33), + throttled = run(1000); + expect(fast.rotation.angleTo(slow.rotation)).toBeLessThan(1e-6); + expect(fast.rotation.angleTo(throttled.rotation)).toBeLessThan(1e-6); + fast.hold(true, 450); + const contact = fast.rotation.clone(); + fast.orbit(100, 100, 500); + fast.advance(10000); + expect(fast.rotation.angleTo(contact)).toBeLessThan(1e-6); + fast.hold(false, 10000); + fast.advance(12000); + expect(fast.needsFrame()).toBe(false); +}); + +it("holds a trackpad orbit through a pause and snaps only when the gesture ends", () => { + const choose = vi.fn(() => new Quaternion()); + const motion = createDeviceMotion({ choose }); + motion.orbit(NaN, 0, 0); + expect(motion.needsFrame()).toBe(false); + motion.orbit(200, 100, 0); + motion.advance(100); + expect(choose).not.toHaveBeenCalled(); + expect(motion.rotation.angleTo(new Quaternion())).toBeGreaterThan(0); + motion.advance(140, true); + expect(choose).not.toHaveBeenCalled(); + const held = motion.rotation.clone(); + expect(held.angleTo(new Quaternion())).toBeGreaterThan(0.5); + expect(motion.needsFrame()).toBe(false); + motion.orbit(80, 0, 200); + motion.advance(340, true); + expect(motion.rotation.angleTo(held)).toBeGreaterThan(0.1); + expect(choose).not.toHaveBeenCalled(); + expect(motion.needsFrame()).toBe(false); + motion.dragActive(false, 340); + motion.advance(2500); + expect(choose).toHaveBeenCalledOnce(); + expect(motion.rotation.angleTo(new Quaternion())).toBeLessThan(1e-6); + expect(motion.needsFrame()).toBe(false); + expect(rotationVector(rotation(0, Math.PI)).length()).toBeCloseTo(Math.PI); +}); + +it("keeps release angular speed bounded even across a back-facing half turn", () => { + const motion = createDeviceMotion({ choose: () => rotation(0, Math.PI) }); + motion.setPose(rotation(0, Math.PI), 0); + let previous = motion.rotation.clone(); + for (let time = 8; time <= 1600; time += 8) { + motion.advance(time); + expect(motion.rotation.angleTo(previous)).toBeLessThanOrEqual(9 * 0.008 + 1e-5); + previous = motion.rotation.clone(); + } + expect(motion.rotation.angleTo(rotation(0, Math.PI))).toBeLessThan(1e-6); + expect(motion.needsFrame()).toBe(false); +}); + +it("a click without dragging resumes the interrupted resting view instead of stranding the device", () => { + const choose = vi.fn(() => new Quaternion()); + const motion = createDeviceMotion({ choose }); + const rest = rotation(0, 0.8); + motion.setPose(rest, 0); + motion.advance(50); + motion.dragActive(true, 50); + motion.advance(60); + motion.dragActive(false, 60); + motion.advance(2100); + expect(motion.rotation.angleTo(rest)).toBeLessThan(1e-6); + expect(choose).not.toHaveBeenCalled(); + expect(motion.needsFrame()).toBe(false); +}); diff --git a/packages/client-runtime/src/device/deviceMotion.ts b/packages/client-runtime/src/device/deviceMotion.ts new file mode 100644 index 000000000000..d2383c65665a --- /dev/null +++ b/packages/client-runtime/src/device/deviceMotion.ts @@ -0,0 +1,279 @@ +import { Quaternion, Vector3 } from "three"; + +/** Shortest rotation vector. q and -q represent the same orientation. */ +export function rotationVector(rotation: Quaternion) { + const q = rotation.clone().normalize(); + if (q.w < 0) q.set(-q.x, -q.y, -q.z, -q.w); + const length = Math.hypot(q.x, q.y, q.z); + return length < 1e-8 + ? new Vector3() + : new Vector3(q.x, q.y, q.z).multiplyScalar((2 * Math.atan2(length, q.w)) / length); +} +function fromVector(vector: Vector3) { + const angle = vector.length(); + return angle < 1e-8 + ? new Quaternion() + : new Quaternion().setFromAxisAngle(vector.clone().divideScalar(angle), angle); +} + +// Recovered from Bitrig 0.25's rotation dynamics: point gain, drag force limit, +// velocity limits, release prediction/blend, and the 0.5 / 0.78 release spring. +const gain = 0.006; +const response = 0.5; +const damping = 0.78; +const frequency = (2 * Math.PI) / response; +const decay = frequency * damping; +const stepSeconds = 1 / 120; +// Fast releases combine exponential friction with a critically damped correction. +const flickThreshold = 4.5; +const flickLimit = 22; +const spinDecay = 1.1; +const settleFrequency = 2; + +/** A moving spring target follows cumulative camera-space gestures; release latches the nearest useful view. */ +export function createDeviceMotion(options: { choose: (rotation: Quaternion) => Quaternion }) { + const rotation = new Quaternion(); + const target = new Quaternion(); + const velocity = new Vector3(); + const gestureVelocity = new Vector3(); + let spring: { at: number; rotation: Quaternion; velocity: Vector3; steps: number } | null = null; + let spin: { at: number; rotation: Quaternion; velocity: Vector3; correction: Vector3 } | null = + null; + let drag: { + start: Quaternion; + rest: Quaternion; + x: number; + y: number; + error: Vector3; + at: number; + } | null = null; + let held = false; + let interruptedDrag = false; + let lastInput = -Infinity; + const beginSpring = (next: Quaternion, now: number) => { + target.copy(next).normalize(); + spring = { + at: now, + rotation: rotation.clone(), + velocity: velocity.clone(), + steps: 0, + }; + }; + const release = (now: number) => { + if (!drag) return; + // A pause before lifting a pointer should not resurrect an old flick. + if (now - lastInput > 100) gestureVelocity.set(0, 0, 0); + const prediction = target + .clone() + .premultiply(fromVector(gestureVelocity.clone().multiplyScalar(0.085))); + velocity.multiplyScalar(0.2).addScaledVector(gestureVelocity, 0.8).clampLength(0, flickLimit); + const moved = drag.x !== 0 || drag.y !== 0; + const rest = drag.rest; + drag = null; + if (moved && velocity.length() >= flickThreshold) { + // Choose where the free spin would finish, then correct toward that view + // throughout the coast rather than handing off to a spring at low speed. + const projected = rotation + .clone() + .premultiply(fromVector(velocity.clone().multiplyScalar(1 / spinDecay))); + target.copy(options.choose(projected)).normalize(); + spin = { + at: now, + rotation: rotation.clone(), + velocity: velocity.clone(), + correction: rotationVector(target.clone().multiply(projected.invert())), + }; + return; + } + if (moved) beginSpring(options.choose(prediction), now); + else beginSpring(rest, now); + }; + const advance = (now: number, reduced = false) => { + if (held || !Number.isFinite(now)) return false; + if (spin) { + const seconds = Math.max(0, (now - spin.at) / 1000); + const coast = Math.exp(-spinDecay * seconds); + const settle = Math.exp(-settleFrequency * seconds); + const correction = fromVector( + spin.correction.clone().multiplyScalar(1 - (1 + settleFrequency * seconds) * settle), + ); + rotation + .copy(spin.rotation) + .premultiply(fromVector(spin.velocity.clone().multiplyScalar((1 - coast) / spinDecay))) + .premultiply(correction) + .normalize(); + velocity + .copy(spin.velocity) + .multiplyScalar(coast) + .applyQuaternion(correction) + .addScaledVector(spin.correction, settleFrequency ** 2 * seconds * settle); + if (reduced || (rotation.angleTo(target) < 0.01 && velocity.length() < 0.08)) { + spin = null; + rotation.copy(target); + velocity.set(0, 0, 0); + } + return true; + } + if (drag) { + // Follow the target during the gesture. Retain the logarithm's winding so + // a long drag cannot reverse its spring force when crossing a half turn. + const seconds = Math.min(0.05, Math.max(0, (now - drag.at) / 1000)); + drag.at = now; + const steps = Math.ceil(seconds * 120); + const dt = steps ? seconds / steps : 0; + for (let step = 0; step < steps; step++) { + const error = rotationVector(target.clone().multiply(rotation.clone().invert())); + if (error.lengthSq() > 1e-10) { + const axis = error.clone().normalize(); + const turns = Math.round((drag.error.dot(axis) - error.length()) / (2 * Math.PI)); + error.addScaledVector(axis, turns * 2 * Math.PI); + } + drag.error.copy(error); + const omega = (2 * Math.PI) / 0.68; + const acceleration = error + .clampLength(0, 1.2) + .multiplyScalar(omega * omega) + .addScaledVector(velocity, -2 * 1.12 * omega); + velocity.addScaledVector(acceleration, dt).clampLength(0, 9); + rotation.premultiply(fromVector(velocity.clone().multiplyScalar(dt))).normalize(); + } + if (reduced) { + rotation.copy(target); + velocity.set(0, 0, 0); + } + return steps > 0 || reduced; + } + if (!spring) return false; + const seconds = Math.max(0, (now - spring.at) / 1000); + const steps = Math.min(240, Math.floor(seconds / stepSeconds)); + const integrate = (q: Quaternion, speed: Vector3, dt: number) => { + const error = rotationVector(target.clone().multiply(q.clone().invert())); + const acceleration = error + .multiplyScalar(frequency * frequency) + .addScaledVector(speed, -2 * decay); + speed.addScaledVector(acceleration, dt).clampLength(0, 9); + q.premultiply(fromVector(speed.clone().multiplyScalar(dt))).normalize(); + }; + // Fixed ticks make release independent of display refresh rate. The final + // partial tick interpolates the next tick without committing it. This also + // keeps displayed angular speed bounded between tick boundaries. + for (; spring.steps < steps; spring.steps++) + integrate(spring.rotation, spring.velocity, stepSeconds); + rotation.copy(spring.rotation); + velocity.copy(spring.velocity); + const nextRotation = spring.rotation.clone(); + const nextVelocity = spring.velocity.clone(); + integrate(nextRotation, nextVelocity, stepSeconds); + const fraction = Math.min(1, (seconds - steps * stepSeconds) / stepSeconds); + rotation.slerp(nextRotation, fraction); + velocity.lerp(nextVelocity, fraction); + if ( + reduced || + steps === 240 || + (rotation.angleTo(target) < 0.0005 && velocity.length() < 0.005) + ) { + rotation.copy(target); + velocity.set(0, 0, 0); + spring = null; + } + return true; + }; + const beginDrag = (now: number) => { + if (held || drag) return; + advance(now); + drag = { + start: rotation.clone(), + rest: target.clone(), + x: 0, + y: 0, + error: new Vector3(), + at: now, + }; + target.copy(rotation); + gestureVelocity.set(0, 0, 0); + lastInput = -Infinity; + spring = null; + spin = null; + }; + return { + rotation, + setPose(next: Quaternion, now: number, immediate = false) { + advance(now); + drag = null; + spin = null; + beginSpring(next, now); + if (immediate) { + rotation.copy(target); + velocity.set(0, 0, 0); + spring = null; + } + }, + dragActive(active: boolean, now: number) { + if (active) { + // A pointer takes over at the displayed pose, even if a wheel gesture + // was paused with its target still ahead of the spring. + if (drag || spin) { + advance(now); + drag = null; + spin = null; + } + beginDrag(now); + } else { + advance(now); + release(now); + } + }, + /** Deltas are CSS pixels, independent of panel size and event partitioning. */ + orbit(x: number, y: number, now: number) { + if (held || ![x, y, now].every(Number.isFinite) || (!x && !y)) return; + advance(now); + beginDrag(now); + if (!drag) return; + const previous = target.clone(); + drag.x += x; + drag.y += y; + target + .copy(fromVector(new Vector3(drag.y * gain, drag.x * gain, 0))) + .multiply(drag.start) + .normalize(); + const seconds = (now - lastInput) / 1000; + gestureVelocity.copy(rotationVector(target.clone().multiply(previous.invert()))); + if (seconds > 0 && seconds < 0.1) + gestureVelocity.divideScalar(seconds).clampLength(0, flickLimit); + else gestureVelocity.set(0, 0, 0); + lastInput = now; + }, + /** Captured device touches and hinge edits freeze projection, independently of orbit dragging. */ + hold(active: boolean, now: number) { + if (held === active) return; + if (active) { + interruptedDrag = drag !== null || spin !== null; + drag = null; + spin = null; + spring = null; + velocity.set(0, 0, 0); + } + held = active; + if (!active) { + if (interruptedDrag) beginSpring(options.choose(rotation.clone()), now); + else if (rotation.angleTo(target) > 0.0005) beginSpring(target, now); + interruptedDrag = false; + } + }, + advance, + needsFrame() { + return ( + !held && + (spring !== null || + spin !== null || + (drag !== null && (rotation.angleTo(target) > 0.0005 || velocity.length() > 0.005))) + ); + }, + reset(next: Quaternion, now: number) { + advance(now); + drag = null; + spin = null; + beginSpring(next, now); + }, + }; +} diff --git a/packages/client-runtime/src/device/deviceViewSnap.ts b/packages/client-runtime/src/device/deviceViewSnap.ts new file mode 100644 index 000000000000..20402a80a35e --- /dev/null +++ b/packages/client-runtime/src/device/deviceViewSnap.ts @@ -0,0 +1,26 @@ +import { Quaternion, Vector3 } from "three"; +const y = new Vector3(0, 1, 0); + +/** Each useful view permits a small yaw. Pick the closest member of each family, then the closest family. */ +export function nearestDeviceView( + rotation: Quaternion, + snaps: readonly T[], +) { + let closest: T | null = null; + let distance = Infinity; + for (const snap of snaps) { + const relative = rotation.clone().multiply(snap.rotation.clone().invert()); + const turn = 2 * Math.atan2(relative.y, relative.w); + const yaw = Math.max( + -snap.yawLimit, + Math.min(snap.yawLimit, Math.atan2(Math.sin(turn), Math.cos(turn))), + ); + const candidate = snap.rotation.clone().premultiply(new Quaternion().setFromAxisAngle(y, yaw)); + const nextDistance = candidate.angleTo(rotation); + if (nextDistance < distance - 1e-8) { + distance = nextDistance; + closest = { ...snap, rotation: candidate }; + } + } + return closest; +} diff --git a/packages/client-runtime/src/device/frame.test.ts b/packages/client-runtime/src/device/frame.test.ts new file mode 100644 index 000000000000..b4ace96cd8da --- /dev/null +++ b/packages/client-runtime/src/device/frame.test.ts @@ -0,0 +1,32 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createCanvasFrameSink } from "./frame.ts"; + +it("retains a borrowed frame at its native size before notifying the viewer", () => { + const drawImage = vi.fn(); + const canvas = { + width: 300, + height: 150, + getContext: () => ({ drawImage }), + } as unknown as HTMLCanvasElement; + const source = {} as CanvasImageSource; + const notify = vi.fn(() => { + expect(canvas.width).toBe(1170); + expect(canvas.height).toBe(2532); + expect(drawImage).toHaveBeenLastCalledWith(source, 0, 0, 1170, 2532); + }); + createCanvasFrameSink(canvas, notify).present(source, 1170, 2532); + expect(notify).toHaveBeenCalledOnce(); +}); + +it("reports when a canvas cannot present a frame", () => { + const canvas = { + width: 300, + height: 150, + getContext: () => null, + } as unknown as HTMLCanvasElement; + const notify = vi.fn(); + expect(createCanvasFrameSink(canvas, notify).present({} as CanvasImageSource, 1170, 2532)).toBe( + false, + ); + expect(notify).not.toHaveBeenCalled(); +}); diff --git a/packages/client-runtime/src/device/frame.ts b/packages/client-runtime/src/device/frame.ts new file mode 100644 index 000000000000..bb875b0b4a24 --- /dev/null +++ b/packages/client-runtime/src/device/frame.ts @@ -0,0 +1,24 @@ +/** A synchronous, borrowed frame. The producer releases its source after present returns. */ +export interface DeviceFrameSink { + readonly present: (source: CanvasImageSource, width: number, height: number) => boolean; +} + +/** Retains the latest frame in a canvas; consumers can invalidate textures after each draw. */ +export function createCanvasFrameSink( + canvas: HTMLCanvasElement, + onFrame?: (width: number, height: number) => void, +): DeviceFrameSink { + return { + present(source, width, height) { + const context = canvas.getContext("2d"); + if (!context) return false; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + context.drawImage(source, 0, 0, width, height); + onFrame?.(width, height); + return true; + }, + }; +} diff --git a/packages/client-runtime/src/device/model.test.ts b/packages/client-runtime/src/device/model.test.ts new file mode 100644 index 000000000000..94db7942176b --- /dev/null +++ b/packages/client-runtime/src/device/model.test.ts @@ -0,0 +1,125 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createDeviceModelSlot, resolveDeviceModelId, type DeviceModelSource } from "./model.ts"; + +const pro: DeviceModelSource = { id: "iphone-18-pro", url: "/pro.glb" }; +const max: DeviceModelSource = { id: "iphone-18-pro-max", url: "/max.glb" }; + +function fixture() { + const pending: { + signal: AbortSignal; + resolve: (model: { dispose: () => void }) => void; + reject: (error: Error) => void; + }[] = []; + const install = vi.fn(); + const load = vi.fn( + (_source: DeviceModelSource, signal: AbortSignal) => + new Promise<{ dispose: () => void }>((resolve, reject) => + pending.push({ signal, resolve, reject }), + ), + ); + const slot = createDeviceModelSlot({ load, install }); + return { slot, pending, install, load }; +} + +it("matches only exact supported hardware, including iPad size and generation", () => { + expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id); + expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id); + expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5"); + for (const name of [ + "iPhone 17 Pro", + "iPhone 18", + "iPad Pro 11-inch (M5)", + "iPad Pro 13-inch (M4)", + "Pixel_10_Pro", + ]) { + expect(resolveDeviceModelId("ios", name)).toBeNull(); + } + expect(resolveDeviceModelId("android", "iPhone 18 Pro")).toBeNull(); +}); + +it("cancels superseded downloads and releases a late decoded model without installing it", async () => { + const { slot, pending, install, load } = fixture(); + slot.set(pro); + slot.set({ ...pro }); + expect(load).toHaveBeenCalledTimes(1); + slot.set(max); + expect(pending[0]!.signal.aborted).toBe(true); + const latest = { dispose: vi.fn() }; + const stale = { dispose: vi.fn() }; + pending[1]!.resolve(latest); + await Promise.resolve(); + pending[0]!.resolve(stale); + await Promise.resolve(); + expect(install).toHaveBeenCalledExactlyOnceWith(latest); + expect(stale.dispose).toHaveBeenCalledOnce(); + slot.set(null); + expect(install).toHaveBeenLastCalledWith(null); + expect(latest.dispose).toHaveBeenCalledOnce(); + slot.dispose(); + expect(latest.dispose).toHaveBeenCalledOnce(); +}); + +it("keeps the fallback on request failure and disposes results that finish after teardown", async () => { + const { slot, pending, install } = fixture(); + slot.set(pro); + pending[0]!.reject(new Error("offline")); + await Promise.resolve(); + expect(install).not.toHaveBeenCalled(); + slot.set(max); + slot.dispose(); + expect(pending[1]!.signal.aborted).toBe(true); + const late = { dispose: vi.fn() }; + pending[1]!.resolve(late); + await Promise.resolve(); + expect(late.dispose).toHaveBeenCalledOnce(); + expect(install).not.toHaveBeenCalled(); +}); + +it("returns the current scene to the fallback before releasing its imported resources", async () => { + const { slot, pending, install } = fixture(); + slot.set(pro); + const model = { dispose: vi.fn(() => expect(install).toHaveBeenLastCalledWith(null)) }; + pending[0]!.resolve(model); + await Promise.resolve(); + slot.dispose(); + slot.dispose(); + expect(model.dispose).toHaveBeenCalledOnce(); +}); + +it("retries a failed source while ignoring rejection of a superseded request", async () => { + const { slot, pending, install, load } = fixture(); + slot.set(pro); + pending[0]!.reject(new Error("offline")); + await Promise.resolve(); + slot.set({ ...pro }); + expect(load).toHaveBeenCalledTimes(2); + slot.set(max); + pending[1]!.reject(new Error("cancelled")); + await Promise.resolve(); + slot.set({ ...max }); + expect(load).toHaveBeenCalledTimes(3); + const recovered = { dispose: vi.fn() }; + pending[2]!.resolve(recovered); + await Promise.resolve(); + expect(install).toHaveBeenCalledExactlyOnceWith(recovered); + slot.dispose(); +}); + +it("releases an invalid model and restores the fallback if scene preparation rejects it", async () => { + const { slot, pending, install } = fixture(); + install.mockImplementationOnce(() => { + throw new Error("invalid display"); + }); + slot.set(pro); + const model = { dispose: vi.fn() }; + pending[0]!.resolve(model); + await Promise.resolve(); + expect(model.dispose).toHaveBeenCalledOnce(); + expect(install).toHaveBeenLastCalledWith(null); + slot.set({ ...pro }); + const recovered = { dispose: vi.fn() }; + pending[1]!.resolve(recovered); + await Promise.resolve(); + expect(install).toHaveBeenLastCalledWith(recovered); + slot.dispose(); +}); diff --git a/packages/client-runtime/src/device/model.ts b/packages/client-runtime/src/device/model.ts new file mode 100644 index 000000000000..1f6524d4277c --- /dev/null +++ b/packages/client-runtime/src/device/model.ts @@ -0,0 +1,87 @@ +import type { DevicePlatform } from "@t3tools/contracts"; + +export type DeviceModelId = "iphone-18-pro" | "iphone-18-pro-max" | "ipad-pro-13-m5"; + +export interface DeviceModelSource { + readonly id: DeviceModelId; + readonly url: string; +} + +export interface DeviceAccessorySource { + readonly id: "ipad-pro-13-m5-magic-keyboard"; + readonly modelId: "ipad-pro-13-m5"; + readonly url: string; +} + +export type DeviceAssetSource = DeviceModelSource | DeviceAccessorySource; + +/** Match actual hardware, never stretch an available model to impersonate another device. */ +export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId | null { + if (platform !== "ios") return null; + if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max"; + if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro"; + if (/^iPad Pro 13-inch \(M5\)$/i.test(name)) return "ipad-pro-13-m5"; + return null; +} + +/** Each request owns its result. A cancelled or superseded parse must still release its resources. */ +export function createDeviceModelSlot< + T extends { dispose: () => void }, + Source extends DeviceAssetSource = DeviceModelSource, +>(options: { + load: (source: Source, signal: AbortSignal) => Promise; + install: (model: T | null) => void; + onError?: ((cause: unknown) => void) | undefined; +}) { + let request: { source: Source; controller: AbortController } | null = null; + let current: T | null = null; + let disposed = false; + const clear = () => { + if (current) options.install(null); + current?.dispose(); + current = null; + }; + return { + set(source: Source | null) { + if (disposed) return; + if (source?.id === request?.source.id && source?.url === request?.source.url) return; + request?.controller.abort(); + request = null; + clear(); + if (!source) return; + const next = { source, controller: new AbortController() }; + request = next; + void options.load(source, next.controller.signal).then( + (model) => { + if (disposed || request !== next) { + model.dispose(); + return; + } + try { + options.install(model); + current = model; + } catch (cause) { + options.install(null); + model.dispose(); + request = null; + options.onError?.(cause); + } + }, + (cause: unknown) => { + if (!disposed && request === next) { + request = null; + options.onError?.(cause); + } + // The procedural body stays installed on download, decoding or validation failure. + }, + ); + }, + dispose() { + if (disposed) return; + disposed = true; + request?.controller.abort(); + request = null; + clear(); + }, + }; +} diff --git a/packages/client-runtime/src/device/modelScene.test.ts b/packages/client-runtime/src/device/modelScene.test.ts new file mode 100644 index 000000000000..f58820d0afe1 --- /dev/null +++ b/packages/client-runtime/src/device/modelScene.test.ts @@ -0,0 +1,121 @@ +import { + BoxGeometry, + Group, + Mesh, + MeshBasicMaterial, + PerspectiveCamera, + PlaneGeometry, + Texture, + Vector3, +} from "three"; +import { expect, it, vi } from "vite-plus/test"; +import { createImportedPhoneScene, disposeDeviceModel } from "./modelScene.ts"; +import { phoneDisplayLayout } from "./phoneScene.ts"; + +function asset() { + const root = new Group(); + const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial()); + body.position.z = -0.02; + const screen = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial()); + screen.geometry.translate(0, 0, 0.043); + screen.name = "device-screen"; + root.add(body, screen); + return { root, screen, body }; +} + +it.each(["portrait", "landscape_left", "landscape_right", "portrait_upside_down"] as const)( + "retains the imported geometry while mapping screen input and framebuffer UVs in %s", + (orientation) => { + const { root, screen } = asset(); + const texture = new Texture(); + const original = screen.material; + // Optimization strips UVs from the placeholder material; the viewer generates framebuffer UVs. + screen.geometry.deleteAttribute("uv"); + const phone = createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)); + const layout = phoneDisplayLayout({ width: 1206, height: 2622, orientation }, 1206, 2622); + phone.setDisplay(texture, layout); + phone.root.rotation.set(0.2, -0.3, 0, "YXZ"); + phone.orientation.rotation.z = layout.rotation; + phone.root.updateMatrixWorld(true); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 6; + camera.updateMatrixWorld(true); + const point = phone.orientation.localToWorld(new Vector3(-0.25, -0.55, 0.043)).project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + const expected = + orientation === "landscape_left" + ? [0.25, 0.25] + : orientation === "landscape_right" + ? [0.75, 0.75] + : orientation === "portrait_upside_down" + ? [0.75, 0.25] + : [0.25, 0.75]; + expect(hit?.x).toBeCloseTo(expected[0]!); + expect(hit?.y).toBeCloseTo(expected[1]!); + const geometry = screen.geometry; + const rotated = phoneDisplayLayout( + { width: 2622, height: 1206, orientation: "landscape_left" }, + 2622, + 1206, + ); + phone.setDisplay(texture, rotated); + expect(screen.geometry).toBe(geometry); + const pos = geometry.getAttribute("position"); + const uv = geometry.getAttribute("uv"); + for (let i = 0; i < pos.count; i++) { + expect(uv.getX(i)).toBeCloseTo(pos.getY(i) / 2.2 + 0.5); + expect(uv.getY(i)).toBeCloseTo(0.5 - pos.getX(i)); + } + phone.dispose(); + expect(screen.material).toBe(original); + disposeDeviceModel(root); + texture.dispose(); + }, +); + +it("rejects touches from the rear and preserves borrowed textures when releasing imported resources", () => { + const { root, screen, body } = asset(); + const texture = new Texture(); + const borrowed = vi.spyOn(texture, "dispose"); + const ownedTexture = new Texture(); + const owned = vi.spyOn(ownedTexture, "dispose"); + body.material.map = ownedTexture; + const geometry = vi.spyOn(body.geometry, "dispose"); + const phone = createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 6; + phone.root.rotation.y = Math.PI; + expect(phone.screenPoint(0.5, 0.5, camera)).toBeNull(); + phone.dispose(); + expect(borrowed).not.toHaveBeenCalled(); + expect(geometry).not.toHaveBeenCalled(); + expect(screen.material.map).toBeNull(); + disposeDeviceModel(root); + expect(owned).toHaveBeenCalledOnce(); + expect(geometry).toHaveBeenCalledOnce(); + expect(borrowed).not.toHaveBeenCalled(); + texture.dispose(); +}); + +it("rejects missing and incorrectly normalized displays before changing their materials", () => { + const { root, screen } = asset(); + const original = screen.material; + const texture = new Texture(); + screen.geometry.scale(1, 2, 1); + expect(() => + createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)), + ).toThrow("not normalized"); + expect(screen.material).toBe(original); + screen.geometry.scale(1, 0.5, 1); + root.rotation.y = Math.PI; + expect(() => + createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)), + ).toThrow("not normalized"); + expect(screen.material).toBe(original); + screen.name = "wrong-screen"; + expect(() => + createImportedPhoneScene(root, texture, phoneDisplayLayout(null, 1206, 2622)), + ).toThrow("one device-screen"); + disposeDeviceModel(root); + texture.dispose(); +}); diff --git a/packages/client-runtime/src/device/modelScene.ts b/packages/client-runtime/src/device/modelScene.ts new file mode 100644 index 000000000000..6e1ce6eb82b3 --- /dev/null +++ b/packages/client-runtime/src/device/modelScene.ts @@ -0,0 +1,118 @@ +// @effect-diagnostics globalFetch:off - This browser-only renderer owns abortable asset requests without an Effect runtime, like the live stream. +import { Box3, Group, Mesh, MeshBasicMaterial, Texture, type Material, type Object3D } from "three"; +import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; +import type { DeviceAssetSource } from "./model.ts"; +import { createDisplayProjection, updateDisplayUv, type PhoneDisplayLayout } from "./phoneScene.ts"; + +/** GLB resources belong to one viewer; the live framebuffer texture belongs to its stream presentation. */ +export function disposeDeviceModel(root: Object3D) { + const geometries = new Set(); + const materials = new Set(); + const textures = new Set(); + root.traverse((object) => { + if (!(object instanceof Mesh)) return; + geometries.add(object.geometry); + for (const material of Array.isArray(object.material) ? object.material : [object.material]) { + materials.add(material); + for (const value of Object.values(material)) { + if (value instanceof Texture) textures.add(value); + } + } + }); + for (const geometry of geometries) geometry.dispose(); + for (const material of materials) material.dispose(); + const images = new Set(); + for (const texture of textures) { + texture.dispose(); + if (typeof ImageBitmap !== "undefined" && texture.image instanceof ImageBitmap) + images.add(texture.image); + } + for (const image of images) image.close(); +} + +/** Assets are converted offline: portrait, front +Z, centered display, height 2.2 and planar display UVs. */ +export function createImportedPhoneScene( + asset: Group, + texture: Texture, + initial: PhoneDisplayLayout, +) { + const screens: Mesh[] = []; + asset.traverse((object) => { + if (object instanceof Mesh && object.name === "device-screen") screens.push(object); + }); + const display = screens[0]; + if (!display || screens.length !== 1) + throw new Error("Device model must have one device-screen mesh"); + asset.updateMatrixWorld(true); + const screenBounds = new Box3().setFromObject(display); + const screenWidth = screenBounds.max.x - screenBounds.min.x; + const screenHeight = screenBounds.max.y - screenBounds.min.y; + if ( + display.matrixWorld.elements.some( + (value, index) => + !Number.isFinite(value) || Math.abs(value - (index % 5 === 0 ? 1 : 0)) > 0.001, + ) || + !Number.isFinite(screenWidth) || + !Number.isFinite(screenHeight) || + !Number.isFinite(screenBounds.min.z) || + !Number.isFinite(screenBounds.max.z) || + screenWidth <= 0 || + Math.abs(screenHeight - 2.2) > 0.001 || + Math.abs(screenBounds.min.x + screenBounds.max.x) > 0.001 || + Math.abs(screenBounds.min.y + screenBounds.max.y) > 0.001 || + screenBounds.max.z - screenBounds.min.z > 0.001 + ) + throw new Error("Device model display is not normalized"); + updateDisplayUv(display.geometry, screenWidth, screenHeight, initial); + const originalMaterial = display.material; + const screenMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false }); + display.material = screenMaterial; + const root = new Group(); + const orientation = new Group(); + orientation.add(asset); + root.add(orientation); + const bounds = new Box3().setFromObject(asset); + let layout = initial; + const projection = createDisplayProjection( + display, + orientation, + screenWidth, + screenHeight, + () => layout, + ); + return { + root, + orientation, + width: bounds.max.x - bounds.min.x, + height: bounds.max.y - bounds.min.y, + setDisplay(nextTexture: Texture, nextLayout: PhoneDisplayLayout) { + layout = nextLayout; + screenMaterial.map = nextTexture; + updateDisplayUv(display.geometry, screenWidth, screenHeight, layout); + }, + screenPoint: projection, + dispose() { + // The model slot owns imported resources. This scene owns only its replacement screen material. + display.material = originalMaterial; + screenMaterial.map = null; + screenMaterial.dispose(); + orientation.remove(asset); + }, + }; +} + +export async function loadDeviceModel(source: DeviceAssetSource, signal: AbortSignal) { + const response = await fetch(source.url, { signal }); + if (!response.ok) throw new Error(`Device model request failed: ${response.status}`); + const data = await response.arrayBuffer(); + signal.throwIfAborted(); + const gltf = await new GLTFLoader().parseAsync( + data, + new URL(".", new URL(source.url, globalThis.location.href)).href, + ); + if (signal.aborted) { + disposeDeviceModel(gltf.scene); + signal.throwIfAborted(); + } + return { asset: gltf.scene, dispose: () => disposeDeviceModel(gltf.scene) }; +} diff --git a/packages/client-runtime/src/device/phoneInteraction.test.ts b/packages/client-runtime/src/device/phoneInteraction.test.ts new file mode 100644 index 000000000000..ddd304f66d25 --- /dev/null +++ b/packages/client-runtime/src/device/phoneInteraction.test.ts @@ -0,0 +1,105 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createPhoneInteraction, phoneWheelNavigation } from "./phoneInteraction.ts"; + +it("keeps a captured screen gesture separate from orbit and ends it once on cancellation", () => { + const touch = vi.fn(); + const orbit = vi.fn(); + const screenPoint = vi + .fn() + .mockReturnValueOnce({ x: 0.2, y: 0.3 }) + .mockReturnValueOnce({ x: 1, y: 0.6 }); + const input = createPhoneInteraction({ screenPoint, touch, orbit, zoomBy: vi.fn() }); + expect(input.begin(1, { x: 0.4, y: 0.3 })).toBe(true); + expect(input.begin(2, { x: 0.2, y: 0.2 })).toBe(false); + input.move(2, { x: 0.8, y: 0.8 }); + input.move(1, { x: 1.3, y: 0.6 }); + input.end(2); + input.end(); + input.end(1); + expect(touch.mock.calls).toEqual([ + ["begin", { x: 0.2, y: 0.3 }], + ["move", { x: 1, y: 0.6 }], + ["end", { x: 1, y: 0.6 }], + ]); + expect(orbit).not.toHaveBeenCalled(); + expect(screenPoint.mock.calls[1]?.[1]).toBe(true); +}); + +it("orbits without sending touches when dragged outside the screen or with the orbit modifier", () => { + const touch = vi.fn(); + const orbit = vi.fn(); + const input = createPhoneInteraction({ screenPoint: () => null, touch, orbit, zoomBy: vi.fn() }); + input.begin(1, { x: 0.1, y: 0.2 }); + input.move(1, { x: 0.4, y: 0.5 }); + input.end(); + input.begin(2, { x: 0.5, y: 0.5 }, true); + input.move(2, { x: 0.6, y: 0.5 }); + input.end(); + expect(orbit).toHaveBeenCalledTimes(2); + expect(orbit.mock.calls[0]?.[0]).toBeCloseTo(0.3); + expect(touch).not.toHaveBeenCalled(); +}); + +it("blocks navigation during either captured gesture, then resumes without sending device touches", () => { + const touch = vi.fn(); + const orbit = vi.fn(); + const zoomBy = vi.fn(); + const input = createPhoneInteraction({ + screenPoint: () => ({ x: 0.5, y: 0.5 }), + touch, + orbit, + zoomBy, + }); + input.begin(1, { x: 0.5, y: 0.5 }); + expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(false); + expect(input.navigate({ type: "orbit", x: 0.1, y: 0.2 })).toBe(false); + input.end(); + input.begin(2, { x: 0.5, y: 0.5 }, true); + expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(false); + input.end(); + expect(input.navigate({ type: "zoom", delta: 0.2 })).toBe(true); + expect(input.navigate({ type: "orbit", x: 0.1, y: 0.2 })).toBe(true); + expect(zoomBy.mock.calls).toEqual([[0.2]]); + expect(orbit.mock.calls).toEqual([[0.1, 0.2]]); + expect(touch.mock.calls.map(([phase]) => phase)).toEqual(["begin", "end"]); +}); + +it("ends wheel orbit without ending a captured screen touch", () => { + const onInteractionActive = vi.fn(); + const input = createPhoneInteraction({ + screenPoint: () => ({ x: 0.5, y: 0.5 }), + touch: vi.fn(), + orbit: vi.fn(), + zoomBy: vi.fn(), + onInteractionActive, + }); + input.endWheel(); + expect(onInteractionActive).toHaveBeenCalledWith(false, "orbit"); + input.begin(1, { x: 0.5, y: 0.5 }); + onInteractionActive.mockClear(); + input.endWheel(); + expect(onInteractionActive).not.toHaveBeenCalled(); + input.end(1); + expect(onInteractionActive).toHaveBeenCalledWith(false, "touch"); +}); + +it("normalizes wheel units, distinguishes pinch, and bounds jumps from coarse scroll wheels", () => { + const wheel = { width: 400, height: 800, deltaX: 16, deltaY: 32, deltaMode: 0, ctrlKey: false }; + expect(phoneWheelNavigation(wheel)).toEqual({ type: "orbit", x: -0.04, y: -0.04 }); + expect(phoneWheelNavigation({ ...wheel, deltaX: 1, deltaY: 2, deltaMode: 1 })).toEqual( + phoneWheelNavigation(wheel), + ); + expect(phoneWheelNavigation({ ...wheel, deltaX: 1, deltaY: 1, deltaMode: 2 })).toEqual({ + type: "orbit", + x: -0.25, + y: -0.25, + }); + expect(phoneWheelNavigation({ ...wheel, ctrlKey: true })).toEqual({ type: "zoom", delta: -0.32 }); + expect(phoneWheelNavigation({ ...wheel, deltaY: -10000, ctrlKey: true })).toEqual({ + type: "zoom", + delta: 1, + }); + expect(phoneWheelNavigation({ ...wheel, width: 0 })).toBeNull(); + expect(phoneWheelNavigation({ ...wheel, deltaY: NaN })).toBeNull(); + expect(phoneWheelNavigation({ ...wheel, deltaMode: 9 })).toBeNull(); +}); diff --git a/packages/client-runtime/src/device/phoneInteraction.ts b/packages/client-runtime/src/device/phoneInteraction.ts new file mode 100644 index 000000000000..dd701f1a8b57 --- /dev/null +++ b/packages/client-runtime/src/device/phoneInteraction.ts @@ -0,0 +1,83 @@ +interface Point { + readonly x: number; + readonly y: number; +} + +export type PhoneNavigation = + | { readonly type: "orbit"; readonly x: number; readonly y: number } + | { readonly type: "zoom"; readonly delta: number }; + +/** Browser wheel units vary by device. Navigation uses viewport fractions and logarithmic zoom. */ +export function phoneWheelNavigation(input: { + readonly deltaX: number; + readonly deltaY: number; + readonly deltaMode: number; + readonly ctrlKey: boolean; + readonly width: number; + readonly height: number; +}): PhoneNavigation | null { + const { deltaX, deltaY, deltaMode, ctrlKey, width, height } = input; + if (![deltaX, deltaY, width, height].every(Number.isFinite) || width <= 0 || height <= 0) + return null; + if (deltaMode !== 0 && deltaMode !== 1 && deltaMode !== 2) return null; + const unit = deltaMode === 1 ? 16 : deltaMode === 2 ? height : 1; + const clamp = (value: number, limit: number) => Math.min(limit, Math.max(-limit, value)); + if (ctrlKey) return { type: "zoom", delta: clamp(-deltaY * unit * 0.01, 1) }; + return { + type: "orbit", + x: clamp((-deltaX * unit) / width, 0.25), + y: clamp((-deltaY * unit) / height, 0.25), + }; +} + +/** A single pointer owns either a device gesture or an orbit until released or cancelled. */ +export function createPhoneInteraction(options: { + readonly screenPoint: (point: Point, captured: boolean) => Point | null; + readonly touch: (phase: "begin" | "move" | "end", point: Point) => void; + readonly orbit: (deltaX: number, deltaY: number) => void; + readonly zoomBy: (logDelta: number) => void; + readonly onInteractionActive?: (active: boolean, mode: "touch" | "orbit") => void; +}) { + let active: { id: number; mode: "touch" | "orbit"; last: Point; screen: Point | null } | null = + null; + return { + navigate(gesture: PhoneNavigation) { + // Moving the camera during a captured device touch would change its projected coordinates. + if (active) return false; + if (gesture.type === "zoom") options.zoomBy(gesture.delta); + else options.orbit(gesture.x, gesture.y); + return true; + }, + endWheel() { + if (!active) options.onInteractionActive?.(false, "orbit"); + }, + begin(id: number, point: Point, forceOrbit = false) { + if (active) return false; + const screen = forceOrbit ? null : options.screenPoint(point, false); + active = { id, mode: screen ? "touch" : "orbit", last: point, screen }; + options.onInteractionActive?.(true, active.mode); + if (screen) options.touch("begin", screen); + return true; + }, + move(id: number, point: Point) { + if (active?.id !== id) return; + if (active.mode === "touch") { + const screen = options.screenPoint(point, true); + if (screen) { + active.screen = screen; + options.touch("move", screen); + } + } else { + options.orbit(point.x - active.last.x, point.y - active.last.y); + } + active.last = point; + }, + end(id?: number) { + if (!active || (id !== undefined && active.id !== id)) return; + const previous = active; + active = null; + if (previous.mode === "touch" && previous.screen) options.touch("end", previous.screen); + options.onInteractionActive?.(false, previous.mode); + }, + }; +} diff --git a/packages/client-runtime/src/device/phoneScene.test.ts b/packages/client-runtime/src/device/phoneScene.test.ts new file mode 100644 index 000000000000..188b8891803f --- /dev/null +++ b/packages/client-runtime/src/device/phoneScene.test.ts @@ -0,0 +1,150 @@ +import { Box3, PerspectiveCamera, Texture, Vector3 } from "three"; +import { describe, expect, it } from "vite-plus/test"; +import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts"; +import { IOS_TABLET_SHAPE, IOS_PHONE_SHAPE, ANDROID_PHONE_SHAPE } from "./shapeProfile.ts"; + +const orientations = [ + "portrait", + "landscape_left", + "landscape_right", + "portrait_upside_down", +] as const; + +describe("3D phone input", () => { + it.each([IOS_TABLET_SHAPE, IOS_PHONE_SHAPE, ANDROID_PHONE_SHAPE])( + "preserves input projection for the $id silhouette in every orientation", + (profile) => { + for (const orientation of orientations) { + const texture = new Texture(); + const aspect = profile.id === "ios-tablet" ? 0.75 : 0.45; + const layout = phoneDisplayLayout( + { width: 1200 * aspect, height: 1200, orientation }, + 1200 * aspect, + 1200, + ); + const phone = createPhoneScene(texture, layout, profile); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 6; + camera.updateMatrixWorld(true); + phone.root.rotation.set(0.2, -0.3, 0, "YXZ"); + phone.orientation.rotation.z = layout.rotation; + phone.root.updateMatrixWorld(true); + const point = new Vector3((-2.2 * aspect) / 4, -2.2 / 4, 0.043); + phone.orientation.localToWorld(point); + point.project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + const expected = + orientation === "landscape_left" + ? { x: 0.25, y: 0.25 } + : orientation === "landscape_right" + ? { x: 0.75, y: 0.75 } + : orientation === "portrait_upside_down" + ? { x: 0.75, y: 0.25 } + : { x: 0.25, y: 0.75 }; + expect(hit?.x).toBeCloseTo(expected.x); + expect(hit?.y).toBeCloseTo(expected.y); + phone.dispose(); + texture.dispose(); + } + }, + ); + it.each([0.6, 2.2])("keeps device coordinates accurate at %s camera zoom", (zoom) => { + const texture = new Texture(); + const layout = phoneDisplayLayout(null, 900, 1950); + const phone = createPhoneScene(texture, layout); + const camera = new PerspectiveCamera(32, 0.65, 0.1, 30); + camera.position.z = 6 / zoom; + camera.updateMatrixWorld(true); + phone.root.rotation.set(0.2, -0.3, 0, "YXZ"); + phone.root.updateMatrixWorld(true); + const point = new Vector3((-2.2 * layout.aspect) / 4, -2.2 / 4, 0.043); + phone.orientation.localToWorld(point); + point.project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + expect(hit?.x).toBeCloseTo(0.25); + expect(hit?.y).toBeCloseTo(0.75); + phone.dispose(); + texture.dispose(); + }); + + it.each(orientations)("maps perspective hits to displayed coordinates in %s", (orientation) => { + const texture = new Texture(); + const layout = phoneDisplayLayout({ width: 1170, height: 2532, orientation }, 1170, 2532); + const phone = createPhoneScene(texture, layout); + const camera = new PerspectiveCamera(32, 0.65, 0.1, 30); + camera.position.z = 6; + camera.updateMatrixWorld(true); + phone.root.rotation.set(0.3, -0.45, 0, "YXZ"); + phone.orientation.rotation.z = layout.rotation; + phone.root.updateMatrixWorld(true); + // A point one quarter across and three quarters down the portrait display. + const point = new Vector3((-2.2 * layout.aspect) / 4, -2.2 / 4, 0.043); + phone.orientation.localToWorld(point); + point.project(camera); + const hit = phone.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera); + const expected = + orientation === "landscape_left" + ? { x: 0.25, y: 0.25 } + : orientation === "landscape_right" + ? { x: 0.75, y: 0.75 } + : orientation === "portrait_upside_down" + ? { x: 0.75, y: 0.25 } + : { x: 0.25, y: 0.75 }; + expect(hit?.x).toBeCloseTo(expected.x); + expect(hit?.y).toBeCloseTo(expected.y); + phone.dispose(); + texture.dispose(); + }); + + it("rejects new gestures outside the display, but clamps captured drags at its edge", () => { + const texture = new Texture(); + const phone = createPhoneScene(texture, phoneDisplayLayout(null, 900, 1950)); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 5; + expect(phone.screenPoint(1, 0.5, camera)).toBeNull(); + expect(phone.screenPoint(1, 0.5, camera, true)).toEqual({ x: 1, y: 0.5 }); + phone.root.rotation.y = Math.PI; + expect(phone.screenPoint(0.5, 0.5, camera)).toBeNull(); + phone.dispose(); + texture.dispose(); + }); + + it.each(["landscape_left", "landscape_right"] as const)( + "unrotates an already-landscape framebuffer in %s", + (orientation) => { + const texture = new Texture(); + const layout = phoneDisplayLayout({ width: 1950, height: 900, orientation }, 1950, 900); + const phone = createPhoneScene(texture, layout); + phone.orientation.rotation.z = layout.rotation; + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 5; + expect(phone.screenPoint(0.5, 0.5, camera)).toEqual({ x: 0.5, y: 0.5 }); + phone.dispose(); + texture.dispose(); + }, + ); +}); + +it.each([IOS_PHONE_SHAPE, IOS_TABLET_SHAPE, ANDROID_PHONE_SHAPE])( + "attaches the $id rear camera plate to the back and seats its lenses on the housing", + (profile) => { + const texture = new Texture(); + const phone = createPhoneScene(texture, phoneDisplayLayout(null, 900, 1950), profile); + phone.root.updateMatrixWorld(true); + const back = new Box3().setFromObject(phone.root.getObjectByName("device-back")!); + const plate = new Box3().setFromObject(phone.root.getObjectByName("camera-plate")!); + expect(plate.min.z).toBeLessThan(back.min.z); + expect(plate.max.z).toBeGreaterThanOrEqual(back.min.z); + const assembly = phone.root.getObjectByName("rear-camera")!; + const rings = assembly.children.filter((part) => part.name === "camera-ring"); + const lenses = assembly.children.filter((part) => part.name === "camera-lens"); + for (const [index, ring] of rings.entries()) { + const ringBounds = new Box3().setFromObject(ring); + expect(ringBounds.intersectsBox(plate)).toBe(true); + const lensBounds = new Box3().setFromObject(lenses[index]!); + expect(Math.abs(lensBounds.max.z - ringBounds.min.z)).toBeLessThan(0.001); + } + phone.dispose(); + texture.dispose(); + }, +); diff --git a/packages/client-runtime/src/device/phoneScene.ts b/packages/client-runtime/src/device/phoneScene.ts new file mode 100644 index 000000000000..824288031452 --- /dev/null +++ b/packages/client-runtime/src/device/phoneScene.ts @@ -0,0 +1,302 @@ +import { + BoxGeometry, + CircleGeometry, + CylinderGeometry, + ExtrudeGeometry, + Float32BufferAttribute, + Group, + Mesh, + MeshBasicMaterial, + MeshPhysicalMaterial, + MeshStandardMaterial, + Plane, + Raycaster, + Shape, + ShapeGeometry, + Vector2, + Vector3, + type Camera, + type BufferGeometry, + type Texture, +} from "three"; +import type { DeviceScreenSize } from "./stream.ts"; +import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts"; + +const SCREEN_HEIGHT = 2.2; + +function roundedPath(width: number, height: number, radius: number, path = new Shape()) { + const x = -width / 2; + const y = -height / 2; + path.moveTo(x + radius, y); + path.lineTo(x + width - radius, y); + path.quadraticCurveTo(x + width, y, x + width, y + radius); + path.lineTo(x + width, y + height - radius); + path.quadraticCurveTo(x + width, y + height, x + width - radius, y + height); + path.lineTo(x + radius, y + height); + path.quadraticCurveTo(x, y + height, x, y + height - radius); + path.lineTo(x, y + radius); + path.quadraticCurveTo(x, y, x + radius, y); + return path; +} + +/** Display orientation is independent of orbit. Texture coordinates stay in raw framebuffer space. */ +export function phoneDisplayLayout( + screen: DeviceScreenSize | null, + rawWidth: number, + rawHeight: number, +) { + const width = rawWidth || screen?.width || 900; + const height = rawHeight || screen?.height || 1950; + const landscape = + screen?.orientation === "landscape_left" || screen?.orientation === "landscape_right"; + const rotation = + screen?.orientation === "landscape_left" + ? -Math.PI / 2 + : screen?.orientation === "landscape_right" + ? Math.PI / 2 + : screen?.orientation === "portrait_upside_down" + ? Math.PI + : 0; + return { + aspect: Math.min(width, height) / Math.max(width, height), + rotation, + landscape, + rawLandscape: width > height, + }; +} + +/** An original procedural device body. It makes no claim to reproduce a particular hardware model. */ +export function createPhoneScene( + texture: Texture, + layout: ReturnType, + profile: DeviceShapeProfile = IOS_PHONE_SHAPE, +) { + const { aspect } = layout; + const root = new Group(); + const orientation = new Group(); + root.add(orientation); + const screenWidth = SCREEN_HEIGHT * aspect; + const width = screenWidth + profile.bezel * 2; + const height = SCREEN_HEIGHT + profile.bezel * 2; + const backZ = 0.01 - profile.depth; + const metal = new MeshStandardMaterial({ color: 0xb5bcc7, metalness: 0.88, roughness: 0.27 }); + const glass = new MeshPhysicalMaterial({ + color: 0x141820, + metalness: 0.15, + roughness: 0.2, + clearcoat: 1, + }); + const backMaterial = new MeshStandardMaterial({ + color: profile.backColor, + metalness: 0.45, + roughness: 0.32, + }); + const lensMaterial = new MeshPhysicalMaterial({ + color: 0x071326, + metalness: 0.6, + roughness: 0.12, + clearcoat: 1, + }); + const materials = [metal, glass, backMaterial, lensMaterial]; + + const body = new Mesh( + new ExtrudeGeometry(roundedPath(width, height, profile.bodyRadius), { + depth: profile.depth, + bevelEnabled: true, + bevelSize: 0.012, + bevelThickness: 0.012, + bevelSegments: 3, + steps: 1, + curveSegments: 12, + }), + metal, + ); + body.position.z = 0.025 - profile.depth; + orientation.add(body); + const face = new Mesh( + new ShapeGeometry(roundedPath(width - 0.014, height - 0.014, profile.bodyRadius - 0.01), 16), + glass, + ); + face.position.z = 0.04; + orientation.add(face); + const back = new Mesh( + new ShapeGeometry(roundedPath(width - 0.012, height - 0.012, profile.bodyRadius - 0.01), 16), + backMaterial, + ); + back.name = "device-back"; + back.rotation.y = Math.PI; + back.position.z = backZ; + orientation.add(back); + + const screenGeometry = new ShapeGeometry( + roundedPath(screenWidth, SCREEN_HEIGHT, profile.screenRadius), + 20, + ); + updateDisplayUv(screenGeometry, screenWidth, SCREEN_HEIGHT, layout); + const screenMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false }); + const display = new Mesh(screenGeometry, screenMaterial); + display.position.z = 0.043; + orientation.add(display); + + for (const { edge, offset, length } of profile.buttons) { + const button = new Mesh( + new BoxGeometry( + edge === "top" ? length : 0.026, + edge === "top" ? 0.026 : length, + profile.depth * 0.65, + ), + metal, + ); + button.position.set( + edge === "top" ? offset : (edge === "left" ? -1 : 1) * (width / 2 + 0.015), + edge === "top" ? height / 2 + 0.015 : offset, + (0.025 + backZ) / 2, + ); + orientation.add(button); + } + // Rear components share a surface-relative coordinate system, with outward positive Z. + const rearCamera = new Group(); + rearCamera.name = "rear-camera"; + rearCamera.position.set( + width / 2 - profile.camera.insetX, + height / 2 - profile.camera.insetY, + back.position.z + 0.001, + ); + rearCamera.rotation.y = Math.PI; + orientation.add(rearCamera); + const plateFront = 0.025 + 0.007; + const ringDepth = 0.025; + const cameraPlate = new Mesh( + new ExtrudeGeometry( + roundedPath( + profile.camera.width, + profile.camera.height, + Math.min(profile.camera.width, profile.camera.height) / 4, + ), + { + depth: 0.025, + bevelEnabled: true, + bevelSize: 0.009, + bevelThickness: 0.007, + bevelSegments: 2, + }, + ), + backMaterial, + ); + cameraPlate.name = "camera-plate"; + rearCamera.add(cameraPlate); + for (const [x, y] of profile.camera.lenses) { + const radius = profile.camera.lensRadius; + const ring = new Mesh(new CylinderGeometry(radius + 0.014, radius + 0.014, 0.025, 32), metal); + ring.rotation.x = Math.PI / 2; + ring.name = "camera-ring"; + ring.position.set(x, y, plateFront + ringDepth / 2 - 0.003); + rearCamera.add(ring); + const lens = new Mesh(new CircleGeometry(radius, 32), lensMaterial); + lens.name = "camera-lens"; + lens.position.set(x, y, ring.position.z + ringDepth / 2 + 0.0005); + rearCamera.add(lens); + } + const flashMaterial = new MeshBasicMaterial({ color: 0xf2ead6 }); + if (profile.camera.flash) { + const flash = new Mesh(new CircleGeometry(0.022, 20), flashMaterial); + flash.position.set(profile.camera.flash[0], profile.camera.flash[1], plateFront + 0.0005); + rearCamera.add(flash); + } + + let activeLayout = layout; + const screenPoint = createDisplayProjection( + display, + orientation, + screenWidth, + SCREEN_HEIGHT, + () => activeLayout, + ); + return { + root, + orientation, + width, + height, + screenPoint, + setDisplay(nextTexture: Texture, nextLayout: PhoneDisplayLayout) { + activeLayout = nextLayout; + screenMaterial.map = nextTexture; + updateDisplayUv(screenGeometry, screenWidth, SCREEN_HEIGHT, activeLayout); + }, + dispose() { + root.traverse((object) => { + if (object instanceof Mesh) object.geometry.dispose(); + }); + for (const material of materials) material.dispose(); + screenMaterial.dispose(); + flashMaterial.dispose(); + }, + }; +} + +export type PhoneDisplayLayout = ReturnType; + +/** Canonical portrait geometry maps to raw framebuffer coordinates in every OS orientation. */ +export function updateDisplayUv( + geometry: BufferGeometry, + width: number, + height: number, + layout: PhoneDisplayLayout, +) { + const position = geometry.getAttribute("position"); + if (!geometry.hasAttribute("uv")) { + geometry.setAttribute( + "uv", + new Float32BufferAttribute(new Float32Array(position.count * 2), 2), + ); + } + const uv = geometry.getAttribute("uv"); + for (let i = 0; i < position.count; i++) { + const u = (position.getX(i) + width / 2) / width; + const v = (position.getY(i) + height / 2) / height; + uv.setXY( + i, + layout.rawLandscape ? (layout.rotation > 0 ? 1 - v : v) : u, + layout.rawLandscape ? (layout.rotation > 0 ? u : 1 - u) : v, + ); + } + uv.needsUpdate = true; +} + +/** New touches hit only the front display; captured drags project onto its plane and clamp. */ +export function createDisplayProjection( + display: Mesh, + orientation: Group, + width: number, + height: number, + getLayout: () => PhoneDisplayLayout, +) { + const raycaster = new Raycaster(); + const pointer = new Vector2(); + const local = new Vector3(); + display.geometry.computeBoundingBox(); + const z = display.position.z + (display.geometry.boundingBox?.max.z ?? 0); + const plane = new Plane(new Vector3(0, 0, 1), -z); + return (x: number, y: number, camera: Camera, captured = false) => { + orientation.updateWorldMatrix(true, true); + camera.updateMatrixWorld(true); + pointer.set(x * 2 - 1, 1 - y * 2); + raycaster.setFromCamera(pointer, camera); + if (!captured) { + const hit = raycaster.intersectObject(display, false)[0]; + if (!hit) return null; + local.copy(hit.point); + orientation.worldToLocal(local); + } else { + const ray = raycaster.ray.clone().applyMatrix4(orientation.matrixWorld.clone().invert()); + if (!ray.intersectPlane(plane, local)) return null; + } + const u = Math.min(1, Math.max(0, (local.x + width / 2) / width)); + const v = Math.min(1, Math.max(0, (local.y + height / 2) / height)); + const { rotation } = getLayout(); + if (rotation === -Math.PI / 2) return { x: v, y: u }; + if (rotation === Math.PI / 2) return { x: 1 - v, y: 1 - u }; + if (rotation === Math.PI) return { x: 1 - u, y: v }; + return { x: u, y: 1 - v }; + }; +} diff --git a/packages/client-runtime/src/device/phoneViewer.test.ts b/packages/client-runtime/src/device/phoneViewer.test.ts new file mode 100644 index 000000000000..ddb08bddf976 --- /dev/null +++ b/packages/client-runtime/src/device/phoneViewer.test.ts @@ -0,0 +1,350 @@ +import type { PerspectiveCamera, Scene, Object3D } from "three"; +import { afterEach, expect, it, vi } from "vite-plus/test"; + +const gpu = vi.hoisted(() => ({ + instances: [] as { + blank: boolean; + allocations: number; + disposed: boolean; + size: { width: number; height: number; pixelRatio: number }; + frames: { + scene: Scene; + phone: Object3D | undefined; + rotation: Quaternion | undefined; + yaw: number | undefined; + cameraZ: number; + }[]; + }[], +})); + +vi.mock("three", async () => { + const actual = await vi.importActual("three"); + return { + ...actual, + WebGLRenderer: class { + outputColorSpace = ""; + state: (typeof gpu.instances)[number] = { + blank: true, + allocations: 0, + disposed: false, + size: { width: 0, height: 0, pixelRatio: 1 }, + frames: [], + }; + constructor() { + gpu.instances.push(this.state); + } + setDrawingBufferSize(width: number, height: number, pixelRatio: number) { + this.state.size = { width, height, pixelRatio }; + this.state.allocations++; + this.state.blank = true; + } + setSize(width: number, height: number) { + this.setDrawingBufferSize(width, height, this.state.size.pixelRatio); + } + setPixelRatio(pixelRatio: number) { + this.setDrawingBufferSize(this.state.size.width, this.state.size.height, pixelRatio); + } + render(scene: Scene, camera: PerspectiveCamera) { + const phone = scene.children.find((child) => child.type === "Group"); + this.state.frames.push({ + scene, + phone, + rotation: phone?.quaternion.clone(), + yaw: phone?.rotation.y, + cameraZ: camera.position.z, + }); + this.state.blank = false; + } + dispose() { + this.state.disposed = true; + } + forceContextLoss() {} + }, + }; +}); + +const models = vi.hoisted(() => ({ + pending: [] as { + signal: AbortSignal; + resolve: (model: { asset: import("three").Group; dispose: () => void }) => void; + }[], +})); +vi.mock("./modelScene.ts", async () => { + const actual = await vi.importActual("./modelScene.ts"); + return { + ...actual, + loadDeviceModel: (_source: unknown, signal: AbortSignal) => + new Promise((resolve) => models.pending.push({ signal, resolve })), + }; +}); + +import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three"; +import { disposeDeviceModel } from "./modelScene.ts"; +import { createPhoneViewer } from "./phoneViewer.ts"; +import { IOS_TABLET_SHAPE } from "./shapeProfile.ts"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + gpu.instances.length = 0; + models.pending.length = 0; +}); + +function fixture() { + const pending = new Map(); + let id = 0; + let now = 0; + vi.spyOn(performance, "now").mockImplementation(() => now); + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + pending.set(++id, callback); + return id; + }); + vi.stubGlobal("cancelAnimationFrame", (id: number) => pending.delete(id)); + const canvas = Object.assign(new EventTarget(), { width: 0, height: 0 }) as HTMLCanvasElement; + const source = { width: 1206, height: 2622 } as HTMLCanvasElement; + const onUnavailable = vi.fn(); + const onFramingAspect = vi.fn(); + const viewer = createPhoneViewer({ canvas, source, onUnavailable, onFramingAspect }); + const draw = (time = now) => { + now = time; + const callbacks = [...pending.values()]; + pending.clear(); + callbacks.forEach((callback) => callback(0)); + }; + viewer.resize(400, 700, 2); + draw(); + return { + viewer, + draw, + pending, + source, + onUnavailable, + onFramingAspect, + state: gpu.instances[0]!, + }; +} + +it("retains the drawn phone until resize and redraw commit together, without replacing scene or pose", () => { + const { viewer, draw, pending, source, state, onUnavailable } = fixture(); + viewer.orbit(0.08, 0.04); + draw(); + const previous = state.frames.at(-1)!; + const allocations = state.allocations; + viewer.resize(500, 700, 2); + viewer.resize(450, 700, 2); + viewer.frameUpdated(); + // ResizeObserver runs after rAF. A cleared buffer here would reach the browser's next paint. + expect(state.blank).toBe(false); + expect(state.allocations).toBe(allocations); + expect(pending.size).toBe(1); + draw(); + expect(state.size).toEqual({ width: 450, height: 700, pixelRatio: 2 }); + expect(state.blank).toBe(false); + expect(state.allocations).toBe(allocations + 1); + expect(state.frames.at(-1)?.scene).toBe(previous.scene); + expect(state.frames.at(-1)?.phone).toBe(previous.phone); + expect(state.frames.at(-1)?.yaw).toBe(previous.yaw); + viewer.resize(400, 700, 2); + draw(); + expect(state.frames.at(-1)?.cameraZ).toBeCloseTo(previous.cameraZ); + expect(source).toMatchObject({ width: 1206, height: 2622 }); + expect(gpu.instances).toHaveLength(1); + expect(onUnavailable).not.toHaveBeenCalled(); + viewer.dispose(); +}); + +it("ignores redundant and invalid sizes and combines a DPR change into one allocation and draw", () => { + const { viewer, draw, pending, state } = fixture(); + const allocations = state.allocations; + viewer.resize(400, 700, 2); + viewer.resize(400, 700, 5); + viewer.resize(0, 700, 2); + viewer.resize(NaN, 700, 2); + expect(pending.size).toBe(0); + viewer.resize(400, 700, 1); + expect(state.blank).toBe(false); + draw(); + expect(state.size.pixelRatio).toBe(1); + expect(state.allocations).toBe(allocations + 1); + viewer.resize(450, 700, 1); + viewer.dispose(); + draw(); + expect(state.allocations).toBe(allocations + 1); + expect(state.disposed).toBe(true); +}); + +it("changes device shape without replacing the renderer, decoded source or pose", () => { + const { viewer, draw, source, state } = fixture(); + viewer.orbit(0.1, 0.05); + draw(); + const previous = state.frames.at(-1)!; + const allocations = state.allocations; + viewer.setScreen(null, IOS_TABLET_SHAPE); + draw(); + expect(state.frames.at(-1)?.phone).not.toBe(previous.phone); + expect(state.frames.at(-1)?.yaw).toBe(previous.yaw); + expect(state.frames.at(-1)?.scene).toBe(previous.scene); + expect(state.allocations).toBe(allocations); + expect(source.width).toBe(1206); + expect(gpu.instances).toHaveLength(1); + const tablet = state.frames.at(-1)?.phone; + viewer.frameUpdated(); + draw(); + expect(state.frames.at(-1)?.phone).toBe(tablet); + viewer.dispose(); +}); + +it("retains the loaded model and pose through rotation and framebuffer resolution changes, then releases it once", async () => { + const { viewer, draw, source, state } = fixture(); + viewer.orbit(0.08, 0.04); + draw(); + const yaw = state.frames.at(-1)!.yaw; + viewer.setModel({ id: "iphone-18-pro", url: "/pro.glb" }); + const asset = new Group(); + const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial()); + body.position.z = -0.02; + const display = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial()); + display.geometry.translate(0, 0, 0.043); + display.name = "device-screen"; + asset.add(body, display); + const release = vi.spyOn(body.geometry, "dispose"); + const dispose = vi.fn(() => disposeDeviceModel(asset)); + models.pending[0]!.resolve({ asset, dispose }); + await Promise.resolve(); + draw(); + const loaded = state.frames.at(-1)!.phone; + expect(loaded?.getObjectByName("device-screen")).toBe(display); + expect(state.frames.at(-1)!.yaw).toBe(yaw); + source.width = 2622; + source.height = 1206; + viewer.setScreen({ width: 2622, height: 1206, orientation: "landscape_left" }); + viewer.frameUpdated(); + viewer.resize(700, 400, 1); + draw(); + expect(state.frames.at(-1)!.phone).toBe(loaded); + expect(state.frames.at(-1)!.yaw).toBe(yaw); + expect(display.material.map?.image).toBe(source); + expect(release).not.toHaveBeenCalled(); + expect(gpu.instances).toHaveLength(1); + viewer.dispose(); + viewer.dispose(); + expect(dispose).toHaveBeenCalledOnce(); + expect(release).toHaveBeenCalledOnce(); +}); + +it("attaches and detaches an accessory without replacing the device, stream texture, renderer or pose", async () => { + const { viewer, draw, source, state, onFramingAspect } = fixture(); + viewer.setModel({ id: "ipad-pro-13-m5", url: "/ipad.glb" }); + const asset = new Group(); + const body = new Mesh(new BoxGeometry(1.8, 2.3, 0.1), new MeshBasicMaterial()); + const display = new Mesh(new PlaneGeometry(1.7, 2.2), new MeshBasicMaterial()); + display.geometry.translate(0, 0, 0.043); + display.name = "device-screen"; + asset.add(body, display); + const releaseBody = vi.fn(() => disposeDeviceModel(asset)); + models.pending[0]!.resolve({ asset, dispose: releaseBody }); + await Promise.resolve(); + viewer.orbit(0.1, 0.05); + draw(); + const before = state.frames.at(-1)!; + const bareAspect = onFramingAspect.mock.lastCall![0]; + const keyboard = new Group(); + const deck = new Mesh(new BoxGeometry(2, 1, 1), new MeshBasicMaterial()); + deck.position.set(1, 0, 0.5); + keyboard.add(deck); + const releaseKeyboard = vi.fn(() => disposeDeviceModel(keyboard)); + viewer.setAccessory({ + id: "ipad-pro-13-m5-magic-keyboard", + modelId: "ipad-pro-13-m5", + url: "/keyboard.glb", + }); + models.pending[1]!.resolve({ asset: keyboard, dispose: releaseKeyboard }); + await Promise.resolve(); + draw(); + expect(state.frames.at(-1)!.phone).toBe(before.phone); + expect(keyboard.parent).toBe(asset.parent); + expect(state.frames.at(-1)!.yaw).toBe(before.yaw); + expect(display.material.map?.image).toBe(source); + expect(state.frames.at(-1)!.cameraZ).toBeGreaterThan(before.cameraZ); + expect(onFramingAspect.mock.lastCall![0]).toBeGreaterThan(bareAspect); + const calls = onFramingAspect.mock.calls.length; + viewer.orbit(0.1, 0.05); + viewer.frameUpdated(); + draw(); + expect(onFramingAspect).toHaveBeenCalledTimes(calls); + viewer.setAccessory(null); + draw(); + expect(keyboard.parent).toBeNull(); + expect(onFramingAspect.mock.lastCall![0]).toBeCloseTo(bareAspect); + expect(releaseKeyboard).toHaveBeenCalledOnce(); + expect(releaseBody).not.toHaveBeenCalled(); + expect(state.frames.at(-1)!.phone).toBe(before.phone); + expect(state.frames.at(-1)!.cameraZ).toBeCloseTo(before.cameraZ); + expect(gpu.instances).toHaveLength(1); + viewer.dispose(); + expect(releaseBody).toHaveBeenCalledOnce(); + expect(releaseKeyboard).toHaveBeenCalledOnce(); +}); + +it("rejects incompatible accessories and releases an accessory that finishes after detaching", async () => { + const { viewer } = fixture(); + const keyboard = { + id: "ipad-pro-13-m5-magic-keyboard", + modelId: "ipad-pro-13-m5", + url: "/keyboard.glb", + } as const; + viewer.setAccessory(keyboard); + expect(models.pending).toHaveLength(0); + viewer.setModel({ id: "ipad-pro-13-m5", url: "/ipad.glb" }); + viewer.setAccessory(keyboard); + viewer.setAccessory(null); + expect(models.pending[1]!.signal.aborted).toBe(true); + const dispose = vi.fn(); + const asset = new Group(); + models.pending[1]!.resolve({ asset, dispose }); + await Promise.resolve(); + expect(asset.parent).toBeNull(); + expect(dispose).toHaveBeenCalledOnce(); + viewer.dispose(); +}); + +it("springs an ordinary device back toward its screen, freezes captured input, and stops rendering at rest", () => { + const { viewer, draw, pending, state } = fixture(); + const initialYaw = state.frames.at(-1)!.yaw!; + viewer.setInteractionActive(true, "orbit"); + viewer.orbit(0.8, 0.4); + draw(80); + const dragged = state.frames.at(-1)!; + expect(Math.abs(dragged.yaw! - initialYaw)).toBeGreaterThan(0.05); + viewer.setInteractionActive(false, "orbit"); + draw(160); + viewer.setInteractionActive(true, "touch"); + draw(180); + const captured = state.frames.at(-1)!; + viewer.orbit(1, 1); + viewer.frameUpdated(); + draw(1000); + expect(state.frames.at(-1)!.yaw).toBe(captured.yaw); + expect(state.frames.at(-1)!.cameraZ).toBe(captured.cameraZ); + expect(pending.size).toBe(0); + viewer.setInteractionActive(false, "touch"); + for (let time = 1016; time <= 4000 && pending.size; time += 16) draw(time); + expect(pending.size).toBe(0); + expect(Math.abs(state.frames.at(-1)!.yaw!)).toBeLessThanOrEqual(Math.PI / 3 + 1e-6); + expect(state.frames.at(-1)!.phone).toBe(dragged.phone); + viewer.dispose(); +}); + +it("resets the device to a square front view after orbiting", () => { + const { viewer, draw, pending, state } = fixture(); + const front = state.frames.at(-1)!.rotation!; + expect(front.angleTo(new Quaternion())).toBeLessThan(1e-6); + viewer.orbit(0.8, 0.4); + draw(80); + expect(state.frames.at(-1)!.rotation!.angleTo(front)).toBeGreaterThan(0.05); + viewer.resetPose(); + for (let time = 96; time <= 3000 && pending.size; time += 16) draw(time); + expect(pending.size).toBe(0); + expect(state.frames.at(-1)!.rotation!.angleTo(new Quaternion())).toBeLessThan(1e-6); + viewer.dispose(); +}); diff --git a/packages/client-runtime/src/device/phoneViewer.ts b/packages/client-runtime/src/device/phoneViewer.ts new file mode 100644 index 000000000000..0794490606d1 --- /dev/null +++ b/packages/client-runtime/src/device/phoneViewer.ts @@ -0,0 +1,332 @@ +import { + AmbientLight, + Box3, + CanvasTexture, + DirectionalLight, + LinearFilter, + Matrix4, + PerspectiveCamera, + Quaternion, + Scene, + SRGBColorSpace, + Vector3, + WebGLRenderer, +} from "three"; +import { + createDeviceModelSlot, + type DeviceModelSource, + type DeviceAccessorySource, +} from "./model.ts"; +import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts"; +import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts"; +import { createRenderScheduler } from "./renderScheduler.ts"; +import { createDeviceMotion } from "./deviceMotion.ts"; +import { createDeviceFraming } from "./deviceFraming.ts"; +import { nearestDeviceView } from "./deviceViewSnap.ts"; +import type { DeviceScreenSize } from "./stream.ts"; +import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts"; + +export interface PhoneViewer { + readonly setModel: (source: DeviceModelSource | null) => void; + readonly setAccessory: (source: DeviceAccessorySource | null) => void; + readonly frameUpdated: () => void; + readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void; + readonly resize: (width: number, height: number, pixelRatio: number) => void; + readonly screenPoint: ( + x: number, + y: number, + captured?: boolean, + ) => { x: number; y: number } | null; + readonly orbit: (deltaX: number, deltaY: number) => void; + readonly setInteractionActive: (active: boolean, mode: "touch" | "orbit") => void; + readonly resetPose: () => void; + readonly dispose: () => void; +} + +/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */ +export function createPhoneViewer(options: { + readonly canvas: HTMLCanvasElement; + readonly source: HTMLCanvasElement; + readonly onUnavailable: () => void; + readonly onModelError?: (cause: unknown) => void; + readonly onFramingAspect?: (aspect: number) => void; + readonly profile?: DeviceShapeProfile; + readonly model?: DeviceModelSource | null; + readonly accessory?: DeviceAccessorySource | null; +}): PhoneViewer { + const renderer = new WebGLRenderer({ + canvas: options.canvas, + alpha: true, + antialias: true, + powerPreference: "low-power", + }); + renderer.outputColorSpace = SRGBColorSpace; + const makeTexture = () => { + const next = new CanvasTexture(options.source); + next.colorSpace = SRGBColorSpace; + next.minFilter = LinearFilter; + next.magFilter = LinearFilter; + next.generateMipmaps = false; + return next; + }; + let texture = makeTexture(); + let textureWidth = options.source.width; + let textureHeight = options.source.height; + const scene = new Scene(); + const camera = new PerspectiveCamera(32, 1, 0.1, 30); + camera.position.z = 5.5; + const ambient = new AmbientLight(0xffffff, 2.4); + const key = new DirectionalLight(0xe4edff, 5); + key.position.set(-3, 4, 5); + const rim = new DirectionalLight(0xffffff, 4); + rim.position.set(3, 1, -3); + const fill = new DirectionalLight(0x9facd4, 2); + fill.position.set(-2, -2, -4); + scene.add(ambient, key, rim, fill); + + let screen: DeviceScreenSize | null = null; + let layout = phoneDisplayLayout(screen, options.source.width, options.source.height); + let profile = options.profile ?? IOS_PHONE_SHAPE; + let imported: Awaited> | null = null; + let modelSource = options.model ?? null; + let accessory: Awaited> | null = null; + let accessoryBounds: Box3 | null = null; + let phone = createPhoneScene(texture, layout, profile); + scene.add(phone.root); + let disposed = false; + const rest = new Quaternion(); + const motion = createDeviceMotion({ + choose: (rotation) => + nearestDeviceView(rotation, [{ rotation: new Quaternion(), yawLimit: Math.PI / 3 }])! + .rotation, + }); + motion.setPose(rest, performance.now(), true); + const framing = createDeviceFraming(); + const reducedMotion = () => + globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false; + let viewport = { width: 0, height: 0, pixelRatio: 1 }; + let drawingBuffer = { width: 0, height: 0, pixelRatio: 0 }; + + let framingAspect: number | null = null; + const fit = (immediate = false) => { + if (!viewport.width || !viewport.height) return; + camera.aspect = viewport.width / viewport.height; + const bounds = new Box3( + new Vector3(-phone.width / 2, -phone.height / 2, 0), + new Vector3(phone.width / 2, phone.height / 2, 0), + ); + if (imported && accessoryBounds) bounds.union(accessoryBounds); + bounds.applyMatrix4(new Matrix4().makeRotationZ(layout.rotation)); + const size = bounds.getSize(new Vector3()); + const aspect = size.x / size.y; + if (aspect !== framingAspect) { + framingAspect = aspect; + options.onFramingAspect?.(aspect); + } + phone.root.updateMatrixWorld(true); + framing.setBounds( + new Box3().setFromObject(phone.root), + (camera.fov * Math.PI) / 360, + camera.aspect, + performance.now(), + immediate, + ); + applyCamera(); + }; + const applyCamera = () => { + camera.position.set(framing.center.x, framing.center.y, framing.distance()); + camera.lookAt(framing.center.x, framing.center.y, 0); + camera.updateProjectionMatrix(); + }; + const applyPose = () => { + phone.root.quaternion.copy(motion.rotation); + phone.orientation.rotation.z = layout.rotation; + }; + const scheduler = createRenderScheduler(() => { + if (disposed || !viewport.width || !viewport.height) return; + try { + if ( + drawingBuffer.width !== viewport.width || + drawingBuffer.height !== viewport.height || + drawingBuffer.pixelRatio !== viewport.pixelRatio + ) { + // Canvas allocation clears the previous image. Commit it with the redraw, + // rather than exposing an empty buffer between ResizeObserver and the next frame. + renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.pixelRatio); + drawingBuffer = viewport; + } + const now = performance.now(); + if (motion.advance(now, reducedMotion())) { + applyPose(); + fit(reducedMotion()); + } + framing.advance(now, reducedMotion()); + applyCamera(); + renderer.render(scene, camera); + if (motion.needsFrame() || framing.needsFrame()) scheduler.invalidate(); + } catch { + options.onUnavailable(); + } + }); + const updateLayout = (nextProfile = profile) => { + const next = phoneDisplayLayout(screen, options.source.width, options.source.height); + const resized = + textureWidth !== options.source.width || textureHeight !== options.source.height; + if ( + resized || + nextProfile !== profile || + next.aspect !== layout.aspect || + next.rawLandscape !== layout.rawLandscape || + next.rotation !== layout.rotation + ) { + // The model and renderer survive framebuffer rotation and native resolution changes. + if (resized) { + const previous = texture; + texture = makeTexture(); + textureWidth = options.source.width; + textureHeight = options.source.height; + phone.setDisplay(texture, next); + previous.dispose(); + } + if (!imported && (nextProfile !== profile || next.aspect !== layout.aspect)) { + scene.remove(phone.root); + phone.dispose(); + phone = createPhoneScene(texture, next, nextProfile); + scene.add(phone.root); + } else { + phone.setDisplay(texture, next); + } + layout = next; + profile = nextProfile; + applyPose(); + fit(true); + } + applyPose(); + }; + const modelSlot = createDeviceModelSlot({ + load: loadDeviceModel, + onError: options.onModelError, + install(model) { + // Validate and prepare the next scene before releasing the visible one. + const next = disposed + ? null + : model + ? createImportedPhoneScene(model.asset, texture, layout) + : createPhoneScene(texture, layout, profile); + scene.remove(phone.root); + phone.dispose(); + imported = model; + if (!next) return; + phone = next; + scene.add(phone.root); + if (accessory) { + if (imported) phone.orientation.add(accessory.asset); + else accessory.asset.removeFromParent(); + } + applyPose(); + fit(true); + scheduler.invalidate(); + }, + }); + const accessorySlot = createDeviceModelSlot({ + load: loadDeviceModel, + onError: options.onModelError, + install(model) { + const bounds = model ? new Box3().setFromObject(model.asset) : null; + if ( + bounds && + (bounds.isEmpty() || + ![...bounds.min.toArray(), ...bounds.max.toArray()].every(Number.isFinite)) + ) + throw new Error("Device accessory has invalid bounds"); + accessory?.asset.removeFromParent(); + accessory = model; + accessoryBounds = bounds; + if (disposed) return; + if (imported && accessory) phone.orientation.add(accessory.asset); + fit(true); + scheduler.invalidate(); + }, + }); + const setAccessory = (source: DeviceAccessorySource | null) => { + accessorySlot.set(source?.modelId === modelSource?.id ? source : null); + }; + const contextLost = (event: Event) => { + event.preventDefault(); + options.onUnavailable(); + }; + options.canvas.addEventListener("webglcontextlost", contextLost); + applyPose(); + modelSlot.set(options.model ?? null); + setAccessory(options.accessory ?? null); + return { + setModel(source) { + if (source?.id !== modelSource?.id || source?.url !== modelSource?.url) + accessorySlot.set(null); + modelSource = source; + modelSlot.set(source); + }, + setAccessory, + frameUpdated() { + if (disposed) return; + updateLayout(); + texture.needsUpdate = true; + scheduler.invalidate(); + }, + setScreen(next, nextProfile = profile) { + if (disposed) return; + screen = next; + updateLayout(nextProfile); + scheduler.invalidate(); + }, + resize(width, height, pixelRatio) { + if (disposed) return; + if (![width, height, pixelRatio].every(Number.isFinite) || width <= 0 || height <= 0) return; + const ratio = Math.min(2, Math.max(1, pixelRatio)); + if (viewport.width === width && viewport.height === height && viewport.pixelRatio === ratio) + return; + viewport = { width, height, pixelRatio: ratio }; + fit(true); + scheduler.invalidate(); + }, + screenPoint(x, y, captured = false) { + if (disposed) return null; + applyPose(); + return phone.screenPoint(x, y, camera, captured); + }, + orbit(deltaX, deltaY) { + if (disposed) return; + motion.orbit(deltaX * viewport.width, deltaY * viewport.height, performance.now()); + scheduler.invalidate(); + }, + setInteractionActive(active, mode) { + if (disposed) return; + const now = performance.now(); + if (mode === "orbit") motion.dragActive(active, now); + else { + motion.hold(active, now); + framing.hold(active, now); + } + scheduler.invalidate(); + }, + resetPose() { + if (disposed) return; + motion.reset(rest, performance.now()); + scheduler.invalidate(); + }, + dispose() { + if (disposed) return; + disposed = true; + scheduler.dispose(); + const hadImported = imported !== null; + accessorySlot.dispose(); + modelSlot.dispose(); + options.canvas.removeEventListener("webglcontextlost", contextLost); + scene.remove(phone.root); + if (!hadImported) phone.dispose(); + texture.dispose(); + renderer.dispose(); + renderer.forceContextLoss(); + }, + }; +} diff --git a/packages/client-runtime/src/device/renderScheduler.test.ts b/packages/client-runtime/src/device/renderScheduler.test.ts new file mode 100644 index 000000000000..9a3e98db7e42 --- /dev/null +++ b/packages/client-runtime/src/device/renderScheduler.test.ts @@ -0,0 +1,27 @@ +import { expect, it, vi } from "vite-plus/test"; +import { createRenderScheduler } from "./renderScheduler.ts"; + +it("coalesces frame and pointer invalidations, then stops scheduling when idle or disposed", () => { + const callbacks: FrameRequestCallback[] = []; + const request = vi.fn((callback: FrameRequestCallback) => { + callbacks.push(callback); + return callbacks.length; + }); + const cancel = vi.fn(); + const render = vi.fn(); + const scheduler = createRenderScheduler(render, request, cancel); + scheduler.invalidate(); + scheduler.invalidate(); + expect(request).toHaveBeenCalledTimes(1); + callbacks[0]?.(0); + expect(render).toHaveBeenCalledTimes(1); + expect(request).toHaveBeenCalledTimes(1); + scheduler.invalidate(); + scheduler.dispose(); + scheduler.dispose(); + scheduler.invalidate(); + callbacks[1]?.(0); + expect(cancel).toHaveBeenCalledExactlyOnceWith(2); + expect(render).toHaveBeenCalledTimes(1); + expect(request).toHaveBeenCalledTimes(2); +}); diff --git a/packages/client-runtime/src/device/renderScheduler.ts b/packages/client-runtime/src/device/renderScheduler.ts new file mode 100644 index 000000000000..203b15bfc054 --- /dev/null +++ b/packages/client-runtime/src/device/renderScheduler.ts @@ -0,0 +1,23 @@ +/** Coalesces invalidations into one render. No work is scheduled while the view is idle. */ +export function createRenderScheduler( + render: () => void, + request: (callback: FrameRequestCallback) => number = requestAnimationFrame, + cancel: (id: number) => void = cancelAnimationFrame, +) { + let pending: number | null = null; + let disposed = false; + return { + invalidate() { + if (disposed || pending !== null) return; + pending = request(() => { + pending = null; + if (!disposed) render(); + }); + }, + dispose() { + disposed = true; + if (pending !== null) cancel(pending); + pending = null; + }, + }; +} diff --git a/packages/client-runtime/src/device/screenshot.test.ts b/packages/client-runtime/src/device/screenshot.test.ts new file mode 100644 index 000000000000..b4530837bd36 --- /dev/null +++ b/packages/client-runtime/src/device/screenshot.test.ts @@ -0,0 +1,50 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { captureDeviceScreenshot, DeviceScreenshotError } from "./screenshot.ts"; + +afterEach(() => vi.unstubAllGlobals()); +const target = { + platform: "android" as const, + deviceId: "phone with spaces", + access: { + httpBase: "https://remote.example/api/device-hub/hosts/remote", + wsBase: "wss://remote.example", + query: { ticket: "capture-ticket" }, + credentials: false, + }, +}; + +it("captures native bytes from the selected remote host with its media credentials", async () => { + const fetch = vi.fn( + async () => + new Response(new Uint8Array([137, 80, 78, 71]), { headers: { "Content-Type": "image/png" } }), + ); + vi.stubGlobal("fetch", fetch); + const image = await captureDeviceScreenshot(target, new AbortController().signal); + expect(image.type).toBe("image/png"); + expect(Array.from(new Uint8Array(await image.arrayBuffer()))).toEqual([137, 80, 78, 71]); + expect(fetch).toHaveBeenCalledWith( + "https://remote.example/api/device-hub/hosts/remote/vendor/serve-emu/api/screenshot?device=phone%20with%20spaces&ticket=capture-ticket", + expect.objectContaining({ method: "POST", credentials: "omit" }), + ); +}); + +it("preserves authorization failures and cancels an in-flight capture when its owner detaches", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("expired", { status: 401 })), + ); + await expect(captureDeviceScreenshot(target, new AbortController().signal)).rejects.toMatchObject( + new DeviceScreenshotError(401), + ); + vi.stubGlobal( + "fetch", + (_url: string, init: RequestInit) => + new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }); + }), + ); + const controller = new AbortController(); + const capture = captureDeviceScreenshot(target, controller.signal); + controller.abort(new Error("detached")); + await expect(capture).rejects.toThrow("detached"); +}); diff --git a/packages/client-runtime/src/device/screenshot.ts b/packages/client-runtime/src/device/screenshot.ts new file mode 100644 index 000000000000..547fabfaa381 --- /dev/null +++ b/packages/client-runtime/src/device/screenshot.ts @@ -0,0 +1,30 @@ +// @effect-diagnostics globalFetch:off - This browser transport runs without an Effect runtime, like the live stream. +import { withDeviceHubQuery } from "./hubAccess.ts"; +import type { DeviceStreamTarget } from "./stream.ts"; + +export class DeviceScreenshotError extends Error { + readonly status: number; + + constructor(status: number) { + super(`Screenshot capture failed (${status}). Try again.`); + this.status = status; + } +} + +/** Captures native pixels through the same host and credentials as the live stream, in either presentation. */ +export async function captureDeviceScreenshot(target: DeviceStreamTarget, signal: AbortSignal) { + const vendor = target.platform === "ios" ? "serve-sim" : "serve-emu"; + const url = withDeviceHubQuery( + `${target.access.httpBase}/vendor/${vendor}/api/screenshot?device=${encodeURIComponent(target.deviceId)}`, + target.access, + ); + const response = await fetch(url, { + method: "POST", + credentials: target.access.credentials ? "include" : "omit", + signal: AbortSignal.any([signal, AbortSignal.timeout(20_000)]), + }); + if (!response.ok) throw new DeviceScreenshotError(response.status); + const image = await response.blob(); + if (!image.size) throw new Error("The device returned an empty screenshot. Try again."); + return image; +} diff --git a/packages/client-runtime/src/device/shapeProfile.test.ts b/packages/client-runtime/src/device/shapeProfile.test.ts new file mode 100644 index 000000000000..7f612769cccf --- /dev/null +++ b/packages/client-runtime/src/device/shapeProfile.test.ts @@ -0,0 +1,27 @@ +import { expect, it } from "vite-plus/test"; +import { resolveDeviceShape } from "./shapeProfile.ts"; + +it("chooses tablet and phone families without mistaking display rotation for device shape", () => { + expect( + resolveDeviceShape({ platform: "ios", name: "iPad Pro 11-inch (M5)", portraitAspect: 0.75 }).id, + ).toBe("ios-tablet"); + expect( + resolveDeviceShape({ platform: "ios", name: "iPhone 18 Pro", portraitAspect: 0.46 }).id, + ).toBe("ios-phone"); + expect( + resolveDeviceShape({ platform: "android", name: "Pixel 9", portraitAspect: 0.45 }).id, + ).toBe("android-phone"); + expect( + resolveDeviceShape({ platform: "android", name: "Pixel Tablet", portraitAspect: 0.625 }).id, + ).toBe("android-tablet"); +}); + +it("uses the screen shape for renamed devices and a phone fallback before metadata arrives", () => { + expect(resolveDeviceShape({ platform: "ios", name: "Julius", portraitAspect: 0.75 }).id).toBe( + "ios-tablet", + ); + expect(resolveDeviceShape({ platform: "android", portraitAspect: 0.45 }).id).toBe( + "android-phone", + ); + expect(resolveDeviceShape({ platform: "ios", portraitAspect: Number.NaN }).id).toBe("ios-phone"); +}); diff --git a/packages/client-runtime/src/device/shapeProfile.ts b/packages/client-runtime/src/device/shapeProfile.ts new file mode 100644 index 000000000000..5ea04778e3ab --- /dev/null +++ b/packages/client-runtime/src/device/shapeProfile.ts @@ -0,0 +1,124 @@ +import type { DevicePlatform } from "@t3tools/contracts"; + +export interface DeviceShapeProfile { + readonly id: "ios-phone" | "ios-tablet" | "android-phone" | "android-tablet"; + readonly bezel: number; + readonly bodyRadius: number; + readonly screenRadius: number; + readonly depth: number; + readonly backColor: number; + readonly buttons: ReadonlyArray<{ + readonly edge: "left" | "right" | "top"; + readonly offset: number; + readonly length: number; + }>; + readonly camera: { + readonly width: number; + readonly height: number; + readonly insetX: number; + readonly insetY: number; + readonly lensRadius: number; + readonly lenses: ReadonlyArray; + readonly flash: readonly [number, number] | null; + }; +} + +/** Original family silhouettes, rather than claims to reproduce individual hardware models. */ +export const IOS_PHONE_SHAPE: DeviceShapeProfile = { + id: "ios-phone", + bezel: 0.055, + bodyRadius: 0.15, + screenRadius: 0.105, + depth: 0.085, + backColor: 0x424b5d, + buttons: [ + { edge: "right", offset: 0.35, length: 0.3 }, + { edge: "left", offset: 0.48, length: 0.18 }, + { edge: "left", offset: 0.22, length: 0.18 }, + ], + camera: { + width: 0.39, + height: 0.44, + insetX: 0.25, + insetY: 0.29, + lensRadius: 0.068, + lenses: [ + [-0.08, 0.095], + [0.08, -0.095], + ], + flash: [0.085, 0.11], + }, +}; + +export const IOS_TABLET_SHAPE: DeviceShapeProfile = { + id: "ios-tablet", + bezel: 0.065, + bodyRadius: 0.105, + screenRadius: 0.045, + depth: 0.055, + backColor: 0x9ca5af, + buttons: [ + { edge: "top", offset: 0.5, length: 0.15 }, + { edge: "right", offset: 0.78, length: 0.13 }, + { edge: "right", offset: 0.58, length: 0.13 }, + ], + camera: { + width: 0.19, + height: 0.19, + insetX: 0.15, + insetY: 0.15, + lensRadius: 0.045, + lenses: [[0, 0]], + flash: null, + }, +}; + +export const ANDROID_PHONE_SHAPE: DeviceShapeProfile = { + id: "android-phone", + bezel: 0.035, + bodyRadius: 0.115, + screenRadius: 0.08, + depth: 0.085, + backColor: 0x344449, + buttons: [ + { edge: "right", offset: 0.2, length: 0.24 }, + { edge: "right", offset: 0.65, length: 0.32 }, + ], + camera: { + width: 0.24, + height: 0.47, + insetX: 0.18, + insetY: 0.29, + lensRadius: 0.056, + lenses: [ + [0, 0.11], + [0, -0.11], + ], + flash: [0.1, 0], + }, +}; + +const ANDROID_TABLET_SHAPE: DeviceShapeProfile = { + ...IOS_TABLET_SHAPE, + id: "android-tablet", + backColor: 0x697b80, +}; + +/** Names identify a family when available; wide unknown screens get a generic tablet shell. */ +export function resolveDeviceShape(options: { + platform: DevicePlatform; + name?: string; + portraitAspect: number; +}): DeviceShapeProfile { + const name = options.name ?? ""; + const namedTablet = /\b(ipad|tablet)\b/i.test(name); + const namedPhone = /\b(iphone|phone)\b/i.test(name); + const tablet = namedTablet || (!namedPhone && options.portraitAspect >= 0.6); + return options.platform === "ios" + ? tablet + ? IOS_TABLET_SHAPE + : IOS_PHONE_SHAPE + : tablet + ? ANDROID_TABLET_SHAPE + : ANDROID_PHONE_SHAPE; +} diff --git a/packages/client-runtime/src/device/stream.test.ts b/packages/client-runtime/src/device/stream.test.ts index b947f5c17a74..7ea5aca27abf 100644 --- a/packages/client-runtime/src/device/stream.test.ts +++ b/packages/client-runtime/src/device/stream.test.ts @@ -111,6 +111,7 @@ describe("native device stream transport", () => { expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host"); expect(fetch).toHaveBeenCalledTimes(1); socket.onopen?.(); + client.setOrientation("landscape_right"); client.pressButton("home"); client.sendTouch("begin", 0.25, 0.75); const messages = socket.send.mock.calls.slice(1).map(([bytes]) => { @@ -121,6 +122,7 @@ describe("native device stream transport", () => { }; }); expect(messages).toEqual([ + { tag: 0x07, body: { orientation: "landscape_right" } }, { tag: 0x04, body: { button: "home" } }, { tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } }, ]); @@ -140,6 +142,7 @@ describe("native device stream transport", () => { client.sendTouch("begin", 0.2, 0.8); client.sendTouch("move", 0.2, 0.4); client.sendTouch("end", 0.2, 0.1); + client.setOrientation("landscape_right"); client.pressButton("home"); client.pressButton("appSwitcher"); expect( diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts index 9275f7be464b..cc849ea3e15e 100644 --- a/packages/client-runtime/src/device/stream.ts +++ b/packages/client-runtime/src/device/stream.ts @@ -19,6 +19,7 @@ * The decoder only runs while frames arrive and the viewer is attached; a * hidden panel calls `stop()` so an idle device costs nothing on the GPU. */ +import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts"; import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; import type { DevicePlatform } from "@t3tools/contracts"; @@ -217,6 +218,7 @@ export interface DeviceStreamClient { readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; readonly pressButton: (button: DeviceHardwareButton) => void; readonly rotate: () => void; + readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void; } const HID_USAGE_BY_CODE: Readonly> = { @@ -282,10 +284,11 @@ const IOS_ORIENTATIONS: ReadonlyArray = [ export function createDeviceStreamClient( target: DeviceStreamTarget, - canvas: HTMLCanvasElement, + output: HTMLCanvasElement | DeviceFrameSink, events: DeviceStreamEvents, ): DeviceStreamClient { const { access, platform, deviceId } = target; + const sink = "present" in output ? output : createCanvasFrameSink(output); const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; const device = encodeURIComponent(deviceId); const httpUrl = (path: string) => @@ -402,17 +405,14 @@ export function createDeviceStreamClient( const paint = (source: CanvasImageSource, width: number, height: number) => { if (stopped) return; - if (canvas.width !== width || canvas.height !== height) { - canvas.width = width; - canvas.height = height; - if (platform === "android") { - screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; - events.onScreen(screen); - } + if (platform === "android" && (screen?.width !== width || screen.height !== height)) { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + if (!sink.present(source, width, height)) { + fail("Could not display the device stream. Reconnect to try again."); + return; } - const context = canvas.getContext("2d"); - if (!context) return fail("Could not display the device stream. Reconnect to try again."); - context.drawImage(source, 0, 0, width, height); frameReceived(); }; @@ -861,5 +861,8 @@ export function createDeviceStreamClient( IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); }, + setOrientation: (orientation) => { + if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation })); + }, }; } diff --git a/packages/client-runtime/src/device/streamFrames.test.ts b/packages/client-runtime/src/device/streamFrames.test.ts new file mode 100644 index 000000000000..8021fe72647a --- /dev/null +++ b/packages/client-runtime/src/device/streamFrames.test.ts @@ -0,0 +1,97 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { createDeviceStreamClient } from "./stream.ts"; + +afterEach(() => vi.unstubAllGlobals()); + +it("delivers borrowed decoded frames, reports presentation failure, and discards late output", async () => { + let decoderOutput: VideoFrameOutputCallback | undefined; + let resolveOutput!: () => void; + const outputReady = new Promise((resolve) => { + resolveOutput = resolve; + }); + class Decoder { + static isConfigSupported = async () => ({ supported: true }); + state = "unconfigured"; + constructor(options: VideoDecoderInit) { + decoderOutput = options.output; + resolveOutput(); + } + configure() { + this.state = "configured"; + } + close() { + this.state = "closed"; + } + } + vi.stubGlobal("VideoDecoder", Decoder); + vi.stubGlobal("EncodedVideoChunk", vi.fn()); + vi.stubGlobal( + "WebSocket", + class { + close() {} + }, + ); + const envelope = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]); + vi.stubGlobal( + "fetch", + vi.fn( + async (url: string) => + new Response( + url.includes("avcc") + ? new ReadableStream({ + start(controller) { + controller.enqueue(envelope); + }, + }) + : "seed", + ), + ), + ); + const present = vi.fn(() => true); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "phone", + access: { + httpBase: "https://test", + wsBase: "wss://test", + credentials: false, + query: {}, + }, + }, + { present }, + events, + ); + client.start(); + await outputReady; + const frame = { + displayWidth: 1170, + displayHeight: 2532, + close: vi.fn(), + } as unknown as VideoFrame; + decoderOutput?.(frame); + expect(present).toHaveBeenCalledExactlyOnceWith(frame, 1170, 2532); + expect(frame.close).toHaveBeenCalledOnce(); + present.mockImplementationOnce(() => { + throw new Error("renderer failed"); + }); + expect(() => decoderOutput?.(frame)).toThrow("renderer failed"); + expect(frame.close).toHaveBeenCalledTimes(2); + present.mockReturnValueOnce(false); + decoderOutput?.(frame); + expect(events.onStatus).toHaveBeenLastCalledWith( + "error", + "Could not display the device stream. Reconnect to try again.", + ); + expect(frame.close).toHaveBeenCalledTimes(3); + decoderOutput?.(frame); + expect(present).toHaveBeenCalledTimes(3); + expect(frame.close).toHaveBeenCalledTimes(4); +}); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index c67ecd27c0e1..ffa8661eb9c8 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1125,6 +1125,7 @@ export interface DesktopBridge { getClientPlatform?: () => string; setNotificationBadge?: (badge: { count: number; image: string | null }) => Promise; onNotificationBadgeClear?: (listener: () => void) => () => void; + onTrackpadScrollEnd?: (listener: () => void) => () => void; /** * The OS locale as a BCP-47 tag, which the renderer cannot read for itself: * the packaged app ships only the `en-US` Chromium locale pak, so diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 59387fdacfd9..431e83189d22 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -590,6 +590,15 @@ export const PreviewAutomationHostFocus = Schema.Struct({ ...PreviewAutomationHostIdentity.fields, connectionId: PreviewAutomationConnectionId, focused: Schema.Boolean, + liveTabs: Schema.optional( + Schema.Array( + Schema.Struct({ + threadId: ThreadId, + tabId: PreviewTabId, + visible: Schema.optional(Schema.Boolean), + }), + ), + ), }); export type PreviewAutomationHostFocus = typeof PreviewAutomationHostFocus.Type; diff --git a/packages/shared/package.json b/packages/shared/package.json index 9e929325a5d4..b810abeaff9c 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -59,6 +59,10 @@ "types": "./src/observability.ts", "import": "./src/observability.ts" }, + "./otelEnvironment": { + "types": "./src/otelEnvironment.ts", + "import": "./src/otelEnvironment.ts" + }, "./httpObservability": { "types": "./src/httpObservability.ts", "import": "./src/httpObservability.ts" diff --git a/packages/shared/src/delimitedPreview.test.ts b/packages/shared/src/delimitedPreview.test.ts index 303e9042855c..0188f9da5023 100644 --- a/packages/shared/src/delimitedPreview.test.ts +++ b/packages/shared/src/delimitedPreview.test.ts @@ -21,6 +21,35 @@ describe("delimited file previews", () => { ["", "three"], ]); }); + describe.each([",", "\t"] as const)("with delimiter %j", (delimiter) => { + it("preserves a final quoted empty record without requiring a line ending", () => { + for (const ending of ["", "\n", "\r\n"]) { + expect(parseDelimitedPreview(`name\r\nAlice\r\n""${ending}`, delimiter)).toEqual({ + rows: [["name"], ["Alice"], [""]], + truncated: false, + }); + } + expect(parseDelimitedPreview('""', delimiter)).toEqual({ + rows: [[""]], + truncated: false, + }); + }); + it("does not invent records for empty input or a trailing line ending", () => { + for (const prefix of ["", "\ufeff"]) { + expect(parseDelimitedPreview(prefix, delimiter).rows).toEqual([]); + expect(parseDelimitedPreview(`${prefix}name\r\n`, delimiter).rows).toEqual([["name"]]); + expect(parseDelimitedPreview(`${prefix}""`, delimiter).rows).toEqual([[""]]); + } + }); + it("keeps a quoted empty record at the row limit", () => { + const text = `${"name\n".repeat(99)}""`; + const preview = parseDelimitedPreview(text, delimiter); + expect(preview.rows).toHaveLength(100); + expect(preview.rows.at(-1)).toEqual([""]); + expect(preview.truncated).toBe(false); + expect(parseDelimitedPreview(`${text}\nextra`, delimiter).truncated).toBe(true); + }); + }); it("bounds rows, columns and cell length and reports partial content", () => { const table = parseDelimitedPreview( Array.from({ length: 101 }, () => diff --git a/packages/shared/src/delimitedPreview.ts b/packages/shared/src/delimitedPreview.ts index d4e24409a06a..1cfbd41a7812 100644 --- a/packages/shared/src/delimitedPreview.ts +++ b/packages/shared/src/delimitedPreview.ts @@ -16,12 +16,13 @@ export function parseDelimitedPreview(text: string, delimiter: "," | "\t") { let cell = ""; let quoted = false; let truncated = false; + let rowStart = text.charCodeAt(0) === 0xfeff ? 1 : 0; const endCell = () => { if (row.length < 30) row.push(cell); else truncated = true; cell = ""; }; - for (let index = text.charCodeAt(0) === 0xfeff ? 1 : 0; index < text.length; index++) { + for (let index = rowStart; index < text.length; index++) { const char = text[index]; if (char === '"') { if (quoted && text[index + 1] === '"') { @@ -41,12 +42,13 @@ export function parseDelimitedPreview(text: string, delimiter: "," | "\t") { rows.push(row); row = []; if (char === "\r" && text[index + 1] === "\n") index++; + rowStart = index + 1; if (rows.length === 100) return { rows, truncated: truncated || index < text.length - 1 }; } } else if (cell.length < 2000) cell += char; else truncated = true; } - if (cell.length || row.length) { + if (rowStart < text.length) { endCell(); rows.push(row); } diff --git a/packages/shared/src/otelEnvironment.test.ts b/packages/shared/src/otelEnvironment.test.ts new file mode 100644 index 000000000000..3b6dac4b501b --- /dev/null +++ b/packages/shared/src/otelEnvironment.test.ts @@ -0,0 +1,82 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; + +import * as OtelEnvironment from "./otelEnvironment.ts"; + +const load = (env: Record) => + OtelEnvironment.load.pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))); + +const SPEC_OFF = + "OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it; set T3CODE_OTEL_SDK_DISABLED=false to export anyway"; +const T3_OFF = + "T3CODE_OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it"; +const specIgnored = (value: string) => + `OTEL_SDK_DISABLED=${value} was read as false; the OpenTelemetry specification recognizes only the string true, so use OTEL_SDK_DISABLED=true or T3CODE_OTEL_SDK_DISABLED to say it any other way`; + +describe("OtelEnvironment", () => { + it.effect.each([ + { name: "nothing set", env: {}, disabled: false, warnings: [] }, + // OTEL_SDK_DISABLED follows the specification: only `true`, case-insensitively. + { name: "spec true", env: { OTEL_SDK_DISABLED: "true" }, disabled: true, warnings: [SPEC_OFF] }, + { name: "spec True", env: { OTEL_SDK_DISABLED: "True" }, disabled: true, warnings: [SPEC_OFF] }, + { + name: "spec padded", + env: { OTEL_SDK_DISABLED: " true " }, + disabled: true, + warnings: [SPEC_OFF], + }, + { name: "spec false", env: { OTEL_SDK_DISABLED: "false" }, disabled: false, warnings: [] }, + { + name: "spec 1", + env: { OTEL_SDK_DISABLED: "1" }, + disabled: false, + warnings: [specIgnored("1")], + }, + { + name: "spec padded yes", + env: { OTEL_SDK_DISABLED: " yes " }, + disabled: false, + warnings: [specIgnored("yes")], + }, + // T3CODE_OTEL_SDK_DISABLED takes Config.Boolean's values, case-insensitively. + { name: "t3 1", env: { T3CODE_OTEL_SDK_DISABLED: "1" }, disabled: true, warnings: [T3_OFF] }, + { + name: "t3 TRUE", + env: { T3CODE_OTEL_SDK_DISABLED: "TRUE" }, + disabled: true, + warnings: [T3_OFF], + }, + { name: "t3 n", env: { T3CODE_OTEL_SDK_DISABLED: "n" }, disabled: false, warnings: [] }, + { + name: "t3 false overrides spec true", + env: { T3CODE_OTEL_SDK_DISABLED: "false", OTEL_SDK_DISABLED: "true" }, + disabled: false, + warnings: [], + }, + { + name: "blank t3 falls through", + env: { T3CODE_OTEL_SDK_DISABLED: " ", OTEL_SDK_DISABLED: "true" }, + disabled: true, + warnings: [SPEC_OFF], + }, + { + name: "unreadable t3 warns and falls through", + env: { T3CODE_OTEL_SDK_DISABLED: "maybe", OTEL_SDK_DISABLED: "true" }, + disabled: true, + warnings: ["T3CODE_OTEL_SDK_DISABLED=maybe is not a yes or a no and was ignored", SPEC_OFF], + }, + { + name: "bad spec value still warns when t3 answered", + env: { T3CODE_OTEL_SDK_DISABLED: "false", OTEL_SDK_DISABLED: "yes" }, + disabled: false, + warnings: [specIgnored("yes")], + }, + ])("$name", ({ env, disabled, warnings }) => + Effect.gen(function* () { + const resolved = yield* load(env); + assert.strictEqual(resolved.disabled, disabled); + assert.deepStrictEqual(resolved.warnings, warnings); + }), + ); +}); diff --git a/packages/shared/src/otelEnvironment.ts b/packages/shared/src/otelEnvironment.ts new file mode 100644 index 000000000000..f05bd9928cd5 --- /dev/null +++ b/packages/shared/src/otelEnvironment.ts @@ -0,0 +1,105 @@ +/** + * otelEnvironment: the OpenTelemetry kill switch, shared by the server and the + * desktop main process so both agree on what turns export off. + * + * `T3CODE_OTEL_SDK_DISABLED` is read first, so a machine that sets + * `OTEL_SDK_DISABLED` for everything else can still opt T3 Code back in. + * + * @module otelEnvironment + */ +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SchemaTransformation from "effect/SchemaTransformation"; + +export interface OtelEnvironment { + /** Whether OTLP export is off, whatever endpoint is configured. */ + readonly disabled: boolean; + /** Messages for the caller to log once at startup. */ + readonly warnings: ReadonlyArray; +} + +interface Flag { + /** `undefined` when the variable is unset, blank, or unreadable. */ + readonly value: boolean | undefined; + readonly warning?: string; +} + +const TrimmedLowercase = Schema.String.pipe( + Schema.decodeTo( + Schema.String, + SchemaTransformation.trim().compose(SchemaTransformation.toLowerCase()), + ), +); + +/** + * Reads a boolean that accepts `truthy` and `falsy`, ignoring case and padding. + * Any other value is ignored with a warning rather than failing startup. + */ +const flag = ( + name: string, + truthy: ReadonlyArray, + falsy: ReadonlyArray, + invalid: (value: string) => string, +) => + Config.schema( + TrimmedLowercase.pipe(Schema.decodeTo(Schema.Literals([...truthy, ...falsy]))), + name, + ).pipe( + Config.map((value): Flag => ({ value: truthy.includes(value) })), + Config.orElse(() => + Config.String(name).pipe( + Config.map((raw): Flag => { + const value = raw.trim(); + return value === "" + ? { value: undefined } + : { value: undefined, warning: invalid(value) }; + }), + ), + ), + Config.withDefault({ value: undefined }), + ); + +// `Config.Boolean`'s literals, which effect does not export on their own. +const T3CODE_TRUE = ["true", "yes", "on", "1", "y"]; +const T3CODE_FALSE = ["false", "no", "off", "0", "n"]; + +export const load: Effect.Effect = Config.all({ + t3: flag( + "T3CODE_OTEL_SDK_DISABLED", + T3CODE_TRUE, + T3CODE_FALSE, + (value) => `T3CODE_OTEL_SDK_DISABLED=${value} is not a yes or a no and was ignored`, + ), + // The specification: a boolean it defines is true "only by the + // case-insensitive string `true`", implementations "MUST NOT" accept other + // values as true, and should warn about unrecognized ones. + spec: flag( + "OTEL_SDK_DISABLED", + ["true"], + ["false"], + (value) => + `OTEL_SDK_DISABLED=${value} was read as false; the OpenTelemetry specification recognizes only the string true, so use OTEL_SDK_DISABLED=true or T3CODE_OTEL_SDK_DISABLED to say it any other way`, + ), +}).pipe( + Effect.map(({ t3, spec }) => { + const disabled = t3.value ?? spec.value ?? false; + const warnings = [t3.warning, spec.warning].filter((warning) => warning !== undefined); + if (disabled) { + warnings.push( + t3.value + ? "T3CODE_OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it" + : "OTEL_SDK_DISABLED is set, so no telemetry is exported, whatever configured it; set T3CODE_OTEL_SDK_DISABLED=false to export anyway", + ); + } + return { disabled, warnings }; + }), + // Every read above falls back instead of failing, so this cannot happen. + Effect.orDie, +); + +/** An environment that asked for nothing, for tests and for the pairing CLI. */ +export const none: OtelEnvironment = { + disabled: false, + warnings: [], +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8922d32ff151..1090902dab8b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -874,6 +874,9 @@ importers: remark-parse: specifier: ^11.0.0 version: 11.0.0 + three: + specifier: 0.180.0 + version: 0.180.0 unified: specifier: ^11.0.5 version: 11.0.5 @@ -881,6 +884,9 @@ importers: '@effect/vitest': specifier: 4.0.0-rc.115 version: 4.0.0-rc.115(patch_hash=172bd95ef73a13d322888984415d5d18d9d2aa8e13fa529eac235efac837e902)(effect@4.0.0-rc.115(patch_hash=0dfc4bb8ebd80fb3e06b91ef61346f5259517ab0f2437644fe95ae531084b1f5)) + '@types/three': + specifier: 0.180.0 + version: 0.180.0 micromark-util-types: specifier: ^2.0.2 version: 2.0.2 @@ -2030,6 +2036,9 @@ packages: '@types/react': optional: true + '@dimforge/rapier3d-compat@0.12.0': + resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==} + '@distilled.cloud/aws@1.0.0-rc.12': resolution: {integrity: sha512-L4TTEwMKB1ymXbYUqvYeoy5UWgmJtqQSzry2qWtAvD2ccwVGy0F7cFViEIYQk7boF4ZIP6SfhAls4+y1fVIi/g==} peerDependencies: @@ -5222,6 +5231,9 @@ packages: '@tiptap/starter-kit@3.31.3': resolution: {integrity: sha512-WKof9RewdmGHvWJ1wn0/HVNG2mV+HOgVRyJkKekuM9fgr6BZAAH/xZsWE1eon+94JnQ+KtK2ThydXQM/qc6b2A==} + '@tweenjs/tween.js@23.1.3': + resolution: {integrity: sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==} + '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} @@ -5359,9 +5371,15 @@ packages: '@types/serve-static@2.2.0': resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==} + '@types/stats.js@0.17.4': + resolution: {integrity: sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==} + '@types/statuses@2.0.6': resolution: {integrity: sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==} + '@types/three@0.180.0': + resolution: {integrity: sha512-ykFtgCqNnY0IPvDro7h+9ZeLY+qjgUWv+qEvUt84grhenO60Hqd4hScHE7VTB9nOQ/3QM8lkbNE+4vKjEpUxKg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -5374,6 +5392,9 @@ packages: '@types/webidl-conversions@7.0.3': resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + '@types/webxr@0.5.24': + resolution: {integrity: sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==} + '@types/whatwg-url@11.0.5': resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} @@ -5706,6 +5727,9 @@ packages: '@vscode/l10n@0.0.18': resolution: {integrity: sha512-KYSIHVmslkaCDyw013pphY+d7x1qV8IZupYfeIfzNA+nsaWHbn5uPuQRvdRFsa9zFzGeudPuoGoZ1Op4jrJXIQ==} + '@webgpu/types@0.1.74': + resolution: {integrity: sha512-lgiI4hbuLcI9unnm2cL/tvCaQU45dp0xcLWJh5uB/9MBGvIA4F8XIk7nSiBeg+K4xWGYwgZKn80LmERI5CxoTA==} + '@xmldom/xmldom@0.8.15': resolution: {integrity: sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==} engines: {node: '>=10.0.0'} @@ -8463,6 +8487,9 @@ packages: merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + meshoptimizer@0.22.0: + resolution: {integrity: sha512-IebiK79sqIy+E4EgOr+CAw+Ke8hAspXKzBd0JdgEmPHiAwmvEj2S4h1rfvo+o/BnfEYd/jAOg5IeeIjzlzSnDg==} + metro-babel-transformer@0.84.4: resolution: {integrity: sha512-rvCfz8snl9h20VcvpOHxZuHP1SlAkv4HXbzw7nyyVwu6Eqo5PRerbakQ9XmUCOsRy70spJ37O+G1TK8oMzo48g==} engines: {node: ^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0} @@ -10224,6 +10251,9 @@ packages: engines: {node: '>=10'} hasBin: true + three@0.180.0: + resolution: {integrity: sha512-o+qycAMZrh+TsE01GqWUxUIKR1AL0S8pq7zDkYOQw8GqfX8b8VoCKYUoHbhiX5j+7hr8XsuHDVU6+gkQJQKg9w==} + throat@5.0.0: resolution: {integrity: sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==} @@ -12223,6 +12253,8 @@ snapshots: optionalDependencies: '@types/react': 19.2.16 + '@dimforge/rapier3d-compat@0.12.0': {} + '@distilled.cloud/aws@1.0.0-rc.12(effect@4.0.0-rc.115(patch_hash=0dfc4bb8ebd80fb3e06b91ef61346f5259517ab0f2437644fe95ae531084b1f5))': dependencies: '@aws-sdk/types': 3.974.5 @@ -15438,6 +15470,8 @@ snapshots: '@tiptap/extensions': 3.31.3(@tiptap/core@3.31.3(@tiptap/pm@3.31.3))(@tiptap/pm@3.31.3) '@tiptap/pm': 3.31.3 + '@tweenjs/tween.js@23.1.3': {} + '@tybys/wasm-util@0.10.3': dependencies: tslib: 2.8.1 @@ -15609,9 +15643,21 @@ snapshots: '@types/http-errors': 2.0.5 '@types/node': 24.12.4 + '@types/stats.js@0.17.4': {} + '@types/statuses@2.0.6': optional: true + '@types/three@0.180.0': + dependencies: + '@dimforge/rapier3d-compat': 0.12.0 + '@tweenjs/tween.js': 23.1.3 + '@types/stats.js': 0.17.4 + '@types/webxr': 0.5.24 + '@webgpu/types': 0.1.74 + fflate: 0.8.3 + meshoptimizer: 0.22.0 + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -15621,6 +15667,8 @@ snapshots: '@types/webidl-conversions@7.0.3': optional: true + '@types/webxr@0.5.24': {} + '@types/whatwg-url@11.0.5': dependencies: '@types/webidl-conversions': 7.0.3 @@ -15958,6 +16006,8 @@ snapshots: '@vscode/l10n@0.0.18': {} + '@webgpu/types@0.1.74': {} + '@xmldom/xmldom@0.8.15': {} '@xmldom/xmldom@0.9.12': {} @@ -19070,6 +19120,8 @@ snapshots: merge-stream@2.0.0: {} + meshoptimizer@0.22.0: {} + metro-babel-transformer@0.84.4: dependencies: '@babel/core': 7.29.7 @@ -21599,6 +21651,8 @@ snapshots: commander: 2.20.3 source-map-support: 0.5.21 + three@0.180.0: {} + throat@5.0.0: {} through@2.3.8: {} diff --git a/scripts/convert-device-model.py b/scripts/convert-device-model.py new file mode 100644 index 000000000000..cba2d9c98ed4 --- /dev/null +++ b/scripts/convert-device-model.py @@ -0,0 +1,108 @@ +# Run with Blender 4.5.9: +# blender --background --python scripts/convert-device-model.py -- +# Then optimize using @gltf-transform/cli 4.2.1 with --compress false --simplify false +# --join false --palette false --flatten false --texture-size 1024 --texture-compress webp. +import hashlib +import json +import sys +from pathlib import Path + +import bpy +import numpy as np +from mathutils import Matrix, Vector + +model_id, source, output = sys.argv[sys.argv.index("--") + 1 :] +manifest_path = ( + Path(__file__).resolve().parents[1] + / "apps/web/src/components/device/models/sources.json" +) +manifest = json.loads(manifest_path.read_text()) +model = next(m for m in manifest["models"] if m["id"] == model_id) +if hashlib.sha256(Path(source).read_bytes()).hexdigest() != model["sourceSha256"]: + raise ValueError("Source checksum differs from the recorded Apple asset") +body_name, screen_name = model["bodyNode"], model["screenNode"] +bpy.ops.wm.read_factory_settings(use_empty=True) +bpy.ops.wm.usd_import(filepath=source, import_materials=True) +body = bpy.data.objects[model.get("accessoryNode", body_name)] +screen = bpy.data.objects[screen_name] +objects = [o for o in body.children_recursive if o.type == "MESH"] +if body.type == "MESH": + objects.append(body) +uv = screen.data.uv_layers.active +points = [] +coords = [] +for loop in screen.data.loops: + points.append( + tuple(screen.matrix_world @ screen.data.vertices[loop.vertex_index].co) + ) + coords.append((*uv.data[loop.index].uv, 1)) +coef = np.linalg.lstsq(np.array(coords), np.array(points), rcond=None)[0] +right = Vector(coef[0]).normalized() +up = Vector(coef[1]).normalized() +normal = right.cross(up).normalized() +# The authored UVs establish the device's front and its physical portrait axes. +# No body proportions are changed. The runtime uses a standard planar framebuffer UV. +rot = Matrix((right, up, normal)).to_4x4() +world = [rot @ Vector(p) for p in points] +lo = Vector(tuple(min(p[i] for p in world) for i in range(3))) +hi = Vector(tuple(max(p[i] for p in world) for i in range(3))) +center = (lo + hi) / 2 +scale = 2.2 / (hi.y - lo.y) +transform = ( + Matrix.Translation(Vector((0, 0, 0.043))) + @ Matrix.Diagonal(Vector((scale, scale, scale, 1))) + @ Matrix.Translation(-center) + @ rot +) +for o in objects: + matrix = transform @ o.matrix_world + o.parent = None + o.matrix_world = Matrix.Identity(4) + o.data.transform(matrix) + # Blender is Z-up; glTF export converts to Y-up. Store canonical Y-up geometry as Blender Z-up. + o.data.transform(Matrix.Rotation(np.pi / 2, 4, "X")) + if o == screen: + o.name = "device-screen" + o.data.name = "device-screen" + for loop in o.data.loops: + p = o.data.vertices[loop.vertex_index].co + uv.data[loop.index].uv = ( + (p.x / ((hi.x - lo.x) * scale)) + 0.5, + p.z / 2.2 + 0.5, + ) + # This material is replaced by the viewer and must not retain the marketing screenshot. + mat = bpy.data.materials.new("device-screen") + mat.diffuse_color = (0, 0, 0, 1) + o.data.materials.clear() + o.data.materials.append(mat) +for o in list(bpy.data.objects): + if o not in objects: + bpy.data.objects.remove(o, do_unlink=True) +for image in bpy.data.images: + if image.size[0] > 1024 or image.size[1] > 1024: + k = 1024 / max(image.size) + image.scale(max(1, round(image.size[0] * k)), max(1, round(image.size[1] * k))) +for o in objects: + o.select_set(True) +bpy.ops.export_scene.gltf( + filepath=output, + export_format="GLB", + use_selection=True, + export_yup=True, + export_animations=False, + export_extras=False, + export_image_format="AUTO", + export_copyright="Apple Inc. Original device assets; see models/sources.json for provenance.", +) +print( + "MODEL_METRICS", + json.dumps( + { + "width": (hi.x - lo.x) * scale, + "height": 2.2, + "bodyMeshes": len(objects), + "screen": screen.name, + "source": source, + } + ), +) diff --git a/scripts/mobile-showcase-environment.ts b/scripts/mobile-showcase-environment.ts index 93738b61325a..209df77bf808 100644 --- a/scripts/mobile-showcase-environment.ts +++ b/scripts/mobile-showcase-environment.ts @@ -11,7 +11,14 @@ export const SHOWCASE_PROJECT_ID = "t3code"; export const SHOWCASE_THREAD_ID = "remote-command-center"; export const SHOWCASE_TERMINAL_ID = "term-1"; -export const SHOWCASE_SCENES = ["threads", "thread", "terminal", "review", "environments"] as const; +export const SHOWCASE_SCENES = [ + "threads", + "thread", + "terminal", + "review", + "environments", + "agent-activity", +] as const; export type ShowcaseScene = (typeof SHOWCASE_SCENES)[number]; const PROJECTOR_NAMES = [ diff --git a/scripts/mobile-showcase.config.ts b/scripts/mobile-showcase.config.ts index 45a9f474fa84..c52d1f1dd116 100644 --- a/scripts/mobile-showcase.config.ts +++ b/scripts/mobile-showcase.config.ts @@ -108,7 +108,7 @@ const config: ShowcaseConfig = { simulatorDeviceType: "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro-Max", appearance: "dark", theme: DEFAULT_SHOWCASE_THEME, - scenes: ["thread", "terminal", "review", "threads", "environments"], + scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"], storeAsset: { store: "apple", directory: "apple/iphone-6.9", @@ -125,7 +125,7 @@ const config: ShowcaseConfig = { simulatorDeviceType: "com.apple.CoreSimulator.SimDeviceType.iPhone-14-Plus", appearance: "dark", theme: DEFAULT_SHOWCASE_THEME, - scenes: ["thread", "terminal", "review", "threads", "environments"], + scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"], storeAsset: { store: "apple", directory: "apple/iphone-6.5", @@ -143,6 +143,8 @@ const config: ShowcaseConfig = { appearance: "dark", theme: DEFAULT_SHOWCASE_THEME, orientation: "landscape", + // The lock screen does not follow the app's self-rotation, so a + // headless landscape capture of it would come out sideways. scenes: ["thread", "terminal", "review", "threads", "environments"], storeAsset: { store: "apple", @@ -167,7 +169,7 @@ const config: ShowcaseConfig = { height: 1920, density: 420, }, - scenes: ["thread", "terminal", "review", "threads", "environments"], + scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"], storeAsset: { store: "google-play", directory: "google-play/phone", @@ -190,7 +192,7 @@ const config: ShowcaseConfig = { height: 1920, density: 288, }, - scenes: ["thread", "terminal", "review", "threads", "environments"], + scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"], storeAsset: { store: "google-play", directory: "google-play/tablet-7", @@ -213,7 +215,7 @@ const config: ShowcaseConfig = { height: 2560, density: 288, }, - scenes: ["thread", "terminal", "review", "threads", "environments"], + scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"], storeAsset: { store: "google-play", directory: "google-play/tablet-10", diff --git a/scripts/mobile-showcase.ts b/scripts/mobile-showcase.ts index ccce919a7443..1fa8e3009f4e 100644 --- a/scripts/mobile-showcase.ts +++ b/scripts/mobile-showcase.ts @@ -26,6 +26,7 @@ import showcaseConfig, { import { SHOWCASE_ENVIRONMENTS, SHOWCASE_PROJECTS, + SHOWCASE_THREADS, seedShowcaseEnvironment, } from "./mobile-showcase-environment.ts"; @@ -934,6 +935,133 @@ async function waitForIosShowcaseScene( throw new Error(`iOS showcase scene '${scene}' did not render within ${timeoutMs}ms.`); } +function iosAxe(): string { + return NodeProcess.env.AXE_PATH ?? "axe"; +} + +async function runAxe(udid: string, args: ReadonlyArray): Promise { + // HID events sent right after the simulator settles are dropped without it. + await runCommand(iosAxe(), [...args, "--udid", udid], { + env: { ...NodeProcess.env, AXE_HID_STABILIZATION_MS: "3000" }, + }).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error( + "The agent-activity scene drives the simulator with AXe. Install it with `brew tap cameroncooke/axe && brew install axe`, or set AXE_PATH.", + ); + } + throw error; + }); +} + +async function iosSimulatorLocked(udid: string): Promise { + const state = await commandOutput("xcrun", [ + "simctl", + "spawn", + udid, + "notifyutil", + "-g", + "com.apple.springboard.lockstate", + ]).catch(() => ""); + return state.trim().endsWith(" 1"); +} + +/** + * The staging app asks for notification permission; simctl has no privacy + * service for it, so answer the prompt until the scene reports ready. + */ +async function allowIosNotificationsUntilReady(udid: string, ready: Promise): Promise { + const state = { settled: false }; + const tapping = (async () => { + while (!state.settled) { + await runAxe(udid, ["tap", "--label", "Allow"]).catch(() => undefined); + if (!state.settled) await delay(1_000); + } + })(); + try { + await ready; + } finally { + state.settled = true; + await tapping; + } +} + +/** + * Locks the simulator over the staged Live Activity and delivers the alert a + * relay push would, so the lock screen shows both. + */ +async function presentIosLockScreen(udid: string): Promise { + await runAxe(udid, ["button", "lock"]); + const deadline = Date.now() + 15_000; + while (!(await iosSimulatorLocked(udid))) { + if (Date.now() > deadline) throw new Error(`Simulator ${udid} did not lock.`); + await delay(500); + } + const alert = showcaseAgentAlert(); + await new Promise((resolve, reject) => { + const child = NodeChildProcess.execFile( + "xcrun", + ["simctl", "push", udid, ANDROID_PACKAGE, "-"], + { cwd: REPO_ROOT }, + (error) => (error ? reject(error) : resolve()), + ); + child.stdin?.end( + JSON.stringify({ + aps: { alert: { title: alert.title, body: alert.body }, sound: "default" }, + }), + ); + }); + await wakeIosLockScreen(udid); + // The first Live Activity on the lock screen asks to keep allowing them. + await delay(2_000); + await runAxe(udid, ["tap", "--label", "Allow"]).catch(() => undefined); +} + +/** + * The alert usually wakes the display, but not always. A home press on a dark + * display only wakes it, so press only after a screenshot proves it is dark; + * pressing on a lit lock screen would unlock the device instead. + */ +async function wakeIosLockScreen(udid: string): Promise { + const probe = NodePath.join(NodeOS.tmpdir(), `t3-showcase-wake-${udid}.png`); + try { + for (let attempt = 0; attempt < 5; attempt += 1) { + await delay(2_000); + await runCommand("xcrun", ["simctl", "io", udid, "screenshot", probe]); + if (!pngIsBlack(await NodeFSP.readFile(probe))) return; + await runAxe(udid, ["button", "home"]); + } + throw new Error(`Simulator ${udid} lock screen stayed dark.`); + } finally { + await NodeFSP.rm(probe, { force: true }); + } +} + +function pngIsBlack(bytes: Uint8Array): boolean { + const { data } = PNG.sync.read(Buffer.from(bytes)); + // Sample a sparse grid; a sleeping display is uniformly black. + for (let offset = 0; offset < data.length; offset += 4 * 997) { + if ((data[offset] ?? 0) > 8 || (data[offset + 1] ?? 0) > 8 || (data[offset + 2] ?? 0) > 8) { + return false; + } + } + return true; +} + +async function unlockIosSimulator(udid: string): Promise { + // The first press wakes the display, the second dismisses the lock screen. + await runAxe(udid, ["button", "home"]); + await delay(2_000); + await runAxe(udid, ["button", "home"]); +} + +/** Mirrors the app's staged hero row (showcaseAgentActivity.ts). */ +function showcaseAgentAlert(): { readonly title: string; readonly body: string } { + const thread = SHOWCASE_THREADS.find((candidate) => candidate.id === "pocket-command-center"); + const project = SHOWCASE_PROJECTS.find((candidate) => candidate.id === thread?.projectId); + if (!thread || !project) throw new Error("The showcase fixture lost its agent-activity thread."); + return { title: thread.title, body: `Approval: ${project.title}` }; +} + async function captureIos( capture: ShowcaseCapture & { readonly device: ShowcaseIosDevice }, appPath: string | null, @@ -1021,6 +1149,12 @@ async function captureIos( for (const [sceneIndex, scene] of capture.scenes.entries()) { if (sceneIndex > 0) await NodeFSP.rm(readyPath, { force: true }); await NodeFSP.writeFile(scenePath, scene); + const waitForScene = (timeoutMs?: number) => { + const ready = waitForIosShowcaseScene(simulator.udid, scene, timeoutMs); + return scene === "agent-activity" + ? allowIosNotificationsUntilReady(simulator.udid, ready) + : ready; + }; if (sceneIndex === 0) { for (let attempt = 0; attempt < 2; attempt += 1) { const isLastAttempt = attempt === 1; @@ -1028,7 +1162,7 @@ async function captureIos( // A freshly installed Expo development build can spend well over 30s // applying an already-bundled update after it reaches 100%. Killing it // at that point sends the next capture back to the dev launcher. - await waitForIosShowcaseScene(simulator.udid, scene, 120_000); + await waitForScene(120_000); break; } catch (error) { if (isLastAttempt) throw error; @@ -1036,8 +1170,9 @@ async function captureIos( } } } else { - await waitForIosShowcaseScene(simulator.udid, scene); + await waitForScene(); } + if (scene === "agent-activity") await presentIosLockScreen(simulator.udid); await delay(scene === "review" ? Math.max(config.settleDelayMs, 8_000) : config.settleDelayMs); const destination = NodePath.join( showcaseCaptureDirectory(outputDirectory, capture), @@ -1053,6 +1188,7 @@ async function captureIos( await runCommand("sips", ["--rotate", "270", destination]); } } + if (scene === "agent-activity") await unlockIosSimulator(simulator.udid); await finalizeCapture(destination, capture.device); } } @@ -1069,6 +1205,22 @@ async function runAdb(serial: string, args: ReadonlyArray): Promise { + const keys = (await adbOutput(serial, ["shell", "cmd", "notification", "list"])) + .split("\n") + .map((line) => line.trim()) + .filter((key) => key.includes("|") && !key.includes(`|${ANDROID_PACKAGE}|`)); + for (const key of keys) { + // adb joins shell args with spaces, so the key needs quoting for the pipes. + await runAdb(serial, ["shell", `cmd notification snooze --for 3600000 '${key}'`]); + } +} + async function runningAndroidAvds(): Promise> { const adb = androidSdkTool("platform-tools/adb"); const devices = (await commandOutput(adb, ["devices"])) @@ -1248,6 +1400,15 @@ async function captureAndroid( await runAdb(serial, ["install", "-r", apkPath]); } await runAdb(serial, ["shell", "pm", "clear", ANDROID_PACKAGE]); + // The agent-activity scene posts notifications; granting up front keeps the + // runtime prompt off every capture. + await runAdb(serial, [ + "shell", + "pm", + "grant", + ANDROID_PACKAGE, + "android.permission.POST_NOTIFICATIONS", + ]); await prepareAndroidShowcaseApp(serial); await runAdb(serial, ["reverse", `tcp:${config.metroPort}`, `tcp:${config.metroPort}`]); const metroUrl = encodeURIComponent(`http://127.0.0.1:${config.metroPort}?disableOnboarding=1`); @@ -1275,6 +1436,10 @@ async function captureAndroid( for (const [sceneIndex, scene] of capture.scenes.entries()) { if (sceneIndex > 0) await writeAndroidShowcaseScene(serial, scene); await waitForAndroidShowcaseScene(serial, scene); + if (scene === "agent-activity") { + await snoozeAndroidSystemNotifications(serial); + await runAdb(serial, ["shell", "cmd", "statusbar", "expand-notifications"]); + } await delay(Math.max(config.settleDelayMs, scene === "review" ? 8_000 : 5_000)); const destination = NodePath.join( showcaseCaptureDirectory(outputDirectory, capture), @@ -1292,6 +1457,9 @@ async function captureAndroid( ); }); await NodeFSP.writeFile(destination, png); + if (scene === "agent-activity") { + await runAdb(serial, ["shell", "cmd", "statusbar", "collapse"]); + } await finalizeCapture(destination, capture.device); } }