From fd8e386f7f259070899553e55c99a2a49a378480 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 12 Nov 2026 08:27:43 -0800 Subject: [PATCH 001/367] fix(mobile): the thread list shows server changes after a tab round trip (#35325) Co-Authored-By: Claude Fable 5.2 --- apps/mobile/src/Stack.tsx | 6 +- .../home/home-route-visibility.test.tsx | 62 ++++++++++++++++++- .../features/home/home-route-visibility.ts | 16 ++++- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index d93bf076a78e..b2ad65cf0537 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -493,11 +493,11 @@ const RootStackConfig = createWorkspaceStackNavigator({ Home: createNativeStackScreen({ screen: WorkspaceRootScreen, linking: "", - listeners: { + listeners: ({ navigation }) => ({ transitionStart: ({ data }) => recordHomeTransitionStart(data.closing), - transitionEnd: ({ data }) => recordHomeTransitionEnd(data.closing), + transitionEnd: ({ data }) => recordHomeTransitionEnd(data.closing, navigation.isFocused()), focus: recordHomeFocus, - }, + }), options: { ...pageRoute({ duoMinimize: "never", root: true }), contentStyle: { backgroundColor: "transparent" }, diff --git a/apps/mobile/src/features/home/home-route-visibility.test.tsx b/apps/mobile/src/features/home/home-route-visibility.test.tsx index b720bbae033a..dae399420944 100644 --- a/apps/mobile/src/features/home/home-route-visibility.test.tsx +++ b/apps/mobile/src/features/home/home-route-visibility.test.tsx @@ -4,7 +4,10 @@ import { act, useLayoutEffect } from "react"; import { createRoot } from "react-dom/client"; import { expect, it, vi } from "vite-plus/test"; -import { NativePrimaryColumnContext } from "../../native/v5-workspace-context"; +import { + NativePrimaryColumnContext, + NativeWorkspaceTabActivityContext, +} from "../../native/v5-workspace-context"; // The Thread in the detail column is the focused route on iPad split view. vi.mock("@react-navigation/native", () => ({ @@ -12,6 +15,7 @@ vi.mock("@react-navigation/native", () => ({ })); import { + recordHomeFocus, recordHomeTransitionEnd, recordHomeTransitionStart, useAtomValueWhileVisible, @@ -54,6 +58,18 @@ function setup() { ), ); }, + /** A phone has no primary column: Home is the root page of its tab's slot. */ + showHomeInTabSlot(active: boolean) { + act(() => + root.render( + + active }}> + + + , + ), + ); + }, [Symbol.dispose]() { act(() => root.unmount()); registry.dispose(); @@ -71,11 +87,51 @@ it("keeps the sidebar thread list live after switching tabs beside a focused Thr // Chats → PRs → Chats: Chats' column will-appear, then the PRs column's did-disappear. act(() => { - recordHomeTransitionEnd(true); + recordHomeTransitionEnd(true, false); recordHomeTransitionStart(false); - recordHomeTransitionEnd(true); + recordHomeTransitionEnd(true, false); }); act(() => home.registry.set(shellsAtom, 2)); expect(home.latest, "new thread shells reach the sidebar list").toBe(2); }); + +// Each phone tab slot shows its own copy of the Home route. PRs keeps Home focused; Settings +// is a route of its own. +it.each([ + { tab: "PRs", homeFocusedUnderTab: true }, + { tab: "Settings", homeFocusedUnderTab: false }, +])( + "keeps the phone thread list live after a round trip through $tab", + ({ homeFocusedUnderTab }) => { + using home = setup(); + recordHomeFocus(); + home.showHomeInTabSlot(true); + act(() => home.registry.set(shellsAtom, 1)); + expect(home.latest).toBe(1); + + // To the other tab: its slot will-appear, then the Chats slot's did-disappear. + act(() => { + recordHomeTransitionStart(false); + recordHomeTransitionEnd(true, homeFocusedUnderTab); + }); + home.showHomeInTabSlot(false); + act(() => home.registry.set(shellsAtom, 2)); + expect(home.latest, "the list under another tab holds its rows").toBe(1); + + // Back to Chats: its slot will-appear, then the other slot's did-disappear comes last. + act(() => { + recordHomeFocus(); + recordHomeTransitionStart(false); + recordHomeTransitionEnd(true, true); + }); + home.showHomeInTabSlot(true); + act(() => home.registry.set(shellsAtom, 3)); + expect(home.latest, "a server rename reaches the list").toBe(3); + + // A pushed Thread takes the focus and still stops the list. + act(() => recordHomeTransitionEnd(true, false)); + act(() => home.registry.set(shellsAtom, 4)); + expect(home.latest, "the list under a Thread holds its rows").toBe(3); + }, +); diff --git a/apps/mobile/src/features/home/home-route-visibility.ts b/apps/mobile/src/features/home/home-route-visibility.ts index 532053f8f8d0..7a8837b582a1 100644 --- a/apps/mobile/src/features/home/home-route-visibility.ts +++ b/apps/mobile/src/features/home/home-route-visibility.ts @@ -3,7 +3,10 @@ import { useNavigation } from "@react-navigation/native"; import { useContext, useRef, useSyncExternalStore } from "react"; import type { Atom } from "effect/reactivity"; -import { NativePrimaryColumnContext } from "../../native/v5-workspace-context"; +import { + NativePrimaryColumnContext, + NativeWorkspaceTabActivityContext, +} from "../../native/v5-workspace-context"; /** * Home stays mounted under a pushed Thread, and native-stack never freezes the @@ -34,8 +37,12 @@ export function recordHomeTransitionStart(closing: boolean): void { if (!closing) setHomeVisible(true); } -export function recordHomeTransitionEnd(closing: boolean): void { - if (closing) setHomeVisible(false); +/** + * Each iOS tab slot shows its own copy of the Home route, and the slot a tab switch leaves + * disappears last. So a disappear hides Home only when another route has the focus. + */ +export function recordHomeTransitionEnd(closing: boolean, homeFocused: boolean): void { + if (closing && !homeFocused) setHomeVisible(false); } /** Popping a sheet or a deep-linked route above Home has no appear event. */ @@ -46,10 +53,13 @@ export function recordHomeFocus(): void { export function useHomeRouteVisible(): boolean { const navigation = useNavigation(); const primaryColumn = useContext(NativePrimaryColumnContext); + const tabActive = useContext(NativeWorkspaceTabActivityContext)?.active ?? true; const visible = useSyncExternalStore(subscribeHomeVisibility, () => homeVisible); // Every tab slot's primary column reports to the one Home route, so a tab // switch can end on the leaving slot's disappear. Trust this column instead. if (primaryColumn) return primaryColumn.presented; + // The store is true for the Home route of every tab; this list shows in one of them. + if (!tabActive) return false; // Before Home's first navigation event: a cold deep link mounts Home under // its Thread without ever showing it. return visible ?? navigation.isFocused(); From 2b4a9c744ec669cbbd31c083819d11d5d3358785 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 12 Nov 2026 08:38:27 -0800 Subject: [PATCH 002/367] test(web): inline confirm tests no longer race the 3s disarm timer (#35361) Co-authored-by: Claude Fable 5.2 --- .../pullRequest/useInlineActionConfirm.browser.test.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/pullRequest/useInlineActionConfirm.browser.test.tsx b/apps/web/src/components/pullRequest/useInlineActionConfirm.browser.test.tsx index c94403abde4d..b96d2df0f368 100644 --- a/apps/web/src/components/pullRequest/useInlineActionConfirm.browser.test.tsx +++ b/apps/web/src/components/pullRequest/useInlineActionConfirm.browser.test.tsx @@ -1,4 +1,4 @@ -import { render } from "../../../test/render"; +import { fakeTimers, render } from "../../../test/render"; import { expect, it, vi } from "vite-plus/test"; import { userEvent } from "vite-plus/test/browser"; @@ -22,6 +22,8 @@ function pressEscape(): KeyboardEvent { it("confirms on the second click and disarms on Escape", async () => { const onConfirm = vi.fn(); await using screen = await render(); + // An armed action disarms after 3s, and a slow runner can take longer between two clicks. + using _timers = fakeTimers(); const button = screen.getByRole("button"); await userEvent.click(button); @@ -99,6 +101,7 @@ it.each(["merge", "enable-auto-merge"] as const)( /> ); await using screen = await render(control("Checking stack membership…")); + using _timers = fakeTimers(); const button = screen.getByRole("button"); await userEvent.tab(); await expect.element(button).toHaveFocus(); From 69dd4d08ea5a977a1cfb77abd5e92b99b7ea7a82 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 12 Nov 2026 08:38:51 -0800 Subject: [PATCH 003/367] test(web): Add Environment dialog test fills fields instead of typing each key (#35360) Co-authored-by: Claude Fable 5.2 --- .../settings/AddEnvironmentDialog.browser.test.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/web/src/components/settings/AddEnvironmentDialog.browser.test.tsx b/apps/web/src/components/settings/AddEnvironmentDialog.browser.test.tsx index 2ef0d474ebb8..6b4d0a08a629 100644 --- a/apps/web/src/components/settings/AddEnvironmentDialog.browser.test.tsx +++ b/apps/web/src/components/settings/AddEnvironmentDialog.browser.test.tsx @@ -88,7 +88,7 @@ describe("AddEnvironmentDialog", () => { await using _screen = await render(); await openDialogOnSsh(); - await userEvent.type(page.getByPlaceholder("Search hosts or type devbox"), "devbox"); + await userEvent.fill(page.getByPlaceholder("Search hosts or type devbox"), "devbox"); await userEvent.click( page.getByRole("dialog").getByRole("button", { name: "Add environment" }), ); @@ -103,12 +103,12 @@ describe("AddEnvironmentDialog", () => { await openDialogOnSsh(); await userEvent.click(remoteCard()); - await userEvent.type(page.getByPlaceholder("backend.example.com"), "devbox.example.com"); - await userEvent.type(page.getByPlaceholder("PAIRCODE"), "PAIRCODE"); + await userEvent.fill(page.getByPlaceholder("backend.example.com"), "devbox.example.com"); + await userEvent.fill(page.getByPlaceholder("PAIRCODE"), "PAIRCODE"); await userEvent.click(sshCard()); - await userEvent.type(page.getByPlaceholder("Search hosts or type devbox"), "devbox"); - await userEvent.type(page.getByPlaceholder("root"), "maria"); - await userEvent.type(page.getByPlaceholder("22"), "2222"); + await userEvent.fill(page.getByPlaceholder("Search hosts or type devbox"), "devbox"); + await userEvent.fill(page.getByPlaceholder("root"), "maria"); + await userEvent.fill(page.getByPlaceholder("22"), "2222"); await userEvent.keyboard("{Escape}"); await expectDefaultModeAfterReopen(); @@ -124,7 +124,7 @@ describe("AddEnvironmentDialog", () => { finishConnection = resolve; }); mocks.connectSshEnvironment.mockImplementationOnce(() => pending); - await userEvent.type(page.getByPlaceholder("Search hosts or type devbox"), "devbox"); + await userEvent.fill(page.getByPlaceholder("Search hosts or type devbox"), "devbox"); await userEvent.click( page.getByRole("dialog").getByRole("button", { name: "Add environment" }), ); From 4e48d9e6a5aba8d0184ac2c953598086ed034c29 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 12 Nov 2026 08:39:08 -0800 Subject: [PATCH 004/367] test(web): HTML preview accordion test waits for the frame's script before it toggles (#35359) Co-authored-by: Claude Fable 5.2 --- .../chat/MessagesTimeline.browser.test.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx b/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx index 3d9eeb2f1b3b..49227eb76ead 100644 --- a/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx @@ -228,9 +228,18 @@ describe("MessagesTimeline work rows", () => { const props = baseProps(); const html = injectHtmlRenderBootstrap(`
Earlier preview content
More rows
More rows
- `); + `); const src = URL.createObjectURL(new Blob([html], { type: "text/html" })); using _url = { [Symbol.dispose]: () => URL.revokeObjectURL(src) }; + // The frame loads on its own clock, and a toggle posted before its script runs is lost. + const listening = new Promise((resolve) => { + const onMessage = (event: MessageEvent) => { + if (event.data !== "listening") return; + window.removeEventListener("message", onMessage); + resolve(); + }; + window.addEventListener("message", onMessage); + }); function Preview() { const [height, setHeight] = useState(1437); return ( @@ -270,6 +279,7 @@ describe("MessagesTimeline work rows", () => { expect(viewport.scrollTop).toBeGreaterThan(0); expect(viewport.scrollHeight - viewport.clientHeight - viewport.scrollTop).toBeLessThan(2); }); + await listening; const top = frame.getBoundingClientRect().top; frame.focus(); frame.contentWindow!.postMessage("toggle", "*"); From 7c22095fcc7ad6b9324da9730960c38c5af53e90 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 08:51:48 -0800 Subject: [PATCH 005/367] fix(client-runtime): a stalled sign-in token or dead route no longer holds a reconnect for 40 seconds (#34623) Co-authored-by: Claude Fable 5.2 --- .../client-runtime/src/connection/driver.ts | 14 +++--- .../src/connection/supervisor.test.ts | 30 +++++++++++++ .../src/relay/managedRelayState.test.ts | 37 ++++++++++++++++ .../src/relay/managedRelayState.ts | 44 +++++++++++++------ 4 files changed, 105 insertions(+), 20 deletions(-) diff --git a/packages/client-runtime/src/connection/driver.ts b/packages/client-runtime/src/connection/driver.ts index 512dba21e2d9..416ca526db7a 100644 --- a/packages/client-runtime/src/connection/driver.ts +++ b/packages/client-runtime/src/connection/driver.ts @@ -116,16 +116,16 @@ export const connectOverRoutes = Effect.fn("ConnectionDriver.connectOverRoutes") // An incompatible server is the same server on every route. if (result.failure.reason === "unsupported") return yield* result.failure; } - // A slow route that answered after the head start goes before one that is - // still dead, which can hold an attempt open far longer than its probe. const late = yield* Effect.forEach(silent, ({ index }) => Fiber.join(checks[index]!), { concurrency: "unbounded", }); - const retry = [ - ...silent.filter((_, position) => late[position] !== "silent"), - ...silent.filter((_, position) => late[position] === "silent"), - ]; - for (const { route } of retry) { + const answeredLate = silent.filter((_, position) => late[position] !== "silent"); + const stillSilent = silent.filter((_, position) => late[position] === "silent"); + for (const { route } of [...answeredLate, ...stillSilent]) { + // A route still silent after its whole check can hold the attempt open far + // longer than the check did. Once another route failed in a way that + // retries, the retry reaches it sooner than waiting here would. + if (transient !== undefined && stillSilent.some((entry) => entry.route === route)) break; const result = yield* attempt(route); if (result._tag === "Success") return result.success; if (result.failure.reason === "unsupported") return yield* result.failure; diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 163075a6efa7..8d23348fead9 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -1871,6 +1871,36 @@ describe("EnvironmentSupervisor routes", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("retries instead of waiting on a LAN route that stayed silent after relay failed", () => + Effect.gen(function* () { + const checkingLan = yield* Deferred.make(); + const attempted: string[] = []; + const harness = yield* makeHarness({ + checkRoute: (route) => + route.target._tag === "BearerConnectionTarget" + ? Deferred.succeed(checkingLan, undefined).pipe( + Effect.andThen(Effect.sleep("2500 millis")), + Effect.as("silent" as const), + ) + : Effect.succeed("unchecked"), + prepare: (_attempt, target) => { + attempted.push(target._tag); + return target._tag === "RelayConnectionTarget" + ? Effect.fail(transient("Relay socket closed.")) + : Effect.never; + }, + }); + const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* Deferred.await(checkingLan); + yield* TestClock.adjust("2500 millis"); + yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); + expect(attempted).toEqual(["RelayConnectionTarget"]); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("skips a silent LAN route and connects over T3 Connect", () => Effect.gen(function* () { const harness = yield* makeHarness({ diff --git a/packages/client-runtime/src/relay/managedRelayState.test.ts b/packages/client-runtime/src/relay/managedRelayState.test.ts index 6b91a2213bbe..41d7327ac14e 100644 --- a/packages/client-runtime/src/relay/managedRelayState.test.ts +++ b/packages/client-runtime/src/relay/managedRelayState.test.ts @@ -8,6 +8,7 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; +import * as TestClock from "effect/testing/TestClock"; import { Atom, AtomRegistry } from "effect/reactivity"; import { afterEach, vi } from "vite-plus/test"; @@ -191,6 +192,42 @@ describe("createManagedRelayQueryManager", () => { }), ); + it.effect("replaces a Clerk token read that has not answered after 4 seconds", () => + Effect.gen(function* () { + const token = clerkToken(4_102_444_800); + const readClerkToken = vi + .fn<() => Promise>() + .mockReturnValueOnce(new Promise(() => {})) + .mockResolvedValue(token); + const session = createManagedRelaySession({ accountId: "account-1", readClerkToken }); + + const readsFiber = yield* Effect.all([session.readClerkToken(), session.readClerkToken()], { + concurrency: "unbounded", + }).pipe(Effect.forkChild); + yield* TestClock.adjust("3999 millis"); + expect(readClerkToken).toHaveBeenCalledTimes(1); + + yield* TestClock.adjust("1 millis"); + expect(yield* Fiber.join(readsFiber)).toEqual([token, token]); + expect(readClerkToken).toHaveBeenCalledTimes(2); + }), + ); + + it.effect("fails a Clerk token read that never answers after three tries", () => + Effect.gen(function* () { + const readClerkToken = vi.fn(() => new Promise(() => {})); + const session = createManagedRelaySession({ accountId: "account-1", readClerkToken }); + + const readFiber = yield* session.readClerkToken().pipe(Effect.flip, Effect.forkChild); + yield* TestClock.adjust("12 seconds"); + + expect((yield* Fiber.join(readFiber)).message).toBe( + "Timed out obtaining the T3 Connect session token.", + ); + expect(readClerkToken).toHaveBeenCalledTimes(3); + }), + ); + it.effect("updates the token provider without replacing a same-account session", () => Effect.gen(function* () { const firstRead = vi.fn(() => Promise.resolve(null)); diff --git a/packages/client-runtime/src/relay/managedRelayState.ts b/packages/client-runtime/src/relay/managedRelayState.ts index 5a4b9a62a547..90e635f4e328 100644 --- a/packages/client-runtime/src/relay/managedRelayState.ts +++ b/packages/client-runtime/src/relay/managedRelayState.ts @@ -3,6 +3,7 @@ import { decodeRelayJwt } from "@t3tools/shared/relayJwt"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; @@ -14,6 +15,11 @@ import { relayProtectedErrorMessage } from "./errorPresentation.ts"; const DEFAULT_STALE_TIME_MS = 15_000; const DEFAULT_IDLE_TTL_MS = 5 * 60_000; const CLERK_TOKEN_EXPIRY_SKEW_MS = 5_000; +// Clerk retries a failed token request itself, but waits 3 s before the first +// retry and longer before each next one. A fresh request usually answers at +// once, so a slow read is replaced instead of waited out. +const CLERK_TOKEN_READ_TIMEOUT_MS = 4_000; +const CLERK_TOKEN_READ_ATTEMPTS = 3; const isManagedRelayRequestFailedError = Schema.is(ManagedRelay.ManagedRelayRequestFailedError); export interface ManagedRelaySession { @@ -66,7 +72,10 @@ const managedRelaySessionControls = new WeakMap | null = null; + let pendingToken: { + readonly operation: Promise; + readonly startedAtMillis: number; + } | null = null; let readClerkToken = input.readClerkToken; let tokenProviderGeneration = 0; @@ -74,8 +83,10 @@ export function createManagedRelaySession(input: ManagedRelaySessionInput): Mana if (cachedToken && cachedToken.expiresAtMillis > nowMillis + CLERK_TOKEN_EXPIRY_SKEW_MS) { return cachedToken.token; } - if (pendingToken) { - return await pendingToken; + // A read that outlived its timeout is left running, since it still fills + // the cache, but later readers start a fresh one instead of joining it. + if (pendingToken && nowMillis - pendingToken.startedAtMillis < CLERK_TOKEN_READ_TIMEOUT_MS) { + return await pendingToken.operation; } const operationGeneration = tokenProviderGeneration; @@ -99,11 +110,12 @@ export function createManagedRelaySession(input: ManagedRelaySessionInput): Mana : null; return token; }); - pendingToken = operation; + const pending = { operation, startedAtMillis: nowMillis }; + pendingToken = pending; try { return await operation; } finally { - if (pendingToken === operation) { + if (pendingToken === pending) { pendingToken = null; } } @@ -112,14 +124,20 @@ export function createManagedRelaySession(input: ManagedRelaySessionInput): Mana const session: ManagedRelaySession = { accountId: input.accountId, readClerkToken: Effect.fn("clientRuntime.managedRelaySession.readClerkToken")(function* () { - const nowMillis = yield* Clock.currentTimeMillis; - return yield* Effect.tryPromise({ - try: () => readCachedClerkToken(nowMillis), - catch: (cause) => - new ManagedRelaySessionError({ - message: "Could not obtain the T3 Connect session token.", - cause, - }), + for (let attempt = 0; attempt < CLERK_TOKEN_READ_ATTEMPTS; attempt += 1) { + const nowMillis = yield* Clock.currentTimeMillis; + const token = yield* Effect.tryPromise({ + try: () => readCachedClerkToken(nowMillis), + catch: (cause) => + new ManagedRelaySessionError({ + message: "Could not obtain the T3 Connect session token.", + cause, + }), + }).pipe(Effect.timeoutOption(CLERK_TOKEN_READ_TIMEOUT_MS)); + if (Option.isSome(token)) return token.value; + } + return yield* new ManagedRelaySessionError({ + message: "Timed out obtaining the T3 Connect session token.", }); }), }; From 34c5faa3c38bbbcc5914862e35c80a9969efd38d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 08:52:59 -0800 Subject: [PATCH 006/367] fix(desktop): release-note entities named like Object.prototype keys stay literal (#35366) --- apps/desktop/src/updates/releaseNotes.test.ts | 12 +++++++++--- apps/desktop/src/updates/releaseNotes.ts | 18 +++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/updates/releaseNotes.test.ts b/apps/desktop/src/updates/releaseNotes.test.ts index 6f05b1e9e94a..a8fcc70ea355 100644 --- a/apps/desktop/src/updates/releaseNotes.test.ts +++ b/apps/desktop/src/updates/releaseNotes.test.ts @@ -241,14 +241,20 @@ describe("normalizeDesktopUpdateReleaseNotes", () => { }); }); - it("does not throw on out-of-range numeric entities and keeps the literal", () => { + it("keeps unsupported entities literal, including out-of-range and Object.prototype names", () => { const result = normalizeDesktopUpdateReleaseNotes( - "- Broken entity �", + "- Broken entity �\n- Mentions &constructor; and &toString;", "1.0.0", "latest", ); expect(result).toEqual({ - releaseNotes: [{ version: "1.0.0", items: ["Broken entity �"], totalItems: 1 }], + releaseNotes: [ + { + version: "1.0.0", + items: ["Mentions &constructor; and &toString;", "Broken entity �"], + totalItems: 2, + }, + ], omittedReleaseCount: 0, }); }); diff --git a/apps/desktop/src/updates/releaseNotes.ts b/apps/desktop/src/updates/releaseNotes.ts index 715c1767e6a0..30bfb9ace64b 100644 --- a/apps/desktop/src/updates/releaseNotes.ts +++ b/apps/desktop/src/updates/releaseNotes.ts @@ -20,14 +20,14 @@ const MAX_RELEASE_NOTE_GROUPS = 6; const MAX_RELEASE_NOTE_ITEMS_PER_GROUP = 8; const MAX_RELEASE_NOTE_ITEM_LENGTH = 220; -const HTML_ENTITY_REPLACEMENTS: Readonly> = { - amp: "&", - apos: "'", - gt: ">", - lt: "<", - nbsp: " ", - quot: '"', -}; +const HTML_ENTITY_REPLACEMENTS: ReadonlyMap = new Map([ + ["amp", "&"], + ["apos", "'"], + ["gt", ">"], + ["lt", "<"], + ["nbsp", " "], + ["quot", '"'], +]); function decodeCodePoint(codePoint: number, entity: string): string { if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) { @@ -37,7 +37,7 @@ function decodeCodePoint(codePoint: number, entity: string): string { } function decodeHtmlEntity(entity: string): string { - const named = HTML_ENTITY_REPLACEMENTS[entity]; + const named = HTML_ENTITY_REPLACEMENTS.get(entity); if (named) return named; if (entity.startsWith("#x")) { return decodeCodePoint(Number.parseInt(entity.slice(2), 16), entity); From 371963ce0b4cfe8bac3911efd57ae3af90750b41 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 08:58:29 -0800 Subject: [PATCH 007/367] fix(desktop): startup log prune keeps files that rotation never names (#35378) --- .../src/app/DesktopObservability.test.ts | 21 +++++++++++++++++++ apps/desktop/src/app/DesktopObservability.ts | 6 ++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/app/DesktopObservability.test.ts b/apps/desktop/src/app/DesktopObservability.test.ts index ab8ea0b8e9d6..ceb29b149e3d 100644 --- a/apps/desktop/src/app/DesktopObservability.test.ts +++ b/apps/desktop/src/app/DesktopObservability.test.ts @@ -281,6 +281,27 @@ describe("DesktopObservability", () => { ), ); + it.effect("startup prunes only canonical rotation suffixes", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-rotating-prune-test-", + }); + const filePath = `${directory}/server-child.log`; + const retained = ["1", "3", "04", "4.0", "4e0", "0x4", "+4", "notes"]; + for (const suffix of [...retained, "4", "50"]) { + yield* fileSystem.writeFileString(`${filePath}.${suffix}`, suffix); + } + + yield* DesktopObservability.makeRotatingLogFileWriter({ filePath, maxFiles: 3 }); + + assert.deepEqual( + (yield* fileSystem.readDirectory(directory)).sort(), + retained.map((suffix) => `server-child.log.${suffix}`).sort(), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("stops every export when the OpenTelemetry SDK is disabled", () => { const requests: Array = []; return Effect.gen(function* () { diff --git a/apps/desktop/src/app/DesktopObservability.ts b/apps/desktop/src/app/DesktopObservability.ts index e9b0ab9379df..16192fff46de 100644 --- a/apps/desktop/src/app/DesktopObservability.ts +++ b/apps/desktop/src/app/DesktopObservability.ts @@ -263,8 +263,10 @@ export const makeRotatingLogFileWriter = Effect.fn("makeRotatingLogFileWriter")( const entries = yield* fileSystem.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); for (const entry of entries) { if (!entry.startsWith(`${baseName}.`)) continue; - const suffix = Number(entry.slice(baseName.length + 1)); - if (!Number.isInteger(suffix) || suffix <= maxFiles) continue; + const suffixText = entry.slice(baseName.length + 1); + const suffix = Number(suffixText); + if (!Number.isInteger(suffix) || String(suffix) !== suffixText || suffix <= maxFiles) + continue; yield* fileSystem.remove(path.join(directory, entry), { force: true }).pipe(Effect.ignore); } }); From f99b2835ef0d18559a59dd954e9f564fcb43e7d3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:02:52 -0800 Subject: [PATCH 008/367] fix(provider-core): a timed-out status probe stops a CLI that ignores SIGTERM (#35379) --- .../src/server/snapshotProbe.test.ts | 18 ++++++++++++++++++ .../provider-core/src/server/snapshotProbe.ts | 16 +++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/provider-core/src/server/snapshotProbe.test.ts b/packages/provider-core/src/server/snapshotProbe.test.ts index 58d52e1002c6..26b66751516c 100644 --- a/packages/provider-core/src/server/snapshotProbe.test.ts +++ b/packages/provider-core/src/server/snapshotProbe.test.ts @@ -1,8 +1,10 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import type { ModelCapabilities } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { createModelCapabilities } from "@t3tools/shared/model"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import * as ChildProcess from "effect/process/ChildProcess"; @@ -158,3 +160,19 @@ describe("ProviderCommandNotFoundError", () => { }); }); }); + +describe("spawnAndCollect", () => { + // regression: #35262 - without forceKillAfter a timed-out probe whose CLI ignores + // SIGTERM waits for it to exit, so the caller's timeout never returns. + it.live.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "lets a timed-out probe return when the CLI ignores SIGTERM", + () => + Effect.gen(function* () { + const result = yield* spawnAndCollect( + "sh", + ChildProcess.make("sh", ["-c", 'trap "" TERM; sleep 30']), + ).pipe(Effect.timeoutOption("200 millis")); + expect(Option.isNone(result)).toBe(true); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/provider-core/src/server/snapshotProbe.ts b/packages/provider-core/src/server/snapshotProbe.ts index 8718231cd967..a30b69ac9866 100644 --- a/packages/provider-core/src/server/snapshotProbe.ts +++ b/packages/provider-core/src/server/snapshotProbe.ts @@ -112,12 +112,26 @@ export function providerProbeEnvironment(environment: NodeJS.ProcessEnv): NodeJS return { ...environment, MISE_OFFLINE: "1" }; } +// A timed-out probe interrupts the child with SIGTERM and then waits for it to exit, +// so a CLI that ignores SIGTERM would hold the caller's timeout open without this. +const PROBE_FORCE_KILL_AFTER = "1 second"; + +function withProbeForceKill(command: ChildProcess.Command): ChildProcess.Command { + if (!ChildProcess.isStandardCommand(command) || command.options.forceKillAfter !== undefined) { + return command; + } + return ChildProcess.make(command.command, command.args, { + ...command.options, + forceKillAfter: PROBE_FORCE_KILL_AFTER, + }); +} + export const spawnAndCollect = Effect.fn("snapshotProbe.spawnAndCollect")(function* ( binaryPath: string, command: ChildProcess.Command, ) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn(command); + const child = yield* spawner.spawn(withProbeForceKill(command)); const [stdout, stderr, exitCode] = yield* Effect.all( [ collectStreamAsString(child.stdout), From 49b858b807faf994569bd435c942abee478adcaf Mon Sep 17 00:00:00 2001 From: kirat02 <142911687+Kirat0201@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:34:21 +0530 Subject: [PATCH 009/367] fix(git): selected-file commits work through symlinked subdirectories (#35278) Co-authored-by: Julius Marminge --- apps/server/src/vcs/GitVcsDriverCore.test.ts | 54 ++++++++++++-------- apps/server/src/vcs/GitVcsDriverCore.ts | 6 ++- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 25401ec98544..6dec0cf0207b 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -6341,28 +6341,42 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("prepares selected files without staging and commits only those files", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - yield* initRepoWithCommit(cwd); - const driver = yield* GitVcsDriver.GitVcsDriver; - - yield* writeTextFile(cwd, "a.txt", "a\n"); - yield* writeTextFile(cwd, "b.txt", "b\n"); - - const context = yield* driver.prepareCommitContext(cwd, ["a.txt"]); - assert.include(context?.stagedSummary ?? "", "a.txt"); - assert.notInclude(context?.stagedSummary ?? "", "b.txt"); - assert.equal(yield* git(cwd, ["diff", "--cached", "--name-only"]), ""); + it.effect.each(["root", "aliased-nested"] as const)( + "prepares selected files without staging and commits only those files from %s", + (location) => + Effect.gen(function* () { + const sandbox = yield* makeTmpDir(); + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const repository = path.join(sandbox, "repo"); + const nested = path.join(repository, "sub", "dir"); + yield* fileSystem.makeDirectory(nested, { recursive: true }); + yield* initRepoWithCommit(repository); + const cwd = location === "aliased-nested" ? path.join(sandbox, "link") : repository; + if (location === "aliased-nested") NodeFS.symlinkSync(nested, cwd, "junction"); + const driver = yield* GitVcsDriver.GitVcsDriver; + const selected = "sub/dir/a.txt"; + yield* writeTextFile(repository, selected, "a\n"); + yield* writeTextFile(repository, "b.txt", "b\n"); - const commit = yield* driver.commit(cwd, "Add a", "", { stage: { filePaths: ["a.txt"] } }); - assert.match(commit.commitSha, /^[a-f0-9]{40}$/); - assert.equal(yield* git(cwd, ["log", "-1", "--pretty=%s"]), "Add a"); + const context = yield* driver.prepareCommitContext(cwd, [selected]); + assert.include(context?.stagedSummary ?? "", "a.txt"); + assert.notInclude(context?.stagedSummary ?? "", "b.txt"); + assert.equal(yield* git(repository, ["diff", "--cached", "--name-only"]), ""); - const status = yield* git(cwd, ["status", "--porcelain"]); - assert.include(status, "?? b.txt"); - assert.notInclude(status, "a.txt"); - }), + const commit = yield* driver.commit(cwd, "Add a", "", { + stage: { filePaths: [selected] }, + }); + assert.match(commit.commitSha, /^[a-f0-9]{40}$/); + assert.equal(yield* git(repository, ["log", "-1", "--pretty=%s"]), "Add a"); + assert.equal( + yield* git(repository, ["diff-tree", "--no-commit-id", "--name-only", "-r", "HEAD"]), + selected, + ); + const status = yield* git(repository, ["status", "--porcelain"]); + assert.include(status, "?? b.txt"); + assert.notInclude(status, "a.txt"); + }), ); it.effect("stops a selected-file commit when another process holds the index lock", () => diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index a8e4fb9265f3..5eae512995d3 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -2633,7 +2633,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd, ["rev-parse", "--git-path", "index", "--git-path", "objects"], )).split("\n"); - const indexPath = path.resolve(cwd, indexValue.trim()); + // Git's relative paths use the physical cwd, even when the project uses a symlink. + const physicalCwd = yield* fileSystem.realPath(cwd); + const indexPath = path.resolve(physicalCwd, indexValue.trim()); const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-commit-index-", }); @@ -2651,7 +2653,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const quoted = (entry: string) => entry.startsWith('"') ? entry : `"${entry.replace(/[\\"]/g, "\\$&")}"`; const alternates = [ - path.resolve(cwd, objectsValue.trim()), + path.resolve(physicalCwd, objectsValue.trim()), ...objectDirectories .split("\n") .flatMap((line) => From 833a15b6c1921dc3ee7a94f5a9fe8db13a6e93ed Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:10:40 -0800 Subject: [PATCH 010/367] fix(opencode): a project opened mid-scan lists its OpenCode skills (#35384) --- .../src/server/driver.test.ts | 67 +++++++++++++++++++ .../provider-opencode/src/server/driver.ts | 2 + .../provider-opencode/src/server/status.ts | 6 +- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/packages/provider-opencode/src/server/driver.test.ts b/packages/provider-opencode/src/server/driver.test.ts index 6d74dc8dd254..ffb91b6a3199 100644 --- a/packages/provider-opencode/src/server/driver.test.ts +++ b/packages/provider-opencode/src/server/driver.test.ts @@ -537,6 +537,73 @@ it.layer(layer)("OpenCodeDriver runtime selection", (it) => { }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), ); + it.effect("caches a cold OpenCode 2 directory's skills only after its scan ends", () => + Effect.gen(function* () { + // OpenCode 2.2.7: touching a location lists it as loaded with built-in commands and skills + // at once, while provider.list waits for the scan that adds the project's skills. + const scanEnded = yield* Deferred.make(); + const builtinSkills = JSON.stringify({ + location: { directory: "/work" }, + data: ( + JSON.parse(OPENCODE_2_WORKSPACE_RESPONSES["/api/skill"].body).data as Array<{ + id: string; + }> + ).filter((skill) => skill.id !== "plum"), + }); + const http = HttpClient.make((request) => { + const path = new URL(request.url).pathname; + const json = (body: string) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json(JSON.parse(body)))); + const afterScan = (body: string) => + Deferred.await(scanEnded).pipe(Effect.andThen(json(body))); + switch (path) { + case "/api/info": + return json( + JSON.stringify({ + ...JSON.parse(OPENCODE_2_RESPONSES["/api/info"].body), + version: "2.2.7", + }), + ); + case "/api/event": + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }), + ), + ); + case "/api/debug/location": + return json('[{"directory":"/work"}]'); + case "/api/provider": + case "/api/model": + return afterScan('{"location":{"directory":"/work"},"data":[]}'); + case "/api/command": + return json(OPENCODE_2_WORKSPACE_RESPONSES["/api/command"].body); + case "/api/skill": + return Deferred.isDone(scanEnded).pipe( + Effect.flatMap((done) => + json(done ? OPENCODE_2_WORKSPACE_RESPONSES["/api/skill"].body : builtinSkills), + ), + ); + default: + return Effect.die(`Unexpected request: ${path}`); + } + }); + const instance = yield* create({ serverUrl: "http://127.0.0.1:4096" }, http); + + const snapshot = yield* instance.snapshotForCwd!("/work").pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* Deferred.succeed(scanEnded, undefined); + yield* TestClock.adjust("30 seconds"); + const { skills } = yield* Fiber.join(snapshot); + assert.include( + skills.map((skill) => skill.name), + "plum", + ); + }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + ); + it.effect.each(["probe", "error", "empty"] as const)( "recovers discovery without a manual refresh after %s", (initialCatalog) => diff --git a/packages/provider-opencode/src/server/driver.ts b/packages/provider-opencode/src/server/driver.ts index 2686de885b3f..a0d2f7905e4a 100644 --- a/packages/provider-opencode/src/server/driver.ts +++ b/packages/provider-opencode/src/server/driver.ts @@ -438,6 +438,8 @@ export const OpenCodeDriver: ProviderDriver< Effect.timeout("2 seconds"), Effect.orElseSucceed(() => true), ), + // OpenCode answers provider.list for a location only once its skill scan has ended. + client.provider.list({ location }).pipe(Effect.timeout("10 seconds"), Effect.ignore), ); }).pipe(Effect.scoped), ); diff --git a/packages/provider-opencode/src/server/status.ts b/packages/provider-opencode/src/server/status.ts index 459904c27eb0..0f24b1e22f47 100644 --- a/packages/provider-opencode/src/server/status.ts +++ b/packages/provider-opencode/src/server/status.ts @@ -578,16 +578,18 @@ export interface OpenCode2Workspace { /** * Reading a location the server has not loaded boots its scan, and only reads after the scan's * `command.updated` plus `skill.updated` are complete; `scanned` must listen before `read`. A - * loaded location listing no commands is still scanning. OpenCode exposes no readiness signal, - * so one another caller booted moments earlier can still read partial. + * loaded location listing no commands is still scanning. A location another caller booted + * moments earlier lists as loaded with only built-ins, so `settled` waits out its scan first. */ export const loadOpenCode2Workspace = Effect.fn("OpencodeProviderStatus.loadOpenCode2Workspace")( function* ( read: Effect.Effect, scanned: Effect.Effect, loaded: Effect.Effect, + settled: Effect.Effect = Effect.void, ) { const wasLoaded = yield* loaded; + yield* settled; const first = yield* read; if (wasLoaded && first.commands.length > 0) return first; yield* scanned.pipe(Effect.timeoutOption("10 seconds")); From a5bd0ba5265fdfbfadc9149a54300b5048f853cb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:11:02 -0800 Subject: [PATCH 011/367] fix(ssh): SSH setup on a host without gzip says gzip is missing (#35389) --- packages/ssh/src/tunnel.test.ts | 59 +++++++++++++++++++++++++++++++++ packages/ssh/src/tunnel.ts | 8 ++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/ssh/src/tunnel.test.ts b/packages/ssh/src/tunnel.test.ts index 6fd9e71510e8..03f8ebe2a62e 100644 --- a/packages/ssh/src/tunnel.test.ts +++ b/packages/ssh/src/tunnel.test.ts @@ -230,6 +230,65 @@ describe("ssh tunnel scripts", () => { }, ); + it("names gzip when a host with tar but no gzip cannot unpack the release archive", () => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-ssh-runner-")); + try { + const bin = NodePath.join(root, "bin"); + const release = NodePath.join(root, "release"); + const archive = `t3-${ARCHIVE.archiveVersion}-linux-x64.tar.gz`; + NodeFS.mkdirSync(bin); + NodeFS.mkdirSync(NodePath.join(root, "home")); + NodeFS.mkdirSync(NodePath.join(release, "t3"), { recursive: true }); + NodeFS.writeFileSync(NodePath.join(release, "t3", "t3"), "#!/bin/sh\necho ok\n", { + mode: 0o755, + }); + NodeChildProcess.execFileSync("tar", ["-czf", archive, "t3"], { cwd: release }); + NodeChildProcess.execFileSync("sh", ["-c", `sha256sum ${archive} > SHA256SUMS`], { + cwd: release, + }); + for (const tool of [ + "sh", + "mkdir", + "mktemp", + "cat", + "rm", + "grep", + "cut", + "sha256sum", + "tar", + "mv", + ]) { + NodeFS.symlinkSync( + NodeChildProcess.execFileSync("sh", ["-c", `command -v ${tool}`], { + encoding: "utf8", + }).trim(), + NodePath.join(bin, tool), + ); + } + NodeFS.writeFileSync( + NodePath.join(bin, "uname"), + '#!/bin/sh\ncase "$1" in -s) echo Linux ;; -m) echo x86_64 ;; esac\n', + { mode: 0o755 }, + ); + NodeFS.writeFileSync( + NodePath.join(bin, "curl"), + `#!/bin/sh\nfor last; do :; done\nfor a; do case "$a" in http*) src="${release}/\${a##*/}" ;; esac; done\nexec cat "$src" > "$last"\n`, + { mode: 0o755 }, + ); + const runner = NodePath.join(root, "run-t3.sh"); + NodeFS.writeFileSync(runner, SshTunnel.buildRemoteT3RunnerScript(ARCHIVE)); + const result = NodeChildProcess.spawnSync(NodePath.join(bin, "sh"), [runner, "--version"], { + encoding: "utf8", + env: { HOME: NodePath.join(root, "home"), PATH: bin }, + }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /(missing|needs) gzip/); + assert.notInclude(result.stderr, "Cannot exec"); + } finally { + NodeFS.rmSync(root, { recursive: true, force: true }); + } + }); + it.effect("accepts launch JSON after remote shell startup noise", () => { const target = { alias: "devbox", diff --git a/packages/ssh/src/tunnel.ts b/packages/ssh/src/tunnel.ts index f981a86ec0a3..5d4ac857b17b 100644 --- a/packages/ssh/src/tunnel.ts +++ b/packages/ssh/src/tunnel.ts @@ -526,7 +526,13 @@ if ! t3_runtime_ready; then if [ -z "$T3_EXPECTED" ] || [ "$T3_ACTUAL" != "$T3_EXPECTED" ]; then printf 'Checksum mismatch for %s.\\n' "$T3_ARCHIVE" >&2; exit 1 fi - tar -xzf "$T3_STAGING/$T3_ARCHIVE" -C "$T3_STAGING" --strip-components=1 + if ! T3_TAR_ERROR="$(tar -xzf "$T3_STAGING/$T3_ARCHIVE" -C "$T3_STAGING" --strip-components=1 2>&1)"; then + # Some tars (BusyBox) gunzip internally, so gzip is only blamed once extraction fails. + if command -v gzip >/dev/null 2>&1; then printf '%s\\n' "$T3_TAR_ERROR" >&2 + else printf 'Remote host needs gzip to unpack %s. Install gzip on it and try again.\\n' "$T3_ARCHIVE" >&2 + fi + exit 1 + fi rm -f "$T3_STAGING/$T3_ARCHIVE" "$T3_STAGING/SHA256SUMS" # Prove the binary runs here (libc, arch) before marking it ready, or every # later launch would exec a broken install instead of retrying. From f03db4fbef59c42113e37c32897bf4d25d31cb2b Mon Sep 17 00:00:00 2001 From: kirat02 <142911687+Kirat0201@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:41:19 +0530 Subject: [PATCH 012/367] fix(pi): blank extension dialog titles no longer fail the turn (#35271) Co-authored-by: Julius Marminge --- .../provider-pi/src/server/adapter.test.ts | 126 ++++++++++-------- packages/provider-pi/src/server/adapter.ts | 5 +- 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index 65f3cc6c4344..5d0c9db05db0 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -4115,32 +4115,35 @@ describe("PiAdapterV2", () => { }).pipe(Effect.scoped, Effect.provide(layerTest)), ); - it.effect("keeps an input dialog's title as the header above its placeholder", () => - Effect.gen(function* () { - const fake = yield* makeFakePi; - const { runtime, takeEvent } = yield* openRuntime(fake); - yield* runtime.ensureThread({ - threadId: THREAD_ID, - modelSelection: modelSelection("default"), - runtimePolicy, - }); - yield* fake.emit({ - type: "extension_ui_request", - id: "ui-input-placeholder", - method: "input", - title: "Name", - placeholder: "Your name", - }); - const event = yield* takeEvent( - (event) => - event.type === "turn_item.updated" && event.turnItem.type === "user_input_request", - ); - if (event.type !== "turn_item.updated" || event.turnItem.type !== "user_input_request") { - return assert.fail("expected a user input request"); - } - assert.equal(event.turnItem.questions[0]?.header, "Name"); - assert.equal(event.turnItem.questions[0]?.question, "Your name"); - }).pipe(Effect.scoped, Effect.provide(layerTest)), + it.effect.each(["Name", " "])( + "keeps input placeholder text answerable with title '%s'", + (title) => + Effect.gen(function* () { + const fake = yield* makeFakePi; + const { runtime, takeEvent } = yield* openRuntime(fake); + yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + yield* fake.emit({ + type: "extension_ui_request", + id: "ui-input-placeholder", + method: "input", + title, + placeholder: "Your name", + }); + const event = yield* takeEvent( + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "user_input_request", + ); + if (event.type !== "turn_item.updated" || event.turnItem.type !== "user_input_request") { + return assert.fail("expected a user input request"); + } + const question = decodeUserInputQuestions(event.turnItem.questions)[0]!; + assert.equal(question.header, title.trim() ? title : "Question"); + assert.equal(question.question, "Your name"); + }).pipe(Effect.scoped, Effect.provide(layerTest)), ); it.effect("emits a valid empty select option and returns its original value", () => @@ -4186,39 +4189,44 @@ describe("PiAdapterV2", () => { }).pipe(Effect.scoped, Effect.provide(layerTest)), ); - it.effect("asks an input dialog's title when its placeholder is blank", () => - Effect.gen(function* () { - const fake = yield* makeFakePi; - const { runtime, takeEvent } = yield* openRuntime(fake); - yield* runtime.ensureThread({ - threadId: THREAD_ID, - modelSelection: modelSelection("default"), - runtimePolicy, - }); - const title = "Which database?\n1. Postgres\n2. Type something.\nType your answer:"; - yield* fake.emit({ - type: "extension_ui_request", - id: "ui-input", - method: "input", - title, - placeholder: "", - }); - const event = yield* takeEvent( - (event) => - event.type === "turn_item.updated" && event.turnItem.type === "user_input_request", - ); - if (event.type !== "turn_item.updated" || event.turnItem.type !== "user_input_request") { - return assert.fail("expected a user input request"); - } - assert.equal(decodeUserInputQuestions(event.turnItem.questions)[0]?.header, "Question"); - assert.equal(decodeUserInputQuestions(event.turnItem.questions)[0]?.question, title); - yield* runtime.respondToRuntimeRequest({ - requestId: event.turnItem.requestId, - answers: { "ui-input": "SQLite" }, - }); - const response = yield* fake.takeRequest("extension_ui_response"); - assert.equal(response["value"], "SQLite"); - }).pipe(Effect.scoped, Effect.provide(layerTest)), + it.effect.each(["Which database?\n1. Postgres\n2. Type something.\nType your answer:", " "])( + "asks an input dialog's title when its placeholder is blank, with title '%s'", + (title) => + Effect.gen(function* () { + const fake = yield* makeFakePi; + const { runtime, takeEvent } = yield* openRuntime(fake); + const providerThread = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + yield* startTurn(runtime, providerThread); + yield* fake.takeRequest("prompt"); + yield* fake.emit({ + type: "extension_ui_request", + id: "ui-input", + method: "input", + title, + placeholder: "", + }); + const event = yield* takeEvent( + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "user_input_request", + ); + if (event.type !== "turn_item.updated" || event.turnItem.type !== "user_input_request") { + return assert.fail("expected a user input request"); + } + const question = decodeUserInputQuestions(event.turnItem.questions)[0]!; + assert.equal(question.header, "Question"); + assert.equal(question.question, title.trim() ? title : "Question"); + yield* runtime.respondToRuntimeRequest({ + requestId: event.turnItem.requestId, + answers: { "ui-input": "SQLite" }, + }); + const response = yield* fake.takeRequest("extension_ui_response"); + assert.equal(response["value"], "SQLite"); + assert.isUndefined(response["cancelled"]); + }).pipe(Effect.scoped, Effect.provide(layerTest)), ); it.effect("keeps the complete editor prefill editable and returns exact edited text", () => diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index c237095e5e16..65f2a778dd32 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -3682,10 +3682,11 @@ function piQuestion( const detail = [recordString(event, "message"), recordString(event, "placeholder")].find( (text) => text !== undefined && text.trim().length > 0, ); + const questionTitle = title.trim() ? title : "Question"; return { id: questionId, - header: detail === undefined ? "Question" : title, - question: detail ?? title, + header: detail === undefined ? "Question" : questionTitle, + question: detail ?? questionTitle, ...(method === "editor" ? { initialAnswer: recordString(event, "prefill") ?? "" } : {}), options, }; From c5f5efdbd2ce025fd5f10fae01d6b97c44e42f3e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:19:26 -0800 Subject: [PATCH 013/367] fix(server): text-generation fallback keeps the thread default's custom model (#35391) --- .../CodexTextGeneration.test.ts | 18 +++++---- .../src/textGeneration/TextGeneration.ts | 39 ++++++++++--------- packages/shared/src/serverSettings.test.ts | 19 +++++++++ packages/shared/src/serverSettings.ts | 32 ++++++++------- 4 files changed, 69 insertions(+), 39 deletions(-) diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index b67d3f0ac583..389a6458677e 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -1,5 +1,4 @@ import { - DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER, ProjectId, ProviderDriverKind, type ModelSelection, @@ -559,6 +558,11 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima model: "chat-model", options: [{ id: "effort", value: "high" }], }; + const threadDefault = { + instanceId: fallback, + model: "z-ai/glm-5.3-flash", + options: [{ id: "thinking", value: false }], + }; const calls: ModelSelection[] = []; const instances: ProviderInstance[] = [primary, fallback].map((instanceId) => { const driver = ProviderDriverKind.make(instanceId === primary ? "codex" : "claudeAgent"); @@ -627,8 +631,12 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima }), ServerSettings.layerTest({ providerInstances: { - [fallback]: { driver: ProviderDriverKind.make("claudeAgent"), config: {} }, + [fallback]: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { customModels: [threadDefault.model] }, + }, }, + defaultModelSelection: threadDefault, ...(primaryState === "not installed, fallback disabled for project" ? { projectSettingsOverrides: { @@ -660,11 +668,7 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima primaryState === "not installed, fallback disabled for project" || primaryState === "signed out, fallback failed" ? selection - : { - instanceId: fallback, - model: - DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[ProviderDriverKind.make("claudeAgent")], - }, + : threadDefault, ]); }), ); diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index 3042c97191cb..b89e264db8d5 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -9,10 +9,7 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import { exhaustedUsageWindow } from "@t3tools/shared/usageLimits"; -import { - defaultTextGenerationModel, - fallbackTextGenerationModelSelection, -} from "@t3tools/shared/serverSettings"; +import { fallbackTextGenerationModelSelection } from "@t3tools/shared/serverSettings"; import * as ServerSettings from "../serverSettings.ts"; import type { ProviderTextGeneration } from "@t3tools/provider-core/server/textGeneration"; import { TextGenerationError } from "@t3tools/contracts"; @@ -125,21 +122,25 @@ export const make = Effect.gen(function* () { ), ); selection = - fallbackTextGenerationModelSelection(settings.providerInstances, (instanceId, driver) => { - const provider = snapshots.find((candidate) => candidate.instanceId === instanceId); - return ( - instanceId !== selection.instanceId && - provider?.enabled === true && - resolveProjectProviderInstanceEnabled(settings, projectId ?? null, instanceId) && - provider.installed && - provider.auth.status !== "unauthenticated" && - provider.status !== "error" && - isProviderAvailable(provider) && - isProviderTextGenerationCapable(provider) && - (operation !== "generateReviewGuide" || INVESTIGATING_DRIVERS.has(driver)) && - !exhaustedUsageWindow(provider, now, defaultTextGenerationModel(driver)) - ); - }) ?? selection; + fallbackTextGenerationModelSelection( + settings.providerInstances, + (instanceId, driver, model) => { + const provider = snapshots.find((candidate) => candidate.instanceId === instanceId); + return ( + instanceId !== selection.instanceId && + provider?.enabled === true && + resolveProjectProviderInstanceEnabled(settings, projectId ?? null, instanceId) && + provider.installed && + provider.auth.status !== "unauthenticated" && + provider.status !== "error" && + isProviderAvailable(provider) && + isProviderTextGenerationCapable(provider) && + (operation !== "generateReviewGuide" || INVESTIGATING_DRIVERS.has(driver)) && + !exhaustedUsageWindow(provider, now, model) + ); + }, + { defaultModelSelection: settings.defaultModelSelection }, + ) ?? selection; } if (textGeneratorCannotRun && selection.instanceId === textGeneratorId) { return yield* new TextGenerationError({ diff --git a/packages/shared/src/serverSettings.test.ts b/packages/shared/src/serverSettings.test.ts index 354260ea3cfb..347e6c746648 100644 --- a/packages/shared/src/serverSettings.test.ts +++ b/packages/shared/src/serverSettings.test.ts @@ -38,6 +38,25 @@ describe("fallbackTextGenerationModelSelection", () => { expect( fallbackTextGenerationModelSelection({}, (instanceId) => instanceId === "opencode"), ).toEqual({ instanceId: "opencode", model: "opencode/big-pickle" }); + const threadDefault = { + instanceId: ProviderInstanceId.make("opencode"), + model: "opencode-go/glm-5.3", + }; + const providerInstances = { + [threadDefault.instanceId]: { + driver: ProviderDriverKind.make("opencode"), + config: { customModels: [threadDefault.model] }, + }, + }; + // An exhausted Go quota leaves the custom model unusable but Big Pickle still works. + expect( + fallbackTextGenerationModelSelection( + providerInstances, + (instanceId, _driver, model) => + instanceId === "opencode" && !model.startsWith("opencode-go/"), + { defaultModelSelection: threadDefault }, + ), + ).toEqual({ instanceId: "opencode", model: "opencode/big-pickle" }); }); it("replaces and clears the fallback without carrying model options", () => { diff --git a/packages/shared/src/serverSettings.ts b/packages/shared/src/serverSettings.ts index 24c418decfa2..24b8d8e39979 100644 --- a/packages/shared/src/serverSettings.ts +++ b/packages/shared/src/serverSettings.ts @@ -73,9 +73,21 @@ const TEXT_GENERATION_FALLBACK_DRIVERS = [ /** Configured instances come after every default instance, so they only win when no default qualifies. */ export function fallbackTextGenerationModelSelection( providerInstances: ServerSettings["providerInstances"], - isEnabled: (instanceId: ProviderInstanceId, driver: ProviderDriverKind) => boolean, + isEnabled: (instanceId: ProviderInstanceId, driver: ProviderDriverKind, model: string) => boolean, options?: { readonly defaultModelSelection?: ModelSelection | null }, ): ModelSelection | null { + const threadDefault = options?.defaultModelSelection; + // The thread default's custom model wins on its instance; the driver default still serves when that model is unusable. + const selectionsFor = (instanceId: ProviderInstanceId, driver: ProviderDriverKind) => { + const config = providerInstances[instanceId]?.config as { customModels?: unknown } | undefined; + const driverDefault = { instanceId, model: defaultTextGenerationModel(driver) }; + return threadDefault?.instanceId === instanceId && + readCustomModelEntries(config?.customModels).some( + (entry) => entry.slug === threadDefault.model, + ) + ? [threadDefault, driverDefault] + : [driverDefault]; + }; const candidates = [ ...TEXT_GENERATION_FALLBACK_DRIVERS.map( (driver) => [defaultInstanceIdForDriver(driver), driver] as const, @@ -86,21 +98,15 @@ export function fallbackTextGenerationModelSelection( : [], ), ]; - const match = candidates.find(([instanceId, driver]) => isEnabled(instanceId, driver)); - if (match === undefined) return null; - const [instanceId, driver] = match; - const threadDefault = options?.defaultModelSelection; - const config = providerInstances[instanceId]?.config as { customModels?: unknown } | undefined; - if ( - threadDefault?.instanceId === instanceId && - readCustomModelEntries(config?.customModels).some((entry) => entry.slug === threadDefault.model) - ) { - return threadDefault; + for (const [instanceId, driver] of candidates) { + for (const selection of selectionsFor(instanceId, driver)) { + if (isEnabled(instanceId, driver, selection.model)) return selection; + } } - return { instanceId, model: defaultTextGenerationModel(driver) }; + return null; } -export function defaultTextGenerationModel(driver: ProviderDriverKind): string { +function defaultTextGenerationModel(driver: ProviderDriverKind): string { return ( DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[driver] ?? DEFAULT_MODEL_BY_PROVIDER[driver] ?? From e96c96eafe2353734e41d5eba473056b2da85480 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:19:41 -0800 Subject: [PATCH 014/367] fix(server): storage cleanup stops holding usage-limited threads too overdue to auto-resume (#35397) --- .../src/orchestration-v2/UsageLimitRecoveryWorker.ts | 2 +- apps/server/src/storageCleanup.test.ts | 7 +++++++ apps/server/src/storageCleanup.ts | 7 +++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts index 985815bd73eb..3992d46e6bf9 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts @@ -12,7 +12,7 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; -const MAX_OVERDUE_ARM_MS = 24 * 60 * 60 * 1000; +export const MAX_OVERDUE_ARM_MS = 24 * 60 * 60 * 1000; export function limitRecoveryCommand( thread: ProjectionStore.ProjectionLimitRecoveryCandidate, diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index b933895c4de1..e7220718f15a 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -205,6 +205,13 @@ describe("V2 storage cleanup eligibility", () => { resetIn: -1_000, idle: false, }, + { + name: "never armed because its reset is days overdue", + armed: false, + autoResumeSetting: true, + resetIn: -5 * DAY_MS, + idle: true, + }, { name: "not armed with recovery settings off", armed: false, diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index d044c2f7bdc4..673c121612c9 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -42,6 +42,7 @@ import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts"; +import { MAX_OVERDUE_ARM_MS } from "./orchestration-v2/UsageLimitRecoveryWorker.ts"; import { forkParked } from "./serverActivation.ts"; import * as Settings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; @@ -182,10 +183,12 @@ function heldForLimitRecovery( // An auto-resume holds until its continuation replaces the failed run. return recovery.autoResume || (recovery.snooze === true && resetMs > now); // The recovery worker arms this failure on a later tick; hold it until then. - // Like the worker, skip a reset that had already passed when the run ended. + // Like the worker, skip a reset that had already passed when the run ended + // or that is too overdue to arm. if ( !Number.isFinite(resetMs) || - resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) + resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) || + now - resetMs > MAX_OVERDUE_ARM_MS ) return false; return settings.autoResumeLimitedThreads || (settings.snoozeLimitedThreads && resetMs > now); From 167d41c09449d2e7554d663bae78646a4ff300fb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:19:54 -0800 Subject: [PATCH 015/367] fix(web): a numeric author or labels in the URL no longer crashes Pull Requests (#35376) --- .../pullRequest/pullRequestListPreferences.ts | 4 ++-- .../pullRequest/pullRequestsRouteSearch.test.ts | 5 +++++ apps/web/src/routes/_chat.pull-requests.tsx | 15 ++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/pullRequest/pullRequestListPreferences.ts b/apps/web/src/components/pullRequest/pullRequestListPreferences.ts index 649515eaf9a6..0c22c528226e 100644 --- a/apps/web/src/components/pullRequest/pullRequestListPreferences.ts +++ b/apps/web/src/components/pullRequest/pullRequestListPreferences.ts @@ -29,8 +29,8 @@ export interface PullRequestListPreferences { readonly draft?: "only" | "hide"; readonly review?: NonNullable; readonly checks?: NonNullable; - readonly author?: string; - readonly labels?: ReadonlyArray; + readonly author?: string | undefined; + readonly labels?: ReadonlyArray | undefined; readonly sort?: PullRequestListSort; } diff --git a/apps/web/src/components/pullRequest/pullRequestsRouteSearch.test.ts b/apps/web/src/components/pullRequest/pullRequestsRouteSearch.test.ts index c2c9de1f9464..716a047655f2 100644 --- a/apps/web/src/components/pullRequest/pullRequestsRouteSearch.test.ts +++ b/apps/web/src/components/pullRequest/pullRequestsRouteSearch.test.ts @@ -36,4 +36,9 @@ describe("pull requests route search", () => { expect(matchedSearch("/pull-requests?q=fix")?.q).toBe("fix"); expect(matchedSearch("/pull-requests?q=%224089%22")?.q).toBe("4089"); }); + + it("drops an author or labels the URL parser turned into a number", () => { + expect(matchedSearch("/pull-requests?author=4089")?.author).toBeUndefined(); + expect(matchedSearch("/pull-requests?labels=4089")?.labels).toBeUndefined(); + }); }); diff --git a/apps/web/src/routes/_chat.pull-requests.tsx b/apps/web/src/routes/_chat.pull-requests.tsx index 95b72f57992e..8090e3a1459b 100644 --- a/apps/web/src/routes/_chat.pull-requests.tsx +++ b/apps/web/src/routes/_chat.pull-requests.tsx @@ -8,7 +8,7 @@ import { PullRequestsPage } from "../components/pullRequest/PullRequestsPage"; const MAX_SEARCH_LABEL_CANDIDATES = 100; -function pullRequestSearchLabels(raw: unknown): Partial> { +function pullRequestSearchLabels(raw: unknown): PullRequestsSearch["labels"] { const values = (Array.isArray(raw) ? raw : typeof raw === "string" ? [raw] : []).slice( 0, MAX_SEARCH_LABEL_CANDIDATES, @@ -24,7 +24,7 @@ function pullRequestSearchLabels(raw: unknown): Partial Date: Thu, 12 Nov 2026 22:51:15 +0530 Subject: [PATCH 016/367] fix(web): bare URLs end at Chinese and Japanese punctuation (#35267) Co-authored-by: Julius Marminge --- packages/shared/src/markdownPipeline.test.ts | 173 +++++++++++++++- packages/shared/src/markdownPipeline.ts | 205 ++++++++++++++++++- 2 files changed, 375 insertions(+), 3 deletions(-) diff --git a/packages/shared/src/markdownPipeline.test.ts b/packages/shared/src/markdownPipeline.test.ts index 06ac60958e80..2319fa447598 100644 --- a/packages/shared/src/markdownPipeline.test.ts +++ b/packages/shared/src/markdownPipeline.test.ts @@ -3,7 +3,7 @@ import { type PluggableList, unified } from "unified"; import rehypeRaw from "rehype-raw"; import remarkParse from "remark-parse"; import remarkRehype from "remark-rehype"; -import type { Root, RootContent } from "mdast"; +import type { Link, Root, RootContent } from "mdast"; import { CHAT_MARKDOWN_REHYPE_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS, @@ -15,7 +15,7 @@ import { it.each([ ["default", CHAT_MARKDOWN_REMARK_PLUGINS], ["with breaks", CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS], -] as const)("preserves trailing underscores and tildes in bare URLs (%s)", (_, plugins) => { +] as const)("preserves bare URL boundaries (%s)", (_, plugins) => { const processor = unified().use(remarkParse).use(plugins); const shape = (node: Root | RootContent): string => { if (node.type === "text") return node.value; @@ -90,6 +90,55 @@ it.each([ ["https\\://x.com/dimasike_", link("https://x.com/dimasike_")], ["https://x.com/a\\_", link("https://x.com/a\\_")], ["https://x.com/a_", `${link("https://x.com/a_")};`], + [ + "可以看了:**http://localhost:18780**(已重启)。", + `可以看了:**${link("http://localhost:18780")}**(已重启)。`, + ], + ["打开 http://localhost:18780,然后刷新", `打开 ${link("http://localhost:18780")},然后刷新`], + ["URL:https://example.com/a。下一句", `URL:${link("https://example.com/a")}。下一句`], + ["(预览:http://localhost:18780)", `(预览:${link("http://localhost:18780")})`], + [ + "能打开 http://localhost:3000?见 http://localhost:3001;「www.example.com」", + `能打开 ${link("http://localhost:3000")}?见 ${link("http://localhost:3001")};「${link("www.example.com")}」`, + ], + [ + "__http://localhost:18780__,*http://localhost:3000*,~~http://localhost:3001~~。", + `**${link("http://localhost:18780")}**,*${link("http://localhost:3000")}*,~~${link("http://localhost:3001")}~~。`, + ], + [ + "foo**http://localhost:18780**(bar)***http://localhost:3000***。", + `foo**${link("http://localhost:18780")}**(bar)***${link("http://localhost:3000")}***。`, + ], + [ + "http://a.com。http://b.com,www.c.com、person@example.com。", + `${link("http://a.com")}。${link("http://b.com")},${link("www.c.com")}、[person@example.com](mailto:person@example.com)。`, + ], + [ + "**http://localhost:18780**(重启)。 https://example.com/a。结束", + `**${link("http://localhost:18780")}**(重启)。 ${link("https://example.com/a")}。结束`, + ], + ["# **http://localhost:3000**,刷新", `**${link("http://localhost:3000")}**,刷新`], + [ + "| a |\n| - |\n| **http://localhost:3000**,刷新 |", + `a**${link("http://localhost:3000")}**,刷新`, + ], + ["www.example.com/preview。下一句", `${link("www.example.com/preview")}。下一句`], + ["https://example.com/a_。下一句", `${link("https://example.com/a_")}。下一句`], + ["https://example.com/a.,下一句", `${link("https://example.com/a")}.,下一句`], + [ + "https://example.com/search?q=hello,world", + `${link("https://example.com/search?q=hello")},world`, + ], + ["https://example.com/日本語,続き", link("https://example.com/日本語,続き")], + [ + "https://zh.wikipedia.org/wiki/北京(城市)", + link("https://zh.wikipedia.org/wiki/北京(城市)"), + ], + ["https://example.com/a%EF%BC%8Cb", link("https://example.com/a%EF%BC%8Cb")], + ["", link("https://example.com/日本語,続き")], + ["[記事](https://example.com/a,b)", "[記事](https://example.com/a,b)"], + ["[http://example.com/a,b](http://example.com/a,b)", link("http://example.com/a,b")], + ["`https://example.com/a。次`", "`https://example.com/a。次`"], [ "https://x.com/a_ https://x.com/b_", `${link("https://x.com/a_")} ${link("https://x.com/b_")}`, @@ -120,6 +169,126 @@ it.each([ } }); +it("ends bare URLs at CJK punctuation with source positions", () => { + const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS); + const parse = (markdown: string) => processor.runSync(processor.parse(markdown), markdown); + const at = (line: number, column: number, offset: number) => ({ line, column, offset }); + const url = "http://localhost:18780"; + expect(parse(`第一行\n可以看了:**${url}**(已重启)。`)).toMatchObject({ + children: [ + { + children: [ + { type: "text", value: "第一行\n可以看了:", position: { end: at(2, 6, 9) } }, + { + type: "strong", + position: { start: at(2, 6, 9), end: at(2, 32, 35) }, + children: [ + { + type: "link", + url, + position: { start: at(2, 8, 11), end: at(2, 30, 33) }, + children: [{ type: "text", value: url, position: { end: at(2, 30, 33) } }], + }, + ], + }, + { + type: "text", + value: "(已重启)。", + position: { start: at(2, 32, 35), end: at(2, 38, 41) }, + }, + ], + }, + ], + }); + // Markup the URL swallowed comes back once the URL ends. + expect(parse(`打开 ${url},然后运行\`pnpm dev\``)).toMatchObject({ + children: [ + { + children: [ + { type: "text", value: "打开 " }, + { type: "link", url }, + { type: "text", value: ",然后运行" }, + { type: "inlineCode", value: "pnpm dev" }, + ], + }, + ], + }); + // Every URL in one paragraph ends at its mark, with positions on its own line. + const lines = Array.from({ length: 300 }, (_, index) => `接口:http://localhost:${index},`); + const [paragraph] = (parse(lines.join("\n")) as Root).children; + const links = + paragraph?.type === "paragraph" + ? paragraph.children.filter((node): node is Link => node.type === "link") + : []; + expect(links.map((node) => node.url)).toEqual( + lines.map((_, index) => `http://localhost:${index}`), + ); + const lastLine = lines.slice(0, -1).join("\n").length + 1; + expect(links.at(-1)).toMatchObject({ + position: { start: at(300, 4, lastLine + 3), end: at(300, 24, lastLine + 23) }, + }); + // Only a table unescapes `\|` in code, so that cell keeps its parse and the URL is cut. + const descendants = (node: Root | RootContent): RootContent[] => + "children" in node + ? (node.children as RootContent[]).flatMap((child) => [child, ...descendants(child)]) + : []; + const cell = descendants(parse("| a |\n| - |\n| http://example.com, `a\\|b` |") as Root); + expect(cell.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([ + "http://example.com", + ]); + expect(cell.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual(["a|b"]); + // A task item's checkbox is not part of its paragraph's text. + const task = descendants(parse("- [x] http://example.com,刷新") as Root); + expect(task.flatMap((node) => (node.type === "text" ? [node.value] : []))).toEqual([ + "http://example.com", + ",刷新", + ]); + // A list item's indentation stays out of a code span that crosses lines. + const code = descendants(parse("- `echo\n hello` http://example.com,结束") as Root); + expect(code.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([ + "http://example.com", + ]); + expect(code.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual([ + "echo\nhello", + ]); + // A list item's continuation line parses the same on its own. + expect(parse(`- 第一行\n **${url}**(已重启)`)).toMatchObject({ + children: [ + { + children: [ + { + children: [ + { + children: [ + { type: "text", value: "第一行\n" }, + { type: "strong", children: [{ type: "link", url }] }, + { type: "text", value: "(已重启)" }, + ], + }, + ], + }, + ], + }, + ], + }); + // A quote's continuation line does not, so the URL ends and the asterisks stay as text. + expect(parse(`> 第一行\n> **${url}**(已重启)`)).toMatchObject({ + children: [ + { + children: [ + { + children: [ + { type: "text", value: "第一行\n**" }, + { type: "link", url, position: { end: at(2, 27, 32) } }, + { type: "text", value: "**(已重启)", position: { start: at(2, 27, 32) } }, + ], + }, + ], + }, + ], + }); +}); + const insightRule = "─".repeat(20); const insightFence = (body: string, wrap = "") => [`${wrap}★ Insight ${insightRule}${wrap}`, body, `${wrap}${insightRule}${wrap}`].join("\n"); diff --git a/packages/shared/src/markdownPipeline.ts b/packages/shared/src/markdownPipeline.ts index 9ad3299809f0..e8042ee6d469 100644 --- a/packages/shared/src/markdownPipeline.ts +++ b/packages/shared/src/markdownPipeline.ts @@ -1,5 +1,14 @@ import { isLocalEditorFileUrl, LOCAL_FILE_LINK_SCHEMES } from "@t3tools/contracts"; -import type { Paragraph, PhrasingContent, Root, RootContent } from "mdast"; +import type { + Heading, + Link, + Paragraph, + PhrasingContent, + Root, + RootContent, + TableCell, + Text, +} from "mdast"; import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts"; import { type PluggableList, type Processor, type Transformer, unified } from "unified"; import remarkParse from "remark-parse"; @@ -236,6 +245,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = { export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [ remarkGfm, + remarkEndAutolinksAtCjkPunctuation, remarkKeepAutolinkSuffixes, remarkLinkAbsoluteFilePaths, remarkKeepWindowsPaths, @@ -250,6 +260,7 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [ export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [ remarkGfm, + remarkEndAutolinksAtCjkPunctuation, remarkKeepAutolinkSuffixes, remarkLinkAbsoluteFilePaths, remarkKeepWindowsPaths, @@ -453,6 +464,198 @@ function remarkKeepAutolinkSuffixes(): Transformer { }; } +// Punctuation and symbols from CJK Symbols and Punctuation and from Halfwidth and Fullwidth Forms. +const CJK_PUNCTUATION = /(?=[ -〿＀-￯])[\p{P}\p{S}]/u; + +/** + * GFM runs a bare URL on until whitespace, so Chinese and Japanese punctuation joins it, along with + * the rest of the sentence and any `**` that should have closed around it. A URL that is ASCII up to + * such a mark ends there. Unicode earlier in the URL means the mark may belong to it, so it stays. + */ +function cjkPunctuationIndex(link: Link): number { + const label = link.children.length === 1 ? link.children[0] : undefined; + if (label?.type !== "text" || !/^https?:\/\//i.test(link.url)) return -1; + if (link.url !== label.value && link.url !== `http://${label.value}`) return -1; + const index = label.value.search(/[^!-~]/u); + return index > 0 && CJK_PUNCTUATION.test(label.value.charAt(index)) ? index : -1; +} + +/** GFM's text pass links bare URLs without positions; its tokenizer's links span their URL. */ +function isBareLink(link: Link, source: string): boolean { + const start = link.position?.start.offset; + const end = link.position?.end.offset; + const label = link.children[0]; + return ( + !link.position || + (start !== undefined && + end !== undefined && + label?.type === "text" && + source.slice(start, end) === label.value) + ); +} + +/** Source offsets of the marks that tokenized bare URLs run past. */ +function cjkPunctuationOffsets(node: RootContent, source: string): number[] { + if (node.type === "link") { + const index = cjkPunctuationIndex(node); + const start = node.position?.start.offset; + return index > 0 && start !== undefined && isBareLink(node, source) ? [start + index] : []; + } + return "children" in node && node.type !== "linkReference" + ? (node.children as RootContent[]).flatMap((child) => cjkPunctuationOffsets(child, source)) + : []; +} + +const hasReference = (node: RootContent): boolean => + node.type === "linkReference" || + node.type === "imageReference" || + node.type === "footnoteReference" || + ("children" in node && (node.children as RootContent[]).some(hasReference)); + +/** How many of the ascending numbers are below `value`. */ +function countBelow(sorted: readonly number[], value: number): number { + let low = 0; + let high = sorted.length; + while (low < high) { + const middle = (low + high) >> 1; + if (sorted[middle]! < value) low = middle + 1; + else high = middle; + } + return low; +} + +/** + * Parses the block's text again with a tab before each mark. The tab ends the URL as a space would, + * so emphasis around the URL closes and later URLs and code get their own nodes. The tabs are then + * dropped and positions mapped back to the source. Returns false and leaves the block alone when its + * text would parse differently on its own, as a quote's continuation lines do. + */ +function reparseAtCjkPunctuation( + processor: Processor, + block: Paragraph | Heading | TableCell, + source: string, + offsets: number[], +): boolean { + // A task item's paragraph starts at its checkbox; its first child starts after it. + const from = block.children[0]?.position?.start; + const to = block.children.at(-1)?.position?.end; + if (from?.offset === undefined || to?.offset === undefined || hasReference(block)) return false; + const base = from.offset; + const text = source.slice(base, to.offset); + // A tab is the marker, a quote's `>` on a later line would read as a new quote, a list's + // indentation would join a code span, and only a table unescapes `\|` in code. + if ( + text.includes("\t") || + /[\r\n][ ]*>/.test(text) || + (/[\r\n] /.test(text) && text.includes("`")) || + (block.type === "tableCell" && text.includes("\\|")) + ) { + return false; + } + const marks = offsets.map((offset) => offset - base); + // Each pass finds the next URL in a run with no spaces, like `http://a,http://b,http://c`. + for (let attempt = 0; attempt < 16; attempt++) { + marks.sort((a, b) => a - b); + const tabs = marks.map((mark, index) => mark + index); + const unmarked = (offset: number) => offset - countBelow(tabs, offset); + const marked = [0, ...marks] + .map((start, index) => text.slice(start, marks[index] ?? text.length)) + .join("\t"); + const [paragraph, ...rest] = (processor.parse(marked) as Root).children; + if (paragraph?.type !== "paragraph" || rest.length > 0) return false; + const more = cjkPunctuationOffsets(paragraph, marked); + if (more.length > 0) { + marks.push(...more.map(unmarked)); + continue; + } + + let tabCount = 0; + const moved = new Set(); + const restore = (node: RootContent): boolean => { + for (const point of node.position ? [node.position.start, node.position.end] : []) { + const offset = point.offset; + if (offset === undefined || moved.has(point)) continue; + moved.add(point); + const tabsOnLine = countBelow(tabs, offset) - countBelow(tabs, offset - point.column + 1); + point.column = (point.line === 1 ? from.column - 1 : 0) + point.column - tabsOnLine; + point.line += from.line - 1; + point.offset = base + unmarked(offset); + } + if (node.type === "text") { + tabCount += node.value.split("\t").length - 1; + node.value = node.value.replaceAll("\t", ""); + } else if ( + ("value" in node && node.value.includes("\t")) || + ("url" in node && node.url.includes("\t")) + ) { + return false; + } + if (!("children" in node)) return true; + const children = node.children as RootContent[]; + if (!children.every(restore)) return false; + node.children = children.filter( + (child) => child.type !== "text" || child.value !== "", + ) as typeof node.children; + return true; + }; + if (!restore(paragraph) || tabCount !== tabs.length) return false; + block.children = paragraph.children; + return true; + } + return false; +} + +/** Ends the bare URLs a reparse could not, and those GFM's text pass linked after emphasis. */ +function splitAtCjkPunctuation(node: RootContent, source: string) { + if (!("children" in node) || node.type === "link" || node.type === "linkReference") return; + const children = node.children as RootContent[]; + for (let index = 0; index < children.length; index++) { + const child = children[index]!; + if (child.type !== "link") { + splitAtCjkPunctuation(child, source); + continue; + } + const at = isBareLink(child, source) ? cjkPunctuationIndex(child) : -1; + const label = child.children[0]; + if (at < 0 || label?.type !== "text") continue; + // GFM drops this punctuation from the end of a URL; remarkKeepAutolinkSuffixes restores `_` and `~`. + const length = label.value.slice(0, at).replace(/[!"'*,.:;?_~]+$/, "").length; + const tail: Text = { type: "text", value: label.value.slice(length) }; + const start = child.position?.start; + if (child.position && start?.offset !== undefined) { + const end = { + line: start.line, + column: start.column + length, + offset: start.offset + length, + }; + tail.position = { start: end, end: child.position.end }; + child.position = { start, end }; + label.position = { start, end }; + } + child.url = child.url.slice(0, child.url.length - label.value.length + length); + label.value = label.value.slice(0, length); + children.splice(index + 1, 0, tail); + index++; + } +} + +function remarkEndAutolinksAtCjkPunctuation(this: Processor): Transformer { + return (tree, file) => { + const source = String(file); + if (!CJK_PUNCTUATION.test(source)) return; + const visit = (node: RootContent) => { + if (node.type === "paragraph" || node.type === "heading" || node.type === "tableCell") { + const offsets = cjkPunctuationOffsets(node, source); + if (offsets.length > 0) reparseAtCjkPunctuation(this, node, source, offsets); + splitAtCjkPunctuation(node, source); + } else if ("children" in node) { + (node.children as RootContent[]).forEach(visit); + } + }; + tree.children.forEach(visit); + }; +} + function remarkLinkAbsoluteFilePaths(): Transformer { return (tree) => { // Raw `` tags are sibling html nodes, so text inside one is not under a link node. From 88bf393b30d031ad998651eb66f4df804442399c Mon Sep 17 00:00:00 2001 From: Sidhartha S <247517582+sidhartha1s@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:51:30 +0530 Subject: [PATCH 017/367] fix(server): @ file search finds filenames containing literal braces (#35239) Co-authored-by: sidhartha1s Co-authored-by: Julius Marminge --- .../src/workspace/WorkspaceEntries.test.ts | 24 ++++++++++--- .../src/workspace/WorkspaceSearchIndex.ts | 35 ++++++++++++------- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 62d6ab0a63b3..87cab4a2c88c 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -457,17 +457,31 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); - it.effect("matches square brackets in the query literally", () => + it.effect("matches glob metacharacters in the query literally", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-brackets-" }); yield* writeTextFile(cwd, "src/app/game/[id]/index.tsx"); + yield* writeTextFile(cwd, "src/foo{bar}.ts"); + yield* writeTextFile(cwd, "src/main.ts"); + yield* writeTextFile(cwd, "src/notes.md"); - for (const query of ["game/[i", "game/[id]"]) { - const result = yield* searchWorkspaceEntries({ cwd, query, limit: 10 }); - expect(result.entries.map((entry) => entry.path)).toContain( - "src/app/game/[id]/index.tsx", + const search = (query: string) => + searchWorkspaceEntries({ cwd, query, limit: 10 }).pipe( + Effect.map((result) => result.entries.map((entry) => entry.path)), ); + + for (const query of ["game/[i", "game/[id]"]) { + expect(yield* search(query)).toContain("src/app/game/[id]/index.tsx"); } + expect(yield* search("foo{bar}.ts")).toContain("src/foo{bar}.ts"); + expect((yield* search("*.{ts,tsx}")).toSorted()).toEqual([ + "src/app/game/[id]/index.tsx", + "src/foo{bar}.ts", + "src/main.ts", + ]); + expect((yield* search("*.{ts}")).toSorted()).toEqual(["src/foo{bar}.ts", "src/main.ts"]); + expect(yield* search("src/{main}.ts")).toEqual(["src/main.ts"]); + expect(yield* search("src/{foo{bar},notes}.md")).toEqual(["src/notes.md"]); }), ); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 11a19e183986..23d17759c9fc 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -447,32 +447,43 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }, ); - const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( - "WorkspaceSearchIndex.search", - )(function* (rawQuery, limit, kind, imageOnly) { - // fff parses any word containing "[" as a glob and has no escape for it; fuzzy - // matching is subsequence-based, so dropping "[" still matches "[id]" folders. - const query = rawQuery.replaceAll("[", ""); + const searchNames = Effect.fn("WorkspaceSearchIndex.searchNames")(function* ( + query: string, + limit: number, + kind: ProjectEntryKind | undefined, + imageOnly: boolean | undefined, + ) { const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); - const incompleteBeforeQuery = hasIncompleteInitialScan(); if (kind === "file" || imageOnly) { const result = yield* runSearch(query, pageSize, "fileSearch", () => finder.fileSearch(query, { pageSize }), ); - const mapped = mapFileSearchResult(result, limit, imageOnly); - return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; + return mapFileSearchResult(result, limit, imageOnly); } if (kind === "directory") { const result = yield* runSearch(query, pageSize, "directorySearch", () => finder.directorySearch(query, { pageSize }), ); - const mapped = mapDirectorySearchResult(result, limit); - return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; + return mapDirectorySearchResult(result, limit); } const result = yield* runSearch(query, pageSize, "mixedSearch", () => finder.mixedSearch(query, { pageSize }), ); - const mapped = mapMixedSearchResult(result, limit); + return mapMixedSearchResult(result, limit); + }); + + const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( + "WorkspaceSearchIndex.search", + )(function* (rawQuery, limit, kind, imageOnly) { + // fff parses any word containing "[" or "{" as a glob and has no escape for them; fuzzy + // matching is subsequence-based, so dropping them still matches "[id]" and "a{b}" paths. + // "{" is dropped only when the query finds nothing as a glob, so "*.{ts,tsx}" keeps working. + const query = rawQuery.replaceAll("[", ""); + const incompleteBeforeQuery = hasIncompleteInitialScan(); + let mapped = yield* searchNames(query, limit, kind, imageOnly); + if (mapped.entries.length === 0 && query.includes("{")) { + mapped = yield* searchNames(query.replaceAll("{", ""), limit, kind, imageOnly); + } return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; }); From 2a1e43e89ebe9cfe75f07056f42469e2b1bb3a07 Mon Sep 17 00:00:00 2001 From: Joseph Vidal <135430578+josephv123@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:23:36 -0500 Subject: [PATCH 018/367] fix(server): t3_thread_search says which thread is the caller's (#35377) Co-authored-by: Julius Marminge --- apps/server/src/mcp/toolkits/core.test.ts | 58 +++++++++++++++++++ .../src/mcp/toolkits/thread/handlers.ts | 1 + apps/server/src/mcp/toolkits/thread/tools.ts | 1 + 3 files changed, 60 insertions(+) diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 99937d68b80d..9682a45f5e59 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -10,6 +10,7 @@ import { EnvironmentId, NodeId, OrchestratorMcpThreadReadResult, + ProjectId, ProviderDriverKind, ProviderInstanceId, RunId, @@ -59,6 +60,7 @@ import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; import { dispatchFailure } from "../threadAccess.ts"; import * as EnvironmentHandlers from "./environment/handlers.ts"; import * as AttachmentHandlers from "./attachment/handlers.ts"; +import { ThreadToolkit } from "./thread/tools.ts"; import * as PeerForwarding from "../../peer/PeerForwarding.ts"; import * as PeerLinkRequests from "../../peer/PeerLinkRequests.ts"; import * as PeerLinks from "../../peer/PeerLinks.ts"; @@ -76,6 +78,9 @@ import { htmlRenderFromToolItem } from "@t3tools/shared/toolOutput"; const decodeMcpAttachmentInput = Schema.decodeUnknownEffect(McpAttachmentInput); const decodeThreadReadResult = Schema.decodeUnknownEffect(OrchestratorMcpThreadReadResult); +const decodeThreadSearchResult = Schema.decodeUnknownEffect( + ThreadToolkit.tools.t3_thread_search.successSchema, +); const layerThreadToolkit = McpHttpServer.layerThreadToolkit.pipe( Layer.provide(Layer.mock(ThreadSearch.ThreadSearch)({})), @@ -589,6 +594,59 @@ it.effect("a client caller targets any thread within its ceiling and cannot act ), ); +it.effect("search names the calling thread, so an agent can tell its own match apart", () => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const search = (invocation: McpInvocationContext.McpInvocationScope) => + server + .callTool({ name: "t3_thread_search", arguments: { query: "release notes" } }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + const fromThread = yield* search(scope); + expect(fromThread.structuredContent).toMatchObject({ + currentThreadId: threadId, + matches: [{ threadId }, { threadId: "earlier-thread" }], + }); + + const fromClient = yield* search(clientScope("auto")); + expect(fromClient.structuredContent).toMatchObject({ currentThreadId: null }); + + // A linked environment on an older version answers without the field. + const fromOlderPeer = yield* decodeThreadSearchResult({ matches: [] }); + expect(fromOlderPeer).toEqual({ matches: [] }); + }).pipe( + Effect.provide( + McpHttpServer.layerThreadToolkit.pipe( + Layer.provideMerge(McpHttpServer.layerInProcessMcpServer), + Layer.provide(NodeCrypto.layer), + Layer.provide(McpToolAccessTestkit.liveThreadsLayer), + Layer.provide( + Layer.mock(ThreadSearch.ThreadSearch)({ + searchThreads: () => + Effect.succeed({ + matches: [threadId, ThreadId.make("earlier-thread")].map((id) => ({ + threadId: id, + projectId: ProjectId.make("project:mcp-test"), + title: "Release notes", + source: "title" as const, + snippet: "", + messageCreatedAt: null, + })), + }), + }), + ), + Layer.provide(Layer.mock(ScheduledTaskService.ScheduledTaskService)({})), + Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), + Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), + Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})), + ), + ), + ), +); + it.effect("a read-only client reads threads and is refused every write before it runs", () => Effect.gen(function* () { const server = yield* McpServer.McpServer; diff --git a/apps/server/src/mcp/toolkits/thread/handlers.ts b/apps/server/src/mcp/toolkits/thread/handlers.ts index 32e45b5d0a1c..62023dd986b5 100644 --- a/apps/server/src/mcp/toolkits/thread/handlers.ts +++ b/apps/server/src/mcp/toolkits/thread/handlers.ts @@ -132,6 +132,7 @@ export const layer = McpToolAccess.toLayer(ThreadToolkit, { .searchThreads({ ...input, projectId: input.projectId ?? caller?.projectId }) .pipe(Effect.mapError(unavailable)); return { + currentThreadId: scope.thread?.threadId ?? null, matches: matches.map((match) => ({ ...match, link: McpInvocationContext.threadLinkFor(scope, match.threadId, match.title), diff --git a/apps/server/src/mcp/toolkits/thread/tools.ts b/apps/server/src/mcp/toolkits/thread/tools.ts index c6670c4e953d..750bf15afc01 100644 --- a/apps/server/src/mcp/toolkits/thread/tools.ts +++ b/apps/server/src/mcp/toolkits/thread/tools.ts @@ -199,6 +199,7 @@ const ThreadSearchTool = Tool.make("t3_thread_search", { projectId: Schema.optional(ProjectId), }), success: Schema.Struct({ + currentThreadId: Schema.optional(Schema.NullOr(ThreadId)), matches: Schema.Array( Schema.Struct({ threadId: OrchestrationThreadSearchMatch.fields.threadId, From a909ea423cbbea68db4b3eb4c6017e06ad6dd50d Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 12 Nov 2026 09:28:37 -0800 Subject: [PATCH 019/367] perf(server): older history pages load without indexing the whole thread (#34573) Co-Authored-By: Claude Fable 5.2 --- .../orchestration-v2/ProjectionStore.test.ts | 490 ++++++++++- .../src/orchestration-v2/ProjectionStore.ts | 808 +++++++++++++++--- .../threadHistoryPaging.test.ts | 32 + .../orchestration-v2/threadHistoryPaging.ts | 54 +- 4 files changed, 1245 insertions(+), 139 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 3f48cbf247fc..fa670c448c89 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -3651,14 +3651,16 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const threadId = ThreadId.make("thread:bounded-sql-history"); // The raw SQL edits below stand in for committed writes, which always append a thread event. let revision = 0; - const historyPage = (...args: Parameters) => + const committed = (read: Effect.Effect) => sql`INSERT INTO orchestration_events ( event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at, actor_kind, payload_json, metadata_json, application_event_version ) VALUES ( ${`event:bounded-sql-history:revision:${++revision}`}, 'thread', ${threadId}, ${revision}, 'turn-item.updated', ${nowIso}, 'user', '{}', '{}', 2 - )`.pipe(Effect.andThen(projectionStore.getThreadHistoryPage(...args))); + )`.pipe(Effect.andThen(read)); + const historyPage = (...args: Parameters) => + committed(projectionStore.getThreadHistoryPage(...args)); yield* projectionStore.apply({ id: EventId.make("event:bounded-sql-history:thread"), type: "thread.created", @@ -4050,6 +4052,134 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { WHERE thread_id = ${threadId} AND ordinal IN (500, 580) `; + // Older pages and timeline pages read the rows near the page, not the whole history. A + // full index that another read left cached gives the same page. + const pageStatements: Array]> = []; + const recordPageStatements = (read: Effect.Effect) => + committed( + read.pipe( + Effect.provideService(Statement.CurrentTransformer, (statement) => { + pageStatements.push(statement.compile()); + return Effect.succeed(statement); + }), + ), + ); + const olderCursor = encodeThreadHistoryCursor({ + snapshotSequence: 0, + sourceThreadId: threadId, + sourceItemId: TurnItemId.make("turn-item:bounded-sql-history:990"), + position: 989, + }); + const ordinalsFrom = (first: number, length: number) => + Array.from({ length }, (_, index) => first + index); + for (const workMode of [undefined, "deferred"] as const) { + const read = (throughEntryId?: string) => + projectionStore.getThreadHistoryPage( + threadId, + olderCursor, + throughEntryId, + false, + workMode, + ); + const page = yield* recordPageStatements(read()); + assert.deepEqual( + page.items.map((row) => row.item.ordinal), + ordinalsFrom(915, 75), + ); + assert.strictEqual(decodeThreadHistoryCursor(page.nextCursor!).p, 914); + // A jump to an entry reads the full index and leaves it cached. With work, it scans + // payloads for inline output only in its page's runs, not in a run after its cursor. + const scannedRuns: Array = []; + yield* read("turn-item:bounded-sql-history:950").pipe( + Effect.provideService(Statement.CurrentTransformer, (statement) => { + const [query, params] = statement.compile(); + if (query.includes("LIKE '%htmlRender%'")) scannedRuns.push(String(params[0])); + return Effect.succeed(statement); + }), + ); + assert.deepEqual( + [ + scannedRuns.length, + scannedRuns.some((runs) => runs.includes('"run:bounded-sql-history:950"')), + scannedRuns.some((runs) => runs.includes('"run:bounded-sql-history:995"')), + ], + workMode === undefined ? [0, false, false] : [1, true, false], + ); + assert.deepEqual(yield* read(), page); + } + for (const [view, ordinals] of [ + ["activity", ordinalsFrom(902, 20)], + ["messages", [1000]], + ] as const) { + const read = projectionStore.getTimelinePage(threadId, { + afterPosition: 900, + limit: 20, + view, + }); + const page = yield* recordPageStatements(read); + assert.deepEqual( + page.items.map((row) => [row.item.ordinal, row.position]), + ordinals.map((ordinal) => [ordinal, ordinal - 1]), + ); + assert.strictEqual(page.totalItems, 1000); + // An attachment lookup reads the full index and leaves it cached. + yield* projectionStore.getTimelinePage(threadId, { attachmentId: "absent", limit: 1 }); + assert.deepEqual(yield* read, page); + } + { + const pagePath = "file:/history-page-steps?vfs=memdb"; + using copy = new NodeSqlite.DatabaseSync(pagePath); + yield* sql`VACUUM INTO ${pagePath}`; + // Only statements that count positions may grow with the history, and those read an + // index alone. Every other statement costs the same when the rows below the page are gone. + const countsPositions = ([query]: readonly [string, ReadonlyArray]) => + /COUNT\(\*\)|OFFSET/.test(query); + for (const [query, params] of pageStatements.filter(countsPositions)) { + const plan = copy + .prepare(`EXPLAIN QUERY PLAN ${query}`) + .all(...(params as ReadonlyArray)) + .map((row) => String(row.detail)) + .filter((detail) => detail.includes("orchestration_v2_projection_turn_items")); + assert.isTrue( + plan.every((detail) => detail.includes("COVERING INDEX")), + `${query}\n${plan.join("\n")}`, + ); + } + const rowReads = pageStatements.filter((statement) => !countsPositions(statement)); + const rowReadSteps = () => + rowReads.reduce((steps, [query, params]) => { + const statement = copy.prepare(query); + statement.all(...(params as ReadonlyArray)); + return steps + statement.stat("vmStep"); + }, 0); + const fullSteps = rowReadSteps(); + assert.isAbove(fullSteps, 0); + copy.exec( + "DELETE FROM orchestration_v2_projection_turn_items WHERE ordinal BETWEEN 2 AND 400", + ); + assert.isAtMost(fullSteps, rowReadSteps() * 1.1); + } + // Rows that a rolled-back run hides below the page still shift every position above them. + const setOldRunStatus = (status: "rolled_back" | "completed") => sql` + UPDATE orchestration_v2_projection_runs + SET status = ${status}, payload_json = json_set(payload_json, '$.status', ${status}) + WHERE thread_id = ${threadId} AND ordinal BETWEEN 100 AND 109`; + yield* setOldRunStatus("rolled_back"); + const shiftedPage = yield* historyPage(threadId, olderCursor); + assert.strictEqual(decodeThreadHistoryCursor(shiftedPage.nextCursor!).p, 904); + const shiftedTimeline = yield* committed( + projectionStore.getTimelinePage(threadId, { + afterPosition: 900, + limit: 1, + view: "activity", + }), + ); + assert.deepEqual( + [shiftedTimeline.totalItems, shiftedTimeline.items[0]?.item.ordinal], + [990, 912], + ); + yield* setOldRunStatus("completed"); + const retainedRequestPlan = yield* sql<{ readonly detail: string }>` EXPLAIN QUERY PLAN WITH selected AS ( @@ -4205,21 +4335,24 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { window.projection.visibleTurnItems.find((row) => row.item.deferredWork)?.item.ordinal, 3, ); + const newestCursor = encodeThreadHistoryCursor({ + snapshotSequence: 0, + sourceThreadId: threadId, + sourceItemId: "turn-item:bounded-sql-history:1000", + position: 1000, + }); + const page = yield* historyPage(threadId, newestCursor, undefined, false, "deferred"); + assert.isUndefined(page.items.find((row) => row.item.ordinal === 2)?.item.deferredWork); + // Tool payloads hold megabytes, so a page that reads the full index, as a jump to an + // entry does, looks for inline output only in its own runs. const pageRead = traceSqlStatements(); - const page = yield* historyPage( + yield* historyPage( threadId, - encodeThreadHistoryCursor({ - snapshotSequence: 0, - sourceThreadId: threadId, - sourceItemId: "turn-item:bounded-sql-history:1000", - position: 1000, - }), - undefined, + newestCursor, + "turn-item:bounded-sql-history:2", false, "deferred", ).pipe(Effect.withTracer(pageRead.tracer)); - assert.isUndefined(page.items.find((row) => row.item.ordinal === 2)?.item.deferredWork); - // Tool payloads hold megabytes, so a page looks for inline output only in its own runs. const payloadScans = pageRead.statements.filter((statement) => statement.includes("LIKE '%htmlRender%'"), ); @@ -5558,6 +5691,339 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); + it.effect("a page read from a window equals the page read from the full index", () => + Effect.gen(function* () { + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const now = yield* DateTime.now; + const nowIso = DateTime.formatIso(now); + // Seeded, so every run builds the same threads. + let seed = 0x5eed; + const random = () => { + seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0; + return seed / 2 ** 32; + }; + const chance = (probability: number) => random() < probability; + const spans: Array = []; + const tracer = Tracer.make({ + span(options) { + spans.push(options.name); + return new Tracer.NativeSpan(options); + }, + }); + let windowReads = 0; + let foldedGroups = 0; + + for (let round = 0; round < 4; round += 1) { + const threadId = ThreadId.make(`thread:window-equivalence:${round}`); + const created = { + id: EventId.make(`event:window-equivalence:${round}`), + type: "thread.created", + threadId, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: threadId, + projectId: ProjectId.make("project:window-equivalence"), + title: "Window equivalence", + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + } satisfies Parameters[0]; + yield* projectionStore.apply(created); + // One run of each kind in a random order, then an active run. Every other thread has + // one user turn and then only wake turns. + // The last thread is a long run that a late notification wakes, then enough one-row + // runs that a page stops at the raw turn limit only if it counts that wake as a turn. + const kinds = + round === 3 + ? ["late wake", ...Array.from({ length: 148 }, () => "one row")] + : [ + "rolled back", + "cancelled queued turn", + "replaced attempt", + "stopped", + "stopped", + "folds work", + "folds work", + "short", + "short", + "short", + ] + .map((kind) => ({ kind, order: random() })) + .toSorted((left, right) => left.order - right.order) + .map(({ kind }) => kind); + const agentDriven = round === 1; + const runs: Array = []; + const attempts: Array = []; + const items: Array = []; + const tiedWithHidden: Array = []; + const repliesInStoppedRuns: Array = []; + let ordinal = 0; + for (const [index, kind] of [...kinds, "active"].entries()) { + const run = index + 1; + const runId = `run:window-equivalence:${round}:${run}`; + const nodeId = `node:window-equivalence:${round}:${run}`; + const status = + kind === "active" + ? "running" + : kind === "rolled back" + ? "rolled_back" + : kind === "cancelled queued turn" + ? "cancelled" + : kind === "short" && chance(0.5) + ? "interrupted" + : "completed"; + runs.push({ + id: runId, + threadId, + ordinal: run, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: `message:window-equivalence:${round}:${run}`, + rootNodeId: nodeId, + activeAttemptId: null, + status, + requestedAt: nowIso, + startedAt: nowIso, + completedAt: nowIso, + checkpointId: null, + contextHandoffId: null, + }); + attempts.push({ + id: `attempt:window-equivalence:${round}:${run}`, + runId, + attemptOrdinal: 1, + rootNodeId: nodeId, + providerInstanceId, + providerThreadId: "provider-thread:window-equivalence", + providerTurnId: null, + reason: "initial", + status: kind === "replaced attempt" || chance(0.2) ? "superseded" : "completed", + startedAt: nowIso, + completedAt: nowIso, + }); + // The first row after a rolled-back run shares the ordinal of that run's last row. + let tie = kinds[index - 1] === "rolled back"; + if (tie) + tiedWithHidden.push( + TurnItemId.make(`turn-item:window-equivalence:${round}:${items.length}`), + ); + const add = (type: string, fields: Record = {}) => { + if (ordinal === 0 || !(tie || chance(0.04))) ordinal += 1; + tie = false; + items.push({ + id: `turn-item:window-equivalence:${round}:${items.length}`, + threadId, + runId, + nodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal, + status: "completed", + title: null, + startedAt: nowIso, + completedAt: nowIso, + updatedAt: nowIso, + type, + ...fields, + }); + }; + const userMessage = (inputIntent: string) => + add("user_message", { + createdBy: "user", + creationSource: "web", + messageId: `message:window-equivalence:${round}:${items.length}`, + inputIntent, + text: "Turn", + attachments: [], + }); + const notification = () => + add("notification", { + source: { kind: "system" }, + outcome: "completed", + summary: "Wake", + }); + const wakes = + kind === "one row" + ? run % 10 !== 0 + : kind !== "late wake" && (agentDriven ? run > 1 : chance(0.2)); + if (wakes) notification(); + else userMessage(kind === "cancelled queued turn" ? "queued_turn" : "turn_start"); + // A stopped run never folds, so it is long enough for page ends to fall inside it. + const work = + kind === "late wake" + ? 420 + Math.floor(random() * 40) + : kind === "stopped" + ? 320 + Math.floor(random() * 60) + : kind === "folds work" + ? 155 + Math.floor(random() * 60) + : kind === "one row" + ? 0 + : Math.floor(random() * 30); + for (let row = 0; row < work; row += 1) { + if (chance(0.5)) add("command_execution", { input: "echo", output: "ok", exitCode: 0 }); + else add("reasoning", { text: "", streaming: false }); + // A reply deep in a stopped run: a page that ends on it has enough work below it + // to fold, and only the stop request above it says that it does not. + const reply = kind === "stopped" && row === 200; + if (reply) + repliesInStoppedRuns.push(`turn-item:window-equivalence:${round}:${items.length}`); + if (reply || chance(0.02)) + add("assistant_message", { + messageId: `message:window-equivalence:${round}:${items.length}`, + text: "Reply", + streaming: false, + }); + if (chance(0.01)) notification(); + if (chance(0.004) && !agentDriven) userMessage("steer"); + } + if (kind === "late wake") notification(); + if (kind === "stopped") add("run_interrupt_request", { message: "Stop" }); + if (kind === "replaced attempt" || (kind === "stopped" && chance(0.5))) + add("run_interrupt_result", { message: "Stopped" }); + if (kind === "short" && chance(0.3)) + add("error", { + status: "failed", + failure: { class: "unknown", message: "Failed", code: null, retryable: false }, + }); + } + yield* sql` + INSERT INTO orchestration_v2_projection_runs ( + run_id, thread_id, ordinal, provider, provider_thread_id, status, + requested_at, completed_at, payload_json + ) + SELECT json_extract(value, '$.id'), ${threadId}, json_extract(value, '$.ordinal'), 'codex', + NULL, json_extract(value, '$.status'), ${nowIso}, ${nowIso}, value + FROM json_each(${encodeUnknownJsonString(runs)})`; + yield* sql` + INSERT INTO orchestration_v2_projection_run_attempts ( + attempt_id, thread_id, run_id, attempt_ordinal, root_node_id, provider, + provider_instance_id, provider_thread_id, provider_turn_id, status, payload_json + ) + SELECT json_extract(value, '$.id'), ${threadId}, json_extract(value, '$.runId'), 1, + json_extract(value, '$.rootNodeId'), ${providerInstanceId}, ${providerInstanceId}, + json_extract(value, '$.providerThreadId'), NULL, json_extract(value, '$.status'), value + FROM json_each(${encodeUnknownJsonString(attempts)})`; + yield* sql` + INSERT INTO orchestration_v2_projection_turn_items ( + turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id, + parent_item_id, ordinal, type, status, updated_at, payload_json + ) + SELECT json_extract(value, '$.id'), ${threadId}, json_extract(value, '$.runId'), + json_extract(value, '$.nodeId'), NULL, NULL, NULL, json_extract(value, '$.ordinal'), + json_extract(value, '$.type'), json_extract(value, '$.status'), ${nowIso}, value + FROM json_each(${encodeUnknownJsonString(items)})`; + + // Applying an event drops the thread's cached index, as each event of a run does, so + // each of these reads a window. + const requests: Array<{ + readonly read: Effect.Effect; + readonly page: unknown; + }> = []; + const windowed = (read: Effect.Effect) => + Effect.gen(function* () { + yield* projectionStore.apply(created); + spans.length = 0; + const page = yield* read.pipe(Effect.withTracer(tracer)); + if (spans.includes("ProjectionStore.readTimelineWindow")) windowReads += 1; + requests.push({ read, page }); + return page; + }); + const history = ( + cursor: string, + throughEntryId?: string, + conversationOnly?: boolean, + workMode?: "deferred" | "work", + ) => + projectionStore.getThreadHistoryPage( + threadId, + cursor, + throughEntryId, + conversationOnly, + workMode, + ); + const timeline = (options: ProjectionStore.ProjectionTimelinePageOptions) => + projectionStore.getTimelinePage(threadId, options); + + const { totalItems } = yield* windowed(timeline({ limit: 0 })); + const newest = (yield* windowed( + timeline({ view: "activity", afterPosition: totalItems - 2, limit: 1 }), + )).items[0]!; + const firstCursor = encodeThreadHistoryCursor({ + snapshotSequence: 0, + sourceThreadId: threadId, + sourceItemId: newest.sourceItemId, + position: newest.position, + }); + const groups = new Map(); + for (const [conversationOnly, workMode] of [ + [false, "deferred"], + [false, undefined], + [true, undefined], + ] as const) { + for (let cursor: string | null = firstCursor; cursor !== null;) { + const page: OrchestrationV2ThreadHistoryPage = yield* windowed( + history(cursor, undefined, conversationOnly, workMode), + ); + for (const { item } of page.items) + if (item.deferredWork !== undefined) groups.set(item.id, item.deferredWork.cursor); + cursor = page.nextCursor; + } + } + for (const sourceItemId of repliesInStoppedRuns) { + const cursor = encodeThreadHistoryCursor({ + snapshotSequence: 0, + sourceThreadId: threadId, + sourceItemId, + position: 0, + }); + yield* windowed(history(cursor, undefined, false, "deferred")); + } + foldedGroups += groups.size; + for (const [anchorId, groupCursor] of groups) { + for (let cursor: string | null = groupCursor; cursor !== null;) + cursor = (yield* windowed(history(cursor, anchorId, false, "work"))).nextCursor; + } + for (const view of ["messages", "activity"] as const) { + for (const afterPosition of [-1, Math.floor(totalItems / 3), totalItems - 5]) + yield* windowed(timeline({ view, afterPosition, limit: 20 })); + } + for (const itemId of [newest.sourceItemId, ...tiedWithHidden]) + yield* windowed(timeline({ itemId, limit: 1 })); + + // An attachment lookup and a jump to an entry read the full index and leave it cached, + // so the same requests now come from it. + yield* timeline({ attachmentId: "absent", limit: 1 }); + yield* history(firstCursor, "absent", false, "deferred"); + spans.length = 0; + for (const { read, page } of requests) + assert.deepEqual(yield* read.pipe(Effect.withTracer(tracer)), page); + assert.notInclude(spans, "ProjectionStore.readTimelineWindow"); + } + assert.isAtLeast(foldedGroups, 2); + assert.isAbove(windowReads, 60); + }), + ); + it.effect("scopes actionable provider state to the requested bounded thread", () => Effect.gen(function* () { const projectionStore = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 4ccb6bbf85ad..22d8cfe25ddb 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -90,6 +90,7 @@ import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type * as Statement from "effect/sql/Statement"; @@ -98,6 +99,7 @@ import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp"; import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts"; import { readableSearchText } from "./messageSearchIndex.ts"; import { + isFoldableWorkItem, isThreadHistoryUserTurn, isConversationHistoryItem, THREAD_HISTORY_MAX_RAW_TURNS, @@ -114,6 +116,8 @@ import { THREAD_HISTORY_PAGE_POLICY, THREAD_HISTORY_INLINE_WORK_ITEM_LIMIT, THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT, + type SelectTimelinePageResult, + type ThreadHistoryPrefix, } from "./threadHistoryPaging.ts"; export class ProjectionStoreApplyEventError extends Schema.TaggedError()( @@ -6451,6 +6455,459 @@ export const layer: Layer.Layer = readTimelineIndex(threadId, new Set()).pipe(Effect.map((index) => index.visible)), ); + /** + * Index columns of the thread's own turn items that `where` selects, in timeline order. The + * same columns as `readTimelineIndex`, which reads every row and stays as it is. + */ + const readTimelineWindow = Effect.fn("ProjectionStore.readTimelineWindow")(function* ( + threadId: ThreadId, + runs: ProjectionRecords<"runs">["runs"], + includePersistentOutput: boolean, + where: Statement.Fragment, + limit: Statement.Fragment = sql``, + ) { + const rows = yield* sql<{ + turn_item_id: string; + run_id: string | null; + node_id: string | null; + parent_item_id: string | null; + type: OrchestrationV2TurnItem["type"]; + input_intent: NonNullable | null; + created_by: "user" | "agent" | "system" | null; + message_id: string | null; + status: OrchestrationV2TurnItem["status"]; + ordinal: number; + persistent_output: number; + attempt_id: string | null; + attempt_status: string | null; + }>`SELECT item.turn_item_id, item.run_id, item.node_id, item.parent_item_id, item.type, item.status, item.ordinal, + ${includePersistentOutput ? sql`attempt.attempt_id` : sql`NULL`} AS attempt_id, + ${includePersistentOutput ? sql`attempt.status` : sql`NULL`} AS attempt_status, + ${ + includePersistentOutput ? sql`CASE WHEN ${inlineWorkOutput} THEN 1 ELSE 0 END` : sql`0` + } AS persistent_output, + CASE WHEN item.type = 'user_message' THEN json_extract(item.payload_json, '$.inputIntent') END AS input_intent, + CASE WHEN item.type = 'user_message' THEN json_extract(item.payload_json, '$.createdBy') END AS created_by, + CASE WHEN item.type IN ('user_message', 'assistant_message') THEN json_extract(item.payload_json, '$.messageId') END AS message_id + FROM orchestration_v2_projection_turn_items AS item + ${ + includePersistentOutput + ? sql`LEFT JOIN orchestration_v2_projection_nodes AS node ON node.node_id = item.node_id + LEFT JOIN orchestration_v2_projection_run_attempts AS attempt ON attempt.attempt_id = ( + SELECT attempt_id FROM orchestration_v2_projection_run_attempts + WHERE run_id = item.run_id AND root_node_id = COALESCE(node.root_node_id, item.node_id) + ORDER BY attempt_ordinal DESC LIMIT 1 + )` + : sql`` + } + WHERE item.thread_id = ${threadId} ${where} + ORDER BY item.ordinal ASC, item.turn_item_id ASC ${limit}`; + const runStatuses = new Map(runs.map((run) => [String(run.id), run.status])); + return rows.map((row): TimelineIndexItem => ({ + id: TurnItemId.make(row.turn_item_id), + threadId, + runId: row.run_id === null ? null : RunId.make(row.run_id), + nodeId: row.node_id === null ? null : NodeId.make(row.node_id), + parentItemId: row.parent_item_id === null ? null : TurnItemId.make(row.parent_item_id), + type: row.type, + status: row.status, + ordinal: row.ordinal, + runStatus: row.run_id === null ? undefined : runStatuses.get(row.run_id), + attemptId: row.attempt_id, + attemptStatus: row.attempt_status, + persistentOutput: row.persistent_output === 1, + ...(row.input_intent === null ? {} : { inputIntent: row.input_intent }), + ...(row.created_by === null ? {} : { createdBy: row.created_by }), + ...(row.message_id === null ? {} : { messageId: MessageId.make(row.message_id) }), + })); + }); + const localTimelineRow = (item: TimelineIndexItem): TimelineIndexRow => ({ + sourceThreadId: item.threadId, + sourceItemId: item.id, + visibility: "local", + item, + }); + + /** + * Reads parts of a thread's own timeline without indexing all of it. Null when callers read + * the full index instead: the thread shows fork ancestry, so its rows come from other threads, + * or a full index of this snapshot is cached already. + */ + const openLocalTimeline = Effect.fn("ProjectionStore.openLocalTimeline")(function* ( + threadId: ThreadId, + includePersistentOutput = false, + ) { + // A read that only the full index can answer left it cached, and it answers faster than + // a window. The key is the one `readSnapshotTimelineIndex` uses. + const cached = timelineIndexes.get( + `${includePersistentOutput ? "work" : "plain"}:${threadId}`, + ); + if ( + cached !== undefined && + (yield* latestThreadEventSequence(threadId, cached.index.threadIds)) === cached.sequence + ) + return null; + // Checked in SQL, so a fork loads no runs and no thread here before the full index does. + const ownRowsOnly = yield* sql` + SELECT 1 FROM orchestration_v2_projection_threads + WHERE thread_id = ${threadId} + AND NOT (json_extract(payload_json, '$.forkedFrom.type') IS 'run' + AND json_extract(payload_json, '$.lineage.sideChat') IS NOT 1)`; + if (ownRowsOnly.length === 0) return null; + const records = yield* getThreadRecords(threadId, ["runs", "attempts"]); + if (inheritedRunFork(records.thread) !== null) return null; + const superseded = records.attempts.filter((attempt) => attempt.status === "superseded"); + const stopRequests = + superseded.length === 0 + ? [] + : yield* sql<{ run_id: string | null }>` + SELECT DISTINCT run_id FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND type = 'run_interrupt_request'`; + const isVisible = createOrchestrationV2TurnItemVisibility({ + runs: records.runs, + attempts: records.attempts, + items: stopRequests.map((row) => ({ + type: "run_interrupt_request" as const, + runId: row.run_id === null ? null : RunId.make(row.run_id), + nodeId: null, + })), + }); + // Only these runs can hide rows: every row of a rolled-back run, the queued turn of a + // cancelled run, and the stop result of a replaced attempt. So the hidden rows are found + // without reading the thread. + const stoppedRuns = new Set(stopRequests.map((row) => row.run_id)); + const runsWithStatus = (status: "rolled_back" | "cancelled") => + records.runs.flatMap((run) => (run.status === status ? [String(run.id)] : [])); + const rolledBack = runsWithStatus("rolled_back"); + const cancelled = runsWithStatus("cancelled"); + const steered = [...new Set(superseded.map((attempt) => String(attempt.runId)))].filter( + (runId) => !stoppedRuns.has(runId) && !rolledBack.includes(runId), + ); + type HiddenCandidate = { + turn_item_id: string; + run_id: string; + node_id: string | null; + type: OrchestrationV2TurnItem["type"]; + ordinal: number; + input_intent: NonNullable | null; + }; + const candidateColumns = sql`item.turn_item_id, item.run_id, item.node_id, item.type, + item.ordinal, CASE WHEN item.type = 'user_message' + THEN json_extract(item.payload_json, '$.inputIntent') END AS input_intent`; + const [rolledBackRows, conversationRows] = yield* Effect.all([ + rolledBack.length === 0 + ? Effect.succeed([]) + : sql` + SELECT ${candidateColumns} + FROM json_each(${encodeIdList(rolledBack)}) AS wanted + CROSS JOIN orchestration_v2_projection_turn_items AS item + INDEXED BY orchestration_v2_projection_turn_items_thread_run_idx + ON item.thread_id = ${threadId} AND item.run_id = wanted.value`, + // The conversation index holds the type and run of these rows, so only the matches are + // fetched. Through the run index, every row of each cancelled run would be. + cancelled.length + steered.length === 0 + ? Effect.succeed([]) + : sql` + SELECT ${candidateColumns} + FROM orchestration_v2_projection_turn_items AS item + INDEXED BY orchestration_v2_projection_turn_items_conversation_idx + WHERE item.thread_id = ${threadId} + AND item.type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES} + AND ((item.type = 'user_message' + AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(cancelled)}))) + OR (item.type = 'run_interrupt_result' + AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(steered)}))))`, + ]); + const hidden = [...rolledBackRows, ...conversationRows].filter( + (row) => + !isVisible({ + type: row.type, + runId: RunId.make(row.run_id), + nodeId: row.node_id === null ? null : NodeId.make(row.node_id), + ...(row.input_intent === null ? {} : { inputIntent: row.input_intent }), + }), + ); + /** Counts the visible rows that sort before the given ordinal and item id. */ + const visibleBefore = Effect.fn("ProjectionStore.countVisibleTimelineRows")(function* ( + ordinal: number, + itemId = "", + ) { + const rows = yield* sql<{ count: number }>` + SELECT COUNT(*) AS count FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND (ordinal, turn_item_id) < (${ordinal}, ${itemId})`; + const hiddenBefore = hidden.filter( + (row) => + row.ordinal < ordinal || + (row.ordinal === ordinal && + Buffer.compare(Buffer.from(row.turn_item_id), Buffer.from(itemId)) < 0), + ); + return (rows[0]?.count ?? 0) - hiddenBefore.length; + }); + return { + records, + isVisible, + hidden, + visibleBefore, + read: (where: Statement.Fragment, limit?: Statement.Fragment) => + readTimelineWindow(threadId, records.runs, includePersistentOutput, where, limit), + }; + }); + type LocalTimeline = NonNullable>>; + type IndexedTimelineRow = TimelineIndexRow & { + readonly position: number; + readonly canonicalPosition: number; + }; + + /** True when a visible user turn starts before the ordinal. Reads the nearest user messages. */ + const olderUserTurn = (local: LocalTimeline, ordinal: number) => + sql<{ + run_id: string | null; + node_id: string | null; + input_intent: NonNullable; + }>` + SELECT item.run_id, item.node_id, + json_extract(item.payload_json, '$.inputIntent') AS input_intent + FROM orchestration_v2_projection_turn_items AS item + INDEXED BY orchestration_v2_projection_turn_items_user_message_idx + WHERE item.thread_id = ${local.records.thread.id} AND item.type = 'user_message' + AND item.ordinal < ${ordinal} + AND json_extract(item.payload_json, '$.inputIntent') IN ('turn_start', 'queued_turn') + AND json_extract(item.payload_json, '$.createdBy') = 'user' + ORDER BY item.ordinal DESC, item.turn_item_id DESC + LIMIT ${local.hidden.length + 1}`.pipe( + Effect.map((rows) => + rows.some((row) => + local.isVisible({ + type: "user_message", + runId: row.run_id === null ? null : RunId.make(row.run_id), + nodeId: row.node_id === null ? null : NodeId.make(row.node_id), + inputIntent: row.input_intent, + }), + ), + ), + ); + + /** + * A notification starts a turn only as its run's first input. For each run whose first input + * in `rows` is a notification, this finds whether an input row sits before the ordinal. + */ + const runsStartedBefore = Effect.fn("ProjectionStore.runsStartedBefore")(function* ( + local: LocalTimeline, + ordinal: number, + rows: ReadonlyArray, + ) { + const seen = new Set(); + const unknown = new Set(); + for (const { item } of rows) { + if (item.runId === null) continue; + if (item.type !== "user_message" && item.type !== "notification") continue; + if (item.type === "notification" && !seen.has(item.runId)) unknown.add(item.runId); + seen.add(item.runId); + } + if (unknown.size === 0) return new Set(); + const counts = yield* sql<{ run_id: string; inputs: number }>` + SELECT wanted.value AS run_id, ( + SELECT COUNT(*) FROM orchestration_v2_projection_turn_items AS input + INDEXED BY orchestration_v2_projection_turn_items_conversation_idx + WHERE input.thread_id = ${local.records.thread.id} + AND input.ordinal >= ( + SELECT MIN(ordinal) FROM orchestration_v2_projection_turn_items + WHERE run_id = wanted.value + ) + AND input.ordinal < ${ordinal} + AND input.type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES} + AND input.type IN ('user_message', 'notification') + AND input.run_id = wanted.value + ) AS inputs + FROM json_each(${encodeIdList([...unknown])}) AS wanted`; + const isHiddenInput = (row: LocalTimeline["hidden"][number], runId: string) => + row.run_id === runId && + row.ordinal < ordinal && + (row.type === "user_message" || row.type === "notification"); + return new Set( + counts.flatMap((row) => + row.inputs > local.hidden.filter((hidden) => isHiddenInput(hidden, row.run_id)).length + ? [row.run_id] + : [], + ), + ); + }); + + /** + * Selects an older history page from the rows near its cursor. The window grows until the + * page ends inside rows whose inputs are all known, so `select` gives the page it gives on + * the full index. Null when the cursor names no visible row, which needs the full index. + */ + const selectHistoryPageInWindow = Effect.fn("ProjectionStore.selectHistoryPageInWindow")( + function* < + Selection extends { + readonly page: SelectTimelinePageResult; + readonly historyItems: ReadonlyArray; + }, + >( + local: LocalTimeline, + input: { + readonly cursor: string; + readonly throughEntryId: string | undefined; + readonly workMode: "deferred" | "work" | undefined; + }, + select: ( + indexed: ReadonlyArray, + prefix?: ThreadHistoryPrefix, + ) => Effect.Effect, + ) { + const threadId = local.records.thread.id; + const decoded = Result.try(() => decodeThreadHistoryCursor(input.cursor)); + if (Result.isFailure(decoded) || decoded.success.st !== threadId) return null; + const cursor = decoded.success; + // Looked up by key alone: with the thread in the query, SQLite scans the thread's index. + const ends = (yield* sql<{ turn_item_id: string; thread_id: string; ordinal: number }>` + SELECT turn_item_id, thread_id, ordinal FROM orchestration_v2_projection_turn_items + WHERE turn_item_id IN (${cursor.si}, ${input.throughEntryId ?? cursor.si})`).filter( + (row) => row.thread_id === threadId, + ); + const upper = ends.find((row) => row.turn_item_id === cursor.si)?.ordinal; + if (upper === undefined) return null; + /* Runs for each window this operation tries; the operation owns the span. */ + const indexFrom = Effect.fnUntraced(function* ( + items: ReadonlyArray, + from: number | null, + ) { + const base = from === null ? 0 : yield* local.visibleBefore(from); + return items.filter(local.isVisible).map((item, offset): IndexedTimelineRow => ({ + ...localTimelineRow(item), + position: base + offset, + canonicalPosition: base + offset, + })); + }); + const isCursorRow = (row: TimelineIndexRow) => row.sourceItemId === cursor.si; + // The ordinals come from the stop request index, so this does not walk the thread. + const interruptRequests = local + .read( + sql`AND item.type = 'run_interrupt_request' AND item.ordinal IN ( + SELECT ordinal FROM orchestration_v2_projection_turn_items + INDEXED BY orchestration_v2_projection_turn_items_stop_request_idx + WHERE thread_id = ${threadId} AND type = 'run_interrupt_request')`, + ) + .pipe(Effect.flatMap((items) => indexFrom(items, null))); + + if (input.workMode === "work") { + // A work group lies between its first row and the cursor, so that range is the read. + const lower = ends.find((row) => row.turn_item_id === input.throughEntryId)?.ordinal; + if (lower === undefined) return null; + const indexed = yield* indexFrom( + yield* local.read(sql`AND item.ordinal >= ${lower} AND item.ordinal <= ${upper}`), + lower, + ); + if (!indexed.some(isCursorRow)) return null; + if (!indexed.some((row) => row.sourceItemId === input.throughEntryId)) return null; + return { + indexed, + selection: yield* select(indexed), + olderUserTurn: olderUserTurn(local, lower), + interruptRequests, + }; + } + + const rowsBelow = (ordinal: number, count: number) => + sql<{ ordinal: number }>` + SELECT ordinal FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND ordinal < ${ordinal} + ORDER BY ordinal DESC LIMIT 1 OFFSET ${count - 1}`.pipe( + Effect.map((rows) => rows[0]?.ordinal ?? null), + ); + // `trusted` is the lowest ordinal the page may reach. `lower` can sit below it, because + // a run that folds work is read whole. + let step = THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT; + let trusted = yield* rowsBelow(upper, step); + let lower = trusted; + let higher = upper; + let items = yield* local.read( + lower === null + ? sql`AND item.ordinal <= ${upper}` + : sql`AND item.ordinal >= ${lower} AND item.ordinal <= ${upper}`, + ); + const wholeRuns = new Set(); + while (true) { + if (input.workMode === "deferred") { + // Folding needs every row of a run: its stop and error rows, and its group sizes. + const folding = new Set(); + for (const item of items) { + if (item.runId === null || wholeRuns.has(item.runId)) continue; + if (item.ordinal > upper || (trusted !== null && item.ordinal < trusted)) continue; + if (isFoldableWorkItem(item) && local.isVisible(item)) folding.add(item.runId); + } + if (folding.size > 0) { + const runs = encodeIdList([...folding]); + const [extent] = yield* sql<{ lower: number; higher: number }>` + SELECT MIN(run.lower) AS lower, MAX(run.higher) AS higher FROM ( + SELECT + (SELECT MIN(ordinal) FROM orchestration_v2_projection_turn_items + WHERE run_id = wanted.value) AS lower, + (SELECT MAX(ordinal) FROM orchestration_v2_projection_turn_items + WHERE run_id = wanted.value) AS higher + FROM json_each(${runs}) AS wanted + ) AS run`; + if (extent !== undefined && lower !== null && extent.lower < lower) { + const below = yield* local.read( + sql`AND item.ordinal >= ${extent.lower} AND item.ordinal < ${lower}`, + ); + items = below.concat(items); + lower = extent.lower; + } + if (extent !== undefined && extent.higher > higher) { + const newer = yield* local.read( + sql`AND item.ordinal > ${higher} AND item.ordinal <= ${extent.higher}`, + ); + items = items.concat(newer); + higher = extent.higher; + } + for (const runId of folding) wholeRuns.add(runId); + } + } + const indexed = yield* indexFrom(items, lower); + const end = indexed.findIndex(isCursorRow); + if (end < 0) return null; + const older = lower === null ? Effect.succeed(false) : olderUserTurn(local, lower); + const prefix: ThreadHistoryPrefix | undefined = + lower === null + ? undefined + : { + hasUserTurn: indexed + .slice(0, end) + .some((row) => isThreadHistoryUserTurn(row.item)) + ? false + : yield* older, + startedRuns: yield* runsStartedBefore(local, lower, indexed), + }; + const selection = yield* select(indexed, prefix); + // Without its row the cursor falls back to a position, which only the full index has. + if (!selection.historyItems.some(isCursorRow)) return null; + const oldest = selection.page.oldestScanned; + if ( + lower === null || + (trusted !== null && + oldest !== undefined && + oldest !== selection.historyItems[0] && + oldest.item.ordinal >= trusted) + ) { + return { indexed, selection, olderUserTurn: older, interruptRequests }; + } + step *= 2; + trusted = trusted === null ? null : yield* rowsBelow(trusted, step); + if (trusted === null || trusted < lower) { + const below = yield* local.read( + trusted === null + ? sql`AND item.ordinal < ${lower}` + : sql`AND item.ordinal >= ${trusted} AND item.ordinal < ${lower}`, + ); + items = below.concat(items); + lower = trusted; + } + } + }, + ); + const getThreadHistoryPage: ProjectionStoreV2Shape["getThreadHistoryPage"] = ( threadId, cursor, @@ -6461,8 +6918,15 @@ export const layer: Layer.Layer = sql .withTransaction( Effect.gen(function* () { - const index = yield* readSnapshotTimelineIndex(threadId, workMode !== undefined); - if (index.records.thread.deletedAt !== null) + const withWork = workMode !== undefined; + // A jump to a target entry selects from the whole index. + const local = + (workMode === "work") === (throughEntryId !== undefined) + ? yield* openLocalTimeline(threadId, withWork) + : null; + const full = + local === null ? yield* readSnapshotTimelineIndex(threadId, withWork) : null; + if ((local ?? full)?.records.thread.deletedAt !== null) return yield* new ProjectionStoreThreadNotFoundError({ threadId }); const revisions = yield* sql<{ sequence: number; @@ -6470,100 +6934,132 @@ export const layer: Layer.Layer = FROM orchestration_events WHERE application_event_version = 2 AND aggregate_kind = 'thread' AND stream_id = ${threadId}`; const snapshotSequence = revisions[0]?.sequence ?? 0; - const indexed = index.visible.map((row, position) => ({ - ...row, - position, - canonicalPosition: position, - })); - const { olderPolicy, cursorPosition } = yield* Effect.try({ - try: () => ({ - olderPolicy: olderHistoryPagePolicy({ items: indexed, cursor, throughEntryId }), - // The cursor's own row, or the row its stale position falls back to. - cursorPosition: Math.min( - findCursorEndIndex(indexed, decodeThreadHistoryCursor(cursor)), - indexed.length - 1, - ), - }), - catch: (cause) => new ProjectionStoreReadError({ threadId, cause }), - }); - const selectPage = (rows: typeof indexed) => + const select = ( + indexed: ReadonlyArray, + prefix?: ThreadHistoryPrefix, + ) => Effect.try({ - try: () => - workMode === "work" - ? selectDeferredWorkPage({ - items: rows, - cursor, - throughEntryId, - snapshotSequence, - rowEncodedBytes: () => 0, - }) - : selectHistoryPageFromCursor({ - items: workMode === "deferred" ? deferWorkHistoryRows(rows) : rows, - cursor, - snapshotSequence, - rowEncodedBytes: () => 0, - policy: olderPolicy, - }), + try: () => { + const historyItems = + workMode === "deferred" ? deferWorkHistoryRows(indexed) : indexed; + const policy = olderHistoryPagePolicy({ items: indexed, cursor, throughEntryId }); + const page = + workMode === "work" + ? selectDeferredWorkPage({ + items: indexed, + cursor, + throughEntryId, + snapshotSequence, + rowEncodedBytes: () => 0, + }) + : selectHistoryPageFromCursor({ + items: historyItems, + cursor, + snapshotSequence, + rowEncodedBytes: () => 0, + policy, + prefix, + }); + return { policy, page, historyItems }; + }, catch: (cause) => new ProjectionStoreReadError({ threadId, cause }), }); - let page = yield* selectPage(indexed); - // Inline output is found by scanning payloads, which costs a whole long thread several - // seconds. Scan only the runs the page spans, then select again with what they hold. - const scannedRuns = new Set(); - const persistentOutputs = new Set(); - while (workMode !== undefined) { - // Rows hidden in a work group move the page, so the scan always reaches the cursor. - const span = [...page.items.slice(0, 1), ...page.items.slice(-1)].map( - (row) => row.canonicalPosition, - ); - if (cursorPosition >= 0) span.push(cursorPosition); - if (span.length === 0) break; - const runs = new Map( - // A work page holds rows of one run, which can be the rows that have no run. - (workMode === "work" - ? page.items.slice(0, 1) - : indexed.slice(Math.min(...span), Math.max(...span) + 1) - ).flatMap(({ sourceThreadId, item }) => - (workMode === "work" || - (item.runId !== null && - (item.runStatus === "completed" || item.attemptStatus === "superseded"))) && - !scannedRuns.has(`${sourceThreadId}:${item.runId}`) - ? [ - [ - `${sourceThreadId}:${item.runId}`, - { threadId: sourceThreadId, id: item.runId }, - ] as const, - ] - : [], - ), - ); - if (runs.size === 0) break; - for (const run of runs.keys()) scannedRuns.add(run); - const found = yield* sql<{ thread_id: string; turn_item_id: string }>` + const window = + local === null + ? null + : yield* selectHistoryPageInWindow( + local, + { cursor, throughEntryId, workMode }, + select, + ); + const whole = Effect.gen(function* () { + const index = full ?? (yield* readSnapshotTimelineIndex(threadId, withWork)); + const indexed = index.visible.map((row, position): IndexedTimelineRow => ({ + ...row, + position, + canonicalPosition: position, + })); + let selection = yield* select(indexed); + // The full index does not say which rows hold inline output: finding it scans + // payloads, which costs a whole long thread several seconds. Scan only the runs + // the page spans, then select again with what they hold. + const scannedRuns = new Set(); + const persistentOutputs = new Set(); + // Rows hidden in a work group move the page, so the scan always reaches the + // cursor's own row, or the row its stale position falls back to. + const cursorPosition = + workMode === undefined + ? -1 + : Math.min( + findCursorEndIndex(indexed, decodeThreadHistoryCursor(cursor)), + indexed.length - 1, + ); + while (workMode !== undefined) { + const { page } = selection; + const span = [...page.items.slice(0, 1), ...page.items.slice(-1)].map( + (row) => row.canonicalPosition, + ); + if (cursorPosition >= 0) span.push(cursorPosition); + if (span.length === 0) break; + const runs = new Map( + // A work page holds rows of one run, which can be the rows that have no run. + (workMode === "work" + ? page.items.slice(0, 1) + : indexed.slice(Math.min(...span), Math.max(...span) + 1) + ).flatMap(({ sourceThreadId, item }) => + (workMode === "work" || + (item.runId !== null && + (item.runStatus === "completed" || item.attemptStatus === "superseded"))) && + !scannedRuns.has(`${sourceThreadId}:${item.runId}`) + ? [ + [ + `${sourceThreadId}:${item.runId}`, + { threadId: sourceThreadId, id: item.runId }, + ] as const, + ] + : [], + ), + ); + if (runs.size === 0) break; + for (const run of runs.keys()) scannedRuns.add(run); + const found = yield* sql<{ thread_id: string; turn_item_id: string }>` SELECT item.thread_id, item.turn_item_id FROM json_each(${yield* encodeTimelineSources([...runs.values()])}) AS wanted CROSS JOIN orchestration_v2_projection_turn_items AS item ON item.thread_id = json_extract(wanted.value, '$.threadId') AND item.run_id IS json_extract(wanted.value, '$.id') WHERE item.status = 'completed' AND ${inlineWorkOutput}`; - if (found.length === 0) break; - for (const row of found) - persistentOutputs.add(`${row.thread_id}:${row.turn_item_id}`); - page = yield* selectPage( - indexed.map((row) => - persistentOutputs.has(`${row.sourceThreadId}:${row.sourceItemId}`) - ? { ...row, item: { ...row.item, persistentOutput: true } } - : row, - ), - ); - } - const policy = workMode === "work" ? THREAD_HISTORY_PAGE_POLICY : olderPolicy; + if (found.length === 0) break; + for (const row of found) + persistentOutputs.add(`${row.thread_id}:${row.turn_item_id}`); + selection = yield* select( + indexed.map((row) => + persistentOutputs.has(`${row.sourceThreadId}:${row.sourceItemId}`) + ? { ...row, item: { ...row.item, persistentOutput: true } } + : row, + ), + ); + } + return { + indexed, + selection, + olderUserTurn: Effect.succeed(false), + interruptRequests: Effect.succeed(indexed), + }; + }); + const { indexed, selection, olderUserTurn, interruptRequests } = + window ?? (yield* whole); + const { page } = selection; + const policy = workMode === "work" ? THREAD_HISTORY_PAGE_POLICY : selection.policy; const end = (page.items.at(-1)?.canonicalPosition ?? -1) + 1; - const hasUserTurns = index.visible - .slice(0, end) - .some((row) => isThreadHistoryUserTurn(row.item)); + const hasUserTurns = + conversationOnly === true && + (indexed.some( + (row) => row.canonicalPosition < end && isThreadHistoryUserTurn(row.item), + ) || + (yield* olderUserTurn)); const materialize = Effect.fn("ProjectionStore.materializeHistoryRows")(function* ( - rows: ReadonlyArray<(typeof indexed)[number]>, + rows: ReadonlyArray, ) { if (rows.length === 0) return []; const sources = yield* encodeTimelineSources( @@ -6606,7 +7102,7 @@ export const layer: Layer.Layer = ); }); const items = yield* materialize( - conversationOnly && hasUserTurns + hasUserTurns ? page.items.filter((row) => isConversationHistoryItem(row.item)) : page.items, ); @@ -6614,7 +7110,14 @@ export const layer: Layer.Layer = const requests = workMode === "work" ? [] - : yield* materialize(missingInterruptRequests(indexed, bounded.items)); + : yield* materialize( + missingInterruptRequests( + bounded.items.some((row) => row.item.type === "run_interrupt_result") + ? yield* interruptRequests + : [], + bounded.items, + ), + ); const pageItems = [...requests, ...bounded.items].map((row, position) => ({ ...row, position, @@ -6631,39 +7134,118 @@ export const layer: Layer.Layer = ) .pipe(Effect.mapError(controlReadError(threadId))); + /** + * The rows `getTimelinePage` matches, cut after the first `limit + 1`, with their positions + * in the full visible index. Reads only the rows from the start position to the page end. + */ + const matchLocalTimelineRows = Effect.fn("ProjectionStore.matchLocalTimelineRows")(function* ( + local: LocalTimeline, + options: ProjectionTimelinePageOptions, + ) { + const threadId = local.records.thread.id; + const totals = yield* sql<{ count: number }>` + SELECT COUNT(*) AS count FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId}`; + const totalItems = (totals[0]?.count ?? 0) - local.hidden.length; + if (options.itemId !== undefined) { + const found = (yield* local.read(sql`AND item.turn_item_id = ${options.itemId}`)).filter( + local.isVisible, + ); + const matching = yield* Effect.forEach(found, (item) => + local + .visibleBefore(item.ordinal, item.id) + .pipe(Effect.map((position) => ({ ...localTimelineRow(item), position }))), + ); + return { matching, totalItems }; + } + const afterPosition = options.afterPosition ?? -1; + const firstPosition = Math.max(afterPosition + 1, 0); + // The row at this raw offset sorts at or before the first wanted row: hidden rows only + // push visible rows to later offsets. + const starts = yield* sql<{ ordinal: number }>` + SELECT ordinal FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} + ORDER BY ordinal ASC, turn_item_id ASC LIMIT 1 OFFSET ${firstPosition}`; + const start = starts[0]?.ordinal; + if (start === undefined) return { matching: [], totalItems }; + const base = yield* local.visibleBefore(start); + const hiddenAfter = local.hidden.filter((row) => row.ordinal >= start).length; + const messagesOnly = options.view !== "activity"; + const candidates = (yield* local.read( + messagesOnly + ? sql`AND item.ordinal >= ${start} + AND item.type IN ('user_message', 'assistant_message', 'proposed_plan')` + : sql`AND item.ordinal >= ${start}`, + sql`LIMIT ${firstPosition - base + Math.min(options.limit, totalItems) + 1 + hiddenAfter}`, + )).filter(local.isVisible); + const positions = new Map(); + const last = candidates.at(-1); + if (messagesOnly && last !== undefined) { + const hiddenIds = new Set(local.hidden.map((row) => row.turn_item_id)); + const span = yield* sql<{ turn_item_id: string }>` + SELECT turn_item_id FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND ordinal >= ${start} AND ordinal <= ${last.ordinal} + ORDER BY ordinal ASC, turn_item_id ASC`; + for (const row of span) { + if (!hiddenIds.has(row.turn_item_id)) + positions.set(row.turn_item_id, base + positions.size); + } + } + const matching = candidates + .map((item, offset) => ({ + ...localTimelineRow(item), + position: messagesOnly ? (positions.get(item.id) ?? -1) : base + offset, + })) + .filter((row) => row.position > afterPosition); + return { matching, totalItems }; + }); + const getTimelinePage: ProjectionStoreV2Shape["getTimelinePage"] = (threadId, options) => sql .withTransaction( Effect.gen(function* () { - const index = yield* readSnapshotTimelineIndex(threadId, false); - const attachmentOwners = - options.attachmentId === undefined - ? undefined - : new Set( - (yield* sql<{ thread_id: string; turn_item_id: string }>` + const local = + options.attachmentId === undefined && + Number.isSafeInteger(options.limit) && + options.limit >= 0 && + Number.isSafeInteger(options.afterPosition ?? -1) + ? yield* openLocalTimeline(threadId) + : null; + const whole = Effect.gen(function* () { + const index = yield* readSnapshotTimelineIndex(threadId, false); + const attachmentOwners = + options.attachmentId === undefined + ? undefined + : new Set( + (yield* sql<{ thread_id: string; turn_item_id: string }>` SELECT item.thread_id, item.turn_item_id FROM orchestration_v2_projection_turn_items AS item, json_each(item.payload_json, '$.attachments') AS attachment WHERE item.thread_id IN ${sql.in([...new Set(index.visible.map((row) => row.sourceThreadId))])} AND item.type = 'user_message' AND json_extract(attachment.value, '$.id') = ${options.attachmentId}`).map( - (row) => `${row.thread_id}:${row.turn_item_id}`, - ), - ); - const matching = index.visible - .map((row, position) => ({ ...row, position })) - .filter((row) => - options.itemId === undefined - ? row.position > (options.afterPosition ?? -1) && - (options.view === "activity" || - ["user_message", "assistant_message", "proposed_plan"].includes( - row.item.type, - )) - : row.sourceItemId === options.itemId, - ) - .filter( - (row) => attachmentOwners?.has(`${row.sourceThreadId}:${row.sourceItemId}`) ?? true, - ); + (row) => `${row.thread_id}:${row.turn_item_id}`, + ), + ); + const matching = index.visible + .map((row, position) => ({ ...row, position })) + .filter((row) => + options.itemId === undefined + ? row.position > (options.afterPosition ?? -1) && + (options.view === "activity" || + ["user_message", "assistant_message", "proposed_plan"].includes( + row.item.type, + )) + : row.sourceItemId === options.itemId, + ) + .filter( + (row) => + attachmentOwners?.has(`${row.sourceThreadId}:${row.sourceItemId}`) ?? true, + ); + return { matching, totalItems: index.visible.length }; + }); + const { matching, totalItems } = + local === null ? yield* whole : yield* matchLocalTimelineRows(local, options); const page = matching.slice(0, options.limit); const sources = yield* encodeTimelineSources( page @@ -6713,11 +7295,7 @@ export const layer: Layer.Layer = }; }), ); - return { - items, - totalItems: index.visible.length, - hasMore: matching.length > page.length, - }; + return { items, totalItems, hasMore: matching.length > page.length }; }), ) .pipe( diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts index 111cf0541358..8d81cbfadda7 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts @@ -396,6 +396,38 @@ describe("threadHistoryPaging", () => { expect([...older.items, ...first.items].map((row) => row.sourceItemId)).toEqual( items.map((row) => row.sourceItemId), ); + + // A caller that holds only the newest rows gets the same page when it says what it left + // out: a user turn, and run 3, whose late notification is then not a new turn. + const late = items.with(402, inRun(402, 3, notification("Late"))); + const cursor = encodeThreadHistoryCursor({ + snapshotSequence: 1, + sourceThreadId: THREAD, + sourceItemId: late.at(-1)!.sourceItemId, + position: late.length - 1, + }); + const prefix = { hasUserTurn: true, startedRuns: new Set(["run-3"]) }; + const whole = selectHistoryPageFromCursor({ items: late, cursor, snapshotSequence: 1, policy }); + expect(whole.items[0]?.sourceItemId).toBe(`item-${11 * 4 + 1}`); + expect( + selectHistoryPageFromCursor({ + items: late.slice(22), + cursor, + snapshotSequence: 1, + policy, + prefix, + }), + ).toEqual({ ...whole, oldestScanned: late[45] }); + // A part that ends above the page reports its first row, so the caller reads further back. + expect( + selectHistoryPageFromCursor({ + items: late.slice(60), + cursor, + snapshotSequence: 1, + policy, + prefix, + }).oldestScanned, + ).toBe(late[60]); }); it("pages agent-only child transcripts instead of dropping their earlier activity", () => { diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index b79038a77a80..f088d389fdd5 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -72,10 +72,22 @@ type HistoryRow = Pick< readonly deferredWork?: OrchestrationV2TurnItem["deferredWork"]; }; }; +/** What a caller that passes only the newest part of a history knows about the rows it left out. */ +export type ThreadHistoryPrefix = { + readonly hasUserTurn: boolean; + /** Runs that have a user message or notification among the rows left out. */ + readonly startedRuns: ReadonlySet; +}; + export type SelectTimelinePageResult = { readonly items: ReadonlyArray; readonly nextCursor: string | null; readonly hasMoreHistory: boolean; + /** + * Set with `prefix`: the oldest row the selection read, which can be older than `items[0]`. + * The page is the same as on the full history only when a row lies before this one. + */ + readonly oldestScanned?: Row | undefined; }; export type BoundedProjectionResult = { @@ -200,6 +212,15 @@ function isDeferrableWorkItem(item: HistoryRow["item"]): boolean { ); } +/** A row that folds when its run is not blocked and its group is large enough. */ +export function isFoldableWorkItem(item: HistoryRow["item"]): boolean { + return ( + item.runId != null && + (item.runStatus === "completed" || item.attemptStatus === "superseded") && + isDeferrableWorkItem(item) + ); +} + export function deferWorkHistoryRows(items: ReadonlyArray): Row[] { const blocked = new Set( items.flatMap(({ item }) => @@ -217,12 +238,7 @@ export function deferWorkHistoryRows(items: ReadonlyArra const item = row.item; if (item.runId == null) continue; if (item.type === "user_message") segments.set(item.runId, (segments.get(item.runId) ?? 0) + 1); - if ( - (item.runStatus !== "completed" && item.attemptStatus !== "superseded") || - blocked.has(item.runId) || - !isDeferrableWorkItem(item) - ) - continue; + if (blocked.has(item.runId) || !isFoldableWorkItem(item)) continue; const key = `${row.sourceThreadId}:${item.runId}:${segments.get(item.runId) ?? 0}:${item.attemptId ?? ""}`; const group = groups.get(key) ?? []; group.push(row); @@ -287,8 +303,11 @@ export function selectDeferredWorkPage(input: { } /** Notifications can start wake turns when they are the run's first input; match ProjectionStore run_starts. */ -function threadHistoryTurnStarts(items: ReadonlyArray): ReadonlyArray { - const startedRuns = new Set(); +function threadHistoryTurnStarts( + items: ReadonlyArray, + earlierRuns?: ReadonlySet, +): ReadonlyArray { + const startedRuns = new Set(earlierRuns); return items.map((item) => { if (item.type !== "user_message" && item.type !== "notification") return false; const firstInput = item.runId != null && !startedRuns.has(item.runId); @@ -312,6 +331,7 @@ function selectOlderTimelinePage(input: { readonly snapshotSequence: number; readonly policy?: ThreadHistoryPagePolicy | undefined; readonly rowEncodedBytes?: ((row: Row) => number) | undefined; + readonly prefix?: ThreadHistoryPrefix | undefined; }): SelectTimelinePageResult { const policy = input.policy ?? THREAD_HISTORY_PAGE_POLICY; const rowCost = input.rowEncodedBytes ?? projectedRowEncodedBytes; @@ -325,10 +345,15 @@ function selectOlderTimelinePage(input: { let userTurns = 0; let rawTurns = 0; const candidates = input.items.slice(0, end); - const turnStarts = threadHistoryTurnStarts(candidates.map((row) => row.item)); - const turnLimit = candidates.some((row) => isThreadHistoryUserTurn(row.item)) - ? policy.maxUserTurns - : undefined; + const turnStarts = threadHistoryTurnStarts( + candidates.map((row) => row.item), + input.prefix?.startedRuns, + ); + const turnLimit = + input.prefix?.hasUserTurn === true || + candidates.some((row) => isThreadHistoryUserTurn(row.item)) + ? policy.maxUserTurns + : undefined; const maxItems = turnLimit === undefined ? policy.maxItems : (policy.maxTurnItems ?? policy.maxItems); for (let index = end - 1; index >= 0; index -= 1) { @@ -351,6 +376,7 @@ function selectOlderTimelinePage(input: { // Inside a turn longer than the window, no selected row starts a user turn. // Later pages and the snapshot query see only such rows and use `maxItems`, // so this page uses it too. + const oldestScanned = selected.at(-1); if (userTurns === 0) selected.splice(policy.maxItems); selected.reverse(); @@ -376,6 +402,7 @@ function selectOlderTimelinePage(input: { items: renumberPositions(selected), nextCursor, hasMoreHistory, + ...(input.prefix === undefined ? {} : { oldestScanned }), }; } @@ -458,6 +485,8 @@ export function selectHistoryPageFromCursor(input: { readonly policy?: ThreadHistoryPagePolicy | undefined; readonly rowEncodedBytes?: ((row: Row) => number) | undefined; readonly throughEntryId?: string | undefined; + /** Set when `items` holds only the newest part of the history. */ + readonly prefix?: ThreadHistoryPrefix | undefined; }): SelectTimelinePageResult { const decoded = decodeThreadHistoryCursor(input.cursor); return selectOlderTimelinePage({ @@ -466,6 +495,7 @@ export function selectHistoryPageFromCursor(input: { snapshotSequence: input.snapshotSequence, ...(input.rowEncodedBytes === undefined ? {} : { rowEncodedBytes: input.rowEncodedBytes }), policy: olderHistoryPagePolicy(input), + prefix: input.prefix, }); } From f4cf69f522a9c7acc29a519debafad37db11f030 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:34:30 -0800 Subject: [PATCH 020/367] fix(server): selecting a remote branch lands on its local branch (#35400) --- apps/server/src/vcs/GitVcsDriverCore.test.ts | 130 ++++++++++++++----- apps/server/src/vcs/GitVcsDriverCore.ts | 33 +++-- 2 files changed, 123 insertions(+), 40 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 6dec0cf0207b..542057e60c3e 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -2184,6 +2184,31 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect( + "switches a remote ref to its tracking branch when a tag shares the branch name", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", "HEAD:refs/heads/v1.2"]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["branch", "--track", "v1.2", "origin/v1.2"]); + yield* git(cwd, ["tag", "v1.2"]); + assert.equal(yield* git(cwd, ["branch", "--show-current"]), initialBranch); + + const result = yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ + cwd, + refName: "origin/v1.2", + }); + + assert.equal(result.refName, "v1.2"); + assert.equal(yield* git(cwd, ["symbolic-ref", "HEAD"]), "refs/heads/v1.2"); + }), + ); + it.effect.each([ // Git cannot tell a missing revision from a missing file in these. { label: "an argument that may be a path", args: ["log", "-1", "missing"] }, @@ -6070,39 +6095,82 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("tracks a remote start point once another process releases the config lock", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - const { initialBranch } = yield* initRepoWithCommit(cwd); - const remote = yield* makeTmpDir("git-remote-"); - yield* git(remote, ["init", "--bare"]); - yield* git(cwd, ["remote", "add", "origin", remote]); - yield* git(cwd, ["push", "origin", initialBranch]); - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const lockPath = path.join(cwd, ".git", "config.lock"); - yield* fileSystem.writeFileString(lockPath, ""); - const { driver, failures } = yield* makeDriverReleasingLock( - (args) => args.some((arg) => arg.startsWith("--set-upstream-to")), - lockPath, - ); + it.effect.each(["createWorktree", "switchRef"] as const)( + "tracks a remote start point once another process releases the config lock: %s", + (operation) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", `${initialBranch}:refs/heads/feature/tracked`]); + yield* git(cwd, ["fetch", "origin"]); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const lockPath = path.join(cwd, ".git", "config.lock"); + yield* fileSystem.writeFileString(lockPath, ""); + const { driver, failures } = yield* makeDriverReleasingLock( + (args) => args.some((arg) => arg.startsWith("--set-upstream-to")), + lockPath, + ); - // The upstream retry waits on the real clock; the lock is already gone by then. - yield* TestClock.withLive( - driver.createWorktree({ - cwd, - path: path.join(yield* makeTmpDir("git-worktrees-"), "tracked"), - refName: `origin/${initialBranch}`, - newRefName: "feature/tracked", - }), - ); + // The upstream retry waits on the real clock; the lock is already gone by then. + if (operation === "createWorktree") { + yield* TestClock.withLive( + driver.createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "tracked"), + refName: "origin/feature/tracked", + newRefName: "feature/tracked", + }), + ); + } else { + const switched = yield* TestClock.withLive( + driver.switchRef({ cwd, refName: "origin/feature/tracked" }), + ); + assert.equal(switched.refName, "feature/tracked"); + } - assert.equal(yield* Ref.get(failures), 1); - assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature/tracked@{upstream}"]), - `origin/${initialBranch}`, - ); - }), + assert.equal(yield* Ref.get(failures), 1); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature/tracked@{upstream}"]), + "origin/feature/tracked", + ); + }), + ); + + it.effect( + "leaves no untracked branch behind when the config stays locked during a switch", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", remote]); + yield* git(cwd, ["push", "upstream", `${initialBranch}:refs/heads/feature`]); + yield* git(cwd, ["fetch", "upstream"]); + const fileSystem = yield* FileSystem.FileSystem; + const lockPath = (yield* Path.Path).join(cwd, ".git", "config.lock"); + yield* fileSystem.writeFileString(lockPath, ""); + const driver = yield* GitVcsDriver.GitVcsDriver; + + // The upstream retry waits on the real clock before giving up. + yield* TestClock.withLive(driver.switchRef({ cwd, refName: "upstream/feature" })).pipe( + Effect.flip, + ); + assert.equal(yield* git(cwd, ["branch", "--show-current"]), initialBranch); + assert.equal(yield* git(cwd, ["branch", "--list", "feature"]), ""); + + yield* fileSystem.remove(lockPath); + const switched = yield* driver.switchRef({ cwd, refName: "upstream/feature" }); + assert.equal(switched.refName, "feature"); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature@{upstream}"]), + "upstream/feature", + ); + }), ); it.effect("fetches a remote branch into a tracking branch past another process's lock", () => diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 5eae512995d3..a0b99e6d6d7f 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -5257,7 +5257,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ? yield* executeGit( "GitVcsDriver.switchRef.localTrackingBranch", input.cwd, - ["for-each-ref", "--format=%(refname:short)\t%(upstream:short)", "refs/heads"], + // `:short` prefixes a name a tag shares with `heads/`, so names are unambiguous here. + ["for-each-ref", "--format=%(refname:lstrip=2)\t%(upstream)", "refs/heads"], { timeoutMs: 5_000, allowNonZeroExit: true, @@ -5265,7 +5266,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ).pipe( Effect.map((result) => result.exitCode === 0 - ? parseTrackingBranchByUpstreamRef(result.stdout, input.refName) + ? parseTrackingBranchByUpstreamRef(result.stdout, `refs/remotes/${input.refName}`) : null, ), ) @@ -5307,15 +5308,29 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* // Created before the checkout so the shared-config lock is not held while files are written. // Not rolled back on failure: Git may already have switched HEAD to it (post-checkout hooks). if (newTrackingBranch) { - yield* executeGit( + const startRef = `refs/remotes/${input.refName}`; + yield* executeGitWithStableDiagnostics( "GitVcsDriver.switchRef.createTrackingBranch", input.cwd, - ["branch", "--track", "--", newTrackingBranch, input.refName], - { - timeoutMs: 10_000, - fallbackErrorDetail: "git branch create failed", - writesSharedConfig: true, - }, + ["branch", "--no-track", "--", newTrackingBranch, startRef], + { fallbackErrorDetail: "git branch create failed" }, + ); + // An untracked branch of this name would make every later switch to the remote detach. + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.switchRef.trackRemote", + input.cwd, + ["branch", `--set-upstream-to=${startRef}`, "--", newTrackingBranch], + { writesSharedConfig: true }, + ).pipe( + retryWhileConfigLocked, + Effect.tapError(() => + executeGit( + "GitVcsDriver.switchRef.removeUntrackedBranch", + input.cwd, + ["update-ref", "-d", `refs/heads/${newTrackingBranch}`, startRef], + { allowNonZeroExit: true }, + ), + ), ); } From a2685e64d34945bed0f6317b48b38ef57a7cb558 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:34:44 -0800 Subject: [PATCH 021/367] fix(web): SCP-style Git remotes no longer become terminal file links (#35404) --- apps/web/src/terminal-links.test.ts | 11 +++++++++++ apps/web/src/terminal-links.ts | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/apps/web/src/terminal-links.test.ts b/apps/web/src/terminal-links.test.ts index 130e4d276ef7..a0fb98529cf0 100644 --- a/apps/web/src/terminal-links.test.ts +++ b/apps/web/src/terminal-links.test.ts @@ -64,6 +64,10 @@ describe("extractTerminalLinks", () => { expect(extractTerminalLinks(`see ${quote}${text}${quote}.`)).toEqual([ { kind: "path", text, start: 5, end: 5 + text.length }, ]); + const atText = "src/my contact@support.vue:12:3"; + expect(extractTerminalLinks(`see ${quote}${atText}${quote}.`)).toEqual([ + { kind: "path", text: atText, start: 5, end: 5 + atText.length }, + ]); }); it.each([ @@ -127,6 +131,11 @@ describe("extractTerminalLinks", () => { "src/𐐀.ts:12", "src/app/[id]/page.tsx:3:10", "src/routes/(app)/+page.svelte:12:4", + "src/routes/(app)/+page@.svelte:12:3", + "src/contact@support.vue:12:3", + String.raw`src\contact@support.ts:12:3`, + "src/équipe@home.ts:12:3", + "src/foo$contact@support.ts:12:3", "node_modules/.pnpm/react@18.3.1/node_modules/react/index.js:12:5", "app/routes/posts.$id.tsx:5", ])("keeps the complete relative path %s", (text) => { @@ -198,6 +207,8 @@ describe("extractTerminalLinks", () => { " at main (file:///Users/jane/app/src/index.mjs:3:7)", "+redis://localhost:6379/0", "-redis://localhost:6379/0", + "git clone git@github.com:acme/widgets.git", + "remote: git@gitlab.com:group/sub/project.git", ])("does not turn part of a non-web URL into an editor link: %s", (line) => { expect(extractTerminalLinks(line)).toEqual([]); }); diff --git a/apps/web/src/terminal-links.ts b/apps/web/src/terminal-links.ts index 114219e721eb..d77848a8c26b 100644 --- a/apps/web/src/terminal-links.ts +++ b/apps/web/src/terminal-links.ts @@ -46,6 +46,9 @@ const REQUEST_ROUTE_PATTERN = // The tail of `redis://host/0` or `file:///a.mjs` is not a path. One-letter schemes stay drives, // a diff's `+` or `-` may lead, and the lookbehind keeps runs from being rescanned at every suffix. const SCHEME_URL_PATTERN = /(?]+/gu; +// Nor is the tail of an SCP-style Git remote such as `git@github.com:acme/widgets.git`. Only a +// path wholly inside the tail is dropped, so `src/contact@support.vue:12:3` keeps its link. +const SCP_REMOTE_PATTERN = /(?]+/gu; function trimClosingDelimiters(value: string, kind: TerminalLinkKind): string { let output = value.replace( @@ -119,6 +122,10 @@ export function extractTerminalLinks(line: string): TerminalLinkMatch[] { [...line.matchAll(REQUEST_ROUTE_PATTERN), ...line.matchAll(SCHEME_URL_PATTERN)], (match) => ({ start: match.index, end: match.index + match[0].length }), ); + const remoteTails = Array.from(line.matchAll(SCP_REMOTE_PATTERN), (match) => ({ + start: match.index + match[0].indexOf(":") + 1, + end: match.index + match[0].length, + })); const urlMatches = collectMatches(line, "url", URL_PATTERN, promptMatches); const quotedMatches = collectMatches(line, "path", QUOTED_FILE_PATH_PATTERN, [ ...promptMatches, @@ -131,7 +138,10 @@ export function extractTerminalLinks(line: string): TerminalLinkMatch[] { ]); return [...promptMatches, ...urlMatches, ...quotedMatches, ...pathMatches] .filter( - (match) => match.kind === "url" || !nonPathRanges.some((range) => overlaps(match, range)), + (match) => + match.kind === "url" || + (!nonPathRanges.some((range) => overlaps(match, range)) && + !remoteTails.some((tail) => match.start >= tail.start && match.end <= tail.end)), ) .toSorted((a, b) => a.start - b.start); } From d0f9d4dcc01ba89f28c5cc3d948829dff6fcf6e5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:37:01 -0800 Subject: [PATCH 022/367] fix(web): sidebar draft previews show cited assistant text, not the raw link (#35406) --- .../sidebar/SidebarDraftBlock.browser.test.tsx | 18 +++++++++++++++++- .../components/sidebar/SidebarDraftBlock.tsx | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/apps/web/src/components/sidebar/SidebarDraftBlock.browser.test.tsx b/apps/web/src/components/sidebar/SidebarDraftBlock.browser.test.tsx index c29932f5757e..41cd795013e4 100644 --- a/apps/web/src/components/sidebar/SidebarDraftBlock.browser.test.tsx +++ b/apps/web/src/components/sidebar/SidebarDraftBlock.browser.test.tsx @@ -1,9 +1,10 @@ import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment"; -import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; +import { EnvironmentId, MessageId, ProjectId, ThreadId } from "@t3tools/contracts"; import { Profiler } from "react"; import { expect, it, vi } from "vite-plus/test"; import { act, render } from "../../../test/render"; +import { formatAssistantCitationForComposer } from "../../composer-logic"; import { DraftId, useComposerDraftStore } from "../../composerDraftStore"; import { SidebarDraftBlock } from "./SidebarDraftBlock"; @@ -53,6 +54,21 @@ it("keeps draft previews current through edits, promotion and removal", async () await act(() => initial.setPrompt(draftId, "Updated draft preview")); await expect.element(row).toHaveAccessibleName("Updated draft preview, Unsent draft"); + const quote = "Cache the parsed config per worktree."; + const citation = formatAssistantCitationForComposer({ + version: 1, + environmentId, + threadId: ThreadId.make("source-thread"), + messageId: MessageId.make("source-message"), + text: quote, + start: 0, + end: quote.length, + prefix: "", + suffix: "", + }); + await act(() => initial.setPrompt(draftId, `${citation}Why per worktree?`)); + await expect.element(row).toHaveAccessibleName(`${quote} Why per worktree?, Unsent draft`); + await act(() => initial.markDraftThreadPromoting(draftId)); await expect.element(row).not.toBeInTheDocument(); await act(() => initial.clearDraftThread(draftId)); diff --git a/apps/web/src/components/sidebar/SidebarDraftBlock.tsx b/apps/web/src/components/sidebar/SidebarDraftBlock.tsx index bc15ca594597..c4868c4776ea 100644 --- a/apps/web/src/components/sidebar/SidebarDraftBlock.tsx +++ b/apps/web/src/components/sidebar/SidebarDraftBlock.tsx @@ -1,4 +1,5 @@ import { discardComposerDraft } from "../../lib/discardComposerDraft"; +import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import { replaceComposerContextReferences } from "@t3tools/shared/composerContextReferences"; import { SquarePenIcon, XIcon } from "lucide-react"; import { @@ -34,7 +35,9 @@ const SidebarDraftRow = memo(function SidebarDraftRow(props: { }) { const { composer, draftId, onContextMenu, onDiscard, onNavigate } = props; const promptPreview = - replaceComposerContextReferences(composer.prompt, (occurrence) => occurrence.label) + assistantCitationsToPlainText( + replaceComposerContextReferences(composer.prompt, (occurrence) => occurrence.label), + ) .trim() .split("\n", 1)[0] ?? ""; // images mirrors persistedAttachments once rehydration finishes; before From 0cd6a73433c3b9c1e328560d591fd297e72d7ef9 Mon Sep 17 00:00:00 2001 From: Srijan Dubey <87586713+Srijan-D@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:08:37 +0530 Subject: [PATCH 023/367] fix(web): pasting an image the draft already has no longer flashes a preparing tile (#35265) Co-authored-by: Julius Marminge --- apps/web/src/components/chat/ChatComposer.tsx | 4 +- .../chat/composerAttachmentFiles.ts | 22 +++ .../chat/preparingComposerImages.ts | 10 +- ...eComposerAttachmentIntake.browser.test.tsx | 125 ++++++++++++++++++ .../chat/useComposerAttachmentIntake.ts | 24 +++- apps/web/src/composerDraftStore.ts | 10 +- apps/web/src/lib/imageCompression.test.ts | 8 +- apps/web/src/lib/imageCompression.ts | 8 +- 8 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 apps/web/src/components/chat/useComposerAttachmentIntake.browser.test.tsx diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index bafafcb9612c..6f6d4fbd50ff 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -698,7 +698,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) uncommittedSnapShotIds, standaloneComposerImages, } = useComposerImageGroups(attachmentDraftTarget, composerImages, composerPreviewAnnotations); - const preparingComposerImages = usePreparingComposerImages(attachmentTargetKey); + const preparingComposerImages = usePreparingComposerImages(attachmentTargetKey).filter( + (image) => !image.hidden, + ); const nonPersistedComposerImageIds = attachmentDraft.nonPersistedImageIds; const uploadsByImageId = useAttachmentUploadStore((state) => state.uploadsByImageId); const openPrLink = useOpenPrLink(routeThreadRef); diff --git a/apps/web/src/components/chat/composerAttachmentFiles.ts b/apps/web/src/components/chat/composerAttachmentFiles.ts index 96575d393018..f613b695ffd5 100644 --- a/apps/web/src/components/chat/composerAttachmentFiles.ts +++ b/apps/web/src/components/chat/composerAttachmentFiles.ts @@ -61,6 +61,28 @@ export function normalizeComposerImageFileMimeType(file: File): File { }); } +export function composerImageDedupKey( + image: Pick, +): string { + // Hydrated images get fresh lastModified values, so exclude it from dedupe. + return `${image.mimeType}\u0000${image.sizeBytes}\u0000${image.name}`; +} + +// Compression changes an image's bytes, so a compressed image answers for the file it came from. +const composerImageSourceKeys = new WeakMap(); + +/** The draft store's image dedupe key, taken from the file as pasted rather than as compressed. */ +export function composerImageSourceKey(file: File): string { + return ( + composerImageSourceKeys.get(file) ?? + composerImageDedupKey({ mimeType: file.type, sizeBytes: file.size, name: file.name || "image" }) + ); +} + +export function rememberComposerImageSource(source: File, prepared: File): void { + if (prepared !== source) composerImageSourceKeys.set(prepared, composerImageSourceKey(source)); +} + function isDngFile(file: Pick): boolean { const mimeType = file.type.split(";", 1)[0]?.trim().toLowerCase(); return ( diff --git a/apps/web/src/components/chat/preparingComposerImages.ts b/apps/web/src/components/chat/preparingComposerImages.ts index 4ba7696a1789..def7f023e861 100644 --- a/apps/web/src/components/chat/preparingComposerImages.ts +++ b/apps/web/src/components/chat/preparingComposerImages.ts @@ -5,6 +5,8 @@ export interface PreparingComposerImage { readonly name: string; /** The uncompressed source, thumbnailed so the placeholder shows the image itself. */ readonly file: File; + /** Looks like an image the draft already has: it holds its slot but shows no tile. */ + readonly hidden: boolean; } const NO_PREPARING_IMAGES: ReadonlyArray = []; @@ -24,8 +26,14 @@ export function usePreparingComposerImages(targetKey: string) { ); } +export function getPreparingComposerImages( + targetKey: string, +): ReadonlyArray { + return usePreparingComposerImageStore.getState().byTargetKey[targetKey] ?? NO_PREPARING_IMAGES; +} + export function countPreparingComposerImages(targetKey: string): number { - return usePreparingComposerImageStore.getState().byTargetKey[targetKey]?.length ?? 0; + return getPreparingComposerImages(targetKey).length; } export function addPreparingComposerImages( diff --git a/apps/web/src/components/chat/useComposerAttachmentIntake.browser.test.tsx b/apps/web/src/components/chat/useComposerAttachmentIntake.browser.test.tsx new file mode 100644 index 000000000000..cfacb19bc410 --- /dev/null +++ b/apps/web/src/components/chat/useComposerAttachmentIntake.browser.test.tsx @@ -0,0 +1,125 @@ +import { useEffect } from "react"; +import { ThreadId } from "@t3tools/contracts"; +import { expect, it, vi } from "vite-plus/test"; + +import { act, render } from "../../../test/render"; +import { + type ComposerImageAttachment, + DraftId, + useComposerDraftStore, +} from "../../composerDraftStore"; +import { getPreparingComposerImages } from "./preparingComposerImages"; +import { useComposerAttachmentIntake } from "./useComposerAttachmentIntake"; + +const compression = vi.hoisted(() => ({ prepare: vi.fn() })); +vi.mock("../../lib/imageCompression", async (importOriginal) => ({ + ...(await importOriginal()), + prepareImageForAttachment: compression.prepare, +})); + +type IntakeInput = Parameters[0]; +type Intake = ReturnType; + +function Harness({ input, ready }: { input: IntakeInput; ready: (value: Intake) => void }) { + const intake = useComposerAttachmentIntake(input); + useEffect(() => ready(intake)); + return null; +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((fulfill) => { + resolve = fulfill; + }); + return { promise, resolve }; +} + +// A clipboard image arrives as a new File named image.png on every paste. +const pastedImage = (fill = 0) => + new File([new Uint8Array(100).fill(fill)], "image.png", { type: "image/png" }); +// Compression shrinks an image, so the draft's copy no longer matches the pasted bytes; images +// with different pixels compress to different sizes. +const compress = async (source: File) => { + const [firstByte = 0] = new Uint8Array(await source.arrayBuffer()); + return { + ok: true as const, + file: new File([new Uint8Array(60 + firstByte)], "image.png", { type: "image/png" }), + recompressed: true as const, + }; +}; +const visiblePreparing = (targetKey: string) => + getPreparingComposerImages(targetKey).filter((image) => !image.hidden); + +it("prepares a repeated paste without a tile, and still keeps a different image", async () => { + const target = DraftId.make("intake-repeat-paste"); + const store = useComposerDraftStore.getState(); + using _draft = { [Symbol.dispose]: () => store.clearComposerContent(target) }; + const firstCompression = deferred(); + compression.prepare + .mockReset() + .mockImplementationOnce(async (file: File) => { + await firstCompression.promise; + return compress(file); + }) + .mockImplementation(compress); + const composerImagesRef = { current: [] as ComposerImageAttachment[] }; + const addImages = (images: ComposerImageAttachment[]) => { + const ids = store.addImages(target, images); + composerImagesRef.current = + useComposerDraftStore.getState().getComposerDraft(target)?.images ?? []; + return ids; + }; + const input = { + activeThreadId: ThreadId.make("thread"), + activePendingProgress: null, + activePendingIsResponding: false, + pendingUserInputs: [], + supportsQuestionAttachments: false, + isLiteralPendingAnswer: false, + projectSelectionRequired: false, + questionAttachmentTarget: null, + attachmentTargetKey: target, + fileStagingLimit: null, + promptRef: { current: "" }, + composerImagesRef, + composerFilesRef: { current: [] }, + composerEditorRef: { current: null }, + isRevertingCheckpointRef: { current: false }, + setThreadError: () => {}, + countReservedAttachments: () => + composerImagesRef.current.length + getPreparingComposerImages(target).length, + addComposerImage: (image) => addImages([image]), + addComposerImagesToDraft: addImages, + addComposerFilesToDraft: () => [], + removeComposerImageFromDraft: () => {}, + insertAttachmentReferences: () => false, + } satisfies IntakeInput; + let intake!: Intake; + await using _screen = await render( + { + intake = value; + }} + />, + ); + + const first = intake.addComposerAttachments([pastedImage()]); + const whileCompressing = intake.addComposerAttachments([pastedImage()]); + expect(visiblePreparing(target)).toHaveLength(1); + await act(async () => { + firstCompression.resolve(); + await Promise.all([first, whileCompressing]); + }); + + const afterCompression = intake.addComposerAttachments([pastedImage()]); + expect(visiblePreparing(target)).toHaveLength(0); + await act(() => afterCompression); + expect(composerImagesRef.current).toHaveLength(1); + + // Same name, type and size, different pixels: it compresses differently and is kept. + const different = intake.addComposerAttachments([pastedImage(7)]); + expect(visiblePreparing(target)).toHaveLength(0); + await act(() => different); + expect(composerImagesRef.current).toHaveLength(2); +}); diff --git a/apps/web/src/components/chat/useComposerAttachmentIntake.ts b/apps/web/src/components/chat/useComposerAttachmentIntake.ts index fe3a56a39220..a4e542b1cbe6 100644 --- a/apps/web/src/components/chat/useComposerAttachmentIntake.ts +++ b/apps/web/src/components/chat/useComposerAttachmentIntake.ts @@ -26,7 +26,9 @@ import { import { classifyComposerAttachmentFile, composerFileAttachmentMimeType, + composerImageSourceKey, normalizeComposerImageFileMimeType, + rememberComposerImageSource, } from "./composerAttachmentFiles"; import { fileAttachmentTooLargeMessage, @@ -48,6 +50,7 @@ import { toastManager } from "../ui/toast"; import { type RefObject, useRef } from "react"; import { addPreparingComposerImages, + getPreparingComposerImages, isPreparingComposerImage, removePreparingComposerImages, } from "./preparingComposerImages"; @@ -241,13 +244,21 @@ export function useComposerAttachmentIntake(input: { } if (acceptedImages.length === 0) return insertedAny; + // The store drops a repeated image only after compressing it, so a likely repeat prepares + // without a tile instead of flashing one. Only the store decides it is a duplicate. + const imageSourceKeys = new Set( + [...composerImagesRef.current, ...getPreparingComposerImages(attachmentTargetKey)].map( + (image) => composerImageSourceKey(image.file), + ), + ); // The draft image reuses its placeholder's id and thumbnail so the strip tile stays exactly // as it is when the compressed file replaces it. - const preparingImages = acceptedImages.map((file) => ({ - id: randomUUID(), - name: file.name || "image", - file, - })); + const preparingImages = acceptedImages.map((file) => { + const sourceKey = composerImageSourceKey(file); + const hidden = imageSourceKeys.has(sourceKey); + imageSourceKeys.add(sourceKey); + return { id: randomUUID(), name: file.name || "image", file, hidden }; + }); addPreparingComposerImages(attachmentTargetKey, preparingImages); if (questionAttachmentTarget) changeQuestionAttachmentPreparation(questionAttachmentTarget, acceptedImages.length); @@ -272,7 +283,8 @@ export function useComposerAttachmentIntake(input: { ); continue; } - inheritComposerImageThumbnail(file, result.file); + await inheritComposerImageThumbnail(file, result.file); + rememberComposerImageSource(file, result.file); compressed.push({ id, file: result.file }); } const questionDiscarded = diff --git a/apps/web/src/composerDraftStore.ts b/apps/web/src/composerDraftStore.ts index 783338bc48e2..f719780357a2 100644 --- a/apps/web/src/composerDraftStore.ts +++ b/apps/web/src/composerDraftStore.ts @@ -1,5 +1,8 @@ import { stripInlineContextReferences } from "./lib/composerContextReferences"; -import { composerFileAttachmentMimeType } from "./components/chat/composerAttachmentFiles"; +import { + composerFileAttachmentMimeType, + composerImageDedupKey, +} from "./components/chat/composerAttachmentFiles"; import { elementContextToPreviewAnnotation } from "./lib/elementContext"; import { ResponseContextRecord, @@ -907,11 +910,6 @@ function createEmptyThreadDraft(): ComposerThreadDraftState { }; } -function composerImageDedupKey(image: ComposerImageAttachment): string { - // Hydrated images get fresh lastModified values, so exclude it from dedupe. - return `${image.mimeType}\u0000${image.sizeBytes}\u0000${image.name}`; -} - export function composerFileDedupKey( file: Pick, ): string { diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts index a8d4cfcc1d54..e40c3968bc85 100644 --- a/apps/web/src/lib/imageCompression.test.ts +++ b/apps/web/src/lib/imageCompression.test.ts @@ -254,11 +254,13 @@ describe("composer image thumbnails", () => { stubCanvasPipeline(() => 3); const source = makeFile(3); const compressed = makeFile(2, "image/webp"); - const sourceThumbnail = await createComposerImageThumbnail(source); - expect(sourceThumbnail).toMatch(/^data:image\/png;base64,/); + // Compression can finish before the placeholder's thumbnail does. + const pendingThumbnail = createComposerImageThumbnail(source); - inheritComposerImageThumbnail(source, compressed); + await inheritComposerImageThumbnail(source, compressed); + const sourceThumbnail = await pendingThumbnail; + expect(sourceThumbnail).toMatch(/^data:image\/png;base64,/); expect(readComposerImageThumbnail(compressed)).toBe(sourceThumbnail); expect(await createComposerImageThumbnail(compressed)).toBe(sourceThumbnail); expect(createImageBitmap).toHaveBeenCalledTimes(1); diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.ts index e174bebf4b9e..53e948e535b7 100644 --- a/apps/web/src/lib/imageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -227,16 +227,18 @@ export function createComposerImageThumbnail(file: File): Promise /** * Pasted images are re-encoded after their placeholder already shows the source's thumbnail. * Reusing it keeps the tile from flashing back to its fallback while the new file is decoded. + * Resolves once the target's thumbnail is readable, so a tile swapped to it paints at once even + * when compression finished before the source's thumbnail did. */ -export function inheritComposerImageThumbnail(source: File, target: File): void { +export async function inheritComposerImageThumbnail(source: File, target: File): Promise { const sourceThumbnail = composerThumbnails.get(source); if (source === target || !sourceThumbnail || composerThumbnails.has(target)) return; - const settled = settledComposerThumbnails.get(source); - if (settled) settledComposerThumbnails.set(target, settled); cacheComposerImageThumbnail( target, sourceThumbnail.then((src) => src ?? renderComposerImageThumbnail(target)), ); + const src = await sourceThumbnail; + if (src) settledComposerThumbnails.set(target, src); } async function renderComposerImageThumbnail(file: File): Promise { From 0b21a82aa3761c6073a8a5282f4a598b9accd4be Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:48:56 -0800 Subject: [PATCH 024/367] fix(shared): pooled limits weight each account by its plan size (#35427) --- packages/shared/src/usageLimits.test.ts | 46 +++++++++++++++++++++++++ packages/shared/src/usageLimits.ts | 44 ++++++++++++++++++++--- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/packages/shared/src/usageLimits.test.ts b/packages/shared/src/usageLimits.test.ts index bed48428a93c..8f7fa26d7e72 100644 --- a/packages/shared/src/usageLimits.test.ts +++ b/packages/shared/src/usageLimits.test.ts @@ -804,6 +804,52 @@ describe("pools", () => { ).toEqual(["weekly", "weekly:secondary"]); }); + it("weights pooled Codex quota and resets by each account's plan capacity", () => { + const [pool] = collectLimitPools( + collectLimitAccounts( + new Map([ + [ + EnvironmentId.make("env-a"), + { + ...laptop, + serverConfig: { + providers: [], + usageLimitSources: [ + { + ...source, + accounts: [ + ["pro", "ChatGPT Pro 200 Subscription", 100], + ["plus", "ChatGPT Plus Subscription", 0], + ].map(([id, plan, usedPercent]) => ({ + id: String(id), + email: `${id}@example.com`, + driver: ProviderDriverKind.make("codex"), + plan: String(plan), + usageLimits: { + checkedAt, + windows: [{ ...weekly, id: "secondary", usedPercent: Number(usedPercent) }], + }, + })), + }, + ], + }, + }, + ], + ]), + ), + now, + ); + const [pooledWeekly] = pool?.windows ?? []; + // Pro 20x holds 20 of the pool's 21 capacity parts. + expect(pooledWeekly?.remainingPercent).toBe(5); + expect( + pooledWeekly?.resets.map(({ member, restoresPercent }) => [ + member.account.email, + restoresPercent, + ]), + ).toContainEqual(["pro@example.com", 95]); + }); + it("pools windows by id across accounts and orders resets by when they land", () => { const input = new Map([ [ diff --git a/packages/shared/src/usageLimits.ts b/packages/shared/src/usageLimits.ts index 716103ac2fc9..ed38fdeb45b4 100644 --- a/packages/shared/src/usageLimits.ts +++ b/packages/shared/src/usageLimits.ts @@ -539,6 +539,25 @@ function accountSortName(account: LimitAccount): string { return (account.displayName ?? account.email ?? account.key).toLowerCase(); } +const CODEX_PLAN_CAPACITY: Record = { + "ChatGPT Plus Subscription": 1, + "ChatGPT Pro 100 Subscription": 5, + "ChatGPT Pro 200 Subscription": 20, +}; +// Claude's published plan multipliers describe the session allowance only. +const CLAUDE_SESSION_CAPACITY: Record = { Pro: 1, Max: 5, Ultra: 20 }; + +function planCapacity(account: LimitAccount, kind: ServerProviderUsageWindow["kind"]) { + if (!account.plan) return undefined; + if (account.driver === "codex" && (kind === "session" || kind === "weekly")) { + return CODEX_PLAN_CAPACITY[account.plan]; + } + if (account.driver === "claudeAgent" && kind === "session") { + return CLAUDE_SESSION_CAPACITY[account.plan]; + } + return undefined; +} + // Codex ids name slots, so pool by kind; other drivers' ids identify distinct allowances. function poolKey(driver: ServerProvider["driver"], window: ServerProviderUsageWindow): string { return driver === "codex" ? window.kind : `${window.kind}:${window.id}`; @@ -607,15 +626,28 @@ function poolWindows(accounts: readonly LimitAccount[], now: number): readonly L const memberByAccount = new Map(members.map((member) => [member.account.key, member])); const first = members[0]!.window; const id = members[0]!.account.driver === "codex" ? key : first.id; - const usedPercent = members.reduce((sum, m) => sum + m.window.usedPercent, 0) / members.length; + // One unknown plan makes the pool incomparable by plan, so every member then counts once. + const capacities = members.map((m) => planCapacity(m.account, first.kind)); + const weights = capacities.every((capacity) => capacity !== undefined) + ? capacities + : members.map(() => 1); + const weightOf = new Map(members.map((m, index) => [m, weights[index]!])); + const totalWeight = weights.reduce((sum, weight) => sum + weight, 0); + const usedPercent = + members.reduce((sum, m) => sum + m.window.usedPercent * weightOf.get(m)!, 0) / totalWeight; // Judge pace only for windows with reset clocks; others add spend without elapsed time. const timed = members.flatMap((m) => { const share = elapsedShare(m.window, now); - return share === null ? [] : [{ used: m.window.usedPercent, elapsed: share }]; + return share === null + ? [] + : [{ used: m.window.usedPercent, elapsed: share, weight: weightOf.get(m)! }]; }); - const timedUsed = timed.reduce((sum, t) => sum + t.used, 0) / timed.length; + const timedWeight = timed.reduce((sum, t) => sum + t.weight, 0); + const timedUsed = timed.reduce((sum, t) => sum + t.used * t.weight, 0) / timedWeight; const meanElapsed = - timed.length > 0 ? timed.reduce((sum, t) => sum + t.elapsed, 0) / timed.length : null; + timed.length > 0 + ? timed.reduce((sum, t) => sum + t.elapsed * t.weight, 0) / timedWeight + : null; const resets = members .flatMap((member) => { const at = resetMillis(member.window); @@ -625,7 +657,9 @@ function poolWindows(accounts: readonly LimitAccount[], now: number): readonly L { member, at, - restoresPercent: Math.round(member.refillPercent / members.length), + restoresPercent: Math.round( + (member.refillPercent * weightOf.get(member)!) / totalWeight, + ), }, ]; }) From 60c3b222db553796ef4c2b093691ad9b7ad6bf34 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:49:10 -0800 Subject: [PATCH 025/367] fix(mobile): browser and MCP-app links with blank IDs no longer crash the screen (#35415) --- .../browser/BrowserPreviewRouteScreen.tsx | 21 +++++++++- .../threads/McpAppFullscreenScreen.tsx | 39 +++++++++++++++++-- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx index d83852f26046..b312eeb64534 100644 --- a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx +++ b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx @@ -35,16 +35,33 @@ type BrowserPreviewRouteScreenProps = StaticScreenProps<{ export function BrowserPreviewRouteScreen({ route }: BrowserPreviewRouteScreenProps) { const navigation = useNavigation(); const onClose = () => navigation.goBack(); + const { environmentId, threadId } = route.params; + if (!environmentId?.trim() || !threadId?.trim()) { + return ; + } return ( ); } +function BrowserUnavailableScreen({ onClose }: { readonly onClose: () => void }) { + const insets = useSafeAreaInsets(); + const contentTopInset = useContentTopInset(); + return ( + + + + ); +} + function BrowserPreviewScreen({ environmentId, threadId, diff --git a/apps/mobile/src/features/threads/McpAppFullscreenScreen.tsx b/apps/mobile/src/features/threads/McpAppFullscreenScreen.tsx index de3041c06b57..747b9a9fc186 100644 --- a/apps/mobile/src/features/threads/McpAppFullscreenScreen.tsx +++ b/apps/mobile/src/features/threads/McpAppFullscreenScreen.tsx @@ -56,6 +56,41 @@ function descendsFrom( /** Resolve apps from stored tool calls so links cannot supply apps the call never produced. */ export function McpAppFullscreenScreen({ route }: McpAppFullscreenScreenProps) { + const navigation = useNavigation(); + const { environmentId, threadId, itemId } = route.params; + if (!environmentId?.trim() || !threadId?.trim() || !itemId?.trim()) { + return ( + + navigation.goBack()} + closeAccessibilityLabel="Exit full screen" + /> + This app cannot be shown. + + ); + } + return ( + + ); +} + +function McpAppFullscreen({ + params, + environmentId, + threadId, + itemId, +}: { + readonly params: McpAppFullscreenScreenProps["route"]["params"]; + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly itemId: TurnItemId; +}) { const navigation = useNavigation(); const insets = useSafeAreaInsets(); // Wait for focus before goBack; an exit under another screen must not pop that screen. @@ -74,10 +109,6 @@ export function McpAppFullscreenScreen({ route }: McpAppFullscreenScreenProps) { }); }; const [size, setSize] = useState({ width: 0, height: 0 }); - const params = route.params; - const environmentId = EnvironmentId.make(params.environmentId); - const threadId = ThreadId.make(params.threadId); - const itemId = TurnItemId.make(params.itemId); const conversationThreadId = useConversationThreadId( environmentId, threadId, From 55081a1f7bcfed8a5ace034f03e6f39017b71979 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:50:27 -0800 Subject: [PATCH 026/367] fix(shared): thread search ignores the t3 prefix of flat temporary branches (#35430) --- packages/shared/src/paletteSearch.test.ts | 8 ++++++++ packages/shared/src/paletteSearch.ts | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/shared/src/paletteSearch.test.ts b/packages/shared/src/paletteSearch.test.ts index 4c00dd40571e..14f7f37dc920 100644 --- a/packages/shared/src/paletteSearch.test.ts +++ b/packages/shared/src/paletteSearch.test.ts @@ -333,6 +333,14 @@ describe("scoreThreadSearch beyond Latin text and titles", () => { ranges: [{ start: 0, end: 7 }], }); expect(scoreThreadSearch("trim", subject, NOW)?.explain?.()).toBeUndefined(); + // Nor is it in the flat t3- form used when a `t3` branch blocks nested refs. + const flat = thread("Add search", { branch: "t3-abcd1234" }); + expect(scoreThreadSearch("t3", flat, NOW)).toBeNull(); + expect(scoreThreadSearch("t3 connect", flat, NOW)).toBeNull(); + expect(scoreThreadSearch("abcd1234", flat, NOW)).not.toBeNull(); + for (const query of ["t3-abcd", "t3-abcd1234 connect", "connect t3-abcd1234"]) { + expect(scoreThreadSearch(query, { ...flat, projectTitle: "connect" }, NOW)).not.toBeNull(); + } }); it("reads letters and numbers typed together, without forgiving typos in numbers", () => { diff --git a/packages/shared/src/paletteSearch.ts b/packages/shared/src/paletteSearch.ts index c163482c3af7..02e29a02f2a5 100644 --- a/packages/shared/src/paletteSearch.ts +++ b/packages/shared/src/paletteSearch.ts @@ -923,9 +923,19 @@ export interface SearchMatchHint { } // Most threads' branches start with the app's own worktree prefix ("t3/", and -// "t3code/" before it), so "t3 connect" would match nearly every thread. +// "t3code/" before it, "t3-" when a "t3" branch blocks nested refs), so +// "t3 connect" would match nearly every thread. // Only before more text, so typing "t3/" does not empty the list for a keystroke. -const WORKTREE_BRANCH_PREFIX_PATTERN = new RegExp(`^(?:${WORKTREE_BRANCH_PREFIX}|t3code)/(?=\\S)`); +const WORKTREE_BRANCH_PREFIX_PATTERN = new RegExp( + `^(?:(?:${WORKTREE_BRANCH_PREFIX}|t3code)/(?=\\S)|${WORKTREE_BRANCH_PREFIX}-(?=[0-9a-f]{8}$))`, +); + +// The flat prefix anywhere in a query, while its hex is still being typed, so +// "t3-abcd" and "connect t3-abcd1234" still reach the stripped branch field. +const FLAT_BRANCH_QUERY_PREFIX_PATTERN = new RegExp( + `(^| )${WORKTREE_BRANCH_PREFIX}-(?=[0-9a-f]{1,8}(?: |$))`, + "g", +); function searchableBranch(branch: string | null | undefined): string | null { return branch ? branch.replace(WORKTREE_BRANCH_PREFIX_PATTERN, "") : null; @@ -1005,7 +1015,8 @@ function threadFieldQuery(normalizedQuery: string): string { fieldQuery: normalizedQuery .replace(/^(?:pr|pull request|pull)\s*(?=#?\d)/, "") .replace(/^#\s*/, "") - .replace(WORKTREE_BRANCH_PREFIX_PATTERN, ""), + .replace(WORKTREE_BRANCH_PREFIX_PATTERN, "") + .replace(FLAT_BRANCH_QUERY_PREFIX_PATTERN, "$1"), }; } return lastFieldQuery.fieldQuery; From 841f98d4d80f88ce7218dc1d8dd388cd1e13aee4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:01:00 -0800 Subject: [PATCH 027/367] fix(mobile): a steer splits a settled turn's "Worked for" fold (#35408) --- .../src/features/threads/ThreadFeed.tsx | 2 +- apps/mobile/src/lib/threadActivity.test.ts | 74 ++++++++- apps/mobile/src/lib/threadActivity.ts | 149 +++++++++++++----- 3 files changed, 177 insertions(+), 48 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index dbbcaa1fd9cf..39cc885e4d2e 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -3326,7 +3326,7 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { const jumpRunId = jumpReady && !presentedFeed.some((entry) => entry.type === "message" && entry.id === jumpToMessageId) - ? messageFoldRunId(props.feed, jumpToMessageId) + ? messageFoldRunId(props.feed, jumpToMessageId, presentedFeed) : null; if (jumpRunId && !expandedTurnIds.has(jumpRunId)) { setInteractionState((current) => ({ diff --git a/apps/mobile/src/lib/threadActivity.test.ts b/apps/mobile/src/lib/threadActivity.test.ts index d810563f7266..86e7f3f7182a 100644 --- a/apps/mobile/src/lib/threadActivity.test.ts +++ b/apps/mobile/src/lib/threadActivity.test.ts @@ -896,16 +896,20 @@ describe("buildThreadFeed", () => { projected(runless ? { ...item, runId: null } : item, position), ), ); - const foldRunId = messageFoldRunId(feed, "message-middle"); - expect(foldRunId, String(runless)).toBe(runless ? "runless:message-user" : runId); const ids = (expanded: ReadonlySet) => deriveThreadFeedPresentation(feed, null, expanded).map((entry) => entry.id); + const foldRunId = messageFoldRunId( + feed, + "message-middle", + deriveThreadFeedPresentation(feed, null, new Set()), + ); + expect(foldRunId, String(runless)).toBe(runless ? "runless:message-user" : runId); expect(ids(new Set())).not.toContain("message-middle"); expect(ids(new Set([foldRunId!]))).toContain("message-middle"); } - expect(messageFoldRunId(buildThreadFeed([projected(userMessage(), 0)]), "message-user")).toBe( - null, - ); + expect( + messageFoldRunId(buildThreadFeed([projected(userMessage(), 0)]), "message-user", []), + ).toBe(null); }); it("folds progress messages while keeping the final assistant answer visible", () => { @@ -1010,6 +1014,66 @@ describe("buildThreadFeed", () => { ]); }); + it("splits a settled turn's work fold at a steer", () => { + const steer = { + ...userMessage("2026-06-20T00:00:04.000Z"), + id: TurnItemId.make("item-steer"), + messageId: MessageId.make("message-steer"), + inputIntent: "steer" as const, + }; + const after = { ...command("2026-06-20T00:00:06.000Z"), id: TurnItemId.make("item-after") }; + const progress = { + ...assistantMessage("2026-06-20T00:00:07.000Z"), + id: TurnItemId.make("item-progress"), + messageId: MessageId.make("message-progress"), + }; + const feed = buildThreadFeed( + [ + userMessage(), + command(), + steer, + after, + progress, + assistantMessage("2026-06-20T00:00:09.000Z"), + ].map((item, position) => projected(item, position)), + ); + + expect( + deriveThreadFeedPresentation(feed, null, new Set()).map((entry) => + entry.type === "message" ? entry.id : entry.type, + ), + ).toEqual(["message-user", "run-fold", "message-steer", "run-fold", "message-assistant"]); + const foldRunId = messageFoldRunId( + feed, + "message-progress", + deriveThreadFeedPresentation(feed, null, new Set()), + ); + expect( + deriveThreadFeedPresentation(feed, null, new Set([foldRunId!])).map((entry) => + entry.type === "message" ? entry.id : entry.type, + ), + ).toEqual([ + "message-user", + "run-fold", + "message-steer", + "run-fold", + "activity-group", + "message-progress", + "message-assistant", + ]); + + // Hiding tools leaves no output before the steer, so nothing splits there. + const hideAll = (expanded: ReadonlySet) => + deriveThreadFeedPresentation(feed, null, expanded, new Set(), null, false, undefined, { + toolCallVisibility: "hideAll", + revealedMessageId: null, + }); + const hiddenToolsFoldRunId = messageFoldRunId(feed, "message-progress", hideAll(new Set())); + expect(hideAll(new Set([hiddenToolsFoldRunId!])).map((entry) => entry.id)).toContain( + "message-progress", + ); + }); + it("folds subagents while keeping created-thread and fork cards visible", () => { const { providerThreadId: _providerThreadId, ...forkBase } = base( "item-fork", diff --git a/apps/mobile/src/lib/threadActivity.ts b/apps/mobile/src/lib/threadActivity.ts index 16f16b200c8d..7591b0060985 100644 --- a/apps/mobile/src/lib/threadActivity.ts +++ b/apps/mobile/src/lib/threadActivity.ts @@ -1058,25 +1058,51 @@ export function failedFeedRunIds( /** * The key a run fold uses for the message `entryId`: its run, or for an * imported message with no run, the runless key of the user message before - * it. Null for user messages, which never fold. + * it, or the segment after its run's last steer when `presented` shows that + * fold. Null for user messages, which never fold. */ export function messageFoldRunId( feed: ReadonlyArray, entryId: string, + presented: ReadonlyArray<{ readonly type: string; readonly runId?: RunId | null }>, ): RunId | null { let runlessKey: RunId | null = null; + const lastSteerIdByRunId = new Map(); for (const entry of feed) { if (entry.type !== "message") continue; if (entry.message.role === "user") { if (entry.id === entryId) return null; + if (isSteerMessage(entry.message) && entry.message.runId != null) { + lastSteerIdByRunId.set(entry.message.runId, entry.id); + } runlessKey = entry.message.runId == null ? RunId.make(`runless:${entry.id}`) : null; } else if (entry.id === entryId) { - return entry.message.runId ?? runlessKey; + const runId = entry.message.runId ?? runlessKey; + const steerId = runId === null ? undefined : lastSteerIdByRunId.get(runId); + if (runId === null || steerId === undefined) return runId; + // Tool visibility can drop all output before the steer, leaving no split. + const steerKey = steerFoldRunId(runId, steerId); + return presented.some((row) => row.type === "run-fold" && row.runId === steerKey) + ? steerKey + : runId; } } return null; } +function steerFoldRunId(runId: RunId, steerEntryId: string): RunId { + return RunId.make(`${runId}:steer:${steerEntryId}`); +} + +function feedEntryFoldRunId(entry: ThreadFeedEntry, runlessKey: RunId | null): RunId | null { + if (entry.type === "message") { + return entry.message.role === "assistant" ? (entry.message.runId ?? runlessKey) : null; + } + return entry.type === "activity-group" || entry.type === "html-render" || entry.type === "mcp-app" + ? (entry.runId ?? runlessKey) + : null; +} + function answersBeforeSteers(feed: ReadonlyArray): ReadonlySet { const ids = new Set(); let previous: ThreadFeedEntry | undefined; @@ -1121,26 +1147,35 @@ function deriveThreadFeedRunFolds( } const groupsByRunId = new Map< RunId, - { entries: ThreadFeedEntry[]; startBoundary: string | null } + { + entries: ThreadFeedEntry[]; + startBoundary: string | null; + /** Each steer after the run has output starts a segment at this index into `entries`. */ + steerSegments: Array<{ startIndex: number; steer: ThreadFeedEntry }>; + } >(); let runlessKey: RunId | null = null; let responseKey: RunId | null = null; let pendingUserBoundary: string | null = null; for (const [index, entry] of feed.entries()) { if (entry.type === "message" && entry.message.role === "user") { + const steerGroup = + isSteerMessage(entry.message) && entry.message.runId != null + ? groupsByRunId.get(entry.message.runId) + : undefined; + if (steerGroup && steerGroup.entries.length > 0) { + // Back-to-back steers split once, after the last of them. + if (steerGroup.steerSegments.at(-1)?.startIndex === steerGroup.entries.length) { + steerGroup.steerSegments.pop(); + } + steerGroup.steerSegments.push({ startIndex: steerGroup.entries.length, steer: entry }); + } pendingUserBoundary = entry.message.createdAt; runlessKey = entry.message.runId == null ? RunId.make(`runless:${entry.id}`) : null; responseKey = entry.message.runId ?? runlessKey; continue; } - const ownRunId = - entry.type === "message" && entry.message.role === "assistant" - ? (entry.message.runId ?? runlessKey) - : entry.type === "activity-group" || - entry.type === "html-render" || - entry.type === "mcp-app" - ? (entry.runId ?? runlessKey) - : null; + const ownRunId = feedEntryFoldRunId(entry, runlessKey); if (!ownRunId) continue; // Resumed subagents keep their original turn position and fold, leaving later run folds under their own prompts. const runId = @@ -1149,7 +1184,7 @@ function deriveThreadFeedRunFolds( : ownRunId; let group = groupsByRunId.get(runId); if (!group) { - group = { entries: [], startBoundary: pendingUserBoundary }; + group = { entries: [], startBoundary: pendingUserBoundary, steerSegments: [] }; pendingUserBoundary = null; groupsByRunId.set(runId, group); } @@ -1215,15 +1250,15 @@ function deriveThreadFeedRunFolds( .map((entry) => entry.id), ); const firstEntry = group.entries[0]; - const firstHiddenEntry = group.entries.find((entry) => hiddenEntryIds.has(entry.id)); const lastEntry = group.entries.at(-1); - if (!firstHiddenEntry || !firstEntry || !lastEntry) continue; - const hidesNonCompactionWork = group.entries.some( - (entry) => - hiddenEntryIds.has(entry.id) && - !(entry.type === "activity-group" && isContextCompactionActivityGroup(entry)), - ); - if (!hidesNonCompactionWork) continue; + if (!firstEntry || !lastEntry) continue; + const hidesNonCompactionWork = (entries: ReadonlyArray) => + entries.some( + (entry) => + hiddenEntryIds.has(entry.id) && + !(entry.type === "activity-group" && isContextCompactionActivityGroup(entry)), + ); + if (!hidesNonCompactionWork(group.entries)) continue; const terminalEntry = terminalAssistantId ? group.entries.find((entry) => entry.id === terminalAssistantId) : null; @@ -1243,28 +1278,58 @@ function deriveThreadFeedRunFolds( const duration = elapsedMs === null ? null : formatDuration(elapsedMs); const interrupted = latestRunMatches && (latestRun.status === "interrupted" || latestRun.status === "cancelled"); - const deferredWorkItemIds = group.entries.flatMap((entry) => - entry.type === "activity-group" - ? entry.activities.flatMap((activity) => - activity.projectedItem.item.deferredWork - ? [String(activity.projectedItem.sourceItemId)] - : [], - ) - : [], - ); - foldsByAnchorId.set(firstHiddenEntry.id, { - ...(deferredWorkItemIds.length > 0 ? { deferredWorkItemIds } : {}), - runId, - createdAt: firstHiddenEntry.createdAt, - hiddenEntryIds, - label: interrupted - ? duration - ? `You stopped after ${duration}` - : "You stopped this response" - : duration - ? `Worked for ${duration}` - : "Worked", - }); + const label = interrupted + ? duration + ? `You stopped after ${duration}` + : "You stopped this response" + : duration + ? `Worked for ${duration}` + : "Worked"; + // The top row keeps the whole turn's time. Work after a steer folds under + // its own row below that steer, keyed apart so each segment opens alone. + const segments = [ + { + runId, + entries: group.entries.slice(0, group.steerSegments[0]?.startIndex), + label, + }, + ...group.steerSegments.map(({ startIndex, steer }, index) => { + const entries = group.entries.slice(startIndex, group.steerSegments[index + 1]?.startIndex); + const segmentMs = computeElapsedMs( + steer.createdAt, + entries + .map((entry) => (entry.type === "message" ? entry.message.updatedAt : entry.createdAt)) + .reduce(maxIsoTimestamp, null) ?? steer.createdAt, + ); + return { + runId: steerFoldRunId(runId, steer.id), + entries, + label: segmentMs !== null ? `Continued for ${formatDuration(segmentMs)}` : "Continued", + }; + }), + ]; + for (const segment of segments) { + const firstHiddenEntry = segment.entries.find((entry) => hiddenEntryIds.has(entry.id)); + if (!firstHiddenEntry || !hidesNonCompactionWork(segment.entries)) continue; + const deferredWorkItemIds = segment.entries.flatMap((entry) => + entry.type === "activity-group" + ? entry.activities.flatMap((activity) => + activity.projectedItem.item.deferredWork + ? [String(activity.projectedItem.sourceItemId)] + : [], + ) + : [], + ); + foldsByAnchorId.set(firstHiddenEntry.id, { + ...(deferredWorkItemIds.length > 0 ? { deferredWorkItemIds } : {}), + runId: segment.runId, + createdAt: firstHiddenEntry.createdAt, + hiddenEntryIds: new Set( + segment.entries.flatMap((entry) => (hiddenEntryIds.has(entry.id) ? [entry.id] : [])), + ), + label: segment.label, + }); + } } const covered = new Set( [...foldsByAnchorId.values()].flatMap((fold) => [...fold.hiddenEntryIds]), From ddd02848e601e63e04686fa636a818df90a8a207 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:01:15 -0800 Subject: [PATCH 028/367] fix(shared): html_render pages show for Grok threads (#35434) --- packages/shared/src/htmlRender.test.ts | 10 ++++++++++ packages/shared/src/toolOutput.ts | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/htmlRender.test.ts b/packages/shared/src/htmlRender.test.ts index 9f6315541547..f76b1c8a5625 100644 --- a/packages/shared/src/htmlRender.test.ts +++ b/packages/shared/src/htmlRender.test.ts @@ -169,6 +169,16 @@ describe("htmlRenderFromToolItem", () => { ["html_render", result], // OpenCode 2 Code Mode reports a script's nested html_render call as one `execute` item. ["execute", JSON.stringify(result)], + // Grok CLI wraps MCP results as { type: "MCP", output: { OkayOutput: "" } }. + [ + "t3-code.html_render", + { + type: "MCP", + tool_name: "html_render", + server_name: "t3-code", + output: { OkayOutput: JSON.stringify(result) }, + }, + ], ] as const) { expect(htmlRenderFromToolItem({ toolName, output })).toEqual(reference); } diff --git a/packages/shared/src/toolOutput.ts b/packages/shared/src/toolOutput.ts index f2773b0045e4..6d63d9425d76 100644 --- a/packages/shared/src/toolOutput.ts +++ b/packages/shared/src/toolOutput.ts @@ -100,7 +100,11 @@ function readResult(value: unknown, budget: ResultReadBudget, depth = 0): Result value.is_error === true || value._tag === "OrchestratorMcpFailure" || value.error != null; - const content = value.structuredContent ?? value.content; + // Grok CLI wraps MCP results as { type: "MCP", output: { OkayOutput: "" } }. + const content = + value.structuredContent ?? + value.content ?? + (Predicate.isObject(value.output) ? value.output.OkayOutput : undefined); if (content !== undefined) { const nested = readResult(content, budget, depth + 1); return { ...nested, failed: failed || nested.failed }; From 5ff3d3c0eccdc5aea62e177967c992fccddecb44 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:01:30 -0800 Subject: [PATCH 029/367] fix(server): Claude 1-hour cache writes are priced at the 1-hour rate (#35423) --- .../src/provider/Drivers/claudeUsage.test.ts | 48 +++++++++++- .../src/provider/Drivers/claudeUsage.ts | 9 +++ apps/server/src/usage/UsageService.test.ts | 2 +- .../server/src/usage/usageAggregation.test.ts | 2 + apps/server/src/usage/usageAggregation.ts | 6 ++ apps/server/src/usage/usagePricing.ts | 74 ++++++++++++------- apps/server/src/usage/usageScanCache.test.ts | 16 +++- apps/server/src/usage/usageScanCache.ts | 27 ++++--- packages/provider-core/src/server/usage.ts | 2 + 9 files changed, 143 insertions(+), 43 deletions(-) diff --git a/apps/server/src/provider/Drivers/claudeUsage.test.ts b/apps/server/src/provider/Drivers/claudeUsage.test.ts index 6f457f4f4cf2..bc10ba5b7b29 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.test.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; +import { parseRateTable, priceUsage } from "../../usage/usagePricing.ts"; import { claudeUsageFormat, parseClaudeLine } from "./claudeUsage.ts"; function claudeLine(overrides: { @@ -115,7 +116,8 @@ describe("Claude advisor iterations", () => { model: "claude-fable-5-1", input_tokens: 54204, cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, + cache_creation_input_tokens: 500, + cache_creation: { ephemeral_5m_input_tokens: 200, ephemeral_1h_input_tokens: 300 }, output_tokens: 4229, }, { @@ -160,10 +162,11 @@ describe("Claude advisor iterations", () => { totals: { uncachedInputTokens: 54204, cachedInputTokens: 0, - cacheCreationTokens: 0, + cacheCreationTokens: 500, outputTokens: 4229, reasoningTokens: 0, }, + cacheCreation1hTokens: 300, dedupeKey: expect.any(String), }, ]; @@ -216,3 +219,44 @@ describe("Claude advisor iterations", () => { expect(parse(unusable).map((record) => record.model)).toEqual(["claude-opus-5-5"]); }); }); + +describe("Claude cache write pricing", () => { + it("prices one-hour cache writes at the one-hour rate and the rest at the five-minute rate", () => { + const model = "claude-opus-4-5-20251101"; + const rates = parseRateTable({ + [model]: { + litellm_provider: "anthropic", + input_cost_per_token: 5e-6, + output_cost_per_token: 25e-6, + cache_read_input_token_cost: 5e-7, + cache_creation_input_token_cost: 6.25e-6, + cache_creation_input_token_cost_above_1hr: 1e-5, + }, + }); + const [record] = parseClaudeLine( + JSON.stringify({ + type: "assistant", + timestamp: "2026-08-07T04:05:13.944Z", + sessionId: "s", + message: { + id: "msg_ttl", + model, + usage: { + input_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 1_000_000, + cache_creation: { + ephemeral_5m_input_tokens: 400_000, + ephemeral_1h_input_tokens: 600_000, + }, + output_tokens: 0, + }, + }, + }), + ); + + if (record === undefined) throw new Error("expected a usage record"); + // 0.4M x $6.25/M + 0.6M x $10/M + expect(priceUsage(rates, record).costUsd).toBeCloseTo(8.5, 9); + }); +}); diff --git a/apps/server/src/provider/Drivers/claudeUsage.ts b/apps/server/src/provider/Drivers/claudeUsage.ts index acb697d2941d..713b35884292 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.ts @@ -30,6 +30,13 @@ export function parseClaudeLine(line: string): readonly UsageRecord[] { return parseClaudeRecord(parsed); } +function cacheCreation1h(usage: Record): { cacheCreation1hTokens?: number } { + const split = usage["cache_creation"]; + if (typeof split !== "object" || split === null) return {}; + const tokens = tokenCount((split as Record)["ephemeral_1h_input_tokens"]); + return tokens > 0 ? { cacheCreation1hTokens: tokens } : {}; +} + function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { if (typeof parsed !== "object" || parsed === null) return []; @@ -70,6 +77,7 @@ function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { // Anthropic folds thinking tokens into output and does not break them out. reasoningTokens: 0, }, + ...cacheCreation1h(usageRecord), reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null, speed: usageRecord["speed"] === "fast" ? "fast" : "standard", dedupeKey, @@ -101,6 +109,7 @@ function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { outputTokens: tokenCount(advisor["output_tokens"]), reasoningTokens: 0, }, + ...cacheCreation1h(advisor), reportedCostUsd: null, speed: executor.speed, dedupeKey: dedupeKey === null ? null : `${dedupeKey}:advisor:${index}`, diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index ae0126d43825..3644a53bbc39 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -2890,7 +2890,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v12.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v13.json"); assert.deepStrictEqual( (yield* Effect.promise(() => NodeFSP.readdir(stateDir))) .filter((name) => name.startsWith("usage-scan-cache")) diff --git a/apps/server/src/usage/usageAggregation.test.ts b/apps/server/src/usage/usageAggregation.test.ts index 625f7a9d63ec..f99615085bd2 100644 --- a/apps/server/src/usage/usageAggregation.test.ts +++ b/apps/server/src/usage/usageAggregation.test.ts @@ -12,11 +12,13 @@ const rates: RateTable = new Map([ outputCostPerToken: 5e-5, cacheReadCostPerToken: 1e-6, cacheCreationCostPerToken: 1.25e-5, + cacheCreation1hCostPerToken: 2e-5, fast: { inputCostPerToken: 2e-5, outputCostPerToken: 1e-4, cacheReadCostPerToken: 2e-6, cacheCreationCostPerToken: 2.5e-5, + cacheCreation1hCostPerToken: 4e-5, }, ultrafast: null, contextTiers: [], diff --git a/apps/server/src/usage/usageAggregation.ts b/apps/server/src/usage/usageAggregation.ts index 8d1d79985973..9ff9cb936946 100644 --- a/apps/server/src/usage/usageAggregation.ts +++ b/apps/server/src/usage/usageAggregation.ts @@ -147,12 +147,18 @@ export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRec b.cacheCreationTokens <= a.cacheCreationTokens && b.outputTokens <= a.outputTokens && b.reasoningTokens <= a.reasoningTokens && + (copy.cacheCreation1hTokens ?? 0) <= (held.cacheCreation1hTokens ?? 0) && reportedCostUsd === held.reportedCostUsd ) { return held; } + const cacheCreation1hTokens = Math.max( + held.cacheCreation1hTokens ?? 0, + copy.cacheCreation1hTokens ?? 0, + ); return { ...held, + ...(cacheCreation1hTokens > 0 ? { cacheCreation1hTokens } : {}), totals: { uncachedInputTokens: Math.max(a.uncachedInputTokens, b.uncachedInputTokens), cachedInputTokens: Math.max(a.cachedInputTokens, b.cachedInputTokens), diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 4a04116db2df..21a1bd02bb17 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -12,6 +12,8 @@ export interface TokenRates { readonly outputCostPerToken: number; readonly cacheReadCostPerToken: number; readonly cacheCreationCostPerToken: number; + /** Cache writes with a 1-hour TTL; the 5-minute rate when unpublished. */ + readonly cacheCreation1hCostPerToken: number; } interface SpeedRates extends TokenRates { @@ -37,20 +39,24 @@ export function createOverrideRateTable( overrides: Readonly>, ): RateTable { return new Map( - Object.entries(overrides).map(([model, prices]) => [ - model.trim(), - { - inputCostPerToken: prices.inputCostPerMillionTokens / 1_000_000, - outputCostPerToken: prices.outputCostPerMillionTokens / 1_000_000, - cacheReadCostPerToken: - (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, - cacheCreationCostPerToken: - (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, - fast: null, - ultrafast: null, - contextTiers: [], - }, - ]), + Object.entries(overrides).map(([model, prices]) => { + const cacheCreationCostPerToken = + (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000; + return [ + model.trim(), + { + inputCostPerToken: prices.inputCostPerMillionTokens / 1_000_000, + outputCostPerToken: prices.outputCostPerMillionTokens / 1_000_000, + cacheReadCostPerToken: + (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, + cacheCreationCostPerToken, + cacheCreation1hCostPerToken: cacheCreationCostPerToken, + fast: null, + ultrafast: null, + contextTiers: [], + }, + ]; + }), ); } @@ -60,7 +66,7 @@ function finiteNumber(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } -// Missing cache rates fall back to input pricing; Anthropic normally discounts reads and charges extra for writes. +// Missing cache rates fall back to input pricing, and a missing 1-hour write rate to the 5-minute one. function readTokenRates( entry: LiteLlmEntry, suffix: string, @@ -70,18 +76,28 @@ function readTokenRates( const input = finiteNumber(entry[`input_cost_per_token${suffix}`]); const output = finiteNumber(entry[`output_cost_per_token${suffix}`]) ?? outputFallback ?? null; if (input === null || output === null) return null; - const cacheRate = (name: string, field: "cacheReadCostPerToken" | "cacheCreationCostPerToken") => + const cacheRate = ( + name: string, + field: "cacheReadCostPerToken" | "cacheCreationCostPerToken" | "cacheCreation1hCostPerToken", + fallback = input, + ) => finiteNumber(entry[`${name}${suffix}`]) ?? (standard !== undefined && standard.inputCostPerToken > 0 ? (standard[field] / standard.inputCostPerToken) * input - : input); + : fallback); + const cacheCreationCostPerToken = cacheRate( + "cache_creation_input_token_cost", + "cacheCreationCostPerToken", + ); return { inputCostPerToken: input, outputCostPerToken: output, cacheReadCostPerToken: cacheRate("cache_read_input_token_cost", "cacheReadCostPerToken"), - cacheCreationCostPerToken: cacheRate( - "cache_creation_input_token_cost", - "cacheCreationCostPerToken", + cacheCreationCostPerToken, + cacheCreation1hCostPerToken: cacheRate( + "cache_creation_input_token_cost_above_1hr", + "cacheCreation1hCostPerToken", + cacheCreationCostPerToken, ), }; } @@ -92,6 +108,7 @@ function scaleTokenRates(rates: TokenRates, multiple: number): TokenRates { outputCostPerToken: rates.outputCostPerToken * multiple, cacheReadCostPerToken: rates.cacheReadCostPerToken * multiple, cacheCreationCostPerToken: rates.cacheCreationCostPerToken * multiple, + cacheCreation1hCostPerToken: rates.cacheCreation1hCostPerToken * multiple, }; } @@ -192,7 +209,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean { a.inputCostPerToken === b.inputCostPerToken && a.outputCostPerToken === b.outputCostPerToken && a.cacheReadCostPerToken === b.cacheReadCostPerToken && - a.cacheCreationCostPerToken === b.cacheCreationCostPerToken + a.cacheCreationCostPerToken === b.cacheCreationCostPerToken && + a.cacheCreation1hCostPerToken === b.cacheCreation1hCostPerToken ); } @@ -301,7 +319,7 @@ function resolveRate(table: RateTable, model: string): ModelRate | null { export type PricedRecord = Pick< UsageRecord, - "model" | "rateModel" | "totals" | "speed" | "reportedCostUsd" + "model" | "rateModel" | "totals" | "cacheCreation1hTokens" | "speed" | "reportedCostUsd" >; function recordRate(table: RateTable, record: PricedRecord): ModelRate | null { @@ -316,11 +334,15 @@ export interface PricedUsage { readonly speedPremiumUsd: number; } -function costByCategory(totals: UsageTokenTotals, rates: TokenRates): UsageCategoryCost { +function costByCategory(record: PricedRecord, rates: TokenRates): UsageCategoryCost { + const { totals } = record; + const oneHour = Math.min(record.cacheCreation1hTokens ?? 0, totals.cacheCreationTokens); return { input: totals.uncachedInputTokens * rates.inputCostPerToken, cacheRead: totals.cachedInputTokens * rates.cacheReadCostPerToken, - cacheWrite: totals.cacheCreationTokens * rates.cacheCreationCostPerToken, + cacheWrite: + (totals.cacheCreationTokens - oneHour) * rates.cacheCreationCostPerToken + + oneHour * rates.cacheCreation1hCostPerToken, output: totals.outputTokens * rates.outputCostPerToken, }; } @@ -352,13 +374,13 @@ export function priceUsage( return reported === null ? unsplit(0, "unpriced") : unsplit(reported, "providerReported"); } - const listCost = costByCategory(totals, ratesAt(rate, record.speed, totals)); + const listCost = costByCategory(record, ratesAt(rate, record.speed, totals)); const listCostUsd = sumCategories(listCost); if (reported !== null && listCostUsd <= 0) return unsplit(reported, "providerReported"); const premiumUsd = record.speed === "standard" ? 0 - : listCostUsd - sumCategories(costByCategory(totals, ratesAt(rate, "standard", totals))); + : listCostUsd - sumCategories(costByCategory(record, ratesAt(rate, "standard", totals))); const scale = reported === null ? 1 : reported / listCostUsd; return { costUsd: reported ?? listCostUsd, diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 2866a295da8f..a8fba4da62ae 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -65,7 +65,15 @@ describe("scan cache round trip", () => { [ "/a.jsonl", 100, - [record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5-5", speed: "fast" })], + [ + record(), + record({ + dedupeKey: "msg_2:", + model: "claude-opus-5-5", + speed: "fast", + cacheCreation1hTokens: 8, + }), + ], ], [ "/b.jsonl", @@ -244,11 +252,11 @@ describe("scan cache round trip", () => { }, ); - it("re-parses a Claude transcript cached before partial copies were merged", () => { + it("re-parses a Claude transcript cached before 1-hour cache writes were counted", () => { const original = cacheWith([["/claude.jsonl", 100, [record()]]]); - const v11 = { ...parse(makeScanCacheWriter()(original, {})), version: 11 }; + const v12 = { ...parse(makeScanCacheWriter()(original, {})), version: 12 }; - const restored = decodeScanCache(JSON.parse(JSON.stringify(v11)), TEST_FORMAT_MAP); + const restored = decodeScanCache(JSON.parse(JSON.stringify(v12)), TEST_FORMAT_MAP); expect(restored.get("/claude.jsonl")).toEqual({ ...original.get("/claude.jsonl"), size: -1, diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 4cd959c75aa2..4f1137f3f7a1 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -10,7 +10,7 @@ import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { mergeUsageCopies } from "./usageAggregation.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 12 as const; +const USAGE_SCAN_CACHE_VERSION = 13 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -18,8 +18,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * the newer one; older files are removed once it is saved. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v12.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v13.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v12.json", "usage-scan-cache-v11.json", "usage-scan-cache-v10.json", "usage-scan-cache-v9.json", @@ -59,7 +60,8 @@ type SerializedRecord = readonly [ reportedCostUsd: number | null, speed: number, rateModelIndex?: number | null, - unresolvedServiceTierTurnId?: string, + unresolvedServiceTierTurnId?: string | null, + cacheCreation1hTokens?: number, ]; interface SerializedFile { @@ -118,9 +120,11 @@ function serializeFile(entry: CachedFile, tables: InternTables): SerializedFile record.rateModel === undefined ? null : intern(tables.models, tables.modelIndex, record.rateModel), - ...(record.unresolvedServiceTierTurnId === undefined - ? ([] as const) - : ([record.unresolvedServiceTierTurnId] as const)), + ...(record.cacheCreation1hTokens !== undefined + ? ([record.unresolvedServiceTierTurnId ?? null, record.cacheCreation1hTokens] as const) + : record.unresolvedServiceTierTurnId !== undefined + ? ([record.unresolvedServiceTierTurnId] as const) + : ([] as const)), ]; return { s: entry.size, @@ -211,6 +215,7 @@ export function decodeScanCache( speedIndex, rateModelIndex, unresolvedServiceTierTurnId, + cacheCreation1h = 0, ] = row as SerializedRecord; const speed = typeof speedIndex === "number" ? SPEEDS[speedIndex] : undefined; @@ -220,8 +225,8 @@ export function decodeScanCache( typeof timestampMs !== "number" || !Number.isFinite(timestampMs) || model === undefined || - (unresolvedServiceTierTurnId !== undefined && - typeof unresolvedServiceTierTurnId !== "string") || + (unresolvedServiceTierTurnId != null && typeof unresolvedServiceTierTurnId !== "string") || + !Number.isFinite(cacheCreation1h) || (rateModelIndex != null && rateModel === undefined) || !Number.isFinite(uncached) || !Number.isFinite(cached) || @@ -250,9 +255,10 @@ export function decodeScanCache( outputTokens: output, reasoningTokens: reasoning, }, + ...(cacheCreation1h > 0 ? { cacheCreation1hTokens: cacheCreation1h } : {}), reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null, speed, - ...(version >= 7 && unresolvedServiceTierTurnId !== undefined + ...(version >= 7 && unresolvedServiceTierTurnId != null ? { unresolvedServiceTierTurnId } : {}), dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, @@ -292,8 +298,9 @@ export function decodeScanCache( // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v9 and v11 repair skipped Codex requests; clean v8+ entries stay served while unchanged. // v10 adds Claude advisor calls; before v12 a repeated Claude key kept its first, possibly partial, copy. + // v13 adds Claude 1-hour cache writes. const legacy = - (entry.p === "claude" && version < 12) || + (entry.p === "claude" && version < 13) || (format.state !== undefined && (version < 6 || (entry.p === "codex" && (version < 8 || (version < 11 && malformedRecords > 0))))); diff --git a/packages/provider-core/src/server/usage.ts b/packages/provider-core/src/server/usage.ts index eaed1d06ece4..c2c75a641e53 100644 --- a/packages/provider-core/src/server/usage.ts +++ b/packages/provider-core/src/server/usage.ts @@ -18,6 +18,8 @@ export interface UsageRecord { readonly rateModel?: string; readonly sessionId: string; readonly totals: UsageTokenTotals; + /** The part of `totals.cacheCreationTokens` written with a 1-hour TTL. */ + readonly cacheCreation1hTokens?: number; readonly reportedCostUsd: number | null; readonly speed: UsageSpeed; readonly unresolvedServiceTierTurnId?: string; From 662a1ce6e27c0384ac8a12b7b0c435285ace56e7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:01:43 -0800 Subject: [PATCH 030/367] fix(opencode): OpenCode Go threads no longer fail on a reasoning level the model lacks (#35432) --- .../Adapters/OpenCode2AdapterV2.test.ts | 15 ++++++++++ .../src/server/status.test.ts | 20 ++++++++++++- .../provider-opencode/src/server/status.ts | 2 +- .../src/server/v2/adapter.ts | 28 +++++++++++++------ 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 1efbbfad09d8..acfbdad41ea8 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -496,6 +496,21 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); + it.effect("never sends a saved variant the catalog model does not list", () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); + yield* runtime.startTurn( + turnInput(thread, { ...bigPickle, options: [{ id: "variant", value: "medium" }] }), + ); + assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + }).pipe(Effect.scoped), + ); + it.effect("renames the session before the first turn after the thread was renamed", () => Effect.gen(function* () { const { runtime, thread } = yield* resumed([ diff --git a/packages/provider-opencode/src/server/status.test.ts b/packages/provider-opencode/src/server/status.test.ts index c07464f03641..4c4f61f03d68 100644 --- a/packages/provider-opencode/src/server/status.test.ts +++ b/packages/provider-opencode/src/server/status.test.ts @@ -850,6 +850,12 @@ it.layer(layerTest)("checkOpenCodeProviderStatus", (it) => { name: "Space Bunny Free", variants: [{ id: "low" }, { id: "medium" }, { id: "high" }], }, + { + providerID: "opencode-go", + id: "deepseek-v4.1-flash", + name: "DeepSeek V4.1 Flash", + variants: [{ id: "low" }, { id: "high" }, { id: "max" }], + }, ]), ); @@ -869,7 +875,7 @@ it.layer(layerTest)("checkOpenCodeProviderStatus", (it) => { ); NodeAssert.deepEqual( snapshot.models.map((model) => model.slug), - ["opencode/big-pickle", "opencode/space-bunny-free"], + ["opencode/big-pickle", "opencode-go/deepseek-v4.1-flash", "opencode/space-bunny-free"], ); const variant = snapshot.models.find((model) => model.slug === "opencode/space-bunny-free") ?.capabilities?.optionDescriptors?.[0]; @@ -877,6 +883,18 @@ it.layer(layerTest)("checkOpenCodeProviderStatus", (it) => { variant?.type === "select" ? variant.options.map((option) => option.id) : [], ["low", "medium", "high"], ); + const goVariant = snapshot.models.find( + (model) => model.slug === "opencode-go/deepseek-v4.1-flash", + )?.capabilities?.optionDescriptors?.[0]; + const goVariantIds = + goVariant?.type === "select" ? goVariant.options.map((option) => option.id) : []; + NodeAssert.deepEqual(goVariantIds, ["low", "high", "max"]); + NodeAssert.ok( + goVariant?.type === "select" && + goVariant.currentValue !== undefined && + goVariantIds.includes(goVariant.currentValue), + "an OpenCode Go model gets a default reasoning variant it lists", + ); NodeAssert.equal(runtimeMock.state.sdkClientInputs.length, 0); }), ); diff --git a/packages/provider-opencode/src/server/status.ts b/packages/provider-opencode/src/server/status.ts index 0f24b1e22f47..26f6380dc0ff 100644 --- a/packages/provider-opencode/src/server/status.ts +++ b/packages/provider-opencode/src/server/status.ts @@ -193,7 +193,7 @@ function inferDefaultVariant( if (providerID === "anthropic" || providerID.startsWith("google")) { return variants.includes("high") ? "high" : undefined; } - if (providerID === "openai" || providerID === "opencode") { + if (providerID === "openai" || providerID === "opencode" || providerID === "opencode-go") { return variants.includes("medium") ? "medium" : variants.includes("high") ? "high" : undefined; } return undefined; diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index b13f6c361bac..4ba6c2ed827c 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -660,10 +660,18 @@ const boundaryAfter = ( const isProviderAdapterError = Schema.is(ProviderAdapter.ProviderAdapterV2Error); -const modelRef = (selection: ProviderAdapter.ProviderAdapterV2TurnInput["modelSelection"]) => { +// A saved or inherited variant the catalog model does not list makes OpenCode reject the turn. +const modelRef = ( + selection: ProviderAdapter.ProviderAdapterV2TurnInput["modelSelection"], + listedVariants: ReadonlySet | undefined, +) => { const parsed = OpenCodeRuntime.parseOpenCodeModelSlug(selection.model); if (parsed === null) return undefined; - const variant = getModelSelectionStringOptionValue(selection, "variant"); + const selected = getModelSelectionStringOptionValue(selection, "variant"); + const variant = + selected === undefined || listedVariants === undefined || listedVariants.has(selected) + ? selected + : undefined; return Model.Ref.make({ providerID: Provider.ID.make(parsed.providerID), id: Model.ID.make(parsed.modelID), @@ -765,9 +773,12 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const sessionScope = yield* Effect.scope; // Project config changes model limits; key by directory because this runtime serves all instance threads. const contextWindows = new Map>(); + const modelVariants = new Map>>(); const directoryOf = (cwd: string | null | undefined) => cwd ?? host.paths.cwd; const windowOf = (cwd: string | null | undefined, model: string) => contextWindows.get(directoryOf(cwd))?.get(model); + const listedModelRef = (selection: ModelSelection, cwd: string | null | undefined) => + modelRef(selection, modelVariants.get(directoryOf(cwd))?.get(selection.model)); const now = yield* DateTime.now; let session: OrchestrationV2ProviderSession = { id: input.providerSessionId, @@ -3699,15 +3710,16 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr Effect.suspend(() => { const windows = contextWindows.get(directory) ?? new Map(); contextWindows.set(directory, windows); + const variants = modelVariants.get(directory) ?? new Map>(); + modelVariants.set(directory, variants); return client.model.list({ location: { directory } }).pipe( Effect.timeout("5 seconds"), Effect.tap((models) => Effect.sync(() => { for (const model of models.data) { - windows.set( - `${model.providerID}/${model.id}`, - model.limit.input ?? model.limit.context, - ); + const slug = `${model.providerID}/${model.id}`; + windows.set(slug, model.limit.input ?? model.limit.context); + variants.set(slug, new Set(model.variants.map((variant) => variant.id))); } }), ), @@ -4434,7 +4446,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }); } yield* readModelsOnce(threadInput.runtimePolicy.cwd); - const model = modelRef(threadInput.modelSelection); + const model = listedModelRef(threadInput.modelSelection, threadInput.runtimePolicy.cwd); if (model === undefined) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver: OPENCODE_PROVIDER, @@ -4611,7 +4623,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr }), }); } - const model = modelRef(turnInput.modelSelection); + const model = listedModelRef(turnInput.modelSelection, turnInput.runtimePolicy.cwd); if (model === undefined) { yield* begin; return yield* finishTurn(state, { From c58450f2cf11d1cee712967d36183b8f9fcc255d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:01:58 -0800 Subject: [PATCH 031/367] fix(ssh): SSH environments connect over links with about a second of round trip (#35418) --- packages/ssh/src/tunnel.test.ts | 49 +++++++++++++++++++++++++++++++++ packages/ssh/src/tunnel.ts | 4 ++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/packages/ssh/src/tunnel.test.ts b/packages/ssh/src/tunnel.test.ts index 03f8ebe2a62e..06a99f0b0eb0 100644 --- a/packages/ssh/src/tunnel.test.ts +++ b/packages/ssh/src/tunnel.test.ts @@ -527,6 +527,55 @@ describe("ssh tunnel scripts", () => { }), ); + it.effect("connects through a tunnel whose responses take longer than a second", () => + Effect.gen(function* () { + // Over a ~1s RTT link each request through the forward takes a few round trips. + const RESPONSE_MS = 2_500; + const tunnelSpawned = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + commandArgs(command).includes("-N") + ? Deferred.succeed(tunnelSpawned, undefined).pipe( + Effect.as(makeRunningProcess(() => undefined)), + ) + : Effect.succeed(makeSuccessfulProcess('{"remotePort":3773}\n')), + ); + const httpClient = HttpClient.make((request) => + Effect.sleep(Duration.millis(RESPONSE_MS)).pipe( + Effect.as(HttpClientResponse.fromWeb(request, new Response("", { status: 200 }))), + ), + ); + const layer = Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + Layer.succeed(HttpClient.HttpClient, httpClient), + Layer.succeed(NetService.NetService, testNetService), + SshAuth.SshPasswordPrompt.disabledLayer, + SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }), + ); + yield* Effect.gen(function* () { + const manager = yield* SshTunnel.SshEnvironmentManager; + const ensure = yield* Effect.forkChild( + Effect.result( + manager.ensureEnvironment({ + alias: "devbox", + hostname: "devbox", + username: null, + port: null, + }), + ), + ); + yield* Deferred.await(tunnelSpawned); + yield* TestClock.adjust(Duration.seconds(60)); + const result = yield* Fiber.join(ensure); + assert.isTrue( + Result.isSuccess(result), + Result.isFailure(result) ? result.failure.message : undefined, + ); + }).pipe(Effect.provide(layer), Effect.scoped); + }), + ); + it.effect("reconnects a saved alias through the port its ssh config names now", () => Effect.gen(function* () { let configPort = 53_791; diff --git a/packages/ssh/src/tunnel.ts b/packages/ssh/src/tunnel.ts index 5d4ac857b17b..be4c72b99223 100644 --- a/packages/ssh/src/tunnel.ts +++ b/packages/ssh/src/tunnel.ts @@ -51,6 +51,8 @@ const DEFAULT_REMOTE_PORT = 3773; const REMOTE_PORT_SCAN_WINDOW = 200; const SSH_READY_TIMEOUT_MS = 20_000; const SSH_READY_PROBE_TIMEOUT_MS = 1_000; +// Probes through the forward pay several link round trips, about 4s at a 1s RTT. +const SSH_TUNNEL_READY_PROBE_TIMEOUT_MS = 5_000; const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000; const TUNNEL_STDERR_TAIL_CHARS = 16_384; const REMOTE_READY_TIMEOUT_MS = 60_000; @@ -1159,7 +1161,7 @@ const waitForHttpReady = (input: { ...(input.path === undefined ? {} : { path: input.path }), ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), ...(input.intervalMs === undefined ? {} : { intervalMs: input.intervalMs }), - probeTimeoutMs: input.probeTimeoutMs ?? SSH_READY_PROBE_TIMEOUT_MS, + probeTimeoutMs: input.probeTimeoutMs ?? SSH_TUNNEL_READY_PROBE_TIMEOUT_MS, makeError: ({ requestUrl, probeTimeoutMs, cause }) => { if (typeof cause === "object" && cause !== null && "kind" in cause) { const kind = (cause as { readonly kind?: unknown }).kind; From 315b06c5ecf6587acf53baa0a5691579e04c3f8c Mon Sep 17 00:00:00 2001 From: Wout Stiens <71498452+StiensWout@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:03:48 +0100 Subject: [PATCH 032/367] feat(providers): managed Antigravity and Codex installs can update past their pin (#21249) Co-authored-by: Claude Opus 5.5 Co-authored-by: maria-rcks Co-authored-by: Yash Singh --- .../provider/AntigravityInstallation.test.ts | 231 +++++++++- .../src/provider/AntigravityInstallation.ts | 424 ++++++++++++++---- .../src/provider/CodexInstallation.test.ts | 216 ++++++++- apps/server/src/provider/CodexInstallation.ts | 178 +++++++- .../server/src/provider/antigravityRelease.ts | 12 + apps/server/src/provider/managedRelease.ts | 29 ++ .../src/provider/providerInstallation.test.ts | 2 + .../src/provider/providerInstallation.ts | 12 + .../components/settings/CodexSetupSection.tsx | 12 + docs/user/install.md | 4 + .../src/server/AcpRegistrySupport.ts | 7 +- 11 files changed, 1028 insertions(+), 99 deletions(-) create mode 100644 apps/server/src/provider/managedRelease.ts diff --git a/apps/server/src/provider/AntigravityInstallation.test.ts b/apps/server/src/provider/AntigravityInstallation.test.ts index f8e6636f8067..f604f5e61edf 100644 --- a/apps/server/src/provider/AntigravityInstallation.test.ts +++ b/apps/server/src/provider/AntigravityInstallation.test.ts @@ -17,12 +17,16 @@ import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import { ACP_REGISTRY_URL } from "@t3tools/provider-acp-registry/server/AcpRegistrySupport"; + import * as AntigravityInstallation from "./AntigravityInstallation.ts"; import { type AntigravityReleaseAsset } from "./antigravityRelease.ts"; +import * as ModelManifest from "./ModelManifest.ts"; const serverContents = "antigravity runtime\n"; const harnessContents = "local harness\n"; @@ -126,6 +130,11 @@ interface HarnessOptions { readonly spawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly validate?: AntigravityInstallation.AntigravityInstallationOptions["validate"]; readonly useDefaultValidation?: boolean; + /** Served for the ACP Registry index. */ + readonly registry?: unknown; + /** Served by URL instead of the pinned archive. */ + readonly archives?: Readonly>; + readonly manifest?: Effect.Effect; readonly connectTimeout?: boolean; } @@ -186,11 +195,31 @@ const makeHarness = Effect.fn("test.makeAntigravityInstallation")(function* ( Effect.provideService(HostProcess.Platform, platform), Effect.provideService(HostProcess.Architecture, arch), Effect.provideService(HostProcess.Environment, { PATH: options.path ?? "" }), + Effect.provideService(ModelManifest.ModelManifest, { + current: options.manifest ?? Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + refresh: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + forceRefresh: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + refreshOnDemand: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + refreshInBackground: Effect.void, + }), Effect.provideService( HttpClient.HttpClient, HttpClient.make((request) => Effect.suspend(() => { requests.push(request.url); + if (request.url === ACP_REGISTRY_URL) { + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + options.registry === undefined + ? new Response(null, { status: 503 }) + : Response.json(options.registry), + ), + ); + } + const served = options.archives?.[request.url]; + if (served) + return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(served))); if (options.connectTimeout) { return Effect.fail( new HttpClientError.HttpClientError({ @@ -254,6 +283,43 @@ const expectPreviousRelease = Effect.fn("test.expectPreviousAntigravityRelease") expect((yield* installation.state).installedVersion).toBe(previousVersion); }); +// A zip comment changes the archive's hash without changing its members. +const withZipComment = (archive: Buffer, comment: string) => { + const bytes = Buffer.concat([archive, Buffer.from(comment)]); + bytes.writeUInt16LE(Buffer.byteLength(comment), archive.byteLength - 2); + return bytes; +}; +const upstreamArchive = withZipComment(completeArchive, "upstream"); +const upstreamUrl = "https://dl.google.com/agy-extensions/releases/test/agy-acp-server-1.4.1.zip"; +const registryListing = (archive = upstreamUrl) => ({ + version: "1.0.0", + agents: [ + { id: "other-agent", version: "9.9.9", distribution: { npx: { package: "other" } } }, + { + id: "antigravity-acp", + version: "1.4.1", + distribution: { + binary: Object.fromEntries( + ["linux-x86_64", "windows-x86_64"].map((target) => [target, { archive, cmd: "./agy" }]), + ), + }, + }, + ], +}); +const withdrawnManifest: ModelManifest.ModelManifestData = { + ...ModelManifest.BUNDLED_MODEL_MANIFEST, + compatibility: [ + { + driver: "antigravity", + t3CodeRange: ">=0.0.42", + ranges: [{ range: ">=1.4.0", status: "broken" }], + }, + ], +}; +const pinnedAsset = Effect.fn("test.pinnedAntigravityAsset")(function* () { + return { ...(yield* releaseAsset()), version: "1.3.0" }; +}); + it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity installation", (it) => { it.effect("reports missing Node before downloading the standalone provider runtime", () => Effect.gen(function* () { @@ -764,7 +830,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst }); const { installation: windows } = yield* makeHarness({ baseDir, platform: "win32" }); - for (const name of ["agy.EXE", "agy.cmd"]) { + for (const name of ["agy.EXE", "agy.CMD"]) { const windowsDirectory = path.join(baseDir, `windows-${name}`); yield* fs.makeDirectory(windowsDirectory); yield* fs.writeFileString(path.join(windowsDirectory, name), "cli"); @@ -1018,4 +1084,167 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst expect(requests).toEqual([]); }), ); + + it.effect("updates past the pinned release to the registry release and never downgrades", () => + Effect.gen(function* () { + const asset = yield* pinnedAsset(); + const h = yield* makeHarness({ + asset, + registry: registryListing(), + archives: { [upstreamUrl]: upstreamArchive }, + }); + yield* h.installation.checkForUpdate; + // A fresh install still gets the pinned release. + expect(yield* h.installation.state).toMatchObject({ version: "1.3.0" }); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.3.0", + version: "1.4.1", + totalBytes: null, + }); + + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.4.1", + version: "1.4.1", + }); + expect(h.validations.at(-1)?.version).toBe("1.4.1"); + expect(h.requests).toEqual([ACP_REGISTRY_URL, asset.url, upstreamUrl]); + const selected = yield* h.installation.resolve(); + expect(selected.version).toBe("1.4.1"); + expect(yield* h.fs.readFileString(selected.executablePath)).toBe(serverContents); + + // Without registry information the older pin is not offered as a reinstall. + const restarted = yield* makeHarness({ baseDir: h.baseDir, asset }); + expect(yield* restarted.installation.state).toMatchObject({ + installedVersion: "1.4.1", + version: "1.4.1", + }); + yield* restarted.installation.start; + expect(yield* terminalState(restarted.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.4.1", + }); + expect(restarted.requests).toEqual([]); + expect(restarted.validations.map((entry) => entry.version)).toEqual(["1.4.1"]); + }), + ); + + it.effect.each([ + { + name: "hosted outside Google's downloads", + registry: registryListing("https://example.com/agy.zip"), + }, + { + name: "from an oversized registry index", + registry: { ...registryListing(), padding: "x".repeat(1024 * 1024) }, + }, + { + name: "escaping Google's releases path", + registry: registryListing( + "https://dl.google.com/agy-extensions/releases/%2e%2e/%2e%2e/other.zip", + ), + }, + { + name: "withdrawn by the model manifest", + registry: registryListing(), + manifest: Effect.succeed(withdrawnManifest), + }, + ])("does not offer a registry release $name", ({ registry, manifest }) => + Effect.gen(function* () { + const h = yield* makeHarness({ + asset: yield* pinnedAsset(), + registry, + ...(manifest ? { manifest } : {}), + }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + expect(yield* h.installation.state).toMatchObject({ + installedVersion: "1.3.0", + version: "1.3.0", + }); + }), + ); + + it.effect("does not retry a failed registry lookup within the hour", () => + Effect.gen(function* () { + const h = yield* makeHarness(); + yield* h.installation.checkForUpdate; + yield* h.installation.checkForUpdate; + expect(h.requests).toEqual([ACP_REGISTRY_URL]); + yield* TestClock.adjust("1 hour"); + yield* h.installation.checkForUpdate; + expect(h.requests).toEqual([ACP_REGISTRY_URL, ACP_REGISTRY_URL]); + }), + ); + + it.effect.each(["update", "reinstall"] as const)( + "rejects withdrawal during $0 validation", + (operation) => + Effect.gen(function* () { + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + let withdraw = operation === "update"; + const h = yield* makeHarness({ + asset: yield* pinnedAsset(), + registry: registryListing(), + archives: { [upstreamUrl]: upstreamArchive }, + manifest: Effect.sync(() => manifest), + validate: (_executable, version) => + Effect.sync(() => { + if (withdraw && version === "1.4.1") manifest = withdrawnManifest; + }), + }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + if (operation === "reinstall") { + yield* h.installation.start; + expect((yield* terminalState(h.installation)).phase).toBe("succeeded"); + withdraw = true; + } + yield* h.installation.start; + expect((yield* terminalState(h.installation)).phase).toBe("failed"); + if (operation === "update") { + expect((yield* h.installation.resolve()).version).toBe("1.3.0"); + } else { + expect((yield* h.installation.state).version).toBe("1.3.0"); + yield* Effect.flip(h.installation.resolve()); + } + }), + ); + + it.effect("returns an install the manifest withdraws to the pinned release", () => + Effect.gen(function* () { + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + const h = yield* makeHarness({ + asset: yield* pinnedAsset(), + registry: registryListing(), + archives: { [upstreamUrl]: upstreamArchive }, + manifest: Effect.sync(() => manifest), + }); + yield* h.installation.checkForUpdate; + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ installedVersion: "1.4.1" }); + + manifest = withdrawnManifest; + expect((yield* Effect.flip(h.installation.resolve())).detail).toContain("no longer works"); + yield* h.installation.checkForUpdate; + expect(yield* h.installation.state).toMatchObject({ + installedVersion: "1.4.1", + version: "1.3.0", + }); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.3.0", + version: "1.3.0", + }); + expect((yield* h.installation.resolve()).version).toBe("1.3.0"); + }), + ); }); diff --git a/apps/server/src/provider/AntigravityInstallation.ts b/apps/server/src/provider/AntigravityInstallation.ts index 6f78dc15e842..be6cba7c7afe 100644 --- a/apps/server/src/provider/AntigravityInstallation.ts +++ b/apps/server/src/provider/AntigravityInstallation.ts @@ -22,13 +22,26 @@ import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; -import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; +import { + FetchHttpClient, + HttpClient, + HttpClientError, + HttpClientRequest, + HttpClientResponse, +} from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as NodeCrypto from "node:crypto"; import * as NodeFSP from "node:fs/promises"; import * as EffectAcpErrors from "effect-acp/errors"; +import { collectUint8StreamText } from "@t3tools/provider-core/server/collectStreamText"; +import { + ACP_REGISTRY_URL, + MAX_REGISTRY_BYTES, + resolveAcpRegistryPlatformTarget, +} from "@t3tools/provider-acp-registry/server/AcpRegistrySupport"; + import * as ServerConfig from "../config.ts"; import { openZipArchive } from "../zipArchive.ts"; import { makeAntigravityAcpRuntime } from "./acp/AntigravityAcpSupport.ts"; @@ -38,8 +51,13 @@ import { } from "./antigravityAuthSupport.ts"; import { resolveAntigravityReleaseAsset, + type AntigravityRelease, type AntigravityReleaseAsset, + type AntigravityUpstreamRelease, } from "./antigravityRelease.ts"; +import { selectManagedRelease } from "./managedRelease.ts"; +import * as ModelManifest from "./ModelManifest.ts"; +import { resolveProviderCompatibility } from "./providerCompatibility.ts"; const DRIVER = ProviderDriverKind.make("antigravity"); const DOWNLOAD_TIMEOUT = "45 minutes"; @@ -47,6 +65,29 @@ const VALIDATION_TIMEOUT = "90 seconds"; const FREE_SPACE_MARGIN = 256 * 1024 * 1024; const RECORD_MAX_BYTES = 8 * 1024; const RELEASE_RECORD = ".install-complete.json"; +const UPDATE_CHECK_INTERVAL_MS = 60 * 60_000; +const UPDATE_CHECK_TIMEOUT = "20 seconds"; +const UPSTREAM_AGENT_ID = "antigravity-acp"; +const UPSTREAM_DOWNLOAD_PREFIX = "https://dl.google.com/agy-extensions/releases/"; +// An upstream release has no pinned sizes. Current archives are about 340 MB +// and extract to about 1.1 GB. +const UPSTREAM_MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024; +const UPSTREAM_MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024; + +const decodeRegistryIndex = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ agents: Schema.Array(Schema.Unknown) })), +); +const isUpstreamAgent = Schema.is(Schema.Struct({ id: Schema.Literal(UPSTREAM_AGENT_ID) })); +const decodeRegistryAgent = Schema.decodeUnknownEffect( + Schema.Struct({ + version: Schema.String, + distribution: Schema.Struct({ + binary: Schema.optionalKey( + Schema.Record(Schema.String, Schema.Struct({ archive: Schema.String })), + ), + }), + }), +); const LARGE_PAGE_KERNEL_MESSAGE = "Google's linux-arm64 Antigravity runtime needs a kernel with 4 KiB memory pages, and this kernel uses larger pages (as on Asahi Linux or Raspberry Pi 5). Use a remote environment or a custom Antigravity executable."; @@ -103,6 +144,11 @@ interface AntigravityInstallationService { ) => Effect.Effect; readonly state: Effect.Effect; readonly changes: Stream.Stream; + /** + * Looks up the newest release in the ACP Registry, at most hourly, and + * offers it through `state.version` when it is newer than the install. + */ + readonly checkForUpdate: Effect.Effect; readonly remove: ( protectedBinaryPaths?: ReadonlyArray, ) => Effect.Effect; @@ -162,6 +208,7 @@ function isRunning(state: ProviderInstallState) { export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.make")(function* ( options: AntigravityInstallationOptions, ) { + const manifestService = yield* ModelManifest.ModelManifest; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const random = yield* SecureRandom.SecureRandom; @@ -185,20 +232,112 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma const versionsDirectory = path.join(managedDirectory, "versions"); const activePath = path.join(managedDirectory, "active.json"); const gate = yield* Semaphore.make(1); + const updateCheckGate = yield* Semaphore.make(1); const leases = new Map(); let running: { readonly operationId: string; readonly fiber: Fiber.Fiber } | undefined; + let upstream: AntigravityUpstreamRelease | null = null; + let upstreamCheckedAt: number | null = null; + + // What `start` installs next. Clients compare `version` with + // `installedVersion` to offer an update. + const offer = (installedVersion: string | null) => { + const release = selectManagedRelease( + installedVersion, + releaseAsset, + upstream, + ); + return { + version: release?.version ?? installedVersion, + totalBytes: release && "archiveBytes" in release ? release.archiveBytes : null, + }; + }; + const state = yield* SubscriptionRef.make({ driver: DRIVER, operationId: null, phase: "idle", downloadedBytes: 0, - totalBytes: releaseAsset?.archiveBytes ?? null, - version: releaseAsset?.version ?? null, + ...offer(null), installedVersion: null, canRemove: false, message: null, }); + // The model manifest can withdraw a bad upstream release without a T3 Code release. + const isWithdrawn = Effect.fn("AntigravityInstallation.isWithdrawn")(function* (version: string) { + const manifest = yield* manifestService.current; + const status = ( + resolveProviderCompatibility(manifest.compatibility, DRIVER, version) ?? + resolveProviderCompatibility( + ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, + DRIVER, + version, + ) + )?.status; + return status === "broken" || status === "unsupported"; + }); + + // Forgets what the manifest withdrew after it was seen. A withdrawn install + // is not kept either, so the pin replaces it rather than a re-check. + const keptInstall = Effect.fn("AntigravityInstallation.keptInstall")(function* ( + installedVersion: string | null, + ) { + if (upstream && (yield* isWithdrawn(upstream.version))) upstream = null; + return installedVersion !== null && !(yield* isWithdrawn(installedVersion)) + ? installedVersion + : null; + }); + + const lookupUpstream = Effect.gen(function* () { + const target = resolveAcpRegistryPlatformTarget(platform, arch); + if (!target) return null; + const response = yield* http + .execute(HttpClientRequest.get(ACP_REGISTRY_URL)) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); + const body = yield* collectUint8StreamText({ + stream: response.stream, + maxBytes: MAX_REGISTRY_BYTES + 1, + }); + if (body.truncated || body.bytes > MAX_REGISTRY_BYTES || body.invalidUtf8) return null; + const index = yield* decodeRegistryIndex(body.text); + const entry = index.agents.find(isUpstreamAgent); + if (!entry) return null; + const agent = yield* decodeRegistryAgent(entry); + // Checked after normalization so an encoded `..` cannot leave the releases path. + const archive = agent.distribution.binary?.[target]?.archive; + const url = archive === undefined ? undefined : URL.parse(archive)?.href; + if (!/^\d+\.\d+\.\d+$/u.test(agent.version) || !url?.startsWith(UPSTREAM_DOWNLOAD_PREFIX)) { + return null; + } + if (yield* isWithdrawn(agent.version)) return null; + return { version: agent.version, url } satisfies AntigravityUpstreamRelease; + }); + + // A check already in flight covers concurrent subscribers, including when it fails. + const checkForUpdate = updateCheckGate + .withPermitsIfAvailable(1)( + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (upstreamCheckedAt === null || now - upstreamCheckedAt >= UPDATE_CHECK_INTERVAL_MS) { + // A failed lookup counts too, and keeps whatever release was offered before. + upstreamCheckedAt = now; + upstream = yield* lookupUpstream.pipe( + Effect.timeout(UPDATE_CHECK_TIMEOUT), + Effect.orElseSucceed(() => upstream), + ); + } + // A running install already published the release it is installing. + yield* SubscriptionRef.updateEffect(state, (current) => + isRunning(current) + ? Effect.succeed(current) + : keptInstall(current.installedVersion).pipe( + Effect.map((kept) => ({ ...current, ...offer(kept) })), + ), + ); + }), + ) + .pipe(Effect.ignore); + const readRecord = Effect.fn("AntigravityInstallation.readRecord")(function* ( filePath: string, schema: Schema.Codec, @@ -343,7 +482,15 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma } if (yield* fs.exists(activePath)) { const active = yield* readRecord(activePath, ActiveRelease); - return yield* completedRelease(active.releaseId); + const installed = yield* completedRelease(active.releaseId); + // Without this a withdrawn install keeps launching until its owner updates. + if (yield* isWithdrawn(installed.version)) { + return yield* installationError( + "resolve", + `Antigravity ${installed.version} no longer works with this T3 Code release. Update it in Antigravity setup.`, + ); + } + return installed; } for (const candidate of yield* commandCandidates(names.executable, processEnvironment)) { const selected = yield* fromExternal(candidate, "path"); @@ -472,7 +619,20 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); const install = Effect.fn("AntigravityInstallation.install")( - function* (asset: AntigravityReleaseAsset) { + function* (release: AntigravityRelease) { + // A pinned release is checked against its recorded hash and sizes. An + // upstream release is bounded instead and identified by what downloads. + const pin = "sha256" in release ? release : null; + const ensureEligible = Effect.fn("AntigravityInstallation.ensureEligible")(function* () { + if (!pin && (yield* isWithdrawn(release.version))) { + return yield* installationError( + "verify", + "This Antigravity release was withdrawn. Check for updates and try again.", + ); + } + }); + const executable = pin?.executable ?? { name: names.executable, bytes: null }; + const harness = pin?.harness ?? { name: names.harness, bytes: null }; yield* resolveNodeExecutable("Antigravity", environment).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), @@ -497,27 +657,27 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma SubscriptionRef.update(state, (current) => ({ ...current, phase, message })); yield* fs.makeDirectory(versionsDirectory, { recursive: true }); yield* SubscriptionRef.update(state, (current) => ({ ...current, canRemove: true })); - const destination = path.join(versionsDirectory, asset.sha256); const activate = Effect.fn("AntigravityInstallation.activate")( - function* () { + function* (releaseId: string) { const pointerDirectory = yield* fs.makeTempDirectoryScoped({ directory: managedDirectory, prefix: "active.json.", }); const pointerPath = path.join(pointerDirectory, "contents.tmp"); - yield* fs.writeFileString( - pointerPath, - yield* encodeActiveRelease({ releaseId: asset.sha256 }), - { flag: "wx", mode: 0o600 }, - ); + yield* fs.writeFileString(pointerPath, yield* encodeActiveRelease({ releaseId }), { + flag: "wx", + mode: 0o600, + }); + yield* ensureEligible(); yield* fs.rename(pointerPath, activePath); yield* SubscriptionRef.update( state, (current) => ({ ...current, + ...offer(release.version), phase: "succeeded", - installedVersion: asset.version, + installedVersion: release.version, message: null, }) satisfies ProviderInstallState, ); @@ -531,37 +691,52 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ), Effect.uninterruptible, ); - - if (yield* fs.exists(destination)) { - const existing = yield* completedRelease(asset.sha256); - if (existing.version !== asset.version) { + const reuse = Effect.fn("AntigravityInstallation.reuse")(function* (releaseId: string) { + const existing = yield* completedRelease(releaseId); + if (existing.version !== release.version) { return yield* installationError( "verify", "The existing managed release has the wrong version. Remove it before reinstalling.", ); } yield* report("verifying", "Checking the installed runtime."); - yield* validate(existing, asset.version).pipe( + yield* ensureEligible(); + yield* validate(existing, release.version).pipe( Effect.scoped, Effect.timeout(VALIDATION_TIMEOUT), ); - yield* activate(); - return; + yield* activate(releaseId); + }); + + if (pin && (yield* fs.exists(path.join(versionsDirectory, pin.sha256)))) { + return yield* reuse(pin.sha256); } - const available = yield* Effect.tryPromise({ - try: () => NodeFSP.statfs(versionsDirectory, { bigint: true }), - catch: (cause) => installationError("download", "Could not check free disk space.", cause), - }).pipe(Effect.option); - const required = - asset.archiveBytes + asset.executable.bytes + asset.harness.bytes + FREE_SPACE_MARGIN; - if ( - Option.isSome(available) && - available.value.bavail * available.value.bsize < BigInt(required) + const ensureFreeSpace = Effect.fn("AntigravityInstallation.ensureFreeSpace")(function* ( + operation: string, + bytes: number, ) { - return yield* installationError( + const available = yield* Effect.tryPromise({ + try: () => NodeFSP.statfs(versionsDirectory, { bigint: true }), + catch: (cause) => installationError(operation, "Could not check free disk space.", cause), + }).pipe(Effect.option); + const required = bytes + FREE_SPACE_MARGIN; + if ( + Option.isSome(available) && + available.value.bavail * available.value.bsize < BigInt(required) + ) { + return yield* installationError( + operation, + `Antigravity needs at least ${Math.ceil(required / 1024 / 1024)} MiB of free space to install.`, + ); + } + }); + // An upstream release has no recorded sizes, so each member is checked + // against free space once the archive declares it. + if (pin) { + yield* ensureFreeSpace( "download", - `Antigravity needs at least ${Math.ceil(required / 1024 / 1024)} MiB of free space to install.`, + pin.archiveBytes + pin.executable.bytes + pin.harness.bytes, ); } const staging = yield* fs.makeTempDirectoryScoped({ @@ -571,40 +746,63 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma const archivePath = path.join(staging, "download.zip"); const pairDirectory = path.join(staging, "runtime"); yield* fs.makeDirectory(pairDirectory); + const maxArchiveBytes = pin?.archiveBytes ?? UPSTREAM_MAX_ARCHIVE_BYTES; const hash = NodeCrypto.createHash("sha256"); let downloadedBytes = 0; let lastProgressAt = yield* Clock.currentTimeMillis; yield* Effect.gen(function* () { - const response = yield* http - .execute(HttpClientRequest.get(asset.url)) - .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); - // dl.google.com may gzip the zip: content-length is encoded, while byte/hash checks use decoded data. + const request = http.execute(HttpClientRequest.get(release.url)); + // Only its URL vouches for an upstream release, so a redirect is + // refused rather than followed somewhere that check never saw. + const response = yield* ( + pin + ? request + : request.pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + ) + ).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); + // dl.google.com gzips the zip when the client accepts it, so + // `content-length` is the encoded size. The decoded stream is still + // checked against the byte limit and, when pinned, the hash below. const contentLength = response.headers["content-length"]; const contentEncoding = response.headers["content-encoding"]?.trim().toLowerCase(); const identityBody = contentEncoding === undefined || contentEncoding === "identity"; - if ( - identityBody && - contentLength !== undefined && - Number(contentLength) !== asset.archiveBytes - ) { - return yield* installationError( - "download", - "The Antigravity download size did not match the pinned release.", - ); + if (identityBody && contentLength !== undefined) { + const declaredBytes = Number(contentLength); + if (pin && declaredBytes !== pin.archiveBytes) { + return yield* installationError( + "download", + "The Antigravity download size did not match the pinned release.", + ); + } + if (!pin && Number.isSafeInteger(declaredBytes) && declaredBytes > 0) { + if (declaredBytes > maxArchiveBytes) { + return yield* installationError( + "download", + "The Antigravity download is larger than T3 Code accepts.", + ); + } + yield* SubscriptionRef.update(state, (current) => ({ + ...current, + totalBytes: declaredBytes, + })); + } } yield* response.stream.pipe( Stream.tap((chunk) => Effect.gen(function* () { downloadedBytes += chunk.byteLength; - if (downloadedBytes > asset.archiveBytes) { + if (downloadedBytes > maxArchiveBytes) { return yield* installationError( "download", - "The Antigravity download exceeded the pinned release size.", + pin + ? "The Antigravity download exceeded the pinned release size." + : "The Antigravity download is larger than T3 Code accepts.", ); } hash.update(chunk); const now = yield* Clock.currentTimeMillis; - if (now - lastProgressAt >= 250 || downloadedBytes === asset.archiveBytes) { + if (now - lastProgressAt >= 250 || downloadedBytes === pin?.archiveBytes) { lastProgressAt = now; yield* SubscriptionRef.update(state, (current) => ({ ...current, @@ -627,14 +825,21 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma : cause, ), ); - if (downloadedBytes !== asset.archiveBytes || hash.digest("hex") !== asset.sha256) { + const releaseId = hash.digest("hex"); + if (pin && (downloadedBytes !== pin.archiveBytes || releaseId !== pin.sha256)) { return yield* installationError( "download", "The Antigravity download failed its size or SHA-256 check. Nothing was installed.", ); } + const destination = path.join(versionsDirectory, releaseId); + if (!pin && (yield* fs.exists(destination))) { + return yield* reuse(releaseId); + } yield* report("extracting", "Extracting the verified runtime."); + const extracted = new Map(); + let extractedTotal = 0; yield* Effect.gen(function* () { const archive = yield* openZipArchive(archivePath, (detail, cause) => installationError("extract", detail, cause), @@ -645,31 +850,33 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma "The archive must contain exactly the Antigravity executable and its harness.", ); } - const seen = new Set(); for (;;) { const entry = yield* archive.next; if (!entry) break; - const expected = [asset.executable, asset.harness].find( - (file) => file.name === entry.fileName, - ); + const expected = [executable, harness].find((file) => file.name === entry.fileName); + const expectedBytes = expected?.bytes ?? entry.uncompressedSize; const unixType = (entry.externalFileAttributes >>> 16) & 0o170000; if ( !expected || - seen.has(entry.fileName) || + extracted.has(entry.fileName) || entry.fileName.includes("/") || entry.fileName.includes("\\") || (unixType !== 0 && unixType !== 0o100000) || (entry.externalFileAttributes & 0x10) !== 0 || (entry.generalPurposeBitFlag & 1) !== 0 || ![0, 8].includes(entry.compressionMethod) || - entry.uncompressedSize !== expected.bytes + entry.uncompressedSize !== expectedBytes || + expectedBytes <= 0 || + extractedTotal + expectedBytes > UPSTREAM_MAX_EXTRACTED_BYTES ) { return yield* installationError( "extract", "The archive contains an unexpected, unsafe, or incorrectly sized member.", ); } - seen.add(entry.fileName); + if (!pin) yield* ensureFreeSpace("extract", expectedBytes); + extracted.set(entry.fileName, expectedBytes); + extractedTotal += expectedBytes; yield* Effect.gen(function* () { const readable = yield* archive.streamEntry(entry); let extractedBytes = 0; @@ -680,10 +887,10 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma Stream.tap((chunk) => Effect.gen(function* () { extractedBytes += chunk.byteLength; - if (extractedBytes > expected.bytes) { + if (extractedBytes > expectedBytes) { return yield* installationError( "extract", - "An archive member exceeded its pinned size.", + "An archive member exceeded its expected size.", ); } }), @@ -692,21 +899,23 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma fs.sink(path.join(pairDirectory, entry.fileName), { flag: "wx", mode: 0o700 }), ), ); - if (extractedBytes !== expected.bytes) { + if (extractedBytes !== expectedBytes) { return yield* installationError("extract", "An archive member was truncated."); } }).pipe(Effect.scoped); } - if (!seen.has(asset.executable.name) || !seen.has(asset.harness.name)) { - return yield* installationError( - "extract", - "The archive is missing the Antigravity executable or its harness.", - ); - } }).pipe(Effect.scoped); + const executableBytes = extracted.get(executable.name); + const harnessBytes = extracted.get(harness.name); + if (executableBytes === undefined || harnessBytes === undefined) { + return yield* installationError( + "extract", + "The archive is missing the Antigravity executable or its harness.", + ); + } yield* fs.remove(archivePath); - const executablePath = path.join(pairDirectory, asset.executable.name); - const harnessPath = path.join(pairDirectory, asset.harness.name); + const executablePath = path.join(pairDirectory, executable.name); + const harnessPath = path.join(pairDirectory, harness.name); if (platform !== "win32") { yield* fs.chmod(executablePath, 0o755); yield* fs.chmod(harnessPath, 0o755); @@ -734,24 +943,25 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma } } yield* report("verifying", "Checking the downloaded runtime."); + yield* ensureEligible(); yield* validate( { executablePath, harnessPath, source: "managed", - version: asset.version, + version: release.version, managedVersionDirectory: pairDirectory, }, - asset.version, + release.version, ).pipe(Effect.scoped, Effect.timeout(VALIDATION_TIMEOUT)); const record: InstalledRelease = { - releaseId: asset.sha256, - version: asset.version, + releaseId, + version: release.version, executable: { - name: asset.executable.name, + name: executable.name, bytes: Number((yield* fs.stat(executablePath)).size), }, - harness: { name: asset.harness.name, bytes: Number((yield* fs.stat(harnessPath)).size) }, + harness: { name: harness.name, bytes: Number((yield* fs.stat(harnessPath)).size) }, }; yield* fs.writeFileString( path.join(pairDirectory, RELEASE_RECORD), @@ -760,10 +970,11 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); yield* fs.rename(pairDirectory, destination).pipe( Effect.catch((cause) => - completedRelease(asset.sha256).pipe( + completedRelease(releaseId).pipe( Effect.flatMap((existing) => - existing.version === asset.version - ? validate(existing, asset.version).pipe( + existing.version === release.version + ? ensureEligible().pipe( + Effect.andThen(validate(existing, release.version)), Effect.scoped, Effect.timeout(VALIDATION_TIMEOUT), ) @@ -784,7 +995,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ), ), ); - yield* activate(); + yield* activate(releaseId); }, Effect.scoped, Effect.mapError( @@ -795,12 +1006,56 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ), ); + // Nothing newer is known, so "reinstall" re-checks the active runtime rather + // than replacing it with an older pinned release. + const reverify = Effect.fn("AntigravityInstallation.reverify")( + function* () { + const active = yield* readRecord(activePath, ActiveRelease); + const installed = yield* completedRelease(active.releaseId); + yield* validate(installed, installed.version).pipe( + Effect.scoped, + Effect.timeout(VALIDATION_TIMEOUT), + ); + const kept = yield* keptInstall(installed.version); + if (kept === null) { + yield* SubscriptionRef.update(state, (current) => ({ ...current, ...offer(kept) })); + return yield* installationError( + "verify", + "This Antigravity release was withdrawn. Check for updates and try again.", + ); + } + yield* SubscriptionRef.update( + state, + (current) => + ({ + ...current, + ...offer(kept), + phase: "succeeded", + installedVersion: installed.version, + message: null, + }) satisfies ProviderInstallState, + ); + }, + Effect.mapError( + wrapFailure( + "verify", + "The installed Antigravity runtime could not start. Remove it and reinstall.", + ), + ), + ); + const start = gate .withPermit( Effect.gen(function* () { const current = yield* SubscriptionRef.get(state); if (isRunning(current)) return current; - if (!releaseAsset) { + const installedVersion = yield* keptInstall(current.installedVersion); + const release = selectManagedRelease( + installedVersion, + releaseAsset, + upstream, + ); + if (!release && installedVersion === null) { return yield* installationError( "start", `Google does not publish an Antigravity runtime for ${platform}-${arch}. Use a supported remote environment or a custom executable.`, @@ -810,16 +1065,17 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma const next: ProviderInstallState = { driver: DRIVER, operationId, - phase: "downloading", + phase: release ? "downloading" : "verifying", downloadedBytes: 0, - totalBytes: releaseAsset.archiveBytes, - version: releaseAsset.version, + ...offer(installedVersion), installedVersion: current.installedVersion, canRemove: current.canRemove, - message: "Downloading Google's official Antigravity runtime.", + message: release + ? "Downloading Google's official Antigravity runtime." + : "Checking the installed runtime.", }; yield* SubscriptionRef.set(state, next); - const work = install(releaseAsset).pipe( + const work = (release ? install(release) : reverify()).pipe( Effect.onExit((exit) => Exit.isFailure(exit) ? SubscriptionRef.update(state, (value) => { @@ -908,6 +1164,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma (current) => ({ ...current, + ...offer(null), operationId: null, phase: "idle", downloadedBytes: 0, @@ -933,8 +1190,10 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma if (!(yield* fs.exists(activePath))) return; const active = yield* readRecord(activePath, ActiveRelease); const installed = yield* completedRelease(active.releaseId); + const kept = yield* keptInstall(installed.version); yield* SubscriptionRef.update(state, (current) => ({ ...current, + ...offer(kept), installedVersion: installed.version, })); }).pipe( @@ -959,6 +1218,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma cancel, state: SubscriptionRef.get(state), changes: SubscriptionRef.changes(state), + checkForUpdate, remove, }); }); diff --git a/apps/server/src/provider/CodexInstallation.test.ts b/apps/server/src/provider/CodexInstallation.test.ts index 98861629c79f..5536fe7a1704 100644 --- a/apps/server/src/provider/CodexInstallation.test.ts +++ b/apps/server/src/provider/CodexInstallation.test.ts @@ -41,6 +41,8 @@ const makeHarness = Effect.fn("test.makeCodexInstallation")(function* ( platform?: NodeJS.Platform; spawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; runtimeRenameFailures?: number; + /** Served by URL instead of the pinned package. */ + responses?: Readonly Response>>; } = {}, ) { const fs = yield* FileSystem.FileSystem; @@ -110,6 +112,8 @@ const makeHarness = Effect.fn("test.makeCodexInstallation")(function* ( HttpClient.make((request) => Effect.sync(() => { downloads++; + const respond = input.responses?.[request.url]; + if (respond) return HttpClientResponse.fromWeb(request, respond()); return Object.defineProperty( HttpClientResponse.fromWeb( request, @@ -529,4 +533,214 @@ it.effect("keeps an activated runtime when a later update fails verification", ( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); -// Stands in for Windows' bsdtar: CRLF listings, and extraction writes the Windows layout. +const upstreamVersion = "0.163.0"; +const upstreamArchive = Buffer.from( + "H4sIAAAAAAACA+3WzY7CIBQF4D5Kw1oqtNQmPoh7rKStP9BQdMaYefcBE43T9RQ1nm8B7Q1Jk94cYN3peW026juZDvOqsrzO3nj2yofnUK9KwZKUJREcByet/2TymabuPby29S3/NIz0EIbWDC5u/sXf/PO8EAL5j+G/ew1vuP/TXrp2bpunnf/5KP+syCvkP4bJmg5vlf96JxuVbQej4+efi2qUf17g/h/Hhezl2RzdStmhM5osUz5Lyen+RljGF0XGiK/6H9UoF4pS2rpdCCr7fq/oRtqvTocVSjt77k2nr6vud0vyg6ABAAAAAAAAAAAAAABE8gvVbjvaACgAAA==", + "base64", +); +const upstreamReleaseUrl = `https://github.com/openai/codex/releases/download/rust-v${upstreamVersion}`; +const upstreamResponses = (checksum: string) => ({ + "https://registry.npmjs.org/@openai/codex/latest": () => + Response.json({ version: upstreamVersion }), + [`${upstreamReleaseUrl}/codex-package_SHA256SUMS`]: () => + new Response( + `${"0".repeat(64)} codex-package-x86_64-apple-darwin.tar.gz\n${checksum} codex-package-aarch64-apple-darwin.tar.gz\n`, + ), + [`${upstreamReleaseUrl}/codex-package-aarch64-apple-darwin.tar.gz`]: () => + new Response(upstreamArchive), +}); + +it.effect("updates a managed install to OpenAI's newest release and never downgrades it", () => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + const h = yield* makeHarness({ responses: upstreamResponses(checksum) }); + yield* h.installation.checkForUpdate; + // A fresh install still gets the pinned release. + expect((yield* h.installation.state).version).toBe("0.156.1"); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "0.156.1", + version: upstreamVersion, + }); + + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: upstreamVersion, + version: upstreamVersion, + }); + expect((yield* h.installation.resolve()).version).toBe(upstreamVersion); + + // Without upstream information the older pin is not offered as a reinstall. + const restarted = yield* makeHarness({ baseDir: h.baseDir }); + expect(yield* restarted.installation.state).toMatchObject({ + installedVersion: upstreamVersion, + version: upstreamVersion, + }); + yield* restarted.installation.start; + expect(yield* terminalState(restarted.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: upstreamVersion, + }); + expect(restarted.downloads()).toBe(0); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("keeps the managed install when an update fails its published checksum", () => + Effect.gen(function* () { + const h = yield* makeHarness({ responses: upstreamResponses("f".repeat(64)) }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "failed", + installedVersion: "0.156.1", + message: expect.stringContaining("SHA-256"), + }); + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("does not install a cached update the manifest has since withdrawn", () => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + const h = yield* makeHarness({ + responses: upstreamResponses(checksum), + manifestCurrent: Effect.sync(() => manifest), + }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + expect((yield* h.installation.state).version).toBe(upstreamVersion); + manifest = { + ...manifest, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + ranges: [ + { range: ">=0.160.0", status: "unsupported" }, + { range: ">=0.156.0 <0.160.0", status: "supported" }, + ], + }, + ], + }; + const downloads = h.downloads(); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "0.156.1", + version: "0.156.1", + }); + expect(h.downloads()).toBe(downloads); + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("returns a managed install the manifest withdraws to the pinned release", () => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + const h = yield* makeHarness({ + responses: upstreamResponses(checksum), + manifestCurrent: Effect.sync(() => manifest), + }); + yield* h.installation.checkForUpdate; + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + installedVersion: upstreamVersion, + }); + + manifest = { + ...manifest, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + ranges: [ + { range: ">=0.160.0", status: "unsupported" }, + { range: ">=0.156.0 <0.160.0", status: "supported" }, + ], + }, + ], + }; + yield* h.installation.checkForUpdate; + expect(yield* h.installation.state).toMatchObject({ + installedVersion: upstreamVersion, + version: "0.156.1", + }); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "0.156.1", + version: "0.156.1", + }); + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect.each(["update", "reinstall"] as const)( + "rejects withdrawal during $0 validation", + (operation) => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + let withdraw = operation === "update"; + const h = yield* makeHarness({ + responses: upstreamResponses(checksum), + manifestCurrent: Effect.sync(() => manifest), + options: { + validate: (_executable, version) => + Effect.sync(() => { + if (withdraw && version === upstreamVersion) + manifest = { + ...ModelManifest.BUNDLED_MODEL_MANIFEST, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + ranges: [ + { range: ">=0.160.0", status: "unsupported" }, + { range: ">=0.156.0 <0.160.0", status: "supported" }, + ], + }, + ], + }; + }), + }, + }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + if (operation === "reinstall") { + yield* h.installation.start; + expect((yield* terminalState(h.installation)).phase).toBe("succeeded"); + withdraw = true; + } + yield* h.installation.start; + expect((yield* terminalState(h.installation)).phase).toBe("failed"); + if (operation === "update") { + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + } else { + expect((yield* h.installation.state).version).toBe("0.156.1"); + yield* Effect.flip(h.installation.resolve()); + } + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); diff --git a/apps/server/src/provider/CodexInstallation.ts b/apps/server/src/provider/CodexInstallation.ts index 8fdcfbebb328..c1e489ec0e1e 100644 --- a/apps/server/src/provider/CodexInstallation.ts +++ b/apps/server/src/provider/CodexInstallation.ts @@ -4,7 +4,7 @@ import { type ProviderInstallState, type ServerProviderCompatibilityAdvisory, } from "@t3tools/contracts"; -import { cliArchiveTarCommand } from "@t3tools/shared/cliRelease"; +import { cliArchiveTarCommand, parseChecksums } from "@t3tools/shared/cliRelease"; import { compareSemverVersions } from "@t3tools/shared/semver"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { resolveCommandPath, resolveSpawnCommand } from "@t3tools/shared/shell"; @@ -29,11 +29,18 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import * as NodeCrypto from "node:crypto"; import * as ServerConfig from "../config.ts"; +import { selectManagedRelease } from "./managedRelease.ts"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; const DRIVER = ProviderDriverKind.make("codex"); const STAGING_STALE_AFTER_MS = 24 * 60 * 60_000; +const UPDATE_CHECK_INTERVAL_MS = 60 * 60_000; +const UPDATE_CHECK_TIMEOUT = "20 seconds"; +const UPSTREAM_LATEST_URL = "https://registry.npmjs.org/@openai/codex/latest"; +const UPSTREAM_RELEASES_URL = "https://github.com/openai/codex/releases/download"; +// An upstream release has no recorded size; current packages are about 160 MB. +const UPSTREAM_MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024; const Version = Schema.String.check(Schema.isPattern(/^\d+\.\d+\.\d+(?:-[a-zA-Z0-9.-]+)?$/u)); const ActiveRelease = Schema.Struct({ version: Version }); const InstalledRelease = Schema.Struct({ @@ -48,6 +55,7 @@ const PackageManifest = Schema.Struct({ entrypoint: Schema.String, }); const decodeVersion = Schema.decodeUnknownEffect(Version); +const decodeNpmLatest = Schema.decodeUnknownEffect(Schema.Struct({ version: Version })); const encodeRecord = Schema.encodeEffect(Schema.fromJsonString(InstalledRelease)); const encodeActive = Schema.encodeEffect(Schema.fromJsonString(ActiveRelease)); export interface CodexReleaseAsset { @@ -55,7 +63,8 @@ export interface CodexReleaseAsset { readonly target: string; readonly url: string; readonly sha256: string; - readonly archiveBytes: number; + /** Null for an upstream release, whose package checksum is published without a size. */ + readonly archiveBytes: number | null; } const RELEASES: Readonly> = { "darwin-arm64": { @@ -139,6 +148,11 @@ interface CodexInstallationService { ) => Effect.Effect; readonly state: Effect.Effect; readonly changes: Stream.Stream; + /** + * Looks up OpenAI's newest Codex release, at most hourly, and offers it + * through `state.version` when it is newer than the managed install. + */ + readonly checkForUpdate: Effect.Effect; readonly remove: ( protectedBinaryPaths?: ReadonlyArray, ) => Effect.Effect; @@ -186,15 +200,31 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function const activePath = path.join(managedDirectory, "active.json"); const executableName = platform === "win32" ? "codex.exe" : "codex"; const gate = yield* Semaphore.make(1); + const updateCheckGate = yield* Semaphore.make(1); let leases = 0; let running: { readonly operationId: string; readonly fiber: Fiber.Fiber } | undefined; + let upstream: CodexReleaseAsset | null = null; + let upstreamCheckedAt: number | null = null; + + // A local Codex is used as is, so only a managed install is ever updated. + const managedVersion = (current: ProviderInstallState) => + current.source === "managed" ? current.installedVersion : null; + // What `start` installs next. Clients compare `version` with + // `installedVersion` to offer an update. + const offer = (installedVersion: string | null) => { + const release = selectManagedRelease(installedVersion, asset, upstream); + return { + version: release?.version ?? installedVersion, + totalBytes: release?.archiveBytes ?? null, + }; + }; + const state = yield* SubscriptionRef.make({ driver: DRIVER, operationId: null, phase: "idle", downloadedBytes: 0, - totalBytes: asset?.archiveBytes ?? null, - version: asset?.version ?? null, + ...offer(null), installedVersion: null, executablePath: null, canRemove: false, @@ -279,6 +309,70 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ) ); }); + const isSupported = (version: string) => + compatibility(version).pipe(Effect.map((advisory) => advisory?.status === "supported")); + // Forgets what the manifest withdrew after it was seen. Managed resolution + // refuses a withdrawn install, so the pin replaces it rather than a re-check. + const keptInstall = Effect.fn("CodexInstallation.keptInstall")(function* ( + installedVersion: string | null, + ) { + if (upstream && !(yield* isSupported(upstream.version))) upstream = null; + return installedVersion !== null && (yield* isSupported(installedVersion)) + ? installedVersion + : null; + }); + // npm names the latest stable release; its GitHub release lists the package + // checksums, so an upstream update stays SHA-256 verified. + const lookupUpstream = Effect.gen(function* () { + if (!asset) return null; + const latest = yield* http + .execute(HttpClientRequest.get(UPSTREAM_LATEST_URL)) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); + const { version } = yield* latest.json.pipe(Effect.flatMap(decodeNpmLatest)); + // Managed resolution refuses anything else, so offering it would break sessions. + if (!(yield* isSupported(version))) return null; + const releaseUrl = `${UPSTREAM_RELEASES_URL}/rust-v${version}`; + const checksums = yield* http + .execute(HttpClientRequest.get(`${releaseUrl}/codex-package_SHA256SUMS`)) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.text), + ); + const fileName = `codex-package-${asset.target}.tar.gz`; + const sha256 = parseChecksums(checksums).get(fileName); + if (!sha256) return null; + return { + version, + target: asset.target, + url: `${releaseUrl}/${fileName}`, + sha256, + archiveBytes: null, + } satisfies CodexReleaseAsset; + }); + // A check already in flight covers concurrent subscribers, including when it fails. + const checkForUpdate = updateCheckGate + .withPermitsIfAvailable(1)( + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (upstreamCheckedAt === null || now - upstreamCheckedAt >= UPDATE_CHECK_INTERVAL_MS) { + // A failed lookup counts too, and keeps whatever release was offered before. + upstreamCheckedAt = now; + upstream = yield* lookupUpstream.pipe( + Effect.timeout(UPDATE_CHECK_TIMEOUT), + Effect.orElseSucceed(() => upstream), + ); + } + // A running install already published the release it is installing. + yield* SubscriptionRef.updateEffect(state, (current) => + isRunning(current) + ? Effect.succeed(current) + : keptInstall(managedVersion(current)).pipe( + Effect.map((kept) => ({ ...current, ...offer(kept) })), + ), + ); + }), + ) + .pipe(Effect.ignore); const resolveManaged = Effect.fn("CodexInstallation.resolveManaged")( function* () { const active = yield* readRecord(activePath, ActiveRelease); @@ -464,6 +558,14 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ); const install = Effect.fn("CodexInstallation.install")( function* (release: CodexReleaseAsset) { + const ensureEligible = Effect.fn("CodexInstallation.ensureEligible")(function* () { + if (release.version !== asset?.version && !(yield* isSupported(release.version))) { + return yield* installationError( + "verify", + "This Codex release was withdrawn. Check for updates and try again.", + ); + } + }); yield* decodeVersion(release.version); yield* fs.makeDirectory(managedDirectory, { recursive: true }); yield* SubscriptionRef.update(state, (current) => ({ ...current, canRemove: true })); @@ -480,12 +582,14 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function mode: 0o600, flag: "wx", }); + yield* ensureEligible(); yield* fs.rename(pointer, activePath); yield* SubscriptionRef.update( state, (current) => ({ ...current, + ...offer(release.version), phase: "succeeded", source: "managed", installedVersion: release.version, @@ -508,6 +612,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function "verify", "The existing managed Codex release differs from the official package. Remove it and reinstall.", ); + yield* ensureEligible(); yield* validate(existing, release.version).pipe( Effect.scoped, Effect.timeout("90 seconds"), @@ -523,6 +628,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function const runtime = path.join(staging, "runtime"); yield* fs.makeDirectory(runtime); const hash = NodeCrypto.createHash("sha256"); + const maxArchiveBytes = release.archiveBytes ?? UPSTREAM_MAX_ARCHIVE_BYTES; let downloadedBytes = 0; let lastProgressAt = yield* Clock.currentTimeMillis; const response = yield* http @@ -532,7 +638,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function Stream.tap((chunk) => Effect.gen(function* () { downloadedBytes += chunk.byteLength; - if (downloadedBytes > release.archiveBytes) + if (downloadedBytes > maxArchiveBytes) return yield* installationError( "download", "The Codex download exceeded the expected release size.", @@ -556,7 +662,10 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ), }), ); - if (downloadedBytes !== release.archiveBytes || hash.digest("hex") !== release.sha256) + if ( + (release.archiveBytes !== null && downloadedBytes !== release.archiveBytes) || + hash.digest("hex") !== release.sha256 + ) return yield* installationError( "download", "The Codex download failed its size or SHA-256 check. Nothing was installed.", @@ -600,6 +709,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function }) satisfies ProviderInstallState, ); const executable = yield* fromDirectory(runtime, release.version, release.target); + yield* ensureEligible(); yield* validate(executable, release.version).pipe( Effect.scoped, Effect.timeout("90 seconds"), @@ -642,13 +752,54 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ), ), ); + // Nothing newer is known, so "reinstall" re-checks the active runtime rather + // than replacing it with an older pinned release. + const reverify = Effect.fn("CodexInstallation.reverify")( + function* () { + const active = yield* readRecord(activePath, ActiveRelease); + const executable = yield* completedRelease(active.version); + yield* validate(executable, executable.version).pipe( + Effect.scoped, + Effect.timeout("90 seconds"), + ); + const kept = yield* keptInstall(executable.version); + if (kept === null) { + yield* SubscriptionRef.update(state, (current) => ({ ...current, ...offer(kept) })); + return yield* installationError( + "verify", + "This Codex release was withdrawn. Check for updates and try again.", + ); + } + yield* SubscriptionRef.update( + state, + (current) => + ({ + ...current, + ...offer(kept), + phase: "succeeded", + source: "managed", + installedVersion: executable.version, + executablePath: executable.executablePath, + message: null, + }) satisfies ProviderInstallState, + ); + }, + Effect.mapError( + wrapFailure( + "verify", + "The installed Codex runtime could not start. Remove it and reinstall.", + ), + ), + ); const start = gate .withPermit( Effect.gen(function* () { const current = yield* SubscriptionRef.get(state); if (isRunning(current)) return current; if (yield* reuseLocal()) return yield* SubscriptionRef.get(state); - if (!asset) { + const installedVersion = yield* keptInstall(managedVersion(current)); + const release = selectManagedRelease(installedVersion, asset, upstream); + if (!release && installedVersion === null) { return yield* installationError( "start", `OpenAI does not publish a Codex runtime for ${platform}-${arch}. Use a supported remote environment or a custom executable.`, @@ -658,14 +809,13 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function const next: ProviderInstallState = { ...current, operationId, - phase: "downloading", + phase: release ? "downloading" : "verifying", downloadedBytes: 0, - totalBytes: asset.archiveBytes, - version: asset.version, - message: "Downloading Codex.", + ...offer(installedVersion), + message: release ? "Downloading Codex." : "Checking Codex.", }; yield* SubscriptionRef.set(state, next); - const work = install(asset).pipe( + const work = (release ? install(release) : reverify()).pipe( Effect.onExit((exit) => Exit.isFailure(exit) ? SubscriptionRef.update(state, (value) => { @@ -745,6 +895,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function (current) => ({ ...current, + ...offer(null), operationId: null, phase: "idle", downloadedBytes: 0, @@ -796,11 +947,13 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function if (!(yield* fs.exists(activePath))) return; const active = yield* readRecord(activePath, ActiveRelease); const installed = yield* completedRelease(active.version); + const kept = yield* keptInstall(installed.version); yield* SubscriptionRef.update( state, (current) => ({ ...current, + ...offer(kept), installedVersion: installed.version, executablePath: installed.executablePath, source: "managed", @@ -832,6 +985,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function cancel, state: SubscriptionRef.get(state), changes: SubscriptionRef.changes(state), + checkForUpdate, remove, }); }); diff --git a/apps/server/src/provider/antigravityRelease.ts b/apps/server/src/provider/antigravityRelease.ts index 255a92761994..ecec1aaa5879 100644 --- a/apps/server/src/provider/antigravityRelease.ts +++ b/apps/server/src/provider/antigravityRelease.ts @@ -17,6 +17,18 @@ export interface AntigravityReleaseAsset { }; } +/** + * A newer release Google lists in the ACP Registry. The registry publishes no + * hash or sizes, so the installer bounds the download, accepts only Google's + * download host, and relies on the runtime's own handshake to identify it. + */ +export interface AntigravityUpstreamRelease { + readonly version: string; + readonly url: string; +} + +export type AntigravityRelease = AntigravityReleaseAsset | AntigravityUpstreamRelease; + const releaseAssets = new Map([ [ "darwin-arm64", diff --git a/apps/server/src/provider/managedRelease.ts b/apps/server/src/provider/managedRelease.ts new file mode 100644 index 000000000000..ca7c55e43b59 --- /dev/null +++ b/apps/server/src/provider/managedRelease.ts @@ -0,0 +1,29 @@ +import { compareSemverVersions, parseSemver } from "@t3tools/shared/semver"; + +// Older pins recorded non-semver versions such as `agy_acp_server_1.1.1`; any +// other release is an update from those. +const isNewer = (candidate: string, current: string) => + parseSemver(candidate) && parseSemver(current) + ? compareSemverVersions(candidate, current) > 0 + : candidate !== current; + +/** + * Picks the release a managed installation's `start` downloads. A fresh install + * gets T3 Code's pinned release; an existing one moves to the newest known + * release above it. Null means nothing newer is known, so `start` re-checks the + * installed runtime instead of downgrading it. + */ +export function selectManagedRelease( + installedVersion: string | null, + pinned: Release | null, + upstream: Release | null, +): Release | null { + if (installedVersion === null) return pinned; + let selected: Release | null = null; + for (const candidate of [pinned, upstream]) { + if (candidate && isNewer(candidate.version, selected?.version ?? installedVersion)) { + selected = candidate; + } + } + return selected; +} diff --git a/apps/server/src/provider/providerInstallation.test.ts b/apps/server/src/provider/providerInstallation.test.ts index 8a3d3316ac32..d4c7467c62d0 100644 --- a/apps/server/src/provider/providerInstallation.test.ts +++ b/apps/server/src/provider/providerInstallation.test.ts @@ -95,6 +95,7 @@ const makeHarness = Effect.fn("providerInstallation.test.makeHarness")(function* }), state: Effect.succeed({ ...state, driver: ProviderDriverKind.make("codex") }), changes: Stream.succeed({ ...state, driver: ProviderDriverKind.make("codex") }), + checkForUpdate: Effect.void, remove: (paths) => Effect.sync(() => { protectedPaths = paths ?? []; @@ -103,6 +104,7 @@ const makeHarness = Effect.fn("providerInstallation.test.makeHarness")(function* }), Layer.mock(AntigravityInstallation.AntigravityInstallation)({ managedDirectory: "/unused-managed-runtime", + checkForUpdate: Effect.void, start: Effect.sync(() => { calls.push("start"); return { ...state, operationId: "install-1", phase: "downloading" as const }; diff --git a/apps/server/src/provider/providerInstallation.ts b/apps/server/src/provider/providerInstallation.ts index 7a628dfa5b31..e5e864517ba6 100644 --- a/apps/server/src/provider/providerInstallation.ts +++ b/apps/server/src/provider/providerInstallation.ts @@ -186,9 +186,21 @@ const make = Effect.fn("makeProviderInstallation")(function* () { .pipe(Effect.mapError(failure(input.instanceId))); }); + // Watching an installation is when its owner can act on an update, so the + // lookup runs then, in the background, unless update checks are off. const subscribe = (input: ProviderSetupInput) => Stream.unwrap( requireInstance(input.instanceId, "observe-install").pipe( + Effect.tap(({ installation }) => + settings.getSettings.pipe( + Effect.flatMap((current) => + current.enableProviderUpdateChecks + ? Effect.forkDetach(installation.checkForUpdate) + : Effect.void, + ), + Effect.ignore, + ), + ), Effect.map(({ installation }) => installation.changes), ), ); diff --git a/apps/web/src/components/settings/CodexSetupSection.tsx b/apps/web/src/components/settings/CodexSetupSection.tsx index dea5f40dfea3..cbbe3d51a4ed 100644 --- a/apps/web/src/components/settings/CodexSetupSection.tsx +++ b/apps/web/src/components/settings/CodexSetupSection.tsx @@ -983,6 +983,18 @@ function ManagedCodexSetup({ ) : authenticated ? ( <> + {updateAvailable ? ( + + ) : null} + + ) : scopedProjectGroups.length > 0 ? ( + <> + + {`No ${settledCount > 0 ? "active threads" : "threads"} in ${ + scopedProjectGroups.length > 1 + ? "these projects" + : scopedProjectGroups[0]!.displayName + }${settledCount > 0 ? "" : " yet"}`} + + + + ) : ( + "No threads yet" + )} + + ) : null} {!isSearchingThreads ? ( ) : null} - {!isSearchingThreads && - !hasProjectFolders && - visibleDraftSessionCount === 0 && - pinnedThreads.length + - activeThreads.length + - workingThreads.length + - snoozedThreads.length + - settledCount === - 0 ? ( -
- {projects.length === 0 ? ( - <> - No projects yet - - - ) : scopedProjectGroups.length > 1 ? ( - "No threads in these projects yet" - ) : scopedProjectGroups[0] ? ( - `No threads in ${scopedProjectGroups[0].displayName} yet` - ) : ( - "No threads yet" - )} -
- ) : null} From b42841808d47194de2713d8322e7222809c88166 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 12 Nov 2026 12:53:31 -0600 Subject: [PATCH 042/367] fix(usage): count Codex request ledgers and replace archived cache entries (#35497) --- .../src/provider/Drivers/codexUsage.test.ts | 55 +++++++++++++++++++ .../server/src/provider/Drivers/codexUsage.ts | 33 ++++++++++- apps/server/src/usage/UsageService.test.ts | 52 +++++++++++++++++- apps/server/src/usage/UsageService.ts | 7 +++ apps/server/src/usage/usageScanCache.test.ts | 18 +++--- apps/server/src/usage/usageScanCache.ts | 11 ++-- .../src/vcs/VcsStatusBroadcaster.test.ts | 4 +- 7 files changed, 159 insertions(+), 21 deletions(-) diff --git a/apps/server/src/provider/Drivers/codexUsage.test.ts b/apps/server/src/provider/Drivers/codexUsage.test.ts index 74812bafb509..a3abb2ccb4fb 100644 --- a/apps/server/src/provider/Drivers/codexUsage.test.ts +++ b/apps/server/src/provider/Drivers/codexUsage.test.ts @@ -450,6 +450,61 @@ describe("parseCodexLine", () => { expect(state.malformedRecords).toBe(0); }); + it("bills request-ledger usage through resumed context counters and compaction", () => { + const state = initialCodexScanState(); + const request = (response_id: string, usage: unknown) => + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-08-01T05:17:45.000Z", + payload: { response_id, usage }, + }); + parseCodexLine(sessionMeta, state); + parseCodexLine(turnContext, state); + parseCodexLine(counted(counters(100, 20), counters(1000, 200)), state); + // Resume restores older context counters, while the request ledger continues. + expect( + parseCodexLine(request("response-1", counters(80, 10)), state)?.totals.outputTokens, + ).toBe(10); + expect(parseCodexLine(counted(counters(80, 10), counters(580, 110)), state)).toBeNull(); + // Rate-limit refreshes repeat the snapshot without another request. + expect(parseCodexLine(counted(counters(80, 10), counters(580, 110)), state)).toBeNull(); + expect( + parseCodexLine(request("compaction", counters(90, 15)), state)?.totals.outputTokens, + ).toBe(15); + parseCodexLine(turnContext, state); + expect( + parseCodexLine(counted({ ...counters(0, 0), total_tokens: 40 }, counters(580, 110)), state), + ).toBeNull(); + expect(parseCodexLine(request("response-2", counters(30, 5)), state)?.totals.outputTokens).toBe( + 5, + ); + expect(parseCodexLine(counted(counters(30, 5), counters(610, 115)), state)).toBeNull(); + expect(state.malformedRecords).toBe(0); + // A later turn from an older CLI can still report usage through snapshots alone. + parseCodexLine(turnContext, state); + expect( + parseCodexLine(counted(counters(20, 2), counters(630, 117)), state)?.totals.outputTokens, + ).toBe(2); + expect( + parseCodexLine(request("interrupted", counters(40, 4)), state)?.totals.outputTokens, + ).toBe(4); + parseCodexLine(turnContext, state); + expect( + parseCodexLine(counted(counters(10, 1), counters(680, 122)), state)?.totals.outputTokens, + ).toBe(1); + expect(parseCodexLine(request("", counters(15, 3)), state)).toBeNull(); + parseCodexLine(turnContext, state); + const transition = parseCodexRecords(request("upgraded", counters(25, 5)), state); + expect(transition.map((record) => record.totals.outputTokens)).toEqual([3, 5]); + expect(parseCodexLine(counted(counters(25, 5), counters(720, 130)), state)).toBeNull(); + expect(parseCodexLine(request("", counters(10, 1)), state)).toBeNull(); + expect( + parseCodexLine(counted(counters(10, 1), counters(730, 131)), state)?.totals.outputTokens, + ).toBe(1); + expect(parseCodexLine(counted(counters(10, 1), counters(730, 131)), state)).toBeNull(); + expect(state.malformedRecords).toBe(0); + }); + it("does not bill a context-window estimate or lose the following real request", () => { const state = initialCodexScanState(); parseCodexLine(turnContext, state); diff --git a/apps/server/src/provider/Drivers/codexUsage.ts b/apps/server/src/provider/Drivers/codexUsage.ts index 90f7c78b849f..540e06fd2b8d 100644 --- a/apps/server/src/provider/Drivers/codexUsage.ts +++ b/apps/server/src/provider/Drivers/codexUsage.ts @@ -127,6 +127,7 @@ export const CodexScanState = Schema.Struct({ unreportedRequests: Schema.mutableKey( Schema.optionalKey(Schema.mutable(Schema.Array(CodexUnreportedRequest))), ), + requestLedgerActive: Schema.mutableKey(Schema.optionalKey(Schema.Boolean)), malformedRecords: Schema.mutableKey(TokenCount), sawSessionMeta: Schema.mutableKey(Schema.Boolean), modelProvider: Schema.mutableKey(Schema.String), @@ -181,8 +182,8 @@ function isForkCopy(state: CodexScanState, timestampMs: number): boolean { } /** - * Cumulative counters validate request deltas. Older events without cumulative counters use - * normalized consecutive-request deduplication. + * Identified request-ledger entries report spending independently of context snapshots. + * Legacy snapshots validate request deltas against cumulative counters. */ export function parseCodexLine(line: string, state: CodexScanState): readonly UsageRecord[] { let parsed: unknown; @@ -221,6 +222,7 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag } if (record["type"] === "turn_context") { + state.requestLedgerActive = false; if (typeof payloadRecord["model"] === "string") state.model = payloadRecord["model"]; // Model carry is useful for usage, but history recovery needs this context's own identity. state.turnId = @@ -232,13 +234,31 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag return []; } - // Several of these may sit between two snapshots, so they only explain a gap in the counters. + // Request ledgers include spending that context snapshots can omit. if (record["type"] === "token_usage_record") { const timestampMs = parseTimestampMs(record["timestamp"]); const usage = readCodexTokenUsage(payloadRecord["usage"]); if (timestampMs === null || usage === null || state.model.length === 0) return []; // Copied history is already billed in the parent. if (isForkCopy(state, timestampMs)) return []; + const responseId = payloadRecord["response_id"]; + if (typeof responseId === "string" && responseId.length > 0) { + const pending = state.unreportedRequests ?? []; + state.requestLedgerActive = true; + for (const request of pending) { + state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, request.usage); + } + state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, usage); + delete state.unreportedRequests; + return [ + ...pending.map((request) => codexRequestRecord(state, request.usage, request)), + { + ...codexRequestRecord(state, usage, codexAttribution(state, timestampMs)), + dedupeKey: JSON.stringify(["codex", state.sessionId, responseId]), + }, + ]; + } + state.requestLedgerActive = false; (state.unreportedRequests ??= []).push({ ...codexAttribution(state, timestampMs), usage }); return []; } @@ -278,6 +298,12 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag return []; } + // Context snapshots can rewind on resume and omit compaction spending; identified requests are authoritative. + if (state.requestLedgerActive) { + if (cumulative !== null) rebaseCodexUsage(state, cumulative); + return []; + } + const previous = state.lastCumulativeUsage; const pending = state.unreportedRequests; // A valid snapshot remains the next baseline even when its paired request is malformed. @@ -418,6 +444,7 @@ export const codexUsageFormat: TranscriptUsageFormat = { session_id: true, model_provider: true, turn_id: true, + response_id: true, model: true, thread_settings: { service_tier: true }, forked_from_id: true, diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index 3644a53bbc39..bff9bdb1f57f 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -577,9 +577,55 @@ describe("UsageService", () => { assert.strictEqual(totalOutputTokens(rewritten), 71); assert.deepStrictEqual((yield* service.readSummary(WINDOW)).buckets, rewritten.buckets); yield* service.awaitPersisted; + const { stateDir } = yield* ServerConfig.ServerConfig; + const cachePath = NodePath.join(stateDir, SCAN_CACHE_FILE_NAME); + const originalPath = (yield* Effect.promise(() => NodeFSP.realpath(compressedPath))).slice( + 0, + -4, + ); + yield* Effect.promise(async () => { + const document = decodeUnknownJsonString(await NodeFSP.readFile(cachePath, "utf8")) as { + files: Record< + string, + { cs: Record; r: unknown[]; t: unknown[]; mr: number } + >; + }; + const archived = document.files[await NodeFSP.realpath(compressedPath)]; + assert.ok(archived); + document.files[originalPath] = { + ...archived, + r: [], + t: [], + mr: 1, + cs: { ...archived.cs, malformedRecords: 1 }, + }; + await NodeFSP.writeFile(cachePath, encodeUnknownJsonString(document)); + }); const restarted = yield* UsageService.make; - assert.deepStrictEqual((yield* restarted.readSummary(WINDOW)).buckets, rewritten.buckets); + const restored = yield* restarted.readSummary(WINDOW); + assert.deepStrictEqual(restored.buckets, rewritten.buckets); + assert.strictEqual( + restored.sources.find((source) => source.fingerprint.provider === "codex") + ?.malformedRecords, + 0, + ); yield* restarted.awaitPersisted; + yield* Effect.promise(async () => { + const document = decodeUnknownJsonString(await NodeFSP.readFile(cachePath, "utf8")) as { + files: Record; + }; + assert.strictEqual(document.files[originalPath], undefined); + await NodeFSP.rm(compressedPath); + }); + const removed = yield* UsageService.make; + const retained = yield* removed.readSummary(WINDOW); + assert.deepStrictEqual(retained.buckets, rewritten.buckets); + assert.strictEqual( + retained.sources.find((source) => source.fingerprint.provider === "codex") + ?.malformedRecords, + 0, + ); + yield* removed.awaitPersisted; }).pipe( Effect.scoped, Effect.provide(layerService({ prefix: "usage-compressed-codex", home, settings })), @@ -2789,7 +2835,7 @@ describe("UsageService", () => { }).pipe(Effect.scoped), ); - it.live.each([4, 5, 6, 7, 8, 10])( + it.live.each([4, 5, 6, 7, 8, 10, 11, 12, 13])( "upgrades a v%s cache: reparses live Codex usage, keeps deleted rollouts, removes the old file", (version) => Effect.gen(function* () { @@ -2890,7 +2936,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v13.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v14.json"); assert.deepStrictEqual( (yield* Effect.promise(() => NodeFSP.readdir(stateDir))) .filter((name) => name.startsWith("usage-scan-cache")) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 341b14d8c628..912ed51c96b3 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -638,6 +638,13 @@ export const make = Effect.gen(function* () { } return { path, records, malformedRecords }; }); + const livePaths = new Set(files.map((file) => file.path)); + for (const file of read) { + if (file.failed || !file.path.endsWith(".jsonl.zst")) continue; + const originalPath = file.path.slice(0, -4); + // A successfully read archive replaces the original's saved records and diagnostics. + if (!livePaths.has(originalPath) && fileCache.delete(originalPath)) cacheDirty = true; + } const unread = failedPaths + read.filter((file) => file.failed).length; yield* Effect.annotateCurrentSpan({ "usage.files": files.length, diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index a8fba4da62ae..a6a0f6d3c41d 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -168,7 +168,7 @@ describe("scan cache round trip", () => { expect(restored).toEqual(original); }); - it.each([5, 6, 7, 8, 10])( + it.each([5, 6, 7, 8, 10, 11, 12, 13])( "retains v%s history and compatible positions but invalidates obsolete state", (version) => { const original = cacheWith([["/gone.jsonl", 100, [record({ speed: "fast" })]]]); @@ -225,11 +225,15 @@ describe("scan cache round trip", () => { }; const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP); - expect(restored.get("/gone.jsonl")).toEqual({ - ...original.get("/gone.jsonl"), - size: -1, - position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, - }); + expect(restored.get("/gone.jsonl")).toEqual( + version >= 13 + ? original.get("/gone.jsonl") + : { + ...original.get("/gone.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }, + ); expect(restored.get("/gone-codex.jsonl")).toEqual({ ...original.get("/gone-codex.jsonl"), size: -1, @@ -237,7 +241,7 @@ describe("scan cache round trip", () => { }); expect(restored.get("/clean-codex.jsonl")).toEqual({ ...cleanCodex, - size: version >= 8 ? cleanCodex.size : -1, + size: -1, position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, }); expect(restored.get("/gone-pi.jsonl")).toEqual( diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 4f1137f3f7a1..604937a6a5e4 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -10,7 +10,7 @@ import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { mergeUsageCopies } from "./usageAggregation.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 13 as const; +const USAGE_SCAN_CACHE_VERSION = 14 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -18,8 +18,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * the newer one; older files are removed once it is saved. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v13.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v14.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v13.json", "usage-scan-cache-v12.json", "usage-scan-cache-v11.json", "usage-scan-cache-v10.json", @@ -296,14 +297,12 @@ export function decodeScanCache( continue; } // Keep history for deleted rollouts, but re-parse incompatible reducer state. - // v9 and v11 repair skipped Codex requests; clean v8+ entries stay served while unchanged. + // v14 uses Codex request ledgers, including compaction spending omitted by context snapshots. // v10 adds Claude advisor calls; before v12 a repeated Claude key kept its first, possibly partial, copy. // v13 adds Claude 1-hour cache writes. const legacy = (entry.p === "claude" && version < 13) || - (format.state !== undefined && - (version < 6 || - (entry.p === "codex" && (version < 8 || (version < 11 && malformedRecords > 0))))); + (format.state !== undefined && (version < 6 || (entry.p === "codex" && version < 14))); // A corrupt state disqualifies the entry: resuming with it would attach // appended usage to the wrong model or replay fork-copied history. if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue; diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index 37d767c29c04..ed9351b37e49 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -1555,13 +1555,13 @@ describe("VcsStatusBroadcaster", () => { yield* Effect.gen(function* () { const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); - assert.deepEqual(yield* branches, [{ refName: "main" }]); + assert.deepEqual(yield* branches, [{ refName: "main", isRepo: true }]); // Every checkout removes HEAD.lock, which leaves the watch running. state.currentLocalStatus = { ...baseLocalStatus, refName: "other" }; yield* Queue.offer(events, { _tag: "Remove", path: "HEAD.lock" }); yield* Queue.offer(events, { _tag: "Update", path: "HEAD" }); - assert.deepEqual(yield* branches, [{ refName: "other" }]); + assert.deepEqual(yield* branches, [{ refName: "other", isRepo: true }]); // `git worktree remove` deletes the watched directory; Windows keeps reporting it. gitDirExists = false; From 8fc1dade71e08dacf08bae997d550ff318a95a0e Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 12 Nov 2026 12:58:40 -0600 Subject: [PATCH 043/367] fix(web): keep settled pagination focused while loading (#35522) --- .../src/components/Sidebar.browser.test.tsx | 106 +++++++++++++++++- apps/web/src/components/Sidebar.tsx | 5 +- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/apps/web/src/components/Sidebar.browser.test.tsx b/apps/web/src/components/Sidebar.browser.test.tsx index 64bead88b35c..f152cabf0185 100644 --- a/apps/web/src/components/Sidebar.browser.test.tsx +++ b/apps/web/src/components/Sidebar.browser.test.tsx @@ -1,3 +1,7 @@ +import appCss from "../index.css?inline"; +import type { EnvironmentServerConfig } from "@t3tools/client-runtime/connection"; +import * as shelfHooks from "../state/olderSettledShelf"; +import { createWorkspaceFocusController, workspaceAreaProps } from "../lib/workspaceFocus"; import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; import { DEFAULT_CLIENT_SETTINGS, @@ -16,7 +20,7 @@ import { makeThreadFixture } from "../test-fixtures"; import { useHandleNewThread } from "../hooks/useHandleNewThread"; import { __setClientSettingsForTests, getClientSettings } from "../hooks/useSettings"; import { useThreadRouteTarget } from "../hooks/useThreadRouteTarget"; -import { useProjects, useQuietThreadShells } from "../state/entities"; +import { useProjects, useServerConfigs, useQuietThreadShells } from "../state/entities"; import { readEnvironmentScope, useEnvironmentScope } from "../state/session"; import { useOrchestrationCommand } from "../state/use-orchestration-command"; import { useTerminalUiStateStore } from "../terminalUiStateStore"; @@ -26,6 +30,10 @@ import { THREAD_JUMP_HINT_SHOW_DELAY_MS } from "./Sidebar.logic"; import { SortableThreadRow } from "./sidebar/SidebarThreadRow"; import { SidebarProvider } from "./ui/sidebar"; +vi.mock(import("../state/olderSettledShelf"), async (original) => { + const actual = await original(); + return { ...actual, useOlderSettledShelf: vi.fn(actual.useOlderSettledShelf) }; +}); vi.mock("../state/entities", { spy: true }); vi.mock("../hooks/useHandleNewThread", { spy: true }); vi.mock("../hooks/useThreadRouteTarget", { spy: true }); @@ -135,6 +143,7 @@ describe("sidebar render isolation", () => { environmentId: activeProject.environmentId, threadId: threads[0]!.id, }; + threads.unshift(threadShell(activeProject.id, 100)); vi.mocked(useProjects).mockReturnValue([activeProject]); vi.mocked(useQuietThreadShells).mockReturnValue(threads); vi.mocked(useThreadRouteTarget).mockReturnValue({ kind: "server", threadRef: routeThreadRef }); @@ -247,3 +256,98 @@ describe("sidebar thread rename", () => { }, ); }); + +describe("sidebar settled pagination", () => { + it("keeps the viewport in place when showing more settled threads", async () => { + using stack = new DisposableStack(); + const style = document.createElement("style"); + style.textContent = appCss; + document.head.append(style); + stack.defer(() => style.remove()); + const focusController = createWorkspaceFocusController(document); + stack.defer(() => focusController.dispose()); + let loading = false; + const originalShelf = vi.mocked(shelfHooks.useOlderSettledShelf).getMockImplementation()!; + using _shelf = vi.spyOn(shelfHooks, "useOlderSettledShelf").mockImplementation((input) => ({ + ...originalShelf(input), + loading, + retry: () => { + loading = true; + }, + })); + const activeProject = project("active"); + const threads = Array.from({ length: 70 }, (_, index) => + makeThreadFixture({ + environmentId: activeProject.environmentId, + projectId: activeProject.id, + id: ThreadId.make(`settled-${index}`), + title: `Settled thread ${index}`, + settledOverride: "settled", + settledAt: "2026-01-01T00:00:00.000Z", + }), + ); + threads.unshift(threadShell(activeProject.id, 100)); + vi.mocked(useProjects).mockReturnValue([activeProject]); + vi.mocked(useQuietThreadShells).mockReturnValue(threads); + vi.mocked(useThreadRouteTarget).mockReturnValue({ + kind: "server", + threadRef: { environmentId: activeProject.environmentId, threadId: threads[0]!.id }, + }); + vi.mocked(useHandleNewThread).mockReturnValue({ + activeDraftThread: null, + activeThread: null, + defaultProjectRef: { + environmentId: activeProject.environmentId, + projectId: activeProject.id, + }, + defaultProjectFallbackReady: true, + handleNewThread: vi.fn().mockResolvedValue(null), + routeDraftId: null, + routeThreadRef: { environmentId: activeProject.environmentId, threadId: threads[0]!.id }, + }); + vi.mocked(useServerConfigs).mockReturnValue( + new Map([ + [ + activeProject.environmentId, + { + scratchWorkspaceRoot: null, + shellThreadFamilies: false, + } as unknown as EnvironmentServerConfig, + ], + ]), + ); + await using screen = await render( + + + , + ); + const shelf = screen.container.querySelector( + '[data-sidebar-row="shelf:settled"]', + )!; + if (shelf.getAttribute("aria-expanded") === "false") await act(() => shelf.click()); + const more = screen.getByRole("button", { name: "Show 25 more", exact: true }); + await expect.element(more).toBeInTheDocument(); + const moreButton = more.element() as HTMLButtonElement; + const viewport = moreButton.closest('[data-slot="scroll-area-viewport"]')!; + viewport.style.height = "300px"; + viewport.style.maxHeight = "300px"; + moreButton.scrollIntoView({ block: "nearest" }); + const before = viewport.scrollTop; + expect(before).toBeGreaterThan(0); + moreButton.focus({ preventScroll: true }); + await act(() => moreButton.click()); + await expect + .element(screen.getByRole("button", { name: "Loading", exact: true })) + .toBeInTheDocument(); + await expect + .element(screen.getByRole("button", { name: "Loading", exact: true })) + .toHaveFocus(); + expect(viewport.scrollTop).toBe(before); + const loadedRows = screen.container.querySelectorAll("[data-thread-item]").length; + await act(() => moreButton.click()); + expect(screen.container.querySelectorAll("[data-thread-item]")).toHaveLength(loadedRows); + }); +}); diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 5c9203742578..b7ec300204a0 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -3644,12 +3644,13 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve type="button" data-sidebar-row="settled:more" data-sidebar-parent="shelf:settled" - disabled={olderSettledShelf.loading} + aria-disabled={olderSettledShelf.loading} onClick={() => { + if (olderSettledShelf.loading) return; olderSettledShelf.retry(); showMoreSettled(); }} - className="flex h-9 w-full cursor-pointer items-center gap-2.5 rounded-md px-2.5 text-left text-sm text-sidebar-muted-foreground hover:bg-sidebar-row-hover hover:text-sidebar-foreground disabled:cursor-default" + className="flex h-9 w-full cursor-pointer items-center gap-2.5 rounded-md px-2.5 text-left text-sm text-sidebar-muted-foreground hover:bg-sidebar-row-hover hover:text-sidebar-foreground aria-disabled:cursor-default" > {/* Plus means "add", so revealing more uses a chevron. */} From 91a76cf5a6a406343f88727d26570914528c5a96 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:58:56 -0800 Subject: [PATCH 044/367] fix(web): voice dictionary keeps the draft when Enter confirms an IME candidate (#35467) --- .../VoiceSettingsPanel.browser.test.tsx | 27 +++++++++++++++++++ .../settings/VoiceSettingsPanel.tsx | 2 ++ 2 files changed, 29 insertions(+) diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index 93187363198d..98558bc26b4b 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -599,6 +599,33 @@ it("keeps punctuation in a saved dictionary term and its alias", async () => { ]); }); +it("keeps a dictionary term and alias draft on the Enter that confirms an IME candidate", async () => { + const previous = getClientSettings(); + using cleanup = new DisposableStack(); + cleanup.defer(() => __setClientSettingsForTests(previous)); + __setClientSettingsForTests({ + ...previous, + speechCustomWords: [{ term: "日本語", aliases: [] }], + }); + await using screen = await render( + {}}> + + , + ); + const composingEnter = () => + new KeyboardEvent("keydown", { key: "Enter", isComposing: true, bubbles: true }); + const term = screen.getByRole("textbox", { name: "Add a word or phrase to the dictionary" }); + await userEvent.fill(term, "にほん"); + term.element().dispatchEvent(composingEnter()); + await expect.element(term).toHaveValue("にほん"); + await userEvent.click(screen.getByRole("button", { name: "Add an alias for 日本語" })); + const alias = screen.getByRole("textbox", { name: "Alias for 日本語" }); + await userEvent.fill(alias, "にほん"); + alias.element().dispatchEvent(composingEnter()); + expect(getClientSettings().speechCustomWords).toEqual([{ term: "日本語", aliases: [] }]); + await expect.element(alias).toHaveValue("にほん"); +}); + it("records from the first connected microphone in the saved order", async () => { const previous = getClientSettings(); using cleanup = new DisposableStack(); diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index cfddfce72d15..f4a62edf6ed1 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -391,6 +391,7 @@ function DictionaryEditor(props: { aria-label={`Add a word or phrase to the ${label}`} onChange={(event) => setTermDraft(event.target.value)} onKeyDown={(event) => { + if (event.nativeEvent.isComposing || event.keyCode === 229) return; if (event.key !== "Enter" || event.shiftKey) return; event.preventDefault(); addTerm(); @@ -427,6 +428,7 @@ function DictionaryEditor(props: { onChange={(event) => setAliasDraft({ term, value: event.target.value })} onBlur={() => setAliasDraft(null)} onKeyDown={(event) => { + if (event.nativeEvent.isComposing || event.keyCode === 229) return; if (event.key === "Escape") setAliasDraft(null); if (event.key !== "Enter") return; event.preventDefault(); From 4c255bc9c3fa45eddc2efa7394967c3a36a89946 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:59:08 -0800 Subject: [PATCH 045/367] fix(preview): Browser pane navigates when the desktop backend has no browser channel (#35424) --- apps/server/src/preview/ServerBrowser.test.ts | 20 +++++++++++++++++++ apps/server/src/preview/ServerBrowser.ts | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index a86dcc415c10..e1085b199a82 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -742,6 +742,26 @@ it.effect("a tab the desktop does not draw is marked headless, so the desktop st ).pipe(Effect.provide(layer)), ); +it.effect("a server started without the desktop browser channel marks its tabs headless", () => + Effect.scoped( + Effect.gen(function* () { + const { tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + + expect(sessions.find((session) => session.tabId === tabId)?.headless).toBe(true); + }), + ).pipe( + // A WSL-only desktop starts its backend without the browser fds. + Effect.provide( + ServerBrowser.layer.pipe( + Layer.provide(DesktopChannel.layer), + Layer.provideMerge(dependencies), + ), + ), + ), +); + it.effect("an agent's tab the reloaded desktop dropped goes back to the desktop", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index ecb3184b0997..53d2a01e6f30 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -887,8 +887,8 @@ const make = Effect.gen(function* () { if (desktop) opened.push(desktop.close); // Attaching over CDP emulates a light scheme; a tab on System follows the desktop's. if (desktop) await desktop.page.emulateMedia({ colorScheme: null }); - // Otherwise a desktop that attaches late shows its own page, and the agent drives another. - if (!desktop && desktopChannel.available) { + // Otherwise a desktop that attaches late, or has no channel to draw it, shows its own page. + if (!desktop) { await runPromise( manager .markHeadless({ threadId: ThreadId.make(snapshot.threadId), tabId: snapshot.tabId }) From ad3d60f689b7f15aa85a3c570ca8feff82c82931 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:59:22 -0800 Subject: [PATCH 046/367] fix(desktop): Nightly update checks find the newest nightly past newer preview builds (#35470) --- patches/electron-updater@6.8.9.patch | 64 +++++++++++++++++++++++++++- pnpm-lock.yaml | 6 +-- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/patches/electron-updater@6.8.9.patch b/patches/electron-updater@6.8.9.patch index 29c0b3bf62b6..00bf9bafec9f 100644 --- a/patches/electron-updater@6.8.9.patch +++ b/patches/electron-updater@6.8.9.patch @@ -1,5 +1,5 @@ diff --git a/out/electronHttpExecutor.js b/out/electronHttpExecutor.js -index 0ae9e1126e1dc136977914ef1bfd46709fc9db1b..3932b6607ca41e6663aae001111c9b18594ed37a 100644 +index 0ae9e1126e1dc136977914ef1bfd46709fc9db1b..c298ffe7808c421c29e16f1e2ba8510f0cd31142 100644 --- a/out/electronHttpExecutor.js +++ b/out/electronHttpExecutor.js @@ -61,6 +61,35 @@ class ElectronHttpExecutor extends builder_util_runtime_1.HttpExecutor { @@ -38,3 +38,65 @@ index 0ae9e1126e1dc136977914ef1bfd46709fc9db1b..3932b6607ca41e6663aae001111c9b18 addRedirectHandlers(request, options, reject, redirectCount, handler) { request.on("redirect", (statusCode, method, redirectUrl) => { // no way to modify request options, abort old and make a new one +diff --git a/out/providers/GitHubProvider.js b/out/providers/GitHubProvider.js +index e768d24e7a957caf32a4b8ca6c68cb50eb5865ee..34a0759220c0cce052285204b2d14fdf0db905e5 100644 +--- a/out/providers/GitHubProvider.js ++++ b/out/providers/GitHubProvider.js +@@ -107,6 +107,12 @@ class GitHubProvider extends BaseGitHubProvider { + catch (e) { + throw (0, builder_util_runtime_1.newError)(`Cannot parse releases feed: ${e.stack || e.message},\nXML:\n${feedXml}`, "ERR_UPDATER_INVALID_RELEASE_FEED"); + } ++ // The atom feed lists only the 10 newest releases, which other prereleases can fill. ++ let apiRelease = null; ++ if (tag == null && this.updater.allowPrerelease && this.updater.channel) { ++ apiRelease = await this.getLatestChannelRelease(this.updater.channel, cancellationToken); ++ tag = apiRelease === null ? null : apiRelease.tag_name; ++ } + if (tag == null) { + throw (0, builder_util_runtime_1.newError)(`No published versions on GitHub`, "ERR_UPDATER_NO_PUBLISHED_VERSIONS"); + } +@@ -144,6 +150,14 @@ class GitHubProvider extends BaseGitHubProvider { + } + } + const result = (0, Provider_1.parseUpdateInfo)(rawData, channelFile, channelFileUrl); ++ if (apiRelease !== null) { ++ if (result.releaseName == null) { ++ result.releaseName = apiRelease.name || apiRelease.tag_name; ++ } ++ if (result.releaseNotes == null) { ++ result.releaseNotes = apiRelease.body || ""; ++ } ++ } + if (result.releaseName == null) { + result.releaseName = latestRelease.elementValueOrEmpty("title"); + } +@@ -155,6 +169,29 @@ class GitHubProvider extends BaseGitHubProvider { + ...result, + }; + } ++ async getLatestChannelRelease(channel, cancellationToken) { ++ for (let page = 1; page <= 5; page++) { ++ const url = new url_1.URL(`${this.computeGithubBasePath(`/repos/${this.options.owner}/${this.options.repo}/releases`)}?per_page=100&page=${page}`, this.baseApiUrl); ++ const rawData = await this.httpRequest(url, { Accept: "application/vnd.github+json" }, cancellationToken); ++ const releases = rawData == null ? [] : JSON.parse(rawData); ++ if (!Array.isArray(releases)) { ++ return null; ++ } ++ for (const release of releases) { ++ if (release == null || release.draft || typeof release.tag_name !== "string" || !semver.valid(release.tag_name)) { ++ continue; ++ } ++ const releaseChannel = semver.prerelease(release.tag_name); ++ if (releaseChannel !== null && releaseChannel[0] === channel) { ++ return release; ++ } ++ } ++ if (releases.length < 100) { ++ return null; ++ } ++ } ++ return null; ++ } + async getLatestTagName(cancellationToken) { + const options = this.options; + // do not use API for GitHub to avoid limit, only for custom host or GitHub Enterprise diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 71a14ccacb0c..a829fa8be77b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -373,7 +373,7 @@ patchedDependencies: alchemy@2.0.0-beta.85: 07cc8c7e14b3b9af143ddd40a059f117c27e55139fbf20bd5dde2eb30509cedd app-builder-lib@26.16.1: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53 dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4 - electron-updater@6.8.9: 2ed21db470f254c5e095a0eb8dabc086c310d180322004bceee7d56e7ce86b84 + electron-updater@6.8.9: 71117ae11bf6058893dca8b40ce46025996924b76e534d56a53e04058781071e expo-audio@58.0.11: 93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483 expo-modules-core@58.0.30: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7 expo-network@58.0.4: 159c2d760d1cd5102e95df51c20600111e7a72096ef0daea8215d50cae48bd98 @@ -463,7 +463,7 @@ importers: version: 8.2.0 electron-updater: specifier: 6.8.9 - version: 6.8.9(patch_hash=2ed21db470f254c5e095a0eb8dabc086c310d180322004bceee7d56e7ce86b84)(supports-color@8.1.1) + version: 6.8.9(patch_hash=71117ae11bf6058893dca8b40ce46025996924b76e534d56a53e04058781071e)(supports-color@8.1.1) electron-webauthn: specifier: 'catalog:' version: 1.3.1(typescript@6.0.3) @@ -19015,7 +19015,7 @@ snapshots: electron-to-chromium@1.5.430: {} - electron-updater@6.8.9(patch_hash=2ed21db470f254c5e095a0eb8dabc086c310d180322004bceee7d56e7ce86b84)(supports-color@8.1.1): + electron-updater@6.8.9(patch_hash=71117ae11bf6058893dca8b40ce46025996924b76e534d56a53e04058781071e)(supports-color@8.1.1): dependencies: builder-util-runtime: 9.7.0(supports-color@8.1.1) fs-extra: 10.1.0 From 4f86f94e151ef6729a645766c034bf0e75010942 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:59:35 -0800 Subject: [PATCH 047/367] fix(mobile): files with # or ? in their path keep audio playback and Markdown Preview (#35448) --- apps/mobile/src/features/files/filePath.test.ts | 13 +++++++++++++ apps/mobile/src/features/files/filePath.ts | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/features/files/filePath.test.ts b/apps/mobile/src/features/files/filePath.test.ts index 46b26d7e3bf3..c58316af3698 100644 --- a/apps/mobile/src/features/files/filePath.test.ts +++ b/apps/mobile/src/features/files/filePath.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vite-plus/test"; import { fileRoutePathSegments, + isAudioPreviewFile, + isMarkdownPreviewFile, needsWorkspaceFileContents, resolveFileLinkPath, resolveWorkspaceRelativeFilePath, @@ -58,3 +60,14 @@ describe("needsWorkspaceFileContents", () => { expect(needsWorkspaceFileContents({ ...input, metadataKind: "directory" })).toBe(false); }); }); + +describe("workspace preview classifiers", () => { + it("treat # and ? as literal filename characters", () => { + for (const path of ["notes/clip#1.wav", "notes/clip?1.wav", "notes#old/clip.wav"]) { + expect(isAudioPreviewFile(path)).toBe(true); + } + for (const path of ["docs/draft#1.md", "docs/draft?1.md", "docs#old/draft.md"]) { + expect(isMarkdownPreviewFile(path)).toBe(true); + } + }); +}); diff --git a/apps/mobile/src/features/files/filePath.ts b/apps/mobile/src/features/files/filePath.ts index e38ff86fa665..330ff5784da6 100644 --- a/apps/mobile/src/features/files/filePath.ts +++ b/apps/mobile/src/features/files/filePath.ts @@ -100,15 +100,15 @@ export function isVideoPreviewFile(path: string): boolean { } export function isAudioPreviewFile(path: string): boolean { - return isWorkspaceAudioPreviewPath(path.split(/[?#]/, 1)[0] ?? ""); + return isWorkspaceAudioPreviewPath(path); } export function isSvgImagePreviewFile(path: string): boolean { - return /\.svg$/i.test(path.split(/[?#]/, 1)[0] ?? ""); + return /\.svg$/i.test(path); } export function isMarkdownPreviewFile(path: string): boolean { - return /\.(?:md|mdx)$/i.test(path.split(/[?#]/, 1)[0] ?? ""); + return /\.(?:md|mdx)$/i.test(path); } export function needsWorkspaceFileContents(input: { From 729687e23c67bf96256fb4915f1847122307d46a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:59:48 -0800 Subject: [PATCH 048/367] fix(web): device tab rename no longer saves on the Enter that confirms an IME candidate (#35468) --- .../RightPanelTabs.browser.test.tsx | 39 +++++++++++++++++++ apps/web/src/components/RightPanelTabs.tsx | 1 + 2 files changed, 40 insertions(+) diff --git a/apps/web/src/components/RightPanelTabs.browser.test.tsx b/apps/web/src/components/RightPanelTabs.browser.test.tsx index cceb8c22c707..f803cb41520b 100644 --- a/apps/web/src/components/RightPanelTabs.browser.test.tsx +++ b/apps/web/src/components/RightPanelTabs.browser.test.tsx @@ -2,6 +2,7 @@ import "~/index.css"; import { EnvironmentId } from "@t3tools/contracts"; import { describe, expect, it, vi } from "vite-plus/test"; +import { userEvent } from "vite-plus/test/browser"; import { InputGroup, InputGroupAddon, InputGroupInput } from "~/components/ui/input-group"; import type { RightPanelSurface } from "~/rightPanelStore"; @@ -249,3 +250,41 @@ describe("right panel thumb buttons", () => { expect(actions).toEqual(["back", "forward", "back"]); }); }); + +const DEVICE_SURFACE: RightPanelSurface = { + id: "device:host:phone", + kind: "device", + target: { hostId: "host", deviceId: "phone", platform: "ios", name: "Phone" }, +}; + +describe("device tab rename", () => { + it("keeps the editor open on an Enter that confirms an IME candidate", async () => { + const onRenameDevice = vi.fn(); + const props = panel(DEVICE_SURFACE).props as RightPanelTabsProps; + await using screen = await render( + + {null} + , + ); + await userEvent.dblClick(screen.getByRole("button", { name: "Phone", exact: true })); + const input = screen.getByRole("textbox", { name: "Device tab name" }); + await userEvent.fill(input, "にほん"); + + for (const shape of [ + { isComposing: true, keyCode: 13 }, + { isComposing: false, keyCode: 229 }, + ]) { + input + .element() + .dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true, cancelable: true, ...shape }), + ); + } + + await expect.element(input).toHaveFocus(); + expect(onRenameDevice).not.toHaveBeenCalled(); + await userEvent.fill(input, "日本語"); + await userEvent.keyboard("{Enter}"); + expect(onRenameDevice).toHaveBeenCalledExactlyOnceWith(DEVICE_SURFACE.id, "日本語"); + }); +}); diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index 223799efb26f..d220385e2814 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -965,6 +965,7 @@ function RightPanelTab( onBlur={(event) => onFinishRename(surface.id, event.currentTarget.value)} onKeyDown={(event) => { event.stopPropagation(); + if (event.nativeEvent.isComposing || event.keyCode === 229) return; if (event.key === "Enter") event.currentTarget.blur(); if (event.key === "Escape") { event.currentTarget.value = title; From c1ea9782d2686ac605982af176e9beb7d061c9ce Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 10:59:58 -0800 Subject: [PATCH 049/367] test(server): HEAD watch removal test expects isRepo in branch status (#35498) From a7779fc558e0f567384a05fb589b02cc520630d5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:00:11 -0800 Subject: [PATCH 050/367] fix(web): a focused folder New thread button no longer draws over the hidden-thread count (#35499) --- .../web/src/components/sidebar/SidebarProjectFolderHeader.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/src/components/sidebar/SidebarProjectFolderHeader.tsx b/apps/web/src/components/sidebar/SidebarProjectFolderHeader.tsx index 38b1e53bf057..1e6510bbe37e 100644 --- a/apps/web/src/components/sidebar/SidebarProjectFolderHeader.tsx +++ b/apps/web/src/components/sidebar/SidebarProjectFolderHeader.tsx @@ -85,7 +85,7 @@ export const SidebarProjectFolderHeader = memo(function SidebarProjectFolderHead ) : null} {collapsed && hiddenThreads.length > 0 ? ( - + {hiddenThreads.length} ) : null} @@ -97,7 +97,7 @@ export const SidebarProjectFolderHeader = memo(function SidebarProjectFolderHead )} /> - {/* Sits over the count, which hides on hover, and left of the chevron. */} + {/* Sits over the count, which hides on hover or focus, and left of the chevron. */} Date: Thu, 12 Nov 2026 11:05:13 -0800 Subject: [PATCH 051/367] fix(grok): Grok MCP calls store the real tool, arguments, and result (#35502) Co-authored-by: Claude Opus 5.5 --- .../src/server/xaiAcpExtension.test.ts | 76 +++++++++++++++++++ .../src/server/xaiAcpExtension.ts | 49 +++++++++++- packages/shared/src/htmlRender.test.ts | 10 --- packages/shared/src/toolOutput.ts | 6 +- 4 files changed, 125 insertions(+), 16 deletions(-) diff --git a/packages/provider-grok/src/server/xaiAcpExtension.test.ts b/packages/provider-grok/src/server/xaiAcpExtension.test.ts index a74078733d5d..b1a24b2cff81 100644 --- a/packages/provider-grok/src/server/xaiAcpExtension.test.ts +++ b/packages/provider-grok/src/server/xaiAcpExtension.test.ts @@ -30,9 +30,11 @@ import { import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; import { type AcpToolCallState, + extractMcpToolCallIdentity, mergeToolCallState, parseSessionUpdateEvent, } from "@t3tools/provider-acp/server/runtimeModel"; +import { compactDynamicToolOutput, htmlRenderFromToolItem } from "@t3tools/shared/toolOutput"; const decodeXAiAskUserQuestionRequest = Schema.decodeUnknownSync(XAiAskUserQuestionRequest); @@ -340,6 +342,80 @@ describe("XAiAcpExtension", () => { expect(extractXAiMonitorTaskId(normalized)).toBe("019f44a5-87d1-7640-8e35-6a4667ffc873"); }); + // regression: #23714 Recorded from Grok 1.0.77 calling stdio MCP tools through `use_tool`. + it("rewrites use_tool MCP calls and results into the generic ACP MCP shapes", () => { + const meta = { "x.ai/tool": { version: 1, name: "use_tool", kind: "use_tool" } }; + const grokCall = ( + toolCallId: string, + tool: string, + toolInput: Record, + completion: Record, + ) => { + const rawInput = { tool_name: tool, tool_input: toolInput }; + const updates = [ + { sessionUpdate: "tool_call", toolCallId, title: "use_tool", rawInput, _meta: meta }, + { + sessionUpdate: "tool_call_update", + toolCallId, + kind: "other", + title: tool, + rawInput: { variant: "UseTool", ...rawInput }, + _meta: meta, + }, + { sessionUpdate: "tool_call_update", toolCallId, ...completion }, + ]; + let state: AcpToolCallState | undefined; + for (const update of updates) { + state = normalizeXAiAcpToolCallState( + mergeToolCallState(state, parseRecordedToolCallUpdate(update)), + ); + } + return state!; + }; + const page = { attachmentId: "abc-html", title: "T", height: 300 }; + + const rendered = grokCall( + "call-85597c6c-6fb8-4c9c-a6a8-12c4a74ebc96-1", + "t3-code__html_render", + { path: "a.html" }, + { + status: "completed", + rawOutput: { + type: "MCP", + tool_name: "html_render", + server_name: "t3-code", + output: { OkayOutput: JSON.stringify({ htmlRender: page, message: "Shown" }) }, + }, + }, + ); + expect(extractMcpToolCallIdentity(rendered)).toEqual({ + server: "t3-code", + tool: "html_render", + }); + expect(rendered.data.rawInput).toMatchObject({ arguments: { path: "a.html" } }); + expect( + htmlRenderFromToolItem({ toolName: "t3-code.html_render", output: rendered.data.rawOutput }), + ).toEqual(page); + + const failed = grokCall( + "call-2baebdc5-2446-4ac2-9652-85f97a76a05b-2", + "pencil__browser", + { action: "load-page" }, + { + status: "failed", + rawOutput: { + type: "MCP", + tool_name: "browser", + server_name: "pencil", + output: { Error: "Page not found" }, + is_error: true, + }, + }, + ); + expect(extractMcpToolCallIdentity(failed)).toEqual({ server: "pencil", tool: "browser" }); + expect(compactDynamicToolOutput(failed.data.rawOutput)?.isError).toBe(true); + }); + it("keeps Monitor ticks running while Grok reports them in progress", () => { // Grok 1.0.41 ticks report in_progress with exit_code 0; only _x.ai/task_completed ends the monitor. const command = "for i in 1 2 3; do sleep 8; echo tick $i; done"; diff --git a/packages/provider-grok/src/server/xaiAcpExtension.ts b/packages/provider-grok/src/server/xaiAcpExtension.ts index 022dd327b55b..7181d1927923 100644 --- a/packages/provider-grok/src/server/xaiAcpExtension.ts +++ b/packages/provider-grok/src/server/xaiAcpExtension.ts @@ -558,7 +558,54 @@ function resolveXAiAcpToolTitle(toolCall: AcpToolCallState): string | undefined } /** Grok 1.0.41 monitor ticks report exit_code 0 before ending via _x.ai/task_completed. */ -export function normalizeXAiAcpToolCallState(toolCall: AcpToolCallState): AcpToolCallState { +// Grok calls MCP tools through its `use_tool` built-in as "__" and wraps results as +// { type: "MCP", output: { OkayOutput } | { Error } }; rewrite both into the generic ACP MCP shapes. +function normalizeXAiMcpToolCall(toolCall: AcpToolCallState): AcpToolCallState { + const meta = unknownRecord(toolCall.data.meta); + const rawInput = unknownRecord(toolCall.data.rawInput); + const toolName = nonEmptyString(rawInput?.tool_name) ?? ""; + const separator = toolName.indexOf("__"); + const isUseTool = + unknownRecord(meta?.["x.ai/tool"])?.name === "use_tool" && + separator > 0 && + separator + 2 < toolName.length; + const rawOutput = unknownRecord(toolCall.data.rawOutput); + const result = rawOutput?.type === "MCP" ? unknownRecord(rawOutput.output) : undefined; + const text = + typeof result?.Error === "string" + ? result.Error + : typeof result?.OkayOutput === "string" + ? result.OkayOutput + : undefined; + if (!isUseTool && text === undefined) return toolCall; + return { + ...toolCall, + data: { + ...toolCall.data, + ...(isUseTool + ? { + meta: { ...meta, is_mcp_tool_call: true }, + rawInput: { + server: toolName.slice(0, separator), + tool: toolName.slice(separator + 2), + arguments: unknownRecord(rawInput?.tool_input) ?? {}, + }, + } + : {}), + ...(text === undefined + ? {} + : { + rawOutput: { + ...(typeof result?.Error === "string" ? { isError: true } : {}), + content: [{ type: "text", text }], + }, + }), + }, + }; +} + +export function normalizeXAiAcpToolCallState(incoming: AcpToolCallState): AcpToolCallState { + const toolCall = normalizeXAiMcpToolCall(incoming); const resolvedTitle = resolveXAiAcpToolTitle(toolCall); const withTitle = resolvedTitle !== undefined && resolvedTitle !== toolCall.title diff --git a/packages/shared/src/htmlRender.test.ts b/packages/shared/src/htmlRender.test.ts index f76b1c8a5625..9f6315541547 100644 --- a/packages/shared/src/htmlRender.test.ts +++ b/packages/shared/src/htmlRender.test.ts @@ -169,16 +169,6 @@ describe("htmlRenderFromToolItem", () => { ["html_render", result], // OpenCode 2 Code Mode reports a script's nested html_render call as one `execute` item. ["execute", JSON.stringify(result)], - // Grok CLI wraps MCP results as { type: "MCP", output: { OkayOutput: "" } }. - [ - "t3-code.html_render", - { - type: "MCP", - tool_name: "html_render", - server_name: "t3-code", - output: { OkayOutput: JSON.stringify(result) }, - }, - ], ] as const) { expect(htmlRenderFromToolItem({ toolName, output })).toEqual(reference); } diff --git a/packages/shared/src/toolOutput.ts b/packages/shared/src/toolOutput.ts index 6d63d9425d76..f2773b0045e4 100644 --- a/packages/shared/src/toolOutput.ts +++ b/packages/shared/src/toolOutput.ts @@ -100,11 +100,7 @@ function readResult(value: unknown, budget: ResultReadBudget, depth = 0): Result value.is_error === true || value._tag === "OrchestratorMcpFailure" || value.error != null; - // Grok CLI wraps MCP results as { type: "MCP", output: { OkayOutput: "" } }. - const content = - value.structuredContent ?? - value.content ?? - (Predicate.isObject(value.output) ? value.output.OkayOutput : undefined); + const content = value.structuredContent ?? value.content; if (content !== undefined) { const nested = readResult(content, budget, depth + 1); return { ...nested, failed: failed || nested.failed }; From 17ddf292aafc0f186fdc907c5ee7423693299ab5 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 12 Nov 2026 11:05:17 -0800 Subject: [PATCH 052/367] fix(server): Load earlier turns no longer drops the middle of threads an agent started (#35405) Co-Authored-By: Claude Fable 5.2 --- .../orchestration-v2/ProjectionStore.test.ts | 74 +++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 11 +++ .../orchestration-v2/threadHistoryPaging.ts | 4 +- 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index fa670c448c89..511c252689cc 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -54,6 +54,7 @@ import { encodeThreadHistoryCursor, selectHistoryPageFromCursor, THREAD_HISTORY_PAGE_POLICY, + THREAD_HISTORY_SNAPSHOT_ROW_LIMIT, THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT, } from "./threadHistoryPaging.ts"; import { buildBoundedThreadStreamSnapshot } from "./ThreadStream.ts"; @@ -5688,6 +5689,79 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { ) `; assert.strictEqual((yield* forkPage()).totalItems, forkTotal + 2); + + // An agent started this thread, so the snapshot folds no work. Its cursor then names a + // command that an older page folds into the run's one work group. + yield* sql` + UPDATE orchestration_v2_projection_turn_items + SET type = CASE WHEN ordinal = 0 THEN type ELSE 'assistant_message' END, + payload_json = CASE + WHEN ordinal = 0 THEN json_set(payload_json, '$.createdBy', 'agent') + ELSE json_set(payload_json, '$.type', 'assistant_message', + '$.messageId', turn_item_id, '$.streaming', json('false')) + END + WHERE thread_id = ${threadId} AND type = 'user_message' + `; + yield* sql` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + 'event:paged-scrollback:agent-started', 'thread', ${threadId}, 2, + 'turn-item.updated', ${nowIso}, 'user', '{}', '{}', 2 + ) + `; + const window = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: THREAD_HISTORY_SNAPSHOT_ROW_LIMIT, + userTurnLimit: THREAD_HISTORY_PAGE_POLICY.maxUserTurns, + deferWork: true, + }); + const bounded = buildBoundedThreadProjection({ + projection: window.projection, + snapshotSequence: window.snapshotSequence, + }); + const loaded = [bounded.projection.visibleTurnItems]; + let earlier = bounded.hasMoreHistory ? bounded.historyCursor : null; + while (earlier !== null) { + assert.isBelow(loaded.length, 20); + const page: OrchestrationV2ThreadHistoryPage = yield* projectionStore.getThreadHistoryPage( + threadId, + earlier, + undefined, + false, + "deferred", + ); + loaded.unshift(page.items); + earlier = page.hasMoreHistory ? page.nextCursor : null; + } + const held = loaded.flat(); + const heldOrdinals = held.map((row) => row.item.ordinal); + assert.deepEqual( + heldOrdinals, + [...new Set(heldOrdinals)].toSorted((left, right) => left - right), + ); + const folded = held.find((row) => row.item.deferredWork !== undefined)?.item; + assert.isAbove(folded?.deferredWork?.itemCount ?? 0, 150); + const everyRow = yield* projectionStore.getTimelinePage(threadId, { + view: "activity", + limit: itemCount + 2, + }); + // Every row is on the client, or is work that the folded row stands for. + assert.deepEqual( + everyRow.items + .filter( + ({ item }) => + !heldOrdinals.includes(item.ordinal) && + !( + item.type === "command_execution" && + folded?.deferredWork !== undefined && + item.ordinal > folded.ordinal && + item.ordinal <= folded.deferredWork.lastOrdinal + ), + ) + .map(({ item }) => item.ordinal), + [], + ); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 22d8cfe25ddb..54ef44c079aa 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -6943,6 +6943,16 @@ export const layer: Layer.Layer = const historyItems = workMode === "deferred" ? deferWorkHistoryRows(indexed) : indexed; const policy = olderHistoryPagePolicy({ items: indexed, cursor, throughEntryId }); + // Folding can hide the cursor's row: a snapshot folds no work in a thread with no + // user turn. The page then ends at the first row kept from the cursor on. + const cursorRow = + indexed[findCursorEndIndex(indexed, decodeThreadHistoryCursor(cursor))]; + const end = + cursorRow === undefined + ? -1 + : historyItems.findIndex( + (row) => row.canonicalPosition >= cursorRow.canonicalPosition, + ); const page = workMode === "work" ? selectDeferredWorkPage({ @@ -6955,6 +6965,7 @@ export const layer: Layer.Layer = : selectHistoryPageFromCursor({ items: historyItems, cursor, + exclusiveEndIndex: end === -1 ? historyItems.length : end, snapshotSequence, rowEncodedBytes: () => 0, policy, diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index f088d389fdd5..cf9a101bba44 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -485,13 +485,15 @@ export function selectHistoryPageFromCursor(input: { readonly policy?: ThreadHistoryPagePolicy | undefined; readonly rowEncodedBytes?: ((row: Row) => number) | undefined; readonly throughEntryId?: string | undefined; + /** Where the page ends in `items`. Set when `items` can lack the cursor's row. */ + readonly exclusiveEndIndex?: number | undefined; /** Set when `items` holds only the newest part of the history. */ readonly prefix?: ThreadHistoryPrefix | undefined; }): SelectTimelinePageResult { const decoded = decodeThreadHistoryCursor(input.cursor); return selectOlderTimelinePage({ items: input.items, - exclusiveEndIndex: findCursorEndIndex(input.items, decoded), + exclusiveEndIndex: input.exclusiveEndIndex ?? findCursorEndIndex(input.items, decoded), snapshotSequence: input.snapshotSequence, ...(input.rowEncodedBytes === undefined ? {} : { rowEncodedBytes: input.rowEncodedBytes }), policy: olderHistoryPagePolicy(input), From 1113e4b510da0983db7cab202fd8b4ce281cad8d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:07:54 -0800 Subject: [PATCH 053/367] fix(scripts): `vp run dev --share` removes its tailnet mapping when the terminal closes (#35534) --- scripts/dev-runner.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index 9cd8e598acf1..0127e76e0384 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -992,6 +992,9 @@ const cliRuntimeLayer = Layer.mergeAll( ); if (import.meta.main) { + // runMain interrupts (running finalizers such as the --share unmapping) only + // on SIGINT/SIGTERM; a closed terminal or dropped SSH session sends SIGHUP. + process.on("SIGHUP", () => process.kill(process.pid, "SIGTERM")); Command.run(devRunnerCli, { version: "0.0.0" }).pipe( Effect.scoped, Effect.provide(cliRuntimeLayer), From dbbd1d253c1ccaa8e08002cac421870c5d4103f0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:13:12 -0800 Subject: [PATCH 054/367] fix(server): "Open in" stops offering the tailnet name after `tailscale down` (#35539) --- .../src/environment/RemoteOpenTargets.test.ts | 26 +++++++++++++++++++ .../src/environment/RemoteOpenTargets.ts | 3 ++- packages/tailscale/src/tailscale.test.ts | 1 + packages/tailscale/src/tailscale.ts | 7 +++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/apps/server/src/environment/RemoteOpenTargets.test.ts b/apps/server/src/environment/RemoteOpenTargets.test.ts index 562321055f86..0d0da302efa5 100644 --- a/apps/server/src/environment/RemoteOpenTargets.test.ts +++ b/apps/server/src/environment/RemoteOpenTargets.test.ts @@ -162,6 +162,32 @@ describe("RemoteOpenTargets", () => { }), ); + it.effect("drops the tailnet name after `tailscale down` while sshd listens", () => + Effect.gen(function* () { + const targets = yield* resolveTargets({ + sshd: { ipv4: true, ipv6: false }, + tailscale: { + // After `tailscale down`, status still exits 0 and keeps Self from the last netmap. + status: { + exitCode: 0, + stdout: JSON.stringify({ + BackendState: "Stopped", + Self: { + DNSName: "bb-1.tail1234.ts.net.", + TailscaleIPs: ["100.64.1.2"], + Online: false, + }, + }), + }, + prefs: prefsJson({ RunSSH: false, WantRunning: false }), + }, + hostname: "bb-1", + username: "t3-user", + }); + expect(targets).toEqual([{ kind: "mdns", host: "bb-1.local", username: "t3-user" }]); + }), + ); + it.effect("accepts an sshd bound to IPv6 loopback only", () => Effect.gen(function* () { const targets = yield* resolveTargets({ diff --git a/apps/server/src/environment/RemoteOpenTargets.ts b/apps/server/src/environment/RemoteOpenTargets.ts index d6837560790f..beb216339f06 100644 --- a/apps/server/src/environment/RemoteOpenTargets.ts +++ b/apps/server/src/environment/RemoteOpenTargets.ts @@ -37,7 +37,8 @@ export const make = Effect.gen(function* () { Effect.orElseSucceed(() => null), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); - const magicDnsName = tailscaleStatus?.magicDnsName ?? null; + const magicDnsName = + tailscaleStatus?.running === false ? null : (tailscaleStatus?.magicDnsName ?? null); const targets: Array = []; diff --git a/packages/tailscale/src/tailscale.test.ts b/packages/tailscale/src/tailscale.test.ts index 569a204c0651..094b86837104 100644 --- a/packages/tailscale/src/tailscale.test.ts +++ b/packages/tailscale/src/tailscale.test.ts @@ -255,6 +255,7 @@ describe("tailscale", () => { Effect.gen(function* () { const status = yield* parseTailscaleStatus(tailscaleStatusJson); assert.deepEqual(status, { + running: null, magicDnsName: "desktop.tail.ts.net", tailnetIpv4Addresses: ["100.100.100.100"], httpsServeEnabled: null, diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index 03ab702c86dd..3b3088e017d7 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -145,12 +145,18 @@ const TailscaleStatusSelf = Schema.Struct({ }); const TailscaleStatusJson = Schema.Struct({ + BackendState: Schema.optional(Schema.Unknown), Self: Schema.optional(TailscaleStatusSelf), }); export type TailscaleStatusJson = typeof TailscaleStatusJson.Type; export interface TailscaleStatus { + /** + * `tailscale down` keeps `Self` from the last netmap, so the DNS name alone + * does not mean the tailnet is reachable. `null` when the CLI omits BackendState. + */ + readonly running: boolean | null; readonly magicDnsName: string | null; readonly tailnetIpv4Addresses: readonly string[]; /** @@ -220,6 +226,7 @@ export const parseTailscaleStatus = ( const capMap = parsed.Self?.CapMap; return { + running: parsed.BackendState === undefined ? null : parsed.BackendState === "Running", magicDnsName: normalizeMagicDnsName(parsed), tailnetIpv4Addresses, httpsServeEnabled: From 13645b190465e614ce5978d274ea7da88879c8bb Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 12 Nov 2026 13:16:46 -0600 Subject: [PATCH 055/367] fix(web): align timeline activity timestamps (#35549) --- .../components/chat/TimelineWorkEntryRow.tsx | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/apps/web/src/components/chat/TimelineWorkEntryRow.tsx b/apps/web/src/components/chat/TimelineWorkEntryRow.tsx index 253c83de3f4d..dfefdedeed52 100644 --- a/apps/web/src/components/chat/TimelineWorkEntryRow.tsx +++ b/apps/web/src/components/chat/TimelineWorkEntryRow.tsx @@ -535,21 +535,17 @@ function WorkEntryLogRow(props: WorkEntryRowProps) { !toolIconAcceptsTint(entryToolIcon) ? ( ) : null} + {canExpandProjectedItem ? ( + + + + ) : null} - - - } > From fd344d88ffbbd21dbabe8672db4a1a39c2182c4f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:18:31 -0800 Subject: [PATCH 056/367] fix(tailscale): Serve setup fails when the tailnet has not enabled HTTPS (#35538) --- packages/tailscale/src/tailscale.test.ts | 25 +++++++++++++++++++++- packages/tailscale/src/tailscale.ts | 27 ++++++++++++++++++++---- scripts/lib/dev-share.ts | 13 ++++++++---- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/packages/tailscale/src/tailscale.test.ts b/packages/tailscale/src/tailscale.test.ts index 094b86837104..bcf4ffec59a6 100644 --- a/packages/tailscale/src/tailscale.test.ts +++ b/packages/tailscale/src/tailscale.test.ts @@ -418,6 +418,29 @@ describe("tailscale", () => { }), ); + // regression: #35510 - without the https capability `tailscale serve` prints an + // enablement notice, writes no Serve config, and still exits 0. + it.effect("fails serve when the tailnet has not enabled HTTPS", () => { + const layer = layerMockSpawner((_command, args) => { + if (args[0] === "status") { + return { + stdout: + '{"BackendState":"Running","Self":{"DNSName":"desktop.tail.ts.net.","CapMap":{}}}', + }; + } + if (args[1] === "status") return { stdout: "{}\n" }; + return { stdout: "\nYou don't have permission to enable this feature.\n" }; + }); + + return Effect.gen(function* () { + const exit = yield* ensureTailscaleServe({ localPort: 3773 }).pipe( + Effect.provide(layer), + Effect.exit, + ); + assert.isTrue(Exit.isFailure(exit), "no Serve route was configured"); + }); + }); + // regression: #21609 - without forceKillAfter a timed-out probe only gets // SIGTERM, so a group member that ignores it outlives the call. it.effect("escalates timed-out probes to SIGKILL", () => { @@ -440,7 +463,7 @@ describe("tailscale", () => { yield* readTailscaleStatus.pipe(Effect.provide(layer)); yield* readTailscaleSshEnabled.pipe(Effect.provide(layer)); yield* ensureTailscaleServe({ localPort: 45_678 }).pipe(Effect.provide(layer)); - assert.deepEqual(killGraces, [500, 500, 500]); + assert.deepEqual(killGraces, [500, 500, 500, 500]); }); }); }); diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index 3b3088e017d7..02e5742b34df 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -450,16 +450,35 @@ const runTailscaleCommand = Effect.fn("tailscale.runTailscaleCommand")(function* ); }); -export const ensureTailscaleServe = (input: { +export class TailscaleServeHttpsDisabledError extends Schema.TaggedError()( + "TailscaleServeHttpsDisabledError", + {}, +) { + override get message(): string { + return "Tailscale Serve needs HTTPS enabled on the tailnet. Enable HTTPS certificates at https://login.tailscale.com/admin/dns."; + } +} + +export const ensureTailscaleServe = Effect.fn("tailscale.ensureTailscaleServe")(function* (input: { readonly localPort: number; readonly servePort?: number; readonly localHost?: string; -}): Effect.Effect => { +}): Effect.fn.Return< + void, + TailscaleCommandError | TailscaleServeHttpsDisabledError, + ChildProcessSpawner.ChildProcessSpawner +> { + // Without the capability `tailscale serve` prints an enablement notice and exits 0, or blocks on + // it, having configured nothing either way. + const status = yield* Effect.option(readTailscaleStatus); + if (Option.isSome(status) && status.value.httpsServeEnabled === false) { + return yield* new TailscaleServeHttpsDisabledError(); + } const servePort = input.servePort ?? DEFAULT_TAILSCALE_SERVE_PORT; const localHost = input.localHost ?? "127.0.0.1"; const args = ["serve", "--bg", `--https=${servePort}`, `http://${localHost}:${input.localPort}`]; - return runTailscaleCommand(args, TAILSCALE_SERVE_TIMEOUT); -}; + return yield* runTailscaleCommand(args, TAILSCALE_SERVE_TIMEOUT); +}); /** A Serve route: a web handler at `mount` on `servePort`, or a TCP forward without one. */ export interface TailscaleServeRoute { diff --git a/scripts/lib/dev-share.ts b/scripts/lib/dev-share.ts index 781443214590..424fc6c86555 100644 --- a/scripts/lib/dev-share.ts +++ b/scripts/lib/dev-share.ts @@ -6,6 +6,7 @@ import { ensureTailscaleServe, readTailscaleStatus, type TailscaleCommandError, + type TailscaleServeHttpsDisabledError, type TailscaleStderrDiagnostic, } from "@t3tools/tailscale"; import * as Effect from "effect/Effect"; @@ -21,10 +22,14 @@ const DIAGNOSTIC_EXPLANATIONS: Record - error._tag === "TailscaleCommandExitError" && error.stderrDiagnostic !== undefined - ? (DIAGNOSTIC_EXPLANATIONS[error.stderrDiagnostic] ?? "run the command by hand to see why") - : undefined; +const explainCommandFailure = ( + error: TailscaleCommandError | TailscaleServeHttpsDisabledError, +): string | undefined => + error._tag === "TailscaleServeHttpsDisabledError" + ? "HTTPS is not enabled on this tailnet — enable HTTPS certificates at https://login.tailscale.com/admin/dns" + : error._tag === "TailscaleCommandExitError" && error.stderrDiagnostic !== undefined + ? (DIAGNOSTIC_EXPLANATIONS[error.stderrDiagnostic] ?? "run the command by hand to see why") + : undefined; export class TailscaleUnavailableError extends Schema.TaggedError()( "TailscaleUnavailableError", From 0c3a9982b1b6d90741ba1d5b1893ccdb1068b2d2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:18:44 -0800 Subject: [PATCH 057/367] fix(scripts): --share --built --host localhost points the tailnet URL at the backend (#35542) --- scripts/dev-runner.test.ts | 9 +++++++++ scripts/dev-runner.ts | 12 ++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/dev-runner.test.ts b/scripts/dev-runner.test.ts index d522bfdb1579..25fb88a80696 100644 --- a/scripts/dev-runner.test.ts +++ b/scripts/dev-runner.test.ts @@ -574,6 +574,15 @@ it.layer(NodeServices.layer)("dev-runner", (it) => { }), ); + // Node resolves localhost to ::1 first, so a 127.0.0.1 target can miss the backend. + it.effect("--built --host localhost keeps the backend's localhost name", () => + Effect.gen(function* () { + const { env, tailscaleCalls } = yield* shareSpawned({ built: true, host: "localhost" }); + const serve = tailscaleCalls.find((args) => args.includes("--bg")); + assert.equal(serve?.at(-1), `http://localhost:${env?.T3CODE_PORT}`); + }), + ); + it.effect("without --built keeps Vite behind the tailnet URL", () => Effect.gen(function* () { const { env, vpArgs, tailscaleCalls } = yield* shareSpawned({}); diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index 0127e76e0384..4a310e9e8117 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -59,6 +59,13 @@ export function isProxiableBindHost(host: string): boolean { ); } +function sharedBackendTargetHost(host: string | undefined): string { + const normalized = host?.trim() ?? ""; + if (normalized === "::1" || normalized === "[::1]") return "[::1]"; + if (normalized === "localhost") return "localhost"; + return "127.0.0.1"; +} + export const DEFAULT_T3_HOME = Effect.map(Effect.service(Path.Path), (path) => path.join(NodeOS.homedir(), ".t3"), ); @@ -788,12 +795,13 @@ export const runDevRunnerWithInput = Effect.fn("dev-runner.runDevRunnerWithInput shareDevServer({ webPort: sharedWebPort, // The backend binds 127.0.0.1 unless --host changes it. A wildcard - // still answers there; `--host ::1` binds only the IPv6 loopback. + // still answers there; `--host ::1` binds only the IPv6 loopback, and + // `localhost` binds whichever loopback Node resolves first. ...(input.built ? { localTarget: { port: Number(env.T3CODE_PORT), - host: ["::1", "[::1]"].includes(input.host?.trim() ?? "") ? "[::1]" : "127.0.0.1", + host: sharedBackendTargetHost(input.host), }, } : {}), From 77f40aff37b8478d39f8f342bdf7417d07459065 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:18:59 -0800 Subject: [PATCH 058/367] fix(server): t3 pair --tailscale no longer replaces another app's Serve route (#35547) --- apps/server/src/cli/pair.test.ts | 89 ++++++++++++++++++++++++++++- apps/server/src/cli/pair.ts | 10 ++++ apps/server/src/server.ts | 20 +------ packages/tailscale/src/tailscale.ts | 22 ++++++- 4 files changed, 121 insertions(+), 20 deletions(-) diff --git a/apps/server/src/cli/pair.test.ts b/apps/server/src/cli/pair.test.ts index e626461914a0..7f5ed1113cea 100644 --- a/apps/server/src/cli/pair.test.ts +++ b/apps/server/src/cli/pair.test.ts @@ -35,7 +35,11 @@ import { type PersistedServerRuntimeState, } from "../serverRuntimeState.ts"; import { resolveHostCliCommand } from "./invocation.ts"; -import { DevServerNotProxiableError, resolveTailscaleLocalTarget } from "./pair.ts"; +import { + DevServerNotProxiableError, + resolveTailscaleLocalTarget, + ServePortOccupiedError, +} from "./pair.ts"; import packageJson from "../../package.json" with { type: "json" }; import { buildPairingAppLink } from "@t3tools/shared/remote"; @@ -136,7 +140,7 @@ describe("t3 pair", () => { Effect.gen(function* () { if (!ChildProcess.isStandardCommand(command)) return yield* Effect.die("Unexpected pipeline"); - if (command.args[0] === "serve") { + if (command.args[0] === "serve" && command.args[1] === "--bg") { const target = String(command.args.at(-1)); const descriptor = yield* Effect.promise(async () => (await fetch(new URL("/.well-known/t3/environment", target))).json(), @@ -194,6 +198,87 @@ describe("t3 pair", () => { ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); + // The MagicDNS probe fails whenever the host cannot reach its own *.ts.net name, + // so it cannot be what decides that the port is free. + it.effect( + "leaves another app's Serve route alone when the host cannot reach its MagicDNS URL", + () => + withDescriptorServer((origin) => + // Another live app on loopback; a dead port's route may be taken back. + withDescriptorServer((otherAppOrigin) => + Effect.gen(function* () { + const fetch = yield* FetchHttpClient.Fetch; + const tailscaleCalls: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + const { args } = command as unknown as { readonly args: ReadonlyArray }; + tailscaleCalls.push(args); + const stdout = + args[0] === "status" + ? '{"BackendState":"Running","Self":{"DNSName":"host-a.tail.ts.net.","TailscaleIPs":["100.64.0.1"],"CapMap":{"https":null}}}' + : args[0] === "serve" && args[1] === "status" + ? JSON.stringify({ + TCP: { "443": { HTTPS: true } }, + Web: { + "host-a.tail.ts.net:443": { + Handlers: { "/": { Proxy: otherAppOrigin } }, + }, + }, + }) + : ""; + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + // A recorded takeover fails fast instead of waiting out the HTTPS retries. + exitCode: Effect.succeed( + ChildProcessSpawner.ExitCode( + args[0] === "serve" && args[1] !== "status" ? 1 : 0, + ), + ), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.make(new TextEncoder().encode(stdout)), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }); + + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-pair-ts-held-")); + yield* persistServerRuntimeState({ + path: NodePath.join(baseDir, "userdata", "server-runtime.json"), + state: yield* makePersistedServerRuntimeState({ + config: { host: "127.0.0.1", devUrl: undefined }, + port: Number(new URL(origin).port), + }), + }); + + const exit = yield* runCli(["pair", "--base-dir", baseDir, "--tailscale"]).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(FetchHttpClient.Fetch, async (input, init) => { + if (String(input).startsWith(origin)) return fetch(input, init); + throw new TypeError("getaddrinfo ENOTFOUND host-a.tail.ts.net"); + }), + Effect.exit, + ); + + assert.deepEqual( + tailscaleCalls.filter((args) => args[0] === "serve" && args[1] !== "status"), + [], + ); + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) { + assert.instanceOf(Cause.squash(exit.cause), ServePortOccupiedError); + } + }), + ), + ).pipe(Effect.provide(layerCliRuntime)), + ); + it.effect("mints a token and prints a QR pairing URL for a live server", () => withDescriptorServer((origin) => Effect.gen(function* () { diff --git a/apps/server/src/cli/pair.ts b/apps/server/src/cli/pair.ts index e1af3be829ab..078f868ab091 100644 --- a/apps/server/src/cli/pair.ts +++ b/apps/server/src/cli/pair.ts @@ -15,6 +15,7 @@ import { buildTailscaleHttpsBaseUrl, DEFAULT_TAILSCALE_SERVE_PORT, ensureTailscaleServe, + readTailscaleServeRootHolder, readTailscaleStatus, } from "@t3tools/tailscale"; import * as Config from "effect/Config"; @@ -405,6 +406,15 @@ const resolveTailscalePairingBase = Effect.fn("pair.resolveTailscalePairingBase" if (isDevServerNotProxiableError(localTarget)) { return yield* localTarget; } + // The probe above fails whenever this host cannot reach its own MagicDNS name, so the Serve + // config, not the probe, decides whether the port is free. + const holder = yield* readTailscaleServeRootHolder(input.servePort, { + host: (localTarget.localHost ?? "127.0.0.1").replace(/^\[|\]$/gu, ""), + port: localTarget.localPort, + }).pipe(Effect.mapError((cause) => new TailscaleUnavailableError({ cause }))); + if (holder === "other") { + return yield* new ServePortOccupiedError({ servePort: input.servePort }); + } yield* ensureTailscaleServe({ localPort: localTarget.localPort, servePort: input.servePort, diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index f92b3cbaaf63..44e4514799d6 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -218,7 +218,7 @@ import * as RelayClient from "@t3tools/shared/relayClient"; import { disableTailscaleServe, ensureTailscaleServe, - readTailscaleServeRootHolders, + readTailscaleServeRootHolder, readTailscaleServeRoutesTo, readTailscaleStatus, } from "@t3tools/tailscale"; @@ -906,22 +906,8 @@ const layerMakeServer = Layer.unwrap( ); }), ); - // Who holds the Serve port's root: nobody, this server, only dead loopback ports (an unclean - // exit leaves one), or another live server or app, whose route is left alone. - const tailscaleServeRootHolder = Effect.fn("server.tailscaleServeRootHolder")( - function* (target: { readonly host: string; readonly port: number }) { - const net = yield* NetService.NetService; - const holders = yield* readTailscaleServeRootHolders(config.tailscaleServePort, target); - if (holders.length === 0) return "none" as const; - if (holders.some((holder) => holder.ours)) return "ours" as const; - const live = yield* Effect.forEach(holders, (holder) => - holder.ours || holder.loopback === undefined - ? Effect.succeed(true) - : net.hasListenerOnHost(holder.loopback.port, holder.loopback.host), - ); - return live.includes(true) ? ("other" as const) : ("dead" as const); - }, - ); + const tailscaleServeRootHolder = (target: { readonly host: string; readonly port: number }) => + readTailscaleServeRootHolder(config.tailscaleServePort, target); const layerTailscaleServe = config.tailscaleServeEnabled ? Layer.effectDiscard( Effect.acquireRelease( diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index 02e5742b34df..c5e9b28e8d3b 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -1,5 +1,6 @@ import { ExecutionEnvironmentDescriptor } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as NetService from "@t3tools/shared/Net"; import { isLocalLoopbackHost, normalizeHostname } from "@t3tools/shared/hostClassification"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -723,7 +724,7 @@ export const parseTailscaleServeRootHolders = ( return [...tcp, ...web]; }; -export const readTailscaleServeRootHolders = ( +const readTailscaleServeRootHolders = ( servePort: number, target: { readonly host: string; readonly port: number }, ) => @@ -736,6 +737,25 @@ export const readTailscaleServeRootHolders = ( }), ); +/** + * Who holds the root of `servePort`: nobody, the server at `target`, only dead loopback ports (an + * unclean exit leaves one), or another live server or app, whose route must be left alone. + */ +export const readTailscaleServeRootHolder = Effect.fn("tailscale.readTailscaleServeRootHolder")( + function* (servePort: number, target: { readonly host: string; readonly port: number }) { + const net = yield* NetService.NetService; + const holders = yield* readTailscaleServeRootHolders(servePort, target); + if (holders.length === 0) return "none" as const; + if (holders.some((holder) => holder.ours)) return "ours" as const; + const live = yield* Effect.forEach(holders, (holder) => + holder.ours || holder.loopback === undefined + ? Effect.succeed(true) + : net.hasListenerOnHost(holder.loopback.port, holder.loopback.host), + ); + return live.includes(true) ? ("other" as const) : ("dead" as const); + }, +); + export const probeTailscaleHttpsEndpoint = Effect.fn("tailscale.probeTailscaleHttpsEndpoint")( function* (input: { readonly baseUrl: string; readonly timeout?: Duration.Input }) { const client = yield* HttpClient.HttpClient; From a05a3cd584cb8b0c4849cf555589b9f2a420a085 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:25:06 -0800 Subject: [PATCH 059/367] fix(shared): voice cleanup falls back after removing an instance named like an Object key (#35489) --- packages/shared/src/serverSettings.test.ts | 17 +++++++++++++++++ packages/shared/src/serverSettings.ts | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/serverSettings.test.ts b/packages/shared/src/serverSettings.test.ts index 347e6c746648..9836fe10fc77 100644 --- a/packages/shared/src/serverSettings.test.ts +++ b/packages/shared/src/serverSettings.test.ts @@ -377,6 +377,23 @@ describe("serverSettings helpers", () => { expect(settings.sourceControlWriterModelSelection).toBe(sourceControlWriterModelSelection); }); + it("falls back from a removed instance whose id matches an inherited object key", () => { + for (const id of ["constructor", "toString", "valueOf"]) { + const speechPostProcessingModelSelection = createModelSelection( + ProviderInstanceId.make(id), + "gpt-5.4-mini", + ); + const settings = { ...DEFAULT_SERVER_SETTINGS, speechPostProcessingModelSelection }; + + expect(isModelSelectionProviderEnabled(settings, speechPostProcessingModelSelection)).toBe( + false, + ); + expect(resolveSpeechPostProcessingModelSelection(settings)).toBe( + settings.textGenerationModelSelection, + ); + } + }); + it("falls back from a writer provider that cannot generate application text", () => { const instanceId = ProviderInstanceId.make("acp_writer"); const sourceControlWriterModelSelection = createModelSelection(instanceId, "default"); diff --git a/packages/shared/src/serverSettings.ts b/packages/shared/src/serverSettings.ts index 24b8d8e39979..023c6eb74333 100644 --- a/packages/shared/src/serverSettings.ts +++ b/packages/shared/src/serverSettings.ts @@ -51,7 +51,9 @@ export function isModelSelectionProviderEnabled( ): boolean { const generator = textGeneratorForSelection(settings, selection.instanceId); if (generator !== undefined) return generator.enabled; - const instanceConfig = settings.providerInstances[selection.instanceId]; + const instanceConfig = Object.hasOwn(settings.providerInstances, selection.instanceId) + ? settings.providerInstances[selection.instanceId] + : undefined; if (instanceConfig !== undefined) { return resolveProviderInstanceEnabled(instanceConfig); } From 7c57291c4b642e12b8e0b330fbb005c73fe6ea8a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:25:19 -0800 Subject: [PATCH 060/367] fix(server): a bind address Tailscale Serve refuses says to bind 127.0.0.1 or 0.0.0.0 (#35545) --- apps/server/src/cli/pair.test.ts | 58 +++++++++++++++++++++++++++++ apps/server/src/cli/pair.ts | 17 ++++++++- apps/server/src/server.ts | 12 +++--- packages/tailscale/src/tailscale.ts | 3 ++ scripts/lib/dev-share.ts | 1 + 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/apps/server/src/cli/pair.test.ts b/apps/server/src/cli/pair.test.ts index 7f5ed1113cea..4f4004b37129 100644 --- a/apps/server/src/cli/pair.test.ts +++ b/apps/server/src/cli/pair.test.ts @@ -553,6 +553,64 @@ describe("t3 pair", () => { }), ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); + + // regression: #35517 - the macOS Tailscale apps refuse Serve targets other than + // localhost, 127.0.0.1 and ::1, so a specific bind needs an explanation. + it.effect("names the bind address when Tailscale refuses it as a Serve target", () => + withDescriptorServer( + (origin) => + Effect.gen(function* () { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-pair-remote-")); + yield* persistServerRuntimeState({ + path: NodePath.join(baseDir, "userdata", "server-runtime.json"), + state: yield* makePersistedServerRuntimeState({ + config: { host: "127.0.0.2", devUrl: undefined }, + port: Number(new URL(origin).port), + }), + }); + const spawner = ChildProcessSpawner.make((command) => { + const { args } = command as unknown as { readonly args: ReadonlyArray }; + const serve = args[0] === "serve"; + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(serve ? 1 : 0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.make( + new TextEncoder().encode( + serve ? "" : '{"Self":{"DNSName":"pair-test.tail.ts.net."}}', + ), + ), + stderr: Stream.make( + new TextEncoder().encode( + serve + ? "Warning: the MacOS extension doesn't support connecting to remote destinations from non-default route\n" + : "", + ), + ), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }); + + const error = yield* provideCliTestLayers( + runCli(["pair", "--base-dir", baseDir, "--tailscale"]).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HostProcess.Platform, "darwin"), + Effect.flip, + ), + ); + + assert.include(String(error), "127.0.0.2"); + }), + "127.0.0.2", + ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); }); describe("auth scope options", () => { diff --git a/apps/server/src/cli/pair.ts b/apps/server/src/cli/pair.ts index 078f868ab091..71f380eb718b 100644 --- a/apps/server/src/cli/pair.ts +++ b/apps/server/src/cli/pair.ts @@ -139,6 +139,15 @@ export class TailscaleServeFailedError extends Schema.TaggedError()( + "TailscaleServeTargetRefusedError", + { localHost: Schema.String }, +) { + override get message(): string { + return `This Tailscale install can only proxy to localhost, and the server listens on ${this.localHost}. Restart it with \`--host 127.0.0.1\` or \`--host 0.0.0.0\`, then run this command again.`; + } +} + export class ServePortOccupiedError extends Schema.TaggedError()( "ServePortOccupiedError", { servePort: Schema.Number }, @@ -420,8 +429,12 @@ const resolveTailscalePairingBase = Effect.fn("pair.resolveTailscalePairingBase" servePort: input.servePort, ...(localTarget.localHost !== undefined ? { localHost: localTarget.localHost } : {}), }).pipe( - Effect.mapError( - (cause) => new TailscaleServeFailedError({ servePort: input.servePort, cause }), + Effect.mapError((cause) => + cause._tag === "TailscaleCommandExitError" && + cause.stderrDiagnostic === "unsupported-target" && + localTarget.localHost !== undefined + ? new TailscaleServeTargetRefusedError({ localHost: localTarget.localHost }) + : new TailscaleServeFailedError({ servePort: input.servePort, cause }), ), ); notes.push( diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 44e4514799d6..c15045e4ccf5 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -954,11 +954,13 @@ const layerMakeServer = Layer.unwrap( }), ), Effect.catch((cause) => - Effect.logWarning("Failed to configure Tailscale Serve", { - cause, - localPort, - servePort: config.tailscaleServePort, - }).pipe(Effect.as(null)), + Effect.logWarning( + cause._tag === "TailscaleCommandExitError" && + cause.stderrDiagnostic === "unsupported-target" + ? `Tailscale Serve left off; this Tailscale install can only proxy to localhost, not ${localHost}. Bind --host 127.0.0.1 or 0.0.0.0 instead` + : "Failed to configure Tailscale Serve", + { cause, localPort, servePort: config.tailscaleServePort }, + ).pipe(Effect.as(null)), ), ); }), diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index c5e9b28e8d3b..a47e84d5d38b 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -47,6 +47,7 @@ export const TailscaleStderrDiagnostic = Schema.Literals([ "not-logged-in", "not-running", "permission-denied", + "unsupported-target", "unknown", ]); export type TailscaleStderrDiagnostic = typeof TailscaleStderrDiagnostic.Type; @@ -63,6 +64,8 @@ const STDERR_DIAGNOSTIC_PATTERNS: ReadonlyArray< "not-running", ], [/permission denied|access denied|must be root|operation not permitted/i, "permission-denied"], + // The macOS apps, and tailscaled without SO_MARK, proxy only to localhost, 127.0.0.1 and ::1. + [/doesn't support connecting to remote destinations/i, "unsupported-target"], ]; const stderrDiagnosticOf = (stderr: string): TailscaleStderrDiagnostic | undefined => { diff --git a/scripts/lib/dev-share.ts b/scripts/lib/dev-share.ts index 424fc6c86555..0c42fe9f3116 100644 --- a/scripts/lib/dev-share.ts +++ b/scripts/lib/dev-share.ts @@ -19,6 +19,7 @@ const DIAGNOSTIC_EXPLANATIONS: Record Date: Thu, 12 Nov 2026 11:25:33 -0800 Subject: [PATCH 061/367] fix(desktop): Tailscale HTTPS counts as on only when Serve reaches this backend (#35544) --- .../backend/tailscaleEndpointProvider.test.ts | 39 ++++++++++++++++++- .../src/backend/tailscaleEndpointProvider.ts | 24 +++++++++--- .../src/environment/DirectEndpoints.test.ts | 8 ++++ .../server/src/environment/DirectEndpoints.ts | 4 +- apps/server/src/server.ts | 2 +- packages/tailscale/src/tailscale.test.ts | 29 ++++++++++++++ packages/tailscale/src/tailscale.ts | 10 +++-- 7 files changed, 104 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts b/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts index 054655917cec..397a75770483 100644 --- a/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts +++ b/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts @@ -1,7 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { HttpClient } from "effect/http"; +import { HttpClient, HttpClientResponse } from "effect/http"; import { ChildProcessSpawner } from "effect/process"; import { resolveTailscaleAdvertisedEndpoints } from "./tailscaleEndpointProvider.ts"; @@ -79,4 +79,41 @@ describe("tailscale endpoint provider", () => { ); }).pipe(Effect.provide(layerUnusedTailscaleExternalServices)), ); + + it.effect("offers Tailscale HTTPS only when Serve reaches this desktop's backend", () => + Effect.gen(function* () { + const magicDnsStatus = (servedEnvironmentId: string) => + resolveTailscaleAdvertisedEndpoints({ + port: 3773, + serveEnabled: true, + networkInterfaces: {}, + statusJson: `{"Self":{"DNSName":"desktop.example.ts.net.","TailscaleIPs":[]}}`, + }).pipe( + Effect.map((endpoints) => endpoints.map((endpoint) => endpoint.status)), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + environmentId: + new URL(request.url).hostname === "127.0.0.1" + ? "desktop-backend" + : servedEnvironmentId, + label: "T3 server", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.1.9", + capabilities: { repositoryIdentity: true }, + }), + ), + ), + ), + ), + ); + + assert.deepEqual(yield* magicDnsStatus("desktop-backend"), ["available"]); + assert.deepEqual(yield* magicDnsStatus("another-server"), ["unavailable"]); + }).pipe(Effect.provide(layerUnusedTailscaleExternalServices)), + ); }); diff --git a/apps/desktop/src/backend/tailscaleEndpointProvider.ts b/apps/desktop/src/backend/tailscaleEndpointProvider.ts index 4b75afccdb5b..1b8d9dff4ed4 100644 --- a/apps/desktop/src/backend/tailscaleEndpointProvider.ts +++ b/apps/desktop/src/backend/tailscaleEndpointProvider.ts @@ -1,5 +1,9 @@ import { createAdvertisedEndpoint } from "@t3tools/shared/advertisedEndpoint"; -import type { AdvertisedEndpoint, AdvertisedEndpointProvider } from "@t3tools/contracts"; +import { + ExecutionEnvironmentDescriptor, + type AdvertisedEndpoint, + type AdvertisedEndpointProvider, +} from "@t3tools/contracts"; import { buildTailscaleHttpsBaseUrl, isTailscaleIpv4Address, @@ -11,6 +15,7 @@ import { import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import type { NetworkInterfaces } from "./DesktopNetworkInterfaces.ts"; @@ -71,10 +76,20 @@ function resolveTailscaleIpAdvertisedEndpoints(input: { return endpoints; } +// The backend may run elsewhere (WSL), so ask it for its id instead of reading the state dir. +const probeServesThisBackend = (localHttpUrl: string) => (baseUrl: string) => + HttpClient.get(new URL("/.well-known/t3/environment", localHttpUrl)).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)), + Effect.flatMap(({ environmentId }) => probeTailscaleHttpsEndpoint({ baseUrl, environmentId })), + Effect.orElseSucceed(() => false), + ); + const resolveTailscaleMagicDnsAdvertisedEndpoint = Effect.fn( "resolveTailscaleMagicDnsAdvertisedEndpoint", )(function* (input: { readonly dnsName: string | null; + readonly localHttpUrl: string; readonly serveEnabled: boolean; readonly servePort?: number; readonly probe?: (baseUrl: string) => Effect.Effect; @@ -87,11 +102,7 @@ const resolveTailscaleMagicDnsAdvertisedEndpoint = Effect.fn( magicDnsName: input.dnsName, ...(input.servePort === undefined ? {} : { servePort: input.servePort }), }); - const probe = - input.probe?.(httpBaseUrl) ?? - probeTailscaleHttpsEndpoint({ - baseUrl: httpBaseUrl, - }); + const probe = (input.probe ?? probeServesThisBackend(input.localHttpUrl))(httpBaseUrl); const isReachable = input.serveEnabled ? yield* probe : false; return Option.some( @@ -140,6 +151,7 @@ export const resolveTailscaleAdvertisedEndpoints = Effect.fn("resolveTailscaleAd const ipEndpoints = resolveTailscaleIpAdvertisedEndpoints({ ...input, status }); const magicDnsEndpoint = yield* resolveTailscaleMagicDnsAdvertisedEndpoint({ dnsName: status?.magicDnsName ?? null, + localHttpUrl: `http://127.0.0.1:${input.port}`, serveEnabled: input.serveEnabled === true, ...(input.servePort === undefined ? {} : { servePort: input.servePort }), ...(input.probe === undefined ? {} : { probe: input.probe }), diff --git a/apps/server/src/environment/DirectEndpoints.test.ts b/apps/server/src/environment/DirectEndpoints.test.ts index 2b022cd069d2..2580aa7f6780 100644 --- a/apps/server/src/environment/DirectEndpoints.test.ts +++ b/apps/server/src/environment/DirectEndpoints.test.ts @@ -1,4 +1,5 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; +import { EnvironmentId } from "@t3tools/contracts"; import { it as effectIt } from "@effect/vitest"; import type * as NodeOS from "node:os"; import * as Effect from "effect/Effect"; @@ -16,6 +17,7 @@ import { describe, expect, it } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import * as DirectEndpoints from "./DirectEndpoints.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; import { resolveBoundEndpoints } from "./DirectEndpoints.ts"; const INTERFACES: ReturnType = { @@ -284,6 +286,12 @@ const layerServeConfig = (host = "127.0.0.1") => ).pipe( Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-direct-" })), Layer.provide(NodeServices.layer), + Layer.merge( + Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed(EnvironmentId.make("serve-test")), + getDescriptor: Effect.die("unused"), + }), + ), ); const resolveWithServe = (probeStatus: number) => diff --git a/apps/server/src/environment/DirectEndpoints.ts b/apps/server/src/environment/DirectEndpoints.ts index 5b9ecc98299c..1f7d9f705620 100644 --- a/apps/server/src/environment/DirectEndpoints.ts +++ b/apps/server/src/environment/DirectEndpoints.ts @@ -23,6 +23,7 @@ import { } from "@t3tools/shared/hostClassification"; import * as ServerConfig from "../config.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; import { isLoopbackHost, isPrivateNetworkAddress, VIRTUAL_INTERFACE } from "../startupAccess.ts"; type NetworkInterfacesMap = ReturnType; @@ -120,6 +121,7 @@ export const make = Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const httpClient = yield* HttpClient.HttpClient; const platform = yield* HostProcess.Platform; + const environmentId = yield* (yield* ServerEnvironment.ServerEnvironment).getEnvironmentId; // Only the service default listens on every interface without being asked, // so only it checks once whether it runs in a cloud. The check starts now so // the first client does not wait for its timeout. @@ -155,7 +157,7 @@ export const make = Effect.gen(function* () { magicDnsName, servePort: config.tailscaleServePort, }); - return probeTailscaleHttpsEndpoint({ baseUrl: httpBaseUrl }).pipe( + return probeTailscaleHttpsEndpoint({ baseUrl: httpBaseUrl, environmentId }).pipe( Effect.map((reachable) => (reachable ? httpBaseUrl : null)), ); }), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c15045e4ccf5..959cad396ac2 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -773,7 +773,7 @@ const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( Layer.provideMerge(AnalyticsService.layer.pipe(Layer.provide(ManagedPolicy.layer))), Layer.provideMerge(ExternalLauncher.layer), Layer.provideMerge(RemoteOpenTargets.layer), - Layer.provideMerge(DirectEndpoints.layer), + Layer.provideMerge(DirectEndpoints.layer.pipe(Layer.provide(layerServerEnvironment))), Layer.provideMerge(ServerLifecycleEvents.layer), Layer.provide(NetService.layer), ); diff --git a/packages/tailscale/src/tailscale.test.ts b/packages/tailscale/src/tailscale.test.ts index bcf4ffec59a6..095323e037ac 100644 --- a/packages/tailscale/src/tailscale.test.ts +++ b/packages/tailscale/src/tailscale.test.ts @@ -110,6 +110,7 @@ describe("tailscale", () => { ] as const) { const reachable = yield* probeTailscaleHttpsEndpoint({ baseUrl: "https://desktop.example.ts.net", + environmentId: "this-environment", }).pipe( Effect.provideService( HttpClient.HttpClient, @@ -123,6 +124,34 @@ describe("tailscale", () => { }), ); + it.effect("does not treat another T3 server behind the Serve URL as this server", () => + Effect.gen(function* () { + const answer = (environmentId: string) => + HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + environmentId, + label: "T3 server", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.1.9", + capabilities: { repositoryIdentity: true }, + }), + ), + ), + ); + const probe = (served: string) => + probeTailscaleHttpsEndpoint({ + baseUrl: "https://desktop.example.ts.net", + environmentId: "this-environment", + }).pipe(Effect.provideService(HttpClient.HttpClient, answer(served))); + + assert.isTrue(yield* probe("this-environment")); + assert.isFalse(yield* probe("another-environment")); + }), + ); + it("finds the Serve routes that forward to the bound address, and only those", () => { const status = JSON.stringify({ TCP: { diff --git a/packages/tailscale/src/tailscale.ts b/packages/tailscale/src/tailscale.ts index a47e84d5d38b..0456f95ab485 100644 --- a/packages/tailscale/src/tailscale.ts +++ b/packages/tailscale/src/tailscale.ts @@ -760,18 +760,22 @@ export const readTailscaleServeRootHolder = Effect.fn("tailscale.readTailscaleSe ); export const probeTailscaleHttpsEndpoint = Effect.fn("tailscale.probeTailscaleHttpsEndpoint")( - function* (input: { readonly baseUrl: string; readonly timeout?: Duration.Input }) { + function* (input: { + readonly baseUrl: string; + readonly environmentId: string; + readonly timeout?: Duration.Input; + }) { const client = yield* HttpClient.HttpClient; const descriptor = yield* Effect.gen(function* () { const url = new URL("/.well-known/t3/environment", input.baseUrl); const response = yield* client .execute(HttpClientRequest.get(url.toString())) .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); - // A Serve mapping to another app also answers 2xx; only a T3 descriptor proves this server. + // Another app or another T3 server can hold the Serve port; only our own descriptor counts. return yield* HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)(response); }).pipe(Effect.timeoutOption(input.timeout ?? TAILSCALE_PROBE_TIMEOUT)); - return Option.isSome(descriptor); + return Option.isSome(descriptor) && descriptor.value.environmentId === input.environmentId; }, Effect.orElseSucceed(() => false), ); From 4ca5b9cdb8f5f8a0e463faba0d5c4b4f6bb322c0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:25:46 -0800 Subject: [PATCH 062/367] fix(server): long startup migrations log progress and resume after a restart (#35486) --- .../server/src/persistence/Migrations.test.ts | 16 +++++++ apps/server/src/persistence/Migrations.ts | 34 +++++++++++--- apps/server/src/persistence/Sqlite.test.ts | 39 ++++++++++++++++ apps/server/src/persistence/Sqlite.ts | 30 +++++++++++-- packages/shared/src/nodeSqliteClient.ts | 45 +++++++++++++++++-- 5 files changed, 152 insertions(+), 12 deletions(-) diff --git a/apps/server/src/persistence/Migrations.test.ts b/apps/server/src/persistence/Migrations.test.ts index 4ab9c86aa10b..64705e4bbb8f 100644 --- a/apps/server/src/persistence/Migrations.test.ts +++ b/apps/server/src/persistence/Migrations.test.ts @@ -151,6 +151,22 @@ describe("runMigrations", () => { }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); + it.effect("keeps the migrations that finished before a later one failed", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 81 }); + yield* sql`CREATE TABLE orchestration_v2_projection_turn_item_counts (thread_id TEXT)`; + + const exit = yield* Effect.exit(runMigrations()); + + assert.isTrue(Exit.isFailure(exit)); + assert.deepStrictEqual( + yield* history, + migrationManifest.filter(([id]) => id <= 84), + ); + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), + ); + it.effect("reapplies peer link migrations whose ledger rows are missing", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 37b74d5c6db0..8b33ba60f8e9 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -268,16 +268,40 @@ export const runMigrations = Effect.fn("runMigrations")(function* ({ if (includesV2) { yield* assertV2MigrationsCanRun(toMigrationInclusive); } - const executedMigrations = [ - ...previewMigrations, - ...(yield* run({ loader: makeMigrationLoader(toMigrationInclusive) })), - ]; + const sql = yield* SqlClient.SqlClient; + const ledger = yield* sql` + SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'effect_sql_migrations' + `; + const [latest] = + ledger.length === 0 + ? [] + : yield* sql<{ + readonly id: number | null; + }>`SELECT MAX(migration_id) AS id FROM effect_sql_migrations`; + const latestRecordedId = latest?.id ?? 0; + const pending = migrationEntries.filter( + ([id]) => + id > latestRecordedId && (toMigrationInclusive === undefined || id <= toMigrationInclusive), + ); + // One transaction per migration: an index build on a large database can take minutes, and a + // restart then resumes after the last finished migration instead of rolling all of them back. + const executedMigrations: Array = [...previewMigrations]; + for (const [index, [id, name]] of pending.entries()) { + if (latestRecordedId > 0) { + yield* Effect.log("Upgrading database").pipe( + Effect.annotateLogs({ + migration: `${id}_${name}`, + progress: `${index + 1} of ${pending.length}`, + }), + ); + } + executedMigrations.push(...(yield* run({ loader: makeMigrationLoader(id) }))); + } const migrations = executedMigrations.map(([id, name]) => `${id}_${name}`); yield* migrations.length === 0 ? Effect.logDebug("Database schema is current") : Effect.log("Migrations ran successfully").pipe(Effect.annotateLogs({ migrations })); - const sql = yield* SqlClient.SqlClient; const recorded = yield* sql<{ readonly migration_id: number; readonly name: string; diff --git a/apps/server/src/persistence/Sqlite.test.ts b/apps/server/src/persistence/Sqlite.test.ts index c042ea9afa7a..8609649c2c1a 100644 --- a/apps/server/src/persistence/Sqlite.test.ts +++ b/apps/server/src/persistence/Sqlite.test.ts @@ -24,6 +24,7 @@ import { migrationManifest, runMigrations } from "./Migrations.ts"; import * as SqlitePersistence from "./Sqlite.ts"; const workers = vi.hoisted((): Array => []); +const workerData = vi.hoisted((): Array => []); vi.mock("node:worker_threads", async (importOriginal) => { const actual = await importOriginal(); @@ -31,6 +32,7 @@ vi.mock("node:worker_threads", async (importOriginal) => { constructor(...args: ConstructorParameters) { super(...args); workers.push(this); + workerData.push(args[1]?.workerData); } } return { ...actual, Worker: RecordedWorker }; @@ -192,6 +194,43 @@ it.effect("drops planner statistics a database already has", () => { ); }); +it.effect("upgrades an existing database on a read-write worker connection", () => { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-upgrade-")); + const dbPath = NodePath.join(tempDir, "state.sqlite"); + + return Effect.gen(function* () { + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`PRAGMA journal_mode = WAL;`; + yield* runMigrations({ toMigrationInclusive: 81 }); + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: dbPath }))); + const startedBefore = workerData.length; + + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly migration_id: number; readonly name: string }>` + SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id + `; + assert.deepStrictEqual( + rows.map((row) => [row.migration_id, row.name] as const), + migrationManifest, + ); + }).pipe( + Effect.provide( + SqlitePersistence.layerFromPath(dbPath).pipe(Layer.provide(NodeServices.layer)), + ), + ); + assert.deepInclude(workerData.slice(startedBefore), { + filename: dbPath, + readOnly: false, + allowExtension: false, + prepareCacheSize: 200, + }); + }).pipe( + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))), + ); +}); + it.effect("names a database file that is not SQLite", () => { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-notadb-")); const dbPath = NodePath.join(tempDir, "state.sqlite"); diff --git a/apps/server/src/persistence/Sqlite.ts b/apps/server/src/persistence/Sqlite.ts index e8471f3f5989..273c2b3c6dc5 100644 --- a/apps/server/src/persistence/Sqlite.ts +++ b/apps/server/src/persistence/Sqlite.ts @@ -64,6 +64,26 @@ const runMigrationsAsOwner = Effect.fn("SqlitePersistence.runMigrationsAsOwner") yield* runMigrations(); }, Effect.scoped); +// Index builds over large tables take minutes; on the main connection they block the event loop, +// so requests hang instead of getting the listener's 503. A fresh database stays on the main +// connection, which must create the first table for its auto_vacuum setting to apply. +const runMigrationsOffMainThread = Effect.fn("SqlitePersistence.runMigrationsOffMainThread")( + function* (dbPath: string) { + const sql = yield* SqlClient.SqlClient; + const tables = yield* sql<{ + readonly count: number; + }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`; + if (!tables[0]?.count || !(yield* hasPendingMigrations())) return yield* runMigrations(); + return yield* Effect.gen(function* () { + const worker = yield* SqlClient.SqlClient; + yield* worker`PRAGMA foreign_keys = ON;`; + yield* worker`PRAGMA synchronous = NORMAL;`; + yield* worker`PRAGMA temp_store = MEMORY;`; + return yield* runMigrations(); + }).pipe(Effect.provide(NodeSqliteClient.layerWorker(dbPath))); + }, +); + const makeLayerSetup = (migrate: Effect.Effect) => Layer.effectDiscard( Effect.gen(function* () { @@ -100,14 +120,14 @@ const makeLayerSetup = (migrate: Effect.Effect) => const layerSetup = makeLayerSetup(runMigrations()); -const makeLayerFromPath = (setup: Layer.Layer) => +const makeLayerFromPath = (makeSetup: (dbPath: string) => Layer.Layer) => Effect.fn("makeSqlitePersistenceLive")(function* (dbPath: string) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); return Layer.provideMerge( - setup, + makeSetup(dbPath), NodeSqliteClient.layer({ filename: dbPath, readerWorker: true, @@ -132,7 +152,9 @@ const makeLayerFromPath = (setup: Layer.Layer) => ); }, Layer.unwrap); -export const layerFromPath = makeLayerFromPath(layerSetup); +export const layerFromPath = makeLayerFromPath((dbPath) => + makeLayerSetup(runMigrationsOffMainThread(dbPath)), +); export const layerMemory = Layer.provideMerge( layerSetup, @@ -154,5 +176,5 @@ export const layerConfig = makeLayerConfig(layerFromPath); /** For CLI commands: pending migrations run only while no server owns the T3 home. */ export const layerCliConfig = makeLayerConfig( - makeLayerFromPath(makeLayerSetup(runMigrationsAsOwner())), + makeLayerFromPath(() => makeLayerSetup(runMigrationsAsOwner())), ); diff --git a/packages/shared/src/nodeSqliteClient.ts b/packages/shared/src/nodeSqliteClient.ts index 251c795c30a9..43098670784f 100644 --- a/packages/shared/src/nodeSqliteClient.ts +++ b/packages/shared/src/nodeSqliteClient.ts @@ -201,7 +201,7 @@ let db; try { // A read-only open cannot be undone by SQL, unlike PRAGMA query_only. const next = new DatabaseSync(workerData.filename, { - readOnly: true, + readOnly: workerData.readOnly, allowExtension: workerData.allowExtension, }); next.exec("PRAGMA busy_timeout = 5000;"); @@ -300,6 +300,7 @@ const READER_POOL_SIZE = 4; const makeReader = Effect.fnUntraced(function* ( options: SqliteClientConfig, pool: { unavailable: boolean }, + readOnly = true, ) { let current: | { @@ -315,6 +316,7 @@ const makeReader = Effect.fnUntraced(function* ( eval: true, workerData: { filename: options.filename, + readOnly, allowExtension: false, prepareCacheSize: 200, }, @@ -358,7 +360,10 @@ const makeReader = Effect.fnUntraced(function* ( const request = (input: ReaderRequest) => Effect.withFiber, SqlError>((fiber) => { - const endsTransaction = endsReadTransaction(input.sql); + // A COMMIT that never reached a writer must fail; only its ROLLBACK is moot. + const endsTransaction = readOnly + ? endsReadTransaction(input.sql) + : /^\s*ROLLBACK\b/i.test(input.sql); if (stoppedMessage !== undefined) { return endsTransaction ? Effect.succeed([]) @@ -411,7 +416,7 @@ const makeReader = Effect.fnUntraced(function* ( }, executeUnprepared(sql, params, rowTransform) { // Read snapshots need deferred BEGIN; readonly connections reject IMMEDIATE and EXCLUSIVE. - const statement = /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql; + const statement = readOnly && /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql; const effect = request({ sql: statement, params: params ?? [], @@ -1288,5 +1293,39 @@ const make = Effect.fn("makeWithDatabase")(function* ( }); }); +const makeWorker = Effect.fn("makeWorkerClient")(function* (filename: string) { + yield* checkNodeSqliteCompat(); + const acquire = yield* makeReader({ filename }, { unavailable: false }, false); + const connection = yield* acquire; + if (connection === undefined) { + return yield* failed( + new Error("database worker is unavailable"), + readerFailureMessages.open, + "open", + ); + } + const { acquirer, borrower, onCommitFailure, transactionAcquirer } = Client.makeSqliteAcquirers({ + connection: Effect.succeed(connection), + semaphore: yield* Semaphore.make(1), + }); + return yield* Client.make({ + driver: "node:sqlite", + acquirer, + borrower, + compiler: Statement.makeCompilerSqlite(), + transactionAcquirer, + onCommitFailure, + beginTransaction: "BEGIN IMMEDIATE", + spanAttributes: [[ATTR_DB_SYSTEM_NAME, "sqlite"]], + }); +}); + +/** + * A read-write client whose one connection runs in a worker thread, so long statements such as + * index builds leave the event loop free. Requires a file database already in WAL mode. + */ +export const layerWorker = (filename: string): Layer.Layer => + Layer.effect(Client.SqlClient, makeWorker(filename)).pipe(Layer.provide(Reactivity.layer)); + export const layer = (config: SqliteClientConfig): Layer.Layer => Layer.effect(Client.SqlClient, make(config)).pipe(Layer.provide(Reactivity.layer)); From 8b05078459aaa55e95075b93abd8aa00afd2b7b1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:25:58 -0800 Subject: [PATCH 063/367] fix(scripts): mobile:e2e asks Tailscale for the address pool devices reach (#35553) --- scripts/mobile-e2e.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/mobile-e2e.ts b/scripts/mobile-e2e.ts index abeb484e158d..4b6121378a35 100644 --- a/scripts/mobile-e2e.ts +++ b/scripts/mobile-e2e.ts @@ -32,14 +32,17 @@ const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`); -/** This machine's Tailscale address (100.64.0.0/10), which pool devices reach it on. */ -function tailnetAddress(): string { - for (const addresses of Object.values(NodeOS.networkInterfaces())) { - for (const address of addresses ?? []) { - const [first, second] = address.address.split(".").map(Number); - if (address.family === "IPv4" && first === 100 && second! >= 64 && second! < 128) - return address.address; - } +/** This machine's Tailscale address, which pool devices reach it on. */ +async function tailnetAddress(): Promise { + // Other VPNs share 100.64.0.0/10, so ask Tailscale instead of matching interface ranges. + // The macOS app bundles its CLI without putting a launcher on PATH. + for (const cli of ["tailscale", "/Applications/Tailscale.app/Contents/MacOS/Tailscale"]) { + const stdout = await run(cli, ["ip", "-4"]).then( + (result) => result.stdout, + () => "", + ); + const address = stdout.trim().split("\n")[0]; + if (address) return address; } throw new Error("No Tailscale IPv4 address: pool devices reach Metro and the fixture over it."); } @@ -173,7 +176,7 @@ async function main() { throw new Error( "Set SIMLOCK_AGENT_DEVICE_URL (/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.", ); - const host = tailnetAddress(); + const host = await tailnetAddress(); const runId = new Date().toISOString().replaceAll(/[:.]/g, "-"); const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-e2e", runId); From c846833a6baa1cfab8e83a7d74d183ca92ad6ba6 Mon Sep 17 00:00:00 2001 From: Logan Date: Thu, 12 Nov 2026 13:26:11 -0600 Subject: [PATCH 064/367] fix(server): provider instances named constructor, toString or valueOf can be added (#35492) Co-authored-by: ixmxvii-hash <235766220+ixmxvii-hash@users.noreply.github.com> Co-authored-by: Julius Marminge --- apps/server/src/serverSettings.test.ts | 11 +++++++++++ apps/server/src/serverSettings.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 347b43dd6f44..e733a7ab8cf8 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -539,6 +539,17 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" (yield* serverSettings.getSettings).providerInstances[instanceId]?.displayName ?? "", ), ); + + // An inherited Object.prototype member is not an instance that already exists. + for (const name of ["constructor", "toString", "valueOf"]) { + const prototypeNamedId = ProviderInstanceId.make(name); + const created = yield* serverSettings.updateProviderInstance({ + operation: "create", + instanceId: prototypeNamedId, + instance: { driver: ProviderDriverKind.make("codex") }, + }); + assert.isTrue(Object.hasOwn(created.providerInstances, prototypeNamedId)); + } }).pipe(Effect.provide(layerServerSettings())), ); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 251becba12f6..4dc30d29a6cf 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -237,7 +237,7 @@ function ensureProviderInstanceMutationAllowed( ): Effect.Effect { if ( mutation.operation === "create" && - settings.providerInstances[mutation.instanceId] !== undefined + Object.hasOwn(settings.providerInstances, mutation.instanceId) ) { return Effect.fail( new ServerSettingsError({ From 78ee8a156dbaafa8c7f931f6e34c1211d795be31 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:28:35 -0800 Subject: [PATCH 065/367] fix(mobile): inline-code file paths with non-ASCII names become file pills (#35552) --- apps/mobile/src/lib/enrichedLinkPresentation.ts | 2 +- apps/mobile/src/lib/markdownInlineFileLinks.test.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/mobile/src/lib/enrichedLinkPresentation.ts b/apps/mobile/src/lib/enrichedLinkPresentation.ts index 859c015c0533..5d2e9431339d 100644 --- a/apps/mobile/src/lib/enrichedLinkPresentation.ts +++ b/apps/mobile/src/lib/enrichedLinkPresentation.ts @@ -92,7 +92,7 @@ export function enrichedLinkVariantPattern(url: string) { } export const ENRICHED_INLINE_FILE_LINK_REGEX = - /^(?!\S*:\/\/)(?:(?:\.{1,2}[\\/]|~[\\/]|\/(?:Users|home|tmp|var|etc|opt|mnt|Volumes|private|root|usr|bin|sbin|lib|lib64|srv|dev|proc|sys|run|boot|media|workspace|workspaces)\/|[A-Za-z]:[\\/])[^\s`]+|(?:[A-Za-z0-9._-]+[\\/])+[A-Za-z0-9._-]+\.[A-Za-z0-9_-]+(?::\d+(?::\d+|-\d+)?)?|[A-Za-z0-9._-]+\.[A-Za-z0-9_-]+:\d+(?::\d+|-\d+)?)$/; + /^(?!\S*:\/\/)(?:(?:\.{1,2}[\\/]|~[\\/]|\/(?:Users|home|tmp|var|etc|opt|mnt|Volumes|private|root|usr|bin|sbin|lib|lib64|srv|dev|proc|sys|run|boot|media|workspace|workspaces)\/|[A-Za-z]:[\\/])[^\s`]+|(?:[\p{L}\p{M}\p{N}._-]+[\\/])+[\p{L}\p{M}\p{N}._-]+\.[A-Za-z0-9_-]+(?::\d+(?::\d+|-\d+)?)?|[\p{L}\p{M}\p{N}._-]+\.[A-Za-z0-9_-]+:\d+(?::\d+|-\d+)?)$/u; export function enrichedSkillLinkRegex(skills?: ReadonlyArray) { const names = [...new Set(skills?.map((skill) => skill.name) ?? [])].filter(Boolean); diff --git a/apps/mobile/src/lib/markdownInlineFileLinks.test.ts b/apps/mobile/src/lib/markdownInlineFileLinks.test.ts index 5cce10e3c3e3..e2d81cf86956 100644 --- a/apps/mobile/src/lib/markdownInlineFileLinks.test.ts +++ b/apps/mobile/src/lib/markdownInlineFileLinks.test.ts @@ -93,6 +93,9 @@ describe("inline code that names a file", () => { expect(rewrite("See `src/math.ts:2` and `./README.md`.")).toBe( "See [src/math.ts:2]() and [./README.md](<./README.md>).", ); + for (const path of ["src/數學.ts:2", "數學/math.ts:2", "數學.ts:2", "src/數學.ts:2-4"]) { + expect(rewrite(`See \`${path}\`.`), path).toBe(`See [${path}](<${path}>).`); + } }); it("keeps commands, identifiers and URLs as code", () => { From 1001bc71fcc80ba8dc32c60a0ae4bbe85efea44b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 11:35:05 -0800 Subject: [PATCH 066/367] fix(voice): dictation survives draft changes and Escape in the citation box (#32682) Co-authored-by: Claude Opus 5.5 --- .../SettingsScheduledTasksRouteScreen.tsx | 4 - .../src/features/threads/ThreadComposer.tsx | 3 - .../voice-input/lazyVoiceRecorder.test.ts | 1 - .../voice-input/useVoiceInputController.ts | 2 - .../voice-input/voiceInputSession.test.ts | 94 ++++++------ .../features/voice-input/voiceInputSession.ts | 32 ++--- ...ditorTiptap.citationDraft.browser.test.tsx | 27 +++- .../components/ComposerPromptEditorTiptap.tsx | 6 +- apps/web/src/components/chat/ChatComposer.tsx | 1 - .../preview/PreviewAnnotationSpeech.tsx | 6 +- .../settings/VoiceSettingsPanel.tsx | 1 - .../useDictationShortcut.browser.test.tsx | 134 +++++++++++++++++- apps/web/src/speech/useDictationShortcut.ts | 36 ++++- ...useEnvironmentSpeechInput.browser.test.tsx | 1 - .../src/speech/useEnvironmentSpeechInput.ts | 29 +--- apps/web/src/speech/useTextFieldSpeech.tsx | 1 - docs/user/voice-input.md | 2 + .../src/voice-input/controller.test.ts | 72 ++++++++-- .../src/voice-input/controller.ts | 57 +++++--- 19 files changed, 350 insertions(+), 159 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index f4604ca4a2f6..3f552c1f0c46 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -87,9 +87,7 @@ import { useScheduledTaskDestinationLabel, useScheduledTaskEditor } from "./sche import { discardScheduledTaskEditor, markScheduledTaskEditorSaved, - scheduledTaskEditorSessionAtom, } from "./scheduled-task-editor-state"; -import { appAtomRegistry } from "../../state/atom-registry"; import { formatNextScheduledTaskRun, formatScheduledTaskInterval, @@ -519,8 +517,6 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } const voiceInput = useVoiceInputController({ ownerKey, label: editor?.draft.title.trim() || title, - subscribeToDraftChanges: (onChange) => - appAtomRegistry.subscribe(scheduledTaskEditorSessionAtom, onChange), selection, readDraftMessage: () => { const current = readEditor(); diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 9c2eb6d4c38d..3b1b927ef8a9 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -57,13 +57,11 @@ import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/re import { scopedThreadKey } from "../../lib/scopedEntities"; import { getComposerDraftSnapshot, - composerDraftsAtom, setComposerDraftText, composerContextImportsAtom, countComposerDraftAttachmentsAfterSelection, useComposerDraft, } from "../../state/use-composer-drafts"; -import { appAtomRegistry } from "../../state/atom-registry"; import type { ComposerDocumentAttachment } from "../../lib/composerContext"; import { useProject, useVisibleThreadShells } from "../../state/entities"; import { useMentionableEnvironments } from "../../state/environments"; @@ -579,7 +577,6 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps ownerKey: composerDraftKey, label: props.selectedThread.title || "Untitled thread", readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, - subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts index c16519c0ec79..da4ecdb402d1 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -143,7 +143,6 @@ describe("createLazyVoiceRecorder", () => { ownerKey: "thread", text: "", selection: { start: 0, end: 0 }, - revision: 0, }), commitDraft: () => undefined, onStateChange: () => undefined, diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index e26672b5b480..e3152f3627fd 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -15,7 +15,6 @@ export function useVoiceInputController(input: { readonly ownerKey: string | null; readonly label: string; readonly readDraftMessage: () => string | null; - readonly subscribeToDraftChanges: (onChange: () => void) => () => void; readonly selection: ComposerEditorSelection; readonly disabled?: boolean; readonly onChangeDraftMessage: (value: string) => void; @@ -58,7 +57,6 @@ export function useVoiceInputController(input: { } }, captured.selection, - captured.subscribeToDraftChanges, ), label: captured.label, }); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 9d30e376b54e..a935422c89df 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -1,6 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import type { PreparedVoiceTranscription } from "@t3tools/client-runtime/voice-input"; -import { Atom, AtomRegistry } from "effect/reactivity"; import { resetVoiceInputGlobalsForTests } from "../../../../../packages/client-runtime/src/voice-input/controller"; import { acquireAudioSession, @@ -23,19 +22,9 @@ import { createVoiceInputTarget, VoiceInputSession, type VoiceInputSessionDependencies, - type VoiceInputTarget, type VoiceRecordingSample, } from "./voiceInputSession"; -function createTarget( - ownerKey: string, - readText: () => string | null, - commit: VoiceInputTarget["commitDraft"], - selection: { start: number; end: number }, -) { - return createVoiceInputTarget(ownerKey, readText, commit, selection, () => () => {}); -} - function createSession( overrides?: Partial< Pick< @@ -116,7 +105,7 @@ describe("global voice input", () => { if (outcome === "activation failure") { audio.setActive.mockRejectedValueOnce(new Error("Audio activation failed")); } - await session.start(createTarget("first", () => "", vi.fn(), { start: 0, end: 0 })); + await session.start(createVoiceInputTarget("first", () => "", vi.fn(), { start: 0, end: 0 })); expect(pause).toHaveBeenCalledOnce(); expect(isAudioSessionOwner(playback)).toBe(false); if (outcome === "complete") { @@ -154,9 +143,9 @@ describe("global voice input", () => { async ({ selection, expected, cursor }) => { const { session } = createSession(); const commit = vi.fn(); - await session.start(createTarget("first", () => "hello world", commit, selection)); + await session.start(createVoiceInputTarget("first", () => "hello world", commit, selection)); await session.start( - createTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }), + createVoiceInputTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }), ); await session.controller.stop(); expect(commit).toHaveBeenCalledWith(expected, { @@ -175,7 +164,7 @@ describe("global voice input", () => { let visibleDraft = "first"; const targetKey = visibleDraft; await session.start( - createTarget( + createVoiceInputTarget( targetKey, () => drafts.get(targetKey) ?? null, (text) => drafts.set(targetKey, text), @@ -207,7 +196,7 @@ describe("global voice input", () => { const firstCommit = vi.fn(); const secondCommit = vi.fn(); const starting = session.start( - createTarget("first", () => "first", firstCommit, { start: 5, end: 5 }), + createVoiceInputTarget("first", () => "first", firstCommit, { start: 5, end: 5 }), ); await preparationEntered.promise; let stopping: Promise | null = null; @@ -226,7 +215,7 @@ describe("global voice input", () => { await transcriptionEntered.promise; } await session.start( - createTarget("second", () => "second", secondCommit, { start: 6, end: 6 }), + createVoiceInputTarget("second", () => "second", secondCommit, { start: 6, end: 6 }), ); expect(session.ownerKey).toBe("first"); expect(session.controller.currentState.phase).toBe(phase); @@ -247,7 +236,9 @@ describe("global voice input", () => { const { session } = createSession(); let text: string | null = "first"; const commit = vi.fn(); - await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); + await session.start( + createVoiceInputTarget("first", () => text, commit, { start: 5, end: 5 }), + ); text = change === "removed" ? null : "edited prompt"; await session.controller.stop(); expect(commit).not.toHaveBeenCalled(); @@ -255,10 +246,26 @@ describe("global voice input", () => { }, ); + it("retries a failed start against the draft as edited since", async () => { + const { session, prepare } = createSession(); + prepare.mockRejectedValueOnce(new Error("model unavailable")); + let text = "first"; + const commit = vi.fn(); + await session.start(createVoiceInputTarget("first", () => text, commit, { start: 5, end: 5 })); + expect(session.controller.currentState.phase).toBe("error"); + text = "first draft"; + await session.retry(); + await session.controller.stop(); + expect(prepare).toHaveBeenCalledTimes(2); + expect(commit).toHaveBeenCalledWith("first spoken text draft", { start: 17, end: 17 }); + }); + it("finishes the original draft at the recording limit while it is off screen", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); - await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + await session.start( + createVoiceInputTarget("first", () => "first", commit, { start: 5, end: 5 }), + ); await session.controller.handleRecorderStatus({ isFinished: true, hasError: false, @@ -269,41 +276,14 @@ describe("global voice input", () => { expect(session.controller.currentState.phase).toBe("idle"); }); - it("rejects a transcript when its off-screen draft changes and returns to the original text", async () => { - const registry = AtomRegistry.make(); - const draft = Atom.make("hello world"); - const unsubscribe = vi.fn(); - const { session } = createSession(); - const commit = vi.fn(); - const target = createVoiceInputTarget( - "first", - () => registry.get(draft), - commit, - { start: 6, end: 6 }, - (onChange) => { - const stop = registry.subscribe(draft, onChange); - return () => { - stop(); - unsubscribe(); - }; - }, - ); - await session.start(target); - registry.set(draft, "changed"); - registry.set(draft, "hello world"); - await session.controller.stop(); - expect(commit).not.toHaveBeenCalled(); - expect(session.controller.currentState.error).toContain("draft changed"); - expect(unsubscribe).toHaveBeenCalledTimes(1); - registry.dispose(); - }); - it("stops recording when its queued edit is discarded", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); let draft: string | null = "queued prompt"; const ownerKey = "thread~queued-edit~run"; - await session.start(createTarget(ownerKey, () => draft, commit, { start: 13, end: 13 })); + await session.start( + createVoiceInputTarget(ownerKey, () => draft, commit, { start: 13, end: 13 }), + ); session.cancel(ownerKey); draft = null; await session.controller.stop(); @@ -319,7 +299,9 @@ describe("global voice input", () => { it("keeps another prompt's recording when a queued edit is discarded", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); - await session.start(createTarget("other prompt", () => "hello", commit, { start: 5, end: 5 })); + await session.start( + createVoiceInputTarget("other prompt", () => "hello", commit, { start: 5, end: 5 }), + ); session.cancel("thread~queued-edit~run"); expect(recorder.stop).not.toHaveBeenCalled(); expect(session.controller.currentState.phase).toBe("recording"); @@ -338,12 +320,12 @@ describe("global voice input", () => { const oldCommit = vi.fn(); const nextCommit = vi.fn(); const firstStart = session.start( - createTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), + createVoiceInputTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), ); await preparationEntered.promise; session.cancel("first"); const nextStart = session.start( - createTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), + createVoiceInputTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), ); expect(prepare).toHaveBeenCalledTimes(1); expect(recorder.record).not.toHaveBeenCalled(); @@ -366,7 +348,9 @@ describe("global voice input", () => { const snapshots: number[] = []; session.subscribe(() => snapshots.push(session.getSnapshot().elapsedSeconds)); - await session.start(createTarget("first", () => "first", vi.fn(), { start: 5, end: 5 })); + await session.start( + createVoiceInputTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }), + ); expect(session.getSnapshot().state.phase).toBe("recording"); expect(keepAwake).toHaveBeenCalledTimes(1); // Silence on a silent history publishes nothing. @@ -388,7 +372,9 @@ describe("global voice input", () => { expect(session.getSnapshot().elapsedSeconds).toBe(2); sample = { durationMillis: 0, metering: -100 }; - await session.start(createTarget("first", () => "first", vi.fn(), { start: 5, end: 5 })); + await session.start( + createVoiceInputTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }), + ); expect(session.getSnapshot().elapsedSeconds).toBe(0); expect(publishAudioLevels).toHaveBeenLastCalledWith(expect.not.arrayContaining([1])); expect(keepAwake).toHaveBeenCalledTimes(2); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 5bf9395daf59..6a5ecd5bde22 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -19,7 +19,6 @@ export type VoiceInputTarget = { readonly label?: string; readonly readDraft: () => VoiceDraftSnapshot | null; readonly commitDraft: VoiceInputControllerDependencies["commitDraft"]; - readonly subscribe: () => () => void; }; export type VoiceInputSnapshot = { @@ -52,27 +51,14 @@ export function createVoiceInputTarget( readText: () => string | null, commitDraft: VoiceInputTarget["commitDraft"], selection: VoiceDraftSnapshot["selection"], - subscribeToChanges: (onChange: () => void) => () => void, ): VoiceInputTarget { - let revision = 0; return { ownerKey, readDraft: () => { const text = readText(); - if (text === null) return null; - return { ownerKey, text, selection, revision }; + return text === null ? null : { ownerKey, text, selection }; }, commitDraft, - subscribe: () => { - let previousText = readText(); - return subscribeToChanges(() => { - const text = readText(); - if (text !== previousText) { - previousText = text; - revision += 1; - } - }); - }, }; } @@ -84,7 +70,7 @@ export class VoiceInputSession { readonly controller: VoiceInputController; private readonly dependencies: VoiceInputSessionDependencies; private target: VoiceInputTarget | null = null; - private unsubscribe: (() => void) | null = null; + private startText: string | null = null; private snapshot: VoiceInputSnapshot = { state: IDLE_STATE, ownerKey: null, @@ -102,7 +88,11 @@ export class VoiceInputSession { this.controller = new VoiceInputController({ ...dependencies, onStateChange: (state) => this.handleStateChange(state), - readDraft: () => this.target?.readDraft() ?? null, + // The target's selection is fixed at start, so a draft changed since refuses the transcript. + readDraft: () => { + const draft = this.target?.readDraft() ?? null; + return draft?.text === this.startText ? draft : null; + }, commitDraft: (text, selection) => this.target?.commitDraft(text, selection), }); } @@ -141,23 +131,17 @@ export class VoiceInputSession { start(target: VoiceInputTarget): Promise { if (voiceInputBlocksSubmission(this.controller.currentState)) return Promise.resolve(); this.target = target; - this.unsubscribe = target.subscribe(); + this.startText = target.readDraft()?.text ?? null; return this.controller.start(); } dispose(): void { this.controller.dispose(); this.stopRecordingActivity(); - this.unsubscribe?.(); - this.unsubscribe = null; this.listeners.clear(); } private handleStateChange(state: VoiceInputState): void { - if (!voiceInputBlocksSubmission(state)) { - this.unsubscribe?.(); - this.unsubscribe = null; - } const previousPhase = this.snapshot.state.phase; let elapsedSeconds = this.snapshot.elapsedSeconds; if (state.phase !== previousPhase) { diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx index 8da54f5f7214..69c2ac1fd4d5 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx @@ -31,10 +31,12 @@ const promptBefore = `See ${serializeAssistantCitation(citation)} `; function Composer({ question, editorRef, + disabled = false, send = "clears", }: { question: boolean; editorRef: React.RefObject; + disabled?: boolean; send?: "clears" | "blocked" | "refusedLater"; }) { const [prompt, setPrompt] = useState(promptBefore); @@ -57,7 +59,7 @@ function Composer({ cursor={0} contextRecords={new Map()} skills={[]} - disabled={false} + disabled={disabled} placeholder="" onChange={(next) => (question ? setAnswer(next) : setPrompt(next))} onCitationDraftSave={(replacedValue, savedValue) => { @@ -159,3 +161,26 @@ it.each(["blocked", "refusedLater"] as const)( .toHaveAttribute("data-prompt", `See ${serializeAssistantCitation(citation)}x `); }, ); + +it("cancels a new citation while dictation makes the composer read-only", async () => { + const editorRef = createRef(); + await using screen = await render(); + + const sourceAnchor = findAssistantCitationSourceAnchor(document, citation); + if (!sourceAnchor) throw new Error("Citation source did not resolve"); + await act(async () => + editorRef.current?.requestCitationComment({ + previousValue: "See ", + value: promptBefore, + citationStart: "See ".length, + sourceAnchor, + insertedSpaces: { before: false, after: true }, + }), + ); + await expect.element(page.getByLabelText("Comment on selected text")).toBeVisible(); + + await page.getByRole("button", { name: "Cancel" }).click(); + + await expect.element(page.getByLabelText("Comment on selected text")).not.toBeInTheDocument(); + expect(screen.container.querySelector("output")?.dataset.prompt).toBe("See "); +}); diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index 5895b63cbb9a..a974c9308844 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -468,7 +468,7 @@ function ComposerCitationNodeView({ node, editor, getPos }: NodeViewProps) { const removeOnCancel = commentTarget?.removeOnCancel; const onRemove = () => { - if (!editor.isEditable || !removeOnCancel) return; + if (!removeOnCancel) return; const pos = nodePos(); if (pos === null) return; const { doc } = editor.state; @@ -477,7 +477,9 @@ function ComposerCitationNodeView({ node, editor, getPos }: NodeViewProps) { const end = pos + current.nodeSize; const from = removeOnCancel.before && doc.textBetween(pos - 1, pos) === " " ? pos - 1 : pos; const to = removeOnCancel.after && doc.textBetween(end, end + 1) === " " ? end + 1 : end; - editor.chain().focus().deleteRange({ from, to }).run(); + // A read-only composer (dictation) can still receive a new citation, so Cancel must undo it. + const chain = editor.chain(); + (editor.isEditable ? chain.focus() : chain).deleteRange({ from, to }).run(); }; const sentRef = useRef(false); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 6f6d4fbd50ff..4150db637d45 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1810,7 +1810,6 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) projectId: activeProjectId, projectName, ownerKey: questionAttachmentTarget ?? composerDraftTargetKey, - draftText: prompt, disabled: dictationDisabled, readDraft: () => { const snapshot = readComposerSnapshot(); diff --git a/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx b/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx index 8d52066cd5f7..c76e39a14731 100644 --- a/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx +++ b/apps/web/src/components/preview/PreviewAnnotationSpeech.tsx @@ -5,7 +5,7 @@ import type { ScopedThreadRef, } from "@t3tools/contracts"; import { useNavigate } from "@tanstack/react-router"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef } from "react"; import { resolveSpeechPresentation } from "~/components/chat/ComposerSpeechButton"; import { VoiceInputSetup } from "~/components/chat/VoiceInputSetup"; @@ -40,7 +40,6 @@ export function PreviewAnnotationSpeech({ projectId ? scopeProjectRef(threadRef.environmentId, projectId) : null, ); const draft = useRef({ text: "", cursor: 0, selectionEnd: 0 }); - const [text, setText] = useState(""); const keys = useRef(null); const guestFocused = useRef(false); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- notices focus returning from the preview guest to this window @@ -60,7 +59,6 @@ export function PreviewAnnotationSpeech({ projectId, projectName: project?.title, ownerKey: JSON.stringify(["preview-annotation", threadRef, tabId, config.sessionId]), - draftText: text, offerSetup: false, readDraft: () => ({ text: draft.current.text, @@ -68,7 +66,6 @@ export function PreviewAnnotationSpeech({ }), commitDraft: (next, selection) => { draft.current = { text: next, cursor: selection.start, selectionEnd: selection.end }; - setText(next); void publish(current.current.voiceState, { text: next, cursor: selection.start }); }, }); @@ -136,7 +133,6 @@ export function PreviewAnnotationSpeech({ cursor: Math.min(event.cursor, event.text.length), selectionEnd: Math.min(event.selectionEnd ?? event.cursor, event.text.length), }; - setText(event.text); if (event.action === "focus" || event.action === "key" || event.action === "start") guestFocused.current = true; if (event.action === "blur") guestFocused.current = false; diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index f4a62edf6ed1..2875a6edbe35 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -1083,7 +1083,6 @@ function TranscriptionTest(props: { projectId: props.projectId, projectName: props.projectName, ownerKey: JSON.stringify([props.environmentId, props.projectId]), - draftText: draft, cleanup: false, readDraft: () => ({ text: draftRef.current, diff --git a/apps/web/src/speech/useDictationShortcut.browser.test.tsx b/apps/web/src/speech/useDictationShortcut.browser.test.tsx index 8cd17f4f368d..5bbc17b1ecc3 100644 --- a/apps/web/src/speech/useDictationShortcut.browser.test.tsx +++ b/apps/web/src/speech/useDictationShortcut.browser.test.tsx @@ -1,17 +1,30 @@ import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings"; +import { createRef, type RefObject } from "react"; import { expect, it, vi } from "vite-plus/test"; import { render } from "../../test/render"; -import { useDictationShortcut } from "./useDictationShortcut"; +import { useDictationShortcut, type DictationKeyHandlers } from "./useDictationShortcut"; vi.mock("../hooks/useSettings", () => ({ useClientSettings: () => "hold" })); -function Probe({ speech }: { speech: Parameters[0]["speech"] }) { +type ShortcutInput = Parameters[0]; + +function Probe({ + speech, + targetRef, + subscribeKeys, +}: { + speech: ShortcutInput["speech"]; + targetRef?: RefObject; + subscribeKeys?: ShortcutInput["subscribeKeys"]; +}) { useDictationShortcut({ keybindings: DEFAULT_RESOLVED_KEYBINDINGS, speech, disabled: false, terminalOpen: false, modelPickerOpen: false, + ...(targetRef ? { targetRef } : {}), + ...(subscribeKeys ? { subscribeKeys, ownsFocus: () => true } : {}), }); return null; } @@ -80,3 +93,120 @@ it("finishes a released hold after startup even if the chord was pressed again w expect(speech.start).toHaveBeenCalledOnce(); expect(speech.stop).toHaveBeenCalledOnce(); }); + +const escape = () => + new KeyboardEvent("keydown", { key: "Escape", code: "Escape", bubbles: true, cancelable: true }); + +const recording = () => ({ + ...actions(true), + state: { phase: "recording" as const, error: null, errorAction: null }, +}); + +it("lets Escape in another dictation editor reach that editor instead of canceling the recording", async () => { + const speech = recording(); + const commentRef = createRef(); + const editorEscape = vi.fn(); + await using screen = await render( + <> + + +
{ + if (event.key === "Escape") editorEscape(); + }} + > +