diff --git a/.changeset/quiet-hounds-shave.md b/.changeset/quiet-hounds-shave.md new file mode 100644 index 00000000000..20e32667ebb --- /dev/null +++ b/.changeset/quiet-hounds-shave.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +The current-worker API now reports each task's queue, so you can see which tasks write to a given queue. diff --git a/.server-changes/dashboard-agent.md b/.server-changes/dashboard-agent.md index 4ecf07bba69..d90442ddd69 100644 --- a/.server-changes/dashboard-agent.md +++ b/.server-changes/dashboard-agent.md @@ -7,6 +7,8 @@ Meet the dashboard agent: a chat in every environment that answers questions abo **Investigate** on a failed run, an error, a backed-up queue or a run that hasn't started gets you a worked-through answer — what happened, why, and how to fix it, with every claim linked to the runs, errors and deploys behind it. +**Watch…** on a run, queue, error or the health report tells you when things change: a run finishes, a queue clears or grows past a number you pick, an error comes back, an environment recovers. The answer arrives in the chat and, if you want, by email, Slack or webhook — and the agent can look into bad news on its own. A watch reaches you on any browser you sign in from, without opening the chat first. + The agent works on preview and dev branches, with that branch's own data. The health report reads the same everywhere — dashboard, terminal, editor. A very long chat keeps working: the agent summarises the earlier part and carries on. The agent's replies no longer show images. diff --git a/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx b/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx index 4ef3a276b74..d4089e51533 100644 --- a/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx +++ b/apps/webapp/app/components/dashboard-agent/ActionsBlock.tsx @@ -4,16 +4,20 @@ import type { } from "@internal/dashboard-agent-contracts"; import { Button } from "~/components/primitives/Buttons"; import { ChatActionsRow } from "./chat-layout"; -import { renderableActions } from "./view-actions"; +import { renderableActions, withoutWatchActions } from "./view-actions"; export function ActionsBlock({ block, onIntent, + dropWatch = false, }: { block: ActionsBlockPayload; onIntent?: (intent: AgentIntent) => void; + /** Set when an investigation card in the same answer already offers the watch. */ + dropWatch?: boolean; }) { - const renderable = renderableActions(block.actions); + const actions = dropWatch ? withoutWatchActions(block.actions) : block.actions; + const renderable = renderableActions(actions); if (!onIntent || renderable.length === 0) return null; return ( diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx index 99e852e6b13..cc581ee39c3 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgent.tsx @@ -1,4 +1,4 @@ -import type { SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import type { SuggestedPrompt, WatchSpec } from "@internal/dashboard-agent-contracts"; import { useLocation } from "@remix-run/react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { @@ -6,6 +6,9 @@ import { ResizablePanel, ResizablePanelGroup, } from "~/components/primitives/Resizable"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; import { useAskAiAvailability } from "~/hooks/useAskAiAvailability"; import { agentDeepLinkParams, ASK_AI_SHORTCUT, askAiChannelTarget } from "./ask-ai-channels"; @@ -18,18 +21,98 @@ import { readAgentFullscreen, writeAgentFullscreen, } from "./panel-layout"; +import { nextPendingTurnChatId } from "./pending-turn"; +import { nextVisibleChat, unreadWorkForDot } from "./unread-counts"; +import { startWakePolling, wakesToToast } from "./wake-poll"; +import { shouldPollWakeFeed, subscribeWatchActivity } from "./watch-activity"; +import { + showWatchWakesSummaryToast, + showWatchWakeToast, + WAKE_TOAST_MAX_INDIVIDUAL, + type WatchWake, +} from "./WatchWakeToast"; + +const TOASTED_WAKES_STORAGE_KEY = "tdev:dashboard-agent:toasted-wakes"; + +// Shorter than the poll interval, so a stuck request is dropped before the next tick. +const UNREAD_REQUEST_TIMEOUT_MS = 30_000; /** `hasAccess` is a UI gate only; the resource routes enforce the same check server-side. */ export function DashboardAgent({ children, hasAccess = false, promotedPrompt, + /** From the page load: unread wakes waiting for this user, whatever this browser remembers. */ + initialUnreadWakes = 0, + initialUnreadWork = 0, + /** Also from the page load: a watch is running, so a wake can still arrive in this tab. */ + hasActiveWatches = false, }: { children: React.ReactNode; hasAccess?: boolean; promotedPrompt?: SuggestedPrompt; + initialUnreadWakes?: number; + /** Chats whose transcript moved on since their owner last looked. */ + initialUnreadWork?: number; + hasActiveWatches?: boolean; }) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const actionPath = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/dashboard-agent`; + const [open, setOpen] = useState(false); + // Seeded from the page load, so the launcher dot is right before the first poll answers. + const [unreadWakes, setUnreadWakes] = useState(initialUnreadWakes); + // Work that finished behind a closed panel. Counted server-side on page load and refreshed + // with the chat list; the wake poll doesn't carry it. + const [unreadWork, setUnreadWork] = useState(initialUnreadWork); + // A turn this tab started may finish after the panel closes; that is exactly the case the + // dot exists for, so the poll has to be running when it lands. + const [pendingTurnChatId, setPendingTurnChatId] = useState(null); + const handleTurnActivityChange = useCallback((chatId: string, active: boolean) => { + setPendingTurnChatId((current) => nextPendingTurnChatId(current, { chatId, active })); + }, []); + const toastedWakes = useRef(new Set()); + // The toast source is recent deliveries, not unread, so the dedupe must survive a reload. + useEffect(() => { + try { + const raw = window.localStorage.getItem(TOASTED_WAKES_STORAGE_KEY); + if (raw) for (const id of JSON.parse(raw) as string[]) toastedWakes.current.add(id); + } catch { + // Storage unavailable; the in-memory dedupe still applies. + } + }, []); + const rememberToasted = useCallback((watchId: string) => { + toastedWakes.current.add(watchId); + try { + // Newest ids only, so the key can't grow unbounded. + window.localStorage.setItem( + TOASTED_WAKES_STORAGE_KEY, + JSON.stringify([...toastedWakes.current].slice(-50)) + ); + } catch { + // Same as the read. + } + }, []); + // A wake in the on-screen chat toasts but must not light the dot. + const visibleChat = useRef(null); + // Read by the poll callback, which outlives the render that started it: `open` in its closure + // is whatever it was when polling began, and opening the panel does not restart the poll. + const panelOpen = useRef(open); + useEffect(() => { + panelOpen.current = open; + }, [open]); + + // Switching environment re-runs the layout loader but does not remount it, so the seeds + // above would keep the old environment's counts. + const seededEnvironment = useRef(environment.id); + useEffect(() => { + if (seededEnvironment.current === environment.id) return; + seededEnvironment.current = environment.id; + setUnreadWakes(initialUnreadWakes); + setUnreadWork(initialUnreadWork); + }, [environment.id, initialUnreadWakes, initialUnreadWork]); // Read lazily so SSR always renders the side panel. const [fullscreen, setFullscreen] = useState(readAgentFullscreen); @@ -55,17 +138,32 @@ export function DashboardAgent({ const [requestedMessage, setRequestedMessage] = useState< { text: string; seq: number } | undefined >(undefined); + // `seq` so the same chat can be asked for twice. + const [openChatRequest, setOpenChatRequest] = useState< + { chatId: string; seq: number } | undefined + >(undefined); + const [watchRequest, setWatchRequest] = useState<{ spec: WatchSpec; seq: number } | undefined>( + undefined + ); const setPanelOpen = useCallback((next: boolean) => { setOpen(next); // Pending requests must be dropped or a stale one re-applies on the next open. if (!next) { + visibleChat.current = null; setFullscreen(false); writeAgentFullscreen(false); setRequestedMessage(undefined); + setOpenChatRequest(undefined); + setWatchRequest(undefined); } }, []); + const openChat = useCallback((chatId: string) => { + setOpen(true); + setOpenChatRequest((current) => ({ chatId, seq: (current?.seq ?? 0) + 1 })); + }, []); + const openWith = useCallback((text: string) => { const trimmed = text.trim(); if (!trimmed) return; @@ -73,6 +171,111 @@ export function DashboardAgent({ setRequestedMessage((current) => ({ text: trimmed, seq: (current?.seq ?? 0) + 1 })); }, []); + const openWithWatch = useCallback((spec: WatchSpec) => { + setOpen(true); + setWatchRequest((current) => ({ spec, seq: (current?.seq ?? 0) + 1 })); + }, []); + + // Nothing to be woken about means nothing to poll for. The page load's unread count and + // active-watch flag are the ungated signals; the browser's own memory of a watch starts the + // poll without a reload. Once any says yes this tab keeps polling, so a wake reaches a tab + // that was open before the watch existed. + const [watching, setWatching] = useState(false); + useEffect(() => { + const sync = () => { + if ( + shouldPollWakeFeed({ + serverUnreadWakes: initialUnreadWakes, + serverHasActiveWatches: hasActiveWatches, + serverUnreadWork: initialUnreadWork, + turnInFlight: pendingTurnChatId !== null, + organizationId: organization.id, + }) + ) + setWatching(true); + }; + sync(); + return subscribeWatchActivity(sync); + }, [organization.id, initialUnreadWakes, hasActiveWatches, initialUnreadWork, pendingTurnChatId]); + + useEffect(() => { + if (!hasAccess || !watching) return; + + let cancelled = false; + const load = async () => { + try { + // Bounded, so one stuck request can't hold the poll's in-flight guard. + const res = await fetch(`${actionPath}?unread=1`, { + signal: AbortSignal.timeout(UNREAD_REQUEST_TIMEOUT_MS), + }); + if (!res.ok) return; + const data = (await res.json()) as { + unreadWakes?: number; + unreadWork?: number; + wakes?: WatchWake[]; + }; + if (cancelled) return; + // The wakes list carries read ones too, so only unread ones are subtracted. + const unreadInView = (data.wakes ?? []).filter( + (wake) => wake.unread && wake.chatId === visibleChat.current + ).length; + setUnreadWakes(Math.max(0, (data.unreadWakes ?? 0) - unreadInView)); + setUnreadWork( + unreadWorkForDot({ + reported: data.unreadWork, + panelOpen: panelOpen.current, + visibleChatId: visibleChat.current, + }) + ); + + const fresh = wakesToToast(data.wakes, toastedWakes.current); + for (const wake of fresh) rememberToasted(wake.watchId); + + if (fresh.length > WAKE_TOAST_MAX_INDIVIDUAL) { + showWatchWakesSummaryToast(fresh.length, () => setPanelOpen(true)); + } else { + for (const wake of [...fresh].reverse()) { + showWatchWakeToast(wake, openChat); + } + } + } catch { + // Try again next tick. + } + }; + + const stop = startWakePolling({ + load, + isHidden: () => document.hidden, + onVisibilityChange: (listener) => { + document.addEventListener("visibilitychange", listener); + return () => document.removeEventListener("visibilitychange", listener); + }, + }); + + return () => { + cancelled = true; + stop(); + }; + }, [hasAccess, watching, actionPath, setPanelOpen, openChat]); + + // Zeroes the wake dot right away; the poll restores the truth if another chat has one. The + // work count is not touched here: the panel derives it from the chat list. + const markChatRead = useCallback( + async (chatId: string, options: { leaving: boolean }) => { + visibleChat.current = nextVisibleChat(chatId, options); + setUnreadWakes(0); + const body = new FormData(); + body.set("intent", "read"); + body.set("chatId", chatId); + try { + await fetch(actionPath, { method: "POST", body }); + } catch { + // Catches up on the next open. + } + }, + [actionPath] + ); + // ⌘J is contextual: closed opens the panel, open starts a new chat. It never closes. useShortcutKeys({ shortcut: TOGGLE_PANEL_SHORTCUT, @@ -107,8 +310,8 @@ export function DashboardAgent({ }); const context = useMemo( - () => ({ open, setOpen: setPanelOpen, openWith }), - [open, setPanelOpen, openWith] + () => ({ open, setOpen: setPanelOpen, openWith, openWithWatch, unreadWakes, unreadWork }), + [open, setPanelOpen, openWith, openWithWatch, unreadWakes, unreadWork] ); if (!hasAccess) { @@ -137,8 +340,13 @@ export function DashboardAgent({ setPanelOpen(false)} requestedMessage={requestedMessage} + openChatRequest={openChatRequest} + watchRequest={watchRequest} newChatSeq={newChatSeq} promotedPrompt={promotedPrompt} + onChatRead={markChatRead} + onUnreadWorkChange={setUnreadWork} + onTurnActivityChange={handleTurnActivityChange} isFullscreen={fullscreen} onToggleFullscreen={toggleFullscreen} /> diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx index 45281c796b3..bfcfad9e491 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentChat.tsx @@ -1,8 +1,13 @@ import { useChat } from "@ai-sdk/react"; import type { UIMessage } from "@ai-sdk/react"; import type { dashboardAgent } from "@internal/dashboard-agent"; -import type { AgentIntent, SuggestedPrompt } from "@internal/dashboard-agent-contracts"; -import { useNavigate } from "@remix-run/react"; +import { + isWatchRequestMessageId, + type AgentIntent, + type SuggestedPrompt, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { useLocation, useNavigate } from "@remix-run/react"; import { useTriggerChatTransport } from "@trigger.dev/sdk/chat/react"; import { useCallback, useEffect, useRef, useState } from "react"; import { useToast } from "~/components/primitives/Toast"; @@ -14,7 +19,7 @@ import { DashboardAgentMessages, type TurnActivity } from "./DashboardAgentMessa import { MESSAGE_TOO_LARGE_ERROR } from "./message-limits"; import { createTranscriptOrder, orderTranscript } from "./message-order"; import { appendRunFilters } from "./navigate-target"; -import { pendingNavigateIntents } from "./pending-intents"; +import { pendingNavigateIntents, pendingWatchIntents } from "./pending-intents"; import type { AgentPageContext } from "./page-context-types"; import { retryAction } from "./retry-action"; import { @@ -22,8 +27,11 @@ import { hasOpenInvestigation, pollSettledTranscript, } from "./settled-transcript"; +import { takeNavigateIntent } from "./turn-navigation"; +import { teardownCancelsTurn, unmountTeardown } from "./turn-teardown"; import { useAgentMessageQuota } from "./useAgentMessageQuota"; import { useTriggerUriResolver } from "./useTriggerUriResolver"; +import { WatchChips, type WatchChip } from "./WatchChips"; // Resuming with `lastEventId` stops the `.out` stream replaying the previous turn. export type DashboardAgentSession = { @@ -54,9 +62,14 @@ export function DashboardAgentChat({ currentPage, pendingFirstMessage, streaming, - prefill, + sendRequest, promotedPrompt, + watches, pagePaths, + watchCard, + appendedMessages, + onWatchIntent, + onCancelWatch, onTurnSettled, onActivityChange, }: { @@ -73,23 +86,29 @@ export function DashboardAgentChat({ // Undefined for head-started and resumed chats. pendingFirstMessage?: string; streaming?: boolean; - // `seq` makes each request distinct so the same text can be sent twice. - prefill?: { text: string; seq: number }; + // A prompt the user asked for by clicking. `seq` makes each request distinct so the same + // text can be sent twice. + sendRequest?: { text: string; seq: number }; promotedPrompt?: SuggestedPrompt; + watches: WatchChip[]; pagePaths?: Record; + watchCard?: React.ReactNode; + appendedMessages?: { messages: UIMessage[]; seq: number }; + /** Nothing is persisted until the user submits the card. */ + onWatchIntent?: (spec: WatchSpec) => void; + onCancelWatch: (watchId: string) => void; onTurnSettled: () => void; onActivityChange?: (chatId: string, activity: TurnActivity | null) => void; }) { const [input, setInput] = useState(""); const navigate = useNavigate(); + const location = useLocation(); const toast = useToast(); - const prefilledSeq = useRef(undefined); - useEffect(() => { - if (!prefill || prefilledSeq.current === prefill.seq) return; - prefilledSeq.current = prefill.seq; - setInput(prefill.text); - }, [prefill]); + // The path this chat last rendered on. React never unmounts on a page teardown, so an + // unmount whose live URL has moved is the router having navigated out from under it. + const renderedPathRef = useRef(location.pathname); + renderedPathRef.current = location.pathname; const transport = useTriggerChatTransport({ task: "dashboard-agent", @@ -174,6 +193,20 @@ export function DashboardAgentChat({ const activity: TurnActivity | null = status === "submitted" ? "thinking" : status === "streaming" ? "working" : null; + // Once per `seq`: the append is already persisted, so a replay would duplicate it. + // Ids are stable, so anything already in the transcript is skipped. + const appendedSeq = useRef(undefined); + useEffect(() => { + if (!appendedMessages || appendedSeq.current === appendedMessages.seq) return; + appendedSeq.current = appendedMessages.seq; + setMessages((current) => { + const missing = appendedMessages.messages.filter( + (message) => !current.some((existing) => existing.id === message.id) + ); + return missing.length === 0 ? current : [...current, ...missing]; + }); + }, [appendedMessages, setMessages]); + const sentFirst = useRef(false); useEffect(() => { if (pendingFirstMessage && !sentFirst.current) { @@ -193,8 +226,19 @@ export function DashboardAgentChat({ [isStreaming, atMessageCap, sendMessage] ); + // The panel only sends when the chat can take it, so this never lands mid-turn. + const sentRequestSeq = useRef(undefined); + useEffect(() => { + if (!sendRequest || sentRequestSeq.current === sendRequest.seq) return; + sentRequestSeq.current = sendRequest.seq; + submit(sendRequest.text); + }, [sendRequest, submit]); + const retry = useCallback(() => { - const action = retryAction(messages); + // A watch's consent record is a user message nobody typed, so retry never treats it as one. + const action = retryAction( + messages.filter((m) => !(m.role === "user" && isWatchRequestMessageId(m.id))) + ); if (!action) return; clearError(); if (action.kind === "regenerate") { @@ -232,6 +276,9 @@ export function DashboardAgentChat({ case "ask": submit(intent.prompt); return; + case "watch": + onWatchIntent?.(intent.spec); + return; case "navigate": void goTo(intent); return; @@ -239,7 +286,7 @@ export function DashboardAgentChat({ console.warn(`Dashboard agent: unhandled intent "${intent.kind}"`); } }, - [submit, goTo] + [submit, goTo, onWatchIntent] ); // Seeded from the loaded transcript before first render, so history never re-navigates. @@ -248,17 +295,54 @@ export function DashboardAgentChat({ navigatedRef.current = new Set(); pendingNavigateIntents(initialMessages, navigatedRef.current); } + // Where the running turn was asked for. Never cleared on settle: the navigate intent can be + // committed alongside the status going ready, and it is the started-at path it belongs to. + const turnStartedPathRef = useRef(null); + const turnWasInFlight = useRef(false); useEffect(() => { - const pending = pendingNavigateIntents(messages, navigatedRef.current!); - const target = pending.at(-1); + const inFlight = status === "submitted" || status === "streaming"; + if (inFlight && !turnWasInFlight.current) turnStartedPathRef.current = renderedPathRef.current; + turnWasInFlight.current = inFlight; + }, [status]); + + useEffect(() => { + const target = takeNavigateIntent({ + messages, + handled: navigatedRef.current!, + startedPath: turnStartedPathRef.current, + currentPath: renderedPathRef.current, + }); if (target) void goTo(target); }, [messages, goTo]); + const watchProposedRef = useRef | null>(null); + if (watchProposedRef.current === null) { + watchProposedRef.current = new Set(); + pendingWatchIntents(initialMessages, watchProposedRef.current); + } + useEffect(() => { + const pending = pendingWatchIntents(messages, watchProposedRef.current!); + const proposed = pending.at(-1); + if (proposed) onWatchIntent?.(proposed.spec); + }, [messages, onWatchIntent]); + const stop = useCallback(() => { transport.stopGeneration(chatId); aiStop(); }, [transport, chatId, aiStop]); + const teardownRef = useRef<() => void>(() => {}); + teardownRef.current = () => { + if (status !== "streaming" && status !== "submitted") return; + const reason = unmountTeardown({ + renderedPath: renderedPathRef.current, + livePath: window.location.pathname, + }); + if (!teardownCancelsTurn(reason)) return; + stop(); + }; + useEffect(() => () => teardownRef.current(), []); + // Read by the settle effect, which must not re-run when the transcript changes. const messagesRef = useRef(messages); messagesRef.current = messages; @@ -288,6 +372,10 @@ export function DashboardAgentChat({ return ( <> + watch.status === "active")} + onCancel={onCancelWatch} + /> {messages.length === 0 && !pendingFirstMessage ? ( )} + {watchCard ?
{watchCard}
: null} {quota.kind === "reached" ? ( submit(input)} onStop={stop} isStreaming={isStreaming} - focusKey={prefill?.seq} + focusKey={sendRequest?.seq} context={ void; projectSlug: string; @@ -22,6 +23,7 @@ export function DashboardAgentDraft({ currentPage: string; pageContext?: AgentPageContext; promotedPrompt?: SuggestedPrompt; + watchCard?: React.ReactNode; }) { const [input, setInput] = useState(""); @@ -56,6 +58,7 @@ export function DashboardAgentDraft({ promoted={promotedPrompt} composer={
+ {watchCard} submit(input)} onStop={() => {}} isStreaming={false} - placeholderSuggestion={placeholderSuggestion} + placeholderSuggestion={watchCard ? undefined : placeholderSuggestion} context={ void; }) { const [isHistoryOpen, setHistoryOpen] = useState(false); + const [pendingDelete, setPendingDelete] = useState(null); return (
@@ -77,11 +82,20 @@ export function DashboardAgentHeader({ setHistoryOpen(false); onSelectChat(chatId); }} - onDelete={onDeleteChat} + onRequestDelete={(chat) => { + setHistoryOpen(false); + setPendingDelete(chat); + }} /> + !open && setPendingDelete(null)} + onConfirm={onDeleteChat} + /> +
{showNewChat && ( - } - cancelButton={ - - } - /> -
- - - +// Rendered outside the history popover: inside it, focus moving to the dialog dismisses the +// popover, which unmounts the dialog before it can be answered. +export function DashboardAgentDeleteChatDialog({ + chat, + onOpenChange, + onConfirm, +}: { + chat: DashboardAgentChat | null; + onOpenChange: (open: boolean) => void; + onConfirm: (chatId: string) => void; +}) { + return ( + + + Delete this chat? +
+ + "{chat?.title}" and everything in it will be deleted. This can't be undone. + + { + if (chat) onConfirm(chat.id); + onOpenChange(false); + }} + > + Delete chat + + } + cancelButton={ + + } + /> +
+ + ); } diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx index 5eda8983952..a0672a3d3bb 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.tsx @@ -16,14 +16,17 @@ import { ChatText, ChatTranscript, ChatTurn, + ChatWakeSlot, } from "./chat-layout"; import { reuseWinners } from "./investigation-winners"; import { stripModelImages } from "./model-markdown"; import { reportBlockFromToolPart } from "./report-block-adapter"; import { shouldShowLiveTurnError } from "./turn-error"; import type { ResolvedUri } from "./ReportView"; -import { answerContinuesAfter } from "./view-actions"; +import { answerContinuesAfter, turnAlreadyOffersWatch } from "./view-actions"; +import { latestRevisionBlocks } from "./view-blocks"; import { ViewBlocks } from "./view-catalog"; +import { findWakeWatch, WakeBanner, wakeRefFromMessageId, type WakeWatch } from "./WakeBanner"; export type { TurnActivity }; @@ -36,6 +39,8 @@ export type DashboardAgentMessagesProps = { onIntent?: (intent: AgentIntent) => void; resolveUri?: (uri: string) => ResolvedUri | null; pagePaths?: Record; + /** Optional: without it a wake banner falls back to kind-agnostic wording. */ + watches?: WakeWatch[]; }; // Returns the same reference when there are no `step-start` parts, so memoization holds. @@ -211,12 +216,14 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ onIntent, resolveUri, pagePaths, + watches, investigationWinners, }: { message: UIMessage; onIntent?: (intent: AgentIntent) => void; resolveUri?: (uri: string) => ResolvedUri | null; pagePaths?: Record; + watches?: WakeWatch[]; /** See {@link winningInvestigationOccurrences}. */ investigationWinners?: Map; }) { @@ -231,17 +238,28 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ const parts = message.parts ?? []; if (parts.length === 0) return null; + // Null for a part that renders no view at all; an empty array for one whose blocks were + // all superseded. Both skip the part, only the first falls through to the other renderers. + const blocksByPart = parts.map((part, i) => { + const raw = blocksFor(part); + return raw + ? withoutSupersededInvestigations(raw, `${message.id}:${i}`, investigationWinners) + : null; + }); + // One answer for the whole turn: two `render_view` parts each deciding for themselves + // would show the watch button twice. `ViewBlocks` collapses revisions the same way. + const watchOfferedInTurn = turnAlreadyOffersWatch( + blocksByPart + .filter((blocks): blocks is unknown[] => blocks !== null) + .map((blocks) => latestRevisionBlocks(blocks as never)) + ); + const body: React.ReactNode[] = []; for (let i = 0; i < parts.length; i++) { const part = parts[i]!; - const rawBlocks = blocksFor(part); - if (rawBlocks) { - const blocks = withoutSupersededInvestigations( - rawBlocks, - `${message.id}:${i}`, - investigationWinners - ); + const blocks = blocksByPart[i]; + if (blocks) { if (blocks.length > 0) { body.push( @@ -251,6 +269,7 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ resolveUri={resolveUri} pagePaths={pagePaths} answered={answerContinuesAfter(parts as never, i)} + watchOfferedInTurn={watchOfferedInTurn} /> ); @@ -275,6 +294,21 @@ const DashboardAgentTurn = memo(function DashboardAgentTurn({ body.push(renderDashboardPart(part, i, resolveUri)); } + const wake = wakeRefFromMessageId(message.id); + if (wake) { + return ( + + + } + > + {body} + + + ); + } + return {body}; }); @@ -287,6 +321,7 @@ export function DashboardAgentTurns({ onIntent, resolveUri, pagePaths, + watches, }: DashboardAgentMessagesProps) { // Must be the exact parts the turns render: the winners map keys by part index. const stripped = useMemo(() => messages.map(stripStepParts), [messages]); @@ -307,6 +342,7 @@ export function DashboardAgentTurns({ onIntent={onIntent} resolveUri={resolveUri} pagePaths={pagePaths} + watches={watches} investigationWinners={investigationWinners} /> ))} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx index 636fe328905..5c077063e81 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentPanel.tsx @@ -1,7 +1,7 @@ import type { UIMessage } from "@ai-sdk/react"; import { useLocation } from "@remix-run/react"; import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from "react"; import { AgentSpinner } from "~/components/primitives/Spinner"; import { useToast } from "~/components/primitives/Toast"; import { useAgentPageContext } from "~/hooks/useAgentPageContext"; @@ -16,13 +16,20 @@ import { type DashboardAgentSession, } from "./DashboardAgentChat"; import { DashboardAgentDraft } from "./DashboardAgentDraft"; +import { WatchCard } from "./WatchCard"; +import { watchDraftFor } from "./watch-card"; +import { NO_WATCH_CARD, watchCardReducer } from "./watch-card-state"; +import { forgetWatchActivity, rememberWatchActivity } from "./watch-activity"; import type { TurnActivity } from "./DashboardAgentMessages"; import { DashboardAgentHeader } from "./DashboardAgentHeader"; import type { DashboardAgentChat as DashboardAgentChatListItem } from "./DashboardAgentHistory"; -import type { SuggestedPrompt } from "@internal/dashboard-agent-contracts"; +import type { SuggestedPrompt, WatchSpec } from "@internal/dashboard-agent-contracts"; import { resolveOpenedChat, type OpenedChatResponse } from "./opened-chat"; import type { AgentPageContext } from "./page-context-types"; import { agentPageLabel } from "./page-label"; +import { explicitPromptTarget } from "./explicit-prompt"; +import { escapeClosesPanel } from "./panel-escape"; +import { markChatListRead, unreadWorkCount } from "./unread-counts"; import { AgentPanelColumn } from "./panel-layout"; import { concurrencyPath } from "~/utils/pathBuilder"; @@ -64,8 +71,13 @@ type ActiveChat = { export function DashboardAgentPanel({ onClose, requestedMessage, + openChatRequest, newChatSeq, promotedPrompt, + watchRequest, + onChatRead, + onUnreadWorkChange, + onTurnActivityChange, isFullscreen = false, onToggleFullscreen, }: { @@ -74,8 +86,15 @@ export function DashboardAgentPanel({ onToggleFullscreen?: () => void; // Every `seq` below distinguishes repeat requests with identical contents. requestedMessage?: { text: string; seq: number }; + openChatRequest?: { chatId: string; seq: number }; newChatSeq?: number; promotedPrompt?: SuggestedPrompt; + watchRequest?: { spec: WatchSpec; seq: number }; + onChatRead?: (chatId: string, options: { leaving: boolean }) => void; + /** How many chats still hold work their owner hasn't seen. */ + onUnreadWorkChange?: (count: number) => void; + /** Whether a turn is running in a chat, so a closed panel still knows to expect an answer. */ + onTurnActivityChange?: (chatId: string, active: boolean) => void; }) { const organization = useOrganization(); const project = useProject(); @@ -89,7 +108,12 @@ export function DashboardAgentPanel({ const actionPath = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/dashboard-agent`; const storageKey = lastChatStorageKey(organization.id); + const panelRef = useRef(null); + // Declared before the chat plumbing: changing chat dispatches into it. + const [watchCard, dispatchWatchCard] = useReducer(watchCardReducer, NO_WATCH_CARD); const [chats, setChats] = useState([]); + // Until the list has arrived, the page load's server count is the better answer. + const [chatsLoaded, setChatsLoaded] = useState(false); const [active, setActive] = useState(null); // Starts true so an `openWith` request waits for the restore instead of racing it. const [loading, setLoading] = useState( @@ -119,14 +143,21 @@ export function DashboardAgentPanel({ ); const [thinkingChatId, setThinkingChatId] = useState(null); - const handleActivityChange = useCallback((chatId: string, activity: TurnActivity | null) => { - setThinkingChatId((previous) => - activity !== null ? chatId : previous === chatId ? null : previous - ); - }, []); + const handleActivityChange = useCallback( + (chatId: string, activity: TurnActivity | null) => { + setThinkingChatId((previous) => + activity !== null ? chatId : previous === chatId ? null : previous + ); + onTurnActivityChange?.(chatId, activity !== null); + }, + [onTurnActivityChange] + ); const historyInFlight = useRef | null>(null); + // The read POST and its reload can land out of order, so mask the next list. + const justRead = useRef>(new Set()); + const loadHistory = useCallback(async () => { if (historyInFlight.current) return historyInFlight.current; const request = (async () => { @@ -134,7 +165,19 @@ export function DashboardAgentPanel({ const res = await fetch(actionPath); if (!res.ok) throw new Error(`History request failed (${res.status})`); const data = (await res.json()) as { chats?: DashboardAgentChatListItem[] }; - setChats(data.chats ?? []); + const read = justRead.current; + justRead.current = new Set(); + const chats = data.chats ?? []; + // Reloaded after every turn and after a watch is created, so this is where the browser + // learns whether the wake feed is worth polling. + const pending = chats.some((chat) => chat.hasActiveWatch || chat.hasUnreadWake); + if (pending) rememberWatchActivity(organization.id); + else forgetWatchActivity(organization.id); + const settled = chats.map((chat) => + read.has(chat.id) ? { ...chat, hasUnreadWake: false, hasUnreadWork: false } : chat + ); + setChats(settled); + setChatsLoaded(true); } catch (error) { console.error("Dashboard agent: failed to load chat history", error); toast.error("We couldn't load your previous chats. Try again in a moment."); @@ -144,14 +187,21 @@ export function DashboardAgentPanel({ })(); historyInFlight.current = request; return request; - }, [actionPath, toast]); + }, [actionPath, organization.id, toast]); // Bumped on each open so a slower earlier open can't overwrite a newer one. const openChatRequestSeq = useRef(0); + // The one way the panel changes chat: it invalidates any in-flight open and abandons a + // half-configured watch card, which would otherwise be submitted against the new chat. + const claimChatSlot = useCallback(() => { + dispatchWatchCard({ type: "chat-changed" }); + return ++openChatRequestSeq.current; + }, []); + const openChat = useCallback( async (id: string) => { - const seq = ++openChatRequestSeq.current; + const seq = claimChatSlot(); setLoading(true); try { const res = await fetch(`${actionPath}?chatId=${encodeURIComponent(id)}`); @@ -171,12 +221,12 @@ export function DashboardAgentPanel({ if (seq === openChatRequestSeq.current) setLoading(false); } }, - [actionPath, toast] + [actionPath, claimChatSlot, toast] ); const createChat = useCallback( async (text: string) => { - const seq = ++openChatRequestSeq.current; + const seq = claimChatSlot(); setLoading(true); try { const userMessage: UIMessage = { @@ -217,7 +267,7 @@ export function DashboardAgentPanel({ if (seq === openChatRequestSeq.current) setLoading(false); } }, - [actionPath, clientData, toast] + [actionPath, claimChatSlot, clientData, toast] ); const restored = useRef(false); @@ -240,12 +290,24 @@ export function DashboardAgentPanel({ useEffect(() => { if (panelOrg.current === organization.id) return; panelOrg.current = organization.id; - openChatRequestSeq.current += 1; + claimChatSlot(); setActive(null); setLoading(false); setChats([]); + setChatsLoaded(false); void loadHistory(); - }, [organization.id, loadHistory]); + }, [organization.id, claimChatSlot, loadHistory]); + + const handledOpenChatSeq = useRef(undefined); + useEffect(() => { + if (!openChatRequest || handledOpenChatSeq.current === openChatRequest.seq) return; + handledOpenChatSeq.current = openChatRequest.seq; + // Reloading the visible transcript would drop a turn in flight. + if (openChatRequest.chatId === active?.chatId) return; + void openChat(openChatRequest.chatId); + // `active` is read, not tracked: a later change must not re-run the request. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [openChatRequest, openChat]); useEffect(() => { if (!active?.chatId) return; @@ -259,28 +321,148 @@ export function DashboardAgentPanel({ } }, [active?.chatId, storageKey, location.pathname]); + useEffect(() => { + if (!active?.chatId) return; + const chatId = active.chatId; + onChatRead?.(chatId, { leaving: false }); + justRead.current.add(chatId); + setChats((previous) => markChatListRead(previous, chatId)); + // Read again on the way out: a wake can land while the chat is open. + return () => { + onChatRead?.(chatId, { leaving: true }); + justRead.current.add(chatId); + setChats((previous) => markChatListRead(previous, chatId)); + }; + }, [active?.chatId, onChatRead]); + + // The one source for the dot's work count: nudging it per open double-subtracts. + useEffect(() => { + if (!chatsLoaded) return; + onUnreadWorkChange?.(unreadWorkCount(chats)); + }, [chats, chatsLoaded, onUnreadWorkChange]); + // Bound to its chat, which remounts with a fresh guard ref on every switch. - const [prefill, setPrefill] = useState<{ text: string; seq: number; chatId: string } | undefined>( - undefined - ); + const [sendRequest, setSendRequest] = useState< + { text: string; seq: number; chatId: string } | undefined + >(undefined); const handledRequestSeq = useRef(undefined); useEffect(() => { - if (!requestedMessage || loading) return; - if (handledRequestSeq.current === requestedMessage.seq) return; + if (!requestedMessage || handledRequestSeq.current === requestedMessage.seq) return; + const target = explicitPromptTarget({ + chat: loading ? "opening" : active ? "open" : "none", + turnInFlight: thinkingChatId !== null && thinkingChatId === active?.chatId, + }); + // Held requests are re-asked by this same effect once the panel settles. + if (target === "hold") return; handledRequestSeq.current = requestedMessage.seq; - if (active) { - setPrefill({ ...requestedMessage, chatId: active.chatId }); - } else { + if (target === "new-chat") { void createChat(requestedMessage.text); + return; + } + setSendRequest({ ...requestedMessage, chatId: active!.chatId }); + }, [requestedMessage, loading, active, thinkingChatId, createChat]); + + // Carries its chat id so a later-mounted chat cannot adopt another chat's block. + const [appendedMessages, setAppendedMessages] = useState< + { chatId: string; messages: UIMessage[]; seq: number } | undefined + >(undefined); + + const handledWatchSeq = useRef(undefined); + useEffect(() => { + if (!watchRequest || handledWatchSeq.current === watchRequest.seq) return; + handledWatchSeq.current = watchRequest.seq; + dispatchWatchCard({ + type: "open", + draft: watchDraftFor(watchRequest.spec), + requestId: generateFriendlyId("wreq"), + }); + }, [watchRequest]); + + // Nothing is posted or persisted until the card is submitted. + const openWatchCard = useCallback((spec: WatchSpec) => { + dispatchWatchCard({ + type: "open", + draft: watchDraftFor(spec), + requestId: generateFriendlyId("wreq"), + }); + }, []); + + const dismissWatchCard = useCallback(() => dispatchWatchCard({ type: "dismissed" }), []); + + const submitWatch = useCallback(async () => { + const draft = watchCard.draft; + if (!draft) return; + // Held across retries, so a resubmit repairs the same pair of records. + const clientRequestId = watchCard.requestId ?? generateFriendlyId("wreq"); + dispatchWatchCard({ type: "submitting", requestId: clientRequestId }); + try { + const body = new FormData(); + body.set("intent", "watch-create"); + body.set("draft", JSON.stringify(draft)); + body.set("clientRequestId", clientRequestId); + // A watch is chat-bound: with no chat open the server creates one. + if (active?.chatId) body.set("chatId", active.chatId); + + const res = await fetch(actionPath, { method: "POST", body }); + const data = (await res.json()) as { + chatId?: string; + messages?: UIMessage[]; + error?: string; + }; + if (!res.ok || !data.chatId || !data.messages) { + dispatchWatchCard({ + type: "failed", + error: data.error ?? "We couldn't start that watch. Try again in a moment.", + }); + return; + } + + const messages = data.messages; + if (active?.chatId === data.chatId) { + setAppendedMessages((current) => ({ + chatId: data.chatId!, + messages, + seq: (current?.seq ?? 0) + 1, + })); + dispatchWatchCard({ type: "submitted" }); + } else { + claimChatSlot(); + // No session: nothing is streaming and the records are the whole chat. + setActive({ chatId: data.chatId, messages, session: null }); + } + void loadHistory(); + } catch (error) { + console.error("Dashboard agent: failed to create watch", error); + dispatchWatchCard({ + type: "failed", + error: "We couldn't start that watch. Try again in a moment.", + }); } - }, [requestedMessage, loading, active, createChat]); + }, [ + watchCard.draft, + watchCard.requestId, + active?.chatId, + actionPath, + claimChatSlot, + loadHistory, + ]); + + const watchCardElement = watchCard.draft ? ( + dispatchWatchCard({ type: "edit", draft })} + onSubmit={() => void submitWatch()} + onCancel={dismissWatchCard} + pending={watchCard.pending} + error={watchCard.error} + /> + ) : null; const newChat = useCallback(() => { - // Invalidate any in-flight open or create so its result can't replace the draft. - openChatRequestSeq.current += 1; + claimChatSlot(); setLoading(false); setActive(null); - }, []); + }, [claimChatSlot]); const switchChat = useCallback( (id: string) => { @@ -317,16 +499,54 @@ export function DashboardAgentPanel({ [actionPath, active?.chatId, newChat, loadHistory, toast] ); + const cancelWatch = useCallback( + async (watchId: string) => { + const chatId = active?.chatId; + if (!chatId) return; + setChats((previous) => + previous.map((chat) => + chat.id === chatId + ? { ...chat, watches: (chat.watches ?? []).filter((watch) => watch.id !== watchId) } + : chat + ) + ); + const body = new FormData(); + body.set("intent", "watch-cancel"); + body.set("chatId", chatId); + body.set("watchId", watchId); + try { + const res = await fetch(actionPath, { method: "POST", body }); + if (!res.ok) throw new Error(`Watch cancel failed (${res.status})`); + } catch (error) { + console.error("Dashboard agent: failed to cancel watch", error); + toast.error("We couldn't stop that watch. Try again in a moment."); + } + void loadHistory(); + }, + [actionPath, active?.chatId, loadHistory, toast] + ); + // Titles are written when the first turn settles, so a new chat has none yet. const activeChat = active ? chats.find((chat) => chat.id === active.chatId) : undefined; const headerTitle = active ? (activeChat?.title ?? "Chat") : "New chat"; + // Not filtered to active: the wake banner needs watches that already fired. + const chatWatches = activeChat?.watches ?? []; + return (
{ - if (event.key !== "Escape" || event.defaultPrevented) return; + if ( + !escapeClosesPanel({ + key: event.key, + defaultPrevented: event.defaultPrevented, + targetInsidePanel: panelRef.current?.contains(event.target as Node) ?? false, + }) + ) + return; event.preventDefault(); onClose(); }} @@ -360,7 +580,9 @@ export function DashboardAgentPanel({ session={active.session} pendingFirstMessage={active.pendingFirstMessage} streaming={active.streaming} - prefill={prefill && prefill.chatId === active.chatId ? prefill : undefined} + sendRequest={ + sendRequest && sendRequest.chatId === active.chatId ? sendRequest : undefined + } clientData={clientData} apiOrigin={apiOrigin} actionPath={actionPath} @@ -368,7 +590,14 @@ export function DashboardAgentPanel({ environmentSlug={environment.slug} currentPage={currentPage} promotedPrompt={promotedPrompt} + watches={chatWatches} pagePaths={pagePaths} + watchCard={watchCardElement} + appendedMessages={ + appendedMessages?.chatId === active.chatId ? appendedMessages : undefined + } + onWatchIntent={openWatchCard} + onCancelWatch={cancelWatch} // The generated chat name is written before the turn-complete chunk lands. onTurnSettled={loadHistory} onActivityChange={handleActivityChange} @@ -381,6 +610,7 @@ export function DashboardAgentPanel({ currentPage={currentPage} pageContext={pageContext} promotedPrompt={promotedPrompt} + watchCard={watchCardElement} /> )} diff --git a/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx b/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx index c7f935b1d9b..b9a53192ba9 100644 --- a/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx +++ b/apps/webapp/app/components/dashboard-agent/DashboardAgentSuggestedPrompts.tsx @@ -1,6 +1,7 @@ import { BookOpenIcon, ChartBarIcon, + EyeIcon, MagnifyingGlassIcon, QuestionMarkCircleIcon, SparklesIcon, @@ -22,6 +23,7 @@ export const PROMPT_SLOT_BUTTON: Record< > = { promoted: { variant: "primary/small", icon: SparklesIcon }, investigate: { variant: "primary/small", icon: MagnifyingGlassIcon }, + watch: { variant: "secondary/small", icon: EyeIcon }, status: { variant: "secondary/small", icon: ChartBarIcon }, explain: { variant: "tertiary/small", icon: QuestionMarkCircleIcon }, docs: { variant: "docs/small", icon: BookOpenIcon }, diff --git a/apps/webapp/app/components/dashboard-agent/ReportView.tsx b/apps/webapp/app/components/dashboard-agent/ReportView.tsx index ae5eff78f88..33b8218801e 100644 --- a/apps/webapp/app/components/dashboard-agent/ReportView.tsx +++ b/apps/webapp/app/components/dashboard-agent/ReportView.tsx @@ -32,6 +32,7 @@ import { type ReportMessages } from "~/presenters/v3/reports/report-messages"; import { AgentBadge } from "./agent-badges"; import { seriesEndMs as toSeriesEndMs } from "./report-spark"; import { + FOOTER_WATCH_CODE, ReportBody, ReportCard, ReportFindingLine, @@ -53,6 +54,9 @@ import { export type ResolvedUri = { label: string; url: string }; +/** How often a recovery watch polls, and how long it lives. Aggregate conditions floor at 5m. */ +const RECOVERY_WATCH = { checkEveryMinutes: 5, maxHours: 6 } as const; + // --- messages --------------------------------------------------------------- /** @@ -313,6 +317,22 @@ export function ReportView({ const linkByKey = (key: string | undefined) => key === undefined ? undefined : vm.links.find((link) => link.key === key)?.url; + // Only offered when there is something to recover from, and only for the health + // report, which is the one with a recovery watch kind. + const recoveryWatch: AgentIntent | null = + vm.title === "health" && (severity === "warn" || severity === "crit") + ? { + kind: "watch", + spec: { + kind: "health_recovery", + report: "health", + fromSeverity: severity, + note: `${vm.scope} health back to normal`, + ...RECOVERY_WATCH, + }, + } + : null; + // Links a footer action already speaks for aren't repeated as reading matter. const footerLinkKeys = new Set(layout.footer.map((entry) => entry.link).filter(Boolean)); @@ -327,6 +347,22 @@ export function ReportView({ }), })); + if (recoveryWatch && onIntent) { + const watchItem: ReportFooterItem = { + code: FOOTER_WATCH_CODE, + // The label is deliberately the same everywhere; only the pre-filled spec is + // contextual, so a per-object label would break the pattern. + node: onIntent(recoveryWatch)}>Watch…, + }; + // The watch joins the other buttons, before the trailing prose entry. + const noteIndex = footerItems.findIndex((item) => reportFooterStyle(item.code) === "note"); + if (noteIndex !== -1) { + footerItems.splice(noteIndex, 0, watchItem); + } else { + footerItems.push(watchItem); + } + } + // Resources the report cites, resolved to dashboard links by the host. Cited, // not offered, so a text link; our docs still get the docs button. for (const link of vm.links) { diff --git a/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx b/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx new file mode 100644 index 00000000000..0a54ddd894f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WakeBanner.tsx @@ -0,0 +1,156 @@ +/** + * The banner above a wake narration. + * + * This component holds no kind-specific wording: tone, semantic icon and headline + * come from contracts and `app/presenters/v3/dashboardAgent`. All it decides is which glyph a + * semantic icon draws and which frame a tone paints. + * + * A wake is identified by its message id, `wake:watch:{watchId}:{fired|expired}`. + * That suffix is the transport encoding, not the outcome; the outcome comes off the + * watch row. + */ +import { + CheckCircleIcon, + ClockIcon, + ExclamationCircleIcon, + ExclamationTriangleIcon, + InformationCircleIcon, +} from "@heroicons/react/20/solid"; +import type { + WatchObservedOutcome, + WatchResolution, + WatchSemanticIcon, +} from "@internal/dashboard-agent-contracts"; +import { cn } from "~/utils/cn"; +import { type AgentTone, TONE_ICON_COLOR } from "./agent-badges"; +import { + presentResolvedWatch, + watchSubline, + WATCH_PRESENTATION_FALLBACK, +} from "~/presenters/v3/dashboardAgent"; + +const WAKE_ID_PREFIX = "wake:watch:"; + +/** + * The wire encoding in a wake's message id, not the resolution: `window_completed` + * and `condition_impossible` are both addressed as `expired`, and the row is the + * authority on which one it was. + */ +export type WakeOutcome = "fired" | "expired"; + +/** The watch fields a banner can use. A `WatchChip` satisfies it. */ +export type WakeWatch = { + id: string; + kind: string; + note: string; + identity: string; + /** How the watch ended. Absent on a row written before the resolution model. */ + resolution?: WatchResolution | null; + /** What the resolving check observed — the other half of the headline. */ + observedOutcome?: WatchObservedOutcome | null; + /** + * Why the watch ended, from its last result. Only used to reconstruct a + * resolution for rows that predate the `resolution` column. + */ + endedReason?: string | null; +}; + +export type WakeRef = { watchId: string; outcome: WakeOutcome }; + +/** + * The watch a message narrates the wake of, or null when the message isn't a wake. + * A watch id never ends in an outcome word, so splitting on the last colon is + * unambiguous. + */ +export function wakeRefFromMessageId(messageId: string): WakeRef | null { + if (!messageId.startsWith(WAKE_ID_PREFIX)) return null; + const rest = messageId.slice(WAKE_ID_PREFIX.length); + const split = rest.lastIndexOf(":"); + if (split <= 0) return null; + const outcome = rest.slice(split + 1); + if (outcome !== "fired" && outcome !== "expired") return null; + return { watchId: rest.slice(0, split), outcome }; +} + +/** The watch a wake belongs to, when the host passed its watches down. */ +export function findWakeWatch(watches: WakeWatch[] | undefined, watchId: string) { + return watches?.find((watch) => watch.id === watchId); +} + +/** + * The watch's resolution, falling back to what the transport can prove for a row + * written before the `resolution` column existed: `fired` is unambiguous, `expired` + * splits on the last check's reason. + */ +export function wakeResolution( + outcome: WakeOutcome, + watch: Pick | undefined +): WatchResolution { + if (watch?.resolution) return watch.resolution; + if (outcome === "fired") return "condition_met"; + return watch?.endedReason === "terminal_unsatisfied" + ? "condition_impossible" + : "window_completed"; +} + +/** What this banner shows, without the markup. */ +export function wakePresentation(outcome: WakeOutcome, watch: WakeWatch | undefined) { + if (!watch) return WATCH_PRESENTATION_FALLBACK; + return presentResolvedWatch({ + kind: watch.kind, + identity: watch.identity, + resolution: wakeResolution(outcome, watch), + observed: watch.observedOutcome ?? null, + }); +} + +/** + * Semantic icon to glyph. Which icon a resolved result deserves is decided in + * contracts, and the rule there is that the icon follows the observed outcome, not + * the resolution: a failed run gets `error`, not the check its `condition_met` + * would suggest. + */ +const SEMANTIC_ICON: Record JSX.Element> = { + success: CheckCircleIcon, + attention: ExclamationTriangleIcon, + error: ExclamationCircleIcon, + waiting: ClockIcon, + info: InformationCircleIcon, +}; + +const TONE_FRAME: Record = { + neutral: "border-l-border-bright bg-background-bright/40", + success: "border-l-success bg-success/10", + warning: "border-l-warning bg-warning/10", + error: "border-l-error bg-error/10", +}; + +export function WakeBanner({ + outcome, + watch, +}: { + /** The wire encoding from the wake's message id. */ + outcome: WakeOutcome; + /** The watch that woke, when the host has it. Absent: the neutral fallback. */ + watch?: WakeWatch; +}) { + const presentation = wakePresentation(outcome, watch); + const tone = presentation.tone as AgentTone; + const Icon = SEMANTIC_ICON[presentation.semanticIcon]; + const note = watchSubline(watch); + + return ( +
+ +
+

+ {presentation.label} +

+

{presentation.headline}

+ {note ?

{note}

: null} +
+
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchButton.tsx b/apps/webapp/app/components/dashboard-agent/WatchButton.tsx new file mode 100644 index 00000000000..25b000cd656 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchButton.tsx @@ -0,0 +1,45 @@ +import { EyeIcon } from "@heroicons/react/20/solid"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { Button } from "~/components/primitives/Buttons"; +import { useDashboardAgent } from "./dashboardAgentLauncher"; +import { watchTooltipLabel } from "~/presenters/v3/dashboardAgent"; + +/** Posts nothing: opens the panel with the card pre-filled. Renders nothing without a provider. */ +export function WatchButton({ + spec, + label = "Watch…", + size = "small", + variant = "secondary", + fullWidth, + className, + tooltip, +}: { + spec: WatchSpec; + label?: string; + size?: "small" | "medium"; + variant?: "primary" | "secondary" | "minimal"; + fullWidth?: boolean; + className?: string; + tooltip?: string; +}) { + const agent = useDashboardAgent(); + if (!agent) { + return null; + } + + return ( + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchCard.tsx b/apps/webapp/app/components/dashboard-agent/WatchCard.tsx new file mode 100644 index 00000000000..e395dcecf10 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchCard.tsx @@ -0,0 +1,330 @@ +/** + * The watch configuration card, opened by the Watch action. + * + * Rules it keeps: the card is ephemeral until submitted (it lives in the panel, + * not the transcript, and only a submitted outcome is persisted as a + * `watch_result` block); Customize expands in place, never a modal; in-chat + * delivery is stated as a line, so the two opt-ins stay independent checkboxes + * and never become a radio group. + * + * Pure component: draft in, markup and callbacks out. Draft rules live in + * `watch-card.ts` and wording in `app/presenters/v3/dashboardAgent`. + */ +import { EyeIcon } from "@heroicons/react/20/solid"; +import { + WATCH_WINDOW_HOURS_OPTIONS, + watchCadenceOptions, + type WatchDraft, + type WatchKind, +} from "@internal/dashboard-agent-contracts"; +import { useId, useState } from "react"; +import { Button } from "~/components/primitives/Buttons"; +import { Checkbox } from "~/components/primitives/Checkbox"; +import { Input } from "~/components/primitives/Input"; +import { AgentSpinner } from "~/components/primitives/Spinner"; +import { cn } from "~/utils/cn"; +import { ChatSystemBlock } from "./chat-layout"; +import { + variantsOf, + watchDraftError, + withAgeMinutes, + withCadence, + withFollowUp, + withThreshold, + withVariant, + withWindow, +} from "./watch-card"; +import { + formatWatchCadence, + formatWatchWindow, + WATCH_IN_CHAT_DELIVERY_LINE, + watchConditionLabel, + watchDurationLabel, + watchSubjectLabel, +} from "~/presenters/v3/dashboardAgent"; + +/** How the condition variants are named in the picker. Short, not sentences. */ +const VARIANT_LABEL: Record = { + run_start: "when it starts", + run_finished: "when it finishes", + run_failed: "if it fails", + backlog_drain: "when it drains", + queue_depth_above: "if it grows", + queue_depth_below: "when it's back below", + queue_stalled: "if it stops moving", + queue_oldest_age: "if runs wait too long", + error_recurrence: "if it recurs", + health_recovery: "when it recovers", +}; + +/** Hoisted so the submit button's icon component keeps a stable identity. */ +function ButtonSpinner() { + return ; +} + +/** Controlled, unlike `CheckboxWithLabel`: the draft is the only thing that says what's on. */ +function Toggle({ + label, + checked, + disabled, + onChange, +}: { + label: string; + checked: boolean; + disabled: boolean; + onChange: (checked: boolean) => void; +}) { + const id = useId(); + return ( +
+ onChange(event.target.checked)} + className="mt-1" + /> + +
+ ); +} + +/** One choice in an inline picker. */ +function Choice({ + selected, + disabled, + onSelect, + children, +}: { + selected: boolean; + disabled: boolean; + onSelect: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ {label} +
{children}
+
+ ); +} + +export function WatchCard({ + draft, + onChange, + onSubmit, + onCancel, + /** Start expanded: the gallery's Customize state, and a free-text pre-fill. */ + defaultExpanded = false, + /** The submit is in flight: the card stays, disabled, so nothing moves. */ + pending = false, + /** A refusal from the server (cap, duplicate, network). */ + error, +}: { + draft: WatchDraft; + onChange: (draft: WatchDraft) => void; + onSubmit: () => void; + onCancel?: () => void; + defaultExpanded?: boolean; + pending?: boolean; + error?: string | null; +}) { + const [expanded, setExpanded] = useState(defaultExpanded); + const { spec } = draft; + const variants = variantsOf(draft); + // Local validation first: a draft the schema would refuse never reaches the server. + const localError = watchDraftError(draft); + const blocked = localError !== null || pending; + + return ( + } + actions={ + <> + {/* One confirm, expanded or not: an expanded card is submitted as shown. */} + + {!expanded ? ( + + ) : null} + {onCancel ? ( + + ) : null} + + } + > +

+ Watch {watchSubjectLabel(spec)} +

+ {!expanded ? ( + <> +

{watchConditionLabel(spec)}

+

{watchDurationLabel(spec)}

+ + ) : null} +

{WATCH_IN_CHAT_DELIVERY_LINE}

+ + {expanded ? ( +
+ {/* Kinds with no second condition variant must not show an empty picker. */} + {variants.length > 1 ? ( + + {variants.map((kind) => ( + { + if (kind !== spec.kind) onChange(withVariant(draft, kind)); + }} + > + {VARIANT_LABEL[kind]} + + ))} + + ) : ( + + {watchConditionLabel(spec)} + + )} + + {/* One contextual parameter per condition, only where one exists. */} + {spec.kind === "queue_depth_above" || spec.kind === "queue_depth_below" ? ( + + + onChange(withThreshold(draft, Number.parseInt(event.target.value, 10))) + } + aria-label="Queue depth threshold" + /> + + ) : null} + + {spec.kind === "queue_oldest_age" ? ( + + + onChange(withAgeMinutes(draft, Number.parseInt(event.target.value, 10))) + } + aria-label="Wait limit in minutes" + /> + minutes + + ) : null} + + + {WATCH_WINDOW_HOURS_OPTIONS.map((hours) => ( + onChange(withWindow(draft, hours))} + > + {formatWatchWindow(hours)} + + ))} + + + {/* Cadence options come from the kind's schema limits, so an aggregate + watch can never be offered a 1-minute hot loop. */} + + {watchCadenceOptions(spec.kind).map((minutes) => ( + onChange(withCadence(draft, minutes))} + > + {formatWatchCadence(minutes)} + + ))} + + + {/* Two independent opt-ins under a fixed delivery line, never a radio + group, so "email instead of chat" is not expressible. */} + +
+ + onChange(withFollowUp(draft, { investigateOnAttention: checked })) + } + /> + onChange(withFollowUp(draft, { notifyExternally: checked }))} + /> +
+ +
+ ) : null} + + {/* Errors live and die with the card: nothing is persisted. */} + {localError || error ? ( +

{localError ?? error}

+ ) : null} + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchChips.test.ts b/apps/webapp/app/components/dashboard-agent/WatchChips.test.ts new file mode 100644 index 00000000000..4b15efc29fc --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchChips.test.ts @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +/** + * There is no rendering harness here, so this pins the prop that decides the tab order + * rather than the tab order itself: `SimpleTooltip` sets `tabIndex={-1}` unless `tabbable` + * is passed. What it does not prove is that focus actually opens the tooltip. + */ +describe("the watch chip's tooltips are reachable by keyboard", () => { + const source = readFileSync(new URL("./WatchChips.tsx", import.meta.url), "utf8"); + + it("marks both tabbable — the label one carries status, cadence and expiry", () => { + expect(source.match(/ JSX.Element> = { + success: CheckCircleIcon, + attention: ExclamationTriangleIcon, + error: ExclamationCircleIcon, + waiting: ClockIcon, + info: InformationCircleIcon, +}; + +/** + * A terminal chip wears the resolved result's icon, not its lifecycle status: a + * `run_finished` watch on a failed run resolves `condition_met`. Cancellation has none. + */ +function StatusIcon({ watch }: { watch: WatchChip }) { + if (watch.status === "active") return ; + + if (watch.status === "cancelled") { + return ; + } + + const presentation = wakePresentation(watch.status === "fired" ? "fired" : "expired", watch); + const Icon = SEMANTIC_ICON[presentation.semanticIcon]; + return ( + + ); +} + +export function WatchChips({ + watches, + onCancel, +}: { + watches: WatchChip[]; + onCancel?: (watchId: string) => void; +}) { + if (watches.length === 0) return null; + + return ( +
+ watches + {watches.map((watch) => { + const label = watchChipLabel(watch); + return ( + + + {label}} + /> + {watch.status === "active" && onCancel ? ( + onCancel(watch.id)} + className="text-text-faint transition-colors hover:text-error focus-visible:text-error focus-custom" + > + + + } + /> + ) : null} + + ); + })} +
+ ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx b/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx new file mode 100644 index 00000000000..551a9683ef5 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchResultBlock.tsx @@ -0,0 +1,48 @@ +/** + * What a submitted watch card leaves in the transcript, in two flavours. + * + * A confirmation states the watch's lifetime facts and is the only transcript record + * of the request. A one-shot result means the immediate check answered outright and + * no watch was created, so no chip appears, no wake arrives and there is nothing to + * cancel. + * + * Pure component: the wording is not computed here, it was frozen into the block at + * append time by `app/presenters/v3/dashboardAgent`, so a later copy change never rewrites what + * a user was already told. + */ +import { CheckCircleIcon, EyeIcon, InformationCircleIcon } from "@heroicons/react/20/solid"; +import type { WatchResultBlock as WatchResultBlockPayload } from "@internal/dashboard-agent-contracts"; +import { ChatSystemBlock } from "./chat-layout"; +import { TONE_ICON_COLOR } from "./agent-badges"; +import { cn } from "~/utils/cn"; + +/** + * Icon and label per outcome. A confirmation is not a success (nothing has happened + * yet) so it wears the neutral eye; the check belongs to the one-shot that did + * answer the question. + */ +const OUTCOME = { + watching: { label: "Watch", Icon: EyeIcon, tone: "neutral" }, + already_true: { label: "Watch", Icon: CheckCircleIcon, tone: "success" }, + impossible: { label: "Watch", Icon: InformationCircleIcon, tone: "neutral" }, +} as const; + +export function WatchResultBlock({ block }: { block: WatchResultBlockPayload }) { + const { label, Icon, tone } = OUTCOME[block.outcome] ?? OUTCOME.watching; + + return ( + } + > +

{block.headline}

+ {block.lifetime ?

{block.lifetime}

: null} + {block.detail ?

{block.detail}

: null} + {(block.followUp ?? []).map((line) => ( +

+ {line} +

+ ))} + + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx b/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx new file mode 100644 index 00000000000..8170c37b07f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/WatchWakeToast.tsx @@ -0,0 +1,133 @@ +/** + * The dashboard-wide signal that a watch woke a chat while the panel was closed. + * + * Persistent by design: a wake answers a question asked minutes or hours ago, so it + * waits until dismissed rather than expiring on a timer. Dismissing does not mark + * the chat read (reading happens in the panel), so the launcher's dot survives a + * swatted toast. + */ +import { toast } from "sonner"; +import { Button } from "~/components/primitives/Buttons"; +import { ToastUI } from "~/components/primitives/Toast"; +import type { WatchObservedOutcome, WatchResolution } from "@internal/dashboard-agent-contracts"; +import { wakeResolution } from "./WakeBanner"; +import { presentResolvedWatch, WATCH_PRESENTATION_FALLBACK } from "~/presenters/v3/dashboardAgent"; + +/** Matches sonner's default toast width, same as the app's other toasts. */ +const TOAST_WIDTH = 356; + +/** More new wakes than this at once collapse into one summary toast. */ +export const WAKE_TOAST_MAX_INDIVIDUAL = 3; + +export type WatchWake = { + watchId: string; + chatId: string; + /** The wire encoding off the row. Not the outcome; see `resolution`. */ + outcome: "fired" | "expired"; + note: string; + /** + * What actually happened, frozen on the row by the resolving check. The toast, + * the banner and the email take their headline from the same presenter so they + * cannot disagree. Absent on a row written before the resolution model, where the + * presenter falls back rather than guessing. + */ + kind?: string; + identity?: string; + resolution?: WatchResolution | null; + observedOutcome?: WatchObservedOutcome | null; + /** Landed after the chat's read marker. The dot counts these; the toast fires either way. */ + unread?: boolean; +}; + +/** + * The toast's title: the fact, or the neutral fallback when this wake predates the + * resolution model. The wording is `app/presenters/v3/dashboardAgent`'s; this only decides + * which watch to ask it about. + */ +export function watchWakeToastTitle(wake: WatchWake): string { + if (!wake.kind || !wake.identity) return WATCH_PRESENTATION_FALLBACK.headline; + return presentResolvedWatch({ + kind: wake.kind, + identity: wake.identity, + resolution: wakeResolution(wake.outcome, { resolution: wake.resolution ?? null }), + observed: wake.observedOutcome ?? null, + }).headline; +} + +function WakeToastUI({ + t, + title, + message, + onOpenChat, +}: { + t: string; + title: string; + message: string; + onOpenChat: () => void; +}) { + return ( + { + onOpenChat(); + toast.dismiss(t); + }} + > + Open chat + + } + /> + ); +} + +function show(node: (t: string) => React.ReactElement, id: string) { + toast.custom((t) => node(t as string), { + // Manual dismissal only — see the file comment. + duration: Infinity, + // Keyed so a re-render or a duplicate poll can't stack the same wake twice. + id, + }); +} + +/** + * One persistent toast for a single wake. `onOpenChat` is given the chat the wake + * happened in, not whichever chat the panel had open last. + */ +export function showWatchWakeToast(wake: WatchWake, onOpenChat: (chatId: string) => void) { + show( + (t) => ( + onOpenChat(wake.chatId)} + /> + ), + `watch-wake-${wake.watchId}` + ); +} + +/** One persistent toast standing in for a batch too large to narrate one by one. */ +export function showWatchWakesSummaryToast(count: number, onOpenChat: () => void) { + show( + (t) => ( + + ), + // One id for all summaries: a later poll rewrites the count in place instead + // of stacking a second never-expiring toast on top of the first. + "watch-wakes-summary" + ); +} diff --git a/apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts b/apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts index 8dde4872db1..56d27ecb7f5 100644 --- a/apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts +++ b/apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { hotkeyOptions } from "~/hooks/useShortcutKeys"; -import { LEGACY_ASK_AI_SHORTCUT, TOGGLE_PANEL_SHORTCUT } from "./dashboardAgentLauncher"; +import { ASK_AI_SHORTCUT } from "./ask-ai-channels"; +import { TOGGLE_PANEL_SHORTCUT } from "./dashboardAgentLauncher"; const enabled = { isEnabled: true }; @@ -20,9 +21,9 @@ describe("the agent's shortcuts", () => { }); it("leaves Cmd-I's default alone", () => { - expect(hotkeyOptions({ shortcut: LEGACY_ASK_AI_SHORTCUT, ...enabled }).preventDefault).toBe( - false - ); + expect(ASK_AI_SHORTCUT.key).toBe("i"); + expect(ASK_AI_SHORTCUT.modifiers).toEqual(["mod"]); + expect(hotkeyOptions({ shortcut: ASK_AI_SHORTCUT, ...enabled }).preventDefault).toBe(false); }); }); diff --git a/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts b/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts index 3dbe69435aa..cb268f7f924 100644 --- a/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts +++ b/apps/webapp/app/components/dashboard-agent/chat-layout.test.ts @@ -80,6 +80,7 @@ describe("chat-layout enforcement", () => { "ChatToolRow", "ChatNote", "ChatStatusLine", + "ChatWakeSlot", "ChatActionsRow", ]) { expect(source, name).toContain(`export function ${name}(`); diff --git a/apps/webapp/app/components/dashboard-agent/chat-layout.tsx b/apps/webapp/app/components/dashboard-agent/chat-layout.tsx index 7139f24b07b..d281f64b6a5 100644 --- a/apps/webapp/app/components/dashboard-agent/chat-layout.tsx +++ b/apps/webapp/app/components/dashboard-agent/chat-layout.tsx @@ -12,6 +12,7 @@ const TURN_GAP = "space-y-4"; const TURN_BODY_GAP = "space-y-2"; const ROW_GAP = "gap-2"; const CHIP_GAP = "gap-1.5"; +const UNIT_GAP = "space-y-1.5"; const SCROLLER = "flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"; @@ -148,6 +149,21 @@ export function ChatStatusLine({ ); } +export function ChatWakeSlot({ + banner, + children, +}: { + banner: React.ReactNode; + children: React.ReactNode; +}) { + return ( +
+ {banner} + {children} +
+ ); +} + const BLOCK_LINE_GAP = "space-y-1"; const BLOCK_INSET = "px-3 py-2.5"; diff --git a/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx b/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx index 48542381fde..052824aa25c 100644 --- a/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx +++ b/apps/webapp/app/components/dashboard-agent/dashboardAgentLauncher.tsx @@ -1,3 +1,4 @@ +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; import { createContext, useContext } from "react"; import { Button } from "~/components/primitives/Buttons"; import { ShortcutKey } from "~/components/primitives/ShortcutKey"; @@ -21,6 +22,12 @@ type DashboardAgentContextValue = { setOpen: (open: boolean) => void; /** Sent as the first message of a new chat; with a chat open it only fills the composer. */ openWith: (text: string) => void; + /** Nothing is posted or persisted until the card is submitted. */ + openWithWatch: (spec: WatchSpec) => void; + /** Polled only while the panel is closed; 0 while it is open. */ + unreadWakes: number; + /** Chats that answered, settled or woke while the panel was closed. */ + unreadWork: number; }; const DashboardAgentContext = createContext(null); @@ -38,11 +45,13 @@ export function DashboardAgentLauncher() { return null; } - const { open, setOpen } = agent; + const { open, setOpen, unreadWakes, unreadWork } = agent; if (open) { return null; } + const hasUnread = unreadWakes > 0 || unreadWork > 0; + return ( + {hasUnread && ( + // The ring matches the `NavBar` surface the launcher sits on. + + )} } /> diff --git a/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts b/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts index 3fd4699146d..2cae37107b9 100644 --- a/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts +++ b/apps/webapp/app/components/dashboard-agent/demo/fixtures/page-context.ts @@ -31,7 +31,7 @@ export const demoConcurrencySaturationSignal: AgentPageSignal = { severity: "crit", }; -// Priority order. +// Priority order. `SIGNAL_PRIORITY` in the registry mirrors this. export const demoSignalsByPriority: AgentPageSignal[] = [ demoFreshFailureSignal, demoWaitingRunSignal, @@ -141,6 +141,12 @@ export const demoPromptSets: Record = { "How many other runs failed with this error in the last hour?", "contextual" ), + prompt( + "watch-retry", + "Tell me when it retries", + `Watch ${DEMO_WORLD.failedRunId} and tell me when it finishes.`, + "contextual" + ), DEFAULT_PROMPTS[1]!, ], waitingRun: [ @@ -195,6 +201,12 @@ export const demoPromptSets: Record = { "Explain this error and what usually causes it.", "promoted" ), + prompt( + "watch-recurrence", + "Tell me if it comes back", + "Watch this error and tell me if it happens again.", + "contextual" + ), DEFAULT_PROMPTS[1]!, ], queue: [ @@ -224,7 +236,7 @@ export const demoPromptSets: Record = { other: DEFAULT_PROMPTS, }; -export const demoDismissedPromptIds: string[] = []; +export const demoDismissedPromptIds: string[] = [demoId("prompt-watch-retry")]; export const demoPromptsAfterDismissal: SuggestedPrompt[] = demoPromptSets.failedRun .filter((p) => !demoDismissedPromptIds.includes(p.id)) diff --git a/apps/webapp/app/components/dashboard-agent/explicit-prompt.test.ts b/apps/webapp/app/components/dashboard-agent/explicit-prompt.test.ts new file mode 100644 index 00000000000..f25023be6de --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/explicit-prompt.test.ts @@ -0,0 +1,62 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { explicitPromptTarget } from "./explicit-prompt"; + +describe("explicitPromptTarget", () => { + it("starts a chat when the panel has none", () => { + expect(explicitPromptTarget({ chat: "none", turnInFlight: false })).toBe("new-chat"); + }); + + it("sends into the chat the user is already in", () => { + expect(explicitPromptTarget({ chat: "open", turnInFlight: false })).toBe("send-to-open-chat"); + }); + + it("holds while a chat is still opening, rather than racing it into a new one", () => { + expect(explicitPromptTarget({ chat: "opening", turnInFlight: false })).toBe("hold"); + }); + + it("holds while the open chat is mid-turn instead of barging in", () => { + expect(explicitPromptTarget({ chat: "open", turnInFlight: true })).toBe("hold"); + }); + + it("never fills the composer and leaves the sending to the user", () => { + const targets = (["none", "opening", "open"] as const).flatMap((chat) => + [true, false].map((turnInFlight) => explicitPromptTarget({ chat, turnInFlight })) + ); + expect(targets).not.toContain("prefill"); + }); +}); + +/** + * Structural guards, not behavioural proof: whether a held request is asked again, and whether + * the old prefill path is really gone, live in the wiring rather than in the rule. + */ +describe("the panel sends every explicit prompt", () => { + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + const chat = readFileSync(new URL("./DashboardAgentChat.tsx", import.meta.url), "utf8"); + + it("routes through the shared rule", () => { + expect(panel).toContain("explicitPromptTarget({"); + }); + + it("keeps a held request pending instead of marking it handled", () => { + const effect = panel.slice(panel.indexOf("const target = explicitPromptTarget({")); + expect(effect.indexOf('if (target === "hold") return;')).toBeLessThan( + effect.indexOf("handledRequestSeq.current = requestedMessage.seq;") + ); + }); + + it("re-asks once the panel settles, so a hold cannot strand the prompt", () => { + expect(panel).toContain("}, [requestedMessage, loading, active, thinkingChatId, createChat]);"); + }); + + it("leaves no prefill path behind", () => { + expect(panel).not.toMatch(/prefill/i); + expect(chat).not.toMatch(/prefill/i); + }); + + it("submits the request in the chat rather than typing it into the composer", () => { + expect(chat).toContain("submit(sendRequest.text);"); + expect(chat).not.toContain("setInput(sendRequest.text)"); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/explicit-prompt.ts b/apps/webapp/app/components/dashboard-agent/explicit-prompt.ts new file mode 100644 index 00000000000..749ee67380e --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/explicit-prompt.ts @@ -0,0 +1,16 @@ +/** What to do with a prompt the user asked for by clicking, rather than by typing. */ +export type ExplicitPromptTarget = "new-chat" | "send-to-open-chat" | "hold"; + +/** + * A click on Investigate or a prompt chip always ends in a sent message; only where it lands + * depends on the panel. `hold` is not a refusal — the request stays pending and is asked again + * once the chat has opened or its turn has finished. + */ +export function explicitPromptTarget(panel: { + chat: "none" | "opening" | "open"; + turnInFlight: boolean; +}): ExplicitPromptTarget { + if (panel.chat === "opening") return "hold"; + if (panel.chat === "none") return "new-chat"; + return panel.turnInFlight ? "hold" : "send-to-open-chat"; +} diff --git a/apps/webapp/app/components/dashboard-agent/list-row.tsx b/apps/webapp/app/components/dashboard-agent/list-row.tsx index 6fe35f267e3..a5b80d873ff 100644 --- a/apps/webapp/app/components/dashboard-agent/list-row.tsx +++ b/apps/webapp/app/components/dashboard-agent/list-row.tsx @@ -20,6 +20,7 @@ export function AgentListRow({ meta, status, variant = "default", + unread = false, onSelect, action, }: { @@ -27,6 +28,7 @@ export function AgentListRow({ meta?: ReactNode; status?: ReactNode; variant?: AgentListRowVariant; + unread?: boolean; onSelect: () => void; /** Use {@link AgentListRowAction}. */ action?: ReactNode; @@ -38,12 +40,19 @@ export function AgentListRow({ onClick={onSelect} className={cn( "flex min-w-0 flex-1 items-center gap-2 rounded-md border px-3 py-2 text-left text-sm outline-hidden transition focus-custom", - ROW_VARIANTS[variant] + ROW_VARIANTS[variant], + unread && "text-text-bright" )} > {status ? ( {status} ) : null} + {unread ? ( + <> + + Unread. + + ) : null} {label} {meta ? {meta} : null} diff --git a/apps/webapp/app/components/dashboard-agent/message-quota.ts b/apps/webapp/app/components/dashboard-agent/message-quota.ts index 21edb9ab9fb..f65481c8705 100644 --- a/apps/webapp/app/components/dashboard-agent/message-quota.ts +++ b/apps/webapp/app/components/dashboard-agent/message-quota.ts @@ -1,3 +1,5 @@ +import { isWatchRequestMessageId } from "@internal/dashboard-agent-contracts"; + // Counted per user across their chats in the org, not per chat, which "New chat" // would reset. export const FREE_PLAN_MESSAGE_LIMIT = 20; @@ -25,6 +27,12 @@ export function resolveMessageQuota({ : { kind: "within", used, limit, remaining }; } -export function countUserMessages(messages: { role: string }[]): number { - return messages.reduce((total, message) => (message.role === "user" ? total + 1 : total), 0); +// A watch's consent record is a user message the person never typed, so it is +// excluded here exactly as the stored count excludes it. +export function countUserMessages(messages: { role: string; id?: string }[]): number { + return messages.reduce( + (total, message) => + message.role === "user" && !isWatchRequestMessageId(message.id) ? total + 1 : total, + 0 + ); } diff --git a/apps/webapp/app/components/dashboard-agent/panel-escape.test.ts b/apps/webapp/app/components/dashboard-agent/panel-escape.test.ts new file mode 100644 index 00000000000..cfb57828394 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/panel-escape.test.ts @@ -0,0 +1,75 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { escapeClosesPanel } from "./panel-escape"; + +/** + * Escape has to reach the thing the user meant. Radix dismisses a popover or a dialog from a + * document listener that runs after the panel's own handler and never marks the event handled, + * so the panel has to decide for itself whether the keystroke came from inside it. + */ +describe("escapeClosesPanel", () => { + it("closes the panel when Escape comes from the panel itself", () => { + expect( + escapeClosesPanel({ key: "Escape", defaultPrevented: false, targetInsidePanel: true }) + ).toBe(true); + }); + + it("leaves the panel open when Escape comes from a portalled layer", () => { + // The history popover and the delete dialog both render outside the panel's DOM subtree. + expect( + escapeClosesPanel({ key: "Escape", defaultPrevented: false, targetInsidePanel: false }) + ).toBe(false); + }); + + it("stays out of the way once something else has handled the key", () => { + expect( + escapeClosesPanel({ key: "Escape", defaultPrevented: true, targetInsidePanel: true }) + ).toBe(false); + }); + + it("ignores every other key", () => { + expect( + escapeClosesPanel({ key: "Enter", defaultPrevented: false, targetInsidePanel: true }) + ).toBe(false); + expect(escapeClosesPanel({ key: "j", defaultPrevented: false, targetInsidePanel: true })).toBe( + false + ); + }); +}); + +/** + * Structural guards, not behavioural proof: the delete confirmation's survival depends on where + * it is mounted in the tree, which these assertions pin down without rendering anything. + */ +describe("the delete confirmation lives outside the history popover", () => { + const header = readFileSync(new URL("./DashboardAgentHeader.tsx", import.meta.url), "utf8"); + const history = readFileSync(new URL("./DashboardAgentHistory.tsx", import.meta.url), "utf8"); + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + + const menuBody = history.slice( + history.indexOf("export function DashboardAgentHistoryMenu"), + history.indexOf("export function DashboardAgentDeleteChatDialog") + ); + + it("keeps no dialog and no pending state inside the popover's menu", () => { + expect(menuBody).not.toContain(" { + const popoverEnd = header.indexOf(""); + const dialog = header.indexOf(" { + expect(header).toContain("const [pendingDelete, setPendingDelete] = useState"); + }); + + it("gates the panel's Escape on the shared rule rather than defaultPrevented alone", () => { + expect(panel).toContain("escapeClosesPanel({"); + expect(panel).toContain("panelRef.current?.contains(event.target as Node)"); + expect(panel).not.toContain('if (event.key !== "Escape" || event.defaultPrevented) return;'); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/panel-escape.ts b/apps/webapp/app/components/dashboard-agent/panel-escape.ts new file mode 100644 index 00000000000..201d7b09c18 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/panel-escape.ts @@ -0,0 +1,15 @@ +/** + * Escape inside the panel closes the panel — but a popover or a dialog is portalled out of + * the panel's DOM subtree while still bubbling through the React tree, and Radix dismisses + * those from a document listener that runs after this handler, so the event arrives here + * undefaulted. Deciding on the DOM target is what tells the two apart. + */ +export function escapeClosesPanel(event: { + key: string; + defaultPrevented: boolean; + /** Whether the event's target is a DOM descendant of the panel. */ + targetInsidePanel: boolean; +}): boolean { + if (event.key !== "Escape" || event.defaultPrevented) return false; + return event.targetInsidePanel; +} diff --git a/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts b/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts index 6cbc02d3573..2ff0d61403f 100644 --- a/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts +++ b/apps/webapp/app/components/dashboard-agent/pending-intents.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { pendingNavigateIntents } from "./pending-intents"; +import { pendingNavigateIntents, pendingWatchIntents } from "./pending-intents"; describe("pendingNavigateIntents", () => { const uri = "trigger://proj_abc/env_123/run/run_abc"; @@ -46,3 +46,75 @@ describe("pendingNavigateIntents", () => { ).toEqual([{ kind: "navigate", target: uri }]); }); }); + +describe("pendingWatchIntents", () => { + const spec = { + kind: "run_finished", + runId: "run_abc", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when the receipt run finishes", + }; + const toolPart = (toolCallId: string, state = "output-available") => ({ + type: "tool-schedule_watch", + state, + toolCallId, + output: { intent: { kind: "watch", spec } }, + }); + + it("returns the proposed spec from a completed schedule_watch call, once", () => { + const seen = new Set(); + const messages = [{ id: "m1", parts: [toolPart("call-1")] }]; + + expect(pendingWatchIntents(messages, seen)).toEqual([{ kind: "watch", spec }]); + expect(pendingWatchIntents(messages, seen)).toEqual([]); + }); + + it("ignores a call still running, and a spec the contract rejects", () => { + expect( + pendingWatchIntents([{ id: "m1", parts: [toolPart("call-1", "input-available")] }], new Set()) + ).toEqual([]); + + const invalid = [ + { + id: "m1", + parts: [ + { + ...toolPart("call-2"), + output: { intent: { kind: "watch", spec: { kind: "run_finished" } } }, + }, + ], + }, + ]; + expect(pendingWatchIntents(invalid, new Set())).toEqual([]); + }); + + it("never reopens a proposal seeded from loaded history", () => { + const history = [{ id: "m1", parts: [toolPart("call-1")] }]; + const seen = new Set(); + pendingWatchIntents(history, seen); + + expect(pendingWatchIntents(history, seen)).toEqual([]); + expect( + pendingWatchIntents([...history, { id: "m2", parts: [toolPart("call-2")] }], seen) + ).toEqual([{ kind: "watch", spec }]); + }); + + it("doesn't confuse a navigate result for a watch", () => { + const messages = [ + { + id: "m1", + parts: [ + { + type: "tool-navigate_to", + state: "output-available", + toolCallId: "call-1", + output: { intent: { kind: "navigate", target: "trigger://p/e/run/run_abc" } }, + }, + ], + }, + ]; + + expect(pendingWatchIntents(messages, new Set())).toEqual([]); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/pending-intents.ts b/apps/webapp/app/components/dashboard-agent/pending-intents.ts index 6b58ba04d31..93a1d790656 100644 --- a/apps/webapp/app/components/dashboard-agent/pending-intents.ts +++ b/apps/webapp/app/components/dashboard-agent/pending-intents.ts @@ -40,3 +40,11 @@ export function pendingNavigateIntents( ): Array> { return pendingToolIntents(messages, seen, "tool-navigate_to", "navigate"); } + +// `schedule_watch` only proposes: the panel creates the watch. +export function pendingWatchIntents( + messages: ReadonlyArray, + seen: Set +): Array> { + return pendingToolIntents(messages, seen, "tool-schedule_watch", "watch"); +} diff --git a/apps/webapp/app/components/dashboard-agent/pending-turn.test.ts b/apps/webapp/app/components/dashboard-agent/pending-turn.test.ts new file mode 100644 index 00000000000..0dee765fb80 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/pending-turn.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { nextPendingTurnChatId } from "./pending-turn"; +import { shouldPollWakeFeed } from "./watch-activity"; + +/** + * The launcher dot only appears if the wake poll is running when the answer lands. A turn + * started in the panel has to keep the poll alive across a close, and let go of it once the + * answer has been seen. + */ +describe("nextPendingTurnChatId", () => { + it("latches onto the chat whose turn started", () => { + expect(nextPendingTurnChatId(null, { chatId: "chat_a", active: true })).toBe("chat_a"); + }); + + it("holds while a newer turn takes over", () => { + const afterA = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(nextPendingTurnChatId(afterA, { chatId: "chat_b", active: true })).toBe("chat_b"); + }); + + it("lets go once that chat's turn is no longer running", () => { + const pending = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(nextPendingTurnChatId(pending, { chatId: "chat_a", active: false })).toBe(null); + }); + + it("keeps waiting when a different chat goes quiet", () => { + const pending = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(nextPendingTurnChatId(pending, { chatId: "chat_b", active: false })).toBe("chat_a"); + }); + + it("stays clear when nothing is pending", () => { + expect(nextPendingTurnChatId(null, { chatId: "chat_a", active: false })).toBe(null); + }); +}); + +describe("a turn started behind a closed panel", () => { + // The page load knew of nothing: no wake, no watch, no unread work. Only the turn can + // start the poll. + const quietPageLoad = { + serverUnreadWakes: 0, + serverHasActiveWatches: false, + serverUnreadWork: 0, + organizationId: "org_quiet", + }; + + it("keeps the poll running until the answer is seen", () => { + expect(shouldPollWakeFeed({ ...quietPageLoad, turnInFlight: false })).toBe(false); + + // Asked a question, then closed the panel: the panel reports no end, so the latch holds. + const pending = nextPendingTurnChatId(null, { chatId: "chat_a", active: true }); + expect(shouldPollWakeFeed({ ...quietPageLoad, turnInFlight: pending !== null })).toBe(true); + + // Re-opened the chat with the turn already over. + const seen = nextPendingTurnChatId(pending, { chatId: "chat_a", active: false }); + expect(shouldPollWakeFeed({ ...quietPageLoad, turnInFlight: seen !== null })).toBe(false); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/pending-turn.ts b/apps/webapp/app/components/dashboard-agent/pending-turn.ts new file mode 100644 index 00000000000..d024d19ea59 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/pending-turn.ts @@ -0,0 +1,17 @@ +/** + * Which chat this tab is still waiting on. A turn started here can finish after the panel + * closes — the case the launcher dot exists for — so the wake poll has to keep running until + * the answer has been seen. The panel reports turn activity while it is mounted; closing it + * reports nothing, which is what leaves the latch set. + */ + +/** `active` is true while a turn is running in `chatId`, false once it is not. */ +export function nextPendingTurnChatId( + current: string | null, + event: { chatId: string; active: boolean } +): string | null { + if (event.active) return event.chatId; + // Only the chat we are waiting on clears the latch; another chat going quiet says nothing + // about this one. + return current === event.chatId ? null : current; +} diff --git a/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx b/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx index eb1c6a84c1a..8acd08e98c4 100644 --- a/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx +++ b/apps/webapp/app/components/dashboard-agent/report-sparkline.tsx @@ -311,6 +311,13 @@ export function ReportNoteBlock({ label, children }: { label: string; children: // and `note` is prose for an option stated rather than offered. export { reportFooterStyle, type ReportFooterStyle }; +/** + * The recovery-watch offer. No report emits it; the card adds it. Two codes + * because it is phrased differently when it is the only thing on offer. + */ +export const FOOTER_WATCH_CODE = "watch_recovery"; +export const FOOTER_WATCH_ONLY_CODE = "watch_recovery_only"; + /** A dimmed line that accompanies a row entry. */ const FOOTER_NOTE_LINES: Record = { check_control_plane: "There's nothing to fix on your side.", diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts index fe9e381e640..5e8c3ff2f47 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts @@ -206,7 +206,7 @@ describe("queueAgentPageContext", () => { const context = queueAgentPageContext(queueLoaderData()); expect(context).toEqual({ - page: { kind: "queue", name: "black-friday", health: "ok" }, + page: { kind: "queue", name: "black-friday", health: "ok", paused: false }, signals: [], }); expect(agentPageContextSchema.safeParse(context).success).toBe(true); @@ -244,6 +244,16 @@ describe("queueAgentPageContext", () => { expect(context?.signals).toEqual([]); }); + it("offers no watch on a paused queue, even when it is at capacity", () => { + // Paused and saturated at once: nothing will drain or grow until it is resumed, so a + // watch would promise an answer that can't come. + const context = queueAgentPageContext( + queueLoaderData({ paused: true, running: 10, queued: 40, concurrencyLimit: 10 }) + ); + + expect(context?.signals).toEqual([]); + }); + it("emits nothing for an unlimited queue, however deep the backlog", () => { const context = queueAgentPageContext( queueLoaderData({ concurrencyLimit: null, running: 99, queued: 99 }) diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts index e79f96d2d7f..1de69bec3d3 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.ts @@ -211,12 +211,13 @@ export function queueAgentPageContext(data: unknown): AgentPageContext | undefin const health = atCapacity ? "crit" : paused || queued > 0 || waitingTooLong ? "warn" : "ok"; const signals: AgentPageSignal[] = []; - if (atCapacity) { + // Nothing to watch on a paused queue: it can neither drain nor grow until it is resumed. + if (atCapacity && !paused) { // A backlog at least as deep as the limit won't clear this cycle. signals.push({ kind: "concurrency_saturation", severity: queued >= limit! ? "crit" : "warn" }); } - return { page: { kind: "queue", name, health }, signals }; + return { page: { kind: "queue", name, health, paused: Boolean(paused) }, signals }; } export function deploymentsAgentPageContext(): AgentPageContext { diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts index 8f594a1183c..1bc7aeb340d 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/page-prompts.ts @@ -97,6 +97,11 @@ export function pageSlotPrompts(page: AgentPage): PageSlotPrompts { "Why does this keep happening?", "Investigate this error — why does it keep coming back, and which runs are affected?" ), + watch: def( + "error-watch-recurrence", + "Tell me if it comes back", + "Watch this error and tell me if it happens again." + ), explain: def( "error-similar", "Find similar failures", @@ -117,14 +122,23 @@ export function pageSlotPrompts(page: AgentPage): PageSlotPrompts { case "queue": return { + // A paused queue is backed up because someone paused it, and nothing it could be + // watched for will happen until they resume it — so neither chip is offered. investigate: - page.health === "warn" || page.health === "crit" + !page.paused && (page.health === "warn" || page.health === "crit") ? def( "queue-backlog-cause", "Why is this queue backed up?", queueBacklogPrompt(page.name) ) : undefined, + watch: page.paused + ? undefined + : def( + "queue-watch-drain", + "Tell me when the backlog drains", + `Watch the ${page.name} queue and tell me when the backlog drains.` + ), status: def( "queue-backlog", "How big is the backlog?", diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts index aae8e0e4cf0..5e5f7e5690a 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/prompt-chips.ts @@ -25,12 +25,13 @@ export const ctx = (id: string, label: string, prompt: string) => make(id, label, prompt, "contextual"); /** The slots after the promoted one, in display order. */ -export const PROMPT_SLOTS = ["investigate", "status", "explain", "docs"] as const; +export const PROMPT_SLOTS = ["investigate", "watch", "status", "explain", "docs"] as const; export type PromptSlot = (typeof PROMPT_SLOTS)[number]; export type PageSlotPrompts = { investigate?: SuggestedPrompt; + watch?: SuggestedPrompt; status?: SuggestedPrompt; explain: SuggestedPrompt; docs: SuggestedPrompt; diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts index 80ce032b0bf..97f4ec558d2 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts @@ -25,27 +25,33 @@ const docsId = (key: keyof typeof demoPageContexts) => pageSlotPrompts(demoPageContexts[key].page).docs.id; describe("resolveSuggestedPrompts", () => { - it("fills every slot the page has, given a promoted chip, signals and defaults", () => { + it("fills all five slots when the page has a promoted chip, signals and defaults", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.error, { promoted, now: NOW }); - expect(prompts).toHaveLength(4); + expect(prompts).toHaveLength(5); expect(ids(prompts)).toEqual([ promoted.id, "sp:fresh-failure", + "sp:error-watch-recurrence", "sp:error-similar", docsId("error"), ]); expect(prompts[0]?.source).toBe("promoted"); }); - it("drops a slot when nothing is promoted", () => { + it("drops to four slots when nothing is promoted", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW }); - expect(prompts).toHaveLength(3); - expect(ids(prompts)).toEqual(["sp:fresh-failure", "sp:error-similar", docsId("error")]); + expect(prompts).toHaveLength(4); + expect(ids(prompts)).toEqual([ + "sp:fresh-failure", + "sp:error-watch-recurrence", + "sp:error-similar", + docsId("error"), + ]); }); - it("shows explain + docs only when no investigate applies", () => { + it("shows explain + docs only when no investigate or watch applies", () => { const prompts = resolveSuggestedPrompts(demoPageContexts.deployment, { now: NOW }); expect(ids(prompts)).toEqual(ids(pageDefaultPrompts(demoPageContexts.deployment.page))); @@ -73,7 +79,7 @@ describe("resolveSuggestedPrompts", () => { } }); - it("orders promoted, then investigate, then status, then explain", () => { + it("orders promoted, then investigate, then watch, then explain", () => { const context = { ...demoPageContexts.queue, signals: [...demoPageContexts.queue.signals, demoFreshFailureSignal], @@ -84,7 +90,8 @@ describe("resolveSuggestedPrompts", () => { expect(ids(prompts)).toEqual([ promoted.id, "sp:fresh-failure", - "sp:queue-backlog", + // waiting_run beats concurrency_saturation for the watch slot. + "sp:waiting-run", "sp:queue-state", docsId("queue"), ]); @@ -122,10 +129,10 @@ describe("resolveSuggestedPrompts", () => { const full = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW }); const dismissed = resolveSuggestedPrompts(demoPageContexts.error, { now: NOW, - dismissedIds: ["sp:error-similar"], + dismissedIds: ["sp:error-watch-recurrence"], }); - expect(ids(dismissed)).not.toContain("sp:error-similar"); + expect(ids(dismissed)).not.toContain("sp:error-watch-recurrence"); expect(dismissed).toHaveLength(full.length - 1); expect(dismissed.at(-1)?.id).toBe(docsId("error")); }); @@ -161,7 +168,12 @@ describe("resolveSuggestedPrompts", () => { expect(failure?.prompt).toContain("12m ago"); }); - it("words the slow-run chip for its slot", () => { + it("words the waiting-run and slow-run chips for their slots", () => { + const waiting = resolveSuggestedPrompts(demoPageContexts.waitingRun, { now: NOW }); + const waitingChip = waiting.find((p) => p.id === "sp:waiting-run"); + expect(waitingChip?.label).toBe("Tell me when this run starts"); + expect(waitingChip?.prompt).toContain("queue"); + const slow = resolveSuggestedPrompts(demoPageContexts.slowRun, { now: NOW }); expect(slow[0]?.label).toBe("~7.8x slower than usual"); }); @@ -223,6 +235,17 @@ describe("pageSlotPrompts", () => { } }); + it("offers a paused queue neither chip, however unhealthy it looks", () => { + // Paused reads as `warn`, so without the guard the backlog chips would both appear — + // asking why a queue someone paused is backed up, and offering to watch it drain. + const slots = pageSlotPrompts({ kind: "queue", name: "emails", health: "warn", paused: true }); + + expect(slots.investigate).toBeUndefined(); + expect(slots.watch).toBeUndefined(); + // The page is still explainable; only the two backlog asks are withheld. + expect(slots.explain).toBeDefined(); + }); + it("offers a deployment investigate chip only for a deploy that didn't land", () => { expect(pageSlotPrompts({ kind: "deployment", version: "1.0" }).investigate).toBeUndefined(); expect( diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts index 9264ae78690..b8575acd459 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.ts @@ -68,7 +68,7 @@ export function resolveSuggestedPromptsBySlot( } // Over the cap, optional slots yield in this order; promoted, explain and docs never yield. - const yieldOrder: ResolvedPromptSlot[] = ["status", "investigate"]; + const yieldOrder: ResolvedPromptSlot[] = ["status", "watch", "investigate"]; let trimmed = resolved; for (const slot of yieldOrder) { if (trimmed.length <= SUGGESTED_PROMPT_CAP) break; diff --git a/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts b/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts index d5faf17653f..6c4a2f74982 100644 --- a/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts +++ b/apps/webapp/app/components/dashboard-agent/suggested-prompts/signal-prompts.ts @@ -10,14 +10,20 @@ import type { } from "@internal/dashboard-agent-contracts"; import { ctx, type PromptSlot } from "./prompt-chips"; -/** A kind with no entry produces no chip. */ -export const SIGNAL_SLOT: Partial> = { +export const SIGNAL_SLOT: Record = { fresh_failure: "investigate", slow_run: "investigate", + waiting_run: "watch", + concurrency_saturation: "watch", }; -/** Signal precedence within a slot. */ -export const SIGNAL_PRIORITY: AgentPageSignalKind[] = ["fresh_failure", "slow_run"]; +/** Signal precedence within a slot. Mirrors `demoSignalsByPriority` in the fixtures. */ +export const SIGNAL_PRIORITY: AgentPageSignalKind[] = [ + "fresh_failure", + "waiting_run", + "slow_run", + "concurrency_saturation", +]; /** "3m", "2h", "4d". */ export function formatAgo(ms: number): string { @@ -47,6 +53,15 @@ export function promptForSignal(signal: AgentPageSignal, now: number): Suggested ); } + case "waiting_run": + return ctx( + "waiting-run", + "Tell me when this run starts", + signal.queue + ? `Watch ${signal.runId} and tell me when it leaves the ${signal.queue} queue.` + : `Watch ${signal.runId} and tell me when it starts running.` + ); + case "slow_run": { if (signal.baselineP95Ms <= 0) return undefined; const factor = formatMultiplier(signal.durationMs / signal.baselineP95Ms); @@ -56,6 +71,13 @@ export function promptForSignal(signal: AgentPageSignal, now: number): Suggested `${signal.runId} is running ~${factor} slower than this task's usual p95. Investigate why.` ); } + + case "concurrency_saturation": + return ctx( + "concurrency-saturation", + "Tell me when the backlog drains", + "Concurrency is saturated right now. Watch it and tell me when the backlog drains." + ); } } @@ -79,18 +101,17 @@ export function contextualPromptsBySlot( ): Record { const bySlot: Record = { investigate: [], + watch: [], status: [], explain: [], docs: [], }; for (const kind of SIGNAL_PRIORITY) { - const slot = SIGNAL_SLOT[kind]; - if (!slot) continue; for (const signal of context.signals) { if (signal.kind !== kind) continue; const prompt = promptForSignal(signal, now); - if (prompt) bySlot[slot].push(prompt); + if (prompt) bySlot[SIGNAL_SLOT[kind]].push(prompt); } } diff --git a/apps/webapp/app/components/dashboard-agent/tool-labels.ts b/apps/webapp/app/components/dashboard-agent/tool-labels.ts index d3bd07f52d4..4972cdb2d5b 100644 --- a/apps/webapp/app/components/dashboard-agent/tool-labels.ts +++ b/apps/webapp/app/components/dashboard-agent/tool-labels.ts @@ -21,6 +21,7 @@ const TOOL_LABELS: Record = { search_docs: "Searching the docs", get_current_page: "Reading the current page", navigate_to: "Opening the page", + schedule_watch: "Filling in a watch", list_alerts: "Listing alerts", create_alert: "Creating an alert", delete_alert: "Deleting an alert", diff --git a/apps/webapp/app/components/dashboard-agent/turn-error.test.ts b/apps/webapp/app/components/dashboard-agent/turn-error.test.ts index 9975bdd1f49..4bf646e4fbb 100644 --- a/apps/webapp/app/components/dashboard-agent/turn-error.test.ts +++ b/apps/webapp/app/components/dashboard-agent/turn-error.test.ts @@ -7,7 +7,7 @@ const failure = { id: "turn-error:0" }; describe("the failed-turn record", () => { it("recognises the agent's failure message id", () => { expect(isTurnErrorMessageId("turn-error:3")).toBe(true); - expect(isTurnErrorMessageId("msg_1")).toBe(false); + expect(isTurnErrorMessageId("wake:watch:watch_1:fired")).toBe(false); expect(isTurnErrorMessageId(undefined)).toBe(false); }); diff --git a/apps/webapp/app/components/dashboard-agent/turn-error.ts b/apps/webapp/app/components/dashboard-agent/turn-error.ts index 1fecf35c1ba..0a451401088 100644 --- a/apps/webapp/app/components/dashboard-agent/turn-error.ts +++ b/apps/webapp/app/components/dashboard-agent/turn-error.ts @@ -1,7 +1,8 @@ /** * A failed turn is recorded in the transcript by the agent, under the message id - * `turn-error:{turn}`. The prefix is the transport convention, recognised here so - * the panel can tell a stored failure record apart from an ordinary answer. + * `turn-error:{turn}`. Same arrangement as a wake's `wake:watch:…` id: the prefix + * is the transport convention, recognised here so the panel can tell a stored + * failure record apart from an ordinary answer. * * Live, a failure arrives as the stream's error chunk and `useChat` surfaces it as * the retry callout. The stored record is what a reload reads. Both must never show diff --git a/apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts b/apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts new file mode 100644 index 00000000000..49813a1a0da --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts @@ -0,0 +1,113 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { navigateIntentApplies, takeNavigateIntent } from "./turn-navigation"; + +const runs = "/orgs/acme/projects/api/env/prod/runs"; +const queues = "/orgs/acme/projects/api/env/prod/queues"; + +describe("navigateIntentApplies", () => { + it("navigates when the user is still where the turn was asked for", () => { + expect(navigateIntentApplies({ startedPath: runs, currentPath: runs })).toBe(true); + }); + + it("drops the navigation once the user has walked to another screen", () => { + expect(navigateIntentApplies({ startedPath: runs, currentPath: queues })).toBe(false); + }); + + it("drops it when this tab never saw the turn start", () => { + // A resumed turn: nothing here knows the page it was asked on. + expect(navigateIntentApplies({ startedPath: null, currentPath: runs })).toBe(false); + }); +}); + +describe("takeNavigateIntent", () => { + const target = "trigger://proj_abc/env_123/run/run_abc"; + + function messages() { + return [ + { + id: "msg_1", + parts: [ + { + type: "tool-navigate_to", + state: "output-available", + toolCallId: "call_1", + output: { intent: { kind: "navigate", target } }, + }, + ], + }, + ]; + } + + it("takes the navigation on the page the turn was asked for", () => { + const taken = takeNavigateIntent({ + messages: messages(), + handled: new Set(), + startedPath: runs, + currentPath: runs, + }); + expect(taken).toMatchObject({ kind: "navigate", target }); + }); + + it("takes nothing once the user has walked to another screen", () => { + expect( + takeNavigateIntent({ + messages: messages(), + handled: new Set(), + startedPath: runs, + currentPath: queues, + }) + ).toBeUndefined(); + }); + + // The property the panel depends on: a commit that drops a navigation still consumes it, so + // walking back to the page it was asked on does not make it fire late. + it("marks a dropped navigation handled, so a later commit cannot fire it", () => { + const handled = new Set(); + const parts = messages(); + + expect( + takeNavigateIntent({ messages: parts, handled, startedPath: runs, currentPath: queues }) + ).toBeUndefined(); + + expect( + takeNavigateIntent({ messages: parts, handled, startedPath: runs, currentPath: runs }) + ).toBeUndefined(); + }); +}); + +/** + * Structural guards, not behavioural proof: whether the started-at path is still right when the + * intent lands depends on effect order and on nothing clearing it, which these assertions pin + * down without rendering anything. + */ +describe("the chat scopes a turn's navigation to the page it started on", () => { + const chat = readFileSync(new URL("./DashboardAgentChat.tsx", import.meta.url), "utf8"); + + it("gates the navigate intent on the shared rule", () => { + expect(chat).toContain("takeNavigateIntent({"); + expect(chat).toContain("startedPath: turnStartedPathRef.current"); + expect(chat).not.toContain("pendingNavigateIntents(messages"); + }); + + it("records the path only as a turn goes in flight", () => { + expect(chat).toContain( + "if (inFlight && !turnWasInFlight.current) turnStartedPathRef.current = renderedPathRef.current;" + ); + }); + + it("never clears the path on settle, which can share a commit with the intent", () => { + const assignments = [...chat.matchAll(/turnStartedPathRef\.current = /g)]; + expect(assignments).toHaveLength(1); + }); + + it("records the path before the intent effect reads it", () => { + expect(chat.indexOf("turnStartedPathRef.current = renderedPathRef.current")).toBeLessThan( + chat.indexOf("takeNavigateIntent({") + ); + }); + + it("hands the persistent handled-set in, so drops are recorded across commits", () => { + expect(chat).toContain("handled: navigatedRef.current!"); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/turn-navigation.ts b/apps/webapp/app/components/dashboard-agent/turn-navigation.ts new file mode 100644 index 00000000000..de08e278ce9 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/turn-navigation.ts @@ -0,0 +1,33 @@ +import { pendingNavigateIntents } from "./pending-intents"; + +/** + * The panel follows the user around the dashboard, so a turn can outlive the page it was asked + * on. Its navigation applies only there: someone who has since walked to another screen keeps + * the screen they chose, and the answer's own button is still theirs to click. + */ +export function navigateIntentApplies(paths: { + /** Null when this tab never saw the turn start, so it cannot claim the user is still there. */ + startedPath: string | null; + currentPath: string; +}): boolean { + return paths.startedPath === paths.currentPath; +} + +type NavigateIntent = ReturnType[number]; + +/** + * The navigation to take on this commit, if any. Every intent is marked handled whether or not + * it applies, so one dropped here cannot fire on a later commit. + */ +export function takeNavigateIntent(args: { + messages: Parameters[0]; + handled: Set; + startedPath: string | null; + currentPath: string; +}): NavigateIntent | undefined { + const target = pendingNavigateIntents(args.messages, args.handled).at(-1); + if (!target) return undefined; + return navigateIntentApplies({ startedPath: args.startedPath, currentPath: args.currentPath }) + ? target + : undefined; +} diff --git a/apps/webapp/app/components/dashboard-agent/turn-teardown.test.ts b/apps/webapp/app/components/dashboard-agent/turn-teardown.test.ts new file mode 100644 index 00000000000..e31a4910fbf --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/turn-teardown.test.ts @@ -0,0 +1,73 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { teardownCancelsTurn, unmountTeardown } from "./turn-teardown"; + +describe("teardownCancelsTurn", () => { + it("cancels when the user clicks Stop", () => { + expect(teardownCancelsTurn("stop-clicked")).toBe(true); + }); + + it("keeps the turn when the panel closes", () => { + expect(teardownCancelsTurn("panel-closed")).toBe(false); + }); + + it("keeps the turn when the panel changes chat", () => { + expect(teardownCancelsTurn("chat-switched")).toBe(false); + }); + + it("cancels when the user has left the page", () => { + expect(teardownCancelsTurn("navigated-away")).toBe(true); + }); +}); + +describe("unmountTeardown", () => { + const path = "/orgs/acme/projects/api/env/prod/runs"; + + it("reads an unmount on the same path as the panel closing", () => { + expect(unmountTeardown({ renderedPath: path, livePath: path })).toBe("panel-closed"); + }); + + it("reads an unmount after the URL moved as a navigation", () => { + expect(unmountTeardown({ renderedPath: path, livePath: "/orgs/acme/settings" })).toBe( + "navigated-away" + ); + }); +}); + +/** + * Structural guards, not behavioural proof: the wiring depends on when React runs the cleanup + * relative to the router, which these assertions pin down without rendering anything. + */ +describe("the chat cancels its turn only on the teardowns that say so", () => { + const chat = readFileSync(new URL("./DashboardAgentChat.tsx", import.meta.url), "utf8"); + + it("decides through the shared rule rather than unmounting straight into a stop", () => { + expect(chat).toContain("teardownCancelsTurn("); + expect(chat).toContain("unmountTeardown({"); + }); + + it("compares the last rendered path against the live one", () => { + expect(chat).toContain("renderedPath: renderedPathRef.current"); + expect(chat).toContain("livePath: window.location.pathname"); + }); + + // Where "filtering a page is not leaving it" actually lives: both sides are pathnames, so a + // query string never reaches the comparison. Widen either side and a filter change reads as a + // navigation, cancelling the turn. + it("tracks the rendered path without its query string", () => { + expect(chat).toContain("useRef(location.pathname)"); + expect(chat).toContain("renderedPathRef.current = location.pathname;"); + expect(chat).not.toContain("location.search"); + }); + + it("runs the cleanup once, not on every path change", () => { + const teardown = chat.slice(chat.indexOf("const teardownRef")); + expect(teardown).toMatch( + /useEffect\(\s*\(\)\s*=>\s*\(\)\s*=>\s*teardownRef\.current\(\),\s*\[\]\)/ + ); + }); + + it("cancels nothing when no turn is in flight", () => { + expect(chat).toContain('if (status !== "streaming" && status !== "submitted") return;'); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/turn-teardown.ts b/apps/webapp/app/components/dashboard-agent/turn-teardown.ts new file mode 100644 index 00000000000..48307a3b76b --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/turn-teardown.ts @@ -0,0 +1,18 @@ +/** Why a chat with a turn in flight is going away. */ +export type TurnTeardown = "stop-clicked" | "panel-closed" | "chat-switched" | "navigated-away"; + +/** + * A turn that finishes behind a closed panel is what the launcher dot exists for, so only a + * deliberate stop and leaving the page end it early. + */ +export function teardownCancelsTurn(reason: TurnTeardown): boolean { + return reason === "stop-clicked" || reason === "navigated-away"; +} + +/** + * The three unmounts look identical from inside React. Only a navigation has already moved the + * URL by the time the cleanup runs; the other two keep the turn, so they share one branch. + */ +export function unmountTeardown(paths: { renderedPath: string; livePath: string }): TurnTeardown { + return paths.renderedPath === paths.livePath ? "panel-closed" : "navigated-away"; +} diff --git a/apps/webapp/app/components/dashboard-agent/unread-counts.test.ts b/apps/webapp/app/components/dashboard-agent/unread-counts.test.ts new file mode 100644 index 00000000000..48c4f07e527 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/unread-counts.test.ts @@ -0,0 +1,113 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { + markChatListRead, + nextVisibleChat, + unreadWorkCount, + unreadWorkForDot, +} from "./unread-counts"; + +const list = () => [ + { id: "chat_a", hasUnreadWake: true, hasUnreadWork: true }, + { id: "chat_b", hasUnreadWork: true }, + { id: "chat_c" }, +]; + +/** + * The dot counts chats, not visits. Reading a chat settles it in the list, and the list is + * what the count is derived from — so one visit, or ten, subtracts the same one chat. + */ +describe("the work count is derived from the list", () => { + it("counts every chat still holding unseen work", () => { + expect(unreadWorkCount(list())).toBe(2); + }); + + it("subtracts a read chat once, however many times it is read", () => { + const once = markChatListRead(list(), "chat_a"); + expect(unreadWorkCount(once)).toBe(1); + // The read effect fires on entry and again on cleanup, and again on every revisit. + const again = markChatListRead(markChatListRead(once, "chat_a"), "chat_a"); + expect(unreadWorkCount(again)).toBe(1); + }); + + it("reaches zero only when every chat has been read", () => { + const all = ["chat_a", "chat_b", "chat_c"].reduce(markChatListRead, list()); + expect(unreadWorkCount(all)).toBe(0); + }); + + it("settles the wake alongside the work, so the row stops looking unread", () => { + const read = markChatListRead(list(), "chat_a"); + expect(read[0]).toEqual({ id: "chat_a", hasUnreadWake: false, hasUnreadWork: false }); + // Every other chat is left exactly as it was. + expect(read.slice(1)).toEqual(list().slice(1)); + }); +}); + +/** + * A wake in the chat on screen must not light the dot — but once the panel has let go of that + * chat, its wakes have to reach the dot again. + */ +describe("nextVisibleChat", () => { + it("holds the chat while it is on screen", () => { + expect(nextVisibleChat("chat_a", { leaving: false })).toBe("chat_a"); + }); + + it("lets go on the way out instead of restoring it", () => { + expect(nextVisibleChat("chat_a", { leaving: true })).toBeNull(); + }); +}); + +/** + * The chat on screen is being read, so it isn't work waiting for anyone — but only while the + * panel is actually open. + */ +describe("unreadWorkForDot", () => { + it("subtracts the chat the panel is showing", () => { + expect(unreadWorkForDot({ reported: 3, panelOpen: true, visibleChatId: "chat_a" })).toBe(2); + }); + + it("counts every chat when the panel is closed", () => { + expect(unreadWorkForDot({ reported: 3, panelOpen: false, visibleChatId: "chat_a" })).toBe(3); + expect(unreadWorkForDot({ reported: 3, panelOpen: true, visibleChatId: null })).toBe(3); + }); + + it("never reports a negative count, or one the poll didn't give", () => { + expect(unreadWorkForDot({ reported: 0, panelOpen: true, visibleChatId: "chat_a" })).toBe(0); + expect(unreadWorkForDot({ reported: undefined, panelOpen: false, visibleChatId: null })).toBe( + 0 + ); + }); +}); + +describe("what the panel and the layout actually do with it", () => { + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + const layout = readFileSync(new URL("./DashboardAgent.tsx", import.meta.url), "utf8"); + + it("reports the count from the list, and only from the list", () => { + expect(panel).toContain("onUnreadWorkChange?.(unreadWorkCount(chats));"); + expect(panel).not.toContain("settled.filter((chat) => chat.hasUnreadWork).length"); + expect(layout).not.toContain("setUnreadWork((count) => Math.max(0, count - 1))"); + }); + + it("tells the read effect's cleanup that it is leaving", () => { + expect(panel).toContain("onChatRead?.(chatId, { leaving: false });"); + expect(panel).toContain("onChatRead?.(chatId, { leaving: true });"); + expect(layout).toContain("visibleChat.current = nextVisibleChat(chatId, options);"); + }); + + /** + * The poll runs for as long as this tab is watching, so anything it reads about the panel has + * to come from a ref. `open` is state: the callback would keep the value it had when polling + * started, which is `false`, and the dot would go on counting the chat on screen. + */ + it("reads the panel's state at poll time, not from the closure", () => { + expect(layout).toContain("panelOpen: panelOpen.current,"); + expect(layout).not.toContain("open && visibleChat.current"); + }); + + it("re-seeds both counts when the environment changes under the layout", () => { + expect(layout).toContain("seededEnvironment.current = environment.id;"); + expect(layout).toContain("setUnreadWakes(initialUnreadWakes);"); + expect(layout).toContain("setUnreadWork(initialUnreadWork);"); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/unread-counts.ts b/apps/webapp/app/components/dashboard-agent/unread-counts.ts new file mode 100644 index 00000000000..ad4958d0e7f --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/unread-counts.ts @@ -0,0 +1,46 @@ +/** + * What the launcher's dot counts. + * + * The counts are derived from the chat list rather than nudged up and down as chats are + * opened: a decrement fires once per read effect and once per cleanup, and again on every + * revisit, none of which the server ever hears about. Reading a chat settles it in the list, + * and the list is what the dot counts — so the same chat read twice counts once. + */ + +type UnreadChat = { id: string; hasUnreadWake?: boolean; hasUnreadWork?: boolean }; + +/** + * The chat the panel has on screen. `leaving` is the read effect's cleanup: it runs after the + * panel has already let go, so restoring the id there would keep hiding that chat's wakes. + */ +export function nextVisibleChat(chatId: string, options: { leaving: boolean }): string | null { + return options.leaving ? null : chatId; +} + +/** + * The work count the launcher's dot shows, given what the poll just reported. A chat open in + * the panel is being read right now, so it is not work anyone is waiting on. + * + * Both inputs have to be read at poll time rather than captured when the poll started: the + * panel opens and closes without restarting it. + */ +export function unreadWorkForDot(params: { + reported: number | undefined; + panelOpen: boolean; + visibleChatId: string | null; +}): number { + const onScreen = params.panelOpen && params.visibleChatId !== null ? 1 : 0; + return Math.max(0, (params.reported ?? 0) - onScreen); +} + +/** Opening a chat settles everything unseen in it, not just the wake. */ +export function markChatListRead(chats: T[], chatId: string): T[] { + return chats.map((chat) => + chat.id === chatId ? { ...chat, hasUnreadWake: false, hasUnreadWork: false } : chat + ); +} + +/** How many chats still hold work their owner hasn't seen. */ +export function unreadWorkCount(chats: UnreadChat[]): number { + return chats.filter((chat) => chat.hasUnreadWork).length; +} diff --git a/apps/webapp/app/components/dashboard-agent/unread-work.test.ts b/apps/webapp/app/components/dashboard-agent/unread-work.test.ts new file mode 100644 index 00000000000..9358d5bf335 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/unread-work.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { chatIsUnread } from "./DashboardAgentHistory"; + +/** + * A chat is unread when its transcript moved on after its owner last looked — whether that + * was a watch waking it or an answer that landed while the panel was closed. Both raise the + * dot and the highlight; only a wake also raises a toast. + */ +describe("chatIsUnread", () => { + const chat = (over: Record = {}) => + ({ id: "chat_1", title: "t", lastMessageAt: null, ...over }) as never; + + it("counts work that finished behind a closed panel", () => { + expect(chatIsUnread(chat({ hasUnreadWork: true }))).toBe(true); + }); + + it("still counts a watch wake", () => { + expect(chatIsUnread(chat({ hasUnreadWake: true }))).toBe(true); + }); + + it("leaves a chat its owner has seen", () => { + expect(chatIsUnread(chat())).toBe(false); + expect(chatIsUnread(chat({ hasUnreadWake: false, hasUnreadWork: false }))).toBe(false); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/view-actions.test.ts b/apps/webapp/app/components/dashboard-agent/view-actions.test.ts index a9914da149a..baf18f3b695 100644 --- a/apps/webapp/app/components/dashboard-agent/view-actions.test.ts +++ b/apps/webapp/app/components/dashboard-agent/view-actions.test.ts @@ -1,7 +1,27 @@ import type { ActionsBlockAction } from "@internal/dashboard-agent-contracts"; import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { answerContinuesAfter, renderableActions } from "./view-actions"; +import { + answerContinuesAfter, + cardAlreadyOffersWatch, + renderableActions, + turnAlreadyOffersWatch, + withoutWatchActions, +} from "./view-actions"; + +const watchAction: ActionsBlockAction = { + label: "Set up a watch", + intent: { + kind: "watch", + spec: { + kind: "error_recurrence", + fingerprint: "a1b2c3", + checkEveryMinutes: 15, + maxHours: 6, + note: "the TypeError in send-order-receipt", + }, + }, +}; const askAction: ActionsBlockAction = { label: "Investigate it", @@ -25,6 +45,10 @@ describe("renderableActions", () => { expect(renderableActions([navigate])).toEqual([navigate]); }); + it("keeps a watch action, spec intact — that spec is what pre-fills the card", () => { + expect(renderableActions([watchAction, askAction])).toEqual([watchAction, askAction]); + }); + it("can filter every action out, leaving nothing to render", () => { expect( renderableActions([{ label: "Nowhere", intent: { kind: "navigate", target: "nope" } }]) @@ -47,6 +71,52 @@ describe("keep digging, only while there is digging left", () => { }); }); +describe("one watch button per answer", () => { + const watchAction = { label: "Watch for a repeat", intent: { kind: "watch" as const, spec: {} } }; + const card = (actions: unknown[]) => + ({ type: "investigation", investigation: {}, capabilities: { actions } }) as never; + + it("sees the card's own watch offer", () => { + expect(cardAlreadyOffersWatch([card([watchAction])])).toBe(true); + }); + + it("leaves an answer whose card offers no watch alone", () => { + expect( + cardAlreadyOffersWatch([ + card([{ label: "Keep digging", intent: { kind: "ask", prompt: "" } }]), + ]) + ).toBe(false); + expect(cardAlreadyOffersWatch([])).toBe(false); + }); + + // The bug this closes: one `render_view` call carries the investigation card and a second + // carries the actions block, so each call asked only about its own blocks and said no. + it("sees a watch offered by another of the same turn's render_view calls", () => { + const investigationCall = [card([watchAction])]; + const actionsCall = [{ type: "actions", actions: [watchAction] }] as never[]; + + expect(cardAlreadyOffersWatch(actionsCall)).toBe(false); + // Either order: the card can be rendered before or after the block that repeats it. + expect(turnAlreadyOffersWatch([investigationCall, actionsCall])).toBe(true); + expect(turnAlreadyOffersWatch([actionsCall, investigationCall])).toBe(true); + }); + + it("says no when no call in the turn has a card offering one", () => { + const plain = [card([{ label: "Keep digging", intent: { kind: "ask", prompt: "" } }])]; + expect(turnAlreadyOffersWatch([plain, []])).toBe(false); + expect(turnAlreadyOffersWatch([])).toBe(false); + }); + + it("drops the model's duplicate offer, keeping everything else", () => { + expect( + withoutWatchActions([ + { label: "Set up a watch", intent: { kind: "watch", spec: {} } }, + { label: "View similar", intent: { kind: "navigate", target: "trigger://x" } }, + ] as never) + ).toEqual([{ label: "View similar", intent: { kind: "navigate", target: "trigger://x" } }]); + }); +}); + describe("ActionsBlock", () => { const source = readFileSync(new URL("./ActionsBlock.tsx", import.meta.url), "utf8"); @@ -56,7 +126,7 @@ describe("ActionsBlock", () => { }); it("filters through the shared filter rather than rendering every action", () => { - expect(source).toContain("renderableActions(block.actions)"); + expect(source).toContain("renderableActions(actions)"); }); it("is a pure component: no app hooks, no server module, no Remix", () => { @@ -65,3 +135,26 @@ describe("ActionsBlock", () => { expect(source).not.toMatch(/\.server"/); }); }); + +/** + * There is no rendering harness here, so this pins the wiring rather than the pixels: the + * turn-wide answer is computed where every part is in scope and reaches every card, and + * `ViewBlocks` can only add to it. What it does not prove is that the button disappears. + */ +describe("the one-watch-button flag is decided per turn, not per render_view call", () => { + const turn = readFileSync(new URL("./DashboardAgentMessages.tsx", import.meta.url), "utf8"); + const catalog = readFileSync(new URL("./view-catalog.tsx", import.meta.url), "utf8"); + + it("computes it over every part's blocks, above the per-part loop", () => { + expect(turn).toContain("turnAlreadyOffersWatch("); + // Above the loop: computed from the whole `parts` map, not from one part. + expect(turn.indexOf("const watchOfferedInTurn")).toBeLessThan( + turn.indexOf("for (let i = 0; i < parts.length; i++)") + ); + expect(turn).toContain("watchOfferedInTurn={watchOfferedInTurn}"); + }); + + it("lets a card add its own offer but never drop the turn's", () => { + expect(catalog).toMatch(/watchOfferedInTurn \|\|\s*cardAlreadyOffersWatch\(/); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/view-actions.ts b/apps/webapp/app/components/dashboard-agent/view-actions.ts index 1d59d20b081..d0253d3d67e 100644 --- a/apps/webapp/app/components/dashboard-agent/view-actions.ts +++ b/apps/webapp/app/components/dashboard-agent/view-actions.ts @@ -4,6 +4,7 @@ import { isTriggerUri, type ActionsBlockAction, type ChartAction, + type ViewBlock, } from "@internal/dashboard-agent-contracts"; type CardAction = ChartAction | ActionsBlockAction; @@ -15,6 +16,31 @@ export function renderableActions(actions: T[]): T[] { }); } +/** + * An investigation card carries its own "watch for a repeat" button, and the model is + * asked to end an unresolved answer with a watch offer — so an answer that does both + * shows the same button twice. The card wins: it is the one with the pre-filled spec. + */ +export function cardAlreadyOffersWatch(blocks: ViewBlock[]): boolean { + return blocks.some( + (block) => + block.type === "investigation" && + (block.capabilities?.actions ?? []).some((action) => action.intent.kind === "watch") + ); +} + +/** + * The same question across every card a turn renders. One `render_view` call can carry the + * investigation card and another the actions block, so a per-call answer misses the pair. + */ +export function turnAlreadyOffersWatch(blockGroups: ViewBlock[][]): boolean { + return blockGroups.some(cardAlreadyOffersWatch); +} + +export function withoutWatchActions(actions: T[]): T[] { + return actions.filter((action) => action.intent.kind !== "watch"); +} + /** * "Keep digging" asks the agent to carry on — which is pointless once it already has. * A turn that renders an inconclusive card and then keeps answering leaves the button diff --git a/apps/webapp/app/components/dashboard-agent/view-catalog.test.ts b/apps/webapp/app/components/dashboard-agent/view-catalog.test.ts index 8b36e0c3f3c..85ea6f0d338 100644 --- a/apps/webapp/app/components/dashboard-agent/view-catalog.test.ts +++ b/apps/webapp/app/components/dashboard-agent/view-catalog.test.ts @@ -55,6 +55,16 @@ const FIXTURES: Record = { footer: [], }, }, + watch_result: { + ...envelope("watch:watch_1"), + type: "watch_result", + outcome: "watching", + headline: "Watching send-order-receipt for failures.", + lifetime: "24h", + detail: null, + followUp: [], + watchId: "watch_1", + }, investigation: { ...envelope("investigation-1"), type: "investigation", diff --git a/apps/webapp/app/components/dashboard-agent/view-catalog.tsx b/apps/webapp/app/components/dashboard-agent/view-catalog.tsx index dfb1034a77b..b8d0a31f014 100644 --- a/apps/webapp/app/components/dashboard-agent/view-catalog.tsx +++ b/apps/webapp/app/components/dashboard-agent/view-catalog.tsx @@ -5,6 +5,8 @@ import { InvestigationCard } from "./InvestigationCard"; import { ReportView, type ResolvedUri } from "./ReportView"; import { RunDiagnosisCard } from "./RunDiagnosisCard"; import { blockKey, latestRevisionEntries } from "./view-blocks"; +import { cardAlreadyOffersWatch } from "./view-actions"; +import { WatchResultBlock } from "./WatchResultBlock"; // Unknown block types are skipped, so an older or newer agent cannot render // arbitrary content. A new block needs a `case` here and a `viewBlockSchema` member. @@ -14,6 +16,7 @@ export function ViewBlocks({ resolveUri, pagePaths, answered = false, + watchOfferedInTurn = false, }: { blocks: ViewBlock[]; onIntent?: (intent: AgentIntent) => void; @@ -21,11 +24,16 @@ export function ViewBlocks({ pagePaths?: Record; /** The turn kept answering after this card, so "keep digging" has nothing to ask for. */ answered?: boolean; + /** A card in another of this turn's parts already offers the watch; see `view-actions`. */ + watchOfferedInTurn?: boolean; }) { if (!Array.isArray(blocks)) return null; + const entries = latestRevisionEntries(blocks); + const watchOfferedOnCard = + watchOfferedInTurn || cardAlreadyOffersWatch(entries.map((entry) => entry.block)); return (
- {latestRevisionEntries(blocks).map(({ block, index }) => { + {entries.map(({ block, index }) => { // The original array's index, so collapsing a revision above an // envelope-less block can't shift its key. const key = blockKey(block, index); @@ -35,7 +43,14 @@ export function ViewBlocks({ case "chart": return ; case "actions": - return ; + return ( + + ); // Revisions share the investigationId, so latest-wins keeps one card. case "investigation": return ( @@ -47,6 +62,9 @@ export function ViewBlocks({ answered={answered} /> ); + // Host-emitted only, so the model cannot fabricate a confirmation. + case "watch_result": + return ; case "report": return ( { + it("still reads the as-built two-value wake id", () => { + expect(wakeRefFromMessageId("wake:watch:watch_1:fired")).toEqual({ + watchId: "watch_1", + outcome: "fired", + }); + expect(wakeRefFromMessageId("wake:watch:watch_1:expired")).toEqual({ + watchId: "watch_1", + outcome: "expired", + }); + expect(wakeRefFromMessageId("msg_1")).toBeNull(); + }); +}); + +describe("wakeResolution", () => { + it("prefers the row's resolution", () => { + expect(wakeResolution("expired", { resolution: "condition_impossible" })).toBe( + "condition_impossible" + ); + }); + + it("reconstructs one for a row written before the resolution column", () => { + expect(wakeResolution("fired", { endedReason: null })).toBe("condition_met"); + expect(wakeResolution("expired", { endedReason: "terminal_unsatisfied" })).toBe( + "condition_impossible" + ); + expect(wakeResolution("expired", { endedReason: "not_met_by_expiry" })).toBe( + "window_completed" + ); + expect(wakeResolution("expired", undefined)).toBe("window_completed"); + }); +}); + +describe("wakePresentation", () => { + it("states the fact, not a generic watch update", () => { + const presented = wakePresentation("fired", { + ...runWatch, + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_SUCCESSFULLY", + durationMs: 4200, + }, + }); + expect(presented.headline).toBe("Run run_abc123 finished"); + expect(presented.label).toBe("Watch update"); + expect(presented.category).toBe("positive"); + }); + + it("shows a failed run as a failure, on the same resolution", () => { + const presented = wakePresentation("fired", { + ...runWatch, + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: null, + }, + }); + expect(presented.headline).toBe("Run run_abc123 failed"); + expect(presented.category).toBe("attention"); + expect(presented.semanticIcon).not.toBe("success"); + }); + + it("names the queue in a drain headline", () => { + expect( + wakePresentation("fired", { + id: "watch_2", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "condition_met", + }).headline + ).toBe("email-sends queue drained"); + }); + + it("reports the threshold watch with its number", () => { + expect( + wakePresentation("fired", { + id: "watch_3", + kind: "queue_depth_above", + identity: "queue_depth_above:email-sends:500", + note: "", + resolution: "condition_met", + observedOutcome: { + kind: "queue_depth_above", + verified: true, + depth: 612, + threshold: 500, + }, + }).headline + ).toBe("email-sends queue is still above 500"); + }); + + it("treats a completed window as an answer, not silence", () => { + const presented = wakePresentation("expired", { + id: "watch_4", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: true, depth: 42 }, + }); + expect(presented.headline).toBe("email-sends queue is still at 42"); + expect(presented.category).toBe("attention"); + }); + + it("says the condition couldn't be confirmed when the final read failed", () => { + expect( + wakePresentation("expired", { + id: "watch_5", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + note: "", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: false, depth: null }, + }).headline + ).toBe("The watch ended without a confirmed answer"); + }); + + it("falls back without guessing an outcome when the watch is gone", () => { + const presented = wakePresentation("fired", undefined); + expect(presented.headline).toBe("The watch woke this chat up on its own."); + expect(presented.category).toBe("neutral"); + }); + + it("says an error recurred, and that a quiet window was good news", () => { + const error = { + id: "watch_6", + kind: "error_recurrence", + identity: "error_recurrence:a1b2c3d4e5f6", + note: "", + }; + expect(wakePresentation("fired", { ...error, resolution: "condition_met" })).toMatchObject({ + headline: "Error a1b2c3d4e5f6 happened again", + category: "attention", + }); + expect(wakePresentation("expired", { ...error, resolution: "window_completed" })).toMatchObject( + { headline: "Error a1b2c3d4e5f6 stayed quiet", category: "positive" } + ); + }); + + it("says a queue came back below its threshold, and when it never did", () => { + const below = { + id: "watch_below", + kind: "queue_depth_below", + identity: "queue_depth_below:email-sends:100", + note: "", + }; + expect( + wakePresentation("fired", { + ...below, + resolution: "condition_met", + observedOutcome: { kind: "queue_depth_below", verified: true, depth: 42, threshold: 100 }, + }) + ).toMatchObject({ headline: "email-sends queue is back below 100", category: "positive" }); + + expect( + wakePresentation("expired", { + ...below, + resolution: "window_completed", + observedOutcome: { kind: "queue_depth_below", verified: true, depth: 780, threshold: 100 }, + }) + ).toMatchObject({ headline: "email-sends queue is still above 100", category: "attention" }); + }); + + it("says a queue is stuck at the depth it stalled on, and that it kept moving", () => { + const stalled = { + id: "watch_stalled", + kind: "queue_stalled", + identity: "queue_stalled:email-sends", + note: "", + }; + expect( + wakePresentation("fired", { + ...stalled, + resolution: "condition_met", + observedOutcome: { + kind: "queue_stalled", + verified: true, + depth: 42, + notDecreasingStreak: 3, + ticks: 3, + }, + }) + ).toMatchObject({ headline: "email-sends queue is stuck at 42", category: "attention" }); + + expect( + wakePresentation("expired", { + ...stalled, + resolution: "window_completed", + observedOutcome: { + kind: "queue_stalled", + verified: true, + depth: 3, + notDecreasingStreak: 1, + ticks: 3, + }, + }) + ).toMatchObject({ headline: "email-sends queue kept moving", category: "positive" }); + }); + + it("states the wait and the limit it passed, in minutes", () => { + const age = { + id: "watch_age", + kind: "queue_oldest_age", + identity: "queue_oldest_age:email-sends:5", + note: "", + }; + expect( + wakePresentation("fired", { + ...age, + resolution: "condition_met", + observedOutcome: { + kind: "queue_oldest_age", + verified: true, + ageMs: 12 * 60_000, + thresholdMinutes: 5, + }, + }) + ).toMatchObject({ + headline: "runs in email-sends are waiting 12m (over your 5m limit)", + category: "attention", + }); + + expect( + wakePresentation("expired", { + ...age, + resolution: "window_completed", + observedOutcome: { + kind: "queue_oldest_age", + verified: true, + ageMs: 30_000, + thresholdMinutes: 5, + }, + }) + ).toMatchObject({ headline: "email-sends queue stayed under 5m", category: "positive" }); + }); + + it("names the queue, not the threshold, when a queue-pack watch's queue is gone", () => { + for (const [kind, identity] of [ + ["queue_depth_below", "queue_depth_below:email-sends:100"], + ["queue_stalled", "queue_stalled:email-sends"], + ["queue_oldest_age", "queue_oldest_age:email-sends:5"], + ] as const) { + expect( + wakePresentation("expired", { + id: `watch_${kind}`, + kind, + identity, + note: "", + resolution: "condition_impossible", + }) + ).toMatchObject({ headline: "email-sends queue no longer exists", category: "neutral" }); + } + }); + + it("recovers health without naming an identity", () => { + expect( + wakePresentation("fired", { + id: "watch_7", + kind: "health_recovery", + identity: "health_recovery:health", + note: "", + resolution: "condition_met", + }).headline + ).toBe("Health recovered"); + }); +}); + +describe("watchWakeToastTitle", () => { + const wake = { + watchId: "watch_1", + chatId: "chat_1", + note: "tell me when the nightly invoice run finishes", + }; + + it("leads with the fact, not the notification", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "fired", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + resolution: "condition_met", + }) + ).toBe("email-sends queue drained"); + }); + + it("follows the observed outcome, so a failed run is never good news", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "fired", + kind: "run_finished", + identity: "run_finished:run_abc123", + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 1200, + }, + }) + ).toBe("Run run_abc123 failed"); + }); + + it("reconstructs a resolution for a row written before the model existed", () => { + expect( + watchWakeToastTitle({ + ...wake, + outcome: "expired", + kind: "backlog_drain", + identity: "backlog_drain:email-sends", + }) + ).toBe("email-sends queue still hasn't drained"); + }); + + it("claims nothing when the wake carries no watch at all", () => { + expect(watchWakeToastTitle({ ...wake, outcome: "fired" })).toBe( + "The watch woke this chat up on its own." + ); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts b/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts new file mode 100644 index 00000000000..cd02e8dcfa1 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/wake-poll.test.ts @@ -0,0 +1,126 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { startWakePolling, UNREAD_POLL_INTERVAL_MS, wakesToToast } from "./wake-poll"; + +function harness() { + let hidden = false; + const listeners = new Set<() => void>(); + const loads: number[] = []; + + const stop = startWakePolling({ + load: async () => { + loads.push(Date.now()); + }, + isHidden: () => hidden, + onVisibilityChange: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + // No jitter, so every delay is exactly one interval. + random: () => 0, + setTimer: (callback, ms) => setTimeout(callback, ms) as unknown as number, + clearTimer: (handle) => clearTimeout(handle as unknown as NodeJS.Timeout), + }); + + return { + loads, + stop, + setHidden(next: boolean) { + hidden = next; + for (const listener of listeners) listener(); + }, + listenerCount: () => listeners.size, + }; +} + +describe("startWakePolling", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("polls once immediately and then once per interval", async () => { + const poll = harness(); + + expect(poll.loads).toHaveLength(1); + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 3); + expect(poll.loads).toHaveLength(4); + + poll.stop(); + }); + + it("asks nothing while hidden and catches up once when visible again", async () => { + const poll = harness(); + poll.setHidden(true); + + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 3); + expect(poll.loads).toHaveLength(1); + + poll.setHidden(false); + await vi.advanceTimersByTimeAsync(0); + expect(poll.loads).toHaveLength(2); + + poll.stop(); + }); + + it("keeps exactly one chain across ten rapid hide/show cycles", async () => { + const poll = harness(); + + for (let cycle = 0; cycle < 10; cycle++) { + poll.setHidden(true); + await vi.advanceTimersByTimeAsync(10); + poll.setHidden(false); + await vi.advanceTimersByTimeAsync(10); + } + + const afterCycles = poll.loads.length; + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 10); + + // One poll per interval, not ten: the resumes replaced the chain instead of + // forking it. + expect(poll.loads.length - afterCycles).toBe(10); + + poll.stop(); + }); + + it("stops every timer and listener on unmount", async () => { + const poll = harness(); + poll.setHidden(true); + poll.setHidden(false); + + poll.stop(); + const settled = poll.loads.length; + + await vi.advanceTimersByTimeAsync(UNREAD_POLL_INTERVAL_MS * 10); + expect(poll.loads).toHaveLength(settled); + expect(poll.listenerCount()).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("wakesToToast", () => { + const wake = (watchId: string, unread: boolean) => ({ watchId, unread }); + + it("skips a wake another machine already read, and keeps the unread one", () => { + const wakes = [wake("watch_read", false), wake("watch_new", true)]; + + expect(wakesToToast(wakes, new Set())).toEqual([wake("watch_new", true)]); + }); + + it("still skips what this browser toasted, read or not", () => { + const wakes = [wake("watch_seen", true), wake("watch_new", true)]; + + expect(wakesToToast(wakes, new Set(["watch_seen"]))).toEqual([wake("watch_new", true)]); + }); + + // The read POST is what clears it, and that only runs once the chat is looked at. + it("toasts a wake that landed in an open chat, because it is still unread", () => { + expect(wakesToToast([wake("watch_in_view", true)], new Set())).toHaveLength(1); + }); + + it("treats a wake with no unread flag as already seen rather than guessing", () => { + expect(wakesToToast([{ watchId: "watch_old" }], new Set())).toEqual([]); + expect(wakesToToast(undefined, new Set())).toEqual([]); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/wake-poll.ts b/apps/webapp/app/components/dashboard-agent/wake-poll.ts new file mode 100644 index 00000000000..4a15f566400 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/wake-poll.ts @@ -0,0 +1,81 @@ +/** + * The wake feed's poll: one self-scheduling chain per mount. A hidden tab asks nothing, a + * resume catches up once, and neither can fork the chain into a second one. + */ + +export const UNREAD_POLL_INTERVAL_MS = 60_000; + +// Added to each delay so open tabs never settle into polling on the same second. +export const UNREAD_POLL_JITTER_MS = 15_000; + +/** + * Which of the feed's wakes this tab should toast. The feed is recent deliveries, not + * unread ones, and the local memory of what was toasted is per browser — so `unread` is the + * only signal shared across machines that a wake has already been seen. A wake landing in + * an open chat stays unread until that chat's next read, so it still toasts. + */ +export function wakesToToast( + wakes: T[] | undefined, + toasted: ReadonlySet +): T[] { + return (wakes ?? []).filter((wake) => wake.unread === true && !toasted.has(wake.watchId)); +} + +export type WakePollOptions = { + load: () => Promise; + isHidden: () => boolean; + /** Subscribe to visibility changes; returns its own unsubscribe. */ + onVisibilityChange: (listener: () => void) => () => void; + /** Seams so a test can drive the chain without real timers. */ + random?: () => number; + setTimer?: (callback: () => void, delayMs: number) => number; + clearTimer?: (handle: number) => void; +}; + +/** Start polling. The returned function stops the chain for good. */ +export function startWakePolling(options: WakePollOptions): () => void { + const random = options.random ?? Math.random; + const setTimer = options.setTimer ?? ((callback, ms) => window.setTimeout(callback, ms)); + const clearTimer = options.clearTimer ?? ((handle) => window.clearTimeout(handle)); + + let stopped = false; + let timer: number | undefined; + let loading = false; + // Each tick carries the chain it belongs to, so an orphaned callback returns + // instead of scheduling itself again. + let chain = 0; + + const tick = (generation: number) => { + if (stopped || generation !== chain) return; + + // Scheduled before the load, so a slow response can't stall the chain. + timer = setTimer( + () => tick(generation), + UNREAD_POLL_INTERVAL_MS + random() * UNREAD_POLL_JITTER_MS + ); + + if (loading || options.isHidden()) return; + loading = true; + const done = () => { + loading = false; + }; + options.load().then(done, done); + }; + + const unsubscribe = options.onVisibilityChange(() => { + if (stopped || options.isHidden()) return; + // One catch-up fetch on a new chain, replacing the pending timer rather than + // adding a second chain. + if (timer !== undefined) clearTimer(timer); + chain += 1; + tick(chain); + }); + + tick(chain); + + return () => { + stopped = true; + if (timer !== undefined) clearTimer(timer); + unsubscribe(); + }; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts b/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts new file mode 100644 index 00000000000..af7ddfc5174 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-activity.test.ts @@ -0,0 +1,155 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type StorageListener = (event: { key: string | null }) => void; + +const store = new Map(); +const storageListeners = new Set(); + +// A minimal `window`: these tests run without a DOM. +const windowStub = { + localStorage: { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => void store.set(key, value), + }, + addEventListener: (_type: string, listener: StorageListener) => + void storageListeners.add(listener), + removeEventListener: (_type: string, listener: StorageListener) => + void storageListeners.delete(listener), +}; + +const { + forgetWatchActivity, + hasWatchActivity, + rememberWatchActivity, + shouldPollWakeFeed, + subscribeWatchActivity, +} = await import("./watch-activity"); + +/** What another tab writing the key looks like here. */ +function otherTabWrote(organizationId: string) { + store.set("tdev:dashboard-agent:watching", JSON.stringify([organizationId])); + for (const listener of storageListeners) listener({ key: "tdev:dashboard-agent:watching" }); +} + +describe("watch activity", () => { + beforeEach(() => { + store.clear(); + storageListeners.clear(); + vi.stubGlobal("window", windowStub); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("knows nothing until a watch shows up", () => { + expect(hasWatchActivity("org_1")).toBe(false); + + rememberWatchActivity("org_1"); + expect(hasWatchActivity("org_1")).toBe(true); + expect(hasWatchActivity("org_2")).toBe(false); + }); + + it("survives a reload", () => { + rememberWatchActivity("org_1"); + storageListeners.clear(); + + expect(hasWatchActivity("org_1")).toBe(true); + }); + + it("tells a tab that was already open", () => { + const woken: number[] = []; + const unsubscribe = subscribeWatchActivity(() => woken.push(1)); + + rememberWatchActivity("org_1"); + expect(woken).toHaveLength(1); + + unsubscribe(); + rememberWatchActivity("org_2"); + expect(woken).toHaveLength(1); + }); + + it("tells a tab about a watch another tab created", () => { + const woken: number[] = []; + const unsubscribe = subscribeWatchActivity(() => woken.push(1)); + + otherTabWrote("org_1"); + + expect(woken).toHaveLength(1); + expect(hasWatchActivity("org_1")).toBe(true); + unsubscribe(); + }); + + it("forgets one organization without forgetting the others", () => { + rememberWatchActivity("org_1"); + rememberWatchActivity("org_2"); + + forgetWatchActivity("org_1"); + + expect(hasWatchActivity("org_1")).toBe(false); + expect(hasWatchActivity("org_2")).toBe(true); + }); + + it("remembers at most ten organizations", () => { + for (let index = 0; index < 12; index++) rememberWatchActivity(`org_${index}`); + + expect(hasWatchActivity("org_0")).toBe(false); + expect(hasWatchActivity("org_11")).toBe(true); + }); + + describe("a corrupt key", () => { + it("reads as nothing known when the value is not an array", () => { + store.set("tdev:dashboard-agent:watching", JSON.stringify({ org_1: true })); + + expect(hasWatchActivity("org_1")).toBe(false); + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(false); + }); + + it("keeps the ids out of an array holding other things", () => { + store.set("tdev:dashboard-agent:watching", JSON.stringify([{ id: "org_1" }, "org_2", 7])); + + expect(hasWatchActivity("org_1")).toBe(false); + expect(hasWatchActivity("org_2")).toBe(true); + + rememberWatchActivity("org_3"); + expect(store.get("tdev:dashboard-agent:watching")).toBe(JSON.stringify(["org_2", "org_3"])); + }); + }); + + describe("shouldPollWakeFeed", () => { + it("polls in a fresh browser the page load says has an unread wake", () => { + expect(shouldPollWakeFeed({ serverUnreadWakes: 1, organizationId: "org_1" })).toBe(true); + }); + + it("stays quiet when neither the page load nor this browser knows of anything", () => { + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(false); + }); + + it("polls in a fresh browser whose only signal is an active watch", () => { + // Created on another machine, nothing woken yet, no local marker. + expect( + shouldPollWakeFeed({ + serverUnreadWakes: 0, + serverHasActiveWatches: true, + organizationId: "org_1", + }) + ).toBe(true); + }); + + it("stays quiet in a fresh browser with no wake and no active watch", () => { + expect( + shouldPollWakeFeed({ + serverUnreadWakes: 0, + serverHasActiveWatches: false, + organizationId: "org_1", + }) + ).toBe(false); + }); + + it("polls without a reload once this browser sees a watch", () => { + rememberWatchActivity("org_1"); + + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_1" })).toBe(true); + expect(shouldPollWakeFeed({ serverUnreadWakes: 0, organizationId: "org_2" })).toBe(false); + }); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-activity.ts b/apps/webapp/app/components/dashboard-agent/watch-activity.ts new file mode 100644 index 00000000000..5acd7d54c9b --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-activity.ts @@ -0,0 +1,97 @@ +/** + * Which organizations this browser has seen agent watches in. This is an accelerator, not the + * gate: a watch created in this tab starts the poll without a reload. The ungated signals are the + * unread count and the active-watch flag the page load carries — see {@link shouldPollWakeFeed}. + * Shared through `localStorage`, so a watch created in one tab wakes the others. + */ + +const STORAGE_KEY = "tdev:dashboard-agent:watching"; + +// Newest ids only, so the key can't grow unbounded. +const MAX_REMEMBERED = 10; + +const listeners = new Set<() => void>(); + +function read(): string[] { + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + if (!raw) return []; + const parsed: unknown = JSON.parse(raw); + // Anything else under this key is another writer's; keep only what we can compare. + return Array.isArray(parsed) ? parsed.filter((id): id is string => typeof id === "string") : []; + } catch { + // Storage unavailable; treated as "nothing known yet". + return []; + } +} + +export function hasWatchActivity(organizationId: string): boolean { + if (typeof window === "undefined") return false; + return read().includes(organizationId); +} + +/** Called whenever a watch shows up for this org: the poll starts from here. */ +export function rememberWatchActivity(organizationId: string): void { + if (typeof window === "undefined" || hasWatchActivity(organizationId)) return; + try { + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify([...read(), organizationId].slice(-MAX_REMEMBERED)) + ); + } catch { + // Same as the read. This tab still starts polling for the rest of the session. + } + for (const listener of listeners) listener(); +} + +/** + * Called when nothing is left to be woken about. The current tab keeps polling for the rest of + * the session; the next reload starts quiet. + */ +export function forgetWatchActivity(organizationId: string): void { + if (typeof window === "undefined" || !hasWatchActivity(organizationId)) return; + try { + window.localStorage.setItem( + STORAGE_KEY, + JSON.stringify(read().filter((id) => id !== organizationId)) + ); + } catch { + // Same as the read. + } +} + +/** + * Whether this browser should poll the wake feed. Both server signals come from the page load, + * so a fresh browser polls without ever opening the panel: `serverUnreadWakes` for a wake that + * already landed, `serverHasActiveWatches` for one created elsewhere that hasn't fired yet. + */ +export function shouldPollWakeFeed(params: { + serverUnreadWakes: number; + serverHasActiveWatches?: boolean; + /** Chats holding work their owner hasn't seen, as the page load counted them. */ + serverUnreadWork?: number; + /** This tab sent a turn that may still be running behind a closed panel. */ + turnInFlight?: boolean; + organizationId: string; +}): boolean { + return ( + params.serverUnreadWakes > 0 || + params.serverHasActiveWatches === true || + (params.serverUnreadWork ?? 0) > 0 || + params.turnInFlight === true || + hasWatchActivity(params.organizationId) + ); +} + +/** Fires when this browser learns of a watch, in this tab or — via `storage` — in another one. */ +export function subscribeWatchActivity(listener: () => void): () => void { + listeners.add(listener); + const onStorage = (event: StorageEvent) => { + if (event.key === null || event.key === STORAGE_KEY) listener(); + }; + window.addEventListener("storage", onStorage); + return () => { + listeners.delete(listener); + window.removeEventListener("storage", onStorage); + }; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts b/apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts new file mode 100644 index 00000000000..9dadd3516cc --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts @@ -0,0 +1,118 @@ +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { NO_WATCH_CARD, watchCardReducer, type WatchCardState } from "./watch-card-state"; + +const draftFor = (note: string): WatchDraft => + ({ + spec: { kind: "error_recurrence", fingerprint: note, checkEveryMinutes: 15, maxHours: 6, note }, + followUp: { investigateOnAttention: false, notifyExternally: false }, + }) as WatchDraft; + +const run = (events: Parameters[1][], from = NO_WATCH_CARD) => + events.reduce(watchCardReducer, from); + +const opened = () => run([{ type: "open", draft: draftFor("the TypeError"), requestId: "wreq_1" }]); + +describe("a watch card belongs to the chat it was configured in", () => { + it("abandons a half-configured card when the chat changes", () => { + expect(run([{ type: "chat-changed" }], opened())).toEqual(NO_WATCH_CARD); + }); + + it("lets go of the request id too, so the next card writes its own records", () => { + const afterFailure = run( + [ + { type: "submitting", requestId: "wreq_1" }, + { type: "failed", error: "nope" }, + ], + opened() + ); + expect(afterFailure.requestId).toBe("wreq_1"); + expect(run([{ type: "chat-changed" }], afterFailure).requestId).toBeUndefined(); + }); + + it("abandons a card that was mid-submit when the chat changed", () => { + const submitting = run([{ type: "submitting", requestId: "wreq_1" }], opened()); + expect(submitting.pending).toBe(true); + expect(run([{ type: "chat-changed" }], submitting)).toEqual(NO_WATCH_CARD); + }); + + it("clears the card once it has been submitted", () => { + expect(run([{ type: "submitted" }], opened())).toEqual(NO_WATCH_CARD); + expect(run([{ type: "dismissed" }], opened())).toEqual(NO_WATCH_CARD); + }); +}); + +describe("the request id survives a retry", () => { + it("keeps the id it was opened with across a failed submit", () => { + const retried = run( + [ + { type: "submitting", requestId: "wreq_1" }, + { type: "failed", error: "nope" }, + { type: "submitting", requestId: "wreq_2" }, + ], + opened() + ); + // A resubmit repairs the same server records; a fresh id would write a second pair. + expect(retried.requestId).toBe("wreq_1"); + expect(retried.error).toBeNull(); + expect(retried.pending).toBe(true); + }); + + it("keeps the edited draft, and edits nothing once the card is gone", () => { + const edited = run([{ type: "edit", draft: draftFor("edited") }], opened()); + expect(edited.draft).toEqual(draftFor("edited")); + expect(edited.requestId).toBe("wreq_1"); + expect(run([{ type: "edit", draft: draftFor("edited") }])).toEqual(NO_WATCH_CARD); + }); + + it("opening a second card starts clean", () => { + const reopened = run( + [ + { type: "failed", error: "nope" }, + { type: "open", draft: draftFor("another"), requestId: "wreq_2" }, + ], + opened() + ); + expect(reopened).toEqual({ + draft: draftFor("another"), + requestId: "wreq_2", + pending: false, + error: null, + }); + }); +}); + +/** + * Structural guard, not behavioural proof: the reducer only sees a chat change if every path + * that changes chat routes through `claimChatSlot`, which is also the only place the in-flight + * open sequence is bumped. + */ +describe("every chat change goes through one door", () => { + const panel = readFileSync(new URL("./DashboardAgentPanel.tsx", import.meta.url), "utf8"); + + it("bumps the open sequence in exactly one place, next to the card reset", () => { + const bumps = panel.match(/openChatRequestSeq\.current\s*(\+\+|\+=)|\+\+openChatRequestSeq/g); + expect(bumps).toHaveLength(1); + expect(panel).toContain( + 'dispatchWatchCard({ type: "chat-changed" });\n return ++openChatRequestSeq.current;' + ); + }); + + it("claims a slot before every setActive that lands in a different chat", () => { + for (const caller of ["openChat", "createChat", "newChat", "submitWatch"]) { + expect(panel).toMatch(new RegExp(`const ${caller} = useCallback\\(`)); + } + // The watch's own landing chat: without the claim, an earlier open still matches its seq. + const submit = panel.slice(panel.indexOf("const submitWatch = useCallback(")); + const claim = submit.indexOf("claimChatSlot();"); + const setActive = submit.indexOf("setActive({ chatId: data.chatId"); + expect(claim).toBeGreaterThan(-1); + expect(setActive).toBeGreaterThan(claim); + }); + + it("leaves no separate watch-draft state for a chat change to miss", () => { + expect(panel).not.toContain("setWatchDraft"); + expect(panel).not.toContain("watchRequestId"); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-card-state.ts b/apps/webapp/app/components/dashboard-agent/watch-card-state.ts new file mode 100644 index 00000000000..59e3c12a0b3 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card-state.ts @@ -0,0 +1,55 @@ +/** + * The panel's watch card, as a pure state machine. + * + * A card is configured against the chat that is open at the time and submitted against + * whatever chat is open when `Start watching` is pressed, so it cannot outlive its chat: + * every chat change abandons it, request id and all. The request id is what makes a retry + * repair the same pair of server records instead of writing a second pair, so it is held + * across a failure and dropped with the card. + */ +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; + +export type WatchCardState = { + draft: WatchDraft | null; + requestId: string | undefined; + pending: boolean; + error: string | null; +}; + +export const NO_WATCH_CARD: WatchCardState = { + draft: null, + requestId: undefined, + pending: false, + error: null, +}; + +export type WatchCardEvent = + | { type: "open"; draft: WatchDraft; requestId: string } + | { type: "edit"; draft: WatchDraft } + | { type: "submitting"; requestId: string } + | { type: "failed"; error: string } + | { type: "submitted" } + | { type: "dismissed" } + | { type: "chat-changed" }; + +export function watchCardReducer(state: WatchCardState, event: WatchCardEvent): WatchCardState { + switch (event.type) { + case "open": + return { draft: event.draft, requestId: event.requestId, pending: false, error: null }; + case "edit": + return state.draft ? { ...state, draft: event.draft } : state; + case "submitting": + return { + ...state, + requestId: state.requestId ?? event.requestId, + pending: true, + error: null, + }; + case "failed": + return { ...state, pending: false, error: event.error }; + case "submitted": + case "dismissed": + case "chat-changed": + return NO_WATCH_CARD; + } +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-card.test.ts b/apps/webapp/app/components/dashboard-agent/watch-card.test.ts new file mode 100644 index 00000000000..de012037b66 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it } from "vitest"; +import { + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_MAX_QUEUE_THRESHOLD, + WATCH_STALL_TICKS_DEFAULT, + watchSpecSchema, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; +import { + clampCadence, + variantsOf, + watchDraftError, + watchDraftFor, + withAgeMinutes, + withCadence, + withFollowUp, + withThreshold, + withVariant, + withWindow, +} from "./watch-card"; +import { + watchConditionLabel, + watchConfirmationBlockBody, + watchDurationLabel, + watchOneShotBlockBody, + watchSubjectLabel, +} from "~/presenters/v3/dashboardAgent"; +import { + errorWatchRecommendation, + healthWatchRecommendation, + queueWatchRecommendation, + runWatchRecommendation, +} from "./watch-recommendations"; + +const queueDraft = () => watchDraftFor(queueWatchRecommendation("email-sends")); +const runDraft = () => watchDraftFor(runWatchRecommendation("run_abc123")); + +describe("the recommendations", () => { + it("gives every entry point a spec the schema accepts", () => { + const specs: WatchSpec[] = [ + runWatchRecommendation("run_abc123"), + queueWatchRecommendation("email-sends"), + errorWatchRecommendation("error_a1b2c3d4"), + healthWatchRecommendation("crit"), + ]; + for (const spec of specs) { + expect(watchSpecSchema.safeParse(spec).success).toBe(true); + } + }); + + it("recommends the condition §2.1 assigns to each object", () => { + expect(runWatchRecommendation("run_abc123").kind).toBe("run_finished"); + expect(queueWatchRecommendation("email-sends").kind).toBe("queue_oldest_age"); + expect(errorWatchRecommendation("error_a1b2c3d4").kind).toBe("error_recurrence"); + expect(healthWatchRecommendation("warn").kind).toBe("health_recovery"); + }); + + it("switches the queue recommendation to the drain once runs are already late", () => { + const late = queueWatchRecommendation("email-sends", { + oldestWaitMs: OLDEST_WAIT_WARNING_MS, + }); + expect(late).toMatchObject({ + kind: "backlog_drain", + queue: "email-sends", + }); + expect(watchSpecSchema.safeParse(late).success).toBe(true); + }); + + it("stays on the age SLA when the queue is merely busy, or the signal is missing", () => { + expect( + queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS - 1 }).kind + ).toBe("queue_oldest_age"); + expect(queueWatchRecommendation("email-sends", { oldestWaitMs: null }).kind).toBe( + "queue_oldest_age" + ); + expect(queueWatchRecommendation("email-sends", {}).kind).toBe("queue_oldest_age"); + expect(queueWatchRecommendation("email-sends").kind).toBe("queue_oldest_age"); + }); + + it("starts both follow-ups off — consent is never assumed", () => { + expect(runDraft().followUp).toEqual({ + investigateOnAttention: false, + notifyExternally: false, + }); + }); +}); + +describe("cadence limits", () => { + it("lets a run watch poll every minute", () => { + expect(clampCadence("run_finished", 1)).toBe(1); + }); + + it("floors an aggregate watch at five minutes — never a hot loop", () => { + expect(clampCadence("backlog_drain", 1)).toBe(5); + expect(clampCadence("queue_depth_above", 1)).toBe(5); + expect(clampCadence("health_recovery", 1)).toBe(5); + }); + + it("keeps an offered cadence and rounds an unknown one up", () => { + expect(clampCadence("backlog_drain", 15)).toBe(15); + expect(clampCadence("backlog_drain", 7)).toBe(15); + expect(clampCadence("backlog_drain", 999)).toBe(60); + }); + + it("re-clamps when the kind changes under the user", () => { + const swapped = withVariant(withCadence(runDraft(), 1), "backlog_drain"); + expect(swapped.spec.checkEveryMinutes).toBe(5); + expect(watchSpecSchema.safeParse(swapped.spec).success).toBe(true); + }); +}); + +describe("condition variants (§3)", () => { + it("offers the run pair and the whole queue family", () => { + expect(variantsOf(runDraft())).toEqual(["run_finished", "run_failed"]); + expect(variantsOf(queueDraft())).toEqual([ + "backlog_drain", + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ]); + expect(variantsOf(watchDraftFor(errorWatchRecommendation("error_a1")))).toHaveLength(1); + expect(variantsOf(watchDraftFor(healthWatchRecommendation("warn")))).toHaveLength(1); + }); + + it("carries the subject and window across a swap, and restates the note", () => { + const draft = withWindow(runDraft(), 6); + const failed = withVariant(draft, "run_failed"); + expect(failed.spec).toMatchObject({ + kind: "run_failed", + runId: "run_abc123", + maxHours: 6, + note: "tell me if run run_abc123 fails", + }); + }); + + it("restates the note when the threshold number changes", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + // Same verb and same SLA format as the card's condition line: both come from + // the presenter's one wording record. + expect(above.spec.note).toBe("tell me if the email-sends queue goes above 500"); + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 90); + expect(age.spec.note).toBe("tell me if runs in email-sends wait longer than 1h 30m"); + }); + + it("gives the threshold variant a usable default", () => { + const above = withVariant(queueDraft(), "queue_depth_above"); + expect(above.spec).toMatchObject({ kind: "queue_depth_above", queue: "email-sends" }); + expect(watchDraftError(above)).toBeNull(); + }); + + it("swaps back without losing the queue", () => { + const roundTrip = withVariant(withVariant(queueDraft(), "queue_depth_above"), "backlog_drain"); + expect(roundTrip.spec).toMatchObject({ kind: "backlog_drain", queue: "email-sends" }); + }); + + it("gives every queue variant a submittable default and keeps the subject", () => { + for (const kind of [ + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ] as const) { + const swapped = withVariant(queueDraft(), kind); + expect(swapped.spec).toMatchObject({ kind, queue: "email-sends" }); + expect(watchDraftError(swapped)).toBeNull(); + expect(watchSpecSchema.safeParse(swapped.spec).success).toBe(true); + } + }); + + it("carries a typed threshold between the two threshold questions", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + const below = withVariant(above, "queue_depth_below"); + expect(below.spec).toMatchObject({ kind: "queue_depth_below", threshold: 500 }); + }); + + it("keeps the stall count internal — the default, never a field", () => { + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(stalled.spec).toMatchObject({ ticks: WATCH_STALL_TICKS_DEFAULT }); + expect(withThreshold(stalled, 5)).toEqual(stalled); + expect(withAgeMinutes(stalled, 5)).toEqual(stalled); + }); +}); + +describe("the window", () => { + it("never leaves the 24-hour ceiling", () => { + expect(withWindow(runDraft(), 999).spec.maxHours).toBe(24); + }); + + it("never goes below the shortest offered window", () => { + expect(withWindow(runDraft(), 0).spec.maxHours).toBe(0.5); + }); +}); + +describe("the follow-up opt-ins (§2.2, binding)", () => { + it("sets them INDEPENDENTLY — never as a radio group", () => { + const both = withFollowUp(withFollowUp(runDraft(), { notifyExternally: true }), { + investigateOnAttention: true, + }); + expect(both.followUp).toEqual({ investigateOnAttention: true, notifyExternally: true }); + }); + + it("turning one off leaves the other alone", () => { + const draft = withFollowUp(runDraft(), { + investigateOnAttention: true, + notifyExternally: true, + }); + expect(withFollowUp(draft, { notifyExternally: false }).followUp).toEqual({ + investigateOnAttention: true, + notifyExternally: false, + }); + }); + + it("has no way to express in-chat delivery at all — it is not a choice", () => { + expect(Object.keys(runDraft().followUp).sort()).toEqual([ + "investigateOnAttention", + "notifyExternally", + ]); + }); +}); + +describe("validation stays inside the card", () => { + it("accepts every recommendation as it opens", () => { + expect(watchDraftError(runDraft())).toBeNull(); + expect(watchDraftError(queueDraft())).toBeNull(); + }); + + it("refuses a half-typed threshold", () => { + const draft = withThreshold(withVariant(queueDraft(), "queue_depth_above"), Number.NaN); + expect(watchDraftError(draft)).toMatch(/whole number/i); + }); + + it("refuses a threshold above the queue-watch ceiling", () => { + const draft = withThreshold( + withVariant(queueDraft(), "queue_depth_above"), + WATCH_MAX_QUEUE_THRESHOLD + 1 + ); + expect(watchDraftError(draft)).toMatch(/too high/i); + }); + + it("ignores a threshold set on a kind that has none", () => { + expect(withThreshold(runDraft(), 5)).toEqual(runDraft()); + }); + + it("refuses a half-typed threshold on the `below` variant too", () => { + const draft = withThreshold(withVariant(queueDraft(), "queue_depth_below"), Number.NaN); + expect(watchDraftError(draft)).toMatch(/whole number/i); + }); + + it("refuses an SLA that is empty, zero, or longer than a watch can run", () => { + const age = withVariant(queueDraft(), "queue_oldest_age"); + expect(watchDraftError(withAgeMinutes(age, Number.NaN))).toMatch(/whole number of minutes/i); + expect(watchDraftError(withAgeMinutes(age, 0))).toMatch(/whole number of minutes/i); + expect(watchDraftError(withAgeMinutes(age, WATCH_MAX_QUEUE_AGE_MINUTES + 1))).toMatch( + /longer than a watch can run/i + ); + expect(watchDraftError(withAgeMinutes(age, 30))).toBeNull(); + }); + + it("ignores an SLA set on a kind that has none", () => { + expect(withAgeMinutes(runDraft(), 5)).toEqual(runDraft()); + }); +}); + +describe("the card's copy", () => { + it("names the subject the way the object does", () => { + expect(watchSubjectLabel(queueWatchRecommendation("email-sends"))).toBe("email-sends"); + expect(watchSubjectLabel(runWatchRecommendation("run_abc123"))).toBe("run run_abc123"); + expect(watchSubjectLabel(healthWatchRecommendation("warn"))).toBe("health"); + }); + + it("says the kind once, and names the error in full", () => { + // Fingerprints are stored prefixed (`error_c4b4a797397a9c43`), so the raw value + // would read "error error_c4b4a797397a9c43". + expect( + watchSubjectLabel({ + kind: "error_recurrence", + fingerprint: "error_c4b4a797397a9c43", + checkEveryMinutes: 5, + maxHours: 0.5, + }) + ).toBe("error c4b4a797397a9c43"); + }); + + it("states the condition and the duration as §2.2 writes them", () => { + const spec = queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS }); + expect(watchConditionLabel(spec)).toBe("Until the queue drains"); + expect(watchDurationLabel(spec)).toBe("For 1 hour · checking every 5 min"); + }); + + it("carries the threshold into the condition line", () => { + const above = withThreshold(withVariant(queueDraft(), "queue_depth_above"), 500); + expect(watchConditionLabel(above.spec)).toBe("If the queue goes above 500"); + }); + + it("states each new queue condition the way the user reads it", () => { + const below = withThreshold(withVariant(queueDraft(), "queue_depth_below"), 100); + expect(watchConditionLabel(below.spec)).toBe("Until the queue is back below 100"); + + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(watchConditionLabel(stalled.spec)).toBe("If the queue stops moving"); + + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 90); + expect(watchConditionLabel(age.spec)).toBe("If runs wait longer than 1h 30m"); + expect(watchSubjectLabel(age.spec)).toBe("email-sends"); + }); + + it("writes the confirmation as one sentence for every queue condition", () => { + const stalled = withVariant(queueDraft(), "queue_stalled"); + expect(watchConfirmationBlockBody({ spec: stalled.spec, watchId: "w" }).headline).toBe( + "Watching email-sends in case it stops moving." + ); + + const age = withAgeMinutes(withVariant(queueDraft(), "queue_oldest_age"), 5); + expect(watchConfirmationBlockBody({ spec: age.spec, watchId: "w" }).headline).toBe( + "Watching email-sends in case runs wait longer than 5m." + ); + + const below = withThreshold(withVariant(queueDraft(), "queue_depth_below"), 100); + expect(watchConfirmationBlockBody({ spec: below.spec, watchId: "w" }).headline).toBe( + "Watching email-sends until it is back below 100." + ); + }); +}); + +describe("the persisted blocks (§2.2)", () => { + it("states all four lifetime facts on a confirmation", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends", { oldestWaitMs: OLDEST_WAIT_WARNING_MS }), + watchId: "watch_1", + }); + expect(body.outcome).toBe("watching"); + expect(body.headline).toBe("Watching email-sends until the queue drains."); + expect(body.lifetime).toBe( + "Checking every 5 min for up to 1 hour. It reports once, then stops." + ); + expect(body.watchId).toBe("watch_1"); + expect(body.detail).toBeNull(); + }); + + it("says plainly when the creation-time check couldn't run", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + unavailable: true, + }); + expect(body.detail).toBe("We couldn't check that just now. Watching anyway."); + }); + + it("only claims a follow-up that actually took effect", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + followUp: { investigateOnAttention: true, external: { status: "not_requested" } }, + }); + expect(body.followUp).toEqual(["If it turns out badly, I'll investigate straight away."]); + }); + + it("says out loud when the email the user asked for couldn't be added", () => { + const body = watchConfirmationBlockBody({ + spec: queueWatchRecommendation("email-sends"), + watchId: "watch_1", + followUp: { external: { status: "unavailable", reason: "email_alerts_not_configured" } }, + }); + expect(body.followUp).toEqual([ + "I couldn't add email notifications, so updates will appear in the dashboard only.", + ]); + }); + + it("makes a one-shot result carry no lifetime and no watch", () => { + const satisfied = watchOneShotBlockBody({ + spec: queueWatchRecommendation("email-sends"), + result: "satisfied", + }); + expect(satisfied.outcome).toBe("already_true"); + expect(satisfied.headline).toBe("That already happened, so there's nothing left to watch."); + expect(satisfied.lifetime).toBeNull(); + expect(satisfied.watchId).toBeNull(); + + const impossible = watchOneShotBlockBody({ + spec: runWatchRecommendation("run_abc123"), + result: "terminal_unsatisfied", + }); + expect(impossible.outcome).toBe("impossible"); + expect(impossible.headline).toBe("That can't happen any more, so there's nothing to watch."); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-card.ts b/apps/webapp/app/components/dashboard-agent/watch-card.ts new file mode 100644 index 00000000000..9d2a9d54425 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-card.ts @@ -0,0 +1,181 @@ +/** + * The watch card's state machine, kept pure so its rules are testable without a DOM. + * + * The card never invents a value the schema would reject: switching condition + * variant re-clamps the cadence, and the window is always one of the offered + * options. The option lists are read from contracts (`watchCadenceOptions`, + * `WATCH_WINDOW_HOURS_OPTIONS`) rather than re-typed, so a picker cannot offer + * something validation would refuse. Nothing here persists: a draft is client-side + * until `Start watching` submits it. + */ +import { + WATCH_DEFAULT_QUEUE_AGE_MINUTES, + WATCH_DEFAULT_QUEUE_THRESHOLD, + WATCH_MAX_HOURS, + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_MAX_QUEUE_THRESHOLD, + WATCH_STALL_TICKS_DEFAULT, + WATCH_WINDOW_HOURS_OPTIONS, + watchCadenceOptions, + watchConditionVariants, + watchSpecSchema, + type WatchDraft, + type WatchFollowUp, + type WatchKind, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { noteFor } from "~/presenters/v3/dashboardAgent"; + +/** A brand-new draft: the recommendation, with both opt-ins off. */ +export function watchDraftFor(spec: WatchSpec): WatchDraft { + return { spec, followUp: { investigateOnAttention: false, notifyExternally: false } }; +} + +/** + * The nearest cadence this kind is allowed to poll at. A 1-minute run watch + * switched to a queue variant must land on 5, not fail validation on submit. + */ +export function clampCadence(kind: WatchKind, minutes: number): number { + const options = watchCadenceOptions(kind); + if (options.includes(minutes)) return minutes; + return options.find((option) => option >= minutes) ?? options[options.length - 1]!; +} + +/** + * Swap the condition for its sibling variant, carrying everything else except the + * note, which is restated to describe the new condition. + */ +export function withVariant(draft: WatchDraft, kind: WatchKind): WatchDraft { + const next = variantSpec(draft, kind); + if (next === draft.spec) return draft; + return { ...draft, spec: { ...next, note: noteFor(next) } as WatchSpec }; +} + +function variantSpec(draft: WatchDraft, kind: WatchKind): WatchSpec { + const { spec } = draft; + const common = { + note: spec.note, + maxHours: spec.maxHours, + checkEveryMinutes: clampCadence(kind, spec.checkEveryMinutes), + } as const; + + switch (kind) { + case "run_finished": + case "run_failed": + case "run_start": { + const runId = "runId" in spec ? spec.runId : ""; + return { ...common, kind, runId } as WatchSpec; + } + case "backlog_drain": { + const queue = "queue" in spec ? spec.queue : ""; + return { ...common, kind, queue } as WatchSpec; + } + case "queue_depth_above": + case "queue_depth_below": { + const queue = "queue" in spec ? spec.queue : ""; + // The number carries across the two threshold questions: someone who typed + // 500 for "above" means the same 500 when they flip to "back below". + const threshold = "threshold" in spec ? spec.threshold : WATCH_DEFAULT_QUEUE_THRESHOLD; + return { ...common, kind, queue, threshold } as WatchSpec; + } + case "queue_stalled": { + const queue = "queue" in spec ? spec.queue : ""; + // Ticks are not user-facing: the card never shows a field for them. + const ticks = "ticks" in spec ? spec.ticks : WATCH_STALL_TICKS_DEFAULT; + return { ...common, kind, queue, ticks } as WatchSpec; + } + case "queue_oldest_age": { + const queue = "queue" in spec ? spec.queue : ""; + const thresholdMinutes = + "thresholdMinutes" in spec ? spec.thresholdMinutes : WATCH_DEFAULT_QUEUE_AGE_MINUTES; + return { ...common, kind, queue, thresholdMinutes } as WatchSpec; + } + // The kinds with no second question keep the draft untouched. + default: + return draft.spec; + } +} + +/** + * The conditions this draft's picker offers, in order, including the current one. + * A single-entry list means the kind has no second question and the card states + * the condition as a fact instead of a choice. + */ +export function variantsOf(draft: WatchDraft): readonly WatchKind[] { + return watchConditionVariants(draft.spec.kind); +} + +export function withCadence(draft: WatchDraft, minutes: number): WatchDraft { + return { + ...draft, + spec: { + ...draft.spec, + checkEveryMinutes: clampCadence(draft.spec.kind, minutes), + } as WatchSpec, + }; +} + +export function withWindow(draft: WatchDraft, maxHours: number): WatchDraft { + const clamped = Math.min(Math.max(maxHours, WATCH_WINDOW_HOURS_OPTIONS[0]), WATCH_MAX_HOURS); + return { ...draft, spec: { ...draft.spec, maxHours: clamped } as WatchSpec }; +} + +/** + * The threshold, as the user is typing it. No range checks here: a half-typed + * field is a draft, and `watchDraftError` is what refuses to submit it. + */ +export function withThreshold(draft: WatchDraft, threshold: number): WatchDraft { + if (draft.spec.kind !== "queue_depth_above" && draft.spec.kind !== "queue_depth_below") { + return draft; + } + // The note quotes the number, so a new number restates the note. + const spec = { ...draft.spec, threshold }; + return { ...draft, spec: { ...spec, note: noteFor(spec) } }; +} + +/** The age SLA in minutes, as the user is typing it. Same rule as the threshold. */ +export function withAgeMinutes(draft: WatchDraft, thresholdMinutes: number): WatchDraft { + if (draft.spec.kind !== "queue_oldest_age") return draft; + // The note quotes the number, so a new number restates the note. + const spec = { ...draft.spec, thresholdMinutes }; + return { ...draft, spec: { ...spec, note: noteFor(spec) } }; +} + +/** + * The two follow-up opt-ins, set independently. There is no way to express + * "external instead of chat": in-chat delivery is not a choice, so it is not here. + */ +export function withFollowUp(draft: WatchDraft, patch: Partial): WatchDraft { + return { ...draft, followUp: { ...draft.followUp, ...patch } }; +} + +/** + * Why this draft can't be submitted, in the user's words, or null when it can. + * The schema is the authority, so the card and the server agree by construction; + * this only translates its refusal into the sentence the card shows inline. + */ +export function watchDraftError(draft: WatchDraft): string | null { + if (draft.spec.kind === "queue_depth_above" || draft.spec.kind === "queue_depth_below") { + const { threshold } = draft.spec; + if (!Number.isInteger(threshold) || threshold < 0) { + return "Enter a whole number to watch for."; + } + if (threshold > WATCH_MAX_QUEUE_THRESHOLD) { + return `That threshold is too high — ${WATCH_MAX_QUEUE_THRESHOLD.toLocaleString()} is the most a queue watch takes.`; + } + } + + if (draft.spec.kind === "queue_oldest_age") { + const { thresholdMinutes } = draft.spec; + if (!Number.isInteger(thresholdMinutes) || thresholdMinutes < 1) { + return "Enter a whole number of minutes to watch for."; + } + if (thresholdMinutes > WATCH_MAX_QUEUE_AGE_MINUTES) { + return `That's longer than a watch can run — ${WATCH_MAX_QUEUE_AGE_MINUTES} minutes is the most.`; + } + } + + return watchSpecSchema.safeParse(draft.spec).success + ? null + : "Something in this watch isn't valid. Check the duration and the condition."; +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts b/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts new file mode 100644 index 00000000000..687bc12cda4 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-chips.test.ts @@ -0,0 +1,107 @@ +import { watchIdentity, type WatchSpec } from "@internal/dashboard-agent-contracts"; +import { describe, expect, it } from "vitest"; +import { immediateWatchMessage, watchChipLabel, watchChipTooltip } from "./watch-chips"; + +const chip = (spec: WatchSpec) => ({ + kind: spec.kind, + identity: watchIdentity(spec), + note: spec.note, +}); + +describe("watchChipLabel", () => { + it("labels a run watch with its run id", () => { + expect( + watchChipLabel( + chip({ + kind: "run_finished", + runId: "run_abc123", + note: "Tell me when the retry finishes.", + maxHours: 2, + checkEveryMinutes: 1, + }) + ) + ).toBe("run_abc123"); + }); + + it("labels a backlog watch with the queue name", () => { + expect( + watchChipLabel( + chip({ + kind: "backlog_drain", + queue: "task/send-email", + note: "Tell me when the backlog clears.", + maxHours: 6, + checkEveryMinutes: 5, + }) + ) + ).toBe("task/send-email"); + }); + + it("labels an error watch by its fingerprint, in full", () => { + expect( + watchChipLabel( + chip({ + kind: "error_recurrence", + fingerprint: "0123456789abcdef0123456789abcdef", + note: "Tell me if the rate-limit error comes back.", + maxHours: 12, + checkEveryMinutes: 15, + }) + ) + ).toBe("0123456789abcdef0123456789abcdef"); + }); + + it("labels a health watch by its kind, not its report", () => { + expect( + watchChipLabel( + chip({ + kind: "health_recovery", + report: "health", + fromSeverity: "crit", + note: "prod health back to normal", + maxHours: 4, + checkEveryMinutes: 15, + }) + ) + ).toBe("health"); + }); + + it("falls back to the first words of the note when the identity is unreadable", () => { + expect( + watchChipLabel({ kind: "run_start", identity: "nonsense", note: "Tell me when it starts" }) + ).toBe("Tell me when"); + }); + + it("falls back to the kind when there is no note either", () => { + expect(watchChipLabel({ kind: "run_start", identity: "", note: " " })).toBe("run_start"); + }); +}); + +describe("watchChipTooltip", () => { + it("carries the note, the cadence and the state", () => { + expect( + watchChipTooltip({ + note: "Tell me when prod recovers.", + checkEveryMinutes: 15, + status: "active", + }) + ).toBe("Tell me when prod recovers. · every 15 min · watching"); + }); + + it("drops an empty note rather than leaving a dangling separator", () => { + expect(watchChipTooltip({ note: "", checkEveryMinutes: 5, status: "fired" })).toBe( + "every 5 min · fired" + ); + }); +}); + +describe("immediateWatchMessage", () => { + it("says the condition already resolved", () => { + expect(immediateWatchMessage("satisfied")).toMatch(/already happened/); + expect(immediateWatchMessage("terminal_unsatisfied")).toMatch(/can't happen any more/); + }); + + it("never falls through to nothing", () => { + expect(immediateWatchMessage("something-new")).toBeTruthy(); + }); +}); diff --git a/apps/webapp/app/components/dashboard-agent/watch-chips.ts b/apps/webapp/app/components/dashboard-agent/watch-chips.ts new file mode 100644 index 00000000000..c8de7a3eed9 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-chips.ts @@ -0,0 +1,76 @@ +/** + * The pure text of the watch UI: a chip's label and its tooltip. + * + * A chip has one line of room in a 380px panel, so the label names the thing being + * watched and the icon carries the state. The label comes from the watch `identity`, + * the same dedup key the store uses, so a chip cannot disagree with the store about + * what it watches. + */ +import type { WatchStatus } from "@internal/dashboard-agent-contracts"; + +// The immediate-check wording lives in the presenter with the rest of the +// user-facing copy. Re-exported here for chip callers. +export { immediateWatchMessage } from "~/presenters/v3/dashboardAgent"; + +import { + formatWatchCadence, + shortFingerprint, + watchIdentityValue, +} from "~/presenters/v3/dashboardAgent"; + +export const WATCH_STATUS_LABEL: Record = { + active: "watching", + fired: "fired", + expired: "expired", + cancelled: "cancelled", +}; + +/** Fingerprints are hashes — a chip shows just enough of one to tell them apart. */ +/** + * The chip label for a watch. `identity` is `{kind}:{value}`, so the value is the + * thing being watched; a health watch has no per-instance value, so its kind is + * the label. Falls back to the note (then the kind) if the identity is unreadable. + */ +export function watchChipLabel(watch: { kind: string; identity: string; note: string }): string { + const value = watch.identity.startsWith(`${watch.kind}:`) + ? watch.identity.slice(watch.kind.length + 1) + : ""; + + switch (watch.kind) { + case "run_start": + case "run_finished": + case "run_failed": + case "backlog_drain": + case "queue_stalled": + return value || fallbackLabel(watch); + // Identity is `{kind}:{queue}:{number}` here. The chip names the queue; the + // number goes in the tooltip's note, where there is room for it. + case "queue_depth_above": + case "queue_depth_below": + case "queue_oldest_age": + return watchIdentityValue(watch.kind, watch.identity) || fallbackLabel(watch); + case "error_recurrence": + return value ? shortFingerprint(value) : fallbackLabel(watch); + case "health_recovery": + return "health"; + default: + return value || fallbackLabel(watch); + } +} + +/** Last resort: the first few words of the note, else the kind as written. */ +function fallbackLabel(watch: { kind: string; note: string }): string { + const words = watch.note.trim().split(/\s+/).filter(Boolean).slice(0, 3).join(" "); + return words || watch.kind; +} + +/** Everything that didn't fit on the chip: why it exists, and its cadence. */ +export function watchChipTooltip(watch: { + note: string; + checkEveryMinutes: number; + status: WatchStatus; +}): string { + const note = watch.note.trim(); + const cadence = formatWatchCadence(watch.checkEveryMinutes); + return [note, cadence, WATCH_STATUS_LABEL[watch.status]].filter(Boolean).join(" · "); +} diff --git a/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts b/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts new file mode 100644 index 00000000000..4ab53c2dc90 --- /dev/null +++ b/apps/webapp/app/components/dashboard-agent/watch-recommendations.ts @@ -0,0 +1,78 @@ +import { + WATCH_DEFAULT_QUEUE_AGE_MINUTES, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; +import { noteFor } from "~/presenters/v3/dashboardAgent"; + +/** Distributes over the spec union, so the kind stays discriminated. */ +type WithoutNote = T extends unknown ? Omit : never; + +/** The note comes from the presenter, so a recommendation reads like an edited one. */ +function withNote(spec: WithoutNote): WatchSpec { + const draft = { ...spec, note: "" } as WatchSpec; + return { ...draft, note: noteFor(draft) }; +} + +export function runWatchRecommendation(runFriendlyId: string): WatchSpec { + return withNote({ + kind: "run_finished", + runId: runFriendlyId, + checkEveryMinutes: 1, + maxHours: 1, + }); +} + +/** + * The recommendation must be a condition that isn't true yet: an already-true watch + * one-shots instead of watching. Past the wait threshold that means the drain, not the SLA. + */ +export function queueWatchRecommendation( + queueName: string, + context?: { oldestWaitMs?: number | null } +): WatchSpec { + const oldestWaitMs = context?.oldestWaitMs ?? null; + if (oldestWaitMs !== null && oldestWaitMs >= OLDEST_WAIT_WARNING_MS) { + return withNote({ + kind: "backlog_drain", + queue: queueName, + checkEveryMinutes: 5, + maxHours: 1, + }); + } + + return queueAgeWatchRecommendation(queueName); +} + +export function queueAgeWatchRecommendation( + queueName: string, + thresholdMinutes: number = WATCH_DEFAULT_QUEUE_AGE_MINUTES +): WatchSpec { + return withNote({ + kind: "queue_oldest_age", + queue: queueName, + thresholdMinutes, + checkEveryMinutes: 5, + maxHours: 1, + }); +} + +export function errorWatchRecommendation(errorFriendlyId: string): WatchSpec { + return withNote({ + kind: "error_recurrence", + fingerprint: errorFriendlyId, + checkEveryMinutes: 5, + maxHours: 6, + }); +} + +/** Only offered on a degraded report. `fromSeverity` is what the recovery is measured from. */ +export function healthWatchRecommendation(fromSeverity: "warn" | "crit"): WatchSpec { + return withNote({ + kind: "health_recovery", + report: "health", + fromSeverity, + checkEveryMinutes: 5, + maxHours: 2, + }); +} diff --git a/apps/webapp/app/components/queues/queue-name.test.ts b/apps/webapp/app/components/queues/queue-name.test.ts new file mode 100644 index 00000000000..6180e26ef4c --- /dev/null +++ b/apps/webapp/app/components/queues/queue-name.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { storedQueueName } from "./queue-name"; + +describe("storedQueueName", () => { + it("adds the prefix a task queue is stored with", () => { + expect(storedQueueName({ type: "task", name: "my-task" })).toBe("task/my-task"); + }); + + it("keeps a prefix that is already there", () => { + expect(storedQueueName({ type: "task", name: "task/my-task" })).toBe("task/my-task"); + }); + + // Malformed input reaches this, and the contract is one prefix, not "one fewer than it had". + it("leaves one prefix however many the name arrived with", () => { + expect(storedQueueName({ type: "task", name: "task/task/my-task" })).toBe("task/my-task"); + expect(storedQueueName({ type: "task", name: "task/task/task/my-task" })).toBe("task/my-task"); + }); + + it("leaves a custom queue alone, prefix-shaped name and all", () => { + expect(storedQueueName({ type: "custom", name: "my-queue" })).toBe("my-queue"); + expect(storedQueueName({ type: "custom", name: "task/my-queue" })).toBe("task/my-queue"); + }); +}); diff --git a/apps/webapp/app/components/queues/queue-name.ts b/apps/webapp/app/components/queues/queue-name.ts new file mode 100644 index 00000000000..7aa81acc60c --- /dev/null +++ b/apps/webapp/app/components/queues/queue-name.ts @@ -0,0 +1,7 @@ +/** + * `TaskQueue.name` as the engine and the watch checks store it. A task queue keeps its + * `task/` prefix there, and the presenters strip it for display only. + */ +export function storedQueueName(queue: { type: string; name: string }): string { + return queue.type === "task" ? `task/${queue.name.replace(/^(?:task\/)+/, "")}` : queue.name; +} diff --git a/apps/webapp/app/components/queues/queue-thresholds.ts b/apps/webapp/app/components/queues/queue-thresholds.ts index 03e68ddc54b..910975afc72 100644 --- a/apps/webapp/app/components/queues/queue-thresholds.ts +++ b/apps/webapp/app/components/queues/queue-thresholds.ts @@ -1,4 +1,4 @@ -/** Head-of-line wait at which a queue reads as stuck. */ +/** Head-of-line wait at which a queue reads as stuck. Shared by the queue page and the watch card. */ export const OLDEST_WAIT_WARNING_MS = 5 * 60_000; export type QueueCapacity = { diff --git a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts index 83ab09c177c..dff916a6aa1 100644 --- a/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts @@ -18,6 +18,7 @@ export const ApiAlertType = z.enum([ "deployment_failure", "deployment_success", "error_group", + "dashboard_agent_watch", ]); export type ApiAlertType = z.infer; @@ -88,6 +89,8 @@ export class ApiAlertChannelPresenter { return "deployment_success"; case "ERROR_GROUP": return "error_group"; + case "DASHBOARD_AGENT_WATCH": + return "dashboard_agent_watch"; default: assertNever(alertType); } @@ -105,6 +108,8 @@ export class ApiAlertChannelPresenter { return "DEPLOYMENT_SUCCESS"; case "error_group": return "ERROR_GROUP"; + case "dashboard_agent_watch": + return "DASHBOARD_AGENT_WATCH"; default: assertNever(alertType); } diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts b/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts new file mode 100644 index 00000000000..df3c791a82e --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/block-text.ts @@ -0,0 +1,98 @@ +/** + * A view block or a resolved watch as plain text. + * + * The panel renders blocks as React; an email, a Slack message, a webhook body or + * a log line cannot. Rather than each of those re-saying the block's contents in + * its own words, they render it here. Pure, no React, no request context. + */ +import type { ViewBlock } from "@internal/dashboard-agent-contracts"; +import { presentResolvedWatch, watchNoteLine, type WatchResolvedInput } from "./watch-wording"; + +/** A labelled scalar the check observed. */ +export type TextFact = { label: string; value: string }; + +/** Facts as one `Label: value` line each. */ +export function renderFactLines(facts: readonly TextFact[]): string[] { + return facts.map((fact) => `${fact.label}: ${fact.value}`); +} + +function lines(...parts: Array): string { + return parts + .filter((part): part is string => typeof part === "string" && part.length > 0) + .join("\n"); +} + +/** + * One view block as plain text. Says what the card says and nothing more — no + * surface may add a sentence of its own on top. + */ +export function renderBlockAsText(block: ViewBlock): string { + switch (block.type) { + case "watch_result": + return lines(block.headline, block.lifetime, block.detail, ...block.followUp); + + case "diagnosis": + return lines( + block.summary, + `Likely cause: ${block.likelyCause}`, + `Confidence: ${block.confidence}`, + block.impact ? `Impact: ${block.impact}` : null, + ...block.evidence.map( + (item) => + `Evidence (${item.type}): ${item.detail}${item.reference ? ` — ${item.reference}` : ""}` + ), + ...block.nextSteps.map((step, index) => `${index + 1}. ${step}`) + ); + + case "investigation": { + const state = block.investigation; + return lines( + state.title, + state.headline, + `Outcome: ${state.outcome} · severity ${state.severity} · confidence ${state.confidence}`, + ...state.hypotheses.map( + (hypothesis) => + `${hypothesis.statement} — ${hypothesis.verdict}${ + hypothesis.finding ? `: ${hypothesis.finding}` : "" + }` + ), + state.remediation ? `Fix: ${state.remediation}` : null, + ...(state.checkNext ?? []).map((step) => `Check next: ${step}`), + state.caveat ? `Caveat: ${state.caveat.message}` : null + ); + } + + case "report": { + const { vm } = block; + return lines( + `${vm.title} report for ${vm.scope} (${vm.period}): ${vm.summary.severity}`, + ...vm.findings.map((finding) => `${finding.type} — ${finding.severity}: ${finding.reason}`) + ); + } + + // A chart is its shape, not its rows: the rows come from running the query. + case "chart": + return lines(`Chart: ${block.title ?? "untitled"} (${block.chartType})`, block.query); + + case "actions": + return lines(...block.actions.map((action) => `- ${action.label}`)); + + default: { + const unreachable: never = block; + throw new Error(`Unhandled view block: ${JSON.stringify(unreachable)}`); + } + } +} + +/** + * A resolved watch as plain text: the fact, why it was being watched, then what the + * resolving check saw. What the email body and the Slack message both say. + */ +export function renderResolvedWatchAsText(args: { + resolved: WatchResolvedInput; + note: string; + facts: readonly TextFact[]; +}): string { + const { headline } = presentResolvedWatch(args.resolved); + return lines(headline, watchNoteLine(args.note), ...renderFactLines(args.facts)); +} diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/index.ts b/apps/webapp/app/presenters/v3/dashboardAgent/index.ts new file mode 100644 index 00000000000..07d781dc6db --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/index.ts @@ -0,0 +1,5 @@ +// The dashboard agent's presenter: the one place a watch, a view block or a watch +// result becomes English. Every surface (card, banner, toast, email, Slack, +// webhook) imports from here. +export * from "./block-text"; +export * from "./watch-wording"; diff --git a/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts b/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts new file mode 100644 index 00000000000..a0da0585206 --- /dev/null +++ b/apps/webapp/app/presenters/v3/dashboardAgent/watch-wording.ts @@ -0,0 +1,39 @@ +/** + * The watch vocabulary moved into the contracts package so the agent's own + * deterministic narration says the same sentences the dashboard does — the agent + * cannot import the webapp, and a second vocabulary would drift within a release. + * + * Re-exported here because every webapp surface imports the presenter, not contracts. + */ +export { + formatWatchCadence, + formatWatchDuration, + formatWatchSla, + formatWatchWait, + formatWatchWindow, + immediateWatchMessage, + noteFor, + presentResolvedWatch, + WATCH_IN_CHAT_DELIVERY_LINE, + WATCH_PRESENTATION_FALLBACK, + WATCH_UPDATE_LABEL, + shortFingerprint, + watchConditionLabel, + watchConditionWording, + watchConfirmationBlockBody, + watchDurationLabel, + watchExternalNotificationLine, + watchFollowUpLines, + watchIdentityValue, + watchLifetimeSentence, + watchNoteLine, + watchOneShotBlockBody, + watchRequestSentence, + watchSubjectLabel, + watchSubline, + watchTooltipLabel, + type WatchConditionWording, + type WatchPresentation, + type WatchResolvedInput, + type WatchSemanticIcon, +} from "@internal/dashboard-agent-contracts"; diff --git a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts index d6a8f2ebd9c..fbe416db338 100644 --- a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts @@ -11,7 +11,7 @@ import { type ReportViewModel } from "./report-view-model"; const DEFAULT_PERIOD = "1h"; -/** How long a finished report stays reusable. */ +/** How long a finished report stays reusable. Must stay under the watch tick cadence. */ export const REPORT_CACHE_TTL_MS = 90_000; /** How many report, environment and period triples one instance keeps. */ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx index e0ee26255b7..41dd31c5d17 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx @@ -53,9 +53,13 @@ export const meta = pageMeta("New alert"); const FormSchema = z .object({ alertTypes: z - .array(z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])) + .array( + z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS", "DASHBOARD_AGENT_WATCH"]) + ) .min(1) - .or(z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])), + .or( + z.enum(["TASK_RUN", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS", "DASHBOARD_AGENT_WATCH"]) + ), environmentTypes: z .array(z.enum(["STAGING", "PRODUCTION", "PREVIEW"])) .min(1) @@ -456,6 +460,18 @@ export default function Page() { defaultChecked /> +
+ + +
+ {alertTypes.errors} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx index a2d77e99edc..7ce5500a875 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx @@ -570,6 +570,8 @@ export function alertTypeTitle(alertType: ProjectAlertType): string { return "Deployment success"; case "ERROR_GROUP": return "Error group"; + case "DASHBOARD_AGENT_WATCH": + return "Dashboard agent watches"; default: { throw new Error(`Unknown alertType: ${alertType}`); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index ff69c89c52b..1946312aac3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -24,6 +24,8 @@ import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; import { RunsIcon } from "~/assets/icons/RunsIcon"; import { CodeBlock } from "~/components/code/CodeBlock"; import { InvestigateButton } from "~/components/dashboard-agent/InvestigateButton"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { errorWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { errorGroupPrompt } from "~/components/dashboard-agent/investigate-prompts"; import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge"; import { @@ -586,7 +588,7 @@ function ErrorDetailSidebar({
Details - {/* Self-hides when the agent isn't available. */} + {/* Both buttons self-hide when the agent isn't available. */}
+
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 89521fff2f7..86b16340dcd 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -8,6 +8,9 @@ import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { AnimatedOrgBannerBar } from "~/components/billing/AnimatedOrgBannerBar"; import { BigNumber } from "~/components/metrics/BigNumber"; import { Header3 } from "~/components/primitives/Headers"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { queueWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; +import { storedQueueName } from "~/components/queues/queue-name"; import { isQueueDegraded, OLDEST_WAIT_WARNING_MS } from "~/components/queues/queue-thresholds"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Spinner } from "~/components/primitives/Spinner"; @@ -121,7 +124,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } const queue = retrieve.queue; - const fullName = queue.type === "task" ? `task/${queue.name}` : queue.name; + const fullName = storedQueueName(queue); const maxPeriodDays = await queueMetricsMaxPeriodDays(environment.organizationId); @@ -333,14 +336,20 @@ export default function Page() { maxPeriodDays={maxPeriodDays} shortcut={{ key: "d" }} /> - {/* Self-hides when the agent isn't available. */} + {/* Both buttons self-hide when the agent isn't available. Watch is + pre-filled with this queue's recommendation. */} {degraded ? ( ) : null} + {/* A paused queue can't drain or grow, so every watch it could offer is a + promise nothing will keep until someone resumes it. */} + {queue.paused ? null : ( + + )} { }) : null; + // One narrow read per page load, so the wake signal reaches a browser that has never opened + // the panel — including one whose watch hasn't fired yet. The poll never asks for this. + let dashboardAgentActivity: DashboardAgentWakeActivity = { + unreadWakes: 0, + hasActiveWatches: false, + }; + let dashboardAgentUnreadWork = 0; + if (hasDashboardAgentAccess) { + try { + [dashboardAgentActivity, dashboardAgentUnreadWork] = await Promise.all([ + readDashboardAgentWakeActivity(dashboardAgentDb, { + organizationId: project.organization.id, + userId: user.id, + }), + countChatsWithUnreadWork(dashboardAgentDb, { + organizationId: project.organization.id, + userId: user.id, + }), + ]); + } catch (error) { + // The dashboard must load even when the agent's store doesn't answer. + logger.error("Failed to read dashboard agent wake activity", { error }); + } + } + return { ...project, hasDashboardAgentAccess, promotedDashboardAgentPrompt, + dashboardAgentActivity, + dashboardAgentUnreadWork, }; }; export default function Page() { - const { hasDashboardAgentAccess, promotedDashboardAgentPrompt } = useLoaderData(); + const { + hasDashboardAgentAccess, + promotedDashboardAgentPrompt, + dashboardAgentActivity, + dashboardAgentUnreadWork, + } = useLoaderData(); return ( diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts new file mode 100644 index 00000000000..10cf14d6abb --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.$channelId.ts @@ -0,0 +1,100 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { resolveAgentAlertContext } from "~/services/dashboardAgentAlertContext.server"; +import { unsubscribeChannelFromWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * `DELETE /api/v1/dashboard-agent/alerts/:channelId` — stop alerting this channel + * when a watch fires. The channel is looked up scoped to the chat's project. + */ + +const ParamsSchema = z.object({ channelId: z.string().min(1) }); + +const BodySchema = z.object({ + chatId: z.string().min(1), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "DELETE") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const authentication = await authenticateUatOrApiRequest(request); + if (!authentication?.userActor) { + return json({ error: "Invalid or missing access token" }, { status: 401 }); + } + if (authentication.userActor.client !== "dashboard-agent") { + return json({ error: "Not allowed", code: "forbidden_client" }, { status: 403 }); + } + const userId = authentication.userActor.userId; + // The turn's environment scope is the authority for the chat's project below. + const environmentId = authentication.userActor.environmentId; + if (!environmentId) { + return json( + { error: "This chat has no environment context.", code: "invalid_target" }, + { status: 400 } + ); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + const body = parsedBody.data; + + try { + const context = await resolveAgentAlertContext({ + userId, + environmentId, + chatId: body.chatId, + claimedEnvironmentId: body.environmentId, + claimedProjectRef: body.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: context.code === "environment_mismatch" ? 400 : 404 } + ); + } + + const result = await unsubscribeChannelFromWatchAlerts(parsedParams.data.channelId, { + projectId: context.environment.project.id, + // A project is shared by every member, so the caller's own address is part of the scope. + organizationId: context.environment.organizationId, + ownerUserId: userId, + }); + if (!result.ok) { + if (result.reason === "conflict") { + return json( + { error: "That alert was being changed elsewhere. Try again.", code: "conflict" }, + { status: 409 } + ); + } + return json({ error: "Alert not found", code: "not_found" }, { status: 404 }); + } + + return json({ ok: true, disabledChannel: result.disabledChannel }); + } catch (error) { + logger.error("Failed to unsubscribe a channel from dashboard agent watch alerts", { + error, + userId, + environmentId, + channelId: parsedParams.data.channelId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts new file mode 100644 index 00000000000..71c7d423292 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.alerts.ts @@ -0,0 +1,221 @@ +import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, +} from "~/models/projectAlert.server"; +import { + resolveAgentAlertContext, + type AgentAlertContextError, +} from "~/services/dashboardAgentAlertContext.server"; +import { + canUseDashboardAgentEmailAlerts, + DASHBOARD_AGENT_WATCH_ALERT_TYPE, + subscribeChannelToWatchAlerts, + watchAlertDeduplicationKey, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * `GET` lists this chat's project's watch alerts; `POST` subscribes the user's email. Only + * the agent's delegated user-actor token is accepted, and the environment comes from it. + */ + +const ListQuerySchema = z.object({ + chatId: z.string().min(1), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +const CreateBodySchema = z.object({ + chatId: z.string().min(1), + channel: z.literal("email"), + /** May only be the authenticated user's own account email. */ + email: z.string().email().optional(), + environmentId: z.string().min(1).optional(), + projectRef: z.string().min(1).optional(), +}); + +/** A token without an environment scope is unusable here. */ +async function authenticate( + request: Request +): Promise<{ userId: string; environmentId: string } | { error: Response }> { + const authentication = await authenticateUatOrApiRequest(request); + const actor = authentication?.userActor; + if (!actor || actor.client !== "dashboard-agent") { + return { error: json({ error: "Invalid or missing access token" }, { status: 401 }) }; + } + if (!actor.environmentId) { + return { + error: json( + { error: "This chat has no environment context.", code: "invalid_target" }, + { status: 400 } + ), + }; + } + return { userId: actor.userId, environmentId: actor.environmentId }; +} + +/** A mismatched claim is the caller's error, the rest are 404s. */ +function contextStatus(code: AgentAlertContextError) { + return code === "environment_mismatch" ? 400 : 404; +} + +export async function loader({ request }: LoaderFunctionArgs) { + const auth = await authenticate(request); + if ("error" in auth) return auth.error; + + const query = ListQuerySchema.safeParse( + Object.fromEntries(new URL(request.url).searchParams.entries()) + ); + if (!query.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const context = await resolveAgentAlertContext({ + userId: auth.userId, + environmentId: auth.environmentId, + chatId: query.data.chatId, + claimedEnvironmentId: query.data.environmentId, + claimedProjectRef: query.data.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: contextStatus(context.code) } + ); + } + + const channels = await $replica.projectAlertChannel.findMany({ + where: { + projectId: context.environment.project.id, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + select: { id: true, type: true, enabled: true, properties: true, environmentTypes: true }, + orderBy: { createdAt: "asc" }, + }); + + return json({ + alerts: channels.map((channel) => ({ + id: channel.id, + type: channel.type, + enabled: channel.enabled, + environmentTypes: channel.environmentTypes, + target: describeTarget(channel.type, channel.properties), + })), + }); +} + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const auth = await authenticate(request); + if ("error" in auth) return auth.error; + const { userId } = auth; + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + + const parsed = CreateBodySchema.safeParse(rawBody); + if (!parsed.success) { + return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 }); + } + const body = parsed.data; + + const context = await resolveAgentAlertContext({ + userId, + environmentId: auth.environmentId, + chatId: body.chatId, + claimedEnvironmentId: body.environmentId, + claimedProjectRef: body.projectRef, + }); + if (!context.ok) { + return json( + { error: context.error, code: context.code }, + { status: contextStatus(context.code) } + ); + } + const { environment } = context; + + const gate = await canUseDashboardAgentEmailAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.project.id, + }); + if (!gate.allowed) { + return json({ error: "Alerts are not available here", code: gate.reason }, { status: 403 }); + } + + // Only the signed-in user's own account email may be subscribed. Read off the + // primary: this is the identity the subscription is pinned to. + const user = await prisma.user.findFirst({ where: { id: userId }, select: { email: true } }); + if (!user) { + return json({ error: "User not found", code: "invalid_request" }, { status: 404 }); + } + const email = user.email; + if (body.email && body.email.trim().toLowerCase() !== email.toLowerCase()) { + return json( + { + error: + "Watch alerts can only go to your own account email. Ask the user to add another address on the Alerts page.", + code: "email_not_allowed", + }, + { status: 400 } + ); + } + + try { + const channel = await subscribeChannelToWatchAlerts({ + userId, + email, + // Stable per (email, project), so asking twice re-enables one channel and asking from + // a second environment adds that environment to it. + deduplicationKey: watchAlertDeduplicationKey(email), + environmentType: environment.type, + project: environment.project, + }); + + return json({ id: channel.id, type: channel.type, target: email, enabled: channel.enabled }); + } catch (error) { + // A thrown Response is Remix control flow, not a failure to report. + if (error instanceof Response) throw error; + logger.error("Failed to create a dashboard agent watch alert channel", { + error, + userId, + organizationId: environment.organizationId, + projectId: environment.project.id, + environmentId: environment.id, + }); + return json({ error: "Internal Server Error", code: "internal" }, { status: 500 }); + } +} + +/** A short, non-secret description of where a channel delivers. */ +function describeTarget(type: string, properties: unknown): string | undefined { + if (type === "EMAIL") { + const parsed = ProjectAlertEmailProperties.safeParse(properties); + return parsed.success ? maskEmail(parsed.data.email) : undefined; + } + if (type === "SLACK") { + const parsed = ProjectAlertSlackProperties.safeParse(properties); + return parsed.success ? `#${parsed.data.channelName}` : undefined; + } + // Webhook URLs stay out of the agent's context entirely. + return undefined; +} + +function maskEmail(email: string): string { + const [local, domain] = email.split("@"); + if (!domain || !local) return "an email address"; + const head = local.slice(0, 2); + return `${head}${local.length > 2 ? "…" : ""}@${domain}`; +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts new file mode 100644 index 00000000000..406f0b5b2e5 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.check.ts @@ -0,0 +1,191 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { cancelWatch, getWatch, recordWatchCheck } from "@internal/dashboard-agent-db"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { checkWatch, previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, +} from "~/services/dashboardAgentWatches.server"; +import { + WATCH_TOKEN_GRACE_MS, + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * Private per-watch check. The token only names a watch; the row is the authority on + * lifecycle and its snapshot, and this route transitions nothing and advances no tick. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +/** Best-effort: a chain that couldn't be armed returns `false` and is retried next check. */ +async function ensureBatchChain(watch: { + id: string; + environmentId: string; + spec: { checkEveryMinutes: number }; +}): Promise { + try { + const { running } = await armDashboardAgentWatchBatch({ + environmentId: watch.environmentId, + cadenceMinutes: watch.spec.checkEveryMinutes, + }); + return running; + } catch (error) { + logger.error("Dashboard agent watch check: couldn't arm the batch chain", { + watchId: watch.id, + environmentId: watch.environmentId, + error, + }); + return false; + } +} + +const BodySchema = z.object({ + /** The expiry evaluation: allowed after `expiresAt`, within the token's grace. */ + final: z.boolean().optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + // A valid token for a different watch is 403, not 401. + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + let rawBody: unknown; + try { + const raw = await request.text(); + rawBody = raw.length > 0 ? JSON.parse(raw) : {}; + } catch { + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) return json({ error: "Invalid request body" }, { status: 400 }); + const body = parsedBody.data; + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + // Terminal watches are never checked again, whatever the token says. + if (watch.status !== "active") { + return json( + { + error: `This watch is ${watch.status}`, + code: watch.status === "cancelled" ? "cancelled" : "not_active", + status: watch.status, + }, + { status: 403 } + ); + } + + const now = new Date(); + const expired = watch.expiresAt.getTime() <= now.getTime(); + if (expired) { + // Past the deadline only the final evaluation is allowed, inside the token's grace. + const graceEnds = watch.expiresAt.getTime() + WATCH_TOKEN_GRACE_MS; + if (body.final !== true || now.getTime() > graceEnds) { + return json( + { error: "This watch has expired", code: "expired", expiresAt: watch.expiresAt }, + { status: 403 } + ); + } + } + + try { + // Re-authorize the initiating user before any environment data is read. + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + // A watch must not outlive the access it was created with. Never notified. + await cancelWatch(dashboardAgentDb, { id: watchId, reason: "access_revoked" }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { status: 403 } + ); + } + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const outcome = await checkWatch( + watch.spec, + watchCheckDeps(authorization.environment, now), + // A tick that couldn't read anything freezes a streak instead of resetting it. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch check failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }) + ); + + // Recorded even on the final evaluation. Guarded on `active`, so a concurrent + // fire/expire wins and this no-ops. + await recordWatchCheck(dashboardAgentDb, { + id: watchId, + lastResult: { + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + final: body.final === true, + }, + }); + + const batched = await ensureBatchChain(watch); + + // `observed` travels with the verdict so no delivery surface re-reads the source. + return json({ + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + batched, + }); + } catch (error) { + logger.error("Dashboard agent watch check tick failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts new file mode 100644 index 00000000000..427435835c1 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.fired.ts @@ -0,0 +1,113 @@ +import { + claimWatchAlertDispatch, + getWatch, + releaseWatchAlertDispatch, +} from "@internal/dashboard-agent-db"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { enqueueWatchFiredAlert } from "~/services/dashboardAgentWatchAlerts.server"; +import { authorizeWatchEnvironment } from "~/services/dashboardAgentWatches.server"; +import { + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; +import { logger } from "~/services/logger.server"; + +/** + * The watcher task reports a fired watch. The row is the authority on whether it fired, and + * the initiating user is re-authorized against its snapshot before any alert is sent. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + // Anything that isn't a fired watch gets no alert, whatever the caller claims. + if (watch.status !== "fired" || !watch.firedAt) { + return json( + { error: `This watch is ${watch.status}`, code: "not_fired", status: watch.status }, + { status: 409 } + ); + } + + try { + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + // Not cancelled here: the watch is already terminal. + logger.info("Dashboard agent watch fired, but access was revoked; no alert", { watchId }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { + status: 403, + } + ); + } + + const claimed = await claimWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + if (!claimed) { + logger.info("Dashboard agent watch fired callback repeated; no second alert", { watchId }); + return json({ ok: true, alerted: false }); + } + + try { + await enqueueWatchFiredAlert(watch, "fired"); + } catch (error) { + await releaseWatchAlertDispatch(dashboardAgentDb, { id: watch.id, terminalStatus: "fired" }); + throw error; + } + + return json({ ok: true, alerted: true }); + } catch (error) { + logger.error("Dashboard agent watch fire callback failed", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts new file mode 100644 index 00000000000..83af5eba382 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.$watchId.investigate.ts @@ -0,0 +1,105 @@ +import { getWatch, isTerminalWatchStatus } from "@internal/dashboard-agent-db"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { + kickWatchInvestigation, + watchWantsInvestigation, +} from "~/services/dashboardAgentWatchInvestigate.server"; +import { authorizeWatchEnvironment } from "~/services/dashboardAgentWatches.server"; +import { + bearerToken, + verifyWatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; +import { logger } from "~/services/logger.server"; + +/** + * The watcher task reports a delivered wake for a pre-approved investigation. The caller's + * body is ignored: consent, outcome, user and environment all come off the row. + */ + +const ParamsSchema = z.object({ watchId: z.string().min(1) }); + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return json({ error: "Invalid params" }, { status: 400 }); + const { watchId } = parsedParams.data; + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + if (claims.watchId !== watchId) { + return json({ error: "Not allowed for this watch", code: "watch_mismatch" }, { status: 403 }); + } + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch) { + return json({ error: "Watch not found", code: "not_found" }, { status: 404 }); + } + + if (!isTerminalWatchStatus(watch.status)) { + return json( + { error: `This watch is ${watch.status}`, code: "not_resolved", status: watch.status }, + { status: 409 } + ); + } + + if (!watchWantsInvestigation(watch)) { + // No consent, or an outcome consent doesn't cover: the wake was the whole delivery. + return json({ ok: true, investigating: false }); + } + + const authorization = await authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + + if (!authorization.ok) { + logger.info("Dashboard agent watch resolved, but access was revoked; no investigation", { + watchId, + }); + return json( + { error: "Access to this environment was revoked", code: "access_revoked" }, + { status: 403 } + ); + } + + // Never an error to the caller: the wake is already delivered and marked, so a failed + // kick must not make the watcher retry. The stale-investigation sweep settles it. + try { + await kickWatchInvestigation({ watch, environment: authorization.environment }); + } catch (error) { + // A thrown Response is Remix control flow, not a failed kick. + if (error instanceof Response) throw error; + logger.error("Dashboard agent watch investigation could not be started", { + error, + watchId, + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); + return json({ ok: true, investigating: false, code: "kick_failed" }); + } + + return json({ ok: true, investigating: true }); +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts new file mode 100644 index 00000000000..ac869d68ea5 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.batch-check.ts @@ -0,0 +1,75 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { runWatchBatchCheck } from "~/services/dashboardAgentWatchBatch.server"; +import { logger } from "~/services/logger.server"; +import { + bearerToken, + verifyWatchBatchTokenFromRequest, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * Private batch check: one call per (environment, cadence) group per cadence. The token + * names the group, the body names the tick. `runWatchBatchCheck` documents the rest. + */ + +const BodySchema = z.object({ + environmentId: z.string().min(1), + cadenceMinutes: z.number().int().positive(), + epoch: z.number().int().nonnegative(), + tick: z.number().int().positive(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const token = bearerToken(request); + if (!token) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + const claims = await verifyWatchBatchTokenFromRequest(token); + if (!claims) { + return json( + { error: "Invalid or missing access token", code: "unauthorized" }, + { status: 401 } + ); + } + + let rawBody: unknown; + try { + const raw = await request.text(); + rawBody = raw.length > 0 ? JSON.parse(raw) : {}; + } catch { + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) return json({ error: "Invalid request body" }, { status: 400 }); + const body = parsedBody.data; + + // A valid token for a different group is 403, not 401. + if ( + claims.environmentId !== body.environmentId || + claims.cadenceMinutes !== body.cadenceMinutes + ) { + return json({ error: "Not allowed for this group", code: "group_mismatch" }, { status: 403 }); + } + + try { + return json(await runWatchBatchCheck(body)); + } catch (error) { + logger.error("Dashboard agent watch batch check failed", { + error, + environmentId: body.environmentId, + cadenceMinutes: body.cadenceMinutes, + epoch: body.epoch, + tick: body.tick, + }); + throw error; + } +} diff --git a/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts new file mode 100644 index 00000000000..0fbe83e3463 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.dashboard-agent.watches.ts @@ -0,0 +1,163 @@ +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { watchSpecSchema } from "@internal/dashboard-agent-contracts"; +import { z } from "zod"; +import { logger } from "~/services/logger.server"; +import { resolveWatchEmailAlertsState } from "~/services/dashboardAgentWatchAlerts.server"; +import { + authorizeWatchEnvironmentById, + createDashboardAgentWatch, + resolveChatWatchContext, +} from "~/services/dashboardAgentWatches.server"; +import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server"; + +/** + * Programmatic watch creation (MCP). Only the agent's delegated user-actor token is + * accepted, and the environment comes from it, never the body or the chat's stored context. + */ + +const BodySchema = z.object({ + spec: watchSpecSchema, + chatId: z.string().min(1), + /** Consent for the wake turn to open an investigation. Off unless explicitly sent. */ + investigateOnAttention: z.boolean().optional(), + /** + * Only checked against the token's environment scope, never used in its place. + * `environmentId` is the canonical `RuntimeEnvironment.id`, not a slug. + */ + projectRef: z.string().min(1).optional(), + environmentId: z.string().min(1).optional(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const authentication = await authenticateUatOrApiRequest(request); + if (!authentication?.userActor) { + return json({ error: "Invalid or missing access token" }, { status: 401 }); + } + if (authentication.userActor.client !== "dashboard-agent") { + return json({ error: "Not allowed", code: "forbidden_client" }, { status: 403 }); + } + const userId = authentication.userActor.userId; + // The environment this turn is scoped to. There is no trusted fallback. + const environmentId = authentication.userActor.environmentId; + if (!environmentId) { + return json( + { error: "This chat has no environment context to watch in.", code: "invalid_target" }, + { status: 400 } + ); + } + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return json({ error: "Invalid watch request", code: "invalid_request" }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(rawBody); + if (!parsedBody.success) { + return json({ error: "Invalid watch request", code: "invalid_request" }, { status: 400 }); + } + const parsed = parsedBody.data; + + // Refuse a body naming a different environment rather than silently picking one. + if (parsed.environmentId && parsed.environmentId !== environmentId) { + return json( + { + error: "That environment isn't the one this chat is open in.", + code: "environment_mismatch", + }, + { status: 400 } + ); + } + + try { + // A chat this user doesn't own does not exist here. + const chat = await resolveChatWatchContext({ chatId: parsed.chatId, userId }); + if (!chat) { + return json({ error: "Chat not found", code: "chat_not_found" }, { status: 404 }); + } + + // The same authorization a background check applies. + const environment = await authorizeWatchEnvironmentById({ userId, environmentId }); + if (!environment) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + // A chat belongs to one org; its watches can't point at another org's env. + if (environment.organizationId !== chat.organizationId) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + // Same check as `environmentId`, for callers that send the project instead. + if (parsed.projectRef && environment.project.externalRef !== parsed.projectRef) { + return json( + { + error: "That project isn't the one this chat is open in.", + code: "environment_mismatch", + }, + { status: 400 } + ); + } + + const result = await createDashboardAgentWatch({ + environment, + userId, + chatId: parsed.chatId, + spec: parsed.spec, + investigateOnAttention: parsed.investigateOnAttention, + }); + + if (!result.ok) { + const status = + result.code === "limit_reached" || result.code === "duplicate" + ? 409 + : result.code === "invalid_target" + ? 404 + : // The chat was deleted while the create was in flight. + result.code === "chat_not_found" + ? 404 + : result.code === "not_configured" + ? 501 + : 500; + return json( + { + error: result.error, + code: result.code, + ...(result.existingId ? { existingId: result.existingId } : {}), + }, + { status } + ); + } + + // One-shot: the immediate check answered, so there is no watch row and no id. + if (!result.watching) { + return json({ + watching: false, + identity: result.identity, + immediate: { result: result.immediate.result, facts: result.immediate.facts }, + }); + } + + return json({ + watching: true, + watchId: result.watchId, + identity: result.identity, + status: result.status, + expiresAt: result.expiresAt.toISOString(), + emailAlerts: await resolveWatchEmailAlertsState({ userId, environment }), + ...(result.unavailable ? { unavailable: true } : {}), + }); + } catch (error) { + // A thrown Response is Remix control flow, not a failure to report. + if (error instanceof Response) throw error; + logger.error("Failed to create a dashboard agent watch", { + error, + userId, + environmentId, + chatId: parsed.chatId, + }); + return json({ error: "Internal Server Error", code: "internal" }, { status: 500 }); + } +} diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts index 31b0f486d31..33cb38a1045 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.workers.$tagName.ts @@ -70,6 +70,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { triggerSource: true, createdAt: true, payloadSchema: true, + queueConfig: true, }, orderBy: { slug: "asc", @@ -100,6 +101,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { triggerSource: task.triggerSource, createdAt: task.createdAt, payloadSchema: task.payloadSchema, + queueConfig: task.queueConfig, })), }, urls, diff --git a/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx b/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx new file mode 100644 index 00000000000..771d326e556 --- /dev/null +++ b/apps/webapp/app/routes/resources.dashboard-agent.alerts.$channelId.unsubscribe.tsx @@ -0,0 +1,164 @@ +import { EnvelopeIcon } from "@heroicons/react/24/solid"; +import { Form, useNavigation } from "@remix-run/react"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedActionData, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { AppContainer, MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { FormTitle } from "~/components/primitives/FormTitle"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { verifyUnsubscribeToken } from "~/services/dashboardAgentAlertUnsubscribeToken.server"; +import { + DASHBOARD_AGENT_WATCH_ALERT_TYPE, + unsubscribeChannelFromWatchAlerts, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { logger } from "~/services/logger.server"; +import { rootPath } from "~/utils/pathBuilder"; + +/** + * The unsubscribe link in a watch alert email. The signed token is the whole authorization + * and names one channel; GET confirms and POST acts, so a link preview can't unsubscribe. + */ + +const ParamsSchema = z.object({ channelId: z.string().min(1) }); + +async function authorize(request: Request, params: Record) { + const parsedParams = ParamsSchema.safeParse(params); + if (!parsedParams.success) return undefined; + + const token = new URL(request.url).searchParams.get("token"); + if (!token) return undefined; + + const claims = await verifyUnsubscribeToken(token); + if (!claims) return undefined; + if (claims.channelId !== parsedParams.data.channelId) return undefined; + if (claims.alertType !== DASHBOARD_AGENT_WATCH_ALERT_TYPE) return undefined; + + return claims; +} + +export async function loader({ request, params }: LoaderFunctionArgs) { + const claims = await authorize(request, params); + // The POST needs the token, and a bare `
` drops search params. + return typedjson({ + valid: claims !== undefined, + formAction: `${new URL(request.url).pathname}${new URL(request.url).search}`, + }); +} + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toUpperCase() !== "POST") { + return typedjson({ success: false as const, message: "Method not allowed" }, { status: 405 }); + } + + const claims = await authorize(request, params); + if (!claims) { + return typedjson( + { + success: false as const, + message: "This link is no longer valid, so we couldn't turn off the alerts.", + }, + { status: 403 } + ); + } + + // Read only for the failure log. The unsubscribe does its own scoped lookup. + const channel = await prisma.projectAlertChannel.findFirst({ + where: { id: claims.channelId }, + select: { projectId: true }, + }); + + try { + const result = await unsubscribeChannelFromWatchAlerts(claims.channelId); + if (!result.ok) { + return result.reason === "conflict" + ? typedjson( + { + success: false as const, + message: "This alert was being changed elsewhere. Please try again.", + }, + { status: 409 } + ) + : typedjson( + { success: false as const, message: "This alert no longer exists." }, + { status: 404 } + ); + } + + return typedjson({ success: true as const, channelName: result.channelName }); + } catch (error) { + logger.error("Failed to turn off watch alerts from an email link", { + error, + channelId: claims.channelId, + projectId: channel?.projectId, + }); + throw error; + } +} + +export default function Page() { + const { valid, formAction } = useTypedLoaderData(); + const result = useTypedActionData(); + const navigation = useNavigation(); + const isLoading = navigation.state !== "idle"; + + if (result?.success) { + return ( + + + {result.channelName} will no longer be alerted when a watch fires. You can turn it back on + from the Alerts page in your project. + + + Dashboard + + + ); + } + + if (!valid || result?.success === false) { + return ( + + + {result?.success === false + ? result.message + : "This link is no longer valid. You can manage alerts from the Alerts page in your project."} + + + Dashboard + + + ); + } + + return ( + + + This stops the alerts this channel receives when a watch you set up with the dashboard agent + fires. Other alerts on the channel are unaffected. + + + + + + ); +} + +function Shell({ title, children }: { title: string; children: React.ReactNode }) { + return ( + + +
+ } + title={title} + /> + {children} +
+ + + ); +} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts index d3451960495..5c8966e7665 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.ts @@ -1,16 +1,24 @@ import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { + cancelWatch, chatExists, + countUnreadWatchWakes, + countChatsWithUnreadWork, countUserMessages, createChat, getChatMessages, getSession, + getWatch, listChatIdsWithOpenInvestigations, + listChatIdsWithUnreadWakes, listChats, + markChatRead, + readWatchWakeFeed, renameChat, setChatPinned, softDeleteChat, } from "@internal/dashboard-agent-db"; +import { watchDraftSchema, type WatchDraft } from "@internal/dashboard-agent-contracts"; import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import type { UIMessage } from "ai"; import { z } from "zod"; @@ -26,8 +34,15 @@ import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { + authorizeWatchEnvironmentById, + deleteChatWithWatches, + listActiveWatchesForChats, + submitDashboardAgentWatch, +} from "~/services/dashboardAgentWatches.server"; import { dashboardAgentApiOrigin, + dashboardAgentWakeFeedCounter, isDashboardAgentConfigured, mintDashboardAgentToken, mintDashboardAgentUserActorToken, @@ -54,8 +69,11 @@ const ActionBody = z.object({ "rename", "pin", "delete", + "read", "resolve", "resolve-many", + "watch-cancel", + "watch-create", ]), // Omitted for `create` (the server generates it); required for the rest. chatId: z.string().min(1).optional(), @@ -68,10 +86,17 @@ const ActionBody = z.object({ uri: z.string().optional(), // A JSON array of `trigger://` URIs, for `resolve-many`. uris: z.string().optional(), + // The watch to cancel, for `watch-cancel`. + watchId: z.string().min(1).optional(), + // The configured card, for `watch-create`: a JSON `WatchDraft`. + draft: z.string().optional(), + // Stable per card submission, so a retried `watch-create` repairs instead of repeating. + // Required for `watch-create`: see the check in that branch. + clientRequestId: z.string().min(1).max(64).optional(), }); // History list by default. `?chatId=` returns the stored transcript plus session, -// `?quota=1` the message count. +// `?unread=1` the unread wake count and recent wakes, `?quota=1` the message count. export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const userId = user.id; @@ -90,6 +115,36 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const searchParams = new URL(request.url).searchParams; + // The wake poll runs once a minute per open tab, so it reads only the org id it needs + // and asks the agent DB one question. The list is recent deliveries, not unread ones; + // the client dedupes by id. + if (searchParams.get("unread") === "1") { + dashboardAgentWakeFeedCounter.inc(); + const scoped = await $replica.project.findFirst({ + where: { + slug: projectParam, + organization: { slug: organizationSlug, members: { some: { userId } } }, + }, + select: { organizationId: true }, + }); + if (!scoped) return json({ error: "Project not found" }, { status: 404 }); + + const [feed, unreadWork] = await Promise.all([ + readWatchWakeFeed(dashboardAgentDb, { + organizationId: scoped.organizationId, + userId, + deliveredAfter: new Date(Date.now() - 15 * 60 * 1000), + }), + // The dot has two sources; the poll is where a closed panel learns about either. + countChatsWithUnreadWork(dashboardAgentDb, { + organizationId: scoped.organizationId, + userId, + }), + ]); + + return json({ ...feed, unreadWork }); + } + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) return json({ error: "Project not found" }, { status: 404 }); @@ -118,17 +173,45 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { userId, }); - // One query for all the listed chats, never one per row. - const investigatingChatIds = await listChatIdsWithOpenInvestigations(dashboardAgentDb, { - organizationId: project.organizationId, - userId, - }); + // One query each for all the listed chats, never one per row. + const [watchesByChat, unreadWakes, unreadChatIds, investigatingChatIds] = await Promise.all([ + listActiveWatchesForChats({ + chatIds: chats.map((chat) => chat.id), + organizationId: project.organizationId, + userId, + }), + countUnreadWatchWakes(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + listChatIdsWithUnreadWakes(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + listChatIdsWithOpenInvestigations(dashboardAgentDb, { + organizationId: project.organizationId, + userId, + }), + ]); return json({ - chats: chats.map((chat) => ({ - ...chat, - hasOpenInvestigation: investigatingChatIds.has(chat.id), - })), + chats: chats.map((chat) => { + const watches = watchesByChat[chat.id] ?? []; + return { + ...chat, + watches, + hasUnreadWake: unreadChatIds.has(chat.id), + // Work that finished while the chat was closed: the transcript moved on after the + // last time its owner looked. A wake is one way that happens, an answer is another. + hasUnreadWork: + chat.lastMessageAt !== null && + (chat.lastReadAt === null || chat.lastMessageAt > chat.lastReadAt), + // `watches` also carries fired and expired rows, so check for active here. + hasActiveWatch: watches.some((watch) => watch.status === "active"), + hasOpenInvestigation: investigatingChatIds.has(chat.id), + }; + }), + unreadWakes, }); }; @@ -376,6 +459,91 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json({ resolved }); } + // The configuration card's submit path. The environment comes from the URL and goes + // through the same re-authorization a background tick passes, never from the body. + if (parsed.data.intent === "watch-create") { + // No fallback: a per-condition key would identify the condition rather than this + // submit, so a re-watch could replay a stale terminal outcome. + const clientRequestId = parsed.data.clientRequestId; + if (!clientRequestId) { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + + let draft: WatchDraft; + try { + const result = watchDraftSchema.safeParse(JSON.parse(parsed.data.draft ?? "")); + if (!result.success) { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + draft = result.data; + } catch { + return json({ error: "That watch isn't valid.", code: "invalid_request" }, { status: 400 }); + } + + const runtimeEnv = await findEnvironmentBySlug(project.id, envParam, userId); + if (!runtimeEnv) return json({ error: "Environment not found" }, { status: 404 }); + + const environment = await authorizeWatchEnvironmentById({ + userId, + environmentId: runtimeEnv.id, + }); + if (!environment) { + return json({ error: "Environment not found", code: "invalid_target" }, { status: 404 }); + } + + // A watch is chat-bound, so a card submitted from a fresh panel creates a chat. + const targetChatId = parsed.data.chatId; + if ( + targetChatId && + !(await chatExists(dashboardAgentDb, { + chatId: targetChatId, + userId, + organizationId: project.organizationId, + })) + ) { + return json({ error: "Chat not found", code: "chat_not_found" }, { status: 404 }); + } + + // The request record is written before the watch and the confirmation after, so a + // half-finished submit is repairable and never leaves a watch nobody can see. + const result = await submitDashboardAgentWatch({ + environment, + userId, + organizationId: project.organizationId, + chatId: targetChatId, + clientRequestId, + draft, + }); + + if (!result.ok) { + const status = + result.code === "limit_reached" || + result.code === "duplicate" || + result.code === "request_conflict" + ? 409 + : result.code === "invalid_target" || result.code === "chat_not_found" + ? 404 + : result.code === "not_configured" + ? 501 + : 500; + return json( + { + error: result.error, + code: result.code, + ...(result.existingId ? { existingId: result.existingId } : {}), + }, + { status } + ); + } + + return json({ + chatId: result.chatId, + watching: result.watching, + watchId: result.watchId, + messages: result.messages, + }); + } + const { intent, chatId } = parsed.data; if (!chatId) return json({ error: "chatId is required" }, { status: 400 }); @@ -452,9 +620,44 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json({ ok: true }); } + // The update is owner-scoped, so a chatId the caller doesn't own is a no-op. + case "read": { + await markChatRead(dashboardAgentDb, { + chatId, + userId, + organizationId: project.organizationId, + }); + return json({ ok: true }); + } + case "delete": { - // `softDeleteChat` is owner-scoped but takes no org, so the org scope has to be - // enforced here. + // `deleteChatWithWatches` is owner-scoped but takes no org, so the org scope has + // to be enforced here. + if ( + !(await chatExists(dashboardAgentDb, { + chatId, + userId, + organizationId: project.organizationId, + })) + ) { + return json({ error: "Chat not found" }, { status: 404 }); + } + // The delete and the watch cancellations land in one transaction. + const { cancelledWatches } = await deleteChatWithWatches({ chatId, userId }); + return json({ ok: true, cancelledWatches }); + } + + // Ownership goes through the chat: the watch must belong to the named chat, and + // that chat to this user in this org. + case "watch-cancel": { + const watchId = parsed.data.watchId; + if (!watchId) return json({ error: "watchId is required" }, { status: 400 }); + + const watch = await getWatch(dashboardAgentDb, { id: watchId }); + if (!watch || watch.chatId !== chatId) { + return json({ error: "Watch not found" }, { status: 404 }); + } + if ( !(await chatExists(dashboardAgentDb, { chatId, @@ -464,7 +667,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { ) { return json({ error: "Chat not found" }, { status: 404 }); } - await softDeleteChat(dashboardAgentDb, { chatId, userId }); + + // `cancelWatch` only touches an active row, so an already-resolved watch keeps + // its outcome and this is a no-op. + await cancelWatch(dashboardAgentDb, { id: watchId, reason: "user" }); return json({ ok: true }); } } diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 288a0c27a84..374440dfa0b 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -33,6 +33,9 @@ import { MachineLabelCombo } from "~/components/MachineLabelCombo"; import { MachineTooltipInfo } from "~/components/MachineTooltipInfo"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { InvestigateButton } from "~/components/dashboard-agent/InvestigateButton"; +import { WatchButton } from "~/components/dashboard-agent/WatchButton"; +import { isFinalRunStatus } from "~/v3/taskStatus"; +import { runWatchRecommendation } from "~/components/dashboard-agent/watch-recommendations"; import { failedRunPrompt, isFailedRunStatus, @@ -1147,6 +1150,16 @@ function RunBody({ runFriendlyId={run.friendlyId} /> ) : null} + {/* The universal `Watch…` entry (§2.1), pre-filled with this run's + recommendation: tell me when it finishes. Only while the run can + still change — a finished run has nothing left to wait for. */} + {isFinalRunStatus(run.status) ? null : ( + + )} {run.error && ( diff --git a/apps/webapp/app/services/dashboardAgent.server.ts b/apps/webapp/app/services/dashboardAgent.server.ts index 5ae9eb25728..1c2e0b5f1cf 100644 --- a/apps/webapp/app/services/dashboardAgent.server.ts +++ b/apps/webapp/app/services/dashboardAgent.server.ts @@ -1,25 +1,44 @@ import { signUserActorToken } from "@trigger.dev/rbac"; import { TriggerClient } from "@trigger.dev/sdk"; import { chat } from "@trigger.dev/sdk/ai"; +import { Counter } from "prom-client"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; +import { metricsRegister } from "~/metrics.server"; +import { singleton } from "~/utils/singleton"; import { runStore } from "~/v3/runStore.server"; import { githubApp } from "./gitHub.server"; import { logger } from "./logger.server"; const TASK_ID = "dashboard-agent"; +// The wake poll runs once a minute per visible tab. That trade-off is only defensible while it +// stays measurable, so count the requests. singleton: module-scope registration double-registers +// under dev HMR. +export const dashboardAgentWakeFeedCounter = singleton( + "dashboardAgentWakeFeedCounter", + () => + new Counter({ + name: "dashboard_agent_wake_feed_requests_total", + help: "Requests to the dashboard agent's wake feed", + registers: [metricsRegister], + }) +); + // Read-only cap on the agent's delegated user-actor token. `read:apiKeys` is // what lets it exchange the token for an env JWT (the gate on the exchange // route); the rest scope the actual reads. No write/admin scopes, so even a // leaked token can't mutate anything. -const DASHBOARD_AGENT_UAT_CAP = [ +export const DASHBOARD_AGENT_UAT_CAP = [ "read:apiKeys", "read:runs", "read:deployments", "read:environments", "read:errors", "read:query", + // Queue metrics ride on `read:query`, but a queue's own row — paused, depth, limit — + // is a `queues` read, and without it the agent can only see the metrics window. + "read:queues", ]; // Minted fresh on every turn (the `in` proxy injects it), so the lifetime only diff --git a/apps/webapp/app/services/dashboardAgentAlertContext.server.ts b/apps/webapp/app/services/dashboardAgentAlertContext.server.ts new file mode 100644 index 00000000000..7266991265f --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentAlertContext.server.ts @@ -0,0 +1,57 @@ +/** + * From the turn's environment scope and a chat id to an authorized environment. Same order + * of authority as the watches route: token environment, chat ownership, re-authorization. + */ + +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + authorizeWatchEnvironmentById, + resolveChatWatchContext, +} from "~/services/dashboardAgentWatches.server"; + +export type AgentAlertContextError = "chat_not_found" | "invalid_target" | "environment_mismatch"; + +export type AgentAlertContext = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; code: AgentAlertContextError; error: string }; + +export async function resolveAgentAlertContext(params: { + userId: string; + chatId: string; + /** The turn's environment scope, off the user-actor token. The authority here. */ + environmentId: string; + /** Optional echoes from the request body. Checked, never trusted. */ + claimedEnvironmentId?: string; + claimedProjectRef?: string; +}): Promise { + if (params.claimedEnvironmentId && params.claimedEnvironmentId !== params.environmentId) { + return { + ok: false, + code: "environment_mismatch", + error: "That environment isn't the one this chat is open in.", + }; + } + + const chat = await resolveChatWatchContext({ chatId: params.chatId, userId: params.userId }); + if (!chat) { + return { ok: false, code: "chat_not_found", error: "Chat not found" }; + } + + const environment = await authorizeWatchEnvironmentById({ + userId: params.userId, + environmentId: params.environmentId, + }); + if (!environment || environment.organizationId !== chat.organizationId) { + return { ok: false, code: "invalid_target", error: "Environment not found" }; + } + + if (params.claimedProjectRef && environment.project.externalRef !== params.claimedProjectRef) { + return { + ok: false, + code: "environment_mismatch", + error: "That project isn't the one this chat is open in.", + }; + } + + return { ok: true, environment }; +} diff --git a/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts b/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts new file mode 100644 index 00000000000..4c43b962f33 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentAlertUnsubscribeToken.server.ts @@ -0,0 +1,63 @@ +/** + * The credential in a watch alert email's unsubscribe link. HS256 over `SESSION_SECRET` with + * a prefix and `kind` claim disjoint from every other token signed with that secret. + */ + +import { generateJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { env } from "~/env.server"; + +const UNSUBSCRIBE_TOKEN_PREFIX = "tr_daau_"; +const UNSUBSCRIBE_TOKEN_KIND = "dashboard_agent_alert_unsubscribe"; +const UNSUBSCRIBE_PURPOSE = "unsubscribe"; + +/** Long-lived: an alert email has to keep working months after it arrived. */ +const UNSUBSCRIBE_TOKEN_TTL = "365d"; + +export type UnsubscribeTokenClaims = { channelId: string; alertType: string }; + +export async function signDashboardAgentAlertUnsubscribeToken( + secret: string, + opts: { channelId: string; alertType: string } +): Promise { + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: UNSUBSCRIBE_TOKEN_KIND, + purpose: UNSUBSCRIBE_PURPOSE, + sub: opts.channelId, + alertType: opts.alertType, + }, + expirationTime: UNSUBSCRIBE_TOKEN_TTL, + }); + + return `${UNSUBSCRIBE_TOKEN_PREFIX}${jwt}`; +} + +export async function verifyDashboardAgentAlertUnsubscribeToken( + secret: string, + token: string +): Promise { + if (!token.startsWith(UNSUBSCRIBE_TOKEN_PREFIX)) return; + + const result = await validateJWT(token.slice(UNSUBSCRIBE_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== UNSUBSCRIBE_TOKEN_KIND) return; + if (payload.purpose !== UNSUBSCRIBE_PURPOSE) return; + if (typeof payload.sub !== "string" || payload.sub.length === 0) return; + if (typeof payload.alertType !== "string" || payload.alertType.length === 0) return; + + return { channelId: payload.sub, alertType: payload.alertType }; +} + +export function mintDashboardAgentAlertUnsubscribeToken(opts: { + channelId: string; + alertType: string; +}): Promise { + return signDashboardAgentAlertUnsubscribeToken(env.SESSION_SECRET, opts); +} + +export function verifyUnsubscribeToken(token: string): Promise { + return verifyDashboardAgentAlertUnsubscribeToken(env.SESSION_SECRET, token); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts b/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts new file mode 100644 index 00000000000..1db0ca73f3c --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchAlerts.server.ts @@ -0,0 +1,378 @@ +/** + * The seam between a watch firing and the standard alert pipeline: the enqueue, plus the + * gate both the fan-out and the agent's subscribe endpoint consult. + */ + +import { type Watch } from "@internal/dashboard-agent-db"; +import { + type PrismaClientOrTransaction, + type ProjectAlertChannel, + type RuntimeEnvironmentType, +} from "@trigger.dev/database"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { alertsWorker } from "~/v3/alertsWorker.server"; +import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; +import { CreateAlertChannelService } from "~/v3/services/alerts/createAlertChannel.server"; + +/** The alert type a watch fires under. */ +export const DASHBOARD_AGENT_WATCH_ALERT_TYPE = "DASHBOARD_AGENT_WATCH" as const; + +/** What the enqueue needs off a watch row, rather than the full row. */ +export type WatchFiredAlertSource = Pick< + Watch, + | "id" + | "identity" + | "spec" + | "organizationId" + | "projectId" + | "environmentId" + | "userId" + | "firedAt" + | "lastResult" + | "resolution" + | "observedOutcome" +>; + +/** + * Queue the alert fan-out for a resolved watch. Only `fired` dispatches; an expiry is + * narrated in the chat. The job id is the idempotency key: one fan-out per watch. + */ +export async function enqueueWatchFiredAlert( + watch: WatchFiredAlertSource, + outcome: "fired" | "expired" +): Promise { + if (outcome !== "fired") return; + + await alertsWorker.enqueue({ + id: `watch-alert:${watch.id}`, + job: "v3.deliverDashboardAgentWatchAlert", + payload: { + watchId: watch.id, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + userId: watch.userId, + identity: watch.identity, + kind: watch.spec.kind, + note: watch.spec.note, + firedAt: (watch.firedAt ?? new Date()).toISOString(), + facts: watch.lastResult ?? {}, + // The frozen resolved result: the email renders from these and never re-reads + // the source. + resolution: watch.resolution ?? "condition_met", + observed: watch.observedOutcome ?? undefined, + }, + }); +} + +export type DashboardAgentAlertDenyReason = + /** The user can't use the dashboard agent, so its watches can't alert either. */ + | "dashboard_agent_disabled" + /** This installation has no alert email transport configured. */ + | "email_alerts_not_configured"; + +export type DashboardAgentAlertGate = + | { allowed: true } + | { allowed: false; reason: DashboardAgentAlertDenyReason }; + +/** + * May this user's watches alert at all? Operational checks only, no plan check: billing + * gates that separately. `organizationId` stays in the signature for that gate. + */ +export async function canUseDashboardAgentAlerts(params: { + userId: string; + organizationSlug: string; + organizationId: string; + isAdmin?: boolean; + orgFeatureFlags?: Record | null; +}): Promise { + const hasAgent = await canAccessDashboardAgent({ + userId: params.userId, + isAdmin: params.isAdmin ?? false, + // Never an impersonated session: this runs in the background. + isImpersonating: false, + organizationSlug: params.organizationSlug, + orgFeatureFlags: params.orgFeatureFlags, + }); + if (!hasAgent) return { allowed: false, reason: "dashboard_agent_disabled" }; + + return { allowed: true }; +} + +/** + * Whether a fired watch in this environment would already reach this user outside the + * chat. Advisory only: the watch already exists, so every failure answers `none`. + */ +export async function resolveWatchEmailAlertsState(params: { + userId: string; + environment: AuthenticatedEnvironment; +}): Promise<"subscribed" | "none" | "unavailable"> { + const { userId, environment } = params; + try { + // Another member's channel mails them, not this user, so only this user's own channel + // answers "subscribed". + const owner = await resolveWatchAlertOwnership(userId, $replica); + const channel = owner + ? await $replica.projectAlertChannel.findFirst({ + where: { + projectId: environment.project.id, + deduplicationKey: owner.deduplicationKey, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + environmentTypes: { has: environment.type }, + }, + select: { id: true }, + }) + : null; + if (channel) return "subscribed"; + + const gate = await canUseDashboardAgentAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + orgFeatureFlags: environment.organization.featureFlags as Record | null, + }); + return gate.allowed ? "none" : "unavailable"; + } catch (error) { + logger.error("Failed to resolve dashboard agent watch alert state", { + error, + userId, + organizationId: environment.organizationId, + projectId: environment.project.id, + environmentId: environment.id, + }); + return "none"; + } +} + +/** The same gate plus an email transport, or the channel would never deliver. */ +export async function canUseDashboardAgentEmailAlerts( + params: Parameters[0] & { projectId: string } +): Promise { + const base = await canUseDashboardAgentAlerts(params); + if (!base.allowed) return base; + + // Mirrors what the alerts email client needs, not resend specifically. + if (env.ALERT_FROM_EMAIL === undefined || env.ALERT_EMAIL_TRANSPORT === undefined) { + return { allowed: false, reason: "email_alerts_not_configured" }; + } + + return { allowed: true }; +} + +// A channel has no owner column, so this key is the only record of whose channel it is. +export function watchAlertDeduplicationKey(email: string): string { + return `dashboard-agent-watch:${email}`; +} + +/** + * The one place a user id becomes watch-alert ownership. Reading state, subscribing and + * unsubscribing all go through here, so they cannot disagree about whose channel is whose. + */ +async function resolveWatchAlertOwnership( + userId: string, + db: PrismaClientOrTransaction = prisma +): Promise<{ email: string; deduplicationKey: string } | undefined> { + const user = await db.user.findFirst({ where: { id: userId }, select: { email: true } }); + if (!user) return undefined; + return { email: user.email, deduplicationKey: watchAlertDeduplicationKey(user.email) }; +} + +/** How many times a lost race is retried before the subscribe is reported as failed. */ +const SUBSCRIBE_ATTEMPTS = 3; + +function withoutDuplicates(list: T[], value: T): T[] { + return list.includes(value) ? list : [...list, value]; +} + +function isUniqueConstraintError(error: unknown): boolean { + return ( + typeof error === "object" && error !== null && (error as { code?: string }).code === "P2002" + ); +} + +/** + * Put this environment type on the user's watch-alert channel, creating the channel if there + * is none. One channel per (email, project), so subscribing in a second environment must add + * to the list rather than replace it — replacing silently stops the first one's mail. + * + * The update is conditional on the lists the read saw, so two environments subscribing at + * once cannot drop each other's addition: the loser sees no updated row and reads again. + */ +export async function subscribeChannelToWatchAlerts(params: { + userId: string; + email: string; + deduplicationKey: string; + environmentType: RuntimeEnvironmentType; + project: { id: string; externalRef: string }; +}): Promise> { + const { userId, email, deduplicationKey, environmentType, project } = params; + const name = `Watch alerts for ${email}`; + + for (let attempt = 0; attempt < SUBSCRIBE_ATTEMPTS; attempt++) { + const existing = await prisma.projectAlertChannel.findFirst({ + where: { projectId: project.id, deduplicationKey }, + select: { id: true, alertTypes: true, environmentTypes: true }, + }); + + if (!existing) { + try { + // The service also checks this user's membership of the project. + return await new CreateAlertChannelService().call(project.externalRef, userId, { + name, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE], + environmentTypes: [environmentType], + deduplicationKey, + channel: { type: "EMAIL", email }, + }); + } catch (error) { + // Another environment created the channel first; the next attempt adds onto it. + if (!isUniqueConstraintError(error)) throw error; + continue; + } + } + + const environmentTypes = withoutDuplicates(existing.environmentTypes, environmentType); + const alertTypes = withoutDuplicates(existing.alertTypes, DASHBOARD_AGENT_WATCH_ALERT_TYPE); + + const { count } = await prisma.projectAlertChannel.updateMany({ + // Compare-and-swap on the lists the read returned. + where: { + id: existing.id, + projectId: project.id, + deduplicationKey, + environmentTypes: { equals: existing.environmentTypes }, + alertTypes: { equals: existing.alertTypes }, + }, + data: { + name, + alertTypes, + environmentTypes, + type: "EMAIL", + properties: { email }, + enabled: true, + }, + }); + + if (count > 0) { + return { id: existing.id, type: "EMAIL", enabled: true, environmentTypes }; + } + } + + throw new Error("Could not subscribe to watch alerts: the channel kept changing underneath"); +} + +export type SubscribeToWatchAlertsResult = + | { ok: true; email: string } + | { ok: false; reason: DashboardAgentAlertDenyReason | "user_not_found" }; + +/** + * Subscribe the signed-in user's own account email to this project's watch alerts. The + * address is never taken from the request, and the dedup key is stable per (email, project). + */ +export async function subscribeUserToWatchAlerts(params: { + userId: string; + environment: { + type: string; + organizationId: string; + organization: { slug: string }; + project: { id: string; externalRef: string }; + }; +}): Promise { + const { userId, environment } = params; + + const gate = await canUseDashboardAgentEmailAlerts({ + userId, + organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, + projectId: environment.project.id, + }); + if (!gate.allowed) return { ok: false, reason: gate.reason }; + + const owner = await resolveWatchAlertOwnership(userId); + if (!owner) return { ok: false, reason: "user_not_found" }; + + await subscribeChannelToWatchAlerts({ + userId, + email: owner.email, + deduplicationKey: owner.deduplicationKey, + environmentType: environment.type as RuntimeEnvironmentType, + project: environment.project, + }); + + return { ok: true, email: owner.email }; +} + +export type UnsubscribeResult = + | { ok: true; channelName: string; disabledChannel: boolean } + | { ok: false; reason: "not_found" | "conflict" }; + +/** How many times a lost race is retried before the caller is told to try again. */ +const UNSUBSCRIBE_ATTEMPTS = 3; + +/** + * Take `DASHBOARD_AGENT_WATCH` off a channel, disabling one left with no alert types. The + * write is conditional on the list the read saw, so a concurrent edit fails this attempt. + * + * A project is shared by every member, so a request-driven caller must pass + * `organizationId` and `ownerUserId` too. + */ +export async function unsubscribeChannelFromWatchAlerts( + channelId: string, + options: { projectId?: string; organizationId?: string; ownerUserId?: string } = {}, + db: PrismaClientOrTransaction = prisma +): Promise { + let ownerKey: string | undefined; + if (options.ownerUserId) { + const owner = await resolveWatchAlertOwnership(options.ownerUserId, db); + if (!owner) return { ok: false, reason: "not_found" }; + ownerKey = owner.deduplicationKey; + } + + const scope = { + id: channelId, + ...(options.projectId ? { projectId: options.projectId } : {}), + ...(options.organizationId ? { project: { organizationId: options.organizationId } } : {}), + ...(ownerKey ? { deduplicationKey: ownerKey } : {}), + }; + + for (let attempt = 0; attempt < UNSUBSCRIBE_ATTEMPTS; attempt++) { + const channel = await db.projectAlertChannel.findFirst({ + where: scope, + select: { name: true, alertTypes: true, projectId: true, deduplicationKey: true }, + }); + // A channel this alert type was never on is out of scope: stripping nothing off it + // would still report success, and an empty list would disable it. + if (!channel || !channel.alertTypes.includes(DASHBOARD_AGENT_WATCH_ALERT_TYPE)) { + return { ok: false, reason: "not_found" }; + } + + const remaining = channel.alertTypes.filter( + (type) => type !== DASHBOARD_AGENT_WATCH_ALERT_TYPE + ); + + const { count } = await db.projectAlertChannel.updateMany({ + // Compare-and-swap on the row the scoped read returned. `updateMany` takes no relation + // filter, so the org scope is carried by the read's `projectId`. + where: { + id: channelId, + projectId: channel.projectId, + deduplicationKey: channel.deduplicationKey, + alertTypes: { equals: channel.alertTypes }, + }, + data: { + alertTypes: remaining, + ...(remaining.length === 0 ? { enabled: false } : {}), + }, + }); + + if (count > 0) { + return { ok: true, channelName: channel.name, disabledChannel: remaining.length === 0 }; + } + } + + return { ok: false, reason: "conflict" }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts b/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts new file mode 100644 index 00000000000..6f7b071d12b --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchBatch.server.ts @@ -0,0 +1,304 @@ +/** + * The batch check: every due watch of one (environment, cadence) group in one pass. Each row is + * the authority on its own snapshot, and each initiating user is re-authorized before any read. + */ + +import { + cancelWatch, + claimWatchBatchTick, + listActiveWatchesForBatch, + listWatchesAwaitingDeliveryForBatch, + recordWatchAttempt, + recordWatchCheck, + stopWatchBatch, + WATCH_DELIVERY_CLAIM_STALE_MS, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchBatchCheckEntry, WatchBatchCheckResponse } from "@internal/dashboard-agent"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { + authorizeWatchEnvironment, + type WatchAuthorization, +} from "~/services/dashboardAgentWatches.server"; +import { + mintDashboardAgentWatchToken, + WATCH_TOKEN_GRACE_MS, +} from "~/services/dashboardAgentWatchToken.server"; + +/** + * How early a watch may be checked and still count as due, so a tick landing seconds + * early doesn't defer it a whole cadence. Capped at half a cadence. + */ +function dueSlackMs(cadenceMinutes: number): number { + return Math.min(30_000, (cadenceMinutes * 60_000) / 2); +} + +/** Small on purpose: it stops one slow condition serializing the group, not to fan out. */ +const EVALUATION_CONCURRENCY = 8; + +export type WatchBatchCheckDeps = { + now?: () => Date; + /** The group's active watches. */ + listActive?: (params: { environmentId: string; cadenceMinutes: number }) => Promise; + /** The group's resolved watches whose wake is still owed. */ + listOwed?: (params: { + environmentId: string; + cadenceMinutes: number; + claimStaleBefore: Date; + }) => Promise; + /** Re-authorization of one watch's initiating user. */ + authorize?: (watch: Watch) => Promise; + /** The environment readers the conditions run against. */ + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + /** The per-watch token the fired / investigate callbacks are made with. */ + mintToken?: (watch: Watch) => Promise; + concurrency?: number; +}; + +/** + * Run one batch tick's checks. The claim decides whether this run owns the tick and keeps the + * schedule single-file; the guarded transition and fenced delivery claim stop a double fire. + */ +export async function runWatchBatchCheck( + params: { environmentId: string; cadenceMinutes: number; epoch: number; tick: number }, + deps: WatchBatchCheckDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const listActive = + deps.listActive ?? ((args) => listActiveWatchesForBatch(dashboardAgentDb, args)); + const listOwed = + deps.listOwed ?? ((args) => listWatchesAwaitingDeliveryForBatch(dashboardAgentDb, args)); + const mintToken = + deps.mintToken ?? + ((watch: Watch) => + mintDashboardAgentWatchToken({ watchId: watch.id, expiresAt: watch.expiresAt })); + + const claimed = await claimWatchBatchTick(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: params.epoch, + // The tick a run carries is the generation it owns. + generation: params.tick, + }); + if (!claimed) { + logger.debug("Dashboard agent watch batch: the tick is stale", params); + return { stale: true }; + } + + const active = await listActive({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + + const due = active.filter((watch) => isDue(watch, params.cadenceMinutes, now)); + const evaluated = await evaluateGroup(due, params, { ...deps, now: () => now }, mintToken); + + // Wakes this group still owes. Read after the evaluation, so a wake this tick resolved + // and failed to deliver is already in it. + const owed = await listOwed({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + claimStaleBefore: new Date(now.getTime() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + + const deliveries = await Promise.all( + owed.map(async (watch) => ({ + watchId: watch.id, + token: await mintToken(watch), + // A delivery decides nothing, so it claims no generation. + tick: 0, + deliverOnly: true as const, + })) + ); + + // The chain only stops with nothing to poll and nothing owed: stopping while a wake is + // owed strands it. Fenced on the epoch, so it can only end this run's own chain. + const continues = active.length > 0 || owed.length > 0; + if (!continues) { + await stopWatchBatch(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: params.epoch, + }); + } + + return { watches: [...evaluated, ...deliveries], continues }; +} + +/** + * A watch whose window closes before the next tick is due now, so its final evaluation is + * never missed. A watch past the token grace is never due: the expiry sweep owns it. + */ +export function isDue(watch: Watch, cadenceMinutes: number, now: Date): boolean { + const nowMs = now.getTime(); + const cadenceMs = cadenceMinutes * 60_000; + + if (nowMs > watch.expiresAt.getTime() + WATCH_TOKEN_GRACE_MS) return false; + if (watch.expiresAt.getTime() <= nowMs + cadenceMs) return true; + + const lastChecked = watch.lastCheckedAt?.getTime(); + return lastChecked === undefined || lastChecked <= nowMs - cadenceMs + dueSlackMs(cadenceMinutes); +} + +/** + * Evaluate the due watches against one set of readers. Authorization is cached per (user, org, + * project); each watch runs in its own try, so a failure is that watch's answer alone. + */ +async function evaluateGroup( + due: Watch[], + params: { environmentId: string; cadenceMinutes: number }, + deps: WatchBatchCheckDeps, + mintToken: (watch: Watch) => Promise +): Promise { + if (due.length === 0) return []; + + const now = deps.now?.() ?? new Date(); + const authorize = deps.authorize ?? defaultAuthorize; + const buildCheckDeps = deps.checkDeps ?? watchCheckDeps; + + const authorizations = new Map>(); + const authorizeOnce = (watch: Watch) => { + const key = `${watch.userId}:${watch.organizationId}:${watch.projectId}`; + const cached = authorizations.get(key); + if (cached) return cached; + const pending = authorize(watch); + authorizations.set(key, pending); + return pending; + }; + + // Built from the first authorization that passes, then shared: every row in the group + // names the same environment. + let readers: WatchCheckDeps | undefined; + + const evaluateOne = async ( + watch: Watch, + base: { watchId: string; token: string; tick: number } + ): Promise => { + const authorization = await authorizeOnce(watch); + if (!authorization.ok) { + // Cancel before anything is read: a watch must not outlive its creator's access. + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "access_revoked" }); + return { ...base, code: "access_revoked", error: "Access to this environment was revoked" }; + } + + readers ??= shareReads(buildCheckDeps(authorization.environment, now)); + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const final = watch.expiresAt.getTime() <= now.getTime(); + const outcome = await checkWatch( + watch.spec, + readers, + // A check that couldn't read anything freezes a streak instead of resetting it. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch batch: a check failed", { + error, + watchId: watch.id, + environmentId: params.environmentId, + }) + ); + + // Only a real evaluation is recorded, final or not: `unavailable` means nothing was read, + // so writing it would move `lastCheckedAt` and overwrite the facts a streak lives in. + // Guarded on `active`, and never touches `tickCount`. + if (outcome.result !== "unavailable") { + await recordWatchCheck(dashboardAgentDb, { + id: watch.id, + lastResult: { + result: outcome.result, + facts: outcome.facts, + observed: outcome.observed, + final, + }, + }); + } else { + // Looked at, not checked: this rotates the watch out of its group's head without + // touching its dueness or the facts its streak lives in. + await recordWatchAttempt(dashboardAgentDb, { id: watch.id }); + } + + return { ...base, result: outcome.result, facts: outcome.facts, observed: outcome.observed }; + }; + + return mapWithConcurrency(due, deps.concurrency ?? EVALUATION_CONCURRENCY, async (watch) => { + // Minted outside the try, because the catch below needs a token it can't fail to have. + const base = { watchId: watch.id, token: await mintToken(watch), tick: watch.tickCount + 1 }; + try { + return await evaluateOne(watch, base); + } catch (error) { + logger.error("Dashboard agent watch batch: a watch couldn't be evaluated", { + watchId: watch.id, + environmentId: params.environmentId, + error, + }); + // `unavailable` is never read as true or false: the watch keeps its state. Still a + // look, so the fairness key moves even when nothing else does. + await recordWatchAttempt(dashboardAgentDb, { id: watch.id }).catch(() => {}); + return { ...base, result: "unavailable" as const, error: (error as Error).message }; + } + }); +} + +/** + * Wrap a batch's readers so each distinct read happens once. `now` is fixed for the batch, so + * a reader's answer is a pure function of its arguments. Failed reads are cached too. + * + * Exported for the expiry sweep, which finalizes the same rows against the same readers. + */ +export function shareReads(readers: WatchCheckDeps): WatchCheckDeps { + const cache = new Map>(); + const once = (name: string, read: (...args: A) => Promise) => { + return (...args: A): Promise => { + const key = `${name}:${JSON.stringify(args)}`; + const cached = cache.get(key); + if (cached) return cached as Promise; + const pending = read(...args); + cache.set(key, pending); + return pending; + }; + }; + + return { + readRun: once("readRun", readers.readRun), + queueExists: once("queueExists", readers.queueExists), + readQueueDepth: once("readQueueDepth", readers.readQueueDepth), + readQueueOldestAge: once("readQueueOldestAge", readers.readQueueOldestAge), + readErrorRecurrence: once("readErrorRecurrence", readers.readErrorRecurrence), + readHealth: once("readHealth", readers.readHealth), + }; +} + +/** `mapper` over `items`, at most `limit` in flight. Order is preserved. */ +export async function mapWithConcurrency( + items: T[], + limit: number, + mapper: (item: T) => Promise +): Promise { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await mapper(items[index]!); + } + }); + await Promise.all(workers); + return results; +} + +function defaultAuthorize(watch: Watch): Promise { + return authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts b/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts new file mode 100644 index 00000000000..fc41bcba038 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchCheckBase.ts @@ -0,0 +1,106 @@ +/** + * The reader contract every watch condition family shares, plus the duration formatting + * they all label with. No IO of its own. + */ + +import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations"; +import type { WatchCheckResult, WatchObservedOutcome } from "@internal/dashboard-agent-contracts"; + +/** The single run point-read. Postgres is authoritative for run state. */ +export type WatchRunRow = { + friendlyId: string; + status: string; + queue: string; + createdAt: Date; + /** Stamped when the run entered the queue. NULL while a run is delayed. */ + queuedAt: Date | null; + /** Set once the run is dequeued. */ + startedAt: Date | null; + completedAt: Date | null; + delayUntil: Date | null; +}; + +export type WatchQueueDepth = { + /** Pending count for the queue, as of `asOf`. */ + depth: number; + source: "live_queue" | "queue_metrics"; + /** A stale reading can never answer "drained". */ + current: boolean; + /** What instant the reading describes, when it isn't the live counter. */ + asOf?: Date; +}; + +/** + * The oldest still-waiting run's age in one queue. A non-current age is wrong in both + * directions, so `checkQueueOldestAge` refuses it rather than comparing it. + */ +export type WatchQueueOldestAge = { + /** Age of the oldest run still waiting, in ms. Null when nothing is waiting. */ + ageMs: number | null; + source: "live_queue" | "queue_metrics"; + current: boolean; + asOf?: Date; +}; + +/** What we know about the watched error's occurrences relative to `since`. */ +export type WatchErrorRecurrence = { + /** Earliest occurrence proven after `since`. Null with a `lastSeenAt` means not since. */ + occurredAt: Date | null; + /** How precisely `occurredAt` is known: to the millisecond, or to its minute. */ + occurredAtPrecision: "exact" | "minute" | null; + /** Occurrences after `since`. A lower bound when `countApproximate`. */ + countSince: number; + /** True when occurrences in the watch's creation minute can't be separated out. */ + countApproximate: boolean; + /** The fingerprint's most recent occurrence, whenever it was. */ + lastSeenAt: Date | null; +}; + +export type WatchHealthSeverity = "ok" | "warn" | "crit"; + +export type WatchHealthSnapshot = { + /** `facts.trustworthy` from the health report. Untrustworthy never fires recovery. */ + trustworthy: boolean; + severity: WatchHealthSeverity; +}; + +/** + * The readers a check may use. Each may throw, which the caller turns into `unavailable`. + * `null` means the source answered and there is nothing there. + */ +export type WatchCheckDeps = { + /** Run point-read by public run id, scoped to the watch's environment. */ + readRun: (runId: string) => Promise; + /** Does this queue exist in the watch's environment? */ + queueExists: (queue: string) => Promise; + /** Current pending count, live run-queue first with a ClickHouse fallback. */ + readQueueDepth: (queue: string) => Promise; + /** Age of the oldest run still waiting in the queue, right now. */ + readQueueOldestAge: (queue: string) => Promise; + /** `null` means the fingerprint has no occurrences at all in this environment. */ + readErrorRecurrence: (fingerprint: string, since: Date) => Promise; + /** The health report's current verdict for the watch's environment. */ + readHealth: () => Promise; +}; + +export type WatchCheckInput = { + now: Date; + /** The recurrence window's start: the server-set `spec.since`, never caller-set. */ + since: Date; + /** + * The previous check's facts, for the stateful kinds. A check's own facts are the only + * storage for its state. Absent means no prior observation, never zero. + */ + previous?: Record | null; +}; + +export type WatchCheckOutcome = { + result: WatchCheckResult; + facts: Record; + /** Frozen onto the row by the resolving transition, so no surface re-reads the source. */ + observed: WatchObservedOutcome; +}; + +export function formatMs(ms: number): string { + return formatDurationMilliseconds(ms, { style: "short", maxDecimalPoints: 0 }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts b/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts new file mode 100644 index 00000000000..ac49aaacf68 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchChecks.server.ts @@ -0,0 +1,337 @@ +/** + * Default IO wiring for the watch checks. Run state and queue existence are authoritative + * Postgres point-reads, and readers throw rather than invent a zero on a broken source. + */ + +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { $replica, prisma } from "~/db.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { ReportPresenter } from "~/presenters/v3/reports/ReportPresenter.server"; +import { engine } from "~/v3/runEngine.server"; +import { runStore } from "~/v3/runStore.server"; +import type { + WatchCheckDeps, + WatchErrorRecurrence, + WatchHealthSeverity, + WatchHealthSnapshot, + WatchQueueDepth, + WatchQueueOldestAge, + WatchRunRow, +} from "./dashboardAgentWatchChecks"; + +const WATCH_RUN_SELECT = { + friendlyId: true, + status: true, + queue: true, + createdAt: true, + queuedAt: true, + startedAt: true, + completedAt: true, + delayUntil: true, +} as const; + +/** The single Postgres point-read: one run, scoped to the watch's environment. */ +export async function readWatchRun( + runFriendlyId: string, + environmentId: string +): Promise { + const run = await runStore.findRun( + { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, + { select: WATCH_RUN_SELECT }, + $replica + ); + return run ?? null; +} + +/** One Postgres point-read: does this queue exist in the environment? */ +export async function watchQueueExists(environmentId: string, queueName: string): Promise { + const queue = await $replica.taskQueue.findFirst({ + where: { runtimeEnvironmentId: environmentId, name: queueName }, + select: { id: true }, + }); + return queue !== null; +} + +/** The same run read on the primary, for a target that may have been created a moment ago. */ +export async function readWatchRunOnPrimary( + runFriendlyId: string, + environmentId: string +): Promise { + const run = await runStore.findRunOnPrimary( + { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, + { select: WATCH_RUN_SELECT } + ); + return run ?? null; +} + +/** The same queue read on the primary. */ +export async function watchQueueExistsOnPrimary( + environmentId: string, + queueName: string +): Promise { + const queue = await prisma.taskQueue.findFirst({ + where: { runtimeEnvironmentId: environmentId, name: queueName }, + select: { id: true }, + }); + return queue !== null; +} + +/** How far back the ClickHouse depth fallback looks when the live counter is down. */ +const DEPTH_FALLBACK_MINUTES = 10; +const DEPTH_FALLBACK_BUCKET_SECONDS = 60; +/** + * How far behind `now` the newest analytics bucket may end and still count as current. + * One bucket of slack: anything older leaves runs queued in the gap invisible. + */ +const DEPTH_FRESH_TOLERANCE_MS = DEPTH_FALLBACK_BUCKET_SECONDS * 1000; + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + +/** ClickHouse renders DateTime without a zone; the column is UTC. */ +function parseClickhouseDateTime(value: string): Date { + return new Date(`${value.replace(" ", "T")}Z`); +} + +/** + * Current pending count for one queue. The live counter is the truth; the ClickHouse fallback + * reports the newest bucket's peak depth and is `current` only if it reaches the present. + */ +export async function readWatchQueueDepth( + environment: AuthenticatedEnvironment, + queueName: string, + now: Date = new Date() +): Promise { + const live = await engine.lengthOfQueue(environment, queueName).catch(() => null); + if (typeof live === "number" && Number.isFinite(live)) { + return { depth: live, source: "live_queue", current: true, asOf: now }; + } + + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "query" + ); + + const bucketMs = DEPTH_FALLBACK_BUCKET_SECONDS * 1000; + const endMs = Math.ceil(now.getTime() / bucketMs) * bucketMs; + const startMs = endMs - DEPTH_FALLBACK_MINUTES * 60_000; + + const [error, rows] = await clickhouse.queueMetrics.depthSparklines({ + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: environment.id, + queueNames: [queueName], + startTime: formatClickhouseDateTime(new Date(startMs)), + endTime: formatClickhouseDateTime(new Date(endMs)), + bucketSeconds: DEPTH_FALLBACK_BUCKET_SECONDS, + }); + + if (error) throw error; + if (!rows || rows.length === 0) return null; + + // Newest bucket wins: the closest the rollup gets to now. + const newest = rows.reduce((best, row) => (row.bucket > best.bucket ? row : best), rows[0]!); + const bucketEnd = new Date(parseClickhouseDateTime(newest.bucket).getTime() + bucketMs); + const current = bucketEnd.getTime() >= now.getTime() - DEPTH_FRESH_TOLERANCE_MS; + + return { depth: newest.depth, source: "queue_metrics", current, asOf: bucketEnd }; +} + +/** + * How long the oldest still-waiting run in a queue has waited: for a concurrency-keyed queue the + * worst across keys with a live backlog. `null` can't be read, `ageMs: null` is empty. + */ +export async function readWatchQueueOldestAge( + environment: AuthenticatedEnvironment, + queueName: string, + now: Date = new Date() +): Promise { + const [breakdown, oldestQueuedAt] = await Promise.all([ + engine + .concurrencyKeyBreakdown(environment, queueName, { limit: OLDEST_AGE_CK_LIMIT }) + .catch(() => null), + engine.oldestMessageInQueue(environment, queueName).catch(() => null), + ]); + + // A partial read would under-report the wait and silently miss the SLA, so either read + // failing makes the whole reading unavailable rather than a healthy zero. + if (breakdown === null || oldestQueuedAt === null) return null; + + const waitingKeys = breakdown.keys.filter((key) => key.queued > 0); + const ageMs = + waitingKeys.length > 0 + ? waitingKeys.reduce((max, key) => Math.max(max, now.getTime() - key.oldestEnqueuedAt), 0) + : typeof oldestQueuedAt === "number" + ? Math.max(0, now.getTime() - oldestQueuedAt) + : null; + + return { ageMs, source: "live_queue", current: true, asOf: now }; +} + +/** Same cap the queue detail page reads keys with. */ +const OLDEST_AGE_CK_LIMIT = 50; + +const MINUTE_MS = 60_000; + +type OrganizationClickhouse = Awaited< + ReturnType +>; + +/** + * The fingerprint's most recent occurrence at millisecond precision, from `errors_v1`. The + * per-minute rollup can't separate the prompting error from a recurrence in the same minute. + */ +async function readErrorLastSeen( + clickhouse: OrganizationClickhouse, + environment: AuthenticatedEnvironment, + fingerprint: string +): Promise { + const builder = clickhouse.errors.activeErrorsSinceQueryBuilder(); + builder.where("organization_id = {organizationId: String}", { + organizationId: environment.organizationId, + }); + builder.where("project_id = {projectId: String}", { projectId: environment.projectId }); + builder.where("environment_id = {environmentId: String}", { environmentId: environment.id }); + builder.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + builder.groupBy("environment_id, task_identifier, error_fingerprint"); + + const [error, rows] = await builder.execute(); + if (error) throw error; + if (!rows || rows.length === 0) return null; + + let lastSeenMs = 0; + for (const row of rows) { + const ms = Number(row.last_seen); + if (Number.isFinite(ms) && ms > lastSeenMs) lastSeenMs = ms; + } + + return lastSeenMs > 0 ? new Date(lastSeenMs) : null; +} + +/** + * What we know about a fingerprint relative to `since`. `errors_v1` decides whether it + * recurred; the rollup's count is a lower bound when the creation-minute bucket has hits. + */ +export async function readWatchErrorRecurrence( + environment: AuthenticatedEnvironment, + fingerprint: string, + since: Date +): Promise { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "logs" + ); + + const lastSeenAt = await readErrorLastSeen(clickhouse, environment, fingerprint); + // Never seen in this environment at all. + if (!lastSeenAt) return null; + + const notRecurred: WatchErrorRecurrence = { + occurredAt: null, + occurredAtPrecision: null, + countSince: 0, + countApproximate: false, + lastSeenAt, + }; + if (lastSeenAt.getTime() <= since.getTime()) return notRecurred; + + // Something landed after `since`; the rollup fills in the count and the minute. + const sinceMinuteMs = Math.floor(since.getTime() / MINUTE_MS) * MINUTE_MS; + const queryBuilder = clickhouse.errors.createOccurrencesQueryBuilder("INTERVAL 1 MINUTE"); + queryBuilder.where("organization_id = {organizationId: String}", { + organizationId: environment.organizationId, + }); + queryBuilder.where("project_id = {projectId: String}", { projectId: environment.projectId }); + queryBuilder.where("environment_id = {environmentId: String}", { environmentId: environment.id }); + queryBuilder.where("error_fingerprint = {fingerprint: String}", { fingerprint }); + // The creation minute is included; its occurrences are counted separately below. + queryBuilder.where("minute >= toStartOfMinute(fromUnixTimestamp64Milli({sinceMs: Int64}))", { + sinceMs: since.getTime(), + }); + queryBuilder.groupBy("error_fingerprint, bucket_epoch"); + queryBuilder.orderBy("bucket_epoch ASC"); + + const [error, rows] = await queryBuilder.execute(); + if (error) throw error; + + let earliestAfterMs: number | null = null; + let countAfter = 0; + let creationMinuteCount = 0; + + for (const row of rows ?? []) { + const bucketMs = row.bucket_epoch * 1000; + if (bucketMs <= sinceMinuteMs) { + creationMinuteCount += row.count; + continue; + } + countAfter += row.count; + if (earliestAfterMs === null || bucketMs < earliestAfterMs) earliestAfterMs = bucketMs; + } + + // The earliest provable occurrence: a bucket starting after the creation minute, or the + // exact `last_seen` when that is the only evidence. + const useBucket = earliestAfterMs !== null && earliestAfterMs < lastSeenAt.getTime(); + + return { + occurredAt: useBucket ? new Date(earliestAfterMs!) : lastSeenAt, + occurredAtPrecision: useBucket ? "minute" : "exact", + // At least the one `errors_v1` proved, even if the rollup lags behind it. + countSince: Math.max(1, countAfter), + countApproximate: creationMinuteCount > 0, + lastSeenAt, + }; +} + +const HEALTH_SEVERITIES = new Set(["ok", "warn", "crit"]); + +/** + * The health report's current verdict, from the existing interpreter. No health reasoning + * is re-implemented here. + */ +export async function readWatchHealth( + environment: AuthenticatedEnvironment +): Promise { + const report = await new ReportPresenter().call({ environment, key: "health" }); + if (!report) return null; + + const severity = report.summary.severity; + if (!HEALTH_SEVERITIES.has(severity)) return null; + + const trustworthy = (report.facts as { trustworthy?: unknown } | undefined)?.trustworthy; + return { + // An absent trust marker counts as untrustworthy. + trustworthy: trustworthy === true, + severity: severity as WatchHealthSeverity, + }; +} + +export function watchCheckDeps( + environment: AuthenticatedEnvironment, + now: Date = new Date() +): WatchCheckDeps { + return { + readRun: (runId) => readWatchRun(runId, environment.id), + queueExists: (queue) => watchQueueExists(environment.id, queue), + readQueueDepth: (queue) => readWatchQueueDepth(environment, queue, now), + readQueueOldestAge: (queue) => readWatchQueueOldestAge(environment, queue, now), + readErrorRecurrence: (fingerprint, since) => + readWatchErrorRecurrence(environment, fingerprint, since), + readHealth: () => readWatchHealth(environment), + }; +} + +/** + * Creation-time deps. The target reads go to the primary, so a run or queue created moments + * ago is visible instead of failing as a non-existent target inside the replication window. + */ +export function watchCreationCheckDeps( + environment: AuthenticatedEnvironment, + now: Date = new Date() +): WatchCheckDeps { + return { + ...watchCheckDeps(environment, now), + readRun: (runId) => readWatchRunOnPrimary(runId, environment.id), + queueExists: (queue) => watchQueueExistsOnPrimary(environment.id, queue), + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchChecks.ts b/apps/webapp/app/services/dashboardAgentWatchChecks.ts new file mode 100644 index 00000000000..717d130ef20 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchChecks.ts @@ -0,0 +1,165 @@ +/** + * Deterministic evaluation of one watch condition, with IO behind `WatchCheckDeps`. `unavailable` + * is never a verdict, durations carry their basis, and `observed` is kept apart from the result. + * + * One module per condition family; this one dispatches and owns the failure envelope. + */ + +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; +import { checkRunFailed, checkRunFinished, checkRunStart } from "./dashboardAgentWatchRunChecks"; +import { + checkBacklogDrain, + checkQueueDepthAbove, + checkQueueDepthBelow, + checkQueueOldestAge, + checkQueueStalled, +} from "./dashboardAgentWatchQueueChecks"; +import { checkErrorRecurrence } from "./dashboardAgentWatchErrorChecks"; +import { checkHealthRecovery } from "./dashboardAgentWatchHealthChecks"; + +export type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, + WatchErrorRecurrence, + WatchHealthSeverity, + WatchHealthSnapshot, + WatchQueueDepth, + WatchQueueOldestAge, + WatchRunRow, +} from "./dashboardAgentWatchCheckBase"; +export { + checkRunFailed, + checkRunFinished, + checkRunStart, + describeRunWait, + isTerminalRunStatus, + type WatchWaitBasis, +} from "./dashboardAgentWatchRunChecks"; +export { + checkBacklogDrain, + checkQueueDepthAbove, + checkQueueDepthBelow, + checkQueueOldestAge, + checkQueueStalled, +} from "./dashboardAgentWatchQueueChecks"; +export { checkErrorRecurrence, normalizeErrorFingerprint } from "./dashboardAgentWatchErrorChecks"; +export { checkHealthRecovery } from "./dashboardAgentWatchHealthChecks"; + +/** + * The previous check's facts out of `lastResult`, which holds raw facts, the check endpoint's + * envelope, or the failure wrapper. The wrapper is unwrapped, so a streak survives a gap. + */ +export function previousCheckFacts(lastResult: unknown): Record | null { + if (!lastResult || typeof lastResult !== "object" || Array.isArray(lastResult)) return null; + const record = lastResult as Record; + + if (record.checkFailed === true) return previousCheckFacts(record.previous); + if (record.facts && typeof record.facts === "object" && !Array.isArray(record.facts)) { + return record.facts as Record; + } + return record; +} + +/** The single place a check failure becomes `unavailable`, never a verdict. */ +export async function checkWatch( + spec: WatchSpec, + deps: WatchCheckDeps, + input: WatchCheckInput, + onError?: (error: unknown) => void +): Promise { + try { + switch (spec.kind) { + case "run_start": + return await checkRunStart(spec, deps, input); + case "run_finished": + return await checkRunFinished(spec, deps, input); + case "run_failed": + return await checkRunFailed(spec, deps, input); + case "backlog_drain": + return await checkBacklogDrain(spec, deps, input); + case "queue_depth_above": + return await checkQueueDepthAbove(spec, deps, input); + case "queue_depth_below": + return await checkQueueDepthBelow(spec, deps, input); + case "queue_stalled": + return await checkQueueStalled(spec, deps, input); + case "queue_oldest_age": + return await checkQueueOldestAge(spec, deps, input); + case "error_recurrence": + return await checkErrorRecurrence(spec, deps, input); + case "health_recovery": + return await checkHealthRecovery(spec, deps, input); + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } + } catch (error) { + onError?.(error); + return { + result: "unavailable", + facts: { kind: spec.kind, reason: "check_failed" }, + observed: unobservedOutcome(spec), + }; + } +} + +/** + * The observation for a check that couldn't run. `verified: false` means the condition + * couldn't be confirmed, not that it didn't happen. + */ +export function unobservedOutcome(spec: WatchSpec): WatchObservedOutcome { + switch (spec.kind) { + case "run_start": + return { kind: "run_start", verified: false, status: null, started: false }; + case "run_finished": + return { kind: "run_finished", verified: false, finalStatus: null, durationMs: null }; + case "run_failed": + return { kind: "run_failed", verified: false, finalStatus: null, durationMs: null }; + case "backlog_drain": + return { kind: "backlog_drain", verified: false, depth: null }; + case "queue_depth_above": + return { + kind: "queue_depth_above", + verified: false, + depth: null, + threshold: spec.threshold, + }; + case "queue_depth_below": + return { + kind: "queue_depth_below", + verified: false, + depth: null, + threshold: spec.threshold, + }; + case "queue_stalled": + return { + kind: "queue_stalled", + verified: false, + depth: null, + notDecreasingStreak: 0, + ticks: spec.ticks, + }; + case "queue_oldest_age": + return { + kind: "queue_oldest_age", + verified: false, + ageMs: null, + thresholdMinutes: spec.thresholdMinutes, + }; + case "error_recurrence": + return { kind: "error_recurrence", verified: false, countSince: 0 }; + case "health_recovery": + return { kind: "health_recovery", verified: false, severity: null }; + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } +} diff --git a/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts b/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts new file mode 100644 index 00000000000..d317426e25e --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchErrorChecks.ts @@ -0,0 +1,72 @@ +/** The error-recurrence condition family. */ + +import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; + +/** + * The model cites the API error id (`error_`) but ClickHouse stores the raw + * fingerprint. Same normalization the errors API route uses. + */ +export function normalizeErrorFingerprint(fingerprint: string): string { + return ErrorId.toId(fingerprint); +} + +/** + * Satisfied on the first occurrence proven to be after the server-set `since`, which is + * never caller-set. The facts carry the precision of what they claim. + */ +export async function checkErrorRecurrence( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const fingerprint = normalizeErrorFingerprint(spec.fingerprint); + const recurrence = await deps.readErrorRecurrence(fingerprint, input.since); + const base = { fingerprint, since: input.since.toISOString() }; + + const quiet: WatchObservedOutcome = { + kind: "error_recurrence", + verified: true, + countSince: 0, + }; + + if (!recurrence) { + return { + result: "pending", + facts: { ...base, countSince: 0, lastSeenAt: null }, + observed: quiet, + }; + } + + const lastSeenAt = recurrence.lastSeenAt?.toISOString() ?? null; + + if (!recurrence.occurredAt) { + return { + result: "pending", + facts: { ...base, countSince: 0, lastSeenAt }, + observed: quiet, + }; + } + + return { + result: "satisfied", + facts: { + ...base, + occurredAt: recurrence.occurredAt.toISOString(), + occurredAtPrecision: recurrence.occurredAtPrecision, + countSince: recurrence.countSince, + countApproximate: recurrence.countApproximate, + lastSeenAt, + }, + observed: { + kind: "error_recurrence", + verified: true, + countSince: recurrence.countSince, + }, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts b/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts new file mode 100644 index 00000000000..443597e5fa9 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchHealthChecks.ts @@ -0,0 +1,49 @@ +/** The health-recovery condition family. */ + +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { + WatchCheckDeps, + WatchCheckInput, + WatchCheckOutcome, +} from "./dashboardAgentWatchCheckBase"; + +/** + * Satisfied only when the health report is both trustworthy and `ok`. An untrustworthy + * report can never fire a recovery. + */ +export async function checkHealthRecovery( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const health = await deps.readHealth(); + if (!health) { + return { + result: "unavailable", + facts: { report: spec.report, reason: "report_unavailable" }, + observed: { kind: "health_recovery", verified: false, severity: null }, + }; + } + + const facts = { + report: spec.report, + fromSeverity: spec.fromSeverity, + severity: health.severity, + trustworthy: health.trustworthy, + }; + + if (!health.trustworthy) { + // An untrustworthy report is not an observation of the severity, so record none. + return { + result: "pending", + facts: { ...facts, reason: "untrustworthy" }, + observed: { kind: "health_recovery", verified: false, severity: null }, + }; + } + + return { + result: health.severity === "ok" ? "satisfied" : "pending", + facts, + observed: { kind: "health_recovery", verified: true, severity: health.severity }, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts b/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts new file mode 100644 index 00000000000..22b6a303a99 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchInvestigate.server.ts @@ -0,0 +1,89 @@ +/** + * The reading turn of a consented watch, kicked from here because the wake turn has no token. + * The token is for `watch.userId` against `watch.environmentId`, off the row, never a body. + */ + +import type { WatchInvestigateAction } from "@internal/dashboard-agent"; +import { type Watch } from "@internal/dashboard-agent-db"; +import { watchResultNeedsAttention } from "@internal/dashboard-agent-contracts"; +import { ApiClient } from "@trigger.dev/core/v3"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + dashboardAgentApiOrigin, + mintDashboardAgentUserActorToken, +} from "~/services/dashboardAgent.server"; +import { dashboardAgentEnvironmentAddress } from "~/services/dashboardAgentEnvironmentAddress.server"; +import { env } from "~/env.server"; + +/** + * Whether the consent covers this resolved watch. The same contracts call the agent's wake + * makes, so the two can't disagree about which outcomes need attention. + */ +export function watchWantsInvestigation(watch: Watch): boolean { + if (!watch.investigateOnAttention) return false; + if (!watch.resolution) return false; + return watchResultNeedsAttention({ + kind: watch.spec.kind, + resolution: watch.resolution, + outcome: watch.observedOutcome, + }); +} + +/** The action the agent receives. Stable id, so a retried kick is a no-op. */ +export function watchInvestigateAction(watch: Watch): WatchInvestigateAction { + return { + type: "watch.investigate" as const, + id: `watch:${watch.id}:${watch.status}:investigate`, + watchId: watch.id, + identity: watch.identity, + spec: watch.spec, + facts: (watch.lastResult ?? {}) as Record, + resolution: watch.resolution ?? undefined, + observed: watch.observedOutcome ?? undefined, + note: watch.spec.note, + }; +} + +/** + * Send the kick. Throws on failure, and every caller treats it as best-effort: the wake is + * already delivered, so nothing here may retry or invalidate it. + */ +export async function kickWatchInvestigation(params: { + watch: Watch; + environment: AuthenticatedEnvironment; +}): Promise { + const { watch, environment } = params; + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + // The watch's immutable tenancy plus the delegated token that lets the turn read. + const metadata = { + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + projectRef: watch.projectRef ?? environment.project.externalRef, + ...dashboardAgentEnvironmentAddress(environment), + apiOrigin, + userActorToken: await mintDashboardAgentUserActorToken(watch.userId, { + environmentId: watch.environmentId, + }), + }; + + const apiClient = new ApiClient(apiOrigin, accessToken); + await apiClient.appendToSessionStream( + // Sessions are addressable by externalId, which is the chat id. + watch.chatId, + "in", + JSON.stringify({ + kind: "message", + payload: { + chatId: watch.chatId, + trigger: "action", + action: watchInvestigateAction(watch), + metadata, + }, + }) + ); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts b/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts new file mode 100644 index 00000000000..3e251818655 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchQueueChecks.ts @@ -0,0 +1,314 @@ +/** + * The queue condition family: drain, the two depth thresholds, stall and oldest age. + * They share one freshness fence — a claim of quiet needs a reading that describes now. + */ + +import type { WatchObservedOutcome, WatchSpec } from "@internal/dashboard-agent-contracts"; +import { + formatMs, + type WatchCheckDeps, + type WatchCheckInput, + type WatchCheckOutcome, + type WatchQueueDepth, +} from "./dashboardAgentWatchCheckBase"; + +/** + * The queue-depth read both threshold kinds share. A missing queue is `terminal_unsatisfied`, + * an unreadable or stale-low depth is `unavailable`, and a stale-high one is approximate. + */ +async function readDepthOrOutcome(args: { + queue: string; + deps: WatchCheckDeps; + /** The observation to record when there is no usable reading. */ + unobserved: (verified: boolean) => WatchObservedOutcome; + /** A non-current reading at or under this is refused; one above it passes through. */ + quietLine: number; + /** + * Stateful kinds only: no non-current reading is usable, because a phantom sample would + * enter the streak as if it had been observed now. + */ + requireCurrent?: boolean; +}): Promise< + | { ok: true; depth: WatchQueueDepth; facts: Record } + | { ok: false; outcome: WatchCheckOutcome } +> { + const { queue, deps, unobserved, quietLine } = args; + const depth = await deps.readQueueDepth(queue); + + if (depth === null) { + // Only a missing queue is terminal, not an unreadable depth. + const exists = await deps.queueExists(queue); + if (!exists) { + return { + ok: false, + outcome: { + result: "terminal_unsatisfied", + facts: { queue, reason: "queue_not_found" }, + observed: unobserved(true), + }, + }; + } + return { + ok: false, + outcome: { + result: "unavailable", + facts: { queue, reason: "depth_unavailable" }, + observed: unobserved(false), + }, + }; + } + + const facts = { + queue, + depth: depth.depth, + depthSource: depth.source, + depthAsOf: depth.asOf?.toISOString() ?? null, + depthApproximate: !depth.current, + }; + + // A claim of quiet needs a reading that describes now: a stale empty bucket is never + // read as drained. + if (!depth.current && (args.requireCurrent || depth.depth <= quietLine)) { + return { + ok: false, + outcome: { + result: "unavailable", + facts: { ...facts, reason: "depth_stale" }, + observed: unobserved(false), + }, + }; + } + + return { ok: true, depth, facts }; +} + +/** + * Satisfied when the queue's current pending count is 0. The observation carries the depth + * read, so a window completing without a drain needs no second read. + */ +export async function checkBacklogDrain( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ kind: "backlog_drain", verified, depth: null }), + quietLine: 0, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth === 0 ? "satisfied" : "pending", + facts: read.facts, + observed: { kind: "backlog_drain", verified: true, depth: read.depth.depth }, + }; +} + +/** + * Satisfied when the pending count rises above `threshold`. No `terminal_unsatisfied` on a + * live queue: only the queue disappearing makes the condition impossible. + */ +export async function checkQueueDepthAbove( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_depth_above", + verified, + depth: null, + threshold: spec.threshold, + }), + quietLine: spec.threshold, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth > spec.threshold ? "satisfied" : "pending", + facts: { ...read.facts, threshold: spec.threshold }, + observed: { + kind: "queue_depth_above", + verified: true, + depth: read.depth.depth, + threshold: spec.threshold, + }, + }; +} + +/** + * The mirror of `queue_depth_above`: satisfied at or under `threshold`, which is also the quiet + * line for the freshness fence. Only the queue disappearing is terminal. + */ +export async function checkQueueDepthBelow( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_depth_below", + verified, + depth: null, + threshold: spec.threshold, + }), + quietLine: spec.threshold, + }); + if (!read.ok) return read.outcome; + + return { + result: read.depth.depth <= spec.threshold ? "satisfied" : "pending", + facts: { ...read.facts, threshold: spec.threshold }, + observed: { + kind: "queue_depth_below", + verified: true, + depth: read.depth.depth, + threshold: spec.threshold, + }, + }; +} + +/** The stall state one check hands the next, read out of the previous facts. */ +type WatchStallState = { depth: number; notDecreasingStreak: number }; + +function readStallState( + previous: Record | null | undefined +): WatchStallState | null { + if (!previous) return null; + const depth = previous.depth; + if (typeof depth !== "number" || !Number.isFinite(depth)) return null; + const streak = previous.notDecreasingStreak; + return { + depth, + notDecreasingStreak: typeof streak === "number" && Number.isFinite(streak) ? streak : 0, + }; +} + +/** + * Satisfied when the depth fails to decrease for `ticks` consecutive checks with runs queued. + * The streak lives only in `input.previous`, a gap freezes it, and depth 0 resets it. + */ +export async function checkQueueStalled( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const previous = readStallState(input.previous); + const read = await readDepthOrOutcome({ + queue: spec.queue, + deps, + unobserved: (verified) => ({ + kind: "queue_stalled", + verified, + depth: null, + // Carry the streak through an unusable check. + notDecreasingStreak: previous?.notDecreasingStreak ?? 0, + ticks: spec.ticks, + }), + quietLine: 0, + requireCurrent: true, + }); + if (!read.ok) return read.outcome; + + const depth = read.depth.depth; + // A first observation has nothing to compare against, so it isn't a stalled tick. + const notDecreasingStreak = + depth === 0 || previous === null + ? 0 + : depth >= previous.depth + ? previous.notDecreasingStreak + 1 + : 0; + + const facts = { + ...read.facts, + previousDepth: previous?.depth ?? null, + notDecreasingStreak, + ticks: spec.ticks, + }; + + return { + result: depth > 0 && notDecreasingStreak >= spec.ticks ? "satisfied" : "pending", + facts, + observed: { + kind: "queue_stalled", + verified: true, + depth, + notDecreasingStreak, + ticks: spec.ticks, + }, + }; +} + +/** + * Satisfied when the oldest waiting run has waited longer than the SLA. Any stale reading is + * `unavailable` rather than compared, and an empty queue is `pending`. + */ +export async function checkQueueOldestAge( + spec: Extract, + deps: WatchCheckDeps, + _input: WatchCheckInput +): Promise { + const thresholdMs = spec.thresholdMinutes * 60_000; + const unobserved = (verified: boolean): WatchObservedOutcome => ({ + kind: "queue_oldest_age", + verified, + ageMs: null, + thresholdMinutes: spec.thresholdMinutes, + }); + + const gone = (): WatchCheckOutcome => ({ + result: "terminal_unsatisfied", + facts: { queue: spec.queue, reason: "queue_not_found" }, + observed: unobserved(true), + }); + + const reading = await deps.readQueueOldestAge(spec.queue); + + if (reading === null) { + if (!(await deps.queueExists(spec.queue))) return gone(); + return { + result: "unavailable", + facts: { queue: spec.queue, reason: "age_unavailable" }, + observed: unobserved(false), + }; + } + + // Nothing waiting reads the same as a deleted queue, and only the second is terminal. + if (reading.ageMs === null && !(await deps.queueExists(spec.queue))) return gone(); + + const facts = { + queue: spec.queue, + ageMs: reading.ageMs, + ageLabel: reading.ageMs === null ? null : formatMs(reading.ageMs), + ageSource: reading.source, + ageAsOf: reading.asOf?.toISOString() ?? null, + thresholdMinutes: spec.thresholdMinutes, + }; + + if (!reading.current) { + return { + result: "unavailable", + facts: { ...facts, reason: "age_stale" }, + observed: unobserved(false), + }; + } + + const observed: WatchObservedOutcome = { + kind: "queue_oldest_age", + verified: true, + ageMs: reading.ageMs, + thresholdMinutes: spec.thresholdMinutes, + }; + + return { + result: reading.ageMs !== null && reading.ageMs > thresholdMs ? "satisfied" : "pending", + facts, + observed, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts b/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts new file mode 100644 index 00000000000..ceccbb43f0f --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchRunChecks.ts @@ -0,0 +1,240 @@ +/** + * The run condition family: start, finished, failed. All three read one run row and + * label the wait with what the data actually supports. + */ + +import { + watchRunDisposition, + type WatchObservedOutcome, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { + formatMs, + type WatchCheckDeps, + type WatchCheckInput, + type WatchCheckOutcome, + type WatchRunRow, +} from "./dashboardAgentWatchCheckBase"; + +// Mirrors ~/v3/taskStatus. Kept local so this module has no server-side import. + +const FINAL_STATUSES = new Set([ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]); + +/** + * Statuses whose `queuedAt` is a leftover from the first enqueue, since resume/retry + * re-enqueues don't restamp it, so a wait computed from it isn't this attempt's. + */ +const STALE_QUEUED_AT_STATUSES = new Set(["WAITING_TO_RESUME", "RETRYING_AFTER_FAILURE", "PAUSED"]); + +export function isTerminalRunStatus(status: string): boolean { + return FINAL_STATUSES.has(status); +} + +/** Which timestamp a wait was measured from. */ +export type WatchWaitBasis = "queued_at" | "delay_until" | "created_at"; + +/** + * The wait a run has accumulated, labelled with what the data supports. A resumed, retried or + * paused run's stale `queuedAt` is never measured from. + */ +export function describeRunWait( + run: WatchRunRow, + now: Date +): { + waitMs: number | null; + waitBasis: WatchWaitBasis; + waitLabel: string; + /** True only when the wait is this attempt's queue wait. */ + queueWaitReliable: boolean; +} { + const queueWaitReliable = run.queuedAt !== null && !STALE_QUEUED_AT_STATUSES.has(run.status); + const end = run.startedAt ?? now; + + if (run.queuedAt && queueWaitReliable) { + const waitMs = Math.max(0, end.getTime() - run.queuedAt.getTime()); + return { + waitMs, + waitBasis: "queued_at", + waitLabel: `queued for ${formatMs(waitMs)}`, + queueWaitReliable, + }; + } + + if (run.delayUntil && run.delayUntil.getTime() > now.getTime()) { + return { + waitMs: null, + waitBasis: "delay_until", + waitLabel: `scheduled to start at ${run.delayUntil.toISOString()}`, + queueWaitReliable, + }; + } + + // No `queuedAt`, or one from an earlier attempt: fall back to the run's age. + const waitMs = Math.max(0, end.getTime() - run.createdAt.getTime()); + const resumeOrRetry = run.queuedAt !== null; + return { + waitMs, + waitBasis: "created_at", + waitLabel: resumeOrRetry + ? `waiting to ${run.status === "RETRYING_AFTER_FAILURE" ? "retry" : "resume"}; time from creation: ${formatMs(waitMs)}` + : `time from creation: ${formatMs(waitMs)}`, + queueWaitReliable, + }; +} + +/** + * Satisfied the moment `startedAt` exists, whatever the current status. Terminal with no + * `startedAt` can never start, so it is `terminal_unsatisfied`. + */ +export async function checkRunStart( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + // Existence was validated at creation, so the run is gone and can never start. + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_start", verified: true, status: null, started: false }, + }; + } + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + status: run.status, + queue: run.queue, + startedAt: run.startedAt?.toISOString() ?? null, + queuedAt: run.queuedAt?.toISOString() ?? null, + ...wait, + }; + const observed: WatchObservedOutcome = { + kind: "run_start", + verified: true, + status: run.status, + started: run.startedAt !== null, + }; + + if (run.startedAt) return { result: "satisfied", facts, observed }; + if (isTerminalRunStatus(run.status)) { + return { + result: "terminal_unsatisfied", + facts: { ...facts, reason: "never_started" }, + observed, + }; + } + return { result: "pending", facts, observed }; +} + +/** + * Satisfied on any terminal status. Finished and failed are both `condition_met`, so only + * `observed.finalStatus` separates them. + */ +export async function checkRunFinished( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_finished", verified: true, finalStatus: null, durationMs: null }, + }; + } + + const finished = isTerminalRunStatus(run.status); + // Execution duration only. The queue wait is reported separately. + const durationMs = + run.startedAt && run.completedAt + ? Math.max(0, run.completedAt.getTime() - run.startedAt.getTime()) + : null; + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + outcome: run.status, + startedAt: run.startedAt?.toISOString() ?? null, + completedAt: run.completedAt?.toISOString() ?? null, + durationMs, + durationLabel: durationMs === null ? null : formatMs(durationMs), + ...wait, + }; + + return { + result: finished ? "satisfied" : "pending", + facts, + observed: { + kind: "run_finished", + verified: true, + // Only a terminal status is a final status. + finalStatus: finished ? run.status : null, + durationMs, + }, + }; +} + +/** + * A failing terminal status satisfies this; a successful completion or a cancellation makes + * the condition impossible rather than merely unmet. + */ +export async function checkRunFailed( + spec: Extract, + deps: WatchCheckDeps, + input: WatchCheckInput +): Promise { + const run = await deps.readRun(spec.runId); + if (!run) { + return { + result: "terminal_unsatisfied", + facts: { runId: spec.runId, reason: "run_not_found" }, + observed: { kind: "run_failed", verified: true, finalStatus: null, durationMs: null }, + }; + } + + const finished = isTerminalRunStatus(run.status); + const durationMs = + run.startedAt && run.completedAt + ? Math.max(0, run.completedAt.getTime() - run.startedAt.getTime()) + : null; + + const wait = describeRunWait(run, input.now); + const facts = { + runId: run.friendlyId, + outcome: run.status, + startedAt: run.startedAt?.toISOString() ?? null, + completedAt: run.completedAt?.toISOString() ?? null, + durationMs, + durationLabel: durationMs === null ? null : formatMs(durationMs), + ...wait, + }; + const observed: WatchObservedOutcome = { + kind: "run_failed", + verified: true, + finalStatus: finished ? run.status : null, + durationMs, + }; + + if (!finished) return { result: "pending", facts, observed }; + + return { + result: watchRunDisposition(run.status) === "failed" ? "satisfied" : "terminal_unsatisfied", + facts: { + ...facts, + ...(watchRunDisposition(run.status) === "failed" ? {} : { reason: "cannot_fail_now" }), + }, + observed, + }; +} diff --git a/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts b/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts new file mode 100644 index 00000000000..45358854413 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchSweep.server.ts @@ -0,0 +1,529 @@ +/** + * The watch backstop for a dead tick chain. Finalization runs even with no agent project, or rows + * would stay `active` forever; what can't be handed over stays owed. Guarded, so a re-run no-ops. + */ + +import { + cancelWatch, + claimWatchAlertDispatch, + deleteTerminalWatchesOlderThan, + deleteWatchSubmissionsOlderThan, + listExpiredActiveWatches, + listWatchBatchGroupsToArm, + listWatchesAwaitingDelivery, + releaseWatchAlertDispatch, + transitionWatchCondition, + type Watch, + type WatchBatchGroup, +} from "@internal/dashboard-agent-db"; +import { + watchResolutionForCheck, + watchResolutionToWireStatus, + type WatchObservedOutcome, + type WatchResolution, +} from "@internal/dashboard-agent-contracts"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { enqueueWatchFiredAlert } from "~/services/dashboardAgentWatchAlerts.server"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, + type WatchCheckOutcome, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { mapWithConcurrency, shareReads } from "~/services/dashboardAgentWatchBatch.server"; +import { isDashboardAgentConfigured } from "~/services/dashboardAgent.server"; +import { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, + scheduleWatchDelivery, + type WatchAuthorization, +} from "~/services/dashboardAgentWatches.server"; +import { logger } from "~/services/logger.server"; + +/** + * How long past `expiresAt` a watch is left to the tick chain. Only has to cover a late + * tick: the chain's own final check happens within a cadence of the deadline. + */ +export const WATCH_EXPIRY_GRACE_MS = 2 * 60 * 1000; + +/** + * How long a resolved watch may owe its wake before the sweep recovers it. Long enough that + * the recovery can't race a delivery still in flight. + */ +export const WATCH_DELIVERY_GRACE_MS = 5 * 60 * 1000; + +/** Per-run cap for each half of the sweep. Oldest first, so the rest land next run. */ +const SWEEP_BATCH_LIMIT = 100; + +/** How long a terminal watch is kept. Its outcome also lives in the chat transcript. */ +export const WATCH_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +/** Higher than the other caps: retention is one statement, not a row-at-a-time loop. */ +const RETENTION_BATCH_LIMIT = 500; + +/** + * How many rows one sweep handles at once. An incident expires a whole group together, and a + * bound is what stops one slow tenant spending the entire visibility window. + */ +const SWEEP_CONCURRENCY = 8; + +/** What one finalization did. */ +export type WatchFinalizeOutcome = + | "fired" + | "expired" + /** The user lost access: cancelled, and deliberately not narrated. */ + | "cancelled" + /** A tick (or another sweep) resolved it first. */ + | "already_resolved"; + +export type WatchSweepResult = { + /** Overdue active rows seen. */ + overdue: number; + fired: number; + expired: number; + cancelled: number; + alreadyResolved: number; + /** Resolved rows whose wake was still owed. */ + undelivered: number; + /** Wakes handed back to the watcher task. */ + redelivered: number; + /** Decided but not handed over, with no agent project. They stay owed. */ + deliveryDeferred: number; + /** Long-terminal rows dropped by retention. */ + purged: number; + /** Ledger rows dropped by retention. */ + purgedSubmissions: number; + failed: number; +}; + +export type WatchSweepDeps = { + now?: () => Date; + limit?: number; + /** Overdue `active` rows. */ + listOverdue?: (params: { now: Date; limit: number }) => Promise; + /** Resolved rows whose wake is still owed. */ + listAwaitingDelivery?: (params: { olderThan: Date; limit: number }) => Promise; + /** Re-authorization of the watch's initiating user. */ + authorize?: (watch: Watch) => Promise; + /** The environment readers the final check runs against. */ + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + /** Hand the wake back to the watcher task. Must throw if it can't be scheduled. */ + deliver?: (watch: Watch) => Promise; + /** Gates the delivery half only. Finalization never depends on it. */ + configured?: () => boolean; + /** Drop terminal rows older than `before`. Returns how many went. */ + purgeTerminal?: (params: { before: Date; limit: number }) => Promise; + /** Drop submission-ledger rows older than `before`. */ + purgeSubmissions?: (params: { before: Date; limit: number }) => Promise; + /** How many rows are handled at once. */ + concurrency?: number; +}; + +/** + * One re-authorization per (user, org, project, environment) for the whole sweep. An incident + * expires a group together, and every row of it names the same access question. + */ +function authorizeOncePerSweep( + deps: WatchSweepDeps +): (watch: Watch) => Promise { + const authorize = deps.authorize ?? defaultAuthorize; + const seen = new Map>(); + return (watch) => { + const key = `${watch.userId}:${watch.organizationId}:${watch.projectId}:${watch.environmentId}`; + const cached = seen.get(key); + if (cached) return cached; + const pending = authorize(watch); + seen.set(key, pending); + return pending; + }; +} + +/** + * One set of readers per environment, read-shared the way the batch's are: `now` is fixed for + * the sweep, so the same read can't answer two ways. + */ +function readersOncePerSweep( + deps: WatchSweepDeps +): (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps { + const build = deps.checkDeps ?? watchCheckDeps; + const seen = new Map(); + return (environment, now) => { + const cached = seen.get(environment.id); + if (cached) return cached; + const readers = shareReads(build(environment, now)); + seen.set(environment.id, readers); + return readers; + }; +} + +/** + * Facts for a swept expiry, in the same shape the tick writes. `verified: false` means the + * condition couldn't be evaluated, and carries the last observation instead. + */ +function expiredFacts( + watch: Watch, + args: { verified: boolean; reason: string; facts?: Record } +): Record { + return { + verified: args.verified, + reason: args.reason, + expiredAt: watch.expiresAt.toISOString(), + checks: watch.tickCount, + ...(args.verified + ? (args.facts ?? {}) + : { + lastObservedAt: watch.lastCheckedAt?.toISOString(), + lastObservation: watch.lastResult, + }), + }; +} + +/** + * How a final check's verdict resolves the row. The final read is a real evaluation, so only + * `pending` and `unavailable` become `window_completed`. `watchResolutionForCheck` owns that. + */ +function resolutionFor( + watch: Watch, + outcome: WatchCheckOutcome +): { + resolution: WatchResolution; + observed: WatchObservedOutcome; + facts: Record; +} { + // Always non-null here: this is the boundary evaluation. + const resolution = watchResolutionForCheck(outcome.result, true)!; + + switch (outcome.result) { + case "satisfied": + return { + resolution, + observed: outcome.observed, + facts: { verified: true, ...outcome.facts }, + }; + case "terminal_unsatisfied": + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { + verified: true, + reason: "terminal_unsatisfied", + facts: outcome.facts, + }), + }; + case "pending": + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { + verified: true, + reason: "not_met_by_expiry", + facts: outcome.facts, + }), + }; + default: + // The check couldn't run, so the window completes unverified. + return { + resolution, + observed: outcome.observed, + facts: expiredFacts(watch, { verified: false, reason: "unverified_at_expiry" }), + }; + } +} + +/** + * Finalize one overdue watch. Re-authorization comes first, before the final check reads + * anything; `canDeliver: false` stops at the resolution, leaving the wake owed. + */ +export async function finalizeOverdueWatch( + watch: Watch, + deps: WatchSweepDeps & { canDeliver?: boolean } = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const authorize = deps.authorize ?? defaultAuthorize; + const buildCheckDeps = deps.checkDeps ?? watchCheckDeps; + const deliver = deps.deliver ?? scheduleWatchDelivery; + const canDeliver = deps.canDeliver ?? true; + + const authorization = await authorize(watch); + if (!authorization.ok) { + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "access_revoked" }); + logger.info("Dashboard agent watch sweep: cancelled a watch whose access was revoked", { + watchId: watch.id, + }); + return "cancelled"; + } + + const since = watch.spec.since ? new Date(watch.spec.since) : watch.createdAt; + const outcome = await checkWatch( + watch.spec, + buildCheckDeps(authorization.environment, now), + // A stateful condition is a transition across checks, so the boundary evaluation needs + // the previous facts to see one — and to record what it saw, not a reset. + { now, since, previous: previousCheckFacts(watch.lastResult) }, + (error) => + logger.error("Dashboard agent watch sweep: the final check failed", { + watchId: watch.id, + error, + }) + ); + + const resolved = resolutionFor(watch, outcome); + const transitioned = await transitionWatchCondition(dashboardAgentDb, { + id: watch.id, + resolution: resolved.resolution, + observedOutcome: resolved.observed, + lastResult: resolved.facts, + }); + + // Guarded on `active`: a tick that resolved it first keeps its outcome and delivery. + if (!transitioned) return "already_resolved"; + + if (resolved.resolution === "condition_met") { + // Claimed first, exactly as the fire callback does: the wake this sweep is about to + // schedule reports the same fired watch, and an unclaimed row alerts a second time. + const claimed = await claimWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + if (claimed) { + try { + await enqueueWatchFiredAlert(transitioned, "fired"); + } catch (error) { + await releaseWatchAlertDispatch(dashboardAgentDb, { + id: watch.id, + terminalStatus: "fired", + }); + logger.error("Dashboard agent watch sweep: failed to enqueue the fired alert", { + watchId: watch.id, + error, + }); + } + } + } + + // Throws if it can't be scheduled, leaving the row terminal with its delivery owed. + if (canDeliver) await deliver(transitioned); + return watchResolutionToWireStatus(resolved.resolution); +} + +/** + * Recover one owed wake, unconditionally: this sweep can't tell whether the user was already + * told. Whether the wake needs prose is decided where the transcript can be read. + */ +export async function recoverWatchDelivery(watch: Watch, deps: WatchSweepDeps = {}): Promise { + const deliver = deps.deliver ?? scheduleWatchDelivery; + await deliver(watch); +} + +/** + * One sweep: finalize what is overdue, then recover what was never delivered. Each row is + * handled on its own, and the run throws at the end if any failed so the job is retried. + */ +export async function sweepDashboardAgentWatches( + deps: WatchSweepDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const limit = deps.limit ?? SWEEP_BATCH_LIMIT; + const configured = deps.configured ?? isDashboardAgentConfigured; + const listOverdue = + deps.listOverdue ?? ((params) => listExpiredActiveWatches(dashboardAgentDb, params)); + const listAwaitingDelivery = + deps.listAwaitingDelivery ?? + ((params) => listWatchesAwaitingDelivery(dashboardAgentDb, params)); + const purgeTerminal = + deps.purgeTerminal ?? ((params) => deleteTerminalWatchesOlderThan(dashboardAgentDb, params)); + const purgeSubmissions = + deps.purgeSubmissions ?? + ((params) => deleteWatchSubmissionsOlderThan(dashboardAgentDb, params)); + + const result: WatchSweepResult = { + overdue: 0, + fired: 0, + expired: 0, + cancelled: 0, + alreadyResolved: 0, + undelivered: 0, + redelivered: 0, + deliveryDeferred: 0, + purged: 0, + purgedSubmissions: 0, + failed: 0, + }; + + // Gates the hand-off only: the rows still have to be finalized. + const canDeliver = configured(); + if (!canDeliver) { + logger.warn( + "Dashboard agent watch sweep: the agent isn't configured, so wakes can't be delivered — finalizing only" + ); + } + + const overdue = await listOverdue({ + now: new Date(now.getTime() - WATCH_EXPIRY_GRACE_MS), + limit, + }); + result.overdue = overdue.length; + + // The authorization and the readers are resolved once for the whole sweep, so a group of + // expiries costs one of each rather than one per row. + const perSweep: WatchSweepDeps & { canDeliver: boolean } = { + ...deps, + now: () => now, + canDeliver, + authorize: authorizeOncePerSweep(deps), + checkDeps: readersOncePerSweep(deps), + }; + + const finalized = await mapWithConcurrency( + overdue, + deps.concurrency ?? SWEEP_CONCURRENCY, + async (watch) => { + try { + return await finalizeOverdueWatch(watch, perSweep); + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to finalize a watch", { + watchId: watch.id, + error, + }); + return null; + } + } + ); + + for (const outcome of finalized) { + if (outcome === null) result.failed++; + else if (outcome === "fired") result.fired++; + else if (outcome === "expired") result.expired++; + else if (outcome === "cancelled") result.cancelled++; + else result.alreadyResolved++; + // Resolved, but nothing carried the wake away: it stays owed. + if (!canDeliver && (outcome === "fired" || outcome === "expired")) { + result.deliveryDeferred++; + } + } + + // Skipped without an agent project: the rows keep their owed wake for the next sweep. + if (canDeliver) { + const owed = await listAwaitingDelivery({ + olderThan: new Date(now.getTime() - WATCH_DELIVERY_GRACE_MS), + limit, + }); + result.undelivered = owed.length; + + const recovered = await mapWithConcurrency( + owed, + deps.concurrency ?? SWEEP_CONCURRENCY, + async (watch) => { + try { + await recoverWatchDelivery(watch, deps); + return true; + } catch (error) { + logger.error("Dashboard agent watch sweep: failed to recover a wake", { + watchId: watch.id, + error, + }); + return false; + } + } + ); + + for (const ok of recovered) { + if (ok) result.redelivered++; + else result.failed++; + } + } + + // Retention runs last, over rows both halves are finished with. Its own try/catch so a + // lost retention pass can't mask the other failures. + try { + const before = new Date(now.getTime() - WATCH_RETENTION_MS); + result.purged = await purgeTerminal({ before, limit: RETENTION_BATCH_LIMIT }); + // The ledger's rows age out on the same window: past it no client is still retrying. + result.purgedSubmissions = await purgeSubmissions({ before, limit: RETENTION_BATCH_LIMIT }); + } catch (error) { + result.failed++; + logger.error("Dashboard agent watch sweep: failed to purge terminal watches", { error }); + } + + if (result.failed > 0) { + throw new Error(`The dashboard agent watch sweep failed on ${result.failed} watches`); + } + + return result; +} + +export type WatchBatchRearmResult = { + /** Groups with active watches and no live chain. */ + stale: number; + armed: number; + failed: number; +}; + +export type WatchBatchRearmDeps = { + now?: () => Date; + limit?: number; + /** Groups whose chain is missing, stopped, or has gone silent. */ + listGroups?: (params: { now: Date; limit: number }) => Promise; + /** Start a chain for one group. */ + arm?: (params: { + environmentId: string; + cadenceMinutes: number; + now?: Date; + }) => Promise<{ running: boolean }>; + configured?: () => boolean; +}; + +const REARM_BATCH_LIMIT = 200; + +/** + * Re-arm batch chains that died. `armWatchBatch` re-checks the same timestamp in the statement + * that arms, so racing a merely slow chain arms nothing and two runs start one chain. + */ +export async function rearmDashboardAgentWatchBatches( + deps: WatchBatchRearmDeps = {} +): Promise { + const now = deps.now?.() ?? new Date(); + const limit = deps.limit ?? REARM_BATCH_LIMIT; + const configured = deps.configured ?? isDashboardAgentConfigured; + const listGroups = + deps.listGroups ?? ((params) => listWatchBatchGroupsToArm(dashboardAgentDb, params)); + const arm = deps.arm ?? armDashboardAgentWatchBatch; + + const result: WatchBatchRearmResult = { stale: 0, armed: 0, failed: 0 }; + + // Nothing to trigger a chain into. The expiry half still finalizes the rows. + if (!configured()) return result; + + const groups = await listGroups({ now, limit }); + result.stale = groups.length; + + for (const group of groups) { + try { + const { running } = await arm({ ...group, now }); + if (running) result.armed++; + } catch (error) { + result.failed++; + logger.error("Dashboard agent watch sweep: failed to re-arm a batch chain", { + ...group, + error, + }); + } + } + + if (result.failed > 0) { + throw new Error(`The dashboard agent batch re-arm failed on ${result.failed} groups`); + } + + return result; +} + +function defaultAuthorize(watch: Watch): Promise { + return authorizeWatchEnvironment({ + userId: watch.userId, + organizationId: watch.organizationId, + projectId: watch.projectId, + environmentId: watch.environmentId, + }); +} diff --git a/apps/webapp/app/services/dashboardAgentWatchToken.server.ts b/apps/webapp/app/services/dashboardAgentWatchToken.server.ts new file mode 100644 index 00000000000..dc56691f560 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatchToken.server.ts @@ -0,0 +1,181 @@ +import { generateJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { env } from "~/env.server"; + +/** + * The credential the watcher task presents to the private check endpoint. It names one watch, is + * kept apart from the UAT by a disjoint prefix and `kind`, and is re-minted, never persisted. + */ + +export const WATCH_TOKEN_PREFIX = "tr_daw_"; + +/** Distinguishes a watch token from every other SESSION_SECRET-signed JWT. */ +const WATCH_TOKEN_KIND = "dashboard_agent_watch"; + +/** Mirrors the UAT's `act.client` with a value no UAT uses. Verified, so no replay. */ +const WATCH_TOKEN_CLIENT = "dashboard-agent-watch"; + +/** + * How long past `expiresAt` the token stays valid. The final check happens after the + * deadline, so the token has to outlive the watch by enough to cover a late tick. + */ +export const WATCH_TOKEN_GRACE_MS = 60 * 60 * 1000; + +export type WatchTokenClaims = { + watchId: string; + /** Token expiry (seconds since epoch), i.e. `expiresAt` + grace. */ + expiresAtSeconds: number; +}; + +export function isDashboardAgentWatchToken(token: string): boolean { + return token.startsWith(WATCH_TOKEN_PREFIX); +} + +/** Deterministic: the same inputs produce the same string. */ +export async function signDashboardAgentWatchToken( + secret: string, + opts: { watchId: string; expiresAt: Date; graceMs?: number } +): Promise { + const expirationTime = Math.floor( + (opts.expiresAt.getTime() + (opts.graceMs ?? WATCH_TOKEN_GRACE_MS)) / 1000 + ); + + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: WATCH_TOKEN_KIND, + // `sub` is the watch: a watch token never authenticates a user. + sub: opts.watchId, + act: { client: WATCH_TOKEN_CLIENT }, + }, + expirationTime, + omitIssuedAt: true, + }); + + return `${WATCH_TOKEN_PREFIX}${jwt}`; +} + +/** `undefined` for anything but a valid watch token, including a valid user-actor token. */ +export async function verifyDashboardAgentWatchToken( + secret: string, + token: string +): Promise { + if (!isDashboardAgentWatchToken(token)) return; + + const result = await validateJWT(token.slice(WATCH_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== WATCH_TOKEN_KIND) return; + if (typeof payload.sub !== "string" || payload.sub.length === 0) return; + + const act = payload.act as { client?: string } | undefined; + if (act?.client !== WATCH_TOKEN_CLIENT) return; + if (typeof payload.exp !== "number") return; + + return { watchId: payload.sub, expiresAtSeconds: payload.exp }; +} + +export function mintDashboardAgentWatchToken(opts: { + watchId: string; + expiresAt: Date; +}): Promise { + return signDashboardAgentWatchToken(env.SESSION_SECRET, opts); +} + +export function verifyWatchTokenFromRequest(token: string): Promise { + return verifyDashboardAgentWatchToken(env.SESSION_SECRET, token); +} + +/** + * Chain tokens name a whole (environment, cadence) group. The prefix is disjoint from + * `tr_daw_` rather than nested under it, so neither verifier sees the other's tokens. + */ +export const WATCH_BATCH_TOKEN_PREFIX = "tr_dab_"; + +const WATCH_BATCH_TOKEN_KIND = "dashboard_agent_watch_batch"; +const WATCH_BATCH_TOKEN_CLIENT = "dashboard-agent-watch-batch"; + +/** + * How long a chain's token lives. A chain has no deadline to pin it to, but it must still + * expire; an expired one is self-healing via the re-arm backstop. + */ +export const WATCH_BATCH_TOKEN_TTL_MS = 30 * 24 * 60 * 60 * 1000; + +export type WatchBatchTokenClaims = { environmentId: string; cadenceMinutes: number }; + +/** + * Like a watch token, a chain token names one group and carries no authority of its own: the + * batch check re-authorizes every watch's initiating user against that watch's snapshot. + */ +export async function signDashboardAgentWatchBatchToken( + secret: string, + opts: { environmentId: string; cadenceMinutes: number; expiresAt: Date } +): Promise { + const jwt = await generateJWT({ + secretKey: secret, + payload: { + kind: WATCH_BATCH_TOKEN_KIND, + // The group, not a user and not a watch. + sub: `${opts.environmentId}:${opts.cadenceMinutes}`, + act: { client: WATCH_BATCH_TOKEN_CLIENT }, + }, + expirationTime: Math.floor(opts.expiresAt.getTime() / 1000), + omitIssuedAt: true, + }); + + return `${WATCH_BATCH_TOKEN_PREFIX}${jwt}`; +} + +/** `undefined` for anything that isn't a valid, unexpired chain token. */ +export async function verifyDashboardAgentWatchBatchToken( + secret: string, + token: string +): Promise { + if (!token.startsWith(WATCH_BATCH_TOKEN_PREFIX)) return; + + const result = await validateJWT(token.slice(WATCH_BATCH_TOKEN_PREFIX.length), secret); + if (!result.ok) return; + + const payload = result.payload; + if (payload.kind !== WATCH_BATCH_TOKEN_KIND) return; + const act = payload.act as { client?: string } | undefined; + if (act?.client !== WATCH_BATCH_TOKEN_CLIENT) return; + if (typeof payload.sub !== "string") return; + + // The cadence is the last segment, split from the right so a colon in an environment id + // could never confuse it. + const separator = payload.sub.lastIndexOf(":"); + if (separator <= 0) return; + const environmentId = payload.sub.slice(0, separator); + const cadenceMinutes = Number(payload.sub.slice(separator + 1)); + if (!Number.isInteger(cadenceMinutes) || cadenceMinutes <= 0) return; + + return { environmentId, cadenceMinutes }; +} + +export function mintDashboardAgentWatchBatchToken(opts: { + environmentId: string; + cadenceMinutes: number; + now?: Date; +}): Promise { + const now = opts.now ?? new Date(); + return signDashboardAgentWatchBatchToken(env.SESSION_SECRET, { + environmentId: opts.environmentId, + cadenceMinutes: opts.cadenceMinutes, + expiresAt: new Date(now.getTime() + WATCH_BATCH_TOKEN_TTL_MS), + }); +} + +export function verifyWatchBatchTokenFromRequest( + token: string +): Promise { + return verifyDashboardAgentWatchBatchToken(env.SESSION_SECRET, token); +} + +/** The bearer value from an `Authorization: Bearer …` header, if present. */ +export function bearerToken(request: Request): string | undefined { + const raw = request.headers.get("Authorization"); + if (!raw) return undefined; + const value = raw.replace(/^Bearer /, "").trim(); + return value.length > 0 ? value : undefined; +} diff --git a/apps/webapp/app/services/dashboardAgentWatches.server.ts b/apps/webapp/app/services/dashboardAgentWatches.server.ts new file mode 100644 index 00000000000..57f38ae8bd4 --- /dev/null +++ b/apps/webapp/app/services/dashboardAgentWatches.server.ts @@ -0,0 +1,1175 @@ +/** + * Watches, webapp half: creation, the re-authorization a background check passes, and the + * chat-delete cascade. The row's snapshot is immutable, and no target comes from client input. + */ + +import { + MAX_ACTIVE_WATCHES_PER_CHAT, + appendChatMessageOnce, + armWatchBatch, + cancelWatch, + chatExists, + claimWatchSubmission, + createChat, + createWatch, + generateWatchId, + getChatWatchContext, + getWatch, + getWatchSubmission, + listActiveWatchesForChats as listActiveWatchesForChatsQuery, + precheckWatchCreation, + recordWatchSubmissionOutcome, + reopenWatchSubmission, + softDeleteChat, + stopWatchBatch, + type ChatWatchContext, + type PersistedWatchSpec, + type Watch, + type WatchStatus, + type WatchSubmission, +} from "@internal/dashboard-agent-db"; +import { + VIEW_BLOCK_VERSION, + WATCH_CONFIRMATION_MESSAGE_ID_PREFIX, + WATCH_REQUEST_MESSAGE_ID_PREFIX, + watchConfirmationBlockBody, + watchDraftSchema, + watchIdentity, + watchOneShotBlockBody, + watchRequestSentence, + watchSubjectLabel, + type WatchDraft, + type WatchExternalNotification, + type WatchObservedOutcome, + type WatchResolution, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { createHash } from "node:crypto"; +import { TriggerClient } from "@trigger.dev/sdk"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { authIncludeWithParent, toAuthenticated } from "~/models/runtimeEnvironment.server"; +import { isReportKey } from "~/presenters/v3/reports/report-registry"; +import { + dashboardAgentApiOrigin, + isDashboardAgentConfigured as isDashboardAgentConfiguredDefault, +} from "~/services/dashboardAgent.server"; +import { dashboardAgentDb } from "~/services/dashboardAgentDb.server"; +import { logger } from "~/services/logger.server"; +import { + checkWatch, + type WatchCheckDeps, + type WatchCheckOutcome, +} from "~/services/dashboardAgentWatchChecks"; +import { watchCreationCheckDeps } from "~/services/dashboardAgentWatchChecks.server"; +import { normalizeErrorFingerprint } from "~/services/dashboardAgentWatchErrorChecks"; +import { subscribeUserToWatchAlerts } from "~/services/dashboardAgentWatchAlerts.server"; +import { + mintDashboardAgentWatchBatchToken, + mintDashboardAgentWatchToken, +} from "~/services/dashboardAgentWatchToken.server"; +import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; + +/** The task that polls a watch. Lives in the agent project, triggered by us. */ +export const WATCH_TASK_ID = "dashboard-agent-watch"; + +export { MAX_ACTIVE_WATCHES_PER_CHAT }; + +export type WatchAuthorization = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; reason: "access_revoked" }; + +/** + * Re-authorize a watch's initiating user against the row's immutable project/environment; a + * partial pass is `access_revoked`. The membership-scoped query is the tenant floor here. + */ +export async function authorizeWatchEnvironment(params: { + userId: string; + organizationId: string; + projectId: string; + environmentId: string; +}): Promise { + // The primary, not the replica: replica lag would extend access the user has lost. + const environment = await prisma.runtimeEnvironment.findFirst({ + where: { + id: params.environmentId, + // The watch's snapshot has to still describe this environment. + projectId: params.projectId, + organizationId: params.organizationId, + archivedAt: null, + project: { deletedAt: null }, + organization: { deletedAt: null, members: { some: { userId: params.userId } } }, + OR: [ + { type: { in: ["PREVIEW", "STAGING", "PRODUCTION"] } }, + // Dev environments are per-member: only their owner may read them. + { type: "DEVELOPMENT", orgMember: { userId: params.userId } }, + ], + }, + include: authIncludeWithParent, + }); + + if (!environment) return { ok: false, reason: "access_revoked" }; + + // The gate only reads `isAdmin` while the admin preview is on, so this read is skipped + // otherwise: it runs on every watch check, batch authorization and sweep finalisation. + let isAdmin = false; + if (env.DASHBOARD_AGENT_ADMIN_PREVIEW === "1") { + // Primary for the same reason as the membership read above. + const user = await prisma.user.findFirst({ + where: { id: params.userId }, + select: { admin: true }, + }); + if (!user) return { ok: false, reason: "access_revoked" }; + isAdmin = user.admin; + } + + const allowed = await canAccessDashboardAgent({ + userId: params.userId, + isAdmin, + // A background check is never an impersonated session. + isImpersonating: false, + organizationSlug: environment.organization.slug, + orgFeatureFlags: environment.organization.featureFlags as Record | null, + }); + if (!allowed) return { ok: false, reason: "access_revoked" }; + + return { ok: true, environment: toAuthenticated(environment) }; +} + +/** + * The same authorization by environment id alone, for the creation path with no watch row + * yet. The id lookup is unscoped and proves nothing; `authorizeWatchEnvironment` is the gate. + */ +export async function authorizeWatchEnvironmentById(params: { + userId: string; + environmentId: string; +}): Promise { + const environment = await $replica.runtimeEnvironment.findFirst({ + where: { id: params.environmentId }, + select: { organizationId: true, projectId: true }, + }); + if (!environment) return null; + + const authorization = await authorizeWatchEnvironment({ + userId: params.userId, + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: params.environmentId, + }); + return authorization.ok ? authorization.environment : null; +} + +export type CreateWatchErrorCode = + | "limit_reached" + | "duplicate" + | "invalid_target" + | "chat_not_found" + | "not_configured" + | "internal"; + +/** + * Either a watch is now running (`watching: true`), or the immediate check answered and no + * row exists at all (`watching: false`), which never enters the delivery state machine. + */ +export type CreateDashboardAgentWatchResult = + | { + ok: true; + watching: true; + watchId: string; + identity: string; + status: WatchStatus; + expiresAt: Date; + /** Set when the creation-time check couldn't run. The watch is active anyway. */ + unavailable?: boolean; + } + | { + ok: true; + watching: false; + identity: string; + /** `satisfied` (already true) or `terminal_unsatisfied` (can't happen now). */ + immediate: WatchCheckOutcome; + } + | { + ok: false; + error: string; + code: CreateWatchErrorCode; + /** The watch already covering this condition, on `duplicate`. */ + existingId?: string | null; + }; + +/** The one spelling of a spec's target that the identity, the checks and the link all share. */ +function normalizeWatchSpec(spec: WatchSpec): WatchSpec { + if (spec.kind !== "error_recurrence") return spec; + return { ...spec, fingerprint: normalizeErrorFingerprint(spec.fingerprint) }; +} + +/** + * Existence check for the thing a spec points at, in this environment. `error_recurrence` + * has nothing to validate: zero occurrences so far is the normal case. + */ +async function validateWatchTarget(spec: WatchSpec, deps: WatchCheckDeps): Promise { + switch (spec.kind) { + case "run_start": + case "run_finished": + case "run_failed": + return (await deps.readRun(spec.runId)) !== null; + case "backlog_drain": + case "queue_depth_above": + case "queue_depth_below": + case "queue_stalled": + case "queue_oldest_age": + return await deps.queueExists(spec.queue); + case "error_recurrence": + return spec.fingerprint.length > 0; + case "health_recovery": + return isReportKey(spec.report); + } +} + +/** + * Create a watch for an already-authorized context. The order is load-bearing (cap, dedup, + * immediate check, create), and a first tick that can't be scheduled cancels the row. + */ +export async function createDashboardAgentWatch(params: { + environment: AuthenticatedEnvironment; + userId: string; + chatId: string; + spec: WatchSpec; + /** Consent to investigate after an attention outcome. Never inferred. */ + investigateOnAttention?: boolean; + /** Reserved by the submission ledger, so a converging retry finds the row by id. */ + watchId?: string; + now?: Date; + /** IO seams: tests inject fakes here instead of mocking the readers. */ + deps?: { + checkDeps?: (environment: AuthenticatedEnvironment, now: Date) => WatchCheckDeps; + scheduleTick?: typeof scheduleWatchTick; + /** Skip the real trigger-config gate when a tick scheduler is injected. */ + configured?: () => boolean; + }; +}): Promise { + const { environment, userId, chatId } = params; + // Normalized before anything reads it: the page cites `error_` and the tools + // cite the bare one, and only one of the two spellings may reach the identity or the link. + const spec = normalizeWatchSpec(params.spec); + const now = params.now ?? new Date(); + // Creation reads the target on the primary; the polling checks stay on the replica. + const buildCheckDeps = params.deps?.checkDeps ?? watchCreationCheckDeps; + const scheduleTick = params.deps?.scheduleTick ?? scheduleWatchTick; + const isDashboardAgentConfigured = params.deps?.configured ?? isDashboardAgentConfiguredDefault; + const checkDeps = buildCheckDeps(environment, now); + + if (!isDashboardAgentConfigured()) { + return { + ok: false, + code: "not_configured", + error: "The dashboard agent is not configured, so watches can't be scheduled.", + }; + } + + if (!(await validateWatchTarget(spec, checkDeps))) { + return { + ok: false, + code: "invalid_target", + error: "That target doesn't exist in this environment.", + }; + } + + const identity = watchIdentity(spec); + + // Advisory only: `createWatch` below re-applies both guardrails atomically and + // stays the authority. + const precheck = await precheckWatchCreation(dashboardAgentDb, { + chatId, + projectId: environment.projectId, + environmentId: environment.id, + identity, + }); + if (!precheck.ok) return creationGuardrailError(precheck); + + // `since` is server-set so the model can't backdate a recurrence window. + const persistedSpec: PersistedWatchSpec = + spec.kind === "error_recurrence" ? { ...spec, since: now.toISOString() } : spec; + + // Answer in the same turn when the condition has already happened. + const immediate = await checkWatch(persistedSpec, checkDeps, { now, since: now }, (error) => + logger.error("Dashboard agent watch: immediate check failed", { chatId, identity, error }) + ); + + if (immediate.result === "satisfied" || immediate.result === "terminal_unsatisfied") { + // Nothing is persisted: no row means no delivery claim and no wake. + return { ok: true, watching: false, identity, immediate }; + } + + const expiresAt = new Date(now.getTime() + spec.maxHours * 60 * 60 * 1000); + + const created = await createWatch(dashboardAgentDb, { + ...(params.watchId ? { id: params.watchId } : {}), + chatId, + identity, + spec: persistedSpec, + organizationId: environment.organizationId, + projectId: environment.projectId, + // The external ref travels with the row: the agent can't translate the internal id. + projectRef: environment.project.externalRef, + environmentId: environment.id, + userId, + expiresAt, + investigateOnAttention: params.investigateOnAttention === true, + }); + + if (!created.ok) { + if (created.error === "chat_not_found") { + // The chat was deleted mid-create. The query layer re-reads it under the + // per-chat lock, so nothing was written. + return { + ok: false, + code: "chat_not_found", + error: "That chat no longer exists, so nothing is being watched.", + }; + } + return creationGuardrailError(created); + } + + const watch = created.watch; + const token = await mintDashboardAgentWatchToken({ watchId: watch.id, expiresAt }); + + try { + await scheduleTick({ + watchId: watch.id, + token, + delayMinutes: spec.checkEveryMinutes, + // Each invocation claims its own generation atomically, so the first is + // `tickCount + 1`. + tick: watch.tickCount + 1, + }); + } catch (error) { + logger.error("Dashboard agent watch: failed to schedule the first tick", { + id: watch.id, + error, + }); + // Cancelled rather than resolved, because the condition was never evaluated. + // Cancellation is silent, so no wake is sent. + await cancelWatch(dashboardAgentDb, { id: watch.id, reason: "scheduling_failed" }); + return { + ok: false, + code: "internal", + error: "The watch couldn't be scheduled. Nothing is being watched.", + }; + } + + return { + ok: true, + watching: true, + watchId: watch.id, + identity, + status: "active", + expiresAt, + ...(immediate.result === "unavailable" ? { unavailable: true } : {}), + }; +} + +/* ------------------------------------------------------------------ * + * The card submit: a durable record of the request, then the watch + * ------------------------------------------------------------------ */ + +/** A stored transcript record. Deterministic, so a retry rewrites the same bytes. */ +export type WatchTranscriptMessage = { + id: string; + role: "user" | "assistant"; + parts: unknown[]; +}; + +/** A submit can also refuse a request id that arrives carrying a different draft. */ +export type SubmitWatchErrorCode = CreateWatchErrorCode | "request_conflict"; + +export type SubmitWatchCardResult = + | { + ok: true; + chatId: string; + watching: boolean; + watchId: string | null; + /** The request record and the confirmation, in transcript order. */ + messages: WatchTranscriptMessage[]; + /** Nothing was created: this call replayed a recorded outcome. */ + repaired: boolean; + } + | { + ok: false; + code: SubmitWatchErrorCode; + error: string; + existingId?: string | null; + /** Set once a chat exists, so the caller can still open it. */ + chatId?: string; + }; + +/** + * A fresh panel's chat id, derived from the request id so a retried submit lands in the + * chat the first attempt created instead of leaving an empty one behind. + * + * The whole tenancy of the request is mixed in, not just the user: `clientRequestId` is + * client-chosen, and one user can be in several organizations, so a user id alone lets two + * organizations derive the same id — where `createChat(...).onConflictDoNothing()` keeps + * the first org's chat and the second org's records land in it. + */ +function chatIdForRequest(params: { + organizationId: string; + userId: string; + environmentId: string; + clientRequestId: string; +}): string { + const digest = createHash("sha256") + .update( + `${params.organizationId}:${params.userId}:${params.environmentId}:${params.clientRequestId}` + ) + .digest("hex"); + return `chat_${digest.slice(0, 24)}`; +} + +/** + * A spec's comparable form. `since` is server-set on every attempt, so it is excluded: + * two attempts at the same request differ by it and are still the same request. + */ +function comparableSpec(spec: WatchSpec | PersistedWatchSpec): string { + const entries = Object.entries(spec as Record) + .filter(([key]) => key !== "since") + .sort(([a], [b]) => a.localeCompare(b)); + return JSON.stringify(entries); +} + +/** + * Whether an existing watch is the one this submission asked for. A retry is byte-identical, + * so anything else — a different window, cadence, note or investigate consent — is a genuinely + * different request and still conflicts. `notifyExternally` is not on the watch row, so it is + * compared through the ledger's draft digest instead, which covers the whole configuration. + */ +function isSameWatchRequest(existing: Watch, draft: WatchDraft): boolean { + return ( + comparableSpec(existing.spec) === comparableSpec(draft.spec) && + existing.investigateOnAttention === draft.followUp.investigateOnAttention + ); +} + +/** The record of what the user confirmed. Written with no model call. */ +function requestMessage(clientRequestId: string, draft: WatchDraft): WatchTranscriptMessage { + return { + id: `${WATCH_REQUEST_MESSAGE_ID_PREFIX}${clientRequestId}`, + role: "user", + parts: [ + { type: "text", text: watchRequestSentence({ spec: draft.spec, followUp: draft.followUp }) }, + ], + }; +} + +/** The confirmation block, keyed on the watch so a repair rebuilds exactly the same record. */ +function confirmationMessage(args: { + id: string; + blockId: string; + body: Record; +}): WatchTranscriptMessage { + return { + id: args.id, + role: "assistant", + parts: [ + { + type: "data-view", + data: { + blocks: [ + { ...args.body, revision: 0, version: VIEW_BLOCK_VERSION, id: `watch:${args.blockId}` }, + ], + }, + }, + ], + }; +} + +/** + * A submitted draft's comparable digest: the whole confirmed configuration, `notifyExternally` + * included. It is user consent, and the transcript records it, so a retry that flips it is a + * different request — not something to converge on behind the durable record. + */ +function draftDigest(draft: WatchDraft): string { + return createHash("sha256") + .update( + JSON.stringify([ + comparableSpec(draft.spec), + draft.followUp.investigateOnAttention, + draft.followUp.notifyExternally, + ]) + ) + .digest("hex"); +} + +/** The recorded outcome of the external consent, replayed rather than re-decided. */ +function recordedExternalNotification(recorded: WatchSubmission): WatchExternalNotification { + if (recorded.externalNotificationStatus === "enabled") return { status: "enabled" }; + if (recorded.externalNotificationStatus === "unavailable") { + return { status: "unavailable", reason: recorded.externalNotificationReason ?? "unknown" }; + } + return { status: "not_requested" }; +} + +/** The draft the ledger recorded, which is what a replay must be built from. */ +function recordedDraft(recorded: WatchSubmission, fallback: WatchDraft): WatchDraft { + const parsed = watchDraftSchema.safeParse(recorded.draft); + return parsed.success ? parsed.data : fallback; +} + +/** The refusal record, keyed off the request so a retry's success can still follow it. */ +function refusalMessage(clientRequestId: string, error: string): WatchTranscriptMessage { + return { + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}refused:${clientRequestId}`, + role: "assistant", + parts: [{ type: "text", text: error }], + }; +} + +/** + * Submit a configured watch card, for an already-authorized environment and a chat the caller + * owns. + * + * The submission ledger is the idempotency boundary, not the transcript ids: a row keyed + * `(chatId, clientRequestId)` is written *before* the condition is evaluated, and it carries + * the outcome once there is one. So a retry looks the submission up first and replays what + * was recorded — it never re-evaluates and never creates a second operation, even after the + * first watch has fired, expired or answered in one shot. Only a `pending` row, left by an + * attempt that died before writing its outcome, is allowed to proceed, and it converges on + * the watch id reserved up front rather than creating another. + * + * The transcript ordering is the second invariant: the record of what the user confirmed is + * written before anything starts running, and the confirmation after, so a crash can leave a + * watch that is visible but unconfirmed, never one that is live and invisible. + */ +export async function submitDashboardAgentWatch(params: { + environment: AuthenticatedEnvironment; + userId: string; + organizationId: string; + /** The chat the card was submitted from. A fresh panel has none, so one is created. */ + chatId?: string; + /** Stable per card submission: both transcript records are keyed off it. */ + clientRequestId: string; + draft: WatchDraft; + now?: Date; + deps?: { + create?: typeof createDashboardAgentWatch; + subscribe?: typeof subscribeUserToWatchAlerts; + } & NonNullable[0]["deps"]>; +}): Promise { + const { environment, userId, organizationId, clientRequestId, draft } = params; + const create = params.deps?.create ?? createDashboardAgentWatch; + const subscribe = params.deps?.subscribe ?? subscribeUserToWatchAlerts; + + const chatId = + params.chatId ?? + chatIdForRequest({ + organizationId, + userId, + environmentId: environment.id, + clientRequestId, + }); + if (!params.chatId) { + // Idempotent on the id, so a retry reuses the same chat rather than making another. + await createChat(dashboardAgentDb, { + id: chatId, + organizationId, + userId, + title: `Watch ${watchSubjectLabel(draft.spec)}`, + }); + } + + const digest = draftDigest(draft); + const request = requestMessage(clientRequestId, draft); + + /** Append-once, then return. Both records are deterministic, so a replay rewrites bytes. */ + const settle = async (args: { + confirmation: WatchTranscriptMessage; + watchId: string | null; + repaired: boolean; + }): Promise => { + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: args.confirmation, + }); + return { + ok: true, + chatId, + watching: args.watchId !== null, + watchId: args.watchId, + messages: [request, args.confirmation], + repaired: args.repaired, + }; + }; + + /** `confirmed` is the draft the confirmation speaks for: the recorded one on a replay. */ + const watchingConfirmation = (args: { + watchId: string; + unavailable: boolean; + external: WatchExternalNotification; + confirmed?: WatchDraft; + }) => { + const confirmed = args.confirmed ?? draft; + return confirmationMessage({ + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}${args.watchId}`, + blockId: args.watchId, + body: watchConfirmationBlockBody({ + spec: confirmed.spec, + watchId: args.watchId, + unavailable: args.unavailable, + followUp: { + investigateOnAttention: confirmed.followUp.investigateOnAttention, + external: args.external, + }, + }), + }); + }; + + const oneShotConfirmation = ( + result: "satisfied" | "terminal_unsatisfied", + confirmed: WatchDraft = draft + ) => + confirmationMessage({ + // No watch exists, so the request id is the only stable key for a one-shot. + id: `${WATCH_CONFIRMATION_MESSAGE_ID_PREFIX}one-shot:${clientRequestId}`, + blockId: watchIdentity(confirmed.spec), + body: watchOneShotBlockBody({ spec: confirmed.spec, result }), + }); + + /** + * Rebuild the transcript from a recorded outcome. Nothing is evaluated or created, and + * the record is built from the recorded draft, never the body of this attempt: the + * durable user message states what the first attempt confirmed. + */ + const replay = async (recorded: WatchSubmission): Promise => { + const confirmed = recordedDraft(recorded, draft); + + if (recorded.state === "created" && recorded.watchId) { + return settle({ + confirmation: watchingConfirmation({ + watchId: recorded.watchId, + unavailable: recorded.unavailable, + // Recorded, never re-decided: the confirmation already in the transcript is + // append-once, so a second decision here would contradict it forever. + external: recordedExternalNotification(recorded), + confirmed, + }), + watchId: recorded.watchId, + repaired: true, + }); + } + + if (recorded.state === "immediate") { + return settle({ + confirmation: oneShotConfirmation( + recorded.immediateResult === "satisfied" ? "satisfied" : "terminal_unsatisfied", + confirmed + ), + watchId: null, + repaired: true, + }); + } + + // Refused. Replayed verbatim, so the transcript and the response agree. + const error = recorded.refusalError ?? "That watch couldn't be started."; + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: refusalMessage(clientRequestId, error), + }); + return { + ok: false, + chatId, + code: (recorded.refusalCode as SubmitWatchErrorCode | null) ?? "internal", + error, + existingId: recorded.refusalExistingId, + }; + }; + + /** + * Record a refusal, then write it under the consent record rather than leaving it to a + * toast the reload forgets. Losing the write means another attempt already settled it. + */ + const refuse = async (refusal: { + code: SubmitWatchErrorCode; + error: string; + existingId?: string | null; + }): Promise => { + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "refused", + refusalCode: refusal.code, + refusalError: refusal.error, + refusalExistingId: refusal.existingId ?? null, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") return replay(winner); + } + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: refusalMessage(clientRequestId, refusal.error), + }); + return { ok: false, chatId, ...refusal }; + }; + + // Step one, before the condition is even read: the ledger row. Its primary key is what + // makes a retry a replay instead of a second operation. + const claim = await claimWatchSubmission(dashboardAgentDb, { + chatId, + clientRequestId, + organizationId, + userId, + projectId: environment.projectId, + environmentId: environment.id, + draftHash: digest, + draft: { spec: draft.spec, followUp: draft.followUp } as unknown as Record, + watchId: generateWatchId(), + }); + + // A chat can by design span environments, so a matching draft is not enough: the row + // has to have been written by this same tenancy, or a staging retry would replay a + // production watch. A mismatch is refused, never replayed. + const recordedScope = claim.submission; + if ( + recordedScope.organizationId !== organizationId || + recordedScope.userId !== userId || + recordedScope.projectId !== environment.projectId || + recordedScope.environmentId !== environment.id + ) { + return { + ok: false, + chatId, + code: "request_conflict", + error: "That request was already submitted somewhere else.", + }; + } + + // A different draft under the same request id is a different request, not a retry. + if (claim.submission.draftHash !== digest) { + return { + ok: false, + chatId, + code: "request_conflict", + error: "That request was already submitted with different settings.", + }; + } + + // Step two, before the watch can exist: a false here means the record is already there + // from an earlier attempt, and a deleted chat is caught by the create below. + await appendChatMessageOnce(dashboardAgentDb, { + chatId, + userId, + organizationId, + message: request, + }); + + let submission = claim.submission; + + // A recorded outcome is replayed. A refusal produced no side effect, so it is the one + // state that may be attempted again — under a fresh reserved id, since the old one may + // already name a cancelled row. + if (submission.state === "refused") { + const reopened = await reopenWatchSubmission(dashboardAgentDb, { + chatId, + clientRequestId, + watchId: generateWatchId(), + }); + if (!reopened) { + const current = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (current && current.state !== "pending") return replay(current); + return refuse({ code: "internal", error: "That watch couldn't be started." }); + } + submission = reopened; + } else if (submission.state !== "pending") { + return replay(submission); + } + + /** Attach the channel, record the outcome, then confirm. A lost race replays the winner. */ + const settleCreated = async (args: { + watchId: string; + unavailable: boolean; + /** The watch was already there: this call adopted it rather than creating it. */ + adopted: boolean; + }): Promise => { + // Attached after the watch exists, and a failure here never fails the creation — it is + // said out loud in the confirmation instead, and recorded so a replay repeats it. + let external: WatchExternalNotification = { status: "not_requested" }; + if (draft.followUp.notifyExternally) { + const subscribed = await subscribe({ userId, environment }); + external = subscribed.ok + ? { status: "enabled" } + : { status: "unavailable", reason: subscribed.reason }; + } + + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "created", + watchId: args.watchId, + unavailable: args.unavailable, + external, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") { + // The winning outcome doesn't name this watch as created, so this watch is an orphan: + // cancel it before replaying, or a refusal would leave a live watch behind. + if (!(winner.state === "created" && winner.watchId === args.watchId)) { + await cancelWatch(dashboardAgentDb, { id: args.watchId, reason: "superseded" }); + } + return replay(winner); + } + } + + return settle({ + confirmation: watchingConfirmation({ + watchId: args.watchId, + unavailable: args.unavailable, + external, + }), + watchId: args.watchId, + repaired: args.adopted, + }); + }; + + // Converge: an attempt that died mid-create left its row under the reserved id. + const reservedWatchId = submission.watchId ?? generateWatchId(); + const reserved = await getWatch(dashboardAgentDb, { id: reservedWatchId }); + if (reserved) { + if (reserved.status === "cancelled") { + // The previous attempt created it and then took it back. The id is spent, so this + // submission can't be completed; a fresh submit gets a fresh request id. + return refuse({ + code: "internal", + error: "The watch couldn't be scheduled. Nothing is being watched.", + }); + } + // `unavailable` isn't recoverable here: it belonged to the attempt that died. + return settleCreated({ watchId: reserved.id, unavailable: false, adopted: true }); + } + + const result = await create({ + environment, + userId, + chatId, + spec: draft.spec, + investigateOnAttention: draft.followUp.investigateOnAttention, + watchId: reservedWatchId, + now: params.now, + deps: params.deps, + }); + + if (!result.ok) { + // Pre-ledger fallback: a submit that started before this ledger existed has no row of + // its own, so an active watch matching the draft is still adopted rather than refused. + // This only ever loads a watch; it never creates one. + if (result.code === "duplicate" && result.existingId) { + const existing = await getWatch(dashboardAgentDb, { id: result.existingId }); + if ( + existing && + existing.chatId === chatId && + existing.status === "active" && + isSameWatchRequest(existing, draft) + ) { + return settleCreated({ watchId: existing.id, unavailable: false, adopted: true }); + } + } + return refuse(result); + } + + if (!result.watching) { + const recorded = await recordWatchSubmissionOutcome(dashboardAgentDb, { + chatId, + clientRequestId, + state: "immediate", + // No watch exists, so the reserved id is released rather than left dangling. + watchId: null, + immediateResult: result.immediate.result, + }); + if (!recorded) { + const winner = await getWatchSubmission(dashboardAgentDb, { chatId, clientRequestId }); + if (winner && winner.state !== "pending") return replay(winner); + } + return settle({ + confirmation: oneShotConfirmation( + result.immediate.result as "satisfied" | "terminal_unsatisfied" + ), + watchId: null, + repaired: false, + }); + } + + return settleCreated({ + watchId: result.watchId, + unavailable: result.unavailable === true, + adopted: false, + }); +} + +/** The two guardrail refusals, worded once for both the pre-check and the insert. */ +function creationGuardrailError( + refusal: + | { error: "limit_reached"; activeCount: number } + | { error: "duplicate"; existingId: string | null } +): CreateDashboardAgentWatchResult { + if (refusal.error === "limit_reached") { + return { + ok: false, + code: "limit_reached", + error: `This chat already has ${MAX_ACTIVE_WATCHES_PER_CHAT} active watches. Cancel one first.`, + }; + } + return { + ok: false, + code: "duplicate", + error: "This chat is already watching that.", + existingId: refusal.existingId, + }; +} + +/** + * Trigger one tick of the watcher task, as the agent's own environment. The token travels + * in the payload, not the database: signing is a pure function of the watch row. + */ +export async function scheduleWatchTick(params: { + watchId: string; + token: string; + delayMinutes: number; + /** The tick generation the scheduled invocation claims. */ + tick: number; +}): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + + await client.tasks.trigger( + WATCH_TASK_ID, + { watchId: params.watchId, token: params.token, apiOrigin, tick: params.tick }, + { + delay: `${params.delayMinutes}m`, + // Keyed on the generation the payload carries, so a retried schedule can't double-tick. + idempotencyKey: `watch:${params.watchId}:tick:${params.tick}`, + // Pin to the same deployed agent version the chat runs on, when set. + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** The task that polls a whole (environment, cadence) group. */ +export const WATCH_BATCH_TASK_ID = "dashboard-agent-watch-batch"; + +/** + * How long a chain may go silent before it is treated as dead and re-armed. Three cadences + * plus two minutes, so a tick's jitter and retries can't trip it. + */ +export function watchBatchStaleMs(cadenceMinutes: number): number { + return cadenceMinutes * 60_000 * 3 + 2 * 60_000; +} + +/** + * Make sure a chain is polling one (environment, cadence) group. A failed trigger un-arms the + * row, since a chain marked running with no run behind it leaves its group unpolled. + */ +export async function armDashboardAgentWatchBatch(params: { + environmentId: string; + cadenceMinutes: number; + now?: Date; + deps?: { + arm?: typeof armWatchBatch; + schedule?: typeof scheduleWatchBatchTick; + stop?: typeof stopWatchBatch; + }; +}): Promise<{ running: boolean }> { + const now = params.now ?? new Date(); + const arm = params.deps?.arm ?? armWatchBatch; + const schedule = params.deps?.schedule ?? scheduleWatchBatchTick; + const stop = params.deps?.stop ?? stopWatchBatch; + + const armed = await arm(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + staleBefore: new Date(now.getTime() - watchBatchStaleMs(params.cadenceMinutes)), + }); + + // A live chain already covers the group. + if (!armed) return { running: true }; + + try { + await schedule({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: armed.epoch, + // A claim lands on `generation + 1`. + tick: armed.generation + 1, + delayMinutes: params.cadenceMinutes, + }); + } catch (error) { + logger.error("Dashboard agent watch: failed to start a batch chain", { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + error, + }); + await stop(dashboardAgentDb, { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: armed.epoch, + }); + return { running: false }; + } + + return { running: true }; +} + +/** + * Trigger one tick of a batch chain, as the agent's own environment. The chain's token names the + * group and nothing else; the batch check re-authorizes every watch against its own snapshot. + */ +export async function scheduleWatchBatchTick(params: { + environmentId: string; + cadenceMinutes: number; + epoch: number; + tick: number; + delayMinutes: number; +}): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + const token = await mintDashboardAgentWatchBatchToken({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + + await client.tasks.trigger( + WATCH_BATCH_TASK_ID, + { + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + apiOrigin, + token, + epoch: params.epoch, + tick: params.tick, + }, + { + delay: `${params.delayMinutes}m`, + // The chain's own key shape, epoch included, so a re-armed chain can't collide + // with its predecessor's keys. + idempotencyKey: `watch-batch:${params.environmentId}:${params.cadenceMinutes}:${params.epoch}:tick:${params.tick}`, + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** + * Hand a resolved watch's wake to the watcher task, since only the agent project may append to a + * chat's `in` stream. Keyed per watch with a short TTL, so a later sweep can still retry. + */ +export async function scheduleWatchDelivery(watch: { id: string; expiresAt: Date }): Promise { + const accessToken = env.DASHBOARD_AGENT_SECRET_KEY; + if (!accessToken) throw new Error("DASHBOARD_AGENT_SECRET_KEY is not set"); + + const apiOrigin = dashboardAgentApiOrigin(); + const client = new TriggerClient({ baseURL: apiOrigin, accessToken }); + const token = await mintDashboardAgentWatchToken({ + watchId: watch.id, + expiresAt: watch.expiresAt, + }); + + await client.tasks.trigger( + WATCH_TASK_ID, + { watchId: watch.id, token, apiOrigin, tick: 0, deliverOnly: true }, + { + idempotencyKey: `watch:${watch.id}:deliver`, + idempotencyKeyTTL: "10m", + ...(env.DASHBOARD_AGENT_VERSION ? { version: env.DASHBOARD_AGENT_VERSION } : {}), + } + ); +} + +/** + * Delete a chat and end its watches in one transaction, so no live watch is left on an + * invisible chat. Owner-scoped, so a chatId the caller doesn't own deletes nothing. + */ +export async function deleteChatWithWatches(params: { + chatId: string; + userId: string; +}): Promise<{ deleted: boolean; cancelledWatches: number }> { + const result = await softDeleteChat(dashboardAgentDb, params); + return { deleted: result.deleted, cancelledWatches: result.cancelledWatches.length }; +} + +/** Dates are strings because this crosses a loader's JSON boundary. */ +export type ChatWatchChip = { + id: string; + identity: string; + status: WatchStatus; + kind: string; + note: string; + checkEveryMinutes: number; + expiresAt: string; + endedReason: string | null; + /** How the watch ended. Null while active. */ + resolution: WatchResolution | null; + /** What the resolving check observed. */ + observedOutcome: WatchObservedOutcome | null; +}; + +/** + * Active watches for many chats in one query, keyed by chatId. The query layer re-scopes the + * chat ids by org and user, so this is safe with ids from any source. + */ +export async function listActiveWatchesForChats(params: { + chatIds: string[]; + organizationId: string; + userId: string; +}): Promise> { + const byChat = await listActiveWatchesForChatsQuery(dashboardAgentDb, params); + + return Object.fromEntries( + Object.entries(byChat).map(([chatId, watches]) => [ + chatId, + watches.map((watch) => ({ + id: watch.id, + identity: watch.identity, + status: watch.status, + kind: watch.kind, + note: watch.note, + checkEveryMinutes: watch.checkEveryMinutes, + expiresAt: watch.expiresAt.toISOString(), + endedReason: watch.endedReason, + resolution: watch.resolution, + observedOutcome: watch.observedOutcome, + })), + ]) + ); +} + +export function chatBelongsToUser(params: { + chatId: string; + userId: string; + organizationId: string; +}): Promise { + return chatExists(dashboardAgentDb, params); +} + +export type { ChatWatchContext }; + +/** + * Ownership check for a chat, plus its org, the tenancy floor its watches can't leave. No + * project or environment: those come from the authorized request context. + */ +export function resolveChatWatchContext(params: { + chatId: string; + userId: string; +}): Promise { + return getChatWatchContext(dashboardAgentDb, params); +} diff --git a/apps/webapp/app/utils/localHostGuard.test.ts b/apps/webapp/app/utils/localHostGuard.test.ts new file mode 100644 index 00000000000..4f4d3e88ad8 --- /dev/null +++ b/apps/webapp/app/utils/localHostGuard.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { checkLocalOrigin, isLocalHost, LOCAL_HOSTS } from "./localHostGuard"; + +describe("checkLocalOrigin", () => { + it("accepts every host the Redis and ClickHouse guards accept", () => { + for (const host of LOCAL_HOSTS) { + const origin = host === "::1" ? "http://[::1]:3030" : `http://${host}:3030`; + expect(checkLocalOrigin(origin)).toEqual({ ok: true, origin }); + } + }); + + it("refuses a remote origin, so a seed script can't send an API key off-box", () => { + expect(checkLocalOrigin("https://cloud.trigger.dev")).toEqual({ + ok: false, + reason: "non_local", + hostname: "cloud.trigger.dev", + }); + expect(checkLocalOrigin("http://10.0.0.7:3030")).toEqual({ + ok: false, + reason: "non_local", + hostname: "10.0.0.7", + }); + }); + + // "localhost.attacker.example" and "notlocalhost" both end or start with a local name. + it("matches the whole hostname, never a prefix or suffix of one", () => { + expect(checkLocalOrigin("http://localhost.attacker.example").ok).toBe(false); + expect(checkLocalOrigin("http://notlocalhost:3030").ok).toBe(false); + expect(isLocalHost("127.0.0.1.attacker.example")).toBe(false); + }); + + it("refuses what it cannot parse rather than passing it through", () => { + expect(checkLocalOrigin("localhost:3030").ok).toBe(false); + expect(checkLocalOrigin("").ok).toBe(false); + }); +}); diff --git a/apps/webapp/app/utils/localHostGuard.ts b/apps/webapp/app/utils/localHostGuard.ts new file mode 100644 index 00000000000..5a81891a77b --- /dev/null +++ b/apps/webapp/app/utils/localHostGuard.ts @@ -0,0 +1,30 @@ +/** + * The one definition of "local" the dev-only seed scripts stage against. They carry API keys + * and destructive writes, so every host they touch — Redis, ClickHouse, the webapp itself — + * is checked here rather than each deciding for itself. + */ + +export const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]); + +/** `URL.hostname` brackets IPv6, so `::1` arrives as `[::1]`. */ +export function isLocalHost(hostname: string): boolean { + return LOCAL_HOSTS.has(hostname.replace(/^\[(.*)\]$/, "$1")); +} + +export type LocalOriginCheck = + | { ok: true; origin: string } + | { ok: false; reason: "unparseable" | "non_local"; hostname?: string }; + +/** Never returns the URL in the failure: an origin can carry credentials. */ +export function checkLocalOrigin(origin: string): LocalOriginCheck { + let parsed: URL; + try { + parsed = new URL(origin); + } catch { + return { ok: false, reason: "unparseable" }; + } + if (!isLocalHost(parsed.hostname)) { + return { ok: false, reason: "non_local", hostname: parsed.hostname }; + } + return { ok: true, origin }; +} diff --git a/apps/webapp/app/v3/alertsWorker.server.ts b/apps/webapp/app/v3/alertsWorker.server.ts index 88637d1c361..a58426821e6 100644 --- a/apps/webapp/app/v3/alertsWorker.server.ts +++ b/apps/webapp/app/v3/alertsWorker.server.ts @@ -5,11 +5,37 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; import { DeliverAlertService } from "./services/alerts/deliverAlert.server"; +import { + DeliverDashboardAgentWatchAlertService, + DeliverDashboardAgentWatchChannelAlertService, +} from "./services/alerts/deliverDashboardAgentWatchAlert.server"; import { DeliverErrorGroupAlertService } from "./services/alerts/deliverErrorGroupAlert.server"; import { ErrorAlertEvaluator } from "./services/alerts/errorAlertEvaluator.server"; +import { + watchObservedOutcomeSchema, + watchResolutionSchema, +} from "@internal/dashboard-agent-contracts"; import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server"; import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; +/** The fired watch, as the fan-out and each per-channel delivery carry it. */ +const DashboardAgentWatchAlertPayload = z.object({ + watchId: z.string(), + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + userId: z.string(), + identity: z.string(), + kind: z.string(), + note: z.string(), + firedAt: z.string(), + facts: z.record(z.unknown()), + // Optional so a job enqueued before this deploy still validates and delivers. + resolution: watchResolutionSchema.optional().catch(undefined), + // `.catch` so an unrecognized observation shape degrades instead of dropping the alert. + observed: watchObservedOutcomeSchema.optional().catch(undefined), +}); + function initializeWorker() { const redisOptions = { keyPrefix: "alerts:worker:", @@ -93,6 +119,24 @@ function initializeWorker() { }, logErrors: true, }, + // The fan-out: resolves the channels and enqueues one delivery job each. + "v3.deliverDashboardAgentWatchAlert": { + schema: DashboardAgentWatchAlertPayload, + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, + // One channel's delivery, so a retry only re-sends the channel that failed. + "v3.deliverDashboardAgentWatchAlertChannel": { + schema: DashboardAgentWatchAlertPayload.extend({ channelId: z.string() }), + visibilityTimeoutMs: 60_000, + retry: { + maxAttempts: 3, + }, + logErrors: true, + }, }, concurrency: { workers: env.ALERTS_WORKER_CONCURRENCY_WORKERS, @@ -126,6 +170,14 @@ function initializeWorker() { const service = new DeliverErrorGroupAlertService(); await service.call(payload); }, + "v3.deliverDashboardAgentWatchAlert": async ({ payload }) => { + const service = new DeliverDashboardAgentWatchAlertService(); + await service.call(payload); + }, + "v3.deliverDashboardAgentWatchAlertChannel": async ({ payload }) => { + const service = new DeliverDashboardAgentWatchChannelAlertService(); + await service.call(payload); + }, }, }); diff --git a/apps/webapp/app/v3/commonWorker.server.ts b/apps/webapp/app/v3/commonWorker.server.ts index 65b59209ec5..f104cb4f15d 100644 --- a/apps/webapp/app/v3/commonWorker.server.ts +++ b/apps/webapp/app/v3/commonWorker.server.ts @@ -13,6 +13,10 @@ import { } from "~/services/attio.server"; import { sweepDashboardAgentTurnEvals } from "~/services/dashboardAgentEvalRetention.server"; import { sweepDashboardAgentInvestigations } from "~/services/dashboardAgentInvestigationSweep.server"; +import { + rearmDashboardAgentWatchBatches, + sweepDashboardAgentWatches, +} from "~/services/dashboardAgentWatchSweep.server"; import { logger } from "~/services/logger.server"; import { MembershipDevEnvironmentsSchema, @@ -158,6 +162,16 @@ function initializeWorker() { maxAttempts: 1, }, }, + // The watch backstops: expiry, wake redelivery, retention and dead batch chains. + "dashboardAgent.watchMaintenance": { + schema: CronSchema, + visibilityTimeoutMs: 60_000 * 5, + cron: "*/5 * * * *", + jitterInMs: 30_000, + retry: { + maxAttempts: 1, + }, + }, }, concurrency: { workers: env.COMMON_WORKER_CONCURRENCY_WORKERS, @@ -239,6 +253,30 @@ function initializeWorker() { failure ??= error; } + if (failure) throw failure; + }, + "dashboardAgent.watchMaintenance": async () => { + // Each backstop runs independently; the first failure is rethrown at the end. + let failure: unknown; + + try { + const watches = await sweepDashboardAgentWatches(); + if (watches.overdue > 0 || watches.undelivered > 0 || watches.purged > 0) { + logger.debug("Dashboard agent watch sweep", watches); + } + } catch (error) { + failure ??= error; + } + + try { + const batches = await rearmDashboardAgentWatchBatches(); + if (batches.stale > 0) { + logger.debug("Dashboard agent watch batch re-arm", batches); + } + } catch (error) { + failure ??= error; + } + if (failure) throw failure; }, }, diff --git a/apps/webapp/app/v3/queryScope.ts b/apps/webapp/app/v3/queryScope.ts index 143fb9bbd9f..039e2c7cf9d 100644 --- a/apps/webapp/app/v3/queryScope.ts +++ b/apps/webapp/app/v3/queryScope.ts @@ -1,3 +1,4 @@ +import type { ApiAuthenticationResultSuccess } from "~/services/apiAuth.server"; import type { QueryScope } from "~/v3/querySchemas"; /** @@ -36,7 +37,14 @@ export function resolveQueryScope(args: { }; } -/** A public access token is environment-bound; every other bearer credential isn't. */ -export function queryScopeCeilingFor(authenticationType: string): QueryScopeCeiling { - return authenticationType === "PUBLIC_JWT" ? "environment" : "unbounded"; +/** + * A public credential is environment-bound; a secret key isn't. `PUBLIC` is the deprecated + * `pk_*` key — same threat model as a public access token, so it caps the same way. It cannot + * reach the query API today (the bearer resolver 401s `pk_*`), which is why capping it costs + * no caller anything and why the helper must not promise it is uncapped. + */ +export function queryScopeCeilingFor( + authenticationType: ApiAuthenticationResultSuccess["type"] +): QueryScopeCeiling { + return authenticationType === "PRIVATE" ? "unbounded" : "environment"; } diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index 20d7c02333a..3ac7f4afb34 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -391,7 +391,8 @@ export class DeliverAlertService extends BaseService { break; } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } @@ -747,7 +748,8 @@ export class DeliverAlertService extends BaseService { break; } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } @@ -1024,7 +1026,8 @@ export class DeliverAlertService extends BaseService { return; } } - case "ERROR_GROUP": { + case "ERROR_GROUP": + case "DASHBOARD_AGENT_WATCH": { // Payload-carried alert types create no ProjectAlert row, so never seen here. break; } diff --git a/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts new file mode 100644 index 00000000000..5c4e7a75a3d --- /dev/null +++ b/apps/webapp/app/v3/services/alerts/deliverDashboardAgentWatchAlert.server.ts @@ -0,0 +1,506 @@ +import { + type ChatPostMessageArguments, + ErrorCode, + type WebAPIPlatformError, + type WebAPIRateLimitedError, +} from "@slack/web-api"; +import type { WatchObservedOutcome, WatchResolution } from "@internal/dashboard-agent-contracts"; +import { type ProjectAlertChannel } from "@trigger.dev/database"; +import assertNever from "assert-never"; +import { subtle } from "crypto"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { + isIntegrationForService, + type OrganizationIntegrationForService, + OrgIntegrationRepository, +} from "~/models/orgIntegration.server"; +import { + ProjectAlertEmailProperties, + ProjectAlertSlackProperties, + ProjectAlertWebhookProperties, +} from "~/models/projectAlert.server"; +import { mintDashboardAgentAlertUnsubscribeToken } from "~/services/dashboardAgentAlertUnsubscribeToken.server"; +import { + canUseDashboardAgentAlerts, + DASHBOARD_AGENT_WATCH_ALERT_TYPE, +} from "~/services/dashboardAgentWatchAlerts.server"; +import { + presentResolvedWatch, + renderFactLines, + watchNoteLine, +} from "~/presenters/v3/dashboardAgent"; +import { sendAlertEmail } from "~/services/email.server"; +import { logger } from "~/services/logger.server"; +import { decryptSecret } from "~/services/secrets/secretStore.server"; +import { v3RunsPath } from "~/utils/pathBuilder"; +import { alertsWorker } from "~/v3/alertsWorker.server"; +import { safeWebhookFetch } from "./safeWebhookFetch.server"; + +/** + * Deliver a fired dashboard-agent watch to the project's alert channels. No `ProjectAlert` row: + * the job ids are the dedupe, and one job per channel keeps a failing webhook to itself. + */ +export type DashboardAgentWatchAlertPayload = { + watchId: string; + organizationId: string; + projectId: string; + environmentId: string; + userId: string; + identity: string; + kind: string; + note: string; + firedAt: string; + facts: Record; + /** Optional so a job from an older build still delivers, falling back to `condition_met`. */ + resolution?: WatchResolution; + observed?: WatchObservedOutcome; +}; + +/** One channel's delivery: the fan-out payload plus the channel it targets. */ +export type DashboardAgentWatchChannelAlertPayload = DashboardAgentWatchAlertPayload & { + channelId: string; +}; + +/** Bumped when the webhook body's shape changes. */ +const WEBHOOK_VERSION = "2026-08-02"; + +/** Wording comes from the shared presenter, so every surface says the same sentence. */ +function presentAlert(payload: DashboardAgentWatchAlertPayload) { + return presentResolvedWatch({ + kind: payload.kind, + identity: payload.identity, + // Only a met condition fans out today; the fallback keeps an older payload + // from silently presenting as something else. + resolution: payload.resolution ?? "condition_met", + observed: payload.observed ?? null, + }); +} + +class SkipRetryError extends Error {} + +type ResolvedContext = { + environmentName: string; + environmentSlug: string; + organizationSlug: string; + organizationTitle: string; + projectName: string; + projectSlug: string; + projectRef: string; + dashboardLink: string; +}; + +type ResolvedEnvironment = NonNullable>>; + +function findEnvironment(payload: DashboardAgentWatchAlertPayload) { + return $replica.runtimeEnvironment.findFirst({ + where: { id: payload.environmentId, projectId: payload.projectId }, + select: { + type: true, + slug: true, + branchName: true, + project: { + select: { + name: true, + slug: true, + externalRef: true, + organization: { select: { slug: true, title: true } }, + }, + }, + }, + }); +} + +function buildContext(environment: ResolvedEnvironment): ResolvedContext { + return { + environmentName: environment.branchName ?? environment.slug, + environmentSlug: environment.slug, + organizationSlug: environment.project.organization.slug, + organizationTitle: environment.project.organization.title, + projectName: environment.project.name, + projectSlug: environment.project.slug, + projectRef: environment.project.externalRef, + dashboardLink: `${env.APP_ORIGIN}${v3RunsPath( + { slug: environment.project.organization.slug }, + { slug: environment.project.slug }, + { slug: environment.slug } + )}`, + }; +} + +/** The fan-out: gate the watch, then enqueue one delivery job per channel. */ +export class DeliverDashboardAgentWatchAlertService { + async call(payload: DashboardAgentWatchAlertPayload): Promise { + const environment = await findEnvironment(payload); + + if (!environment) { + logger.warn("[DeliverDashboardAgentWatchAlert] Environment not found", { + watchId: payload.watchId, + }); + return; + } + + // Gate at delivery time, not only at subscribe time, so a plan change or a revoked + // feature flag stops the alerts without anyone cleaning up channels. + const gate = await canUseDashboardAgentAlerts({ + userId: payload.userId, + organizationId: payload.organizationId, + organizationSlug: environment.project.organization.slug, + }); + if (!gate.allowed) { + logger.info("[DeliverDashboardAgentWatchAlert] Not allowed for this organization", { + watchId: payload.watchId, + reason: gate.reason, + }); + return; + } + + const channels = await $replica.projectAlertChannel.findMany({ + where: { + projectId: payload.projectId, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + environmentTypes: { has: environment.type }, + }, + select: { id: true }, + }); + + for (const channel of channels) { + await alertsWorker.enqueue({ + // Stable per channel, so a fan-out retry re-enqueues the same job ids + // rather than a second alert per channel. + id: `watch-alert:${payload.watchId}:channel:${channel.id}`, + job: "v3.deliverDashboardAgentWatchAlertChannel", + payload: { ...payload, channelId: channel.id }, + }); + } + } +} + +/** One channel's delivery. A retry here can only re-send this channel. */ +export class DeliverDashboardAgentWatchChannelAlertService { + async call(payload: DashboardAgentWatchChannelAlertPayload): Promise { + // Re-read the channel rather than trusting the fan-out's snapshot: an unsubscribe + // between fan-out and delivery should stop the alert. The primary, since the + // unsubscribe writes there and replica lag would send the mail anyway. + const channel = await prisma.projectAlertChannel.findFirst({ + where: { + id: payload.channelId, + projectId: payload.projectId, + enabled: true, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + }); + + if (!channel) { + logger.info("[DeliverDashboardAgentWatchAlert] Channel gone or unsubscribed", { + watchId: payload.watchId, + channelId: payload.channelId, + }); + return; + } + + const environment = await findEnvironment(payload); + + if (!environment) { + logger.warn("[DeliverDashboardAgentWatchAlert] Environment not found", { + watchId: payload.watchId, + }); + return; + } + + const context = buildContext(environment); + + try { + switch (channel.type) { + case "EMAIL": + await this.#sendEmail(channel, payload, context); + break; + case "SLACK": + await this.#sendSlack(channel, payload, context); + break; + case "WEBHOOK": + await this.#sendWebhook(channel, payload, context); + break; + default: + assertNever(channel.type); + } + } catch (error) { + if (error instanceof SkipRetryError) { + logger.warn("[DeliverDashboardAgentWatchAlert] Skipping retry", { + watchId: payload.watchId, + channelId: channel.id, + reason: error.message, + }); + return; + } + throw error; + } + } + + async #sendEmail( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const emailProperties = ProjectAlertEmailProperties.safeParse(channel.properties); + if (!emailProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse email properties", { + issues: emailProperties.error.issues, + }); + return; + } + + const token = await mintDashboardAgentAlertUnsubscribeToken({ + channelId: channel.id, + alertType: DASHBOARD_AGENT_WATCH_ALERT_TYPE, + }); + + const presentation = presentAlert(payload); + + await sendAlertEmail({ + email: "alert-dashboard-agent-watch", + to: emailProperties.data.email, + identity: payload.identity, + kind: payload.kind, + headline: presentation.headline, + tone: presentation.tone, + note: payload.note, + // The sentence that quotes the note, rendered by the shared presenter so the + // email and the Slack message say it the same way. + noteLine: watchNoteLine(payload.note) ?? undefined, + firedAt: payload.firedAt, + facts: factList(payload.facts), + dashboardLink: context.dashboardLink, + unsubscribeLink: `${env.APP_ORIGIN}/resources/dashboard-agent/alerts/${channel.id}/unsubscribe?token=${encodeURIComponent(token)}`, + organization: context.organizationTitle, + project: context.projectName, + environment: context.environmentName, + }); + } + + async #sendSlack( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const slackProperties = ProjectAlertSlackProperties.safeParse(channel.properties); + if (!slackProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse slack properties", { + issues: slackProperties.error.issues, + }); + return; + } + + const integration = slackProperties.data.integrationId + ? await prisma.organizationIntegration.findFirst({ + where: { + id: slackProperties.data.integrationId, + organizationId: payload.organizationId, + }, + include: { tokenReference: true }, + }) + : await prisma.organizationIntegration.findFirst({ + where: { service: "SLACK", organizationId: payload.organizationId }, + orderBy: { createdAt: "desc" }, + include: { tokenReference: true }, + }); + + if (!integration || !isIntegrationForService(integration, "SLACK")) { + logger.error("[DeliverDashboardAgentWatchAlert] Slack integration not found"); + return; + } + + await this.#postSlackMessage(integration, { + channel: slackProperties.data.channelId, + ...this.#buildSlackMessage(payload, context), + } as ChatPostMessageArguments); + } + + async #sendWebhook( + channel: ProjectAlertChannel, + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): Promise { + const webhookProperties = ProjectAlertWebhookProperties.safeParse(channel.properties); + if (!webhookProperties.success) { + logger.error("[DeliverDashboardAgentWatchAlert] Failed to parse webhook properties", { + issues: webhookProperties.error.issues, + }); + return; + } + + const rawPayload = JSON.stringify({ + // Stable across attempts, so a receiver can dedupe a redelivery. Unlike the + // error-group webhook, which mints a nanoid per attempt. + id: `watch:${payload.watchId}:channel:${payload.channelId}`, + created: new Date(payload.firedAt), + webhookVersion: WEBHOOK_VERSION, + type: "alert.dashboard_agent_watch", + object: { + watch: { + id: payload.watchId, + identity: payload.identity, + kind: payload.kind, + note: payload.note, + // `outcome` keeps its two-value encoding for receivers that already parse it; + // `resolution` and `observed` carry the detail. + outcome: "fired", + resolution: payload.resolution ?? "condition_met", + observed: payload.observed ?? null, + firedAt: payload.firedAt, + facts: payload.facts, + }, + environment: { id: payload.environmentId, name: context.environmentName }, + organization: { + id: payload.organizationId, + slug: context.organizationSlug, + name: context.organizationTitle, + }, + project: { + id: payload.projectId, + ref: context.projectRef, + slug: context.projectSlug, + name: context.projectName, + }, + dashboardUrl: context.dashboardLink, + }, + }); + + const secret = await decryptSecret(env.ENCRYPTION_KEY, webhookProperties.data.secret); + const key = await subtle.importKey( + "raw", + Buffer.from(secret, "utf-8"), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + const signature = await subtle.sign("HMAC", key, Buffer.from(rawPayload, "utf-8")); + + // Deliver via the SSRF-safe wrapper (see safeWebhookFetch.server.ts). + const response = await safeWebhookFetch(webhookProperties.data.url, { + method: "POST", + headers: { + "content-type": "application/json", + "x-trigger-signature-hmacsha256": Buffer.from(signature).toString("hex"), + }, + body: rawPayload, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + logger.info("[DeliverDashboardAgentWatchAlert] Failed to send webhook", { + status: response.status, + url: webhookProperties.data.url, + }); + throw new Error(`Failed to send watch alert webhook to ${webhookProperties.data.url}`); + } + } + + async #postSlackMessage( + integration: OrganizationIntegrationForService<"SLACK">, + message: ChatPostMessageArguments + ) { + const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration( + integration, + { forceBotToken: true } + ); + + try { + return await client.chat.postMessage({ + ...message, + unfurl_links: false, + unfurl_media: false, + }); + } catch (error) { + if (isWebAPIRateLimitedError(error)) { + throw new Error("Slack rate limited"); + } + if (isWebAPIPlatformError(error)) { + const code = (error as WebAPIPlatformError).data.error; + if (code === "invalid_blocks" || code === "account_inactive") { + throw new SkipRetryError(`Slack: ${code}`); + } + throw new Error("Slack platform error"); + } + throw error; + } + } + + #buildSlackMessage( + payload: DashboardAgentWatchChannelAlertPayload, + context: ResolvedContext + ): { text: string; blocks: object[] } { + const facts = factList(payload.facts); + const { headline } = presentAlert(payload); + const noteLine = watchNoteLine(payload.note); + + return { + // The notification text is the plain-text rendering, so what a phone shows + // matches what the blocks below say. + text: [`${headline} [${context.environmentName}]`, noteLine, ...renderFactLines(facts)] + .filter(Boolean) + .join("\n"), + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: [`*${headline}* [${context.environmentName}]`, noteLine] + .filter(Boolean) + .join("\n"), + }, + }, + ...(facts.length > 0 + ? [ + { + type: "section", + fields: facts.slice(0, 10).map((fact) => ({ + type: "mrkdwn", + text: `*${fact.label}:*\n${fact.value}`, + })), + }, + ] + : []), + { + type: "actions", + elements: [ + { + type: "button", + text: { type: "plain_text", text: "Open dashboard" }, + url: context.dashboardLink, + style: "primary", + }, + ], + }, + ], + }; + } +} + +/** + * The check's facts, flattened for display. Generic because the bag is per-watch-kind and + * open-ended, and capped so a big bag can't blow up an email or a Slack block. + */ +function factList(facts: Record): Array<{ label: string; value: string }> { + return Object.entries(facts) + .filter(([, value]) => value !== null && value !== undefined && value !== "") + .slice(0, 12) + .map(([key, value]) => ({ + label: humanizeFactKey(key), + value: typeof value === "object" ? JSON.stringify(value).slice(0, 200) : String(value), + })); +} + +function humanizeFactKey(key: string): string { + const spaced = key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[_-]+/g, " "); + return spaced.charAt(0).toUpperCase() + spaced.slice(1); +} + +function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError { + return (error as WebAPIPlatformError).code === ErrorCode.PlatformError; +} + +function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError { + return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError; +} diff --git a/apps/webapp/test/__snapshots__/dashboardAgentWatchWording.test.ts.snap b/apps/webapp/test/__snapshots__/dashboardAgentWatchWording.test.ts.snap new file mode 100644 index 00000000000..5d48e01a760 --- /dev/null +++ b/apps/webapp/test/__snapshots__/dashboardAgentWatchWording.test.ts.snap @@ -0,0 +1,291 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`the watch presenter > gives every kind and resolution one headline 1`] = ` +[ + { + "category": "positive", + "headline": "Run run_abc123 started", + "kind": "run_start", + "resolution": "condition_met", + "tone": "success", + }, + { + "category": "attention", + "headline": "Run run_abc123 hasn't started yet", + "kind": "run_start", + "resolution": "window_completed", + "tone": "warning", + }, + { + "category": "neutral", + "headline": "Run run_abc123 will never start", + "kind": "run_start", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "positive", + "headline": "Run run_abc123 finished", + "kind": "run_finished", + "resolution": "condition_met", + "tone": "success", + }, + { + "category": "attention", + "headline": "Run run_abc123 is still running", + "kind": "run_finished", + "resolution": "window_completed", + "tone": "warning", + }, + { + "category": "neutral", + "headline": "Run run_abc123 is no longer there", + "kind": "run_finished", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "attention", + "headline": "Run run_abc123 failed", + "kind": "run_failed", + "resolution": "condition_met", + "tone": "error", + }, + { + "category": "positive", + "headline": "Run run_abc123 hasn't failed", + "kind": "run_failed", + "resolution": "window_completed", + "tone": "success", + }, + { + "category": "neutral", + "headline": "Run run_abc123 is no longer there", + "kind": "run_failed", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "positive", + "headline": "email-sends queue drained", + "kind": "backlog_drain", + "resolution": "condition_met", + "tone": "success", + }, + { + "category": "attention", + "headline": "email-sends queue still hasn't drained", + "kind": "backlog_drain", + "resolution": "window_completed", + "tone": "warning", + }, + { + "category": "neutral", + "headline": "email-sends queue no longer exists", + "kind": "backlog_drain", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "attention", + "headline": "email-sends queue is above the threshold", + "kind": "queue_depth_above", + "resolution": "condition_met", + "tone": "warning", + }, + { + "category": "positive", + "headline": "email-sends queue stayed below the threshold", + "kind": "queue_depth_above", + "resolution": "window_completed", + "tone": "success", + }, + { + "category": "neutral", + "headline": "email-sends queue no longer exists", + "kind": "queue_depth_above", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "positive", + "headline": "email-sends queue is back below the threshold", + "kind": "queue_depth_below", + "resolution": "condition_met", + "tone": "success", + }, + { + "category": "attention", + "headline": "email-sends queue is still above the threshold", + "kind": "queue_depth_below", + "resolution": "window_completed", + "tone": "warning", + }, + { + "category": "neutral", + "headline": "email-sends queue no longer exists", + "kind": "queue_depth_below", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "attention", + "headline": "email-sends queue isn't moving", + "kind": "queue_stalled", + "resolution": "condition_met", + "tone": "warning", + }, + { + "category": "positive", + "headline": "email-sends queue kept moving", + "kind": "queue_stalled", + "resolution": "window_completed", + "tone": "success", + }, + { + "category": "neutral", + "headline": "email-sends queue no longer exists", + "kind": "queue_stalled", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "attention", + "headline": "runs in email-sends are waiting too long", + "kind": "queue_oldest_age", + "resolution": "condition_met", + "tone": "warning", + }, + { + "category": "positive", + "headline": "email-sends queue stayed within its wait limit", + "kind": "queue_oldest_age", + "resolution": "window_completed", + "tone": "success", + }, + { + "category": "neutral", + "headline": "email-sends queue no longer exists", + "kind": "queue_oldest_age", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "attention", + "headline": "Error a1b2c3d4e5f6 happened again", + "kind": "error_recurrence", + "resolution": "condition_met", + "tone": "error", + }, + { + "category": "positive", + "headline": "Error a1b2c3d4e5f6 stayed quiet", + "kind": "error_recurrence", + "resolution": "window_completed", + "tone": "success", + }, + { + "category": "neutral", + "headline": "Error a1b2c3d4e5f6 stayed quiet", + "kind": "error_recurrence", + "resolution": "condition_impossible", + "tone": "neutral", + }, + { + "category": "positive", + "headline": "Health recovered", + "kind": "health_recovery", + "resolution": "condition_met", + "tone": "success", + }, + { + "category": "attention", + "headline": "Health hasn't recovered", + "kind": "health_recovery", + "resolution": "window_completed", + "tone": "warning", + }, + { + "category": "neutral", + "headline": "Health couldn't be read", + "kind": "health_recovery", + "resolution": "condition_impossible", + "tone": "neutral", + }, +] +`; + +exports[`the watch presenter > says each condition the same way on every surface 1`] = ` +[ + { + "confirmation": "Watching run run_abc123 until it starts.", + "kind": "run_start", + "label": "Until it starts", + "note": "tell me when run run_abc123 starts", + "tooltip": "Get notified when this run starts", + }, + { + "confirmation": "Watching run run_abc123 until it finishes.", + "kind": "run_finished", + "label": "Until it finishes", + "note": "tell me when run run_abc123 finishes", + "tooltip": "Get notified when this run finishes", + }, + { + "confirmation": "Watching run run_abc123 in case it fails.", + "kind": "run_failed", + "label": "If it fails", + "note": "tell me if run run_abc123 fails", + "tooltip": "Get notified if this run fails", + }, + { + "confirmation": "Watching email-sends until the queue drains.", + "kind": "backlog_drain", + "label": "Until the queue drains", + "note": "tell me when the email-sends queue drains", + "tooltip": "Get notified when this queue drains", + }, + { + "confirmation": "Watching email-sends in case the queue goes above 500.", + "kind": "queue_depth_above", + "label": "If the queue goes above 500", + "note": "tell me if the email-sends queue goes above 500", + "tooltip": "Get notified if this queue goes above 500", + }, + { + "confirmation": "Watching email-sends until it is back below 500.", + "kind": "queue_depth_below", + "label": "Until the queue is back below 500", + "note": "tell me when the email-sends queue is back below 500", + "tooltip": "Get notified when this queue is back below 500", + }, + { + "confirmation": "Watching email-sends in case it stops moving.", + "kind": "queue_stalled", + "label": "If the queue stops moving", + "note": "tell me if the email-sends queue stops moving", + "tooltip": "Get notified if this queue stops moving", + }, + { + "confirmation": "Watching email-sends in case runs wait longer than 1h 30m.", + "kind": "queue_oldest_age", + "label": "If runs wait longer than 1h 30m", + "note": "tell me if runs in email-sends wait longer than 1h 30m", + "tooltip": "Get notified if runs wait longer than 1h 30m", + }, + { + "confirmation": "Watching error a1b2c3d4e5f6 in case it happens again.", + "kind": "error_recurrence", + "label": "If it happens again", + "note": "ping me if error a1b2c3d4e5f6 happens again", + "tooltip": "Get notified if this error happens again", + }, + { + "confirmation": "Watching health until it recovers.", + "kind": "health_recovery", + "label": "Until it recovers", + "note": "tell me when health is back to normal", + "tooltip": "Get notified when health recovers", + }, +] +`; diff --git a/apps/webapp/test/dashboardAgentBodyCap.test.ts b/apps/webapp/test/dashboardAgentBodyCap.test.ts index 599b15df79b..7c83277ad70 100644 --- a/apps/webapp/test/dashboardAgentBodyCap.test.ts +++ b/apps/webapp/test/dashboardAgentBodyCap.test.ts @@ -118,17 +118,17 @@ describe("the dashboard agent's ingress cap", () => { const { url } = await listen(); const response = await postChunked( - `${url}/api/v1/Dashboard-Agent/eval-policy`, + `${url}/api/v1/Dashboard-Agent/watches/batch-check`, DASHBOARD_AGENT_MAX_INGRESS_BYTES * 4 ); expect(response.status).toBe(413); }); - it("caps a DELETE, which can read a body too", async () => { + it("caps a DELETE, which reads a body on the alerts route", async () => { const { url } = await listen(); - const response = await fetch(`${url}/api/v1/dashboard-agent/eval-policy`, { + const response = await fetch(`${url}/api/v1/dashboard-agent/alerts/ch_1`, { method: "DELETE", body: "x".repeat(DASHBOARD_AGENT_MAX_INGRESS_BYTES + 1024), }); diff --git a/apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts b/apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts index f395e981e8a..a50fed55374 100644 --- a/apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts +++ b/apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts @@ -41,22 +41,22 @@ vi.mock("~/services/dashboardAgent.server", () => ({ mintDashboardAgentUserActorToken: mocks.mintUserActorToken, resolveDashboardAgentRepoSnapshot: async () => null, startDashboardAgentSession: mocks.startSession, + dashboardAgentWakeFeedCounter: { inc: vi.fn() }, })); vi.mock("~/services/dashboardAgentHeadStart.server", () => ({ startDashboardAgentHeadStart: mocks.headStart, })); +// The chat route reaches the ClickHouse factory through the watch services, and the factory +// builds its client at import time from an env var no test sets. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: async () => ({}) }, +})); vi.mock("~/services/dashboardAgentDb.server", () => ({ dashboardAgentDb: {} })); vi.mock("~/services/resolveTriggerUri.server", () => ({ resolveTriggerUri: () => null })); -vi.mock("@internal/dashboard-agent-db", () => ({ - chatExists: vi.fn(), - countUserMessages: vi.fn(), +// Spread the real module so this doesn't have to track every query the route imports. +vi.mock("@internal/dashboard-agent-db", async (importOriginal) => ({ + ...((await importOriginal()) as Record), createChat: mocks.createChat, - getChatMessages: vi.fn(), - getSession: vi.fn(), - listChatIdsWithOpenInvestigations: vi.fn(), - listChats: vi.fn(), - renameChat: vi.fn(), - setChatPinned: vi.fn(), softDeleteChat: mocks.softDeleteChat, })); vi.mock("~/services/logger.server", () => ({ logger: mocks.logger })); @@ -93,7 +93,7 @@ describe("dashboard agent chat creation — nothing fallible after the row exist mocks.mintUserActorToken.mockReset().mockResolvedValue("tr_uat_real"); mocks.mintPublicToken.mockReset().mockResolvedValue("pat_public"); mocks.startSession.mockReset().mockResolvedValue(undefined); - mocks.softDeleteChat.mockReset().mockResolvedValue({ deleted: true }); + mocks.softDeleteChat.mockReset().mockResolvedValue({ deleted: true, cancelledWatches: [] }); mocks.env.ANTHROPIC_API_KEY = "sk-test"; }); @@ -144,7 +144,7 @@ describe("dashboard agent chat creation — a start that fails part way", () => mocks.mintUserActorToken.mockReset().mockResolvedValue("tr_uat_real"); mocks.mintPublicToken.mockReset().mockResolvedValue("pat_public"); mocks.startSession.mockReset().mockResolvedValue(undefined); - mocks.softDeleteChat.mockReset().mockResolvedValue({ deleted: true }); + mocks.softDeleteChat.mockReset().mockResolvedValue({ deleted: true, cancelledWatches: [] }); mocks.env.ANTHROPIC_API_KEY = "sk-test"; mocks.logger.error.mockReset(); }); diff --git a/apps/webapp/test/dashboardAgentLastReadBackfill.test.ts b/apps/webapp/test/dashboardAgentLastReadBackfill.test.ts new file mode 100644 index 00000000000..97cee33d122 --- /dev/null +++ b/apps/webapp/test/dashboardAgentLastReadBackfill.test.ts @@ -0,0 +1,137 @@ +import { + countChatsWithUnreadWork, + createDashboardAgentDb, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect } from "vitest"; + +/** + * `chats.last_read_at` is nullable and every reader treats NULL as unread, so without a + * backfill the first load after rollout reports every pre-existing chat unread. Migration + * 0002 backfills it; this replays the migrations against a real Postgres to prove it does. + */ + +const DRIZZLE = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + +const MIGRATIONS = [ + "0000_magenta_lilandra.sql", + "0001_slimy_living_tribunal.sql", + "0002_watches_and_chat_messages.sql", +]; + +/** The statement under test, located by shape so removing it fails rather than silently passing. */ +const BACKFILL = /^update\s+"trigger_dashboard_agent"\."chats"\s+set\s+"last_read_at"/i; + +function statementsOf(file: string): string[] { + return readFileSync(path.join(DRIZZLE, file), "utf8") + .split("--> statement-breakpoint") + .map((statement) => + statement + .split("\n") + .filter((line) => !line.trimStart().startsWith("--")) + .join("\n") + .trim() + ) + .filter((statement) => statement.length > 0); +} + +async function run(prisma: PrismaClient, statements: string[]) { + for (const statement of statements) await prisma.$executeRawUnsafe(statement); +} + +const SCOPE = { organizationId: "org_1", userId: "user_1" }; + +const CREATED_AT = new Date("2026-01-01T00:00:00.000Z"); +const LAST_MESSAGE_AT = new Date("2026-02-01T00:00:00.000Z"); +/** After the last message, so the chat is genuinely read and dropping the `where` moves it back. */ +const ALREADY_READ_AT = new Date("2026-03-01T00:00:00.000Z"); + +/** Chats as they exist before 0002 runs — no `last_read_at` column yet. */ +async function seedPreExistingChats(prisma: PrismaClient) { + for (const [id, lastMessageAt] of [ + ["chat_with_messages", LAST_MESSAGE_AT], + ["chat_never_messaged", null], + ["chat_already_read", LAST_MESSAGE_AT], + ] as const) { + await prisma.$executeRawUnsafe( + `insert into "trigger_dashboard_agent"."chats" + ("id", "organization_id", "user_id", "created_at", "updated_at", "last_message_at") + values ($1, $2, $3, $4, $4, $5)`, + id, + SCOPE.organizationId, + SCOPE.userId, + CREATED_AT, + lastMessageAt + ); + } +} + +async function readLastReadAt(prisma: PrismaClient): Promise> { + const rows = await prisma.$queryRawUnsafe>( + `select "id", "last_read_at" from "trigger_dashboard_agent"."chats" order by "id"` + ); + return Object.fromEntries(rows.map((row) => [row.id, row.last_read_at])); +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("the last_read_at backfill in migration 0002", () => { + postgresTest( + "starts pre-existing chats read, and does not overwrite a chat already read", + async ({ prisma, postgresContainer }) => { + await run(prisma, statementsOf(MIGRATIONS[0]!)); + await run(prisma, statementsOf(MIGRATIONS[1]!)); + + const statements = statementsOf(MIGRATIONS[2]!); + const backfillAt = statements.findIndex((statement) => BACKFILL.test(statement)); + expect(backfillAt, "0002 contains no last_read_at backfill statement").toBeGreaterThan(-1); + + // Everything up to the backfill: the column exists, the chats predate it. + await run(prisma, statements.slice(0, backfillAt)); + await seedPreExistingChats(prisma); + // A deploy that rolled the column out ahead of the backfill could already have a value. + await prisma.$executeRawUnsafe( + `update "trigger_dashboard_agent"."chats" set "last_read_at" = $1 where "id" = 'chat_already_read'`, + ALREADY_READ_AT + ); + expect(await readLastReadAt(prisma)).toEqual({ + chat_with_messages: null, + chat_never_messaged: null, + chat_already_read: ALREADY_READ_AT, + }); + + await run(prisma, statements.slice(backfillAt)); + + expect(await readLastReadAt(prisma)).toEqual({ + // Read as of its last message: a later message still lights the dot. + chat_with_messages: LAST_MESSAGE_AT, + // Nothing was ever said in it, so it is read as of the moment it existed. + chat_never_messaged: CREATED_AT, + // Already read; the backfill must not move it back or forward. + chat_already_read: ALREADY_READ_AT, + }); + + agentDbClient = createDashboardAgentDb(postgresContainer.getConnectionUri(), { max: 2 }); + const agentDb: DashboardAgentDb = agentDbClient.db; + + // The user-visible claim: the launcher dot is dark on the first load after rollout. + expect(await countChatsWithUnreadWork(agentDb, SCOPE)).toBe(0); + + // And a positive control, so a backfill that marked everything read forever would fail. + await prisma.$executeRawUnsafe( + `update "trigger_dashboard_agent"."chats" set "last_message_at" = now() where "id" = 'chat_never_messaged'` + ); + expect(await countChatsWithUnreadWork(agentDb, SCOPE)).toBe(1); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentToolScopes.test.ts b/apps/webapp/test/dashboardAgentToolScopes.test.ts new file mode 100644 index 00000000000..6eabb2d8eb7 --- /dev/null +++ b/apps/webapp/test/dashboardAgentToolScopes.test.ts @@ -0,0 +1,99 @@ +import { DASHBOARD_AGENT_ENV_JWT_SCOPES } from "@internal/dashboard-agent/tool-schemas"; +import { buildJwtAbility } from "@trigger.dev/rbac"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("~/env.server", () => ({ env: { SESSION_SECRET: "secret" } })); +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +import { DASHBOARD_AGENT_UAT_CAP } from "~/services/dashboardAgent.server"; + +/** + * The agent reaches the API two ways, and they are authorized by different lists. + * + * Most tools spend an environment JWT minted with a fixed set of scopes; the delegated + * token's own cap only ceilings that exchange. A few call the API as the delegated token + * itself. A route whose resource is in neither list answers 403 — which reaches the model + * as missing data, not as a permission problem, and it then tells the user the thing does + * not exist. That is how a queue holding 4800 runs was reported as never created. + */ +type Read = { tool: string; path: string; resource: { type: string; id?: string } }; + +const VIA_ENV_JWT: Read[] = [ + { tool: "list_runs", path: "/api/v1/runs", resource: { type: "runs" } }, + { tool: "get_run_trace", path: "/api/v1/runs/:id/trace", resource: { type: "runs" } }, + { tool: "list_errors", path: "/api/v1/errors", resource: { type: "errors" } }, + { tool: "get_error", path: "/api/v1/errors/:id", resource: { type: "errors" } }, + { tool: "list_deploys", path: "/api/v1/deployments", resource: { type: "deployments" } }, + { tool: "get_deploy", path: "/api/v1/deployments/current", resource: { type: "deployments" } }, + { + tool: "get_query_schema", + path: "/api/v1/query/schema", + resource: { type: "query", id: "schema" }, + }, + { tool: "run_query", path: "/api/v1/query", resource: { type: "query", id: "runs" } }, + { + tool: "get_queue (metrics)", + path: "/api/v1/queues/:name/metrics", + resource: { type: "query", id: "queue_metrics" }, + }, + { tool: "get_queue (live row)", path: "/api/v1/queues/:name", resource: { type: "queues" } }, + { + tool: "get_report", + path: "/api/v1/reports/:key", + resource: { type: "query", id: "env_metrics" }, + }, +]; + +const VIA_DELEGATED_TOKEN: Read[] = [ + { + tool: "list_environments", + path: "/api/v1/projects/:ref/environments", + resource: { type: "environments" }, + }, + { + tool: "repo snapshot", + path: "/api/v1/projects/:ref/:env/repo/snapshot", + resource: { type: "apiKeys" }, + }, +]; + +describe("what the agent's environment JWT may read", () => { + const ability = buildJwtAbility([...DASHBOARD_AGENT_ENV_JWT_SCOPES]); + + it.each(VIA_ENV_JWT)("$tool reads $path", ({ resource }) => { + expect(ability.can("read", resource)).toBe(true); + }); +}); + +describe("what the agent's delegated token may read", () => { + const ability = buildJwtAbility(DASHBOARD_AGENT_UAT_CAP); + + it.each(VIA_DELEGATED_TOKEN)("$tool reads $path", ({ resource }) => { + expect(ability.can("read", resource)).toBe(true); + }); + + it("ceilings the exchange: every JWT scope is one the cap already allows", () => { + // The exchange clamps against this cap, so a scope missing here is silently dropped + // from the minted JWT rather than refused loudly. + for (const scope of DASHBOARD_AGENT_ENV_JWT_SCOPES) { + expect(DASHBOARD_AGENT_UAT_CAP, scope).toContain(scope); + } + }); + + it("carries read:queues on both sides, since read:query only buys the metrics", () => { + // A queue's own row — paused, depth, limit — is a `queues` read; its metrics are a + // `query` read. Drop the scope and the live lookup 403s, which the model reads as a + // queue that was never created. + expect(DASHBOARD_AGENT_ENV_JWT_SCOPES).toContain("read:queues"); + expect(DASHBOARD_AGENT_UAT_CAP).toContain("read:queues"); + expect(buildJwtAbility(["read:query"]).can("read", { type: "queues" })).toBe(false); + }); + + it("stays read-only on both sides", () => { + for (const scope of [...DASHBOARD_AGENT_UAT_CAP, ...DASHBOARD_AGENT_ENV_JWT_SCOPES]) { + expect(scope.startsWith("read:"), scope).toBe(true); + } + expect(ability.can("write", { type: "runs" })).toBe(false); + expect(ability.canSuper()).toBe(false); + }); +}); diff --git a/apps/webapp/test/dashboardAgentTranscriptStore.test.ts b/apps/webapp/test/dashboardAgentTranscriptStore.test.ts index e033bd538a1..525b889c065 100644 --- a/apps/webapp/test/dashboardAgentTranscriptStore.test.ts +++ b/apps/webapp/test/dashboardAgentTranscriptStore.test.ts @@ -1,5 +1,6 @@ import { appendChatMessageOnceByChatId, + countChatsWithUnreadWork, countUserMessages, createChat, createDashboardAgentDb, @@ -8,9 +9,11 @@ import { getInvestigation, investigationSettlementMessageId, persistMessages, + seedInvestigation, persistTurn, settleInvestigationAndCloseCard, upsertInvestigationRevision, + watchInvestigationId, type DashboardAgentDb, type DashboardAgentDbClient, } from "@internal/dashboard-agent-db"; @@ -72,6 +75,12 @@ function textMessage(id: string, text = id) { return { id, role: "assistant" as const, parts: [{ type: "text", text }] }; } +// Compile-time: the insert reads `role` off the body and throws without one, so a +// message that satisfies the signature must never be able to lack it. +const _roleIsRequired = (message: { id: string }) => + // @ts-expect-error a message with no role is not appendable + appendChatMessageOnceByChatId(agentDb, { chatId: "chat_x", message }); + function toolMessage(id: string, state: "input-available" | "output-available") { return { id, @@ -152,14 +161,14 @@ describe("invariant 1: a repeated message id creates no row and keeps its positi await persistMessages(agentDb, { chatId, messages: [textMessage("u1")] }); expect( - await appendChatMessageOnceByChatId(agentDb, { chatId, message: textMessage("ev:1") }) + await appendChatMessageOnceByChatId(agentDb, { chatId, message: textMessage("wake:w1") }) ).toBe(true); const before = await rows(prisma, chatId); // The same durable event, redelivered. expect( - await appendChatMessageOnceByChatId(agentDb, { chatId, message: textMessage("ev:1") }) + await appendChatMessageOnceByChatId(agentDb, { chatId, message: textMessage("wake:w1") }) ).toBe(false); expect(await rows(prisma, chatId)).toEqual(before); @@ -196,7 +205,7 @@ describe("invariant 2: concurrent different messages get distinct positions", () const chatId = "chat_concurrent"; await boot(prisma, postgresContainer.getConnectionUri(), chatId); - const ids = Array.from({ length: 8 }, (_, i) => `ev:${i}`); + const ids = Array.from({ length: 8 }, (_, i) => `wake:w${i}`); const results = await Promise.all( ids.map((id) => appendChatMessageOnceByChatId(agentDb, { chatId, message: textMessage(id) }) @@ -277,10 +286,10 @@ describe("invariant 3: an ordinary transcript write can never change a stored me await boot(prisma, postgresContainer.getConnectionUri(), chatId); await persistMessages(agentDb, { chatId, messages: [textMessage("u1")] }); - // A durable event, appended outside a turn. + // A durable event: the wake that actually fired. await appendChatMessageOnceByChatId(agentDb, { chatId, - message: textMessage("ev:fired", "send-order-receipt resolved."), + message: textMessage("wake:watch_1:fired", "The watch on send-order-receipt resolved."), }); const before = await rows(prisma, chatId); @@ -288,7 +297,7 @@ describe("invariant 3: an ordinary transcript write can never change a stored me // not a finalisation, so it must not be able to rewrite it. await persistMessages(agentDb, { chatId, - messages: [textMessage("u1"), textMessage("ev:fired", "something else entirely")], + messages: [textMessage("u1"), textMessage("wake:watch_1:fired", "something else entirely")], }); expect(await rows(prisma, chatId)).toEqual(before); @@ -573,10 +582,10 @@ describe("a write can no longer lose a message another process appended", () => const snapshot = [textMessage("u1"), textMessage("a1")]; await persistMessages(agentDb, { chatId, messages: snapshot }); - // Another process appends while the turn is running. + // Another process — a wake delivery — appends while the turn is running. await appendChatMessageOnceByChatId(agentDb, { chatId, - message: textMessage("ev:fired"), + message: textMessage("wake:watch_1:fired"), }); // The turn ends and writes its own snapshot plus what it produced. @@ -586,12 +595,12 @@ describe("a write can no longer lose a message another process appended", () => session: { publicAccessToken: "pat_store", lastEventId: "1", runId: "run_store" }, }); - // It is still there, and sits where it happened: after the turn's snapshot, + // The wake is still there, and sits where it happened: after the turn's snapshot, // before the reply the turn went on to produce. expect((await transcript(chatId)).map((message) => message.id)).toEqual([ "u1", "a1", - "ev:fired", + "wake:watch_1:fired", "a2", ]); }, @@ -659,6 +668,40 @@ describe("a write can no longer lose a message another process appended", () => ); }); +describe("countChatsWithUnreadWork", () => { + postgresTest( + "counts a chat whose transcript moved on after its owner last looked", + async ({ prisma, postgresContainer }) => { + const chatId = "chat_unread_work"; + await boot(prisma, postgresContainer.getConnectionUri(), chatId); + const scope = { organizationId: ORG_ID, userId: USER_ID }; + + // A chat nobody has written in is not unread. + expect(await countChatsWithUnreadWork(agentDb, scope)).toBe(0); + + await persistMessages(agentDb, { chatId, messages: [textMessage("a1")] }); + expect(await countChatsWithUnreadWork(agentDb, scope)).toBe(1); + + // Opening it clears the state... + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.chats set last_read_at = now() where id = $1`, + chatId + ); + expect(await countChatsWithUnreadWork(agentDb, scope)).toBe(0); + + // ...until the next answer lands behind a closed panel. + await persistMessages(agentDb, { chatId, messages: [textMessage("a2")] }); + expect(await countChatsWithUnreadWork(agentDb, scope)).toBe(1); + + // Another user's chat is never counted here. + expect( + await countChatsWithUnreadWork(agentDb, { ...scope, userId: "user_someone_else" }) + ).toBe(0); + }, + 30_000 + ); +}); + describe("countUserMessages", () => { postgresTest( "counts a user's own messages, and only those", @@ -677,7 +720,8 @@ describe("countUserMessages", () => { await persistMessages(agentDb, { chatId: "chat_a", - messages: [userMessage("u1"), textMessage("a1")], + // A watch's consent record is a user message but not a turn the user spent. + messages: [userMessage("u1"), textMessage("a1"), userMessage("watch-request:watch_1")], }); await persistMessages(agentDb, { chatId: "chat_b", messages: [userMessage("u2")] }); await persistMessages(agentDb, { chatId: "chat_gone", messages: [userMessage("u3")] }); @@ -696,3 +740,43 @@ describe("countUserMessages", () => { 30_000 ); }); + +/** + * A consented watch seeds its card in one run and revises it in another, with no + * hand-off between them: both name the row off the watch. So seeding twice has to + * converge on one card, and a row under that id in another chat must be refused + * rather than revised. + */ +describe("seedInvestigation", () => { + postgresTest( + "opens the watch's card once and hands the same row back after that", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri(), "chat_seed"); + await createChat(agentDb, { + id: "chat_seed_other", + organizationId: ORG_ID, + userId: USER_ID, + }); + + const id = watchInvestigationId("watch_seed"); + const seed = (chatId: string) => + seedInvestigation(agentDb, { + id, + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state: openState(), + }); + + expect(await seed("chat_seed")).toMatchObject({ ok: true, id, created: true }); + // The investigating lane, arriving after the wake already opened it. + expect(await seed("chat_seed")).toMatchObject({ ok: true, id, created: false }); + expect(await seed("chat_seed_other")).toEqual({ ok: false, error: "context_mismatch" }); + + const row = await getInvestigation(agentDb, { id }); + expect(row?.chatId).toBe("chat_seed"); + expect(row?.revision).toBe(0); + }, + 30_000 + ); +}); diff --git a/apps/webapp/test/dashboardAgentWakeActivity.test.ts b/apps/webapp/test/dashboardAgentWakeActivity.test.ts new file mode 100644 index 00000000000..891610f6154 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWakeActivity.test.ts @@ -0,0 +1,88 @@ +import { + cancelWatch, + createChat, + createDashboardAgentDb, + createWatch, + readDashboardAgentWakeActivity, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect } from "vitest"; + +// What the environment layout loader hands the browser. An active watch has to be part of it: +// without it a fresh browser with a watch created elsewhere never starts polling. + +let agentDbClient: DashboardAgentDbClient | undefined; +let agentDb: DashboardAgentDb; + +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +const SCOPE = { organizationId: "org_1", userId: "user_1" }; + +async function seedWatch(): Promise { + await createChat(agentDb, { id: "chat_1", ...SCOPE }); + const created = await createWatch(agentDb, { + chatId: "chat_1", + identity: "run_finished:run_1", + spec: { kind: "run_finished", runId: "run_1", checkEveryMinutes: 5, maxHours: 6, note: "" }, + projectId: "proj_1", + environmentId: "env_1", + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + ...SCOPE, + }); + expect(created.ok).toBe(true); + return created.ok ? created.watch.id : ""; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("the page load's wake activity", () => { + postgresTest( + "reports an active watch that has never woken anyone", + async ({ prisma, postgresContainer }) => { + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(postgresContainer.getConnectionUri(), { max: 2 }); + agentDb = agentDbClient.db; + + expect(await readDashboardAgentWakeActivity(agentDb, SCOPE)).toEqual({ + unreadWakes: 0, + hasActiveWatches: false, + }); + + const watchId = await seedWatch(); + expect(await readDashboardAgentWakeActivity(agentDb, SCOPE)).toEqual({ + unreadWakes: 0, + hasActiveWatches: true, + }); + + // Another user in the same org sees nothing. + expect(await readDashboardAgentWakeActivity(agentDb, { ...SCOPE, userId: "user_2" })).toEqual( + { unreadWakes: 0, hasActiveWatches: false } + ); + + await cancelWatch(agentDb, { id: watchId, reason: "user" }); + expect(await readDashboardAgentWakeActivity(agentDb, SCOPE)).toEqual({ + unreadWakes: 0, + hasActiveWatches: false, + }); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts b/apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts new file mode 100644 index 00000000000..6f5c63840f1 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts @@ -0,0 +1,293 @@ +/** + * One channel per (email, project) carries every environment the user subscribed from. + * Subscribing in a second environment must add to that list: replacing it would stop the + * first environment's mail without telling anyone. + */ + +import { + createChat, + createDashboardAgentDb, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient, RuntimeEnvironment } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + actor: undefined as undefined | { userId: string; client?: string; environmentId?: string }, +})); + +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-alert-environments"; +process.env.DASHBOARD_AGENT_SECRET_KEY = "tr_pat_test_dashboard_agent"; +// The subscribe path refuses outright without an email transport configured. +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; + +const { action: alertsAction } = await import("~/routes/api.v1.dashboard-agent.alerts"); +const { DASHBOARD_AGENT_WATCH_ALERT_TYPE } = + await import("~/services/dashboardAgentWatchAlerts.server"); +const { DeliverDashboardAgentWatchAlertService } = + await import("~/v3/services/alerts/deliverDashboardAgentWatchAlert.server"); +const { alertsWorker } = await import("~/v3/alertsWorker.server"); + +const enqueue = alertsWorker.enqueue as unknown as ReturnType; + +/** Replays every migration in order, so a new migration can't leave this file on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + const migrations = readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort(); + for (const name of migrations) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + ctx.actor = undefined; + enqueue.mockClear(); + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +/** One organization with a production and a staging environment, and one member. */ +async function seedProject(prisma: PrismaClient) { + const slug = `alertenvs_${suffix()}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = async (type: "PRODUCTION" | "STAGING", envSlug: string) => + prisma.runtimeEnvironment.create({ + data: { + slug: envSlug, + type, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_${envSlug}_${slug}`, + pkApiKey: `pk_${envSlug}_${slug}`, + shortcode: `${envSlug.slice(0, 2)}${suffix()}`, + }, + }); + const production = await environment("PRODUCTION", "prod"); + const staging = await environment("STAGING", "stg"); + + const user = await prisma.user.create({ + data: { email: `member_${suffix()}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "MEMBER" }, + }); + + return { organization, project, production, staging, user }; +} + +type Seeded = Awaited>; + +/** Subscribes through the real endpoint, in the environment the chat is open in. */ +async function subscribeIn( + seeded: Seeded, + environment: RuntimeEnvironment, + chatId: string +): Promise> { + await createChat(ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: environment.id, + }; + + const response = (await alertsAction({ + request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { + method: "POST", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify({ chatId, channel: "email" }), + }), + params: {}, + context: {} as never, + } as never)) as Response; + + expect(response.status).toBe(200); + return (await response.json()) as Record; +} + +/** Runs the real fan-out for a watch that fired in this environment. */ +async function fanOut(seeded: Seeded, environment: RuntimeEnvironment): Promise { + enqueue.mockClear(); + await new DeliverDashboardAgentWatchAlertService().call({ + watchId: `watch_${environment.slug}`, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: environment.id, + userId: seeded.user.id, + identity: "queue:my-queue", + kind: "queue_depth", + note: "the queue drains", + firedAt: new Date().toISOString(), + facts: { depth: 0 }, + resolution: "condition_met", + } as never); + + return enqueue.mock.calls + .map(([job]) => job as { payload?: { channelId?: string } }) + .flatMap((job) => (job.payload?.channelId ? [job.payload.channelId] : [])); +} + +describe("subscribing to watch alerts in a second environment", () => { + postgresTest( + "keeps the first environment's alerts delivering", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedProject(prisma); + + const first = await subscribeIn(seeded, seeded.production, "chat_prod"); + expect(await fanOut(seeded, seeded.production)).toEqual([first.id]); + + const second = await subscribeIn(seeded, seeded.staging, "chat_staging"); + // The same channel, re-used through the per-email deduplication key. + expect(second.id).toBe(first.id); + + // The point of the test: production still fans out after the staging subscribe. + expect(await fanOut(seeded, seeded.production)).toEqual([first.id]); + expect(await fanOut(seeded, seeded.staging)).toEqual([first.id]); + + const channels = await prisma.projectAlertChannel.findMany({ + where: { projectId: seeded.project.id }, + select: { environmentTypes: true, alertTypes: true, enabled: true }, + }); + expect(channels).toHaveLength(1); + expect([...channels[0].environmentTypes].sort()).toEqual(["PRODUCTION", "STAGING"]); + expect(channels[0].alertTypes).toEqual([DASHBOARD_AGENT_WATCH_ALERT_TYPE]); + }, + 60_000 + ); + + postgresTest( + "keeps an addition made between this subscribe's read and its write", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedProject(prisma); + + await subscribeIn(seeded, seeded.production, "chat_prod"); + const channel = await prisma.projectAlertChannel.findFirstOrThrow({ + where: { projectId: seeded.project.id }, + select: { id: true }, + }); + + // Stands in for a third environment whose subscribe commits after the staging + // subscribe has read the row and before it writes it back. + const read = prisma.projectAlertChannel.findFirst.bind(prisma.projectAlertChannel); + let raced = false; + const spy = vi + .spyOn(prisma.projectAlertChannel, "findFirst") + .mockImplementation(async (args: never) => { + const result = await read(args); + if (!raced && result) { + raced = true; + await prisma.projectAlertChannel.update({ + where: { id: channel.id }, + data: { environmentTypes: ["PRODUCTION", "DEVELOPMENT"] }, + }); + } + return result; + }); + + try { + await subscribeIn(seeded, seeded.staging, "chat_staging"); + } finally { + spy.mockRestore(); + } + expect(raced).toBe(true); + + const after = await prisma.projectAlertChannel.findFirstOrThrow({ + where: { id: channel.id }, + select: { environmentTypes: true }, + }); + expect([...after.environmentTypes].sort()).toEqual(["DEVELOPMENT", "PRODUCTION", "STAGING"]); + }, + 60_000 + ); + + postgresTest( + "leaves the other alert types on a channel it re-uses", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedProject(prisma); + + await subscribeIn(seeded, seeded.production, "chat_prod"); + await prisma.projectAlertChannel.updateMany({ + where: { projectId: seeded.project.id }, + data: { alertTypes: ["TASK_RUN", DASHBOARD_AGENT_WATCH_ALERT_TYPE] }, + }); + + await subscribeIn(seeded, seeded.staging, "chat_staging"); + + const channel = await prisma.projectAlertChannel.findFirstOrThrow({ + where: { projectId: seeded.project.id }, + select: { alertTypes: true }, + }); + expect([...channel.alertTypes].sort()).toEqual( + [DASHBOARD_AGENT_WATCH_ALERT_TYPE, "TASK_RUN"].sort() + ); + }, + 60_000 + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts b/apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts new file mode 100644 index 00000000000..476cf733fd6 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts @@ -0,0 +1,228 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type * as OrgIntegrationModule from "~/models/orgIntegration.server"; +import type * as SecretStoreModule from "~/services/secrets/secretStore.server"; + +const EMAIL_CHANNEL = { + id: "chan_email", + type: "EMAIL" as const, + properties: { email: "watcher@example.com" }, +}; +const SLACK_CHANNEL = { + id: "chan_slack", + type: "SLACK" as const, + properties: { channelId: "C123", channelName: "#alerts" }, +}; +const WEBHOOK_CHANNEL = { + id: "chan_webhook", + type: "WEBHOOK" as const, + properties: { + url: "https://example.com/hook", + secret: { nonce: "n", ciphertext: "c", tag: "t" }, + }, +}; + +const ctx = vi.hoisted(() => ({ + channels: [] as Array<{ id: string; type: string; properties: unknown }>, + /** Set to model replica lag: what the replica still sees. Null means "same as primary". */ + replicaChannels: null as Array<{ id: string; type: string; properties: unknown }> | null, + gateAllowed: true, + webhookFails: false, +})); + +const sendAlertEmail = vi.hoisted(() => vi.fn(async () => undefined)); +const postMessage = vi.hoisted(() => vi.fn(async () => ({ ok: true }))); +const safeWebhookFetch = vi.hoisted(() => + vi.fn(async (_url: string, _init: { body: string }) => ({ + ok: !ctx.webhookFails, + status: ctx.webhookFails ? 500 : 200, + })) +); + +vi.mock("~/db.server", () => { + const channelReader = (rows: () => Array<{ id: string; type: string; properties: unknown }>) => ({ + findMany: async () => rows(), + findFirst: async ({ where }: { where: { id: string } }) => + rows().find((channel) => channel.id === where.id) ?? null, + }); + const db = { + runtimeEnvironment: { + findFirst: async () => ({ + type: "PRODUCTION", + slug: "prod", + branchName: null, + project: { + name: "My Project", + slug: "my-project-abcd", + externalRef: "proj_abc", + organization: { slug: "acme", title: "Acme" }, + }, + }), + }, + organizationIntegration: { + findFirst: async () => ({ + id: "int_1", + service: "SLACK", + organizationId: "org_1", + tokenReference: { provider: "DATABASE", key: "k" }, + }), + }, + }; + return { + prisma: { ...db, projectAlertChannel: channelReader(() => ctx.channels) }, + $replica: { + ...db, + projectAlertChannel: channelReader(() => ctx.replicaChannels ?? ctx.channels), + }, + sqlDatabaseSchema: undefined, + }; +}); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.gateAllowed, +})); + +vi.mock("~/services/email.server", () => ({ sendAlertEmail })); +vi.mock("~/services/dashboardAgentAlertUnsubscribeToken.server", () => ({ + mintDashboardAgentAlertUnsubscribeToken: async () => "unsub-token", +})); +vi.mock("~/v3/services/alerts/safeWebhookFetch.server", () => ({ safeWebhookFetch })); + +vi.mock("~/services/secrets/secretStore.server", async (importOriginal) => ({ + ...(await importOriginal()), + decryptSecret: async () => "webhook-secret", +})); + +vi.mock("~/models/orgIntegration.server", async (importOriginal) => ({ + ...(await importOriginal()), + OrgIntegrationRepository: { + getAuthenticatedClientForIntegration: async () => ({ chat: { postMessage } }), + }, +})); + +const { DeliverDashboardAgentWatchAlertService, DeliverDashboardAgentWatchChannelAlertService } = + await import("~/v3/services/alerts/deliverDashboardAgentWatchAlert.server"); +const { alertsWorker } = await import("~/v3/alertsWorker.server"); + +const enqueue = alertsWorker.enqueue as unknown as ReturnType; + +const payload = { + watchId: "watch_1", + organizationId: "org_1", + projectId: "proj_1", + environmentId: "env_1", + userId: "user_1", + identity: "queue:my-queue", + kind: "queue_depth", + note: "the queue drains", + firedAt: "2026-07-30T10:00:00.000Z", + facts: { depth: 0 }, +}; + +beforeEach(() => { + ctx.channels = [EMAIL_CHANNEL, SLACK_CHANNEL, WEBHOOK_CHANNEL]; + ctx.replicaChannels = null; + ctx.gateAllowed = true; + ctx.webhookFails = false; + enqueue.mockClear(); + sendAlertEmail.mockClear(); + postMessage.mockClear(); + safeWebhookFetch.mockClear(); +}); + +describe("dashboard agent watch alert fan-out", () => { + test("enqueues one delivery job per channel, keyed per channel", async () => { + await new DeliverDashboardAgentWatchAlertService().call(payload); + + expect(enqueue).toHaveBeenCalledTimes(3); + const calls = enqueue.mock.calls.map(([arg]) => arg); + + expect(calls.map((call) => call.id)).toEqual([ + "watch-alert:watch_1:channel:chan_email", + "watch-alert:watch_1:channel:chan_slack", + "watch-alert:watch_1:channel:chan_webhook", + ]); + for (const call of calls) { + expect(call.job).toBe("v3.deliverDashboardAgentWatchAlertChannel"); + } + expect(calls[2].payload).toMatchObject({ ...payload, channelId: "chan_webhook" }); + + expect(sendAlertEmail).not.toHaveBeenCalled(); + expect(postMessage).not.toHaveBeenCalled(); + expect(safeWebhookFetch).not.toHaveBeenCalled(); + }); + + test("the fan-out is idempotent: a retry re-enqueues the same job ids", async () => { + await new DeliverDashboardAgentWatchAlertService().call(payload); + const first = enqueue.mock.calls.map(([arg]) => arg.id); + enqueue.mockClear(); + + await new DeliverDashboardAgentWatchAlertService().call(payload); + expect(enqueue.mock.calls.map(([arg]) => arg.id)).toEqual(first); + }); + + test("a denied gate enqueues nothing", async () => { + ctx.gateAllowed = false; + await new DeliverDashboardAgentWatchAlertService().call(payload); + expect(enqueue).not.toHaveBeenCalled(); + }); +}); + +describe("dashboard agent watch alert per-channel delivery", () => { + test("a failing webhook retry re-sends only the webhook", async () => { + const service = new DeliverDashboardAgentWatchChannelAlertService(); + + await service.call({ ...payload, channelId: "chan_email" }); + await service.call({ ...payload, channelId: "chan_slack" }); + expect(sendAlertEmail).toHaveBeenCalledTimes(1); + expect(postMessage).toHaveBeenCalledTimes(1); + + ctx.webhookFails = true; + await expect(service.call({ ...payload, channelId: "chan_webhook" })).rejects.toThrow( + /Failed to send watch alert webhook/ + ); + await expect(service.call({ ...payload, channelId: "chan_webhook" })).rejects.toThrow( + /Failed to send watch alert webhook/ + ); + + expect(sendAlertEmail).toHaveBeenCalledTimes(1); + expect(postMessage).toHaveBeenCalledTimes(1); + expect(safeWebhookFetch).toHaveBeenCalledTimes(2); + }); + + test("the webhook event id and created are stable across attempts", async () => { + const service = new DeliverDashboardAgentWatchChannelAlertService(); + ctx.webhookFails = true; + + await expect(service.call({ ...payload, channelId: "chan_webhook" })).rejects.toThrow(); + await expect(service.call({ ...payload, channelId: "chan_webhook" })).rejects.toThrow(); + + const bodies = safeWebhookFetch.mock.calls.map(([, init]) => JSON.parse(init.body)); + + expect(bodies).toHaveLength(2); + expect(bodies[0].id).toBe("watch:watch_1:channel:chan_webhook"); + expect(bodies[1].id).toBe(bodies[0].id); + expect(bodies[0].created).toBe(payload.firedAt); + expect(bodies[1].created).toBe(bodies[0].created); + }); + + test("an unsubscribe the replica hasn't caught up on still stops the email", async () => { + ctx.channels = []; + ctx.replicaChannels = [EMAIL_CHANNEL]; + + await new DeliverDashboardAgentWatchChannelAlertService().call({ + ...payload, + channelId: "chan_email", + }); + + expect(sendAlertEmail).not.toHaveBeenCalled(); + }); + + test("an unsubscribed channel delivers nothing", async () => { + ctx.channels = []; + await new DeliverDashboardAgentWatchChannelAlertService().call({ + ...payload, + channelId: "chan_email", + }); + expect(sendAlertEmail).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts b/apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts new file mode 100644 index 00000000000..6331a07fcc1 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts @@ -0,0 +1,281 @@ +/** + * `emailAlerts` in the create-watch response is a statement about the caller's own + * subscription. A project is shared by every member, so another member's channel must + * never be reported as the caller's. + */ + +import { + createChat, + createDashboardAgentDb, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type * as SdkModule from "@trigger.dev/sdk"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps, WatchRunRow } from "~/services/dashboardAgentWatchChecks"; +import type * as WatchChecksModule from "~/services/dashboardAgentWatchChecks.server"; + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + actor: undefined as undefined | { userId: string; client?: string; environmentId?: string }, +})); + +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +// The watch target's readers are the IO seam. The alert channels this file is about stay +// real rows in the container. +vi.mock("~/services/dashboardAgentWatchChecks.server", async (importOriginal) => ({ + ...(await importOriginal()), + watchCreationCheckDeps: (): WatchCheckDeps => fakeCheckDeps(), +})); + +// The first tick is scheduled by triggering a task over the network. +vi.mock("@trigger.dev/sdk", async (importOriginal) => ({ + ...(await importOriginal()), + TriggerClient: class { + tasks = { trigger: async () => ({ id: "run_tick" }) }; + }, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-alert-owner-scope"; +// Without a secret key the create path refuses as "not configured". +process.env.DASHBOARD_AGENT_SECRET_KEY = "tr_pat_test_dashboard_agent"; +// The subscribe path refuses outright without an email transport configured. +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; + +const { action: watchesAction } = await import("~/routes/api.v1.dashboard-agent.watches"); +const { subscribeUserToWatchAlerts, DASHBOARD_AGENT_WATCH_ALERT_TYPE } = + await import("~/services/dashboardAgentWatchAlerts.server"); + +/** Replays every migration in order, so a new migration can't leave this file on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + const migrations = readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort(); + for (const name of migrations) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + ctx.actor = undefined; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +function runRow(): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date(), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + }; +} + +/** Keeps the condition pending with a live target, so a create always makes a watch. */ +function fakeCheckDeps(): WatchCheckDeps { + return { + readRun: async () => runRow(), + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ ageMs: 30_000, source: "live_queue", current: true }), + readErrorRecurrence: async () => null, + readHealth: async () => ({ trustworthy: true, severity: "warn" }), + }; +} + +/** One organization with a production environment, and two members of it. */ +async function seedProject(prisma: PrismaClient) { + const slug = `alertscope_${suffix()}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `pr${suffix()}`, + }, + }); + return { organization, project, environment }; +} + +type Seeded = Awaited>; + +async function seedMember(prisma: PrismaClient, seeded: Seeded, name: string) { + const user = await prisma.user.create({ + data: { email: `${name}_${suffix()}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: user.id, role: "MEMBER" }, + }); + return user; +} + +function subscribeEnvironment(seeded: Seeded) { + return { + type: seeded.environment.type as string, + organizationId: seeded.organization.id, + organization: { slug: seeded.organization.slug }, + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + }; +} + +function spec(runId: string): WatchSpec { + return { kind: "run_start", runId, checkEveryMinutes: 1, maxHours: 2, note: "tell me" }; +} + +/** + * Creates a watch through the real endpoint as `user`, and answers with the response body + * the caller is told — the only place the subscription claim is visible. + */ +async function createWatchAs( + seeded: Seeded, + user: { id: string }, + chatId: string +): Promise> { + await createChat(ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: user.id, + }); + + ctx.actor = { + userId: user.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const response = (await watchesAction({ + request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/watches", { + method: "POST", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify({ spec: spec(`run_${chatId}`), chatId }), + }), + params: {}, + context: {} as never, + } as never)) as Response; + + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body).toMatchObject({ watching: true }); + return body; +} + +describe("the create-watch response's email alert state", () => { + postgresTest( + "reports only the caller's own subscription, never another member's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedProject(prisma); + const alice = await seedMember(prisma, seeded, "alice"); + const bob = await seedMember(prisma, seeded, "bob"); + + const subscribed = await subscribeUserToWatchAlerts({ + userId: alice.id, + environment: subscribeEnvironment(seeded), + }); + expect(subscribed).toMatchObject({ ok: true, email: alice.email }); + + // One channel exists in the project, and it mails Alice. Bob is not on it. + expect( + await prisma.projectAlertChannel.count({ + where: { + projectId: seeded.project.id, + alertTypes: { has: DASHBOARD_AGENT_WATCH_ALERT_TYPE }, + }, + }) + ).toBe(1); + + const bobBefore = await createWatchAs(seeded, bob, "chat_bob_before"); + expect(bobBefore.emailAlerts).toBe("none"); + + const bobSubscribed = await subscribeUserToWatchAlerts({ + userId: bob.id, + environment: subscribeEnvironment(seeded), + }); + expect(bobSubscribed).toMatchObject({ ok: true, email: bob.email }); + + // His own channel, so now the claim is true. + const bobAfter = await createWatchAs(seeded, bob, "chat_bob_after"); + expect(bobAfter.emailAlerts).toBe("subscribed"); + + // Bob subscribing did not take over or restate Alice's own channel. + const aliceState = await createWatchAs(seeded, alice, "chat_alice"); + expect(aliceState.emailAlerts).toBe("subscribed"); + + const channels = await prisma.projectAlertChannel.findMany({ + where: { projectId: seeded.project.id }, + select: { properties: true }, + }); + expect(channels).toHaveLength(2); + expect( + channels.map((channel) => (channel.properties as { email: string }).email).sort() + ).toEqual([alice.email, bob.email].sort()); + }, + 30_000 + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts b/apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts new file mode 100644 index 00000000000..1a1670860a0 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts @@ -0,0 +1,127 @@ +import { + createChat, + createDashboardAgentDb, + createWatch, + listActiveWatchesForBatch, + recordWatchCheck, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect } from "vitest"; + +// A group larger than the batch cap must rotate: the same prefix winning every tick would +// starve the rest for as long as the group stays full. + +let agentDbClient: DashboardAgentDbClient | undefined; +let agentDb: DashboardAgentDb; + +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +const ENVIRONMENT_ID = "env_batch_fairness"; +const CADENCE = 5; + +/** `count` active watches in one (environment, cadence) group, spread over enough chats. */ +async function seedGroup(count: number, expiresAt: Date, prefix = "a"): Promise { + const ids: string[] = []; + for (let index = 0; index < count; index++) { + const chatId = `chat_${prefix}_${Math.floor(index / 3)}`; + if (index % 3 === 0) { + await createChat(agentDb, { id: chatId, organizationId: "org_1", userId: "user_1" }); + } + const created = await createWatch(agentDb, { + chatId, + identity: `run_finished:run_${prefix}_${index}`, + spec: { + kind: "run_finished", + runId: `run_${prefix}_${index}`, + checkEveryMinutes: CADENCE, + maxHours: 6, + note: "", + }, + organizationId: "org_1", + projectId: "proj_1", + environmentId: ENVIRONMENT_ID, + userId: "user_1", + // Staggered, so soonest-deadline-first would deterministically pick the same prefix. + expiresAt: new Date(expiresAt.getTime() + index * 60_000), + }); + expect(created.ok).toBe(true); + if (created.ok) ids.push(created.watch.id); + } + return ids; +} + +/** One tick: take a capped page and mark every watch on it as checked. */ +async function tick(limit: number): Promise { + const page = await listActiveWatchesForBatch(agentDb, { + environmentId: ENVIRONMENT_ID, + cadenceMinutes: CADENCE, + limit, + }); + for (const watch of page) { + await recordWatchCheck(agentDb, { id: watch.id }); + } + return page.map((watch) => watch.id); +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("the batch group's fairness", () => { + postgresTest( + "checks every watch of an over-cap group within a bounded number of ticks", + async ({ prisma, postgresContainer }) => { + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(postgresContainer.getConnectionUri(), { max: 4 }); + agentDb = agentDbClient.db; + + const inSixHours = new Date(Date.now() + 6 * 60 * 60 * 1000); + const all = await seedGroup(12, inSixHours); + + const cap = 5; + const checked = new Set(); + // ceil(12 / 5) = 3 ticks is the whole group, and the fourth must start over. + for (let round = 0; round < 3; round++) { + for (const id of await tick(cap)) checked.add(id); + } + + expect(checked.size).toBe(all.length); + expect([...checked].sort()).toEqual([...all].sort()); + } + ); + + postgresTest( + "a watch whose window closes within a cadence is never deferred by the cap", + async ({ prisma, postgresContainer }) => { + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(postgresContainer.getConnectionUri(), { max: 4 }); + agentDb = agentDbClient.db; + + const inSixHours = new Date(Date.now() + 6 * 60 * 60 * 1000); + await seedGroup(6, inSixHours); + // Checked a moment ago, so pure least-recently-checked order would put it last. + const [closing] = await seedGroup(1, new Date(Date.now() + 60_000), "b"); + await recordWatchCheck(agentDb, { id: closing!, lastCheckedAt: new Date() }); + + const page = await tick(2); + expect(page[0]).toBe(closing); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts b/apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts new file mode 100644 index 00000000000..c042f191657 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts @@ -0,0 +1,305 @@ +import { + armWatchBatch, + createChat, + createDashboardAgentDb, + createWatch, + getWatch, + listActiveWatchesForBatch, + recordWatchCheck, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; + +// What a batch tick records. A check that read nothing is not an observation: recording it +// would move the watch down the rotation and overwrite the facts a streak lives in. + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-batch"; + +const { isDue, runWatchBatchCheck } = await import("~/services/dashboardAgentWatchBatch.server"); +const { previousCheckFacts } = await import("~/services/dashboardAgentWatchChecks"); + +/** Replays every migration in order, so a new migration can't leave the suite on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +const ORGANIZATION_ID = "org_batch"; +const PROJECT_ID = "proj_batch"; +const ENVIRONMENT_ID = "env_batch_recording"; +const USER_ID = "user_batch"; +const CADENCE = 5; + +const environment = { + id: ENVIRONMENT_ID, + organizationId: ORGANIZATION_ID, + projectId: PROJECT_ID, + slug: "prod", + type: "PRODUCTION", + project: { id: PROJECT_ID, externalRef: "proj_external" }, + organization: { id: ORGANIZATION_ID, slug: "batch" }, +} as any; + +const STALLED: WatchSpec = { + kind: "queue_stalled", + queue: "task/send-receipt", + ticks: 3, + checkEveryMinutes: CADENCE, + maxHours: 6, + note: "tell me if the queue stops moving", +}; + +/** A watch with two no-progress checks already behind it, last looked at `checkedAt`. */ +async function seedStalling(checkedAt: Date): Promise { + await createChat(ctx.agentDb, { + id: "chat_batch", + organizationId: ORGANIZATION_ID, + userId: USER_ID, + }); + const created = await createWatch(ctx.agentDb, { + chatId: "chat_batch", + identity: "queue_stalled:task/send-receipt", + spec: STALLED as any, + organizationId: ORGANIZATION_ID, + projectId: PROJECT_ID, + environmentId: ENVIRONMENT_ID, + userId: USER_ID, + expiresAt: new Date(Date.now() + 6 * 60 * 60 * 1000), + }); + if (!created.ok) throw new Error(`the watch wasn't created: ${created.error}`); + + await recordWatchCheck(ctx.agentDb, { + id: created.watch.id, + lastCheckedAt: checkedAt, + lastResult: { + result: "pending", + facts: { + queue: "task/send-receipt", + depth: 412, + notDecreasingStreak: 2, + ticks: STALLED.kind === "queue_stalled" ? STALLED.ticks : 3, + }, + }, + }); + return created.watch.id; +} + +function readers(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 412, source: "live_queue", current: true }), + readQueueOldestAge: async () => null, + readErrorRecurrence: async () => null, + readHealth: async () => null, + ...overrides, + }; +} + +/** The chain this group's ticks run under. Armed once, like a live chain's. */ +async function armChain(): Promise { + const armed = await armWatchBatch(ctx.agentDb, { + environmentId: ENVIRONMENT_ID, + cadenceMinutes: CADENCE, + staleBefore: new Date(), + }); + if (!armed) throw new Error("the batch chain wasn't armed"); + return armed.epoch; +} + +/** A second watch in the same group, on its own queue so one reader can fail alone. */ +async function seedSecondQueue(queue: string): Promise { + const created = await createWatch(ctx.agentDb, { + chatId: "chat_batch", + identity: `queue_stalled:${queue}`, + spec: { ...STALLED, queue } as any, + organizationId: ORGANIZATION_ID, + projectId: PROJECT_ID, + environmentId: ENVIRONMENT_ID, + userId: USER_ID, + expiresAt: new Date(Date.now() + 6 * 60 * 60 * 1000), + }); + if (!created.ok) throw new Error(`the watch wasn't created: ${created.error}`); + return created.watch.id; +} + +async function tick(params: { + epoch: number; + tick: number; + checkDeps: WatchCheckDeps; + /** The group's per-tick cap, so an over-cap group can be exercised. */ + limit?: number; +}) { + return runWatchBatchCheck( + { + environmentId: ENVIRONMENT_ID, + cadenceMinutes: CADENCE, + epoch: params.epoch, + tick: params.tick, + }, + { + checkDeps: () => params.checkDeps, + authorize: async () => ({ ok: true, environment }) as const, + mintToken: async () => "watch_token", + ...(params.limit + ? { + listActive: (args: { environmentId: string; cadenceMinutes: number }) => + listActiveWatchesForBatch(ctx.agentDb, { ...args, limit: params.limit }), + } + : {}), + } + ); +} + +describe("what a batch tick records", () => { + postgresTest( + "a check that couldn't read anything leaves the row's last look and facts alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const checkedAt = new Date(Date.now() - 60 * 60 * 1000); + const watchId = await seedStalling(checkedAt); + const epoch = await armChain(); + + const response = await tick({ + epoch, + tick: 1, + checkDeps: readers({ + readQueueDepth: async () => { + throw new Error("the queue reader is down"); + }, + }), + }); + + expect(response.watches?.[0]).toMatchObject({ watchId, result: "unavailable" }); + + const row = await getWatch(ctx.agentDb, { id: watchId }); + // Nothing was checked, so the watch is still due at the next tick. + expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); + // And the streak the ticks built is still there to be continued. + expect(previousCheckFacts(row?.lastResult)).toMatchObject({ + depth: 412, + notDecreasingStreak: 2, + }); + } + ); + + postgresTest( + "the next readable check continues the frozen streak and fires the stall", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const checkedAt = new Date(Date.now() - 60 * 60 * 1000); + const watchId = await seedStalling(checkedAt); + const epoch = await armChain(); + + await tick({ + epoch, + tick: 1, + checkDeps: readers({ + readQueueDepth: async () => { + throw new Error("the queue reader is down"); + }, + }), + }); + const response = await tick({ epoch, tick: 2, checkDeps: readers() }); + + expect(response.watches?.[0]).toMatchObject({ watchId, result: "satisfied" }); + + const row = await getWatch(ctx.agentDb, { id: watchId }); + // A real evaluation does move the row's last look on. + expect(row?.lastCheckedAt?.getTime()).toBeGreaterThan(checkedAt.getTime()); + expect(previousCheckFacts(row?.lastResult)).toMatchObject({ notDecreasingStreak: 3 }); + } + ); + + postgresTest( + "a permanently unreadable watch rotates out of an over-cap group's head", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const checkedAt = new Date(Date.now() - 60 * 60 * 1000); + // Looked at an hour ago, so it leads the group; its reader never comes back. + const broken = await seedStalling(checkedAt); + const neighbour = await seedSecondQueue("task/send-invoice"); + const epoch = await armChain(); + + const brokenQueue = STALLED.kind === "queue_stalled" ? STALLED.queue : ""; + const oneReaderDown = readers({ + readQueueDepth: async (queue: string) => { + if (queue === brokenQueue) throw new Error("the queue reader is down"); + return { depth: 412, source: "live_queue", current: true }; + }, + }); + + // A cap of one: whatever leads the group is the only watch the tick reaches. + const first = await tick({ epoch, tick: 1, checkDeps: oneReaderDown, limit: 1 }); + expect(first.watches?.map((entry) => entry.watchId)).toEqual([broken]); + expect(first.watches?.[0]).toMatchObject({ result: "unavailable" }); + + // The neighbour is no longer crowded out by a watch that never reads anything. + const second = await tick({ epoch, tick: 2, checkDeps: oneReaderDown, limit: 1 }); + expect(second.watches?.map((entry) => entry.watchId)).toEqual([neighbour]); + expect(second.watches?.[0]).toMatchObject({ result: "pending" }); + + // And nothing about the unreadable watch's own state moved: not its last check, not + // the streak the earlier ticks built, and not its dueness. + const row = await getWatch(ctx.agentDb, { id: broken }); + expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); + expect(previousCheckFacts(row?.lastResult)).toMatchObject({ + depth: 412, + notDecreasingStreak: 2, + }); + expect(isDue(row!, CADENCE, new Date())).toBe(true); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts b/apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts new file mode 100644 index 00000000000..363a17600f4 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts @@ -0,0 +1,261 @@ +import { + createChat, + createDashboardAgentDb, + getChatMessages, + getInvestigation, + listStaleOpenInvestigations, + settleInvestigationStateAndCloseCard, + softDeleteChat, + upsertInvestigationRevision, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { + forceSettledInvestigationState, + investigationStateSchema, + type InvestigationState, +} from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect } from "vitest"; + +/** + * The write the consented watch investigation closes its card with. + * + * The lane has no `onTurnComplete` to hand settlements to, so it closes the card + * itself. Settling the row and appending the terminal card used to be two operations + * with the append's error swallowed: the row went terminal, the card never arrived, + * the stale sweep stopped selecting the row, and the panel kept spinning for ever. + */ + +let agentDb: DashboardAgentDb; +let agentDbClient: DashboardAgentDbClient | undefined; + +/** Replays every migration in order, so a new migration can't leave the suite on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +const ORG_ID = "org_watch_card"; +const USER_ID = "user_watch_card"; +const PROJECT_REF = "proj_watch_card"; +const ENV_REF = "env_watch_card"; +const MESSAGE_ID = "investigate:watch:watch_1:fired:investigate:settled"; + +async function boot(prisma: PrismaClient, connectionUri: string, chatId: string) { + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 2 }); + agentDb = agentDbClient.db; + await createChat(agentDb, { id: chatId, organizationId: ORG_ID, userId: USER_ID }); +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function openState(): InvestigationState { + return investigationStateSchema.parse({ + outcome: "in_progress", + severity: "warn", + confidence: "low", + title: "Investigating run_abc123", + headline: "The run finished with errors. Looking into why.", + hypotheses: [], + evidence: [], + }); +} + +async function seed(chatId: string, state: unknown): Promise { + const created = await upsertInvestigationRevision(agentDb, { + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state, + }); + if (!created.ok) throw new Error("the fixture investigation wasn't created"); + return created.id; +} + +async function transcript(chatId: string): Promise<{ id: string; parts: any[] }[]> { + return (await getChatMessages(agentDb, { + chatId, + userId: USER_ID, + organizationId: ORG_ID, + })) as { id: string; parts: any[] }[]; +} + +describe("closing a consented watch investigation's card", () => { + postgresTest( + "commits the terminal revision and the card together, under the lane's own message id", + async ({ prisma, postgresContainer }) => { + const chatId = "chat_watch_card"; + await boot(prisma, postgresContainer.getConnectionUri(), chatId); + const id = await seed(chatId, openState()); + + const result = await settleInvestigationStateAndCloseCard(agentDb, { + id, + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state: forceSettledInvestigationState(openState()), + messageId: MESSAGE_ID, + }); + expect(result).toMatchObject({ ok: true, id, revision: 1, closed: true }); + + const stored = await transcript(chatId); + expect(stored.map((message) => message.id)).toEqual([MESSAGE_ID]); + expect(stored[0]!.parts[0]!.output.blocks[0]).toMatchObject({ id, revision: 1 }); + expect(stored[0]!.parts[0]!.output.blocks[0].investigation.outcome).toBe("inconclusive"); + expect( + investigationStateSchema.parse((await getInvestigation(agentDb, { id }))?.state).outcome + ).toBe("inconclusive"); + + // The lane dedupes on the action, so a redelivered kick closes nothing twice — + // and must not bump the revision, or the row runs ahead of the stored card. + const again = await settleInvestigationStateAndCloseCard(agentDb, { + id, + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state: forceSettledInvestigationState(openState()), + messageId: MESSAGE_ID, + }); + expect(again).toMatchObject({ ok: true, id, revision: 1, closed: false }); + expect((await getInvestigation(agentDb, { id }))?.revision).toBe(1); + + const after = await transcript(chatId); + expect(after.map((message) => message.id)).toEqual([MESSAGE_ID]); + expect(after[0]!.parts[0]!.output.blocks[0]).toMatchObject({ id, revision: 1 }); + // The replayed result is the card the transcript holds, not a second rendering. + expect((again as { card: unknown }).card).toEqual(after[0]); + }, + 30_000 + ); + + postgresTest( + "settles nothing when the chat was deleted, so the sweep still selects the row", + async ({ prisma, postgresContainer }) => { + const chatId = "chat_watch_card_deleted"; + await boot(prisma, postgresContainer.getConnectionUri(), chatId); + const id = await seed(chatId, openState()); + await softDeleteChat(agentDb, { chatId, userId: USER_ID }); + + expect( + await settleInvestigationStateAndCloseCard(agentDb, { + id, + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state: forceSettledInvestigationState(openState()), + messageId: MESSAGE_ID, + }) + ).toEqual({ ok: false, error: "chat_missing" }); + + const row = await getInvestigation(agentDb, { id }); + expect(row?.revision).toBe(0); + expect((row!.state as { outcome?: string }).outcome).toBe("in_progress"); + }, + 30_000 + ); + + postgresTest( + "settles nothing when the chat row was never there", + async ({ prisma, postgresContainer }) => { + const chatId = "chat_watch_card_absent"; + await boot(prisma, postgresContainer.getConnectionUri(), "chat_watch_card_present"); + const id = await seed(chatId, openState()); + + expect( + await settleInvestigationStateAndCloseCard(agentDb, { + id, + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state: forceSettledInvestigationState(openState()), + messageId: MESSAGE_ID, + }) + ).toEqual({ ok: false, error: "chat_missing" }); + + const row = await getInvestigation(agentDb, { id }); + expect(row?.revision).toBe(0); + expect((row!.state as { outcome?: string }).outcome).toBe("in_progress"); + }, + 30_000 + ); + + /** + * The regression. A terminal row with no terminal card must be impossible: if the + * card can't be written the settle rolls back, so the row stays `in_progress` and the + * stale sweep still selects it. + */ + postgresTest( + "a card that can't be written rolls the settle back, leaving the row in_progress", + async ({ prisma, postgresContainer }) => { + const chatId = "chat_watch_card_fails"; + await boot(prisma, postgresContainer.getConnectionUri(), chatId); + const id = await seed(chatId, openState()); + + // A state the row accepts but no card can be rendered from, so the delivery half + // genuinely fails against a real database. + await expect( + settleInvestigationStateAndCloseCard(agentDb, { + id, + chatId, + projectRef: PROJECT_REF, + environmentRef: ENV_REF, + state: { outcome: "inconclusive" }, + messageId: MESSAGE_ID, + }) + ).rejects.toThrow(/isn't renderable/); + + const row = await getInvestigation(agentDb, { id }); + expect(row?.revision).toBe(0); + expect((row!.state as { outcome?: string }).outcome).toBe("in_progress"); + expect(await transcript(chatId)).toEqual([]); + + // Still selectable, so the backstop sweep can finish the job. + const stale = await listStaleOpenInvestigations(agentDb, { + olderThan: new Date(), + limit: 10, + }); + expect(stale.map((candidate) => candidate.id)).toEqual([id]); + }, + 30_000 + ); + + postgresTest( + "refuses a row that belongs to another project, and writes nothing", + async ({ prisma, postgresContainer }) => { + const chatId = "chat_watch_card_tenancy"; + await boot(prisma, postgresContainer.getConnectionUri(), chatId); + const id = await seed(chatId, openState()); + + expect( + await settleInvestigationStateAndCloseCard(agentDb, { + id, + chatId, + projectRef: "proj_someone_else", + environmentRef: ENV_REF, + state: forceSettledInvestigationState(openState()), + messageId: MESSAGE_ID, + }) + ).toEqual({ ok: false, error: "context_mismatch" }); + + expect(await transcript(chatId)).toEqual([]); + expect((await getInvestigation(agentDb, { id }))?.revision).toBe(0); + }, + 30_000 + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts b/apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts new file mode 100644 index 00000000000..52ee1d8c2af --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts @@ -0,0 +1,163 @@ +import { + createDashboardAgentDb, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; + +// The submit's idempotency key identifies one card submission. A per-condition fallback would +// identify the condition instead, so a re-watch could replay a stale terminal outcome inside the +// retention window. The key is required, and a submit without one must create nothing. + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + userId: "", +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/session.server", () => ({ + requireUser: async () => ({ id: ctx.userId, admin: false, isImpersonating: false }), +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-card-request-id"; + +const { action } = + await import("~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent"); + +/** Replays every migration in order, so a new migration can't leave the suite on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function seed(prisma: PrismaClient) { + const slug = `card_req_${Math.random().toString(36).slice(2, 10)}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 6)}`, + }, + }); + ctx.userId = user.id; + return { user, organization, project }; +} + +const DRAFT = JSON.stringify({ + spec: { + kind: "run_start", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it starts", + }, + followUp: { investigateOnAttention: false, notifyExternally: false }, +}); + +function submitRequest(slug: string, body: Record) { + const form = new URLSearchParams(body); + return action({ + request: new Request( + `https://app.trigger.dev/resources/orgs/${slug}/projects/${slug}/env/prod/dashboard-agent`, + { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: form.toString(), + } + ), + params: { organizationSlug: slug, projectParam: slug, envParam: "prod" }, + context: {}, + } as never) as Promise; +} + +async function countRows(prisma: PrismaClient, table: "watches" | "watch_submissions") { + const rows = await prisma.$queryRawUnsafe>( + `select count(*)::bigint as count from trigger_dashboard_agent.${table}` + ); + return Number(rows[0]?.count ?? 0); +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("the watch card submit's request id", () => { + postgresTest( + "refuses a submit with no clientRequestId, and creates nothing", + async ({ prisma, postgresContainer }) => { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(postgresContainer.getConnectionUri(), { max: 4 }); + ctx.agentDb = agentDbClient.db; + + const seeded = await seed(prisma); + + const response = await submitRequest(seeded.organization.slug, { + intent: "watch-create", + draft: DRAFT, + }); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "invalid_request" }); + + // No ledger row and no watch: the submit was refused before either could be written. + expect(await countRows(prisma, "watch_submissions")).toBe(0); + expect(await countRows(prisma, "watches")).toBe(0); + + // The same body with a key gets past this refusal, so the 400 above is the key's. + const withKey = await submitRequest(seeded.organization.slug, { + intent: "watch-create", + draft: DRAFT, + clientRequestId: "wreq_1", + }); + expect(await withKey.json()).not.toMatchObject({ code: "invalid_request" }); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchChecks.test.ts b/apps/webapp/test/dashboardAgentWatchChecks.test.ts new file mode 100644 index 00000000000..2f62539e63a --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchChecks.test.ts @@ -0,0 +1,1002 @@ +import { describe, expect, it } from "vitest"; +import { + checkWatch, + previousCheckFacts, + type WatchCheckDeps, + type WatchErrorRecurrence, + type WatchQueueDepth, + type WatchRunRow, +} from "~/services/dashboardAgentWatchChecks"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; + +const NOW = new Date("2026-07-27T12:00:00.000Z"); +const SINCE = new Date("2026-07-27T11:00:00.000Z"); + +function deps(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: async () => true, + readQueueDepth: async () => null, + readQueueOldestAge: async () => null, + readErrorRecurrence: async () => null, + readHealth: async () => null, + ...overrides, + }; +} + +function live(depth: number): WatchQueueDepth { + return { depth, source: "live_queue", current: true }; +} + +function stale(depth: number): WatchQueueDepth { + return { + depth, + source: "queue_metrics", + current: false, + asOf: new Date("2026-07-27T11:40:00.000Z"), + }; +} + +function run(overrides: Partial = {}): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date("2026-07-27T11:55:00.000Z"), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + ...overrides, + }; +} + +function check(spec: WatchSpec, d: WatchCheckDeps, previous?: Record | null) { + return checkWatch(spec, d, { now: NOW, since: SINCE, previous }); +} + +const runStart: WatchSpec = { + kind: "run_start", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it starts", +}; + +const runFinished: WatchSpec = { + kind: "run_finished", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it finishes", +}; + +const backlogDrain: WatchSpec = { + kind: "backlog_drain", + queue: "task/my-task", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when the backlog clears", +}; + +const errorRecurrence: WatchSpec = { + kind: "error_recurrence", + fingerprint: "fp_1", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it comes back", +}; + +const healthRecovery: WatchSpec = { + kind: "health_recovery", + report: "health", + fromSeverity: "crit", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when prod is healthy", +}; + +describe("run_start", () => { + it("is satisfied once startedAt exists, whatever the current status is", async () => { + const outcome = await check( + runStart, + deps({ + readRun: async () => + run({ + status: "COMPLETED_WITH_ERRORS", + queuedAt: new Date("2026-07-27T11:56:00.000Z"), + startedAt: new Date("2026-07-27T11:58:00.000Z"), + }), + }) + ); + + expect(outcome.result).toBe("satisfied"); + expect(outcome.facts.waitMs).toBe(2 * 60_000); + expect(outcome.facts.waitBasis).toBe("queued_at"); + expect(outcome.facts.waitLabel).toBe("queued for 2m"); + }); + + it("labels a wait with no queuedAt as time from creation, never as a queue wait", async () => { + const outcome = await check( + runStart, + deps({ readRun: async () => run({ status: "PENDING" }) }) + ); + + expect(outcome.result).toBe("pending"); + expect(outcome.facts.waitBasis).toBe("created_at"); + expect(outcome.facts.waitLabel).toBe("time from creation: 5m"); + expect(outcome.facts.queueWaitReliable).toBe(false); + }); + + // A resume/retry doesn't restamp queuedAt, so the leftover value is not a queue wait. + it("does not measure a resumed run's wait from its stale queuedAt", async () => { + const outcome = await check( + runStart, + deps({ + readRun: async () => + run({ status: "WAITING_TO_RESUME", queuedAt: new Date("2026-07-27T11:50:00.000Z") }), + }) + ); + + expect(outcome.facts.queueWaitReliable).toBe(false); + expect(outcome.facts.waitBasis).toBe("created_at"); + // 11:55 -> 12:00, not 11:50 -> 12:00. + expect(outcome.facts.waitMs).toBe(5 * 60_000); + expect(outcome.facts.waitLabel).toBe("waiting to resume; time from creation: 5m"); + }); + + it("says retry, not resume, for a run waiting on a retry", async () => { + const outcome = await check( + runStart, + deps({ + readRun: async () => + run({ + status: "RETRYING_AFTER_FAILURE", + queuedAt: new Date("2026-07-27T11:50:00.000Z"), + }), + }) + ); + + expect(outcome.facts.waitLabel).toBe("waiting to retry; time from creation: 5m"); + }); + + it("is terminal_unsatisfied when the run reached a terminal status without starting", async () => { + const outcome = await check( + runStart, + deps({ readRun: async () => run({ status: "CANCELED" }) }) + ); + + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.facts.reason).toBe("never_started"); + }); + + it("is terminal_unsatisfied when the run is gone from the environment", async () => { + const outcome = await check(runStart, deps({ readRun: async () => null })); + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.facts.reason).toBe("run_not_found"); + }); + + it("is unavailable — never a verdict — when the reader fails", async () => { + const outcome = await check( + runStart, + deps({ + readRun: async () => { + throw new Error("postgres is down"); + }, + }) + ); + + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts.reason).toBe("check_failed"); + }); +}); + +describe("run_finished", () => { + it("is satisfied on a terminal status, with the outcome and execution duration", async () => { + const outcome = await check( + runFinished, + deps({ + readRun: async () => + run({ + status: "COMPLETED_SUCCESSFULLY", + queuedAt: new Date("2026-07-27T11:56:00.000Z"), + startedAt: new Date("2026-07-27T11:57:00.000Z"), + completedAt: new Date("2026-07-27T11:59:30.000Z"), + }), + }) + ); + + expect(outcome.result).toBe("satisfied"); + expect(outcome.facts.outcome).toBe("COMPLETED_SUCCESSFULLY"); + expect(outcome.facts.durationMs).toBe(150_000); + }); + + it("is pending while the run is still executing", async () => { + const outcome = await check( + runFinished, + deps({ + readRun: async () => + run({ status: "EXECUTING", startedAt: new Date("2026-07-27T11:58:00.000Z") }), + }) + ); + + expect(outcome.result).toBe("pending"); + expect(outcome.facts.durationMs).toBeNull(); + }); +}); + +describe("backlog_drain", () => { + it("is satisfied at depth 0", async () => { + const outcome = await check( + backlogDrain, + deps({ readQueueDepth: async () => ({ depth: 0, source: "live_queue", current: true }) }) + ); + + expect(outcome.result).toBe("satisfied"); + expect(outcome.facts).toMatchObject({ depth: 0, depthSource: "live_queue" }); + }); + + it("is pending while runs are still queued", async () => { + const outcome = await check( + backlogDrain, + deps({ + readQueueDepth: async () => ({ depth: 42, source: "queue_metrics", current: true }), + }) + ); + + expect(outcome.result).toBe("pending"); + expect(outcome.facts.depth).toBe(42); + }); + + it("is unavailable — never drained — when a zero comes from a stale bucket", async () => { + const asOf = new Date("2026-07-27T11:50:00.000Z"); + const outcome = await check( + backlogDrain, + deps({ + readQueueDepth: async () => ({ + depth: 0, + source: "queue_metrics", + current: false, + asOf, + }), + }) + ); + + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts).toMatchObject({ + reason: "depth_stale", + depth: 0, + depthAsOf: asOf.toISOString(), + depthApproximate: true, + }); + }); + + it("stays pending on a stale non-zero depth, marked approximate", async () => { + const outcome = await check( + backlogDrain, + deps({ + readQueueDepth: async () => ({ + depth: 7, + source: "queue_metrics", + current: false, + asOf: new Date("2026-07-27T11:50:00.000Z"), + }), + }) + ); + + expect(outcome.result).toBe("pending"); + expect(outcome.facts).toMatchObject({ depth: 7, depthApproximate: true }); + }); + + it("is terminal_unsatisfied when the queue doesn't exist", async () => { + const outcome = await check( + backlogDrain, + deps({ readQueueDepth: async () => null, queueExists: async () => false }) + ); + + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.facts.reason).toBe("queue_not_found"); + }); + + it("is unavailable when the queue exists but its depth can't be read", async () => { + const outcome = await check( + backlogDrain, + deps({ readQueueDepth: async () => null, queueExists: async () => true }) + ); + + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts.reason).toBe("depth_unavailable"); + }); + + it("is unavailable when the depth reader throws", async () => { + const outcome = await check( + backlogDrain, + deps({ + readQueueDepth: async () => { + throw new Error("clickhouse timeout"); + }, + }) + ); + + expect(outcome.result).toBe("unavailable"); + }); +}); + +function recurrence(overrides: Partial = {}): WatchErrorRecurrence { + return { + occurredAt: new Date("2026-07-27T11:30:00.000Z"), + occurredAtPrecision: "minute", + countSince: 3, + countApproximate: false, + lastSeenAt: new Date("2026-07-27T11:45:00.000Z"), + ...overrides, + }; +} + +describe("error_recurrence", () => { + it("is satisfied on the first occurrence after `since`", async () => { + const outcome = await check( + errorRecurrence, + deps({ readErrorRecurrence: async () => recurrence() }) + ); + + expect(outcome.result).toBe("satisfied"); + expect(outcome.facts).toMatchObject({ + occurredAt: "2026-07-27T11:30:00.000Z", + occurredAtPrecision: "minute", + countSince: 3, + countApproximate: false, + since: SINCE.toISOString(), + }); + }); + + it("carries the precision of an occurrence in the watch's creation minute", async () => { + const occurredAt = new Date("2026-07-27T11:00:40.000Z"); + const outcome = await check( + errorRecurrence, + deps({ + readErrorRecurrence: async () => + recurrence({ + occurredAt, + occurredAtPrecision: "exact", + countSince: 1, + countApproximate: true, + lastSeenAt: occurredAt, + }), + }) + ); + + expect(outcome.result).toBe("satisfied"); + expect(outcome.facts).toMatchObject({ + occurredAt: occurredAt.toISOString(), + occurredAtPrecision: "exact", + countSince: 1, + countApproximate: true, + }); + }); + + it("is pending when the error has never been seen at all", async () => { + const outcome = await check(errorRecurrence, deps({ readErrorRecurrence: async () => null })); + expect(outcome.result).toBe("pending"); + expect(outcome.facts).toMatchObject({ countSince: 0, lastSeenAt: null }); + }); + + it("is pending when the error was last seen before `since`", async () => { + const lastSeenAt = new Date("2026-07-27T10:30:00.000Z"); + const outcome = await check( + errorRecurrence, + deps({ + readErrorRecurrence: async () => + recurrence({ occurredAt: null, occurredAtPrecision: null, countSince: 0, lastSeenAt }), + }) + ); + + expect(outcome.result).toBe("pending"); + expect(outcome.facts).toMatchObject({ countSince: 0, lastSeenAt: lastSeenAt.toISOString() }); + }); + + it("passes the watch's `since` to the reader, not the clock", async () => { + let seen: Date | undefined; + await check( + errorRecurrence, + deps({ + readErrorRecurrence: async (_fingerprint, since) => { + seen = since; + return null; + }, + }) + ); + expect(seen).toEqual(SINCE); + }); + + it("is unavailable when the reader throws", async () => { + const outcome = await check( + errorRecurrence, + deps({ + readErrorRecurrence: async () => { + throw new Error("clickhouse down"); + }, + }) + ); + expect(outcome.result).toBe("unavailable"); + }); + + // The model cites the API error id; ClickHouse stores the raw fingerprint. + it("strips the `error_` prefix before reading, and reports the raw fingerprint", async () => { + let seen: string | undefined; + const outcome = await check( + { ...errorRecurrence, fingerprint: "error_abc123" } as WatchSpec, + deps({ + readErrorRecurrence: async (fingerprint) => { + seen = fingerprint; + return null; + }, + }) + ); + + expect(seen).toBe("abc123"); + expect(outcome.facts.fingerprint).toBe("abc123"); + }); + + it("passes a raw fingerprint through unchanged", async () => { + let seen: string | undefined; + await check( + { ...errorRecurrence, fingerprint: "abc123" } as WatchSpec, + deps({ + readErrorRecurrence: async (fingerprint) => { + seen = fingerprint; + return null; + }, + }) + ); + + expect(seen).toBe("abc123"); + }); +}); + +describe("health_recovery", () => { + it("is satisfied when the report is trustworthy and ok", async () => { + const outcome = await check( + healthRecovery, + deps({ readHealth: async () => ({ trustworthy: true, severity: "ok" }) }) + ); + + expect(outcome.result).toBe("satisfied"); + expect(outcome.facts).toMatchObject({ severity: "ok", trustworthy: true }); + }); + + it("is pending while the report is still warn or crit", async () => { + for (const severity of ["warn", "crit"] as const) { + const outcome = await check( + healthRecovery, + deps({ readHealth: async () => ({ trustworthy: true, severity }) }) + ); + expect(outcome.result).toBe("pending"); + expect(outcome.facts.severity).toBe(severity); + } + }); + + it("NEVER fires recovery off an untrustworthy report, even when it says ok", async () => { + const outcome = await check( + healthRecovery, + deps({ readHealth: async () => ({ trustworthy: false, severity: "ok" }) }) + ); + + expect(outcome.result).toBe("pending"); + expect(outcome.facts).toMatchObject({ trustworthy: false, reason: "untrustworthy" }); + }); + + it("is unavailable when the report can't be produced", async () => { + const outcome = await check(healthRecovery, deps({ readHealth: async () => null })); + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts.reason).toBe("report_unavailable"); + }); +}); + +const queueAbove: WatchSpec = { + kind: "queue_depth_above", + queue: "email-sends", + threshold: 500, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it grows past 500", +}; + +describe("run_finished — status awareness", () => { + // Without the final status, "finished" and "failed" are the same `condition_met`. + it("keeps the final status on a completion, whatever it was", async () => { + for (const status of ["COMPLETED_SUCCESSFULLY", "COMPLETED_WITH_ERRORS", "CRASHED"]) { + const outcome = await check( + runFinished, + deps({ + readRun: async () => + run({ + status, + startedAt: new Date("2026-07-27T11:56:00.000Z"), + completedAt: new Date("2026-07-27T11:59:00.000Z"), + }), + }) + ); + expect(outcome.result).toBe("satisfied"); + expect(outcome.observed).toMatchObject({ + kind: "run_finished", + verified: true, + finalStatus: status, + durationMs: 180_000, + }); + } + }); + + it("never claims a final status for a run that is still going", async () => { + const outcome = await check( + runFinished, + deps({ readRun: async () => run({ status: "EXECUTING" }) }) + ); + expect(outcome.result).toBe("pending"); + expect(outcome.observed).toMatchObject({ kind: "run_finished", finalStatus: null }); + }); + + it("observes nothing verifiable when the run is gone", async () => { + const outcome = await check(runFinished, deps({ readRun: async () => null })); + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.observed).toMatchObject({ kind: "run_finished", finalStatus: null }); + }); +}); + +describe("run_failed", () => { + const runFailed: WatchSpec = { + kind: "run_failed", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me if it fails", + }; + + const finished = (status: string) => + run({ + status, + startedAt: new Date("2026-07-27T11:56:00.000Z"), + completedAt: new Date("2026-07-27T11:59:00.000Z"), + }); + + it("is satisfied by a failing terminal status", async () => { + for (const status of ["COMPLETED_WITH_ERRORS", "CRASHED", "SYSTEM_FAILURE", "TIMED_OUT"]) { + const outcome = await check(runFailed, deps({ readRun: async () => finished(status) })); + expect(outcome.result).toBe("satisfied"); + expect(outcome.observed).toMatchObject({ + kind: "run_failed", + verified: true, + finalStatus: status, + durationMs: 180_000, + }); + } + }); + + it("becomes impossible — not pending — once the run succeeds", async () => { + const outcome = await check( + runFailed, + deps({ readRun: async () => finished("COMPLETED_SUCCESSFULLY") }) + ); + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.observed).toMatchObject({ finalStatus: "COMPLETED_SUCCESSFULLY" }); + }); + + it("treats a cancellation as terminal too — it will not fail now", async () => { + const outcome = await check(runFailed, deps({ readRun: async () => finished("CANCELED") })); + expect(outcome.result).toBe("terminal_unsatisfied"); + }); + + it("keeps waiting while the run is still going, with no verdict on the row", async () => { + const outcome = await check( + runFailed, + deps({ readRun: async () => run({ status: "EXECUTING" }) }) + ); + expect(outcome.result).toBe("pending"); + expect(outcome.observed).toMatchObject({ kind: "run_failed", finalStatus: null }); + }); + + it("is unavailable, never a verdict, when the reader throws", async () => { + const outcome = await check( + runFailed, + deps({ + readRun: async () => { + throw new Error("postgres is down"); + }, + }) + ); + expect(outcome.result).toBe("unavailable"); + expect(outcome.observed).toMatchObject({ kind: "run_failed", verified: false }); + }); +}); + +describe("queue_depth_above", () => { + it("is the drain check with the comparison inverted", async () => { + const above = await check( + queueAbove, + deps({ readQueueDepth: async () => ({ depth: 612, source: "live_queue", current: true }) }) + ); + expect(above.result).toBe("satisfied"); + expect(above.observed).toMatchObject({ + kind: "queue_depth_above", + verified: true, + depth: 612, + threshold: 500, + }); + + const below = await check( + queueAbove, + deps({ readQueueDepth: async () => ({ depth: 500, source: "live_queue", current: true }) }) + ); + // Exactly at the threshold is not above it. + expect(below.result).toBe("pending"); + }); + + it("stays pending on an empty queue and is terminal only when the queue is gone", async () => { + const empty = await check( + queueAbove, + deps({ readQueueDepth: async () => ({ depth: 0, source: "live_queue", current: true }) }) + ); + expect(empty.result).toBe("pending"); + + const gone = await check( + queueAbove, + deps({ readQueueDepth: async () => null, queueExists: async () => false }) + ); + expect(gone.result).toBe("terminal_unsatisfied"); + }); + + it("refuses a stale zero, and marks a stale non-zero approximate", async () => { + const stale = await check( + queueAbove, + deps({ + readQueueDepth: async () => ({ + depth: 0, + source: "queue_metrics", + current: false, + asOf: new Date("2026-07-27T11:40:00.000Z"), + }), + }) + ); + expect(stale.result).toBe("unavailable"); + expect(stale.observed).toMatchObject({ kind: "queue_depth_above", verified: false }); + + const staleAbove = await check( + queueAbove, + deps({ + readQueueDepth: async () => ({ + depth: 900, + source: "queue_metrics", + current: false, + asOf: new Date("2026-07-27T11:40:00.000Z"), + }), + }) + ); + expect(staleAbove.result).toBe("satisfied"); + expect(staleAbove.facts).toMatchObject({ depthApproximate: true, threshold: 500 }); + }); + + it("reports the depth it read, so the headline needs no second look", async () => { + const outcome = await check( + queueAbove, + deps({ readQueueDepth: async () => ({ depth: 612, source: "live_queue", current: true }) }) + ); + expect(outcome.observed).toMatchObject({ depth: 612, threshold: 500 }); + }); +}); + +const queueBelow: WatchSpec = { + kind: "queue_depth_below", + queue: "email-sends", + threshold: 100, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it's back below 100", +}; + +const queueStalled: WatchSpec = { + kind: "queue_stalled", + queue: "email-sends", + ticks: 3, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it stops moving", +}; + +const queueAge: WatchSpec = { + kind: "queue_oldest_age", + queue: "email-sends", + thresholdMinutes: 5, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if runs wait longer than 5 minutes", +}; + +describe("queue_depth_below", () => { + it("is satisfied at or under the threshold, and the boundary counts", async () => { + const under = await check(queueBelow, deps({ readQueueDepth: async () => live(42) })); + expect(under.result).toBe("satisfied"); + expect(under.observed).toMatchObject({ + kind: "queue_depth_below", + verified: true, + depth: 42, + threshold: 100, + }); + + // Unlike `above`, the boundary itself answers: 100 is back below 100. + const boundary = await check(queueBelow, deps({ readQueueDepth: async () => live(100) })); + expect(boundary.result).toBe("satisfied"); + + const over = await check(queueBelow, deps({ readQueueDepth: async () => live(101) })); + expect(over.result).toBe("pending"); + }); + + it("never satisfies off a stale reading, however low it looks", async () => { + const outcome = await check(queueBelow, deps({ readQueueDepth: async () => stale(3) })); + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts).toMatchObject({ reason: "depth_stale", depthApproximate: true }); + expect(outcome.observed).toMatchObject({ kind: "queue_depth_below", verified: false }); + }); + + it("stays pending on a stale reading that is still above the line", async () => { + const outcome = await check(queueBelow, deps({ readQueueDepth: async () => stale(400) })); + expect(outcome.result).toBe("pending"); + expect(outcome.facts).toMatchObject({ depth: 400, depthApproximate: true }); + }); + + it("is terminal only when the queue is gone, and unavailable when unreadable", async () => { + const gone = await check( + queueBelow, + deps({ readQueueDepth: async () => null, queueExists: async () => false }) + ); + expect(gone.result).toBe("terminal_unsatisfied"); + expect(gone.facts.reason).toBe("queue_not_found"); + + const unreadable = await check( + queueBelow, + deps({ readQueueDepth: async () => null, queueExists: async () => true }) + ); + expect(unreadable.result).toBe("unavailable"); + }); +}); + +describe("queue_stalled — the stateful check", () => { + const depths = (depth: number) => deps({ readQueueDepth: async () => live(depth) }); + + it("counts checks that watched the depth fail to fall, and fires at K", async () => { + const first = await check(queueStalled, depths(42)); + expect(first.result).toBe("pending"); + expect(first.facts).toMatchObject({ notDecreasingStreak: 0, previousDepth: null, ticks: 3 }); + + const second = await check(queueStalled, depths(42), first.facts); + expect(second.facts.notDecreasingStreak).toBe(1); + + const third = await check(queueStalled, depths(45), second.facts); + expect(third.facts.notDecreasingStreak).toBe(2); + expect(third.result).toBe("pending"); + + const fourth = await check(queueStalled, depths(45), third.facts); + expect(fourth.result).toBe("satisfied"); + expect(fourth.observed).toMatchObject({ + kind: "queue_stalled", + verified: true, + depth: 45, + notDecreasingStreak: 3, + ticks: 3, + }); + }); + + it("resets the streak the moment the queue makes progress", async () => { + const first = await check(queueStalled, depths(42)); + const second = await check(queueStalled, depths(42), first.facts); + expect(second.facts.notDecreasingStreak).toBe(1); + + const moved = await check(queueStalled, depths(30), second.facts); + expect(moved.facts.notDecreasingStreak).toBe(0); + expect(moved.result).toBe("pending"); + }); + + // An `unavailable` tick never overwrites the previous observation, so the streak freezes across the gap. + it("freezes the streak across a data gap and resumes counting after it", async () => { + const first = await check(queueStalled, depths(42)); + const second = await check(queueStalled, depths(42), first.facts); + expect(second.facts.notDecreasingStreak).toBe(1); + + const gap = await check( + queueStalled, + deps({ readQueueDepth: async () => null, queueExists: async () => true }), + second.facts + ); + expect(gap.result).toBe("unavailable"); + expect(gap.observed).toMatchObject({ verified: false, notDecreasingStreak: 1 }); + + const parked = { checkFailed: true, detail: "clickhouse down", previous: second.facts }; + const resumed = await check(queueStalled, depths(42), previousCheckFacts(parked)); + expect(resumed.facts.notDecreasingStreak).toBe(2); + expect(resumed.result).toBe("pending"); + + const fires = await check(queueStalled, depths(42), resumed.facts); + expect(fires.result).toBe("satisfied"); + }); + + it("refuses a stale reading outright rather than sampling it", async () => { + const first = await check(queueStalled, depths(42)); + const outcome = await check( + queueStalled, + deps({ readQueueDepth: async () => stale(42) }), + first.facts + ); + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts.reason).toBe("depth_stale"); + }); + + it("is never satisfied by an empty queue — that is a drain, not a stall", async () => { + const first = await check(queueStalled, depths(0)); + const second = await check(queueStalled, depths(0), first.facts); + const third = await check(queueStalled, depths(0), second.facts); + const fourth = await check(queueStalled, depths(0), third.facts); + expect(fourth.result).toBe("pending"); + expect(fourth.facts.notDecreasingStreak).toBe(0); + }); + + it("starts over rather than trusting junk state", async () => { + for (const previous of [null, {}, { depth: "42" }, { severity: "ok" }]) { + const outcome = await check(queueStalled, depths(42), previous as Record); + expect(outcome.facts.notDecreasingStreak).toBe(0); + expect(outcome.result).toBe("pending"); + } + }); + + it("is terminal when the queue is gone", async () => { + const outcome = await check( + queueStalled, + deps({ readQueueDepth: async () => null, queueExists: async () => false }) + ); + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.facts.reason).toBe("queue_not_found"); + }); +}); + +describe("previousCheckFacts", () => { + it("reads the tick's raw facts, the endpoint's envelope, and unwraps a failure", () => { + const facts = { depth: 42, notDecreasingStreak: 2 }; + expect(previousCheckFacts(facts)).toEqual(facts); + expect(previousCheckFacts({ result: "pending", facts, observed: {}, final: false })).toEqual( + facts + ); + expect( + previousCheckFacts({ + checkFailed: true, + previous: { checkFailed: true, previous: facts }, + }) + ).toEqual(facts); + }); + + it("has no previous observation to offer when there was none", () => { + expect(previousCheckFacts(null)).toBeNull(); + expect(previousCheckFacts(undefined)).toBeNull(); + expect(previousCheckFacts("nonsense")).toBeNull(); + expect(previousCheckFacts([1, 2])).toBeNull(); + expect(previousCheckFacts({ checkFailed: true })).toBeNull(); + }); +}); + +describe("queue_oldest_age", () => { + const age = (ageMs: number | null, current = true) => + deps({ + readQueueOldestAge: async () => ({ + ageMs, + source: "live_queue" as const, + current, + asOf: NOW, + }), + }); + + it("is satisfied once the oldest wait passes the SLA, and not on the boundary", async () => { + const over = await check(queueAge, age(12 * 60_000)); + expect(over.result).toBe("satisfied"); + expect(over.facts).toMatchObject({ ageMs: 720_000, ageLabel: "12m", thresholdMinutes: 5 }); + expect(over.observed).toMatchObject({ + kind: "queue_oldest_age", + verified: true, + ageMs: 720_000, + thresholdMinutes: 5, + }); + + const exactly = await check(queueAge, age(5 * 60_000)); + expect(exactly.result).toBe("pending"); + + const under = await check(queueAge, age(5 * 60_000 - 1)); + expect(under.result).toBe("pending"); + }); + + it("is pending on an empty queue", async () => { + const outcome = await check(queueAge, age(null)); + expect(outcome.result).toBe("pending"); + expect(outcome.observed).toMatchObject({ ageMs: null, verified: true }); + }); + + it("never satisfies, and never clears, off a stale reading", async () => { + const stalePastSla = await check(queueAge, age(60 * 60_000, false)); + expect(stalePastSla.result).toBe("unavailable"); + expect(stalePastSla.facts.reason).toBe("age_stale"); + expect(stalePastSla.observed).toMatchObject({ kind: "queue_oldest_age", verified: false }); + + const staleUnderSla = await check(queueAge, age(1_000, false)); + expect(staleUnderSla.result).toBe("unavailable"); + }); + + it("is terminal when the queue is gone, unavailable when it can't be read", async () => { + const gone = await check( + queueAge, + deps({ readQueueOldestAge: async () => null, queueExists: async () => false }) + ); + expect(gone.result).toBe("terminal_unsatisfied"); + expect(gone.facts.reason).toBe("queue_not_found"); + + const unreadable = await check( + queueAge, + deps({ readQueueOldestAge: async () => null, queueExists: async () => true }) + ); + expect(unreadable.result).toBe("unavailable"); + expect(unreadable.facts.reason).toBe("age_unavailable"); + }); + + it("is unavailable — never a verdict — when the reader throws", async () => { + const outcome = await check( + queueAge, + deps({ + readQueueOldestAge: async () => { + throw new Error("redis is down"); + }, + }) + ); + expect(outcome.result).toBe("unavailable"); + expect(outcome.observed).toMatchObject({ kind: "queue_oldest_age", verified: false }); + }); +}); + +describe("observations", () => { + it("marks the observation unverified when a reader throws", async () => { + const outcome = await check( + runFinished, + deps({ + readRun: async () => { + throw new Error("postgres is down"); + }, + }) + ); + expect(outcome.result).toBe("unavailable"); + expect(outcome.observed).toMatchObject({ kind: "run_finished", verified: false }); + }); + + it("never records a severity off an untrustworthy health report", async () => { + const outcome = await check( + healthRecovery, + deps({ readHealth: async () => ({ trustworthy: false, severity: "ok" }) }) + ); + expect(outcome.result).toBe("pending"); + expect(outcome.observed).toMatchObject({ + kind: "health_recovery", + verified: false, + severity: null, + }); + }); + + it("gives every kind an observation of its own kind", async () => { + const specs: WatchSpec[] = [ + runStart, + runFinished, + backlogDrain, + queueAbove, + queueBelow, + queueStalled, + queueAge, + errorRecurrence, + healthRecovery, + ]; + for (const spec of specs) { + const outcome = await check(spec, deps()); + expect(outcome.observed.kind).toBe(spec.kind); + } + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatchCreationReads.test.ts b/apps/webapp/test/dashboardAgentWatchCreationReads.test.ts new file mode 100644 index 00000000000..8d0c295c951 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchCreationReads.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; + +// The replica lags: it has neither the just-triggered run nor the just-created queue. +const ctx = vi.hoisted(() => ({ + primaryReads: [] as string[], + replicaReads: [] as string[], +})); + +vi.mock("~/db.server", () => ({ + prisma: { + taskQueue: { + findFirst: async () => { + ctx.primaryReads.push("queue"); + return { id: "queue_1" }; + }, + }, + }, + $replica: { + taskQueue: { + findFirst: async () => { + ctx.replicaReads.push("queue"); + return null; + }, + }, + }, + sqlDatabaseSchema: undefined, +})); + +vi.mock("~/v3/runStore.server", () => ({ + runStore: { + findRunOnPrimary: async () => { + ctx.primaryReads.push("run"); + return { friendlyId: "run_1", status: "PENDING" }; + }, + findRun: async () => { + ctx.replicaReads.push("run"); + return null; + }, + }, +})); + +const { watchCheckDeps, watchCreationCheckDeps } = + await import("~/services/dashboardAgentWatchChecks.server"); + +const environment = { id: "env_1" } as AuthenticatedEnvironment; + +beforeEach(() => { + ctx.primaryReads = []; + ctx.replicaReads = []; +}); + +describe("the watch target reads", () => { + test("creation reads the run and the queue on the primary", async () => { + const deps = watchCreationCheckDeps(environment); + + expect(await deps.readRun("run_1")).not.toBeNull(); + expect(await deps.queueExists("task/my-task")).toBe(true); + expect(ctx.primaryReads).toEqual(["run", "queue"]); + expect(ctx.replicaReads).toEqual([]); + }); + + test("polling keeps both reads on the replica", async () => { + const deps = watchCheckDeps(environment); + + expect(await deps.readRun("run_1")).toBeNull(); + expect(await deps.queueExists("task/my-task")).toBe(false); + expect(ctx.replicaReads).toEqual(["run", "queue"]); + expect(ctx.primaryReads).toEqual([]); + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts b/apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts new file mode 100644 index 00000000000..95e8eb839d7 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts @@ -0,0 +1,195 @@ +/** + * The errors page cites an error as `error_` and the agent's own tools cite the + * bare fingerprint. Both name the same error, so both must produce the same watch identity — + * otherwise one error carries two watches, two wakes and two emails per recurrence. + */ + +import { + createChat, + createDashboardAgentDb, + getWatch, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +const { createDashboardAgentWatch } = await import("~/services/dashboardAgentWatches.server"); + +/** Replays every migration in order, so a new migration can't leave this file on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +async function seed(prisma: PrismaClient) { + const slug = `fingerprint_${suffix()}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `pr${suffix()}`, + }, + }); + + const chatId = `chat_${suffix()}`; + await createChat(ctx.agentDb, { id: chatId, organizationId: organization.id, userId: user.id }); + + return { user, organization, project, environment, chatId }; +} + +type Seeded = Awaited>; + +function authenticated(seeded: Seeded) { + return { + id: seeded.environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: seeded.environment.slug, + type: seeded.environment.type, + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; +} + +/** Quiet so far, so a recurrence watch is created rather than answered on the spot. */ +function checkDeps(): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: async () => true, + readQueueDepth: async () => null, + readQueueOldestAge: async () => null, + readErrorRecurrence: async () => null, + readHealth: async () => null, + }; +} + +const FINGERPRINT = "9f3c1ab27de4"; + +function recurrenceSpec(fingerprint: string): WatchSpec { + return { + kind: "error_recurrence", + fingerprint, + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me if it comes back", + }; +} + +function createFor(seeded: Seeded, fingerprint: string) { + return createDashboardAgentWatch({ + environment: authenticated(seeded), + userId: seeded.user.id, + chatId: seeded.chatId, + spec: recurrenceSpec(fingerprint), + deps: { + configured: () => true, + checkDeps: () => checkDeps(), + scheduleTick: async () => {}, + }, + }); +} + +describe("a watch on one error", () => { + postgresTest( + "stores the bare fingerprint, whichever spelling the caller cites", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma); + + const created = await createFor(seeded, `error_${FINGERPRINT}`); + expect(created).toMatchObject({ + ok: true, + watching: true, + identity: `error_recurrence:${FINGERPRINT}`, + }); + if (!created.ok || !created.watching) return; + + // The wake builds its `trigger://` link straight from the stored spec. + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(row?.spec).toMatchObject({ fingerprint: FINGERPRINT }); + } + ); + + postgresTest( + "is one watch, not two, when the two spellings are both submitted", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma); + + const first = await createFor(seeded, `error_${FINGERPRINT}`); + expect(first).toMatchObject({ ok: true, watching: true }); + + const second = await createFor(seeded, FINGERPRINT); + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchInvestigate.test.ts b/apps/webapp/test/dashboardAgentWatchInvestigate.test.ts new file mode 100644 index 00000000000..599f3ab6d8c --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchInvestigate.test.ts @@ -0,0 +1,335 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; + +const ctx = vi.hoisted(() => ({ + claims: { watchId: "watch_1" } as undefined | { watchId: string }, + watch: undefined as any, + authorized: true, + appends: [] as Array<{ sessionId: string; io: string; body: any }>, + appendThrows: false, +})); + +vi.mock("~/env.server", () => ({ + env: { DASHBOARD_AGENT_SECRET_KEY: "tr_dashboard_agent", APP_ORIGIN: "https://app.example.com" }, +})); + +vi.mock("~/services/logger.server", () => ({ + logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }, +})); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ dashboardAgentDb: {} })); + +vi.mock("@internal/dashboard-agent-db", async (importOriginal) => ({ + ...((await importOriginal()) as any), + getWatch: async () => ctx.watch, +})); + +vi.mock("~/services/dashboardAgentWatchToken.server", () => ({ + bearerToken: (request: Request) => + request.headers.get("authorization")?.replace(/^Bearer /, "") ?? undefined, + verifyWatchTokenFromRequest: async () => ctx.claims, +})); + +vi.mock("~/services/dashboardAgentWatches.server", () => ({ + authorizeWatchEnvironment: async () => + ctx.authorized + ? { + ok: true, + environment: { + id: "env_1", + type: "PRODUCTION", + project: { id: "project_1", externalRef: "proj_from_env" }, + }, + } + : { ok: false, reason: "access_revoked" }, +})); + +const mints = vi.hoisted(() => [] as Array<{ userId: string; environmentId?: string }>); +vi.mock("~/services/dashboardAgent.server", () => ({ + dashboardAgentApiOrigin: () => "https://api.example.com", + dashboardAgentEnvironmentName: (type: string | undefined) => + type === "PRODUCTION" ? "prod" : undefined, + mintDashboardAgentUserActorToken: async ( + userId: string, + opts: { environmentId?: string } = {} + ) => { + mints.push({ userId, environmentId: opts.environmentId }); + return `uat_for_${userId}`; + }, + resolveDashboardAgentRepoSnapshot: async () => null, +})); + +vi.mock("@trigger.dev/core/v3", async (importOriginal) => ({ + ...((await importOriginal()) as any), + ApiClient: class { + constructor( + public baseUrl: string, + public accessToken: string + ) {} + async appendToSessionStream(sessionId: string, io: string, body: string) { + if (ctx.appendThrows) throw new Error("session not found"); + ctx.appends.push({ sessionId, io, body: JSON.parse(body) }); + return { ok: true }; + } + }, +})); + +vi.mock("~/services/session.server", () => ({ + requireUser: async () => ({ id: "user_1", admin: false, isImpersonating: false }), +})); +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => ({ id: "project_1", externalRef: "proj_1" }), +})); +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + sqlDatabaseSchema: undefined, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => ({ id: "env_1", type: "PRODUCTION" }), +})); + +const { action } = await import("~/routes/api.v1.dashboard-agent.watches.$watchId.investigate"); +const { action: inProxyAction } = + await import("~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboard-agent.in.$"); + +function watchRow(overrides: Record = {}) { + return { + id: "watch_1", + chatId: "chat_1", + identity: "run_finished:run_a1", + spec: { + kind: "run_finished", + runId: "run_a1", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when the receipt run finishes", + }, + status: "fired", + deliveryStatus: "delivered", + resolution: "condition_met", + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 4200, + }, + investigateOnAttention: true, + organizationId: "org_1", + projectId: "project_1", + projectRef: "proj_1", + environmentId: "env_1", + userId: "user_1", + lastResult: { outcome: "COMPLETED_WITH_ERRORS", durationMs: 4200 }, + firedAt: new Date("2026-01-01T12:00:00.000Z"), + ...overrides, + }; +} + +function post(watchId = "watch_1") { + return action({ + request: new Request( + `https://app.example.com/api/v1/dashboard-agent/watches/${watchId}/investigate`, + { method: "POST", headers: { Authorization: "Bearer watch_token" }, body: "{}" } + ), + params: { watchId }, + context: {} as never, + } as never) as Promise; +} + +beforeEach(() => { + ctx.claims = { watchId: "watch_1" }; + ctx.watch = watchRow(); + ctx.authorized = true; + ctx.appends = []; + ctx.appendThrows = false; + mints.length = 0; +}); + +describe("POST /api/v1/dashboard-agent/watches/:watchId/investigate", () => { + it("sends the investigate action with a token minted for the watch's own user and environment", async () => { + const res = await post(); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, investigating: true }); + + expect(ctx.appends).toHaveLength(1); + const append = ctx.appends[0]!; + expect(append.sessionId).toBe("chat_1"); + expect(append.io).toBe("in"); + expect(append.body.payload.trigger).toBe("action"); + expect(append.body.payload.action).toMatchObject({ + type: "watch.investigate", + // Stable per (watch, outcome), so a retried kick is a no-op on the agent side. + id: "watch:watch_1:fired:investigate", + watchId: "watch_1", + identity: "run_finished:run_a1", + resolution: "condition_met", + note: "tell me when the receipt run finishes", + }); + expect(append.body.payload.action.observed.finalStatus).toBe("COMPLETED_WITH_ERRORS"); + + expect(append.body.payload.metadata).toMatchObject({ + userId: "user_1", + organizationId: "org_1", + projectId: "project_1", + projectRef: "proj_1", + environmentId: "env_1", + environmentName: "prod", + userActorToken: "uat_for_user_1", + }); + // Minted for the row's user and the row's environment, never anything a request body names. + expect(mints).toEqual([{ userId: "user_1", environmentId: "env_1" }]); + }); + + it("investigates an attention outcome that expired, not just a fired one", async () => { + ctx.watch = watchRow({ + spec: { + kind: "backlog_drain", + queue: "task/send-receipt", + checkEveryMinutes: 5, + maxHours: 2, + }, + identity: "backlog_drain:task/send-receipt", + status: "expired", + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: true, depth: 412 }, + }); + + const res = await post(); + + expect(await res.json()).toEqual({ ok: true, investigating: true }); + expect(ctx.appends[0]!.body.payload.action.id).toBe("watch:watch_1:expired:investigate"); + }); + + it("starts nothing on a positive outcome, consent or not", async () => { + ctx.watch = watchRow({ + observedOutcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_SUCCESSFULLY", + }, + }); + + const res = await post(); + + expect(await res.json()).toEqual({ ok: true, investigating: false }); + expect(ctx.appends).toHaveLength(0); + expect(mints).toHaveLength(0); + }); + + it("starts nothing without the consent", async () => { + ctx.watch = watchRow({ investigateOnAttention: false }); + + const res = await post(); + + expect(await res.json()).toEqual({ ok: true, investigating: false }); + expect(ctx.appends).toHaveLength(0); + }); + + it("refuses a watch that hasn't resolved", async () => { + ctx.watch = watchRow({ status: "active", resolution: null, deliveryStatus: "not_required" }); + + const res = await post(); + + expect(res.status).toBe(409); + expect((await res.json()).code).toBe("not_resolved"); + expect(ctx.appends).toHaveLength(0); + }); + + it("mints nothing once access has been revoked", async () => { + ctx.authorized = false; + + const res = await post(); + + expect(res.status).toBe(403); + expect((await res.json()).code).toBe("access_revoked"); + expect(mints).toHaveLength(0); + expect(ctx.appends).toHaveLength(0); + }); + + it("rejects a token minted for another watch", async () => { + ctx.claims = { watchId: "watch_other" }; + + const res = await post(); + + expect(res.status).toBe(403); + expect((await res.json()).code).toBe("watch_mismatch"); + expect(ctx.appends).toHaveLength(0); + }); + + it("401s without a valid watch token", async () => { + ctx.claims = undefined; + + const res = await post(); + + expect(res.status).toBe(401); + expect(ctx.appends).toHaveLength(0); + }); + + it("404s when the row is gone", async () => { + ctx.watch = null; + + const res = await post(); + + expect(res.status).toBe(404); + }); + + it("does not fail when the kick itself fails", async () => { + ctx.appendThrows = true; + + const res = await post(); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, investigating: false, code: "kick_failed" }); + }); +}); + +describe("the dashboard-agent in proxy", () => { + const upstream = vi.fn(async () => new Response("{}", { status: 200 })); + + function append(payload: Record) { + return inProxyAction({ + request: new Request( + "https://app.example.com/resources/orgs/acme/projects/checkout/env/prod/dashboard-agent/in/realtime/v1/sessions/chat_1/in/append", + { method: "POST", body: JSON.stringify({ kind: "message", payload }) } + ), + params: { + organizationSlug: "acme", + projectParam: "checkout", + envParam: "prod", + "*": "realtime/v1/sessions/chat_1/in/append", + }, + context: {} as never, + } as never) as Promise; + } + + beforeEach(() => { + upstream.mockClear(); + vi.stubGlobal("fetch", upstream); + }); + + it("refuses a browser-supplied action, forwarding nothing and minting nothing", async () => { + const res = await append({ + chatId: "chat_1", + trigger: "action", + action: { type: "watch.investigate", id: "forged", watchId: "watch_1", spec: { kind: "x" } }, + }); + + expect(res.status).toBe(403); + expect(upstream).not.toHaveBeenCalled(); + expect(mints).toHaveLength(0); + }); + + it("still forwards a normal turn, with the token injected", async () => { + const res = await append({ chatId: "chat_1", trigger: "submit-message" }); + + expect(res.status).toBe(200); + expect(upstream).toHaveBeenCalledTimes(1); + const body = JSON.parse((upstream.mock.calls[0]![1] as RequestInit).body as string); + expect(body.payload.metadata.userActorToken).toBe("uat_for_user_1"); + expect(mints).toEqual([{ userId: "user_1", environmentId: "env_1" }]); + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatchQueueAge.test.ts b/apps/webapp/test/dashboardAgentWatchQueueAge.test.ts new file mode 100644 index 00000000000..f3b0dbaca30 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchQueueAge.test.ts @@ -0,0 +1,112 @@ +/** + * A wait-time watch has to be able to end. The reader must say "unavailable" when the engine + * can't answer rather than report a healthy zero, and a queue that no longer exists has to + * resolve the watch instead of leaving it pending for its whole window. + */ + +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; + +const ctx = vi.hoisted(() => ({ + breakdown: null as null | { keys: Array<{ queued: number; oldestEnqueuedAt: number }> }, + breakdownThrows: false, + oldest: undefined as number | undefined, + oldestThrows: false, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + concurrencyKeyBreakdown: async () => { + if (ctx.breakdownThrows) throw new Error("redis is down"); + return ctx.breakdown ?? { keys: [] }; + }, + oldestMessageInQueue: async () => { + if (ctx.oldestThrows) throw new Error("redis is down"); + return ctx.oldest; + }, + }, +})); + +const { readWatchQueueOldestAge } = await import("~/services/dashboardAgentWatchChecks.server"); +const { checkWatch } = await import("~/services/dashboardAgentWatchChecks"); + +const NOW = new Date("2026-08-07T12:00:00.000Z"); +const environment = { + id: "env_1", + organizationId: "org_1", + projectId: "proj_1", +} as AuthenticatedEnvironment; + +const SPEC: WatchSpec = { + kind: "queue_oldest_age", + queue: "task/send-receipt", + thresholdMinutes: 5, + checkEveryMinutes: 5, + maxHours: 1, + note: "tell me if anything waits too long", +}; + +function deps(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: async () => true, + readQueueDepth: async () => null, + readQueueOldestAge: (queueName: string) => readWatchQueueOldestAge(environment, queueName, NOW), + readErrorRecurrence: async () => null, + readHealth: async () => null, + ...overrides, + }; +} + +beforeEach(() => { + ctx.breakdown = null; + ctx.breakdownThrows = false; + ctx.oldest = undefined; + ctx.oldestThrows = false; +}); + +describe("the wait-time reading", () => { + test("is unavailable when an engine read fails, not a zero wait", async () => { + ctx.oldestThrows = true; + expect(await readWatchQueueOldestAge(environment, "task/send-receipt", NOW)).toBeNull(); + + ctx.oldestThrows = false; + ctx.breakdownThrows = true; + expect(await readWatchQueueOldestAge(environment, "task/send-receipt", NOW)).toBeNull(); + }); + + test("reports an empty queue as a reading with no age", async () => { + expect(await readWatchQueueOldestAge(environment, "task/send-receipt", NOW)).toMatchObject({ + ageMs: null, + source: "live_queue", + current: true, + }); + }); +}); + +describe("a wait-time watch on a queue that is no longer there", () => { + test("resolves terminally instead of sitting pending", async () => { + const outcome = await checkWatch(SPEC, deps({ queueExists: async () => false }), { + now: NOW, + since: NOW, + }); + + expect(outcome.result).toBe("terminal_unsatisfied"); + expect(outcome.facts).toMatchObject({ reason: "queue_not_found" }); + }); + + test("stays pending while the queue exists and is simply empty", async () => { + const outcome = await checkWatch(SPEC, deps(), { now: NOW, since: NOW }); + expect(outcome.result).toBe("pending"); + }); + + test("is unavailable, not terminal, when the engine is down but the queue exists", async () => { + ctx.breakdownThrows = true; + const outcome = await checkWatch(SPEC, deps(), { now: NOW, since: NOW }); + + expect(outcome.result).toBe("unavailable"); + expect(outcome.facts).toMatchObject({ reason: "age_unavailable" }); + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatchQueueName.test.ts b/apps/webapp/test/dashboardAgentWatchQueueName.test.ts new file mode 100644 index 00000000000..dae122c0ce6 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchQueueName.test.ts @@ -0,0 +1,203 @@ +/** + * The queue detail page presents a task queue's name with the `task/` prefix stripped, but + * `TaskQueue.name` keeps it — so the name the page hands a watch has to be the stored one, + * or every task-queue watch is refused as a missing target. + */ + +import { + createChat, + createDashboardAgentDb, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-queue-names"; + +const { createDashboardAgentWatch } = await import("~/services/dashboardAgentWatches.server"); +const { watchQueueExistsOnPrimary } = await import("~/services/dashboardAgentWatchChecks.server"); +const { storedQueueName } = await import("~/components/queues/queue-name"); +const { queueWatchRecommendation } = + await import("~/components/dashboard-agent/watch-recommendations"); + +/** Replays every migration in order, so a new migration can't leave this file on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +/** One org, project and production environment, holding one virtual queue for `send-receipt`. */ +async function seed(prisma: PrismaClient) { + const slug = `queue_name_${suffix()}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `pr${suffix()}`, + }, + }); + // A task's own queue, exactly as `createBackgroundWorker` writes it. + await prisma.taskQueue.create({ + data: { + friendlyId: `queue_${suffix()}`, + name: "task/send-receipt", + orderableName: "task/send-receipt", + type: "VIRTUAL", + projectId: project.id, + runtimeEnvironmentId: environment.id, + }, + }); + + await createChat(ctx.agentDb, { + id: `chat_${suffix()}`, + organizationId: organization.id, + userId: user.id, + }); + + return { user, organization, project, environment }; +} + +type Seeded = Awaited>; + +function authenticated(seeded: Seeded) { + return { + id: seeded.environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: seeded.environment.slug, + type: seeded.environment.type, + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; +} + +/** Only `queueExists` is real: it is the read the target validation is decided by. */ +function checkDeps(seeded: Seeded): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: (name: string) => watchQueueExistsOnPrimary(seeded.environment.id, name), + readQueueDepth: async () => ({ depth: 3, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ ageMs: 1_000, source: "live_queue", current: true }), + readErrorRecurrence: async () => null, + readHealth: async () => null, + }; +} + +async function createFor(seeded: Seeded, spec: WatchSpec) { + const chatId = `chat_${suffix()}`; + await createChat(ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + return createDashboardAgentWatch({ + environment: authenticated(seeded), + userId: seeded.user.id, + chatId, + spec, + deps: { + configured: () => true, + checkDeps: () => checkDeps(seeded), + scheduleTick: async () => {}, + }, + }); +} + +/** The queue as `QueueRetrievePresenter` hands it to the page: the prefix already stripped. */ +const PRESENTED = { type: "task", name: "send-receipt" }; + +describe("a watch on a task's own queue", () => { + postgresTest( + "validates, because the page sends the stored name", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma); + + const spec = queueWatchRecommendation(storedQueueName(PRESENTED), { oldestWaitMs: null }); + expect(spec).toMatchObject({ queue: "task/send-receipt" }); + + const created = await createFor(seeded, spec); + expect(created).toMatchObject({ ok: true, watching: true }); + } + ); + + postgresTest( + "is refused when the display name reaches the spec instead", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma); + + const created = await createFor(seeded, queueWatchRecommendation(PRESENTED.name)); + expect(created).toMatchObject({ ok: false, code: "invalid_target" }); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts b/apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts new file mode 100644 index 00000000000..e85ab277c7c --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts @@ -0,0 +1,210 @@ +/** + * A watch the sweep finalizes fires once, and the wake it then schedules reports the same + * fired watch to the fire callback. Both fan out an alert, so exactly one of them may win + * the dispatch claim — the count of enqueued alerts is what this file asserts. + */ + +import { + createChat, + createDashboardAgentDb, + createWatch, + type DashboardAgentDb, + type DashboardAgentDbClient, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +const SESSION_SECRET = "test-session-secret-for-watch-sweep-alerts"; +process.env.SESSION_SECRET = SESSION_SECRET; + +const { sweepDashboardAgentWatches } = await import("~/services/dashboardAgentWatchSweep.server"); +const { action: firedAction } = + await import("~/routes/api.v1.dashboard-agent.watches.$watchId.fired"); +const { signDashboardAgentWatchToken } = await import("~/services/dashboardAgentWatchToken.server"); +const { alertsWorker } = await import("~/v3/alertsWorker.server"); + +const enqueue = alertsWorker.enqueue as unknown as ReturnType; + +/** Replays every migration in order, so a new migration can't leave this file on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +beforeEach(() => { + enqueue.mockClear(); +}); + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +/** A real org, project, environment and member: the sweep and the callback both re-authorize. */ +async function seed(prisma: PrismaClient) { + const slug = `sweep_alert_${suffix()}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `pr${suffix()}`, + }, + }); + + const chatId = `chat_${suffix()}`; + await createChat(ctx.agentDb, { + id: chatId, + organizationId: organization.id, + userId: user.id, + }); + + return { user, organization, project, environment, chatId }; +} + +/** Below its threshold on the boundary check, so the sweep resolves it `condition_met`. */ +const DRAINING: WatchSpec = { + kind: "queue_depth_below", + queue: "task/send-receipt", + threshold: 10, + checkEveryMinutes: 5, + maxHours: 1, + note: "tell me when it drains", +}; + +function checkDeps(): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 0, source: "live_queue", current: true }), + readQueueOldestAge: async () => null, + readErrorRecurrence: async () => null, + readHealth: async () => null, + }; +} + +function firedRequest(watchId: string, token: string) { + return { + request: new Request( + `https://app.trigger.dev/api/v1/dashboard-agent/watches/${watchId}/fired`, + { method: "POST", headers: { Authorization: `Bearer ${token}` } } + ), + params: { watchId }, + context: {} as never, + } as never; +} + +function alertCalls() { + return enqueue.mock.calls.filter( + (call) => (call[0] as { job: string }).job === "v3.deliverDashboardAgentWatchAlert" + ); +} + +describe("a watch the sweep finalizes", () => { + postgresTest( + "alerts exactly once, however the wake reports it afterwards", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma); + + const created = await createWatch(ctx.agentDb, { + chatId: seeded.chatId, + identity: "queue_depth_below:task/send-receipt:10", + spec: DRAINING as never, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + userId: seeded.user.id, + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + }); + if (!created.ok) throw new Error(`the watch wasn't created: ${created.error}`); + + // Past the sweep's grace window, so this run owns the final evaluation. + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, + created.watch.id + ); + + const token = await signDashboardAgentWatchToken(SESSION_SECRET, { + watchId: created.watch.id, + expiresAt: new Date(Date.now() + 60_000), + }); + + // The wake the sweep schedules ends at the fire callback, which fans out again. + const result = await sweepDashboardAgentWatches({ + checkDeps: () => checkDeps(), + configured: () => true, + deliver: async (watch: Watch) => { + const response = (await firedAction(firedRequest(watch.id, token))) as Response; + expect(response.status).toBe(200); + }, + }); + + expect(result).toMatchObject({ overdue: 1, fired: 1, failed: 0 }); + expect(alertCalls()).toHaveLength(1); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts b/apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts new file mode 100644 index 00000000000..bfb97ba2a15 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts @@ -0,0 +1,316 @@ +import { + createChat, + createDashboardAgentDb, + createWatch, + getWatch, + recordWatchCheck, + type DashboardAgentDb, + type DashboardAgentDbClient, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; + +// The boundary evaluation the sweep runs is a real check: it sees the streak the ticks +// built, and one incident's worth of expiries must not re-read the same authorization. + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +vi.mock("~/services/dashboardAgentWatchAlerts.server", () => ({ + DASHBOARD_AGENT_WATCH_ALERT_TYPE: "DASHBOARD_AGENT_WATCH", + enqueueWatchFiredAlert: async () => {}, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-sweep"; + +const { sweepDashboardAgentWatches } = await import("~/services/dashboardAgentWatchSweep.server"); + +/** Replays every migration in order, so a new migration can't leave the suite on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + for (const name of readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort()) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +const ORGANIZATION_ID = "org_sweep"; +const PROJECT_ID = "proj_sweep"; +const ENVIRONMENT_ID = "env_sweep"; +const USER_ID = "user_sweep"; + +const environment = { + id: ENVIRONMENT_ID, + organizationId: ORGANIZATION_ID, + projectId: PROJECT_ID, + slug: "prod", + type: "PRODUCTION", + project: { id: PROJECT_ID, externalRef: "proj_external" }, + organization: { id: ORGANIZATION_ID, slug: "sweep" }, +} as any; + +/** The stall spec the boundary check has to decide: three no-progress checks in a row. */ +const STALLED: WatchSpec = { + kind: "queue_stalled", + queue: "task/send-receipt", + ticks: 3, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if the queue stops moving", +}; + +async function seedOverdueWatch(args: { + chatId: string; + spec?: WatchSpec; + identity?: string; + userId?: string; + lastResult?: Record; +}): Promise { + const created = await createWatch(ctx.agentDb, { + chatId: args.chatId, + identity: args.identity ?? `queue_stalled:${args.chatId}`, + spec: (args.spec ?? STALLED) as any, + organizationId: ORGANIZATION_ID, + projectId: PROJECT_ID, + environmentId: ENVIRONMENT_ID, + userId: args.userId ?? USER_ID, + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + }); + if (!created.ok) throw new Error(`the watch wasn't created: ${created.error}`); + + if (args.lastResult) { + await recordWatchCheck(ctx.agentDb, { id: created.watch.id, lastResult: args.lastResult }); + } + // Past the sweep's grace window, so this run owns the final evaluation. + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, + created.watch.id + ); + return created.watch.id; +} + +function checkDepsWithDepth(depth: number): WatchCheckDeps { + return { + readRun: async () => null, + queueExists: async () => true, + readQueueDepth: async () => ({ depth, source: "live_queue", current: true }), + readQueueOldestAge: async () => null, + readErrorRecurrence: async () => null, + readHealth: async () => null, + }; +} + +describe("the sweep's boundary evaluation", () => { + postgresTest( + "fires a stall the final check completes, because it carries the streak the ticks built", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + await createChat(ctx.agentDb, { + id: "chat_stall", + organizationId: ORGANIZATION_ID, + userId: USER_ID, + }); + + // Two no-progress checks are already recorded, so a third at the same depth is the + // stall: the transition happens exactly on the window boundary. + const watchId = await seedOverdueWatch({ + chatId: "chat_stall", + lastResult: { + result: "pending", + facts: { + queue: "task/send-receipt", + depth: 412, + notDecreasingStreak: 2, + ticks: 3, + }, + }, + }); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches({ + checkDeps: () => checkDepsWithDepth(412), + authorize: async () => ({ ok: true, environment }) as const, + deliver: async (watch: Watch) => void delivered.push(watch.id), + configured: () => true, + }); + + expect(result).toMatchObject({ overdue: 1, fired: 1, expired: 0, failed: 0 }); + const row = await getWatch(ctx.agentDb, { id: watchId }); + expect(row).toMatchObject({ status: "fired", resolution: "condition_met" }); + // The facts the wake narrates are the streak that closed, not a reset one. + const facts = row?.lastResult as { notDecreasingStreak?: number } | undefined; + expect(facts?.notDecreasingStreak).toBe(3); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "still completes the window when the streak is one short of the stall", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + await createChat(ctx.agentDb, { + id: "chat_no_stall", + organizationId: ORGANIZATION_ID, + userId: USER_ID, + }); + + const watchId = await seedOverdueWatch({ + chatId: "chat_no_stall", + lastResult: { + result: "pending", + facts: { queue: "task/send-receipt", depth: 412, notDecreasingStreak: 1, ticks: 3 }, + }, + }); + + const result = await sweepDashboardAgentWatches({ + checkDeps: () => checkDepsWithDepth(412), + authorize: async () => ({ ok: true, environment }) as const, + deliver: async () => {}, + configured: () => true, + }); + + expect(result).toMatchObject({ overdue: 1, fired: 0, expired: 1, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + resolution: "window_completed", + }); + } + ); +}); + +describe("the sweep's reads per incident", () => { + postgresTest( + "re-authorizes and builds readers once per user and environment, whatever the group's size", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + + // One chat each, because a chat caps its own active watches. Same user and + // environment throughout: that is the pair the sweep must only resolve once. + for (let index = 0; index < 6; index++) { + await createChat(ctx.agentDb, { + id: `chat_incident_${index}`, + organizationId: ORGANIZATION_ID, + userId: USER_ID, + }); + await seedOverdueWatch({ + chatId: `chat_incident_${index}`, + identity: `queue_stalled:queue_${index}`, + spec: { ...STALLED, queue: `task/queue_${index}` }, + }); + } + + let authorizations = 0; + let readerBuilds = 0; + let inFlight = 0; + let peakInFlight = 0; + + const result = await sweepDashboardAgentWatches({ + concurrency: 3, + authorize: async () => { + authorizations++; + return { ok: true, environment } as const; + }, + checkDeps: () => { + readerBuilds++; + return { + ...checkDepsWithDepth(7), + readQueueDepth: async () => { + inFlight++; + peakInFlight = Math.max(peakInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 20)); + inFlight--; + return { depth: 7, source: "live_queue", current: true }; + }, + }; + }, + deliver: async () => {}, + configured: () => true, + }); + + expect(result).toMatchObject({ overdue: 6, expired: 6, fired: 0, failed: 0 }); + // One authorization and one set of readers for the whole group. + expect(authorizations).toBe(1); + expect(readerBuilds).toBe(1); + // Bounded, so one slow tenant can't hold the visibility window open. + expect(peakInFlight).toBeGreaterThan(1); + expect(peakInFlight).toBeLessThanOrEqual(3); + } + ); + + postgresTest( + "authorizes each initiating user separately", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + for (const suffix of ["a", "b"]) { + await createChat(ctx.agentDb, { + id: `chat_${suffix}`, + organizationId: ORGANIZATION_ID, + userId: `user_${suffix}`, + }); + await seedOverdueWatch({ + chatId: `chat_${suffix}`, + userId: `user_${suffix}`, + identity: `queue_stalled:queue_${suffix}`, + }); + } + + const authorized: string[] = []; + await sweepDashboardAgentWatches({ + authorize: async (watch: Watch) => { + authorized.push(watch.userId); + return { ok: true, environment } as const; + }, + checkDeps: () => checkDepsWithDepth(7), + deliver: async () => {}, + configured: () => true, + }); + + expect([...authorized].sort()).toEqual(["user_a", "user_b"]); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchTenancy.test.ts b/apps/webapp/test/dashboardAgentWatchTenancy.test.ts new file mode 100644 index 00000000000..de39103a584 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchTenancy.test.ts @@ -0,0 +1,527 @@ +/** + * The tenancy floor of the watch submission ledger, plus the two boundaries that hang off + * it: the fire callback's alert (once per terminal outcome) and the alert unsubscribe + * (the caller's own channel only). + */ + +import { + createChat, + createDashboardAgentDb, + getChatMessages, + getWatchSubmission, + transitionWatchCondition, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import type { WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps, WatchRunRow } from "~/services/dashboardAgentWatchChecks"; + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + actor: undefined as undefined | { userId: string; client?: string; environmentId?: string }, +})); + +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => true, +})); + +const SESSION_SECRET = "test-session-secret-for-watch-tenancy"; +process.env.SESSION_SECRET = SESSION_SECRET; +// The subscribe path refuses outright without an email transport configured. +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; + +const { submitDashboardAgentWatch } = await import("~/services/dashboardAgentWatches.server"); +const { action: firedAction } = + await import("~/routes/api.v1.dashboard-agent.watches.$watchId.fired"); +const { action: alertChannelAction } = + await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); +const { signDashboardAgentWatchToken } = await import("~/services/dashboardAgentWatchToken.server"); +const { DASHBOARD_AGENT_WATCH_ALERT_TYPE, watchAlertDeduplicationKey } = + await import("~/services/dashboardAgentWatchAlerts.server"); +const { alertsWorker } = await import("~/v3/alertsWorker.server"); + +const enqueue = alertsWorker.enqueue as unknown as ReturnType; + +/** Replays every migration in order, so a new migration can't leave this file on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + const migrations = readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort(); + for (const name of migrations) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + ctx.actor = undefined; + await applyAgentSchema(prisma); + agentDbClient = createDashboardAgentDb(connectionUri, { max: 4 }); + ctx.agentDb = agentDbClient.db; +} + +beforeEach(() => { + enqueue.mockClear(); +}); + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +/** One organization, one project, one production environment, and a user who is a member. */ +async function seedOrg(prisma: PrismaClient, slugBase: string, user?: { id: string }) { + const slug = `${slugBase}_${suffix()}`; + const owner = + user ?? + (await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + })); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: owner.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await environmentFor(prisma, organization.id, project.id, slug, "prod"); + return { user: owner, organization, project, environment }; +} + +async function environmentFor( + prisma: PrismaClient, + organizationId: string, + projectId: string, + slugBase: string, + slug: "prod" | "staging" +) { + return prisma.runtimeEnvironment.create({ + data: { + slug, + type: slug === "prod" ? "PRODUCTION" : "STAGING", + projectId, + organizationId, + apiKey: `tr_${slug}_${slugBase}`, + pkApiKey: `pk_${slug}_${slugBase}`, + shortcode: `${slug.slice(0, 2)}${suffix()}`, + }, + }); +} + +type Seeded = Awaited>; + +function authenticated(seeded: Seeded, environment = seeded.environment) { + return { + id: environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: environment.slug, + type: environment.type, + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; +} + +const RUN_START: WatchSpec = { + kind: "run_start", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it starts", +}; + +function draftFor(followUp: Partial = {}): WatchDraft { + return { + spec: RUN_START, + followUp: { investigateOnAttention: false, notifyExternally: false, ...followUp }, + }; +} + +function runRow(): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date(), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + }; +} + +/** Keeps the condition pending with a live target, so a submit always creates a watch. */ +function fakeCheckDeps(): WatchCheckDeps { + return { + readRun: async () => runRow(), + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ ageMs: 30_000, source: "live_queue", current: true }), + readErrorRecurrence: async () => null, + readHealth: async () => ({ trustworthy: true, severity: "warn" }), + }; +} + +function submit(args: { + seeded: Seeded; + environment?: { id: string; slug: string; type: string }; + chatId?: string; + clientRequestId?: string; + draft?: WatchDraft; + /** Omitted: the real subscribe runs, so a test can assert on real channels. */ + subscribe?: (args: unknown) => Promise<{ ok: boolean; reason?: string; email?: string }>; +}) { + return submitDashboardAgentWatch({ + environment: authenticated(args.seeded, args.environment as never), + userId: args.seeded.user.id, + organizationId: args.seeded.organization.id, + chatId: args.chatId, + clientRequestId: args.clientRequestId ?? "wreq_1", + draft: args.draft ?? draftFor(), + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(), + scheduleTick: async () => {}, + ...(args.subscribe ? { subscribe: args.subscribe as never } : {}), + }, + }); +} + +function messagesIn(chatId: string, organizationId: string, userId: string) { + return getChatMessages(ctx.agentDb, { chatId, userId, organizationId }) as Promise | null>; +} + +describe("the submission ledger's tenancy", () => { + postgresTest( + "one user, one request id, two organizations: two chats, and neither transcript holds the other's record", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const first = await seedOrg(prisma, "tenancy-a"); + // The same person, in a second organization. `clientRequestId` is client-chosen, so + // both organizations can send the same one. + const second = await seedOrg(prisma, "tenancy-b", first.user); + + const inFirst = await submit({ seeded: first, clientRequestId: "wreq_shared" }); + const inSecond = await submit({ seeded: second, clientRequestId: "wreq_shared" }); + + // A shared chat id would have let the second organization's records land in the + // first organization's chat, so this is asserted before anything else. + const chatIdOf = (result: { chatId?: string }) => result.chatId; + expect(chatIdOf(inSecond)).not.toBe(chatIdOf(inFirst)); + + expect(inFirst.ok && inSecond.ok).toBe(true); + if (!inFirst.ok || !inSecond.ok) return; + + const firstChat = await messagesIn(inFirst.chatId, first.organization.id, first.user.id); + const secondChat = await messagesIn(inSecond.chatId, second.organization.id, second.user.id); + expect(firstChat).toHaveLength(2); + expect(secondChat).toHaveLength(2); + + // Each chat belongs to exactly one organization, so neither is readable as the other. + expect(await messagesIn(inSecond.chatId, first.organization.id, first.user.id)).toBeNull(); + expect(await messagesIn(inFirst.chatId, second.organization.id, second.user.id)).toBeNull(); + }, + 30_000 + ); + + postgresTest( + "the same chat and request id in a second environment is refused, not replayed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedOrg(prisma, "tenancy-env"); + const staging = await environmentFor( + prisma, + seeded.organization.id, + seeded.project.id, + `env_${suffix()}`, + "staging" + ); + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + + const production = await submit({ seeded, chatId: "chat_1" }); + expect(production.ok).toBe(true); + if (!production.ok) return; + + // A chat spans environments by design, so the draft matching is not enough: this + // would otherwise replay production's watch as staging's answer. + const inStaging = await submit({ seeded, chatId: "chat_1", environment: staging }); + expect(inStaging).toMatchObject({ ok: false, code: "request_conflict" }); + + const recorded = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(recorded).toMatchObject({ environmentId: seeded.environment.id }); + }, + 30_000 + ); + + postgresTest( + "flipping the email consent under the same request id conflicts and subscribes nobody", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedOrg(prisma, "tenancy-consent"); + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + + const withoutEmail = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ notifyExternally: false }), + }); + expect(withoutEmail.ok).toBe(true); + + // The durable record in the transcript says "chat only". A retry may not quietly + // turn email on behind it. + const withEmail = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ notifyExternally: true }), + }); + expect(withEmail).toMatchObject({ ok: false, code: "request_conflict" }); + + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(0); + }, + 30_000 + ); + + postgresTest( + "an email the user asked for and didn't get is stated, and replayed the same way", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedOrg(prisma, "tenancy-email"); + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + + const failing = async () => ({ ok: false, reason: "email_alerts_not_configured" }); + const created = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ notifyExternally: true }), + subscribe: failing, + }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const line = + "I couldn't add email notifications, so updates will appear in the dashboard only."; + expect(JSON.stringify(created.messages[1])).toContain(line); + + // Normalised on the row, so the replay says the same thing rather than guessing. + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ + externalNotificationStatus: "unavailable", + externalNotificationReason: "email_alerts_not_configured", + }); + + const retry = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ notifyExternally: true }), + subscribe: failing, + }); + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(JSON.stringify(retry.messages[1])).toContain(line); + }, + 30_000 + ); +}); + +describe("the fire callback", () => { + function firedRequest(watchId: string, token: string) { + return { + request: new Request( + `https://app.trigger.dev/api/v1/dashboard-agent/watches/${watchId}/fired`, + { method: "POST", headers: { Authorization: `Bearer ${token}` } } + ), + params: { watchId }, + context: {} as never, + } as never; + } + + postgresTest( + "sends exactly one alert however many times it is called", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedOrg(prisma, "fired-once"); + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + + const created = await submit({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok || !created.watchId) return; + + await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + resolution: "condition_met", + }); + + const token = await signDashboardAgentWatchToken(SESSION_SECRET, { + watchId: created.watchId, + expiresAt: new Date(Date.now() + 60_000), + }); + + const first = (await firedAction(firedRequest(created.watchId, token))) as Response; + expect(first.status).toBe(200); + expect(await first.json()).toMatchObject({ ok: true, alerted: true }); + + // The same token, the same row: a token holder must not be able to mail the user + // again by repeating the call. + const second = (await firedAction(firedRequest(created.watchId, token))) as Response; + expect(second.status).toBe(200); + expect(await second.json()).toMatchObject({ ok: true, alerted: false }); + + const alerts = enqueue.mock.calls.filter( + (call) => (call[0] as { job: string }).job === "v3.deliverDashboardAgentWatchAlert" + ); + expect(alerts).toHaveLength(1); + }, + 30_000 + ); +}); + +describe("the alert unsubscribe", () => { + async function seedMember(prisma: PrismaClient, seeded: Seeded) { + const member = await prisma.user.create({ + data: { email: `member_${suffix()}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, + }); + return member; + } + + async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { + return prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${suffix()}`, + name: `Watch alerts for ${email}`, + projectId: seeded.project.id, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email }, + deduplicationKey: watchAlertDeduplicationKey(email), + }, + }); + } + + function deleteRequest(channelId: string, chatId: string) { + return { + request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { + method: "DELETE", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify({ chatId }), + }), + params: { channelId }, + context: {} as never, + } as never; + } + + postgresTest( + "a member can't turn off another member's watch alerts in the same project", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seedOrg(prisma, "unsub-owner"); + const caller = await seedMember(prisma, seeded); + const other = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_caller", + organizationId: seeded.organization.id, + userId: caller.id, + }); + + const own = await seedWatchChannel(prisma, seeded, caller.email); + const theirs = await seedWatchChannel(prisma, seeded, other.email); + + ctx.actor = { + userId: caller.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + // Same project, same organization, a different owner: out of scope. + const refused = (await alertChannelAction( + deleteRequest(theirs.id, "chat_caller") + )) as Response; + expect(refused.status).toBe(404); + expect( + await prisma.projectAlertChannel.findFirst({ where: { id: theirs.id } }) + ).toMatchObject({ enabled: true, alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE] }); + + // The caller's own channel still comes off. + const removed = (await alertChannelAction(deleteRequest(own.id, "chat_caller"))) as Response; + expect(removed.status).toBe(200); + expect(await prisma.projectAlertChannel.findFirst({ where: { id: own.id } })).toMatchObject({ + enabled: false, + alertTypes: [], + }); + }, + 30_000 + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatchToken.test.ts b/apps/webapp/test/dashboardAgentWatchToken.test.ts new file mode 100644 index 00000000000..7fbf7178eb3 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchToken.test.ts @@ -0,0 +1,106 @@ +import { signUserActorToken, verifyUserActorToken } from "@trigger.dev/rbac"; +import { describe, expect, it } from "vitest"; +import { + WATCH_TOKEN_GRACE_MS, + WATCH_TOKEN_PREFIX, + isDashboardAgentWatchToken, + signDashboardAgentWatchToken, + verifyDashboardAgentWatchToken, +} from "~/services/dashboardAgentWatchToken.server"; + +const SECRET = "test-session-secret-for-watch-tokens"; +const USER_ACTOR_PREFIX = "tr_uat_"; + +function inAnHour(): Date { + return new Date(Date.now() + 60 * 60 * 1000); +} + +describe("dashboard agent watch tokens", () => { + it("round-trips the watch id", async () => { + const expiresAt = inAnHour(); + const token = await signDashboardAgentWatchToken(SECRET, { watchId: "watch_abc", expiresAt }); + + expect(isDashboardAgentWatchToken(token)).toBe(true); + + const claims = await verifyDashboardAgentWatchToken(SECRET, token); + expect(claims?.watchId).toBe("watch_abc"); + // exp = expiresAt + the grace window, to the second. + expect(claims?.expiresAtSeconds).toBe( + Math.floor((expiresAt.getTime() + WATCH_TOKEN_GRACE_MS) / 1000) + ); + }); + + it("is deterministic, so the scheduler can re-mint instead of storing it", async () => { + const expiresAt = inAnHour(); + const a = await signDashboardAgentWatchToken(SECRET, { watchId: "watch_abc", expiresAt }); + const b = await signDashboardAgentWatchToken(SECRET, { watchId: "watch_abc", expiresAt }); + expect(a).toBe(b); + }); + + it("rejects another secret's signature", async () => { + const token = await signDashboardAgentWatchToken(SECRET, { + watchId: "watch_abc", + expiresAt: inAnHour(), + }); + expect(await verifyDashboardAgentWatchToken("a-different-secret", token)).toBeUndefined(); + }); + + it("stays valid through the grace window and dies after it", async () => { + // expiresAt just passed: the token still verifies, because the final check happens after the deadline. + const justExpired = new Date(Date.now() - 60_000); + const graceful = await signDashboardAgentWatchToken(SECRET, { + watchId: "watch_abc", + expiresAt: justExpired, + }); + expect(await verifyDashboardAgentWatchToken(SECRET, graceful)).toMatchObject({ + watchId: "watch_abc", + }); + + const longGone = await signDashboardAgentWatchToken(SECRET, { + watchId: "watch_abc", + expiresAt: new Date(Date.now() - WATCH_TOKEN_GRACE_MS - 60_000), + }); + expect(await verifyDashboardAgentWatchToken(SECRET, longGone)).toBeUndefined(); + }); + + describe("cross-rejection with user-actor tokens", () => { + it("the UAT verifier rejects a watch token", async () => { + const watchToken = await signDashboardAgentWatchToken(SECRET, { + watchId: "watch_abc", + expiresAt: inAnHour(), + }); + expect(await verifyUserActorToken(SECRET, watchToken)).toBeUndefined(); + }); + + it("the watch verifier rejects a user-actor token", async () => { + const uat = await signUserActorToken(SECRET, { + userId: "user_1", + client: "dashboard-agent", + cap: ["read:runs"], + }); + expect(await verifyDashboardAgentWatchToken(SECRET, uat)).toBeUndefined(); + }); + + it("re-prefixing a UAT as a watch token doesn't help — the kind claim disagrees", async () => { + const uat = await signUserActorToken(SECRET, { + userId: "user_1", + client: "dashboard-agent-watch", + cap: ["read:runs"], + }); + const disguised = `${WATCH_TOKEN_PREFIX}${uat.slice(USER_ACTOR_PREFIX.length)}`; + + expect(isDashboardAgentWatchToken(disguised)).toBe(true); + expect(await verifyDashboardAgentWatchToken(SECRET, disguised)).toBeUndefined(); + }); + + it("re-prefixing a watch token as a UAT doesn't help either", async () => { + const watchToken = await signDashboardAgentWatchToken(SECRET, { + watchId: "watch_abc", + expiresAt: inAnHour(), + }); + const disguised = `${USER_ACTOR_PREFIX}${watchToken.slice(WATCH_TOKEN_PREFIX.length)}`; + + expect(await verifyUserActorToken(SECRET, disguised)).toBeUndefined(); + }); + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatchWording.test.ts b/apps/webapp/test/dashboardAgentWatchWording.test.ts new file mode 100644 index 00000000000..efcf837f590 --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatchWording.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from "vitest"; +import { + WATCH_KINDS, + watchIdentity, + watchResolutions, + type WatchKind, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { + noteFor, + presentResolvedWatch, + renderBlockAsText, + renderResolvedWatchAsText, + watchConditionLabel, + watchConfirmationBlockBody, + watchNoteLine, + watchTooltipLabel, +} from "~/presenters/v3/dashboardAgent"; + +/** One spec per kind, so the table below covers every watch the product offers. */ +function specFor(kind: WatchKind): WatchSpec { + const common = { maxHours: 1, note: "", checkEveryMinutes: 5 } as const; + switch (kind) { + case "run_start": + case "run_finished": + case "run_failed": + return { ...common, kind, runId: "run_abc123", checkEveryMinutes: 1 }; + case "backlog_drain": + return { ...common, kind, queue: "email-sends" }; + case "queue_depth_above": + case "queue_depth_below": + return { ...common, kind, queue: "email-sends", threshold: 500 }; + case "queue_stalled": + return { ...common, kind, queue: "email-sends", ticks: 3 }; + case "queue_oldest_age": + return { ...common, kind, queue: "email-sends", thresholdMinutes: 90 }; + case "error_recurrence": + return { ...common, kind, fingerprint: "a1b2c3d4e5f6" }; + case "health_recovery": + return { ...common, kind, report: "health", fromSeverity: "crit" }; + } +} + +describe("the watch presenter", () => { + // Every kind's four registers in one snapshot: the card line, the tooltip, the + // note and the confirmation headline all come from `watchConditionWording`, so a + // change to one of them shows up here rather than drifting on one surface. + it("says each condition the same way on every surface", () => { + const table = WATCH_KINDS.map((kind) => { + const spec = { ...specFor(kind), note: noteFor(specFor(kind)) }; + return { + kind, + label: watchConditionLabel(spec), + tooltip: watchTooltipLabel(spec), + note: spec.note, + confirmation: watchConfirmationBlockBody({ spec, watchId: "watch_1" }).headline, + }; + }); + + expect(table).toMatchSnapshot(); + }); + + // The wake banner, the toast, the email subject and the Slack line all render + // `presentResolvedWatch`, so every kind x resolution pair has exactly one + // sentence. A missing cell would throw rather than fall through to silence. + it("gives every kind and resolution one headline", () => { + const table = WATCH_KINDS.flatMap((kind) => + watchResolutions.map((resolution) => { + const spec = specFor(kind); + const presented = presentResolvedWatch({ + kind, + identity: watchIdentity(spec), + resolution, + }); + return { + kind, + resolution, + headline: presented.headline, + category: presented.category, + tone: presented.tone, + }; + }) + ); + + expect(table).toMatchSnapshot(); + }); + + it("quotes the note in one sentence, and says nothing when there is no note", () => { + expect(watchNoteLine("tell me when run run_abc123 finishes")).toBe( + "You asked to be told when: tell me when run run_abc123 finishes" + ); + expect(watchNoteLine(" ")).toBeNull(); + }); + + it("restates the SLA the way the card does, in the note too", () => { + const spec = specFor("queue_oldest_age"); + expect(noteFor(spec)).toBe("tell me if runs in email-sends wait longer than 1h 30m"); + expect(watchConditionLabel(spec)).toBe("If runs wait longer than 1h 30m"); + }); +}); + +describe("renderBlockAsText", () => { + it("renders a watch-result block as the lines the card shows", () => { + const spec = specFor("backlog_drain"); + const block = { + ...watchConfirmationBlockBody({ + spec: { ...spec, note: noteFor(spec) }, + watchId: "watch_1", + followUp: { external: { status: "enabled" } }, + }), + id: "watch_1", + revision: 0, + version: 1, + }; + + expect(renderBlockAsText(block)).toBe( + [ + "Watching email-sends until the queue drains.", + "Checking every 5 min for up to 1 hour. It reports once, then stops.", + "You'll get an email as well as the chat.", + ].join("\n") + ); + }); + + it("renders a confirmation whose email couldn't be attached", () => { + const spec = specFor("backlog_drain"); + const block = { + ...watchConfirmationBlockBody({ + spec: { ...spec, note: noteFor(spec) }, + watchId: "watch_1", + followUp: { external: { status: "unavailable", reason: "email_alerts_not_configured" } }, + }), + id: "watch_1", + revision: 0, + version: 1, + }; + + expect(renderBlockAsText(block)).toBe( + [ + "Watching email-sends until the queue drains.", + "Checking every 5 min for up to 1 hour. It reports once, then stops.", + "I couldn't add email notifications, so updates will appear in the dashboard only.", + ].join("\n") + ); + }); + + it("renders an actions block as its labels", () => { + expect( + renderBlockAsText({ + type: "actions", + actions: [{ label: "Set up a watch", intent: { kind: "ask", prompt: "watch it" } }], + }) + ).toBe("- Set up a watch"); + }); + + it("renders a resolved watch as headline, note and facts", () => { + const spec = specFor("run_finished"); + expect( + renderResolvedWatchAsText({ + resolved: { + kind: "run_finished", + identity: watchIdentity(spec), + resolution: "condition_met", + }, + note: "tell me when run run_abc123 finishes", + facts: [{ label: "Status", value: "COMPLETED_SUCCESSFULLY" }], + }) + ).toBe( + [ + "Run run_abc123 finished", + "You asked to be told when: tell me when run run_abc123 finishes", + "Status: COMPLETED_SUCCESSFULLY", + ].join("\n") + ); + }); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.test.ts b/apps/webapp/test/dashboardAgentWatches.test.ts new file mode 100644 index 00000000000..6615975d17c --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.test.ts @@ -0,0 +1,3268 @@ +import { + appendChatMessageOnce, + armWatchBatch, + cancelWatch, + chatExists, + claimWatchBatchTick, + claimWatchDelivery, + claimWatchTick, + getWatchSubmission, + listActiveWatchesForBatch, + listWatchBatchGroupsToArm, + stopWatchBatch, + countUnreadWatchWakes, + countUserMessages, + createChat, + createDashboardAgentDb, + getChatMessages, + getWatch, + listActiveWatchesForChat, + listChatIdsWithUnreadWakes, + listRecentWatchWakes, + markWatchDelivered, + readWatchWakeFeed, + recordWatchCheck, + recordWatchSubmissionOutcome, + releaseWatchDelivery, + transitionWatchCondition, + WATCH_DELIVERY_CLAIM_STALE_MS, + type DashboardAgentDb, + type DashboardAgentDbClient, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import { + previousCheckFacts, + type WatchCheckDeps, + type WatchRunRow, +} from "~/services/dashboardAgentWatchChecks"; + +// Every test here boots a container and replays the migrations inside its own budget, +// which does not fit vitest's 5s default on a loaded CI host. +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted(() => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined as undefined | { userId: string; client?: string; environmentId?: string }, +})); + +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.canAccess, +})); + +const SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.SESSION_SECRET = SESSION_SECRET; +// The agent's subscribe endpoint refuses without an email transport configured. +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; + +const { + armDashboardAgentWatchBatch, + authorizeWatchEnvironment, + createDashboardAgentWatch, + deleteChatWithWatches, + listActiveWatchesForChats, + submitDashboardAgentWatch, + watchBatchStaleMs, +} = await import("~/services/dashboardAgentWatches.server"); +const { action: checkAction } = + await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); +const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); +const { action: batchCheckAction } = + await import("~/routes/api.v1.dashboard-agent.watches.batch-check"); +const { + rearmDashboardAgentWatchBatches, + sweepDashboardAgentWatches, + WATCH_DELIVERY_GRACE_MS, + WATCH_EXPIRY_GRACE_MS, +} = await import("~/services/dashboardAgentWatchSweep.server"); +const { runWatchBatchCheck } = await import("~/services/dashboardAgentWatchBatch.server"); +const { signDashboardAgentWatchBatchToken, signDashboardAgentWatchToken } = + await import("~/services/dashboardAgentWatchToken.server"); +const { loader: alertsLoader, action: alertsAction } = + await import("~/routes/api.v1.dashboard-agent.alerts"); +const { action: alertChannelAction } = + await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); +const { findProjectBySlug } = await import("~/models/project.server"); +const { DASHBOARD_AGENT_WATCH_ALERT_TYPE, subscribeUserToWatchAlerts } = + await import("~/services/dashboardAgentWatchAlerts.server"); + +/** Replays every migration in order, so a new migration can't leave the suite on a stale schema. */ +async function applyAgentSchema(prisma: PrismaClient) { + const folder = path.resolve(__dirname, "../../../internal-packages/dashboard-agent-db/drizzle"); + const migrations = readdirSync(folder) + .filter((file) => file.endsWith(".sql")) + .sort(); + for (const name of migrations) { + const sql = readFileSync(path.join(folder, name), "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed.length > 0) await prisma.$executeRawUnsafe(trimmed); + } + } +} + +let agentDbClient: DashboardAgentDbClient | undefined; + +async function boot(prisma: PrismaClient, connectionUri: string) { + ctx.prisma = prisma; + await applyAgentSchema(prisma); + // A pool, not a single connection: the concurrent-create test needs the advisory lock to span connections. + agentDbClient = createDashboardAgentDb(connectionUri, { max: 8 }); + ctx.agentDb = agentDbClient.db; +} + +async function seed(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 6)}`, + }, + }); + return { user, organization, project, environment }; +} + +type Seeded = Awaited>; + +function authenticated(seeded: Seeded) { + return { + id: seeded.environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: "prod", + type: "PRODUCTION", + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; +} + +async function seedChat(seeded: Seeded, chatId = "chat_1") { + await createChat(ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + return chatId; +} + +function runRow(overrides: Partial = {}): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date(), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + ...overrides, + }; +} + +/** Injected readers. Defaults keep every condition pending with a live target. */ +function fakeCheckDeps(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => runRow(), + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ ageMs: 30_000, source: "live_queue", current: true }), + readErrorRecurrence: async () => null, + readHealth: async () => ({ trustworthy: true, severity: "warn" }), + ...overrides, + }; +} + +const RUN_START: WatchSpec = { + kind: "run_start", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it starts", +}; + +const BACKLOG: WatchSpec = { + kind: "backlog_drain", + queue: "task/my-task", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it drains", +}; + +/** A run that exists for target validation and is gone when the immediate check reads it. */ +function readRunOnce(first: WatchRunRow) { + let calls = 0; + return async () => (calls++ === 0 ? first : null); +} + +function create(args: { + seeded: Seeded; + spec?: WatchSpec; + chatId?: string; + environmentId?: string; + investigateOnAttention?: boolean; + watchId?: string; + checkDeps?: Partial; + scheduled?: Array<{ watchId: string; token: string; tick: number }>; + onSchedule?: () => void; +}) { + const environment = authenticated(args.seeded); + return createDashboardAgentWatch({ + environment: args.environmentId ? { ...environment, id: args.environmentId } : environment, + userId: args.seeded.user.id, + chatId: args.chatId ?? "chat_1", + spec: args.spec ?? RUN_START, + investigateOnAttention: args.investigateOnAttention, + watchId: args.watchId, + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(args.checkDeps), + scheduleTick: async (params) => { + args.onSchedule?.(); + args.scheduled?.push({ + watchId: params.watchId, + token: params.token, + tick: params.tick, + }); + }, + }, + }); +} + +beforeEach(() => { + ctx.canAccess = true; + ctx.actor = undefined; +}); + +afterEach(async () => { + await agentDbClient?.close(); + agentDbClient = undefined; +}); + +describe("createDashboardAgentWatch", () => { + postgresTest( + "creates an active watch and schedules its first tick", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const scheduled: Array<{ watchId: string; token: string; tick: number }> = []; + const result = await create({ seeded, scheduled }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.status).toBe("active"); + expect(result.identity).toBe("run_start:run_1"); + expect(result.immediate).toBeUndefined(); + + expect(scheduled).toHaveLength(1); + expect(scheduled[0]!.watchId).toBe(result.watchId); + expect(scheduled[0]!.tick).toBe(1); + expect(scheduled[0]!.token.startsWith("tr_daw_")).toBe(true); + + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + expect(row).toMatchObject({ + status: "active", + deliveryStatus: "not_required", + environmentId: seeded.environment.id, + projectId: seeded.project.id, + organizationId: seeded.organization.id, + userId: seeded.user.id, + tickCount: 0, + investigateOnAttention: false, + projectRef: seeded.project.externalRef, + }); + } + ); + + postgresTest( + "records the investigate-on-attention consent when the caller asks for it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ seeded, investigateOnAttention: true }); + + expect(result.ok).toBe(true); + if (!result.ok || !result.watching) return; + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + expect(row?.investigateOnAttention).toBe(true); + expect(result.identity).toBe("run_start:run_1"); + } + ); + + postgresTest( + "stamps a server-set `since` on an error_recurrence watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const before = Date.now(); + const result = await create({ + seeded, + spec: { + kind: "error_recurrence", + fingerprint: "fp_1", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it comes back", + }, + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + const since = (row?.spec as { since?: string } | undefined)?.since; + expect(since).toBeDefined(); + expect(new Date(since!).getTime()).toBeGreaterThanOrEqual(before - 1000); + } + ); + + postgresTest( + "answers with a one-shot result and writes no row when the condition already holds", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + let ticks = 0; + const result = await create({ + seeded, + checkDeps: { + readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), + }, + onSchedule: () => { + ticks += 1; + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok || result.watching) throw new Error("expected a one-shot result"); + expect(result.immediate.result).toBe("satisfied"); + expect(result.immediate.observed).toMatchObject({ kind: "run_start", started: true }); + expect(ticks).toBe(0); + + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + expect( + await listActiveWatchesForChats({ + chatIds: ["chat_1"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toEqual({}); + } + ); + + postgresTest( + "answers with a one-shot result when the condition can no longer happen", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + checkDeps: { readRun: readRunOnce(runRow({ status: "QUEUED" })) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok || result.watching) throw new Error("expected a one-shot result"); + expect(result.immediate.result).toBe("terminal_unsatisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a duplicate before running the immediate check", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const first = await create({ seeded }); + expect(first.ok).toBe(true); + + let checks = 0; + const second = await create({ + seeded, + checkDeps: { + readRun: async () => { + checks += 1; + return runRow({ status: "EXECUTING", startedAt: new Date() }); + }, + }, + }); + + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + expect(checks).toBe(1); + } + ); + + postgresTest( + "cancels the row silently when the first tick can't be scheduled", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + onSchedule: () => { + throw new Error("no agent project"); + }, + }); + + expect(result).toMatchObject({ ok: false, code: "internal" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + const rows = await ctx.prisma.$queryRawUnsafe< + { status: string; cancel_reason: string; delivery_status: string }[] + >( + `select status, cancel_reason, delivery_status + from trigger_dashboard_agent.watches where chat_id = 'chat_1'` + ); + expect(rows).toMatchObject([ + { + status: "cancelled", + cancel_reason: "scheduling_failed", + delivery_status: "not_required", + }, + ]); + } + ); + + postgresTest( + "rejects a target that doesn't exist, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: BACKLOG, + checkDeps: { queueExists: async () => false }, + }); + + expect(result).toMatchObject({ ok: false, code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "dedups the same condition and allows it in another environment", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const first = await create({ seeded }); + expect(first.ok).toBe(true); + + const second = await create({ seeded }); + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + if (!second.ok && first.ok) expect(second.existingId).toBe(first.watchId); + + const otherEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "stg", + type: "STAGING", + projectId: seeded.project.id, + organizationId: seeded.organization.id, + apiKey: `tr_stg_${seeded.project.slug}`, + pkApiKey: `pk_stg_${seeded.project.slug}`, + shortcode: `s${seeded.project.slug.slice(0, 6)}`, + }, + }); + const third = await create({ seeded, environmentId: otherEnv.id }); + expect(third.ok).toBe(true); + } + ); + + postgresTest( + "refuses a 4th active watch in the same chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + for (const runId of ["run_1", "run_2", "run_3"]) { + const created = await create({ seeded, spec: { ...RUN_START, runId } }); + expect(created.ok).toBe(true); + } + + const fourth = await create({ seeded, spec: { ...RUN_START, runId: "run_4" } }); + expect(fourth).toMatchObject({ ok: false, code: "limit_reached" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); + } + ); + + postgresTest( + "holds the ≤3 limit against four concurrent creates", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + await seedChat(seeded); + + const results = await Promise.all( + ["run_1", "run_2", "run_3", "run_4"].map((runId) => + create({ seeded, spec: { ...RUN_START, runId } }) + ) + ); + + expect(results.filter((result) => result.ok)).toHaveLength(3); + expect( + results.filter((result) => !result.ok && result.code === "limit_reached") + ).toHaveLength(1); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); + } + ); +}); + +describe("the createWatch endpoint's authorization", () => { + function post(body: unknown) { + return createAction({ + request: new Request("https://example.com/api/v1/dashboard-agent/watches", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {}, + }); + } + + const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); + + postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const response = await post(validBody("chat_1")); + expect(response.status).toBe(401); + }); + + postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "adapter"); + ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "forbidden_client" }); + }); + + postgresTest( + "refuses a chat the authenticated user doesn't own, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const owner = await seed(prisma, "owner"); + const stranger = await seed(prisma, "stranger"); + await createChat(ctx.agentDb, { + id: "chat_victim", + organizationId: owner.organization.id, + userId: owner.user.id, + }); + + ctx.actor = { + userId: stranger.user.id, + client: "dashboard-agent", + environmentId: stranger.environment.id, + }; + + const response = await post(validBody("chat_victim")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( + 0 + ); + } + ); + + postgresTest( + "refuses a token with no environment scope", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "noscope"); + await seedChat(seeded, "chat_1"); + ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a body naming a different environment than the token's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "mismatch"); + const other = await seed(prisma, "othermismatch"); + await seedChat(seeded, "chat_1"); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + environmentId: other.environment.id, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "binds to the token's environment, not the chat's stored context", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "binding"); + const otherProject = await prisma.project.create({ + data: { + name: `${seeded.project.slug}_b`, + slug: `${seeded.project.slug}_b`, + organizationId: seeded.organization.id, + externalRef: `proj_${seeded.project.slug}_b`, + }, + }); + const otherEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: otherProject.id, + organizationId: seeded.organization.id, + apiKey: `tr_prod_${otherProject.slug}`, + pkApiKey: `pk_prod_${otherProject.slug}`, + shortcode: `b${otherProject.slug.slice(0, 6)}`, + }, + }); + + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + metadata: { + context: { + environmentId: seeded.environment.id, + projectRef: seeded.project.externalRef, + }, + }, + }); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: otherEnvironment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + projectRef: seeded.project.externalRef, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses an environment in another org than the chat's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "crossorg"); + const other = await seed(prisma, "otherorg"); + await prisma.orgMember.create({ + data: { + organizationId: other.organization.id, + userId: seeded.user.id, + role: "ADMIN", + }, + }); + await seedChat(seeded, "chat_1"); + + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: other.environment.id, + }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); +}); + +describe("the chat cascade and the list view", () => { + postgresTest( + "deleting a chat soft-deletes it and cancels its active watches in one call", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "cascade"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const mine = await create({ seeded, chatId: "chat_1" }); + const theirs = await create({ seeded, chatId: "chat_2" }); + expect(mine.ok && theirs.ok).toBe(true); + if (!mine.ok || !theirs.ok) return; + + expect(await deleteChatWithWatches({ chatId: "chat_1", userId: seeded.user.id })).toEqual({ + deleted: true, + cancelledWatches: 1, + }); + + expect( + await chatExists(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toBe(false); + expect(await getWatch(ctx.agentDb, { id: mine.watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "chat_deleted", + deliveryStatus: "not_required", + }); + expect(await getWatch(ctx.agentDb, { id: theirs.watchId })).toMatchObject({ + status: "active", + }); + } + ); + + postgresTest( + "aggregates active watches per chat in one query", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "chips"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const a = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_1" } }); + const b = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_2" } }); + const c = await create({ seeded, chatId: "chat_2" }); + expect(a.ok && b.ok && c.ok).toBe(true); + + const byChat = await listActiveWatchesForChats({ + chatIds: ["chat_1", "chat_2", "chat_missing"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + expect(byChat.chat_1).toHaveLength(2); + expect(byChat.chat_2).toHaveLength(1); + expect(byChat.chat_missing).toBeUndefined(); + expect(byChat.chat_2![0]).toMatchObject({ + identity: "run_start:run_1", + status: "active", + kind: "run_start", + note: RUN_START.note, + }); + + if (a.ok) await cancelWatch(ctx.agentDb, { id: a.watchId, reason: "user" }); + if (b.ok) await cancelWatch(ctx.agentDb, { id: b.watchId, reason: "user" }); + expect( + ( + await listActiveWatchesForChats({ + chatIds: ["chat_1"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).chat_1 + ).toBeUndefined(); + } + ); + + postgresTest("returns nothing for an empty chat list", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + expect( + await listActiveWatchesForChats({ chatIds: [], organizationId: "org_x", userId: "user_x" }) + ).toEqual({}); + }); +}); + +describe("unread watch wakes", () => { + postgresTest( + "only signals a wake once its delivery landed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "unread"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const scope = { organizationId: seeded.organization.id, userId: seeded.user.id }; + const recent = { ...scope, deliveredAfter: new Date(Date.now() - 15 * 60 * 1000) }; + + if (!created.watching) throw new Error("expected a watch"); + await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + resolution: "condition_met", + }); + expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(0); + expect(await listRecentWatchWakes(ctx.agentDb, recent)).toEqual([]); + expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set()); + + await markWatchDelivered(ctx.agentDb, { id: created.watchId }); + expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(1); + expect(await listRecentWatchWakes(ctx.agentDb, recent)).toMatchObject([ + { watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }, + ]); + expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set(["chat_1"])); + + // The poll's single query answers both halves the same way. + expect(await readWatchWakeFeed(ctx.agentDb, recent)).toMatchObject({ + unreadWakes: 1, + wakes: [{ watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }], + }); + + // An unread wake from before the window still counts, but isn't narrated again. + expect( + await readWatchWakeFeed(ctx.agentDb, { + ...scope, + deliveredAfter: new Date(Date.now() + 60_000), + }) + ).toMatchObject({ unreadWakes: 1, wakes: [] }); + } + ); +}); + +describe("authorizeWatchEnvironment", () => { + postgresTest( + "passes for a member and fails once membership is gone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + + const params = { + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + }; + + expect((await authorizeWatchEnvironment(params)).ok).toBe(true); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + expect(await authorizeWatchEnvironment(params)).toEqual({ + ok: false, + reason: "access_revoked", + }); + } + ); + + postgresTest("fails when the feature gate is revoked", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + ctx.canAccess = false; + + expect( + await authorizeWatchEnvironment({ + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + }) + ).toEqual({ ok: false, reason: "access_revoked" }); + }); + + postgresTest( + "fails when the snapshot names a different project", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + const other = await seed(prisma, "other"); + + expect( + await authorizeWatchEnvironment({ + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: other.project.id, + environmentId: seeded.environment.id, + }) + ).toEqual({ ok: false, reason: "access_revoked" }); + } + ); +}); + +describe("the watch sweep", () => { + async function overdueWatch(seeded: Seeded, chatId = "chat_1") { + const created = await create({ seeded, chatId }); + if (!created.ok) throw new Error("the watch wasn't created"); + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, + created.watchId + ); + return created.watchId; + } + + function sweepDeps(args: { + seeded: Seeded; + checkDeps?: Partial; + revoked?: boolean; + now?: Date; + failDelivery?: boolean; + delivered: string[]; + }) { + return { + now: () => args.now ?? new Date(), + checkDeps: () => fakeCheckDeps(args.checkDeps), + authorize: async () => + args.revoked + ? ({ ok: false, reason: "access_revoked" } as const) + : ({ ok: true, environment: authenticated(args.seeded) } as const), + deliver: async (watch: Watch) => { + if (args.failDelivery) throw new Error("the delivery couldn't be scheduled"); + args.delivered.push(watch.id); + }, + configured: () => true, + }; + } + + postgresTest( + "runs the final check on an overdue watch and fires it at the buzzer", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches( + sweepDeps({ + seeded, + delivered, + checkDeps: { + readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), + }, + }) + ); + + expect(result).toMatchObject({ overdue: 1, fired: 1, expired: 0, cancelled: 0, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "fired", + deliveryStatus: "pending", + }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "expires an overdue watch the check says hasn't happened, as verified", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); + + expect(result).toMatchObject({ overdue: 1, expired: 1, failed: 0 }); + const row = await getWatch(ctx.agentDb, { id: watchId }); + expect(row).toMatchObject({ status: "expired", deliveryStatus: "pending" }); + expect(row?.lastResult).toMatchObject({ verified: true, reason: "not_met_by_expiry" }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "cancels an overdue watch whose user lost access, and never wakes the chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches( + sweepDeps({ seeded, delivered, revoked: true }) + ); + + expect(result).toMatchObject({ overdue: 1, cancelled: 1, expired: 0, fired: 0, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "leaves a watch that is still inside its deadline alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); + + expect(result).toMatchObject({ overdue: 0, undelivered: 0 }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "recovers a wake the delivery lost, through the real query, exactly once", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + await expect( + sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, failDelivery: true })) + ).rejects.toThrow(/failed on 1 watches/); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + deliveryStatus: "pending", + }); + expect(delivered).toEqual([]); + + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const second = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + expect(second).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(delivered).toEqual([watchId]); + + await markWatchDelivered(ctx.agentDb, { id: watchId }); + const third = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + expect(third).toMatchObject({ undelivered: 0, redelivered: 0 }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "a deliverer that died mid-delivery is recovered, but a fresh claim is left alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); + + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_status = 'delivering', + delivery_claimed_at = now(), + last_checked_at = now() - interval '1 hour' + where id = $1`, + watchId + ); + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ + undelivered: 0, + }); + + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_claimed_at = now() - interval '1 hour' where id = $1`, + watchId + ); + const recovered: string[] = []; + expect( + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: recovered })) + ).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(recovered).toEqual([watchId]); + } + ); + + postgresTest( + "leaves nothing owed for a request the immediate check already answered", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + + const created = await create({ + seeded, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(created.ok).toBe(true); + if (!created.ok || created.watching) throw new Error("expected a one-shot result"); + + const delivered: string[] = []; + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + + expect(result).toMatchObject({ overdue: 0, undelivered: 0, redelivered: 0 }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "finalizes overdue watches even with no agent to deliver to, and delivers once it's back", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const unconfigured = await sweepDashboardAgentWatches({ + ...sweepDeps({ seeded, delivered }), + configured: () => false, + }); + + expect(unconfigured).toMatchObject({ + overdue: 1, + expired: 1, + deliveryDeferred: 1, + undelivered: 0, + redelivered: 0, + failed: 0, + }); + expect(delivered).toEqual([]); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + deliveryStatus: "pending", + }); + + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const restored = await sweepDashboardAgentWatches( + sweepDeps({ seeded, delivered, now: later }) + ); + expect(restored).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "the expiry grace keeps the sweep off a watch the tick chain is still finishing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + // A second past the deadline, so the chain's own final check owns this window. + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 second' where id = $1`, + created.watchId + ); + const delivered: string[] = []; + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered }))).toMatchObject({ + overdue: 0, + }); + + const later = new Date(Date.now() + WATCH_EXPIRY_GRACE_MS + 60_000); + expect( + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })) + ).toMatchObject({ overdue: 1, expired: 1 }); + } + ); + + postgresTest( + "retention drops long-terminal rows and nothing else", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + await seedChat(seeded, "chat_2"); + await seedChat(seeded, "chat_3"); + + const old = await overdueWatch(seeded, "chat_1"); + const recent = await overdueWatch(seeded, "chat_2"); + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_status = 'delivered', delivered_at = now() + where id in ($1, $2)`, + old, + recent + ); + + const active = await create({ seeded, chatId: "chat_3" }); + if (!active.ok || !active.watching) throw new Error("expected an active watch"); + + // Backdate every timestamp the age is measured from, so `greatest(...)` really is old. + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set created_at = now() - interval '30 days', + fired_at = now() - interval '30 days', + last_checked_at = now() - interval '30 days', + delivered_at = now() - interval '30 days' + where id = $1`, + old + ); + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set created_at = now() - interval '30 days' where id = $1`, + active.watchId + ); + + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ + purged: 1, + failed: 0, + }); + expect(await getWatch(ctx.agentDb, { id: old })).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: recent })).not.toBeNull(); + expect(await getWatch(ctx.agentDb, { id: active.watchId })).not.toBeNull(); + } + ); + + postgresTest( + "retention never takes a row whose wake is still owed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_status = 'pending', + created_at = now() - interval '30 days', + fired_at = now() - interval '30 days', + last_checked_at = now() - interval '30 days' + where id = $1`, + watchId + ); + + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ + purged: 0, + }); + expect(await getWatch(ctx.agentDb, { id: watchId })).not.toBeNull(); + } + ); +}); + +describe("the tick claim", () => { + postgresTest( + "claiming a generation is not an observation: only a recorded check stamps one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const claimed = await claimWatchTick(ctx.agentDb, { id: created.watchId, generation: 1 }); + expect(claimed).toMatchObject({ tickCount: 1, lastCheckedAt: null, lastResult: null }); + + await recordWatchCheck(ctx.agentDb, { id: created.watchId, lastResult: { pending: 4 } }); + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(row?.lastCheckedAt).toBeInstanceOf(Date); + expect(row?.lastResult).toMatchObject({ pending: 4 }); + expect(row?.tickCount).toBe(1); + } + ); +}); + +// The delivery claim's fencing token: a hung deliverer is taken over, so an unfenced release or mark would touch the new owner's claim. +describe("the delivery claim", () => { + async function firedWatch(seeded: Seeded) { + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) throw new Error("the watch wasn't created"); + const transitioned = await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + status: "fired", + lastResult: { result: "satisfied", facts: { verified: true } }, + }); + expect(transitioned).toMatchObject({ deliveryStatus: "pending" }); + return created.watchId; + } + + function staleBefore() { + return new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS); + } + + async function ageClaim(watchId: string) { + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_claimed_at = now() - interval '1 hour' where id = $1`, + watchId + ); + } + + postgresTest( + "a stale takeover makes the old owner's release a no-op, and the new owner delivers once", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-fence"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(a).not.toBeNull(); + if (!a) return; + + await ageClaim(watchId); + const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(b).not.toBeNull(); + if (!b) return; + expect(b.claimId).not.toBe(a.claimId); + + expect( + await releaseWatchDelivery(ctx.agentDb, { id: watchId, claimId: a.claimId }) + ).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivering", + deliveryClaimId: b.claimId, + }); + + expect( + await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) + ).toBeNull(); + + expect( + await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) + ).toMatchObject({ deliveryStatus: "delivered" }); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId })).toBeNull(); + } + ); + + postgresTest( + "a late delivered-mark from the old owner completes nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-late"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(a).not.toBeNull(); + if (!a) return; + await ageClaim(watchId); + const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(b).not.toBeNull(); + if (!b) return; + + expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: a.claimId })).toBeNull(); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivering", + deliveredAt: null, + }); + + expect( + await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) + ).toMatchObject({ deliveryStatus: "delivered" }); + } + ); + + postgresTest( + "the inline path marks a pending delivery without a claim", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-inline"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivered", + }); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); + expect( + await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) + ).toBeNull(); + } + ); +}); + +describe("deleting a chat while a watch is being created", () => { + postgresTest("holds in both orders", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + + for (const deleteFirst of [true, false]) { + const chatId = `chat_${deleteFirst ? "del" : "add"}`; + await seedChat(seeded, chatId); + + const creating = () => create({ seeded, chatId }); + const deleting = () => deleteChatWithWatches({ chatId, userId: seeded.user.id }); + const [a, b] = deleteFirst + ? await Promise.all([deleting(), creating()]) + : await Promise.all([creating(), deleting()]); + expect(a).toBeDefined(); + expect(b).toBeDefined(); + + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId })).toEqual([]); + expect( + await chatExists(ctx.agentDb, { + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toBe(false); + } + }); + + postgresTest( + "refuses a create against an already-deleted chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + await seedChat(seeded); + await deleteChatWithWatches({ chatId: "chat_1", userId: seeded.user.id }); + + expect(await create({ seeded })).toMatchObject({ ok: false, code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); +}); + +describe("the check endpoint", () => { + function request(token: string, body: unknown = {}) { + return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } + + async function activeWatch(seeded: Seeded) { + const result = await create({ seeded }); + if (!result.ok) throw new Error(`watch not created: ${result.code}`); + return result; + } + + function tokenFor(watchId: string, expiresAt: Date) { + return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); + } + + postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + const response = await checkAction({ + request: request("tr_daw_nonsense"), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(401); + }); + + postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor("watch_someone_else", watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); + }); + + postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.result).toBe("terminal_unsatisfied"); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row?.lastCheckedAt).not.toBeNull(); + expect(row?.tickCount).toBe(0); + expect(row?.status).toBe("active"); + }); + + postgresTest( + "refuses an ordinary check after expiry but allows the final one in grace", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, + watch.watchId + ); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const refused = await checkAction({ + request: request(token, {}), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(refused.status).toBe(403); + expect(await refused.json()).toMatchObject({ code: "expired" }); + + const allowed = await checkAction({ + request: request(token, { final: true }), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(allowed.status).toBe(200); + } + ); + + postgresTest( + "cancels the watch on revoked access, without reading environment data", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "access_revoked" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(row?.tickCount).toBe(0); + expect(row?.lastResult).toBeNull(); + } + ); + + postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, + watch.watchId + ); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "cancelled" }); + }); +}); + +/** A configured card, with both follow-ups off unless a test turns one on. */ +function draftFor(spec: WatchSpec, followUp: Partial = {}): WatchDraft { + return { + spec, + followUp: { investigateOnAttention: false, notifyExternally: false, ...followUp }, + }; +} + +function submit(args: { + seeded: Seeded; + draft?: WatchDraft; + chatId?: string; + clientRequestId?: string; + checkDeps?: Partial; + subscribed?: boolean; + /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ + subscribe?: typeof subscribeUserToWatchAlerts; + onSchedule?: () => void; + /** Wraps the creation step, so a test can die at the exact point after it. */ + create?: typeof createDashboardAgentWatch; +}) { + return submitDashboardAgentWatch({ + environment: authenticated(args.seeded), + userId: args.seeded.user.id, + organizationId: args.seeded.organization.id, + chatId: args.chatId, + clientRequestId: args.clientRequestId ?? "wreq_1", + draft: args.draft ?? draftFor(RUN_START), + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(args.checkDeps), + scheduleTick: async () => args.onSchedule?.(), + ...(args.create ? { create: args.create } : {}), + subscribe: + args.subscribe ?? + (async () => + args.subscribed === false + ? { ok: false, reason: "dashboard_agent_disabled" } + : { ok: true, email: args.seeded.user.email }), + }, + }); +} + +function storedMessages(seeded: Seeded, chatId: string) { + return getChatMessages(ctx.agentDb, { + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) as Promise | null>; +} + +/** + * The Alerts page authorizes with `findProjectBySlug` alone (see + * `_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx`): + * every organization member may list, create and delete a project's alert channels, with + * no role check. These tests pin that policy and prove the agent's routes never write + * wider than it. + */ +describe("the agent's alert boundary", () => { + /** A second, plain member of the same organization. */ + async function seedMember(prisma: PrismaClient, seeded: Seeded) { + const member = await prisma.user.create({ + data: { + email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, + }); + return member; + } + + async function seedOutsider(prisma: PrismaClient) { + return prisma.user.create({ + data: { + email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + } + + async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { + return prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: `Watch alerts for ${email}`, + projectId: seeded.project.id, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email }, + deduplicationKey: `dashboard-agent-watch:${email}`, + }, + }); + } + + function listRequest(chatId: string) { + return { + request: new Request( + `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, + { headers: { Authorization: "Bearer tr_uat_test" } } + ), + params: {}, + context: {} as never, + } as never; + } + + function createRequest(body: Record) { + return { + request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { + method: "POST", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {} as never, + } as never; + } + + function deleteRequest(channelId: string, body: Record) { + return { + request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { + method: "DELETE", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: { channelId }, + context: {} as never, + } as never; + } + + postgresTest( + "the dashboard lets any organization member manage a project's alerts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-policy"); + const member = await seedMember(prisma, seeded); + const outsider = await seedOutsider(prisma); + + // The whole of the Alerts page's authorization, for list, create and delete alike. + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) + ).not.toBeNull(); + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) + ).toBeNull(); + } + ); + + postgresTest( + "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-member"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + await seedWatchChannel(prisma, seeded, member.email); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const listed = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(listed.status).toBe(200); + // The same channel the Alerts page would show this member. + expect((await listed.json()).alerts).toHaveLength(1); + + // An outsider has no chat here and no membership, so nothing resolves. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(refused.status).toBe(404); + } + ); + + postgresTest( + "the agent only ever subscribes the caller's own address", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-create"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const own = (await alertsAction( + createRequest({ chatId: "chat_member", channel: "email" }) + )) as Response; + expect(own.status).toBe(200); + expect((await own.json()).target).toBe(member.email); + + // The Alerts page would let this member add anyone; the agent may not. + const other = (await alertsAction( + createRequest({ + chatId: "chat_member", + channel: "email", + email: "someone-else@example.com", + }) + )) as Response; + expect(other.status).toBe(400); + expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); + + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(1); + } + ); + + postgresTest( + "the agent's delete only takes the watch type off a watch channel", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-delete"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + const watchChannel = await seedWatchChannel(prisma, seeded, member.email); + + // A channel the agent never created and has no business touching. + const runAlerts = await prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: "Run failures", + projectId: seeded.project.id, + alertTypes: ["TASK_RUN"], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email: member.email }, + }, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const removed = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(removed.status).toBe(200); + expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); + + // The Alerts page would let a member delete this outright; the agent gets a 404. + const untouched = (await alertChannelAction( + deleteRequest(runAlerts.id, { chatId: "chat_member" }) + )) as Response; + expect(untouched.status).toBe(404); + expect( + await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) + ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); + + // An outsider can't reach the channel at all. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(refused.status).toBe(404); + } + ); +}); + +describe("the watch card submit", () => { + postgresTest( + "records what the user confirmed before the watch, and confirms it after", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(true); + expect(result.repaired).toBe(false); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${result.watchId}`, + ]); + // The consent record is the user's, and it states the condition and the lifetime. + expect(stored?.[0]).toMatchObject({ role: "user" }); + expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); + expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); + expect(result.messages.map((message) => message.id)).toEqual( + stored?.map((message) => message.id) + ); + } + ); + + postgresTest( + "leaves a repairable state when the confirmation never lands, and the retry repairs it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-repair"); + await seedChat(seeded); + + // The crash state: the request record is written and the watch is live, but the + // process died before the confirmation was appended. + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, + }); + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok || !created.watching) return; + + const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(created.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${created.watchId}`, + ]); + + // Still exactly one watch: the repair loaded it rather than creating another. + const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); + expect(active).toHaveLength(1); + } + ); + + postgresTest( + "a retried submit duplicates neither record", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-retry"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + const second = await submit({ seeded, chatId: "chat_1" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.repaired).toBe(true); + expect(second.watchId).toBe(first.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a genuinely different request still conflicts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-conflict"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + // Same condition, so the same identity, but a different window: not a retry. + const longer = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_2", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); + + // Same spec, different consent: also not a retry. + const investigating = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_3", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); + + // The refused attempts are recorded under their own consent records, so the + // transcript never shows a request with no answer. + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + "watch-request:wreq_2", + "watch-confirmation:refused:wreq_2", + "watch-request:wreq_3", + "watch-confirmation:refused:wreq_3", + ]); + } + ); + + postgresTest( + "a fresh panel's retry reuses the chat the first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fresh"); + + const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); + const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.chatId).toBe(first.chatId); + + const stored = await storedMessages(seeded, first.chatId); + expect(stored).toHaveLength(2); + } + ); + + postgresTest( + "an answered condition records the request and a one-shot result, and never a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + expect(result.watchId).toBeNull(); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ + async function countWatchRows(prisma: PrismaClient, chatId: string) { + const rows = await prisma.$queryRawUnsafe>( + `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, + chatId + ); + return Number(rows[0]?.count ?? 0); + } + + postgresTest( + "a retry after the watch has already fired creates no second watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + // The watch resolves and leaves the active set, so a duplicate check would find + // nothing. Only the ledger still knows this request already ran. + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a retry of an answered one-shot never becomes a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot-retry"); + await seedChat(seeded); + + const first = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(first.ok && first.watching === false).toBe(true); + + // The world moved on: the same condition would now be pending, so a re-evaluation + // would start a real watch. The recorded outcome is replayed instead. + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.watching).toBe(false); + expect(retry.watchId).toBeNull(); + expect(retry.repaired).toBe(true); + expect(await countWatchRows(prisma, "chat_1")).toBe(0); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + } + ); + + postgresTest( + "the same request id carrying a different draft is a conflict", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-hash"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + const changed = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); + + // A conflict writes nothing at all: no watch, and no record under the request. + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a pending submission converges on the watch its first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge"); + await seedChat(seeded); + + // The crash state the ledger exists for: the row is reserved, the watch is live + // under the reserved id, and the process died before the outcome was written. + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + const pending = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Reached the reserved row rather than creating another. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const settled = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); + } + ); + + postgresTest( + "a refusal that wins the race leaves no live watch behind", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-refused-race"); + await seedChat(seeded); + + // A concurrent attempt refuses this submission after the watch exists under the + // reserved id, so the ledger's winner keeps naming that id. + let reservedWatchId = ""; + const result = await submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + const created = await createDashboardAgentWatch(createParams); + const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + state: "refused", + refusalCode: "internal", + refusalError: "That watch couldn't be started.", + }); + expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); + return created; + }, + }); + + // The user is told nothing is being watched, so nothing may be watching. + expect(result.ok).toBe(false); + const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); + expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); + } + ); + + postgresTest( + "the consent record never spends a message from the cap", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-quota"); + await seedChat(seeded); + + await submit({ seeded, chatId: "chat_1" }); + + expect( + await countUserMessages(ctx.agentDb, { + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toBe(0); + } + ); + + postgresTest( + "a replay repeats the recorded email outcome and subscribes nobody", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-external-replay"); + await seedChat(seeded); + + const draft = draftFor(RUN_START, { notifyExternally: true }); + + // The first attempt asked for email and couldn't get it, so `unavailable` is what + // the transcript says and what the ledger records. + const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); + expect(first.ok).toBe(true); + if (!first.ok) return; + expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); + + const transcript = await storedMessages(seeded, "chat_1"); + + // The retry gets the real subscribe, which would succeed here. A replay that took the + // decision again would leave a channel row and an `enabled` answer the transcript — + // append-once, so never rewritten — contradicts for good. + let subscribeCalls = 0; + const retry = await submit({ + seeded, + chatId: "chat_1", + draft, + subscribe: async (subscribeParams) => { + subscribeCalls++; + return subscribeUserToWatchAlerts(subscribeParams); + }, + }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(subscribeCalls).toBe(0); + + expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); + expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(0); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ externalNotificationStatus: "unavailable" }); + + // The symptom: what the user is told after a refresh has to agree with the answer. + expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); + } + ); +}); + +describe("appendChatMessageOnce", () => { + postgresTest( + "appends in order without rewriting the transcript", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "append"); + await seedChat(seeded); + + const first = { id: "watch-card:watch_1", role: "assistant", parts: [] }; + const second = { id: "watch-card:watch_2", role: "assistant", parts: [] }; + + expect( + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: first, + }) + ).toBe(true); + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: second, + }); + + const messages = await getChatMessages(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + expect(messages).toEqual([first, second]); + } + ); + + postgresTest( + "appends nothing for a chat the caller doesn't own", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "append-owner"); + await seedChat(seeded); + + expect( + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: "user_someone_else", + organizationId: seeded.organization.id, + message: { id: "watch-card:watch_1", role: "assistant", parts: [] }, + }) + ).toBe(false); + + const messages = await getChatMessages(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + expect(messages).toEqual([]); + } + ); +}); + +describe("run_failed creation", () => { + const RUN_FAILED: WatchSpec = { + kind: "run_failed", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me if it fails", + }; + + postgresTest( + "watches a running run and dedups against the finished variant separately", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "runfailed"); + await seedChat(seeded); + + const failed = await create({ + seeded, + spec: RUN_FAILED, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, + }); + expect(failed.ok).toBe(true); + if (!failed.ok || !failed.watching) return; + expect(failed.identity).toBe("run_failed:run_1"); + + const finished = await create({ + seeded, + spec: { ...RUN_FAILED, kind: "run_finished" } as WatchSpec, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, + }); + expect(finished.ok).toBe(true); + if (!finished.ok || !finished.watching) return; + expect(finished.identity).toBe("run_finished:run_1"); + } + ); + + postgresTest( + "answers outright, with no watch row, once the run has succeeded", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "runfailed-done"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: RUN_FAILED, + checkDeps: { + readRun: async () => + runRow({ status: "COMPLETED_SUCCESSFULLY", completedAt: new Date() }), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + if (result.watching) return; + expect(result.immediate.result).toBe("terminal_unsatisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); +}); + +describe("the queue pack creation", () => { + const QUEUE = "task/my-task"; + + const BELOW: WatchSpec = { + kind: "queue_depth_below", + queue: QUEUE, + threshold: 100, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it's back below 100", + }; + + const STALLED: WatchSpec = { + kind: "queue_stalled", + queue: QUEUE, + ticks: 3, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it stops moving", + }; + + const AGE: WatchSpec = { + kind: "queue_oldest_age", + queue: QUEUE, + thresholdMinutes: 5, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if runs wait longer than 5 minutes", + }; + + postgresTest( + "creates each kind with its own identity on the same queue", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuepack"); + await seedChat(seeded); + + const busy = { + readQueueDepth: async () => ({ + depth: 780, + source: "live_queue" as const, + current: true, + }), + }; + + const below = await create({ seeded, spec: BELOW, checkDeps: busy }); + expect(below.ok && below.watching).toBe(true); + if (!below.ok || !below.watching) return; + expect(below.identity).toBe(`queue_depth_below:${QUEUE}:100`); + + const stalled = await create({ seeded, spec: STALLED, checkDeps: busy }); + expect(stalled.ok && stalled.watching).toBe(true); + if (!stalled.ok || !stalled.watching) return; + expect(stalled.identity).toBe(`queue_stalled:${QUEUE}`); + + const age = await create({ seeded, spec: AGE, checkDeps: busy }); + expect(age.ok && age.watching).toBe(true); + if (!age.ok || !age.watching) return; + expect(age.identity).toBe(`queue_oldest_age:${QUEUE}:5`); + + const drain = await create({ + seeded, + spec: { ...BELOW, kind: "backlog_drain" } as WatchSpec, + checkDeps: busy, + }); + expect(drain.ok).toBe(false); + if (drain.ok) return; + expect(drain.code).toBe("limit_reached"); + } + ); + + postgresTest( + "dedups the same SLA and allows a different one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queueage"); + await seedChat(seeded); + + const first = await create({ seeded, spec: AGE }); + expect(first.ok && first.watching).toBe(true); + + const same = await create({ seeded, spec: AGE }); + expect(same.ok).toBe(false); + if (same.ok) return; + expect(same.code).toBe("duplicate"); + + const other = await create({ seeded, spec: { ...AGE, thresholdMinutes: 30 } as WatchSpec }); + expect(other.ok && other.watching).toBe(true); + if (!other.ok || !other.watching) return; + expect(other.identity).toBe(`queue_oldest_age:${QUEUE}:30`); + } + ); + + postgresTest( + "answers a back-below ask outright when the queue is already quiet", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuebelow"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: BELOW, + checkDeps: { + readQueueDepth: async () => ({ depth: 4, source: "live_queue", current: true }), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + if (result.watching) return; + expect(result.immediate.result).toBe("satisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); + + postgresTest( + "round-trips the stall state through the row's existing facts column", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuestall"); + await seedChat(seeded); + + const created = await create({ + seeded, + spec: STALLED, + checkDeps: { + readQueueDepth: async () => ({ depth: 42, source: "live_queue", current: true }), + }, + }); + expect(created.ok && created.watching).toBe(true); + if (!created.ok || !created.watching) return; + + const facts = { queue: QUEUE, depth: 42, notDecreasingStreak: 2, ticks: 3 }; + await recordWatchCheck(ctx.agentDb, { + id: created.watchId, + lastResult: { + result: "pending", + facts, + observed: { + kind: "queue_stalled", + verified: true, + depth: 42, + notDecreasingStreak: 2, + ticks: 3, + }, + final: false, + }, + }); + + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(previousCheckFacts(row?.lastResult)).toEqual(facts); + + await recordWatchCheck(ctx.agentDb, { + id: created.watchId, + lastResult: { checkFailed: true, detail: "clickhouse down", previous: facts }, + }); + const afterGap = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(previousCheckFacts(afterGap?.lastResult)).toEqual(facts); + } + ); +}); + +const HEALTH: WatchSpec = { + kind: "health_recovery", + report: "health", + fromSeverity: "warn", + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me when health recovers", +}; + +describe("the batch chain registry", () => { + postgresTest("arms one chain per group, and only one", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batcharm"); + const now = new Date(); + + const scheduled: Array<{ epoch: number; tick: number }> = []; + const arm = () => + armDashboardAgentWatchBatch({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + now, + deps: { + schedule: async (params) => + void scheduled.push({ epoch: params.epoch, tick: params.tick }), + }, + }); + + expect(await arm()).toEqual({ running: true }); + expect(scheduled).toEqual([{ epoch: 1, tick: 1 }]); + + expect(await arm()).toEqual({ running: true }); + expect(await arm()).toEqual({ running: true }); + expect(scheduled).toHaveLength(1); + }); + + postgresTest( + "a chain whose run died is re-armed on a fresh epoch, and the zombie claims nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchdead"); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + const scheduled: Array<{ epoch: number; tick: number }> = []; + const arm = (now: Date) => + armDashboardAgentWatchBatch({ + ...group, + now, + deps: { + schedule: async (params) => + void scheduled.push({ epoch: params.epoch, tick: params.tick }), + }, + }); + + const armedAt = new Date(); + await arm(armedAt); + expect( + await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 1 }) + ).toMatchObject({ epoch: 1, generation: 1 }); + + await arm(new Date(armedAt.getTime() + 60_000)); + expect(scheduled).toHaveLength(1); + + await arm(new Date(armedAt.getTime() + watchBatchStaleMs(5) + 60_000)); + expect(scheduled).toEqual([ + { epoch: 1, tick: 1 }, + { epoch: 2, tick: 1 }, + ]); + + expect(await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 2 })).toBe( + null + ); + expect( + await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 2, generation: 1 }) + ).toMatchObject({ epoch: 2, generation: 1 }); + } + ); + + postgresTest( + "a chain that couldn't be triggered is not left marked as running", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchfail"); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + expect( + await armDashboardAgentWatchBatch({ + ...group, + deps: { + schedule: async () => { + throw new Error("the trigger failed"); + }, + }, + }) + ).toEqual({ running: false }); + + const scheduled: number[] = []; + expect( + await armDashboardAgentWatchBatch({ + ...group, + deps: { schedule: async (params) => void scheduled.push(params.epoch) }, + }) + ).toEqual({ running: true }); + expect(scheduled).toEqual([2]); + } + ); + + postgresTest( + "the re-arm backstop finds groups with active watches and no chain", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchrearm"); + await seedChat(seeded); + const created = await create({ + seeded, + spec: HEALTH, + checkDeps: { readHealth: async () => null }, + }); + expect(created.ok).toBe(true); + + const groups = await listWatchBatchGroupsToArm(ctx.agentDb); + expect(groups).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + + const armed: Array<{ environmentId: string; cadenceMinutes: number }> = []; + expect( + await rearmDashboardAgentWatchBatches({ + configured: () => true, + arm: async (params) => { + armed.push({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + return { running: true }; + }, + }) + ).toEqual({ stale: 1, armed: 1, failed: 0 }); + expect(armed).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + + // The staleness window is the group's own cadence: a five-minute group goes stale 17 minutes later. + await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + staleBefore: new Date(), + }); + expect(await listWatchBatchGroupsToArm(ctx.agentDb)).toEqual([]); + expect( + await listWatchBatchGroupsToArm(ctx.agentDb, { + now: new Date(Date.now() + watchBatchStaleMs(5) + 60_000), + }) + ).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + } + ); + + postgresTest( + "groups are per environment and per cadence, never mixed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchgroup"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + expect((await create({ seeded, chatId: "chat_1", spec: HEALTH })).ok).toBe(true); + expect((await create({ seeded, chatId: "chat_2", spec: RUN_START })).ok).toBe(true); + + const five = await listActiveWatchesForBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + }); + const one = await listActiveWatchesForBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 1, + }); + + expect(five.map((watch) => watch.chatId)).toEqual(["chat_1"]); + expect(one.map((watch) => watch.chatId)).toEqual(["chat_2"]); + expect( + (await listWatchBatchGroupsToArm(ctx.agentDb)).sort( + (a, b) => a.cadenceMinutes - b.cadenceMinutes + ) + ).toEqual([ + { environmentId: seeded.environment.id, cadenceMinutes: 1 }, + { environmentId: seeded.environment.id, cadenceMinutes: 5 }, + ]); + } + ); +}); + +describe("the batch check", () => { + async function healthGroup(seeded: Seeded, count = 3) { + const ids: string[] = []; + for (let index = 0; index < count; index++) { + const chatId = `chat_${index + 1}`; + await seedChat(seeded, chatId); + const created = await create({ + seeded, + chatId, + spec: HEALTH, + // `warn` keeps them all pending, so the group stays whole for the assertions below. + checkDeps: { readHealth: async () => ({ trustworthy: true, severity: "warn" }) }, + }); + if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); + ids.push(created.watchId); + } + return ids; + } + + async function otherUsersWatch(seeded: Seeded, prisma: PrismaClient) { + const user = await prisma.user.create({ + data: { + email: `other_${seeded.organization.slug}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: user.id, role: "MEMBER" }, + }); + await createChat(ctx.agentDb, { + id: "chat_other", + organizationId: seeded.organization.id, + userId: user.id, + }); + const created = await createDashboardAgentWatch({ + environment: authenticated(seeded), + userId: user.id, + chatId: "chat_other", + spec: HEALTH, + deps: { + configured: () => true, + checkDeps: () => + fakeCheckDeps({ readHealth: async () => ({ trustworthy: true, severity: "warn" }) }), + scheduleTick: async () => {}, + }, + }); + if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); + return { userId: user.id, watchId: created.watchId }; + } + + async function armChain(seeded: Seeded, cadenceMinutes = 5) { + const row = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes, + staleBefore: new Date(), + }); + if (!row) throw new Error("the chain wasn't armed"); + return row; + } + + postgresTest( + "authorizes once and loads the shared report once for the whole group", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchcheck"); + const ids = await healthGroup(seeded); + const chain = await armChain(seeded); + + let healthReads = 0; + let authorizations = 0; + + const response = await runWatchBatchCheck( + { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }, + { + authorize: async () => { + authorizations++; + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => + fakeCheckDeps({ + readHealth: async () => { + healthReads++; + return { trustworthy: true, severity: "warn" }; + }, + }), + } + ); + + expect(authorizations).toBe(1); + expect(healthReads).toBe(1); + + expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual([...ids].sort()); + expect(response.watches?.every((entry) => entry.result === "pending")).toBe(true); + expect(response.watches?.every((entry) => entry.tick === 1)).toBe(true); + expect(response.watches?.every((entry) => entry.token.length > 0)).toBe(true); + expect(response.continues).toBe(true); + expect(response.stale).toBeUndefined(); + + for (const id of ids) { + expect((await getWatch(ctx.agentDb, { id }))?.lastResult).toMatchObject({ + result: "pending", + final: false, + }); + } + } + ); + + postgresTest( + "authorizes each distinct user, so sharing readers never shares access", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchusers"); + await healthGroup(seeded, 2); + const other = await otherUsersWatch(seeded, prisma); + + const chain = await armChain(seeded); + const authorized: string[] = []; + + await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async (watch) => { + authorized.push(watch.userId); + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(authorized.sort()).toEqual([other.userId, seeded.user.id].sort()); + } + ); + + postgresTest( + "cancels a watch whose user lost access, and still answers for its neighbours", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchrevoked"); + const ids = await healthGroup(seeded, 2); + const chain = await armChain(seeded); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async () => ({ ok: false, reason: "access_revoked" }), + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(response.watches?.every((entry) => entry.code === "access_revoked")).toBe(true); + for (const id of ids) { + expect(await getWatch(ctx.agentDb, { id })).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + } + } + ); + + postgresTest( + "checks what is due, skips what isn't, and never skips a window boundary", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchdue"); + const [fresh, overdue, boundary] = await healthGroup(seeded, 3); + const chain = await armChain(seeded); + const now = new Date(); + + await recordWatchCheck(ctx.agentDb, { id: fresh!, lastCheckedAt: now }); + await recordWatchCheck(ctx.agentDb, { + id: overdue!, + lastCheckedAt: new Date(now.getTime() - 10 * 60_000), + }); + // `boundary`'s window closes before the next tick, so its final evaluation must still happen. + await recordWatchCheck(ctx.agentDb, { id: boundary!, lastCheckedAt: now }); + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() + interval '1 minute' where id = $1`, + boundary + ); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + now: () => now, + authorize: async () => ({ ok: true, environment: authenticated(seeded) }), + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual( + [boundary!, overdue!].sort() + ); + expect(response.continues).toBe(true); + } + ); + + postgresTest( + "a stale tick claims nothing and checks nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchstale"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch }; + expect((await runWatchBatchCheck({ ...group, tick: 1 })).stale).toBeUndefined(); + expect((await runWatchBatchCheck({ ...group, tick: 2 })).stale).toBeUndefined(); + + const late = await runWatchBatchCheck({ ...group, tick: 1 }); + expect(late).toEqual({ stale: true }); + + expect(await runWatchBatchCheck({ ...group, epoch: chain.epoch - 1, tick: 1 })).toEqual({ + stale: true, + }); + expect((await getWatch(ctx.agentDb, { id: ids[0]! }))?.status).toBe("active"); + } + ); + + postgresTest( + "stops the chain when the group's last watch is gone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchempty"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + await cancelWatch(ctx.agentDb, { id: ids[0]!, reason: "user" }); + + const response = await runWatchBatchCheck({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }); + + expect(response).toMatchObject({ watches: [], continues: false }); + + const rearmed = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + // Deliberately in the past: only a stopped chain can be re-armed this way. + staleBefore: new Date(Date.now() - 60 * 60_000), + }); + expect(rearmed).toMatchObject({ epoch: chain.epoch + 1, status: "running" }); + } + ); + + postgresTest( + "hands the group's owed wakes back for redelivery", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchowed"); + const ids = await healthGroup(seeded, 2); + const chain = await armChain(seeded); + + await transitionWatchCondition(ctx.agentDb, { + id: ids[0]!, + resolution: "condition_met", + lastResult: { verified: true }, + }); + + const response = await runWatchBatchCheck({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }); + + const owed = response.watches?.filter((entry) => entry.deliverOnly === true) ?? []; + expect(owed.map((entry) => entry.watchId)).toEqual([ids[0]!]); + expect(owed[0]?.tick).toBe(0); + expect( + response.watches?.filter((entry) => !entry.deliverOnly).map((entry) => entry.watchId) + ).toEqual([ids[1]!]); + } + ); + + postgresTest( + "keeps the chain alive while a wake is still owed, even with nothing left to watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchowedlast"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + await transitionWatchCondition(ctx.agentDb, { + id: ids[0]!, + resolution: "condition_met", + lastResult: { verified: true }, + }); + + const first = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 1 }); + expect(first.continues).toBe(true); + expect(first.watches?.map((entry) => entry.deliverOnly)).toEqual([true]); + + const claim = await claimWatchDelivery(ctx.agentDb, { + id: ids[0]!, + staleBefore: new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + await markWatchDelivered(ctx.agentDb, { id: ids[0]!, claimId: claim!.claimId }); + + const second = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 2 }); + expect(second).toMatchObject({ watches: [], continues: false }); + expect(await stopWatchBatch(ctx.agentDb, { ...group, epoch: chain.epoch })).toBe(null); + } + ); + + postgresTest( + "one watch that throws mid-evaluation costs only that watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchthrow"); + const mine = await healthGroup(seeded, 2); + const theirs = await otherUsersWatch(seeded, prisma); + const chain = await armChain(seeded); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async (watch) => { + if (watch.userId === theirs.userId) throw new Error("the authorization query failed"); + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => fakeCheckDeps(), + concurrency: 1, + } + ); + + const byId = new Map(response.watches?.map((entry) => [entry.watchId, entry])); + expect(byId.get(theirs.watchId)).toMatchObject({ result: "unavailable" }); + expect((await getWatch(ctx.agentDb, { id: theirs.watchId }))?.status).toBe("active"); + for (const id of mine) { + expect(byId.get(id)).toMatchObject({ result: "pending" }); + } + } + ); +}); + +describe("the batch check endpoint's authorization", () => { + function batchRequest(body: unknown, token?: string) { + return new Request("https://app.trigger.dev/api/v1/dashboard-agent/watches/batch-check", { + method: "POST", + headers: { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + } + + const batchToken = (environmentId: string, cadenceMinutes: number) => + signDashboardAgentWatchBatchToken(SESSION_SECRET, { + environmentId, + cadenceMinutes, + expiresAt: new Date(Date.now() + 60 * 60_000), + }); + + postgresTest("refuses a missing or bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const body = { environmentId: "env_1", cadenceMinutes: 5, epoch: 1, tick: 1 }; + + expect( + (await batchCheckAction({ request: batchRequest(body), params: {}, context: {} })).status + ).toBe(401); + const watchToken = await signDashboardAgentWatchToken(SESSION_SECRET, { + watchId: "watch_1", + expiresAt: new Date(Date.now() + 60 * 60_000), + }); + expect( + (await batchCheckAction({ request: batchRequest(body, watchToken), params: {}, context: {} })) + .status + ).toBe(401); + }); + + postgresTest( + "refuses a token minted for another group", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const token = await batchToken("env_1", 5); + + const wrongCadence = await batchCheckAction({ + request: batchRequest( + { environmentId: "env_1", cadenceMinutes: 15, epoch: 1, tick: 1 }, + token + ), + params: {}, + context: {}, + }); + expect(wrongCadence.status).toBe(403); + expect(await wrongCadence.json()).toMatchObject({ code: "group_mismatch" }); + + const wrongEnvironment = await batchCheckAction({ + request: batchRequest( + { environmentId: "env_2", cadenceMinutes: 5, epoch: 1, tick: 1 }, + token + ), + params: {}, + context: {}, + }); + expect(wrongEnvironment.status).toBe(403); + } + ); + + postgresTest( + "answers a group it does own, through the real registry", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchroute"); + const chain = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + staleBefore: new Date(), + }); + const token = await batchToken(seeded.environment.id, 5); + + const response = await batchCheckAction({ + request: batchRequest( + { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain!.epoch, + tick: 1, + }, + token + ), + params: {}, + context: {}, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ watches: [], continues: false }); + expect( + await stopWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain!.epoch, + }) + ).toBe(null); + } + ); +}); diff --git a/apps/webapp/test/queryScope.test.ts b/apps/webapp/test/queryScope.test.ts index 8aadd2439df..69e99d58036 100644 --- a/apps/webapp/test/queryScope.test.ts +++ b/apps/webapp/test/queryScope.test.ts @@ -9,9 +9,23 @@ describe("the query scope ceiling", () => { expect(queryScopeCeilingFor("PUBLIC_JWT")).toBe("environment"); }); - it("leaves every other bearer credential uncapped", () => { + // `PUBLIC` is the deprecated `pk_*` key: browser-shipped and environment-bound, the same + // shape of credential a public access token is. Nothing routes it here today, so this is + // the helper refusing to hand it the organization if anything ever does. + it("caps a public API key at its own environment", () => { + expect(queryScopeCeilingFor("PUBLIC")).toBe("environment"); + }); + + it("leaves the secret key uncapped", () => { expect(queryScopeCeilingFor("PRIVATE")).toBe("unbounded"); }); + + // Compile-time: the fallback is the cap, but a misspelled credential kind must still not + // be able to name itself into the uncapped branch. + it("takes only the credential kinds that exist", () => { + // @ts-expect-error not an authentication type + expect(() => queryScopeCeilingFor("private")).not.toThrow(); + }); }); describe("resolveQueryScope", () => { diff --git a/apps/webapp/test/reportHealth.test.ts b/apps/webapp/test/reportHealth.test.ts index d5ac18a6612..77d971241eb 100644 --- a/apps/webapp/test/reportHealth.test.ts +++ b/apps/webapp/test/reportHealth.test.ts @@ -214,7 +214,7 @@ describe("liveness trust guard (telemetry freshness)", () => { }); it("no freshness signal is never TRUSTWORTHY, even though the human verdict stays neutral", () => { - // The machine field must not claim trust it lacks. + // The machine field must not claim trust it lacks: a health-recovery watch would fire off silence. const vm = interpret({ ...INPUT_B, liveness: { telemetryAgeMs: null } }); expect(vm.summary.severity).toBe("ok"); expect(vm.facts).toMatchObject({ diff --git a/apps/webapp/test/setup.ts b/apps/webapp/test/setup.ts index 521a9624b7d..153cf759eef 100644 --- a/apps/webapp/test/setup.ts +++ b/apps/webapp/test/setup.ts @@ -13,6 +13,7 @@ config({ path: path.resolve(__dirname, "../.env") }); // the pair — the ioredis mock below forces lazyConnect, so nothing ever dials. process.env.REDIS_HOST ??= "localhost"; process.env.REDIS_PORT ??= "6379"; +process.env.SESSION_SECRET ??= "test-session-secret"; process.env.PROVIDER_SECRET ??= "test-provider-secret"; process.env.COORDINATOR_SECRET ??= "test-coordinator-secret"; process.env.MANAGED_WORKER_SECRET ??= "test-managed-worker-secret"; diff --git a/internal-packages/dashboard-agent-contracts/src/blocks.test.ts b/internal-packages/dashboard-agent-contracts/src/blocks.test.ts index 7d830be0978..04856b1a7bc 100644 --- a/internal-packages/dashboard-agent-contracts/src/blocks.test.ts +++ b/internal-packages/dashboard-agent-contracts/src/blocks.test.ts @@ -14,6 +14,7 @@ import { type EnvelopedViewBlock, type ViewBlock, } from "./blocks.js"; +import { SIMPLE_EVIDENCE_KINDS } from "./evidence.js"; const legacyDiagnosis = { type: "diagnosis", @@ -232,9 +233,18 @@ describe("chart actions", () => { }); describe("actions block", () => { - const navigateAction = { - label: "See its failed runs", - intent: { kind: "navigate", target: "trigger://runs?status=FAILED" }, + const watchAction = { + label: "Set up a watch", + intent: { + kind: "watch", + spec: { + kind: "error_recurrence", + fingerprint: "a1b2c3", + checkEveryMinutes: 15, + maxHours: 6, + note: "the TypeError in send-order-receipt", + }, + }, }; const askAction = { @@ -243,11 +253,11 @@ describe("actions block", () => { }; it("round-trips through both schemas", () => { - const body = { type: "actions", actions: [navigateAction, askAction] }; + const body = { type: "actions", actions: [watchAction, askAction] }; const input = viewBlockInputSchema.parse(body); expect(input.type === "actions" && input.actions).toHaveLength(2); const strict = viewBlockSchema.parse({ ...body, ...envelope }); - expect(strict.type === "actions" && strict.actions[0].intent.kind).toBe("navigate"); + expect(strict.type === "actions" && strict.actions[0].intent.kind).toBe("watch"); expect(parseStoredViewBlock(body).type).toBe("actions"); }); @@ -521,10 +531,50 @@ describe("investigation evidence refs (the model-facing boundary)", () => { }); it("still takes one bare id for the simple kinds", () => { - expect(withEvidence({ kind: "error", uri: "error_c4b4a797", label: "the group" }).success).toBe( - true - ); - expect(withEvidence({ kind: "run", uri: "", label: "a run" }).success).toBe(false); + for (const kind of SIMPLE_EVIDENCE_KINDS) { + expect(withEvidence({ kind, uri: "abc123", label: "a thing" }).success, kind).toBe(true); + expect(withEvidence({ kind, label: "a thing" }).success, `${kind} with no uri`).toBe(false); + expect(withEvidence({ kind, uri: "", label: "a thing" }).success, `${kind} empty`).toBe( + false + ); + } + }); + + // The seven simple kinds share one member, so `kind` must still be closed and the + // two shaped kinds must still be unreachable through it. + it("refuses a kind outside the catalog, and the shaped kinds' fields as a bare id", () => { + expect(withEvidence({ kind: "trace", uri: "trace_1", label: "a trace" }).success).toBe(false); + expect(withEvidence({ kind: "span", uri: "span_1", label: "a span" }).success).toBe(false); + expect(withEvidence({ kind: "source", uri: "src/a.ts", label: "a file" }).success).toBe(false); + expect( + withEvidence({ kind: "run", runId: "run_abc123", spanId: "span_1", label: "x" }).success + ).toBe(false); + expect(withEvidence({ uri: "run_abc123", label: "a run" }).success).toBe(false); + }); +}); + +describe("host-emitted blocks are not model-facing", () => { + it("refuses a block the model may not produce, and one that is not in the catalog", () => { + expect( + viewBlockInputSchema.safeParse({ + type: "watch_result", + outcome: "watching", + headline: "Watching the email-sends queue.", + }).success + ).toBe(false); + expect( + viewBlockInputSchema.safeParse({ type: "report", vm: reportVm, asOf: "x" }).success + ).toBe(false); + expect(viewBlockInputSchema.safeParse({ type: "timeline", items: [] }).success).toBe(false); + // …while the host's own union still takes them. + expect( + viewBlockSchema.safeParse({ + type: "watch_result", + outcome: "watching", + headline: "Watching the email-sends queue.", + ...envelope, + }).success + ).toBe(true); }); }); diff --git a/internal-packages/dashboard-agent-contracts/src/blocks.ts b/internal-packages/dashboard-agent-contracts/src/blocks.ts index b695f99b962..7648d92b887 100644 --- a/internal-packages/dashboard-agent-contracts/src/blocks.ts +++ b/internal-packages/dashboard-agent-contracts/src/blocks.ts @@ -6,6 +6,7 @@ import { evidenceRefSchema, evidenceSchema } from "./evidence.js"; import { agentIntentSchema } from "./intent.js"; import { runFiltersSchema } from "./run-filters.js"; import { triggerUriSchema } from "./trigger-uri.js"; +import { watchSpecSchema } from "./watch.js"; import { z } from "zod"; /** @@ -121,7 +122,7 @@ export const chartActionSchema = z.object({ .string() .describe("The button text, naming the thing, e.g. 'Investigate send-order-receipt'."), intent: chartActionIntentSchema.describe( - "What the button does. `ask` is the default and always works: phrase the user's own follow-up in their voice ('Investigate the send-order-receipt failures — why are they failing?'), and the click sends it as their next message. `navigate` takes them to the matching page — ONLY when you already hold a canonical `trigger://` URI for it (e.g. one a tool returned); an invalid target is silently dropped, so when in doubt use `ask`." + "What the button does. `ask` is the default and always works: phrase the user's own follow-up in their voice, and the click sends it as their next message. `navigate` takes them to the matching page — ONLY with a canonical `trigger://` URI you already hold; an invalid target is silently dropped, so when in doubt use `ask`." ), }); @@ -195,12 +196,16 @@ const actionIntentSchema = z.union([ target: z.string().min(1), filters: runFiltersSchema.optional(), }), + z.object({ + kind: z.literal("watch"), + spec: watchSpecSchema, + }), ]); export const actionsBlockActionSchema = z.object({ - label: z.string().min(1).describe("The button text, e.g. 'See its failed runs'."), + label: z.string().min(1).describe("The button text, e.g. 'Set up a watch'."), intent: actionIntentSchema.describe( - "What the button does. `ask` sends the prompt as the user's next message, phrased in their voice. `navigate` takes them to a page — ONLY with a canonical `trigger://` URI you already hold; an invalid target is silently dropped, so when in doubt use `ask`." + "What the button does. `watch` opens the watch configuration card pre-filled with your spec — the user confirming it is what starts the watch. `ask` sends the prompt as the user's next message, in their voice. `navigate` takes them to a page — ONLY with a canonical `trigger://` URI you already hold; an invalid target is silently dropped." ), }); @@ -213,7 +218,7 @@ const actionsBlockBodySchema = z.object({ .min(1) .max(3) .describe( - "1-3 buttons, the one to take first. Keep labels short and imperative ('See its failed runs', 'Show the code')." + "1-3 buttons, the one to take first. Keep labels short and imperative ('Set up a watch', 'See its failed runs')." ), }); @@ -511,7 +516,12 @@ export function forceSettledInvestigationState(state: InvestigationState): Inves */ export const INVESTIGATION_CAPABILITIES_VERSION = 1; -export const investigationActionKindSchema = z.enum(["show_code", "view_similar", "ask_follow_up"]); +export const investigationActionKindSchema = z.enum([ + "show_code", + "watch_recurrence", + "view_similar", + "ask_follow_up", +]); export const investigationActionSchema = z.object({ kind: investigationActionKindSchema, @@ -539,8 +549,34 @@ const investigationBlockBodyInputSchema = z.object({ }); /** - * What `render_view` accepts from the model. `report` is host-emitted and so - * absent here. + * Host-emitted only, so it is absent from `viewBlockInputSchema`. Wording is + * frozen at append time: the block carries final English, not a message key. + */ +export const watchResultOutcomeSchema = z.enum(["watching", "already_true", "impossible"]); +export type WatchResultOutcome = z.infer; + +const watchResultBlockBodySchema = z.object({ + type: z.literal("watch_result"), + outcome: watchResultOutcomeSchema, + headline: z.string(), + /** Null on a one-shot result: nothing is watching. */ + lifetime: z.string().nullable().default(null), + detail: z.string().nullable().default(null), + followUp: z.array(z.string()).max(4).default([]), + /** The live watch this confirms. Null on a one-shot result. */ + watchId: z.string().nullable().default(null), +}); + +export const watchResultBlockSchema = watchResultBlockBodySchema.merge(blockEnvelopeSchema); +export const legacyWatchResultBlockSchema = + watchResultBlockBodySchema.extend(optionalEnvelopeShape); + +export type EnvelopedWatchResultBlock = z.infer; +export type WatchResultBlock = z.infer; + +/** + * What `render_view` accepts from the model. `report` and `watch_result` are + * host-emitted and so absent here. */ export const viewBlockInputSchema = z.discriminatedUnion("type", [ diagnosisBlockBodySchema, @@ -588,6 +624,7 @@ export const viewBlockSchema = z.discriminatedUnion("type", [ actionsBlockSchema, reportBlockSchema, investigationBlockSchema, + watchResultBlockSchema, ]); export type EnvelopedDiagnosisBlock = z.infer; @@ -611,6 +648,7 @@ export const legacyViewBlockSchema = z.discriminatedUnion("type", [ legacyActionsBlockSchema, legacyReportBlockSchema, legacyInvestigationBlockSchema, + legacyWatchResultBlockSchema, ]); /** diff --git a/internal-packages/dashboard-agent-contracts/src/contracts.test.ts b/internal-packages/dashboard-agent-contracts/src/contracts.test.ts index a9f7bf627b7..9a61636a5e5 100644 --- a/internal-packages/dashboard-agent-contracts/src/contracts.test.ts +++ b/internal-packages/dashboard-agent-contracts/src/contracts.test.ts @@ -49,8 +49,20 @@ describe("intents", () => { expect(isExecutableIntent(parsed)).toBe(true); }); - it("parses ask", () => { + it("parses ask and watch", () => { expect(agentIntentSchema.safeParse({ kind: "ask", prompt: "why?" }).success).toBe(true); + expect( + agentIntentSchema.safeParse({ + kind: "watch", + spec: { + kind: "backlog_drain", + queue: "email-sends", + checkEveryMinutes: 5, + maxHours: 2, + note: "waiting for the backlog", + }, + }).success + ).toBe(true); }); it("keeps propose_fix in the wire format but marks it non-executable", () => { diff --git a/internal-packages/dashboard-agent-contracts/src/evidence.ts b/internal-packages/dashboard-agent-contracts/src/evidence.ts index 0b83971c796..2d5e015652c 100644 --- a/internal-packages/dashboard-agent-contracts/src/evidence.ts +++ b/internal-packages/dashboard-agent-contracts/src/evidence.ts @@ -29,11 +29,8 @@ export type EvidenceKind = Evidence["kind"]; * canonicalizes them into the strict schema above. */ const evidenceLabelShape = { - label: z.string().describe('Short human label, e.g. "run_abc123 failed span".'), - excerpt: z - .string() - .optional() - .describe("Optional verbatim snippet (error message, log line, source lines)."), + label: z.string().describe('Short label, e.g. "run_abc123 failed span".'), + excerpt: z.string().optional().describe("Optional verbatim snippet."), }; /** The kinds whose canonical URI is built from exactly one bare id. */ @@ -47,22 +44,22 @@ export const SIMPLE_EVIDENCE_KINDS = [ "investigation", ] as const; -const simpleEvidenceRef = (kind: K) => - z.object({ - kind: z.literal(kind), - uri: z - .string() - .min(1) - .describe( - "The resource id exactly as a tool returned it: a run id (run_...) for run, an error fingerprint for error, a queue name for queue, a deployment version for deployment, a report key for report. A full trigger:// URI for this same kind and environment is also accepted." - ), - ...evidenceLabelShape, - }); +/** One member, not seven: they differ only in the value of `kind`. */ +export const simpleEvidenceRefSchema = z.object({ + kind: z.enum(SIMPLE_EVIDENCE_KINDS), + uri: z + .string() + .min(1) + .describe( + "The id exactly as a tool returned it: a run id, error fingerprint, queue name, deployment version, or report key. A trigger:// URI of the same kind and environment also works." + ), + ...evidenceLabelShape, +}); export const spanEvidenceRefSchema = z.object({ kind: z.literal("span"), - runId: z.string().min(1).describe("The run the span belongs to, e.g. run_abc123."), - spanId: z.string().min(1).describe("The span's id, as the trace returned it."), + runId: z.string().min(1).describe("The run it belongs to, e.g. run_abc123."), + spanId: z.string().min(1).describe("As the trace returned it."), ...evidenceLabelShape, }); @@ -72,7 +69,7 @@ export const sourceEvidenceRefSchema = z.object({ .string() .min(1) .describe( - 'Repo-relative path of the file you read, e.g. "src/tasks/send-order-receipt.ts". Never a line suffix — put the line in `line`.' + 'Repo-relative path of the file you read, e.g. "src/tasks/send-order-receipt.ts". Never a line suffix — the line goes in `line`.' ), line: z .number() @@ -84,20 +81,12 @@ export const sourceEvidenceRefSchema = z.object({ .string() .min(1) .optional() - .describe( - "The commit the file was read at. Omit it and the executor pins the citation to the snapshot this turn read." - ), + .describe("The commit you read it at. Omit it and the executor pins this turn's snapshot."), ...evidenceLabelShape, }); export const evidenceRefSchema = z.discriminatedUnion("kind", [ - simpleEvidenceRef("runs"), - simpleEvidenceRef("run"), - simpleEvidenceRef("error"), - simpleEvidenceRef("queue"), - simpleEvidenceRef("deployment"), - simpleEvidenceRef("report"), - simpleEvidenceRef("investigation"), + simpleEvidenceRefSchema, spanEvidenceRefSchema, sourceEvidenceRefSchema, ]); diff --git a/internal-packages/dashboard-agent-contracts/src/index.ts b/internal-packages/dashboard-agent-contracts/src/index.ts index e6021735731..76a43ef3594 100644 --- a/internal-packages/dashboard-agent-contracts/src/index.ts +++ b/internal-packages/dashboard-agent-contracts/src/index.ts @@ -9,3 +9,5 @@ export * from "./page-context.js"; export * from "./run-filters.js"; export * from "./suggested-prompts.js"; export * from "./trigger-uri.js"; +export * from "./watch.js"; +export * from "./watch-wording.js"; diff --git a/internal-packages/dashboard-agent-contracts/src/intent.ts b/internal-packages/dashboard-agent-contracts/src/intent.ts index b711b2bbd66..36c1e74e0e3 100644 --- a/internal-packages/dashboard-agent-contracts/src/intent.ts +++ b/internal-packages/dashboard-agent-contracts/src/intent.ts @@ -1,6 +1,7 @@ /** An intent is a request to the host, never an action. The host decides. */ import { runFiltersSchema } from "./run-filters.js"; import { triggerUriSchema } from "./trigger-uri.js"; +import { watchSpecSchema } from "./watch.js"; import { z } from "zod"; export const agentIntentSchema = z.discriminatedUnion("kind", [ @@ -10,6 +11,7 @@ export const agentIntentSchema = z.discriminatedUnion("kind", [ filters: runFiltersSchema.optional(), }), z.object({ kind: z.literal("ask"), prompt: z.string() }), + z.object({ kind: z.literal("watch"), spec: watchSpecSchema }), /** Reserved: nothing may emit or execute this until write actions ship. */ z.object({ kind: z.literal("propose_fix"), investigationId: z.string() }), ]); diff --git a/internal-packages/dashboard-agent-contracts/src/page-context.ts b/internal-packages/dashboard-agent-contracts/src/page-context.ts index 7d45b3b05d2..eda3f8c510a 100644 --- a/internal-packages/dashboard-agent-contracts/src/page-context.ts +++ b/internal-packages/dashboard-agent-contracts/src/page-context.ts @@ -20,6 +20,8 @@ export const agentPageSchema = z.discriminatedUnion("kind", [ kind: z.literal("queue"), name: z.string(), health: z.enum(["ok", "warn", "crit"]).optional(), + /** A paused queue can neither drain nor grow, so it earns no watch and no backlog ask. */ + paused: z.boolean().optional(), }), z.object({ kind: z.literal("deployments") }), z.object({ diff --git a/internal-packages/dashboard-agent-contracts/src/watch-wording.ts b/internal-packages/dashboard-agent-contracts/src/watch-wording.ts new file mode 100644 index 00000000000..4d8559f7054 --- /dev/null +++ b/internal-packages/dashboard-agent-contracts/src/watch-wording.ts @@ -0,0 +1,643 @@ +/** + * The words a watch is said in — every surface's single source. + * + * `watch.ts` owns the meaning (which resolution and observation mean which + * category, tone, icon and headline key); this module owns the final English and + * the value formatting. Pure functions of the contract types, no React and no + * request context, so the card, the banner, the toast, the email, the Slack + * message, the webhook and the agent's own deterministic narration all read the + * same sentence. + * + * It lives here rather than in the webapp's presenter because the agent package + * cannot import the webapp, and a second vocabulary would drift within a release. + * + * Nothing outside this module may write a kind-specific watch sentence. + * + * Headlines state the fact first. The micro-label carries the "this is a wake" + * signal. + */ +import { + isWatchKind, + resolveWatchResult, + type WatchExternalNotification, + type WatchHeadlineKey, + type WatchKind, + type WatchObservedOutcome, + type WatchResolution, + type WatchResolvedPresentation, + type WatchSemanticIcon, + type WatchSpec, +} from "./watch.js"; + +/** The micro-label above a wake headline. Not part of the fact. */ +export const WATCH_UPDATE_LABEL = "Watch update"; + +/** + * A fingerprint carries its own `error_` prefix, and every surface names the kind + * itself — so the prefix is dropped, or the line reads "error error_c4b4a797397a9c43". + * The rest is shown whole: a truncated hash is not something anyone can look up. + */ +export function shortFingerprint(fingerprint: string): string { + return fingerprint.replace(/^error_/, ""); +} + +/* ------------------------------------------------------------------ * + * Identity formatting + * ------------------------------------------------------------------ */ + +/** + * The value half of a watch `identity` (`{kind}:{value}`). Read from the identity + * rather than the spec because the identity is the store's dedup key, so a surface + * can never disagree with the store about what is being watched. The threshold + * kinds append their threshold, so only the first segment is the queue name. + */ +const IDENTITY_KINDS_WITH_TRAILING_VALUE = new Set([ + "queue_depth_above", + "queue_depth_below", + "queue_oldest_age", +]); + +export function watchIdentityValue(kind: string, identity: string): string { + const value = identity.startsWith(`${kind}:`) ? identity.slice(kind.length + 1) : ""; + if (IDENTITY_KINDS_WITH_TRAILING_VALUE.has(kind)) { + const lastColon = value.lastIndexOf(":"); + return lastColon > 0 ? value.slice(0, lastColon) : value; + } + return value; +} + +/** How a run is named in a sentence. */ +function runName(identity: string, kind: string): string { + const value = watchIdentityValue(kind, identity); + return value ? `Run ${value}` : "The run"; +} + +/** How a queue is named in a sentence: the name, then the word "queue". */ +function queueName(identity: string, kind: string): string { + const value = watchIdentityValue(kind, identity); + return value ? `${value} queue` : "The queue"; +} + +/** The bare queue name, for sentences that read better without the word "queue". */ +function bareQueueName(identity: string, kind: string): string { + return watchIdentityValue(kind, identity) || "this queue"; +} + +/** How an error group is named in a sentence. */ +function errorName(identity: string, kind: string): string { + const value = watchIdentityValue(kind, identity); + return value ? `Error ${shortFingerprint(value)}` : "The error"; +} + +/* ------------------------------------------------------------------ * + * Value formatting + * ------------------------------------------------------------------ */ + +/** A duration in the shortest honest form. Never invents precision. */ +export function formatWatchDuration(ms: number | null | undefined): string | null { + if (ms === null || ms === undefined || !Number.isFinite(ms) || ms < 0) return null; + if (ms < 1000) return `${Math.round(ms)}ms`; + const seconds = ms / 1000; + if (seconds < 60) return `${seconds < 10 ? seconds.toFixed(1) : Math.round(seconds)}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${Math.round(seconds % 60)}s`; + const hours = Math.floor(minutes / 60); + return `${hours}h ${minutes % 60}m`; +} + +/** + * A wait, in whole minutes once it is minutes. An SLA is stated in minutes, so the + * wait it is compared against must not carry false seconds-precision. + */ +export function formatWatchWait(ms: number | null | undefined): string | null { + if (ms === null || ms === undefined || !Number.isFinite(ms) || ms < 0) return null; + if (ms < 60_000) return `${Math.max(1, Math.round(ms / 1000))}s`; + const minutes = Math.floor(ms / 60_000); + if (minutes < 60) return `${minutes}m`; + return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +/** An SLA, as the card and the headline state it. */ +export function formatWatchSla(minutes: number): string { + if (minutes < 60) return `${minutes}m`; + const hours = Math.floor(minutes / 60); + const rest = minutes % 60; + return rest === 0 ? `${hours}h` : `${hours}h ${rest}m`; +} + +/** A window length, as the confirmation and the tooltip state it. */ +export function formatWatchWindow(maxHours: number): string { + if (maxHours < 1) return `${Math.round(maxHours * 60)} min`; + return maxHours === 1 ? "1 hour" : `${maxHours} hours`; +} + +/** A cadence, as the confirmation and the tooltip state it. */ +export function formatWatchCadence(checkEveryMinutes: number): string { + return checkEveryMinutes === 60 ? "every hour" : `every ${checkEveryMinutes} min`; +} + +/* ------------------------------------------------------------------ * + * Headlines + * ------------------------------------------------------------------ */ + +export type WatchResolvedInput = { + kind: WatchKind | string; + /** The store's dedup key for the watched thing. */ + identity: string; + resolution: WatchResolution; + observed?: WatchObservedOutcome | null; +}; + +/** + * The final English for one headline key. Every kind-specific sentence in the + * product is in this switch and nowhere else. Numbers come from the frozen + * observation, never a fresh read, so a retry produces the same sentence. + */ +function headlineFor(key: WatchHeadlineKey, input: WatchResolvedInput): string { + const { kind, identity, observed } = input; + + switch (key) { + case "run_started": + return `${runName(identity, kind)} started`; + case "run_not_started": + return `${runName(identity, kind)} hasn't started yet`; + case "run_never_starts": + return `${runName(identity, kind)} will never start`; + + case "run_finished": + return `${runName(identity, kind)} finished`; + case "run_failed": + return `${runName(identity, kind)} failed`; + case "run_cancelled": + return `${runName(identity, kind)} was cancelled`; + case "run_still_running": + return `${runName(identity, kind)} is still running`; + case "run_gone": + return `${runName(identity, kind)} is no longer there`; + + case "run_no_failure": + return `${runName(identity, kind)} hasn't failed`; + case "run_succeeded": + return `${runName(identity, kind)} succeeded`; + + case "queue_drained": + return `${queueName(identity, kind)} drained`; + case "queue_not_drained": { + // Without an observed depth, stay vague rather than invent a number. + const depth = observed?.kind === "backlog_drain" ? observed.depth : null; + return depth === null + ? `${queueName(identity, kind)} still hasn't drained` + : `${queueName(identity, kind)} is still at ${depth}`; + } + case "queue_gone": + return `${queueName(identity, kind)} no longer exists`; + + case "queue_above_threshold": { + const threshold = observed?.kind === "queue_depth_above" ? observed.threshold : null; + return threshold === null + ? `${queueName(identity, kind)} is above the threshold` + : `${queueName(identity, kind)} is still above ${threshold}`; + } + case "queue_stayed_below": { + const threshold = observed?.kind === "queue_depth_above" ? observed.threshold : null; + return threshold === null + ? `${queueName(identity, kind)} stayed below the threshold` + : `${queueName(identity, kind)} stayed below ${threshold}`; + } + + case "queue_back_below": { + const threshold = observed?.kind === "queue_depth_below" ? observed.threshold : null; + return threshold === null + ? `${queueName(identity, kind)} is back below the threshold` + : `${queueName(identity, kind)} is back below ${threshold}`; + } + case "queue_still_above": { + const threshold = observed?.kind === "queue_depth_below" ? observed.threshold : null; + return threshold === null + ? `${queueName(identity, kind)} is still above the threshold` + : `${queueName(identity, kind)} is still above ${threshold}`; + } + + case "queue_stalled": { + // Without an observed depth, say only the fact we do have. + const depth = observed?.kind === "queue_stalled" ? observed.depth : null; + return depth === null + ? `${queueName(identity, kind)} isn't moving` + : `${queueName(identity, kind)} is stuck at ${depth}`; + } + case "queue_kept_moving": + return `${queueName(identity, kind)} kept moving`; + + case "queue_wait_over_sla": { + const sla = + observed?.kind === "queue_oldest_age" ? formatWatchSla(observed.thresholdMinutes) : null; + const wait = observed?.kind === "queue_oldest_age" ? formatWatchWait(observed.ageMs) : null; + const queue = bareQueueName(identity, kind); + if (wait === null) return `runs in ${queue} are waiting too long`; + return sla === null + ? `runs in ${queue} are waiting ${wait}` + : `runs in ${queue} are waiting ${wait} (over your ${sla} limit)`; + } + case "queue_wait_under_sla": { + const sla = + observed?.kind === "queue_oldest_age" ? formatWatchSla(observed.thresholdMinutes) : null; + return sla === null + ? `${queueName(identity, kind)} stayed within its wait limit` + : `${queueName(identity, kind)} stayed under ${sla}`; + } + + case "error_recurred": + return `${errorName(identity, kind)} happened again`; + case "error_quiet": + return `${errorName(identity, kind)} stayed quiet`; + + case "health_recovered": + return "Health recovered"; + case "health_not_recovered": + return "Health hasn't recovered"; + case "health_unavailable": + return "Health couldn't be read"; + + // The window ran out while the source was unreadable, so this deliberately + // says nothing about the condition itself. + case "unverified_at_window_end": + return "The watch ended without a confirmed answer"; + + default: { + const unreachable: never = key; + throw new Error(`Unhandled watch headline key: ${JSON.stringify(unreachable)}`); + } + } +} + +/** Everything a surface needs to render one resolved watch. */ +export type WatchPresentation = WatchResolvedPresentation & { + /** The fact, in final English. Complete without any narration under it. */ + headline: string; + /** The micro-label that marks this as an unprompted wake. */ + label: string; +}; + +/** + * Present one resolved watch. The single entry point for the banner, the toast and + * the email; they render this and add nothing of their own. + */ +export function presentResolvedWatch(input: WatchResolvedInput): WatchPresentation { + // A kind the store knows and this build doesn't must not crash a banner or + // silence an email, so it degrades to the fallback, which claims no outcome. + if (!isWatchKind(input.kind)) return WATCH_PRESENTATION_FALLBACK; + + const resolved = resolveWatchResult({ + kind: input.kind, + resolution: input.resolution, + outcome: input.observed ?? null, + }); + return { + ...resolved, + headline: headlineFor(resolved.headlineKey, { ...input, kind: input.kind }), + label: WATCH_UPDATE_LABEL, + }; +} + +/** + * The presentation for a watch whose row this surface couldn't load. It never + * guesses an outcome. + */ +export const WATCH_PRESENTATION_FALLBACK: WatchPresentation = { + category: "neutral", + tone: "neutral", + semanticIcon: "info", + headlineKey: "unverified_at_window_end", + headline: "The watch woke this chat up on its own.", + label: WATCH_UPDATE_LABEL, +}; + +/** + * What the watch was for, under a headline: the user's own words, else whatever + * names it. The chat banner and the toast both show this line. + */ +export function watchSubline( + watch: { note?: string | null; identity?: string | null; kind?: string | null } | undefined +): string | null { + const note = watch?.note?.trim(); + if (note) return note; + return watch?.identity || watch?.kind || null; +} + +/** + * The note, as a surface with room for a label states it. The email and the Slack + * message both quote the note, so they quote it the same way. + */ +export function watchNoteLine(note: string): string | null { + const trimmed = note.trim(); + return trimmed ? `You asked to be told when: ${trimmed}` : null; +} + +/* ------------------------------------------------------------------ * + * The one-shot result block + * ------------------------------------------------------------------ */ + +/** + * What the immediate check answered with, when it answered outright. No watch + * exists in either case: the check is the delivery, so there is no chip and no wake. + */ +export function immediateWatchMessage(result: string): string { + switch (result) { + case "satisfied": + return "That already happened, so there's nothing left to watch."; + case "terminal_unsatisfied": + return "That can't happen any more, so there's nothing to watch."; + // Not one-shot outcomes: the watch is created and running. Worded here so a + // confirmation can never fall through to nothing. + case "unavailable": + return "We couldn't check that just now. Watching anyway."; + default: + return "Watching."; + } +} + +/** + * The lifetime facts a confirmation always states: how often it checks, that it + * reports once, and when it gives up. + */ +export function watchLifetimeSentence(args: { + checkEveryMinutes: number; + maxHours: number; +}): string { + return `Checking ${formatWatchCadence(args.checkEveryMinutes)} for up to ${formatWatchWindow( + args.maxHours + )}. It reports once, then stops.`; +} + +/** + * The icon a surface should draw, keyed by meaning. Re-exported so the mapping to + * a concrete glyph lives in the component that owns the icon set. + */ +export type { WatchSemanticIcon }; + +/* ------------------------------------------------------------------ * + * The condition, in the four registers the product says it in + * ------------------------------------------------------------------ */ + +/** Fixed and always on: the card states it as a fact, not as a choice. */ +export const WATCH_IN_CHAT_DELIVERY_LINE = "When there's an answer: tell me in chat"; + +/** + * One condition, said four ways. They live in one record per kind rather than in + * four switches so a reviewer sees them together and they cannot drift apart: + * + * - `label` — the card's condition line, read under the subject. + * - `clause` — follows "Watching {subject} …" in the confirmation. + * - `tooltip` — the Watch button's tooltip. + * - `note` — why the watch exists, in the user's voice. The wake quotes it. + */ +export type WatchConditionWording = { + label: string; + clause: string; + tooltip: string; + note: string; +}; + +export function watchConditionWording(spec: WatchSpec): WatchConditionWording { + switch (spec.kind) { + case "run_start": + return { + label: "Until it starts", + clause: "until it starts", + tooltip: "Get notified when this run starts", + note: `tell me when run ${spec.runId} starts`, + }; + case "run_finished": + return { + label: "Until it finishes", + clause: "until it finishes", + tooltip: "Get notified when this run finishes", + note: `tell me when run ${spec.runId} finishes`, + }; + case "run_failed": + return { + label: "If it fails", + clause: "in case it fails", + tooltip: "Get notified if this run fails", + note: `tell me if run ${spec.runId} fails`, + }; + case "backlog_drain": + return { + label: "Until the queue drains", + clause: "until the queue drains", + tooltip: "Get notified when this queue drains", + note: `tell me when the ${spec.queue} queue drains`, + }; + case "queue_depth_above": + return { + label: `If the queue goes above ${spec.threshold}`, + clause: `in case the queue goes above ${spec.threshold}`, + tooltip: `Get notified if this queue goes above ${spec.threshold}`, + note: `tell me if the ${spec.queue} queue goes above ${spec.threshold}`, + }; + case "queue_depth_below": + return { + label: `Until the queue is back below ${spec.threshold}`, + clause: `until it is back below ${spec.threshold}`, + tooltip: `Get notified when this queue is back below ${spec.threshold}`, + note: `tell me when the ${spec.queue} queue is back below ${spec.threshold}`, + }; + case "queue_stalled": + return { + label: "If the queue stops moving", + clause: "in case it stops moving", + tooltip: "Get notified if this queue stops moving", + note: `tell me if the ${spec.queue} queue stops moving`, + }; + case "queue_oldest_age": + return { + label: `If runs wait longer than ${formatWatchSla(spec.thresholdMinutes)}`, + clause: `in case runs wait longer than ${formatWatchSla(spec.thresholdMinutes)}`, + tooltip: `Get notified if runs wait longer than ${formatWatchSla(spec.thresholdMinutes)}`, + note: `tell me if runs in ${spec.queue} wait longer than ${formatWatchSla( + spec.thresholdMinutes + )}`, + }; + case "error_recurrence": + return { + label: "If it happens again", + clause: "in case it happens again", + tooltip: "Get notified if this error happens again", + note: `ping me if error ${spec.fingerprint} happens again`, + }; + case "health_recovery": + return { + label: "Until it recovers", + clause: "until it recovers", + tooltip: "Get notified when health recovers", + note: "tell me when health is back to normal", + }; + } +} + +/** + * What is being watched, as the card's title names it. Read from the spec rather + * than the identity because the card exists before any watch row does. + */ +export function watchSubjectLabel(spec: WatchSpec): string { + switch (spec.kind) { + case "run_start": + case "run_finished": + case "run_failed": + return `run ${spec.runId}`; + case "backlog_drain": + case "queue_depth_above": + case "queue_depth_below": + case "queue_stalled": + case "queue_oldest_age": + return spec.queue; + case "error_recurrence": + return `error ${shortFingerprint(spec.fingerprint)}`; + case "health_recovery": + return "health"; + } +} + +/** + * The condition line. Written to read under the subject, so the two lines together + * are one sentence without repeating the subject. + */ +export function watchConditionLabel(spec: WatchSpec): string { + return watchConditionWording(spec).label; +} + +/** The Watch button's tooltip. */ +export function watchTooltipLabel(spec: WatchSpec): string { + return watchConditionWording(spec).tooltip; +} + +/** + * The note, restated from the spec. The wake narration quotes the note, so + * changing the condition or its number must rewrite it. Edits that keep the + * condition (window, cadence) keep the user's own words. + */ +export function noteFor(spec: WatchSpec): string { + return watchConditionWording(spec).note; +} + +/** The duration line of the card. */ +export function watchDurationLabel(spec: WatchSpec): string { + return `For ${formatWatchWindow(spec.maxHours)} · checking ${formatWatchCadence( + spec.checkEveryMinutes + )}`; +} + +/* ------------------------------------------------------------------ * + * The persisted blocks + * ------------------------------------------------------------------ */ + +export function watchExternalNotificationLine(external: WatchExternalNotification): string | null { + switch (external.status) { + case "enabled": + return "You'll get an email as well as the chat."; + case "not_requested": + return null; + case "unavailable": + return "I couldn't add email notifications, so updates will appear in the dashboard only."; + } +} + +/** The follow-up lines a confirmation states, for the opt-ins that took effect. */ +export function watchFollowUpLines(followUp: { + investigateOnAttention?: boolean; + external?: WatchExternalNotification; +}): string[] { + const lines: string[] = []; + if (followUp.investigateOnAttention) { + lines.push("If it turns out badly, I'll investigate straight away."); + } + const external = followUp.external ? watchExternalNotificationLine(followUp.external) : null; + if (external) lines.push(external); + return lines; +} + +/** + * What the user confirmed on the card, in their own voice. Written into the + * transcript before the watch is created, so a running watch can never be + * missing from the chat that owns it. Deterministic: no model writes this. + */ +export function watchRequestSentence(args: { + spec: WatchSpec; + followUp?: { investigateOnAttention?: boolean; notifyExternally?: boolean }; +}): string { + const parts = [ + `Watch ${watchSubjectLabel(args.spec)} ${watchConditionWording(args.spec).clause}.`, + watchLifetimeSentence({ + checkEveryMinutes: args.spec.checkEveryMinutes, + maxHours: args.spec.maxHours, + }), + ]; + if (args.followUp?.investigateOnAttention) { + parts.push("Investigate straight away if it turns out badly."); + } + if (args.followUp?.notifyExternally) parts.push("Email me as well as the chat."); + return parts.join(" "); +} + +/** + * The confirmation block: a watch is running. It states the lifetime facts and + * nothing else, because this block is the transcript record of the request. + */ +export function watchConfirmationBlockBody(args: { + spec: WatchSpec; + watchId: string; + /** The creation-time check couldn't run. Stated plainly rather than hidden. */ + unavailable?: boolean; + followUp?: { investigateOnAttention?: boolean; external?: WatchExternalNotification }; +}): { + type: "watch_result"; + outcome: "watching"; + headline: string; + lifetime: string; + detail: string | null; + followUp: string[]; + watchId: string; +} { + return { + type: "watch_result", + outcome: "watching", + headline: `Watching ${watchSubjectLabel(args.spec)} ${ + watchConditionWording(args.spec).clause + }.`, + lifetime: watchLifetimeSentence({ + checkEveryMinutes: args.spec.checkEveryMinutes, + maxHours: args.spec.maxHours, + }), + detail: args.unavailable ? immediateWatchMessage("unavailable") : null, + followUp: watchFollowUpLines(args.followUp ?? {}), + watchId: args.watchId, + }; +} + +/** + * The one-shot result block: the immediate check answered outright, so no watch was + * created. Nothing is running, so there is no lifetime and no follow-ups. + */ +export function watchOneShotBlockBody(args: { + spec: WatchSpec; + result: "satisfied" | "terminal_unsatisfied"; +}): { + type: "watch_result"; + outcome: "already_true" | "impossible"; + headline: string; + lifetime: null; + detail: null; + followUp: never[]; + watchId: null; +} { + const satisfied = args.result === "satisfied"; + return { + type: "watch_result", + outcome: satisfied ? "already_true" : "impossible", + headline: immediateWatchMessage(args.result), + lifetime: null, + detail: null, + followUp: [], + watchId: null, + }; +} diff --git a/internal-packages/dashboard-agent-contracts/src/watch.test.ts b/internal-packages/dashboard-agent-contracts/src/watch.test.ts new file mode 100644 index 00000000000..4f5c19df9b0 --- /dev/null +++ b/internal-packages/dashboard-agent-contracts/src/watch.test.ts @@ -0,0 +1,582 @@ +import { describe, expect, it } from "vitest"; +import { + WATCH_FAILED_RUN_STATUSES, + WATCH_KINDS, + WATCH_MAX_QUEUE_AGE_MINUTES, + WATCH_STALL_TICKS_DEFAULT, + WATCH_STALL_TICKS_MAX, + WATCH_STALL_TICKS_MIN, + resolveWatchResult, + watchConditionVariants, + watchCheckResultSchema, + watchDeliveryStatusSchema, + watchHeadlineKeys, + watchIdentity, + watchObservedOutcomeSchema, + watchResolutionSchema, + watchResolutionToWireStatus, + watchResolutions, + watchResultNeedsAttention, + watchRunDisposition, + watchSpecSchema, + watchStatusSchema, + type WatchKind, + type WatchSpec, +} from "./watch.js"; + +const common = { maxHours: 6, note: "because I asked" }; + +const specs = { + run_start: { ...common, kind: "run_start", runId: "run_123", checkEveryMinutes: 1 }, + run_finished: { ...common, kind: "run_finished", runId: "run_x", checkEveryMinutes: 5 }, + run_failed: { ...common, kind: "run_failed", runId: "run_y", checkEveryMinutes: 5 }, + backlog_drain: { ...common, kind: "backlog_drain", queue: "email-sends", checkEveryMinutes: 5 }, + queue_depth_above: { + ...common, + kind: "queue_depth_above", + queue: "email-sends", + threshold: 500, + checkEveryMinutes: 5, + }, + queue_depth_below: { + ...common, + kind: "queue_depth_below", + queue: "email-sends", + threshold: 100, + checkEveryMinutes: 5, + }, + queue_stalled: { + ...common, + kind: "queue_stalled", + queue: "email-sends", + ticks: 3, + checkEveryMinutes: 5, + }, + queue_oldest_age: { + ...common, + kind: "queue_oldest_age", + queue: "email-sends", + thresholdMinutes: 5, + checkEveryMinutes: 5, + }, + error_recurrence: { + ...common, + kind: "error_recurrence", + fingerprint: "a1b2c3", + checkEveryMinutes: 15, + }, + health_recovery: { + ...common, + kind: "health_recovery", + report: "health", + fromSeverity: "warn", + checkEveryMinutes: 60, + }, +} satisfies Record; + +describe("watchSpecSchema", () => { + it("accepts every kind", () => { + for (const spec of Object.values(specs)) { + expect(watchSpecSchema.safeParse(spec).success).toBe(true); + } + }); + + it("allows a 1-minute cadence for run-state watches", () => { + expect(watchSpecSchema.safeParse({ ...specs.run_finished, checkEveryMinutes: 1 }).success).toBe( + true + ); + }); + + it("rejects a 1-minute cadence for aggregate watches", () => { + expect( + watchSpecSchema.safeParse({ ...specs.backlog_drain, checkEveryMinutes: 1 }).success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ ...specs.error_recurrence, checkEveryMinutes: 1 }).success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ ...specs.queue_depth_above, checkEveryMinutes: 1 }).success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ ...specs.health_recovery, checkEveryMinutes: 1 }).success + ).toBe(false); + }); + + it("rejects an off-grid cadence", () => { + expect(watchSpecSchema.safeParse({ ...specs.run_start, checkEveryMinutes: 3 }).success).toBe( + false + ); + expect( + watchSpecSchema.safeParse({ ...specs.backlog_drain, checkEveryMinutes: 30 }).success + ).toBe(false); + }); + + it("enforces the 24 hour ceiling", () => { + expect(watchSpecSchema.safeParse({ ...specs.run_start, maxHours: 24 }).success).toBe(true); + expect(watchSpecSchema.safeParse({ ...specs.run_start, maxHours: 25 }).success).toBe(false); + expect(watchSpecSchema.safeParse({ ...specs.run_start, maxHours: 0 }).success).toBe(false); + }); + + it("requires a note", () => { + const { note, ...withoutNote } = specs.run_start; + expect(watchSpecSchema.safeParse(withoutNote).success).toBe(false); + }); + + it("does not accept a client-supplied `since` on error_recurrence", () => { + const parsed = watchSpecSchema.parse({ + ...specs.error_recurrence, + since: "2026-01-01T00:00:00.000Z", + }); + expect(parsed).not.toHaveProperty("since"); + }); + + it("rejects an unknown kind", () => { + expect(watchSpecSchema.safeParse({ ...common, kind: "run_slow", runId: "run_1" }).success).toBe( + false + ); + }); + + /** + * Kinds that take the same fields share one member, so each of these asks the + * grouped member to still hold every kind to its own subject and thresholds. + */ + it("keeps each kind's own required fields", () => { + for (const [kind, spec] of Object.entries(specs)) { + for (const field of ["runId", "queue", "fingerprint", "threshold", "thresholdMinutes"]) { + if (!(field in spec)) continue; + const { [field]: _dropped, ...without } = spec as Record; + expect(watchSpecSchema.safeParse(without).success, `${kind} without ${field}`).toBe(false); + } + } + }); + + it("does not let one kind borrow another's subject", () => { + expect( + watchSpecSchema.safeParse({ ...common, ...specs.run_start, runId: undefined, queue: "q" }) + .success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ + ...specs.backlog_drain, + queue: undefined, + runId: "run_123", + }).success + ).toBe(false); + // A depth kind needs its threshold; `backlog_drain` is not a threshold in disguise. + expect( + watchSpecSchema.safeParse({ ...specs.backlog_drain, kind: "queue_depth_above" }).success + ).toBe(false); + }); + + it("still separates the run cadence floor from the aggregate one, kind by kind", () => { + for (const [kind, spec] of Object.entries(specs)) { + const runState = kind.startsWith("run_"); + expect( + watchSpecSchema.safeParse({ ...spec, checkEveryMinutes: 1 }).success, + `${kind} at 1 minute` + ).toBe(runState); + } + }); +}); + +describe("watchIdentity", () => { + it("identifies the condition, not the cadence", () => { + expect(watchIdentity(specs.run_start)).toBe("run_start:run_123"); + expect(watchIdentity(specs.run_finished)).toBe("run_finished:run_x"); + expect(watchIdentity(specs.run_failed)).toBe("run_failed:run_y"); + expect(watchIdentity(specs.backlog_drain)).toBe("backlog_drain:email-sends"); + expect(watchIdentity(specs.queue_depth_above)).toBe("queue_depth_above:email-sends:500"); + expect(watchIdentity(specs.queue_depth_below)).toBe("queue_depth_below:email-sends:100"); + expect(watchIdentity(specs.queue_stalled)).toBe("queue_stalled:email-sends"); + expect(watchIdentity(specs.queue_oldest_age)).toBe("queue_oldest_age:email-sends:5"); + expect(watchIdentity(specs.error_recurrence)).toBe("error_recurrence:a1b2c3"); + expect(watchIdentity(specs.health_recovery)).toBe("health_recovery:health"); + }); + + it("ignores cadence, note, and maxHours", () => { + expect( + watchIdentity({ + ...specs.backlog_drain, + checkEveryMinutes: 60, + note: "different", + maxHours: 1, + }) + ).toBe(watchIdentity(specs.backlog_drain)); + }); + + it("covers every kind exhaustively", () => { + for (const kind of WATCH_KINDS) { + expect(watchIdentity(specs[kind])).toContain(`${kind}:`); + } + }); +}); + +// Compile-time exhaustiveness: adding a WatchSpec variant breaks this switch. +function describeWatch(spec: WatchSpec): string { + switch (spec.kind) { + case "run_start": + return `start of ${spec.runId}`; + case "run_finished": + return `finish of ${spec.runId}`; + case "run_failed": + return `failure of ${spec.runId}`; + case "backlog_drain": + return `drain of ${spec.queue}`; + case "queue_depth_above": + return `${spec.queue} above ${spec.threshold}`; + case "queue_depth_below": + return `${spec.queue} back below ${spec.threshold}`; + case "queue_stalled": + return `${spec.queue} stalled for ${spec.ticks} checks`; + case "queue_oldest_age": + return `${spec.queue} waits over ${spec.thresholdMinutes}m`; + case "error_recurrence": + return `recurrence of ${spec.fingerprint}`; + case "health_recovery": + return `recovery from ${spec.fromSeverity}`; + default: { + const unreachable: never = spec; + throw new Error(`Unhandled: ${JSON.stringify(unreachable)}`); + } + } +} + +describe("exhaustiveness", () => { + it("handles every kind", () => { + expect(Object.values(specs).map(describeWatch)).toHaveLength(WATCH_KINDS.length); + expect(WATCH_KINDS).toHaveLength(10); + }); +}); + +describe("enums", () => { + it("check results", () => { + expect(watchCheckResultSchema.options).toEqual([ + "pending", + "satisfied", + "terminal_unsatisfied", + "unavailable", + ]); + }); + + it("statuses", () => { + expect(watchStatusSchema.options).toEqual(["active", "fired", "expired", "cancelled"]); + expect(watchDeliveryStatusSchema.options).toEqual(["not_required", "pending", "delivered"]); + }); +}); + +describe("queue_depth_above", () => { + it("requires a non-negative integer threshold", () => { + expect(watchSpecSchema.safeParse({ ...specs.queue_depth_above, threshold: 0 }).success).toBe( + true + ); + expect(watchSpecSchema.safeParse({ ...specs.queue_depth_above, threshold: -1 }).success).toBe( + false + ); + expect(watchSpecSchema.safeParse({ ...specs.queue_depth_above, threshold: 1.5 }).success).toBe( + false + ); + }); + + it("treats the threshold as part of the identity", () => { + expect(watchIdentity({ ...specs.queue_depth_above, threshold: 5000 })).not.toBe( + watchIdentity(specs.queue_depth_above) + ); + expect( + watchIdentity({ ...specs.queue_depth_above, checkEveryMinutes: 60, note: "other" }) + ).toBe(watchIdentity(specs.queue_depth_above)); + }); +}); + +describe("the queue pack (TRI-12890)", () => { + it("floors all three at the 5-minute aggregate cadence", () => { + for (const spec of [specs.queue_depth_below, specs.queue_stalled, specs.queue_oldest_age]) { + expect(watchSpecSchema.safeParse({ ...spec, checkEveryMinutes: 1 }).success).toBe(false); + expect(watchSpecSchema.safeParse({ ...spec, checkEveryMinutes: 5 }).success).toBe(true); + } + }); + + it("defaults the stall count rather than asking for it", () => { + const { ticks, ...withoutTicks } = specs.queue_stalled; + const parsed = watchSpecSchema.parse(withoutTicks); + expect(parsed).toMatchObject({ kind: "queue_stalled", ticks: WATCH_STALL_TICKS_DEFAULT }); + }); + + it("bounds the stall count", () => { + expect( + watchSpecSchema.safeParse({ ...specs.queue_stalled, ticks: WATCH_STALL_TICKS_MIN - 1 }) + .success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ ...specs.queue_stalled, ticks: WATCH_STALL_TICKS_MAX + 1 }) + .success + ).toBe(false); + expect(watchSpecSchema.safeParse({ ...specs.queue_stalled, ticks: 1.5 }).success).toBe(false); + }); + + it("requires a positive whole-minute SLA under the watch ceiling", () => { + expect( + watchSpecSchema.safeParse({ ...specs.queue_oldest_age, thresholdMinutes: 0 }).success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ ...specs.queue_oldest_age, thresholdMinutes: 1.5 }).success + ).toBe(false); + expect( + watchSpecSchema.safeParse({ + ...specs.queue_oldest_age, + thresholdMinutes: WATCH_MAX_QUEUE_AGE_MINUTES + 1, + }).success + ).toBe(false); + }); + + it("treats the depth threshold and the SLA as part of the identity", () => { + expect(watchIdentity({ ...specs.queue_depth_below, threshold: 10 })).not.toBe( + watchIdentity(specs.queue_depth_below) + ); + expect(watchIdentity({ ...specs.queue_oldest_age, thresholdMinutes: 30 })).not.toBe( + watchIdentity(specs.queue_oldest_age) + ); + expect(watchIdentity({ ...specs.queue_stalled, ticks: 8 })).toBe( + watchIdentity(specs.queue_stalled) + ); + }); + + it("offers the whole queue family under Customize, and the run pair separately", () => { + expect(watchConditionVariants("backlog_drain")).toEqual([ + "backlog_drain", + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + ]); + for (const kind of ["queue_depth_below", "queue_stalled", "queue_oldest_age"] as const) { + expect(watchConditionVariants(kind)).toContain("backlog_drain"); + expect(watchConditionVariants(kind)).toContain(kind); + } + expect(watchConditionVariants("run_finished")).toEqual(["run_finished", "run_failed"]); + expect(watchConditionVariants("health_recovery")).toEqual(["health_recovery"]); + expect(watchConditionVariants("error_recurrence")).toEqual(["error_recurrence"]); + }); + + it("presents each new kind per §9.1: crossing back below is good, a stall isn't", () => { + expect( + resolveWatchResult({ kind: "queue_depth_below", resolution: "condition_met" }) + ).toMatchObject({ category: "positive", headlineKey: "queue_back_below" }); + expect( + resolveWatchResult({ kind: "queue_depth_below", resolution: "window_completed" }) + ).toMatchObject({ category: "attention", headlineKey: "queue_still_above" }); + expect( + resolveWatchResult({ kind: "queue_stalled", resolution: "condition_met" }) + ).toMatchObject({ category: "attention", headlineKey: "queue_stalled" }); + expect( + resolveWatchResult({ kind: "queue_stalled", resolution: "window_completed" }) + ).toMatchObject({ category: "positive", headlineKey: "queue_kept_moving" }); + expect( + resolveWatchResult({ kind: "queue_oldest_age", resolution: "condition_met" }) + ).toMatchObject({ category: "attention", headlineKey: "queue_wait_over_sla" }); + expect( + resolveWatchResult({ kind: "queue_oldest_age", resolution: "window_completed" }) + ).toMatchObject({ category: "positive", headlineKey: "queue_wait_under_sla" }); + for (const kind of ["queue_depth_below", "queue_stalled", "queue_oldest_age"] as const) { + expect(resolveWatchResult({ kind, resolution: "condition_impossible" })).toMatchObject({ + category: "neutral", + headlineKey: "queue_gone", + }); + } + }); + + it("answers the attention question for every surface, and never for an unknown kind", () => { + expect( + watchResultNeedsAttention({ kind: "backlog_drain", resolution: "window_completed" }) + ).toBe(true); + expect(watchResultNeedsAttention({ kind: "backlog_drain", resolution: "condition_met" })).toBe( + false + ); + expect( + watchResultNeedsAttention({ + kind: "run_finished", + resolution: "condition_met", + outcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 4200, + }, + }) + ).toBe(true); + expect( + watchResultNeedsAttention({ + kind: "run_finished", + resolution: "condition_met", + outcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_SUCCESSFULLY", + durationMs: 4200, + }, + }) + ).toBe(false); + expect(watchResultNeedsAttention({ kind: "not_a_kind", resolution: "condition_met" })).toBe( + false + ); + }); +}); + +describe("resolutions", () => { + it("has three values, and `unavailable` is not one of them", () => { + expect(watchResolutionSchema.options).toEqual([ + "condition_met", + "window_completed", + "condition_impossible", + ]); + expect(watchResolutions).not.toContain("unavailable"); + }); + + it("encodes onto the stable two-value wire status", () => { + expect(watchResolutionToWireStatus("condition_met")).toBe("fired"); + expect(watchResolutionToWireStatus("window_completed")).toBe("expired"); + expect(watchResolutionToWireStatus("condition_impossible")).toBe("expired"); + }); +}); + +describe("watchRunDisposition", () => { + it("splits success from failure from cancellation", () => { + expect(watchRunDisposition("COMPLETED_SUCCESSFULLY")).toBe("succeeded"); + expect(watchRunDisposition("CANCELED")).toBe("cancelled"); + expect(watchRunDisposition(null)).toBe("unknown"); + for (const status of WATCH_FAILED_RUN_STATUSES) { + expect(watchRunDisposition(status)).toBe("failed"); + } + }); +}); + +describe("watchObservedOutcomeSchema", () => { + it("accepts one shape per kind", () => { + const outcomes = [ + { kind: "run_start", started: true, status: "EXECUTING" }, + { kind: "run_finished", finalStatus: "COMPLETED_SUCCESSFULLY", durationMs: 1200 }, + { kind: "run_failed", finalStatus: "COMPLETED_WITH_ERRORS", durationMs: 900 }, + { kind: "backlog_drain", depth: 0 }, + { kind: "queue_depth_above", depth: 612, threshold: 500 }, + { kind: "queue_depth_below", depth: 42, threshold: 100 }, + { kind: "queue_stalled", depth: 42, notDecreasingStreak: 3, ticks: 3 }, + { kind: "queue_oldest_age", ageMs: 720_000, thresholdMinutes: 5 }, + { kind: "error_recurrence", countSince: 3 }, + { kind: "health_recovery", severity: "ok" }, + ]; + for (const outcome of outcomes) { + expect(watchObservedOutcomeSchema.safeParse(outcome).success).toBe(true); + } + expect(outcomes).toHaveLength(WATCH_KINDS.length); + }); + + it("defaults `verified` to true", () => { + const parsed = watchObservedOutcomeSchema.parse({ kind: "backlog_drain", depth: 0 }); + expect(parsed.verified).toBe(true); + }); +}); + +describe("resolveWatchResult", () => { + it("covers every kind × resolution cell", () => { + for (const kind of WATCH_KINDS) { + for (const resolution of watchResolutions) { + const result = resolveWatchResult({ kind, resolution }); + expect(watchHeadlineKeys).toContain(result.headlineKey); + expect(["positive", "attention", "neutral"]).toContain(result.category); + } + } + }); + + it("splits run_finished on the observed final status", () => { + const ok = resolveWatchResult({ + kind: "run_finished", + resolution: "condition_met", + outcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_SUCCESSFULLY", + durationMs: null, + }, + }); + const failed = resolveWatchResult({ + kind: "run_finished", + resolution: "condition_met", + outcome: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: null, + }, + }); + + expect(ok).toMatchObject({ category: "positive", headlineKey: "run_finished" }); + expect(failed).toMatchObject({ category: "attention", headlineKey: "run_failed" }); + }); + + it("gives the failed run a non-success icon", () => { + const failed = resolveWatchResult({ + kind: "run_finished", + resolution: "condition_met", + outcome: { kind: "run_finished", verified: true, finalStatus: "CRASHED", durationMs: null }, + }); + expect(failed.semanticIcon).not.toBe("success"); + expect(failed.tone).toBe("error"); + }); + + it("presents a cancelled run neutrally, not as a success", () => { + expect( + resolveWatchResult({ + kind: "run_finished", + resolution: "condition_met", + outcome: { + kind: "run_finished", + verified: true, + finalStatus: "CANCELED", + durationMs: null, + }, + }) + ).toMatchObject({ category: "neutral", headlineKey: "run_cancelled" }); + }); + + it("does not infer tone from a good-news kind list", () => { + expect( + resolveWatchResult({ kind: "backlog_drain", resolution: "window_completed" }).category + ).toBe("attention"); + expect( + resolveWatchResult({ kind: "error_recurrence", resolution: "window_completed" }).category + ).toBe("positive"); + expect( + resolveWatchResult({ kind: "queue_depth_above", resolution: "window_completed" }).category + ).toBe("positive"); + expect( + resolveWatchResult({ kind: "queue_depth_above", resolution: "condition_met" }).category + ).toBe("attention"); + }); + + it("says the condition could not be confirmed when the final read failed", () => { + const required: Partial>> = { + queue_depth_above: { threshold: 500 }, + queue_depth_below: { threshold: 100 }, + queue_stalled: { ticks: 3 }, + queue_oldest_age: { thresholdMinutes: 5 }, + }; + + for (const kind of WATCH_KINDS) { + const outcome = watchObservedOutcomeSchema.parse({ + kind, + verified: false, + ...(required[kind] ?? {}), + }); + expect( + resolveWatchResult({ kind: kind as WatchKind, resolution: "window_completed", outcome }) + ).toMatchObject({ category: "neutral", headlineKey: "unverified_at_window_end" }); + } + }); + + it("never claims a met condition was unverified", () => { + const outcome = watchObservedOutcomeSchema.parse({ kind: "backlog_drain", verified: false }); + expect( + resolveWatchResult({ kind: "backlog_drain", resolution: "condition_met", outcome }) + .headlineKey + ).toBe("queue_drained"); + }); +}); diff --git a/internal-packages/dashboard-agent-contracts/src/watch.ts b/internal-packages/dashboard-agent-contracts/src/watch.ts new file mode 100644 index 00000000000..e85965ce543 --- /dev/null +++ b/internal-packages/dashboard-agent-contracts/src/watch.ts @@ -0,0 +1,614 @@ +/** + * Cadence limits are enforced by the schema: run-state watches may poll every + * minute, aggregate conditions are floored at 5. + */ +import { z } from "zod"; + +export const runStateCadenceSchema = z.object({ + checkEveryMinutes: z.union([z.literal(1), z.literal(5), z.literal(15), z.literal(60)]), +}); + +export const standardCadenceSchema = z.object({ + checkEveryMinutes: z.union([z.literal(5), z.literal(15), z.literal(60)]), +}); + +export type RunStateCadence = z.infer; +export type StandardCadence = z.infer; + +/** Hard ceiling on how long a watch may live. */ +export const WATCH_MAX_HOURS = 24; + +export const watchCommonSchema = z.object({ + maxHours: z.number().positive().max(WATCH_MAX_HOURS), + /** Why this watch exists, in the user's terms. Shown when it fires. */ + note: z.string(), +}); + +export type WatchCommon = z.infer; + +/** Hard ceiling on a queue-depth threshold. */ +export const WATCH_MAX_QUEUE_THRESHOLD = 1_000_000; + +/** Consecutive no-progress checks `queue_stalled` waits for. Not offered by the card. */ +export const WATCH_STALL_TICKS_DEFAULT = 3; +export const WATCH_STALL_TICKS_MIN = 2; +export const WATCH_STALL_TICKS_MAX = 12; + +/** Ceiling on the `queue_oldest_age` SLA. */ +export const WATCH_MAX_QUEUE_AGE_MINUTES = 24 * 60; + +/** + * Kinds taking the same fields share one member with a `kind` enum. They still ask + * different questions — `run_failed` inverts `run_finished`, `queue_depth_below` is + * not `backlog_drain` — that difference just isn't in the fields. + */ +export const watchSpecSchema = z.discriminatedUnion("kind", [ + watchCommonSchema + .extend({ + kind: z.enum(["run_start", "run_finished", "run_failed"]), + runId: z.string(), + }) + .merge(runStateCadenceSchema), + watchCommonSchema + .extend({ kind: z.literal("backlog_drain"), queue: z.string() }) + .merge(standardCadenceSchema), + watchCommonSchema + .extend({ + kind: z.enum(["queue_depth_above", "queue_depth_below"]), + queue: z.string(), + threshold: z.number().int().nonnegative().max(WATCH_MAX_QUEUE_THRESHOLD), + }) + .merge(standardCadenceSchema), + // The one stateful kind: the streak lives in each check's facts and is handed to + // the next as `previous`. An `unavailable` tick freezes it rather than resetting. + watchCommonSchema + .extend({ + kind: z.literal("queue_stalled"), + queue: z.string(), + ticks: z + .number() + .int() + .min(WATCH_STALL_TICKS_MIN) + .max(WATCH_STALL_TICKS_MAX) + .default(WATCH_STALL_TICKS_DEFAULT), + }) + .merge(standardCadenceSchema), + watchCommonSchema + .extend({ + kind: z.literal("queue_oldest_age"), + queue: z.string(), + thresholdMinutes: z.number().int().positive().max(WATCH_MAX_QUEUE_AGE_MINUTES), + }) + .merge(standardCadenceSchema), + // `since` is absent on purpose: it is server-set at persist time, so nothing can + // backdate the recurrence window. + watchCommonSchema + .extend({ kind: z.literal("error_recurrence"), fingerprint: z.string() }) + .merge(standardCadenceSchema), + watchCommonSchema + .extend({ + kind: z.literal("health_recovery"), + report: z.literal("health"), + fromSeverity: z.enum(["warn", "crit"]), + }) + .merge(standardCadenceSchema), +]); + +/** Splits a grouped member into one type per kind, so `Extract` still names one shape. */ +type PerKind = T extends { kind: infer K extends string } + ? K extends K + ? Omit & { kind: K } + : never + : never; + +export type WatchSpec = PerKind>; +export type WatchKind = WatchSpec["kind"]; + +export const WATCH_KINDS = [ + "run_start", + "run_finished", + "run_failed", + "backlog_drain", + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", + "error_recurrence", + "health_recovery", +] as const satisfies readonly WatchKind[]; + +export function isWatchKind(kind: string): kind is WatchKind { + return (WATCH_KINDS as readonly string[]).includes(kind); +} + +/** + * The transcript id of the record of what a user confirmed on a watch card, keyed + * by the card's request id. Stable, so a retried submit repairs rather than repeats. + */ +export const WATCH_REQUEST_MESSAGE_ID_PREFIX = "watch-request:"; + +/** The transcript id of the confirmation that a watch is running, keyed by the watch. */ +export const WATCH_CONFIRMATION_MESSAGE_ID_PREFIX = "watch-confirmation:"; + +/** + * A deterministic consent record, not a turn the user spent, so it never counts + * against the message cap and the retry button never resends it. + */ +export function isWatchRequestMessageId(id: string | undefined | null): boolean { + return typeof id === "string" && id.startsWith(WATCH_REQUEST_MESSAGE_ID_PREFIX); +} + +/** + * The dedup key for a watched condition, scoped to one environment. Cadence, note + * and maxHours are deliberately not part of it. + */ +export function watchIdentity(spec: WatchSpec): string { + switch (spec.kind) { + case "run_start": + case "run_finished": + case "run_failed": + return `${spec.kind}:${spec.runId}`; + case "backlog_drain": + return `backlog_drain:${spec.queue}`; + // The threshold is part of the identity. + case "queue_depth_above": + return `queue_depth_above:${spec.queue}:${spec.threshold}`; + case "queue_depth_below": + return `queue_depth_below:${spec.queue}:${spec.threshold}`; + // `ticks` is not in the identity: like the cadence, it only tunes sensitivity. + case "queue_stalled": + return `queue_stalled:${spec.queue}`; + // The SLA is part of the identity. + case "queue_oldest_age": + return `queue_oldest_age:${spec.queue}:${spec.thresholdMinutes}`; + case "error_recurrence": + return `error_recurrence:${spec.fingerprint}`; + case "health_recovery": + return `health_recovery:${spec.report}`; + default: { + const unreachable: never = spec; + throw new Error(`Unhandled watch kind: ${JSON.stringify(unreachable)}`); + } + } +} + +/** + * `terminal_unsatisfied` means it can never happen now, so stop checking. + * `unavailable` means the check itself couldn't run: keep the watch alive and retry. + */ +export const watchCheckResults = [ + "pending", + "satisfied", + "terminal_unsatisfied", + "unavailable", +] as const; + +export const watchCheckResultSchema = z.enum(watchCheckResults); +export type WatchCheckResult = z.infer; + +/** + * `window_completed` is an answer and gets reported. The resolution alone does not + * decide what the user sees; see {@link resolveWatchResult}. + */ +export const watchResolutions = [ + "condition_met", + "window_completed", + "condition_impossible", +] as const; +export const watchResolutionSchema = z.enum(watchResolutions); +export type WatchResolution = z.infer; + +/** + * Wake action ids, delivery ids and banner render keys keep this two-value suffix + * so persisted wakes and dedup keys stay valid. The resolution travels in the facts. + */ +export function watchResolutionToWireStatus(resolution: WatchResolution): "fired" | "expired" { + return resolution === "condition_met" ? "fired" : "expired"; +} + +/** + * A check landing on the deadline may still resolve `condition_met` or + * `condition_impossible`. Only `pending`/`unavailable` there become `window_completed`. + */ +export function watchResolutionForCheck( + result: WatchCheckResult, + atWindowBoundary: boolean +): WatchResolution | null { + switch (result) { + case "satisfied": + return "condition_met"; + case "terminal_unsatisfied": + return "condition_impossible"; + case "pending": + case "unavailable": + return atWindowBoundary ? "window_completed" : null; + default: { + const unreachable: never = result; + throw new Error(`Unhandled watch check result: ${JSON.stringify(unreachable)}`); + } + } +} + +export const watchStatuses = ["active", "fired", "expired", "cancelled"] as const; +export const watchStatusSchema = z.enum(watchStatuses); +export type WatchStatus = z.infer; + +/** Whether the user still needs to be told this watch fired. */ +export const watchDeliveryStatuses = ["not_required", "pending", "delivered"] as const; +export const watchDeliveryStatusSchema = z.enum(watchDeliveryStatuses); +export type WatchDeliveryStatus = z.infer; + +/** + * `run_finished` resolves `condition_met` on any terminal status, so this set, not + * the resolution, separates "run finished" from "run failed". + */ +export const WATCH_FAILED_RUN_STATUSES = [ + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", + "INTERRUPTED", +] as const; + +/** Neither a success nor a failure, so its own presentation. */ +export const WATCH_CANCELLED_RUN_STATUSES = ["CANCELED"] as const; + +export type WatchRunDisposition = "succeeded" | "failed" | "cancelled" | "unknown"; + +export function watchRunDisposition(status: string | null | undefined): WatchRunDisposition { + if (!status) return "unknown"; + if (status === "COMPLETED_SUCCESSFULLY") return "succeeded"; + if ((WATCH_FAILED_RUN_STATUSES as readonly string[]).includes(status)) return "failed"; + if ((WATCH_CANCELLED_RUN_STATUSES as readonly string[]).includes(status)) return "cancelled"; + return "unknown"; +} + +/** + * Frozen with the facts next to the resolution, so no delivery surface re-reads the + * source. `verified: false` means the window completed with the source unavailable. + */ +export const watchObservedOutcomeSchema = z.union([ + z.object({ + kind: z.literal("run_start"), + verified: z.boolean().default(true), + status: z.string().nullable().default(null), + started: z.boolean().default(false), + }), + z.object({ + kind: z.literal("run_finished"), + verified: z.boolean().default(true), + /** The observation the presentation splits on. */ + finalStatus: z.string().nullable().default(null), + durationMs: z.number().nullable().default(null), + }), + z.object({ + kind: z.literal("run_failed"), + verified: z.boolean().default(true), + /** Null while the run is still going. */ + finalStatus: z.string().nullable().default(null), + durationMs: z.number().nullable().default(null), + }), + z.object({ + kind: z.literal("backlog_drain"), + verified: z.boolean().default(true), + /** Null when the depth could not be read. */ + depth: z.number().nullable().default(null), + }), + z.object({ + kind: z.literal("queue_depth_above"), + verified: z.boolean().default(true), + depth: z.number().nullable().default(null), + threshold: z.number(), + }), + z.object({ + kind: z.literal("queue_depth_below"), + verified: z.boolean().default(true), + depth: z.number().nullable().default(null), + threshold: z.number(), + }), + z.object({ + kind: z.literal("queue_stalled"), + verified: z.boolean().default(true), + depth: z.number().nullable().default(null), + /** Consecutive checks that saw no progress, as of this one. */ + notDecreasingStreak: z.number().default(0), + ticks: z.number(), + }), + z.object({ + kind: z.literal("queue_oldest_age"), + verified: z.boolean().default(true), + /** Null when nothing was waiting, or it was unreadable. */ + ageMs: z.number().nullable().default(null), + thresholdMinutes: z.number(), + }), + z.object({ + kind: z.literal("error_recurrence"), + verified: z.boolean().default(true), + /** Occurrences proven to be after the server-set `since`. */ + countSince: z.number().default(0), + }), + z.object({ + kind: z.literal("health_recovery"), + verified: z.boolean().default(true), + severity: z.enum(["ok", "warn", "crit"]).nullable().default(null), + }), +]); + +export type WatchObservedOutcome = z.infer; + +/** + * Declared per kind, never inferred: `window_completed` is bad news for a drain + * watch and good news for an error-recurrence one. + */ +export const watchPresentationCategories = ["positive", "attention", "neutral"] as const; +export const watchPresentationCategorySchema = z.enum(watchPresentationCategories); +export type WatchPresentationCategory = z.infer; + +export const watchPresentationTones = ["success", "warning", "error", "neutral"] as const; +export const watchPresentationToneSchema = z.enum(watchPresentationTones); +export type WatchPresentationTone = z.infer; + +/** Named by meaning, not glyph. Follows the presentation outcome, not the resolution. */ +export const watchSemanticIcons = ["success", "attention", "error", "waiting", "info"] as const; +export const watchSemanticIconSchema = z.enum(watchSemanticIcons); +export type WatchSemanticIcon = z.infer; + +/** The English wording lives in the webapp's `watch-presentation.ts`. */ +export const watchHeadlineKeys = [ + // run_start + "run_started", + "run_not_started", + "run_never_starts", + // run_finished + "run_finished", + "run_failed", + "run_cancelled", + "run_still_running", + "run_gone", + // run_failed + "run_no_failure", + "run_succeeded", + // backlog_drain + "queue_drained", + "queue_not_drained", + "queue_gone", + // queue_depth_above + "queue_above_threshold", + "queue_stayed_below", + // queue_depth_below + "queue_back_below", + "queue_still_above", + // queue_stalled + "queue_stalled", + "queue_kept_moving", + // queue_oldest_age + "queue_wait_over_sla", + "queue_wait_under_sla", + // error_recurrence + "error_recurred", + "error_quiet", + // health_recovery + "health_recovered", + "health_not_recovered", + "health_unavailable", + // Any kind, when the window completed without a usable final read. + "unverified_at_window_end", +] as const; +export const watchHeadlineKeySchema = z.enum(watchHeadlineKeys); +export type WatchHeadlineKey = z.infer; + +export type WatchResolvedPresentation = { + category: WatchPresentationCategory; + tone: WatchPresentationTone; + semanticIcon: WatchSemanticIcon; + headlineKey: WatchHeadlineKey; +}; + +const POSITIVE: Omit = { + category: "positive", + tone: "success", + semanticIcon: "success", +}; +const ATTENTION_WARN: Omit = { + category: "attention", + tone: "warning", + semanticIcon: "attention", +}; +const ATTENTION_ERROR: Omit = { + category: "attention", + tone: "error", + semanticIcon: "error", +}; +const NEUTRAL: Omit = { + category: "neutral", + tone: "neutral", + semanticIcon: "info", +}; +const WAITING: Omit = { + category: "attention", + tone: "warning", + semanticIcon: "waiting", +}; + +/** + * Total over `kind × resolution` on purpose: adding either fails to compile until + * every cell is filled. Cells refined by the observed outcome carry the default. + */ +const RESOLVED_RESULTS: Record> = { + run_start: { + condition_met: { ...POSITIVE, headlineKey: "run_started" }, + window_completed: { ...WAITING, headlineKey: "run_not_started" }, + condition_impossible: { ...NEUTRAL, headlineKey: "run_never_starts" }, + }, + run_finished: { + // Refined below by the observed final status. + condition_met: { ...POSITIVE, headlineKey: "run_finished" }, + window_completed: { ...WAITING, headlineKey: "run_still_running" }, + condition_impossible: { ...NEUTRAL, headlineKey: "run_gone" }, + }, + // The inverse question, so the presentation inverts too. `condition_impossible` + // is refined below into the success headline when a final status proves it. + run_failed: { + condition_met: { ...ATTENTION_ERROR, headlineKey: "run_failed" }, + window_completed: { ...POSITIVE, headlineKey: "run_no_failure" }, + condition_impossible: { ...NEUTRAL, headlineKey: "run_gone" }, + }, + backlog_drain: { + condition_met: { ...POSITIVE, headlineKey: "queue_drained" }, + window_completed: { ...ATTENTION_WARN, headlineKey: "queue_not_drained" }, + condition_impossible: { ...NEUTRAL, headlineKey: "queue_gone" }, + }, + queue_depth_above: { + condition_met: { ...ATTENTION_WARN, headlineKey: "queue_above_threshold" }, + window_completed: { ...POSITIVE, headlineKey: "queue_stayed_below" }, + condition_impossible: { ...NEUTRAL, headlineKey: "queue_gone" }, + }, + queue_depth_below: { + condition_met: { ...POSITIVE, headlineKey: "queue_back_below" }, + window_completed: { ...ATTENTION_WARN, headlineKey: "queue_still_above" }, + condition_impossible: { ...NEUTRAL, headlineKey: "queue_gone" }, + }, + queue_stalled: { + condition_met: { ...ATTENTION_WARN, headlineKey: "queue_stalled" }, + window_completed: { ...POSITIVE, headlineKey: "queue_kept_moving" }, + condition_impossible: { ...NEUTRAL, headlineKey: "queue_gone" }, + }, + queue_oldest_age: { + condition_met: { ...ATTENTION_WARN, headlineKey: "queue_wait_over_sla" }, + window_completed: { ...POSITIVE, headlineKey: "queue_wait_under_sla" }, + condition_impossible: { ...NEUTRAL, headlineKey: "queue_gone" }, + }, + error_recurrence: { + condition_met: { ...ATTENTION_ERROR, headlineKey: "error_recurred" }, + window_completed: { ...POSITIVE, headlineKey: "error_quiet" }, + // The fingerprint is gone, so it can't recur under this identity. + condition_impossible: { ...NEUTRAL, headlineKey: "error_quiet" }, + }, + health_recovery: { + condition_met: { ...POSITIVE, headlineKey: "health_recovered" }, + window_completed: { ...ATTENTION_WARN, headlineKey: "health_not_recovered" }, + condition_impossible: { ...NEUTRAL, headlineKey: "health_unavailable" }, + }, +}; + +/** + * The only place a resolved watch becomes something to show. No surface may + * present from the resolution alone. + */ +export function resolveWatchResult(args: { + kind: WatchKind; + resolution: WatchResolution; + outcome?: WatchObservedOutcome | null; +}): WatchResolvedPresentation { + const { kind, resolution, outcome } = args; + + // Unconfirmed is not "it didn't happen". + if (resolution === "window_completed" && outcome && outcome.verified === false) { + return { ...NEUTRAL, headlineKey: "unverified_at_window_end" }; + } + + // `unknown` keeps the plain "finished" headline: never claim an unobserved failure. + if (kind === "run_finished" && resolution === "condition_met") { + const disposition = watchRunDisposition( + outcome?.kind === "run_finished" ? outcome.finalStatus : null + ); + if (disposition === "failed") return { ...ATTENTION_ERROR, headlineKey: "run_failed" }; + if (disposition === "cancelled") return { ...NEUTRAL, headlineKey: "run_cancelled" }; + } + + if (kind === "run_failed" && resolution === "condition_impossible") { + const disposition = watchRunDisposition( + outcome?.kind === "run_failed" ? outcome.finalStatus : null + ); + if (disposition === "succeeded") return { ...POSITIVE, headlineKey: "run_succeeded" }; + if (disposition === "cancelled") return { ...NEUTRAL, headlineKey: "run_cancelled" }; + } + + return RESOLVED_RESULTS[kind][resolution]; +} + +/** + * What the "investigate attention outcomes" consent covers. Both the agent's wake + * and the webapp's kick must call this rather than judge for themselves. + */ +export function watchResultNeedsAttention(args: { + kind: string; + resolution: WatchResolution; + outcome?: WatchObservedOutcome | null; +}): boolean { + if (!isWatchKind(args.kind)) return false; + const { category } = resolveWatchResult({ + kind: args.kind, + resolution: args.resolution, + outcome: args.outcome, + }); + return category === "attention"; +} + +/** + * In-chat delivery is always on and absent here. These two are independent + * opt-ins, never a radio group. + */ +export const watchFollowUpSchema = z.object({ + /** Open an investigation when the outcome is an attention one. */ + investigateOnAttention: z.boolean().default(false), + /** Attach an external delivery subscription (email). */ + notifyExternally: z.boolean().default(false), +}); + +export type WatchFollowUp = z.infer; + +/** What the card submits. */ +export const watchDraftSchema = z.object({ + spec: watchSpecSchema, + followUp: watchFollowUpSchema, +}); + +export type WatchDraft = z.infer; + +export const watchExternalNotificationSchema = z.discriminatedUnion("status", [ + z.object({ status: z.literal("enabled") }), + z.object({ status: z.literal("not_requested") }), + z.object({ status: z.literal("unavailable"), reason: z.string() }), +]); + +export type WatchExternalNotification = z.infer; +export type WatchExternalNotificationStatus = WatchExternalNotification["status"]; + +/** The window lengths the card offers, in hours. Capped by {@link WATCH_MAX_HOURS}. */ +export const WATCH_WINDOW_HOURS_OPTIONS = [0.5, 1, 2, 6, 12, 24] as const; + +const RUN_STATE_KINDS = ["run_start", "run_finished", "run_failed"] as const; + +export function isRunStateWatchKind(kind: WatchKind): boolean { + return (RUN_STATE_KINDS as readonly string[]).includes(kind); +} + +/** Must stay in step with the cadence schemas, or the picker offers invalid options. */ +export function watchCadenceOptions(kind: WatchKind): readonly number[] { + return isRunStateWatchKind(kind) ? [1, 5, 15, 60] : [5, 15, 60]; +} + +// One family per array, in the order the picker lists them. +const RUN_CONDITION_VARIANTS = ["run_finished", "run_failed"] as const; +const QUEUE_CONDITION_VARIANTS = [ + "backlog_drain", + "queue_depth_above", + "queue_depth_below", + "queue_stalled", + "queue_oldest_age", +] as const; + +export function watchConditionVariants(kind: WatchKind): readonly WatchKind[] { + if ((RUN_CONDITION_VARIANTS as readonly string[]).includes(kind)) return RUN_CONDITION_VARIANTS; + if ((QUEUE_CONDITION_VARIANTS as readonly string[]).includes(kind)) { + return QUEUE_CONDITION_VARIANTS; + } + return [kind]; +} + +export const WATCH_DEFAULT_QUEUE_THRESHOLD = 100; + +/** In minutes. Must match the threshold the queue page tints Oldest wait at. */ +export const WATCH_DEFAULT_QUEUE_AGE_MINUTES = 5; diff --git a/internal-packages/dashboard-agent-db/drizzle/0002_watches_and_chat_messages.sql b/internal-packages/dashboard-agent-db/drizzle/0002_watches_and_chat_messages.sql index fd9ee099663..f05ac7039a2 100644 --- a/internal-packages/dashboard-agent-db/drizzle/0002_watches_and_chat_messages.sql +++ b/internal-packages/dashboard-agent-db/drizzle/0002_watches_and_chat_messages.sql @@ -88,6 +88,9 @@ CREATE TABLE "trigger_dashboard_agent"."watches" ( --> statement-breakpoint ALTER TABLE "trigger_dashboard_agent"."chats" ADD COLUMN IF NOT EXISTS "last_read_at" timestamp with time zone; --> statement-breakpoint +-- Chats that predate the column start read, so only activity after this migration lights the dot. +UPDATE "trigger_dashboard_agent"."chats" SET "last_read_at" = coalesce("last_message_at", "created_at") WHERE "last_read_at" IS NULL; +--> statement-breakpoint ALTER TABLE "trigger_dashboard_agent"."chats" ADD COLUMN IF NOT EXISTS "next_message_position" integer DEFAULT 1 NOT NULL; --> statement-breakpoint CREATE INDEX "chat_messages_chat_user_role_idx" ON "trigger_dashboard_agent"."chat_messages" USING btree ("chat_id","message_id") WHERE "trigger_dashboard_agent"."chat_messages"."role" = 'user'; diff --git a/internal-packages/dashboard-agent-db/src/ids.ts b/internal-packages/dashboard-agent-db/src/ids.ts index c2bfb964754..4ac6a37b54b 100644 --- a/internal-packages/dashboard-agent-db/src/ids.ts +++ b/internal-packages/dashboard-agent-db/src/ids.ts @@ -1,4 +1,4 @@ -import { randomInt } from "node:crypto"; +import { createHash, randomInt } from "node:crypto"; // Same shape as the platform's `generateFriendlyId`, on `node:crypto` so this leaf // package stays dependency-free. @@ -14,4 +14,23 @@ export function generateId(prefix: string, size: number = SIZE): string { } export const generateInvestigationId = () => generateId("inv"); + +/** + * The id of the investigation a consented watch opens, derived from the watch itself. + * + * The wake seeds the row and a later action revises it, in different runs with no + * hand-off between them, so the id has to be a function of the watch — otherwise the + * second lane can only guess, and guessing picks up the user's own open card. A watch + * reaches exactly one terminal outcome, so the watch id alone is the whole key. + */ +export function watchInvestigationId(watchId: string): string { + const digest = createHash("sha256") + .update(`dashboard-agent:watch-investigation:${watchId}`) + .digest(); + let body = ""; + for (let i = 0; i < SIZE; i++) body += ALPHABET[digest[i]! % ALPHABET.length]; + return `inv_${body}`; +} export const generateWatchId = () => generateId("watch"); +/** Fencing token for one wake-delivery claim. */ +export const generateWatchDeliveryClaimId = () => generateId("wdc"); diff --git a/internal-packages/dashboard-agent-db/src/internal.ts b/internal-packages/dashboard-agent-db/src/internal.ts index cb54b31993a..a9dc4c6531b 100644 --- a/internal-packages/dashboard-agent-db/src/internal.ts +++ b/internal-packages/dashboard-agent-db/src/internal.ts @@ -1,7 +1,18 @@ +import { sql } from "drizzle-orm"; import type { DashboardAgentDb } from "./client.js"; -// Shared by the query modules. Not part of the package's surface. +// Shared by `queries.ts` and `watch-queries.ts`. Not part of the package's surface. export type DashboardAgentDbOrTx = | DashboardAgentDb | Parameters[0]>[0]; + +/** Advisory-lock namespace (ASCII `watc`), so keys can't collide with another lock. */ +const WATCH_CHAT_LOCK_NAMESPACE = 0x77617463; + +/** Serializes creating a watch against deleting the chat under it. Transaction-scoped. */ +export function lockChatForWatches(tx: DashboardAgentDbOrTx, chatId: string) { + return tx.execute( + sql`select pg_advisory_xact_lock(${WATCH_CHAT_LOCK_NAMESPACE}, hashtext(${chatId}))` + ); +} diff --git a/internal-packages/dashboard-agent-db/src/queries.ts b/internal-packages/dashboard-agent-db/src/queries.ts index 592eec37790..34ab919255a 100644 --- a/internal-packages/dashboard-agent-db/src/queries.ts +++ b/internal-packages/dashboard-agent-db/src/queries.ts @@ -1,8 +1,12 @@ -import { investigationBlockSchema, VIEW_BLOCK_VERSION } from "@internal/dashboard-agent-contracts"; -import { and, desc, eq, inArray, ne, sql, isNull, type SQL } from "drizzle-orm"; +import { + investigationBlockSchema, + VIEW_BLOCK_VERSION, + WATCH_REQUEST_MESSAGE_ID_PREFIX, +} from "@internal/dashboard-agent-contracts"; +import { and, desc, eq, inArray, ne, notLike, sql, isNull, type SQL } from "drizzle-orm"; import type { DashboardAgentDb } from "./client.js"; import { generateInvestigationId } from "./ids.js"; -import { type DashboardAgentDbOrTx } from "./internal.js"; +import { lockChatForWatches, type DashboardAgentDbOrTx } from "./internal.js"; import { chatMessages, chats, @@ -12,7 +16,13 @@ import { type ChatSession, type Investigation, type NewChatTurnEval, + type Watch, } from "./schema.js"; +import { cancelActiveWatchesForChat } from "./watch-queries.js"; + +// The watch, wake and batch-chain queries live in `watch-queries.js`, re-exported +// here so every existing import path still resolves. +export * from "./watch-queries.js"; // Every query that touches user data must be scoped by `organizationId` and/or // `userId`. This file is where tenant isolation lives. @@ -24,6 +34,8 @@ export interface ChatListItem { title: string; pinnedAt: Date | null; lastMessageAt: Date | null; + /** When the owner last had this chat open. Older than `lastMessageAt` means unread. */ + lastReadAt: Date | null; createdAt: Date; updatedAt: Date; metadata: Record; @@ -40,6 +52,7 @@ export async function listChats( title: chats.title, pinnedAt: chats.pinnedAt, lastMessageAt: chats.lastMessageAt, + lastReadAt: chats.lastReadAt, createdAt: chats.createdAt, updatedAt: chats.updatedAt, metadata: chats.metadata, @@ -83,6 +96,8 @@ export async function getChatMessages( /** * Counted from the stored messages, not a counter column, so a deleted chat stops * counting. `excludeChatId` is for a caller that counts that chat's live messages itself. + * A watch's consent record is a user message but not a turn the user spent, so it is + * excluded here and by the client-side count. */ export async function countUserMessages( db: DashboardAgentDb, @@ -98,12 +113,37 @@ export async function countUserMessages( eq(chats.userId, params.userId), isNull(chats.deletedAt), eq(chatMessages.role, "user"), + notLike(chatMessages.messageId, `${WATCH_REQUEST_MESSAGE_ID_PREFIX}%`), params.excludeChatId ? ne(chatMessages.chatId, params.excludeChatId) : undefined ) ); return rows[0]?.count ?? 0; } +/** + * Chats whose transcript moved on after their owner last looked. A watch wake is one way + * that happens; an answer that landed while the panel was closed is another, and the panel + * shows them the same way — a dot on the launcher, the chat lifted and highlighted. + */ +export async function countChatsWithUnreadWork( + db: DashboardAgentDb, + params: { organizationId: string; userId: string } +): Promise { + const rows = await db + .select({ count: sql`count(*)::int` }) + .from(chats) + .where( + and( + eq(chats.organizationId, params.organizationId), + eq(chats.userId, params.userId), + isNull(chats.deletedAt), + sql`${chats.lastMessageAt} is not null`, + sql`(${chats.lastReadAt} is null or ${chats.lastMessageAt} > ${chats.lastReadAt})` + ) + ); + return rows[0]?.count ?? 0; +} + /** Joins `chats` to scope by owner, because `chat_sessions` has no `userId`. */ export async function getSession( db: DashboardAgentDb, @@ -220,18 +260,51 @@ export async function setChatPinned( ); } -/** Owner-scoped: a client chatId can only delete the caller's own chat. */ +/** Owner-scoped: a client chatId can only clear the caller's own unread state. */ +export async function markChatRead( + db: DashboardAgentDb, + params: { chatId: string; userId: string; organizationId: string; at?: Date } +): Promise { + await db + .update(chats) + .set({ lastReadAt: params.at ?? sql`now()` }) + .where( + and( + eq(chats.id, params.chatId), + eq(chats.userId, params.userId), + eq(chats.organizationId, params.organizationId) + ) + ); +} + +/** + * One transaction on purpose: a crash between the two halves would leave live + * watches ticking against a chat the user can no longer see. Owner-scoped. + */ export async function softDeleteChat( db: DashboardAgentDb, params: { chatId: string; userId: string } -): Promise<{ deleted: boolean }> { - const deleted = await db - .update(chats) - .set({ deletedAt: sql`now()`, updatedAt: sql`now()` }) - .where(and(eq(chats.id, params.chatId), eq(chats.userId, params.userId))) - .returning({ id: chats.id }); +): Promise<{ deleted: boolean; cancelledWatches: Watch[] }> { + return db.transaction(async (tx) => { + // The same lock `createWatch` takes, or a concurrent create lands an active + // watch on a chat this transaction already deleted. + await lockChatForWatches(tx, params.chatId); + + const deleted = await tx + .update(chats) + .set({ deletedAt: sql`now()`, updatedAt: sql`now()` }) + .where(and(eq(chats.id, params.chatId), eq(chats.userId, params.userId))) + .returning({ id: chats.id }); + + if (deleted.length === 0) return { deleted: false, cancelledWatches: [] }; + + const cancelledWatches = await cancelActiveWatchesForChat(tx, { + chatId: params.chatId, + reason: "chat_deleted", + }); - return { deleted: deleted.length > 0 }; + return { deleted: true, cancelledWatches }; + }); } /** Enough of the payload to recognise it in an error, without logging a whole transcript. */ @@ -472,7 +545,7 @@ export async function appendChatMessageOnce( chatId: string; userId: string; organizationId?: string; - message: { id: string }; + message: { id: string; role: string }; } ): Promise { return appendOneMessage(db, { @@ -491,7 +564,7 @@ export async function appendChatMessageOnce( */ export async function appendChatMessageOnceByChatId( db: DashboardAgentDbOrTx, - params: { chatId: string; message: { id: string } } + params: { chatId: string; message: { id: string; role: string } } ): Promise { return appendOneMessage(db, { chatId: params.chatId, message: params.message, scope: [] }); } @@ -695,6 +768,65 @@ export async function upsertInvestigationRevision( return { ok: false, error: existing.length > 0 ? "context_mismatch" : "not_found" }; } +export type SeedInvestigationResult = + | { ok: true; id: string; created: boolean } + | { ok: false; error: "context_mismatch" }; + +/** + * Open an investigation under an id the caller chose, or report that it is already open. + * + * The wake and the investigating lane both call this with the same derived id, so + * whichever runs first opens the row and the other one finds it. A row under that id in + * another chat or environment is refused rather than revised. + */ +export async function seedInvestigation( + db: DashboardAgentDbOrTx, + params: { + id: string; + chatId: string; + projectRef: string; + environmentRef: string; + state: unknown; + } +): Promise { + const inserted = await db + .insert(investigations) + .values({ + id: params.id, + chatId: params.chatId, + projectRef: params.projectRef, + environmentRef: params.environmentRef, + revision: 0, + state: params.state, + }) + .onConflictDoNothing({ target: investigations.id }) + .returning({ id: investigations.id }); + + if (inserted[0]) return { ok: true, id: inserted[0].id, created: true }; + + const rows = await db + .select({ + id: investigations.id, + chatId: investigations.chatId, + projectRef: investigations.projectRef, + environmentRef: investigations.environmentRef, + }) + .from(investigations) + .where(eq(investigations.id, params.id)) + .limit(1); + + const existing = rows[0]; + if ( + !existing || + existing.chatId !== params.chatId || + existing.projectRef !== params.projectRef || + existing.environmentRef !== params.environmentRef + ) { + return { ok: false, error: "context_mismatch" }; + } + return { ok: true, id: existing.id, created: false }; +} + /** Structural: this package stores the transcript, the UI types it. */ export type InvestigationCardMessage = { id: string; @@ -764,30 +896,6 @@ export async function getInvestigation( return rows[0] ?? null; } -/** - * The wake-to-turn hand-off: the turn revises this row instead of opening a second - * card. The window keeps an abandoned card from being picked up as this watch's. - */ -export async function findOpenInvestigationForChat( - db: DashboardAgentDb, - params: { chatId: string; createdAfter: Date } -): Promise { - const rows = await db - .select() - .from(investigations) - .where( - and( - eq(investigations.chatId, params.chatId), - sql`${investigations.state}->>'outcome' = 'in_progress'`, - // A string bind: postgres-js won't serialize a Date into a raw fragment. - sql`${investigations.createdAt} >= ${params.createdAfter.toISOString()}::timestamptz` - ) - ) - .orderBy(desc(investigations.createdAt)) - .limit(1); - return rows[0] ?? null; -} - export async function listInvestigationsForChat( db: DashboardAgentDb, params: { chatId: string; limit?: number } diff --git a/internal-packages/dashboard-agent-db/src/watch-queries.ts b/internal-packages/dashboard-agent-db/src/watch-queries.ts new file mode 100644 index 00000000000..39364511d16 --- /dev/null +++ b/internal-packages/dashboard-agent-db/src/watch-queries.ts @@ -0,0 +1,1236 @@ +import { and, desc, eq, inArray, or, sql, isNull } from "drizzle-orm"; +import { + watchResolutionToWireStatus, + type WatchExternalNotification, + type WatchObservedOutcome, + type WatchResolution, +} from "@internal/dashboard-agent-contracts"; +import type { DashboardAgentDb } from "./client.js"; +import { generateWatchDeliveryClaimId, generateWatchId } from "./ids.js"; +import { lockChatForWatches, type DashboardAgentDbOrTx } from "./internal.js"; +import { chats } from "./schema.js"; +import { + watchBatches, + watches, + watchSubmissions, + type PersistedWatchSpec, + type Watch, + type WatchBatch, + type WatchCancelReason, + type WatchStatus, + type WatchSubmission, + type WatchSubmissionState, +} from "./watch-schema.js"; + +// The watch, wake and batch-chain half of the query layer. Same tenancy rule as +// `queries.ts`: every read is scoped by organization and/or user. + +export const MAX_ACTIVE_WATCHES_PER_CHAT = 3; + +/** Terminal statuses are immutable. Every transition guards on `active`. */ +export function isTerminalWatchStatus(status: string): boolean { + return status === "fired" || status === "expired" || status === "cancelled"; +} + +/** Whether a claim is still someone's to hold is {@link claimWatchDelivery}'s call. */ +export function isWatchDeliveryOwed(status: string): boolean { + return status === "pending" || status === "delivering"; +} + +export type CreateWatchResult = + | { ok: true; watch: Watch } + | { ok: false; error: "limit_reached"; activeCount: number } + | { ok: false; error: "duplicate"; existingId: string | null } + /** The chat is gone (or was deleted while this create was in flight). */ + | { ok: false; error: "chat_not_found" }; + +const PG_UNIQUE_VIOLATION = "23505"; + +function isUniqueViolation(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { code?: unknown }).code === PG_UNIQUE_VIOLATION + ); +} + +/** + * Dedup is guaranteed by `watches_chat_active_identity_key`, not the pre-check below. + * The cap holds because the advisory lock makes count-then-insert atomic. + */ +export async function createWatch( + db: DashboardAgentDb, + params: { + chatId: string; + identity: string; + spec: PersistedWatchSpec; + organizationId: string; + projectId: string; + /** The project's external `proj_…` ref, which a wake scopes an investigation by. */ + projectRef?: string | null; + environmentId: string; + userId: string; + expiresAt: Date; + /** Consent, given at creation, to investigate after an attention outcome. */ + investigateOnAttention?: boolean; + id?: string; + } +): Promise { + try { + return await db.transaction(async (tx) => { + // Makes count-then-insert atomic against a concurrent create and a delete. + await lockChatForWatches(tx, params.chatId); + + // Re-read under the lock, or a delete that committed while this call was + // validating gets overtaken by the insert below. Owner-scoped: a chat in another + // org, or another user's, does not exist for this create. + const chat = await tx + .select({ id: chats.id }) + .from(chats) + .where( + and( + eq(chats.id, params.chatId), + eq(chats.organizationId, params.organizationId), + eq(chats.userId, params.userId), + isNull(chats.deletedAt) + ) + ) + .limit(1); + if (chat.length === 0) return { ok: false, error: "chat_not_found" } as const; + + const active = await tx + .select({ + id: watches.id, + identity: watches.identity, + projectId: watches.projectId, + environmentId: watches.environmentId, + }) + .from(watches) + .where(and(eq(watches.chatId, params.chatId), eq(watches.status, "active"))); + + const duplicate = active.find( + (w) => + w.identity === params.identity && + w.projectId === params.projectId && + w.environmentId === params.environmentId + ); + if (duplicate) { + return { ok: false, error: "duplicate", existingId: duplicate.id }; + } + + if (active.length >= MAX_ACTIVE_WATCHES_PER_CHAT) { + return { ok: false, error: "limit_reached", activeCount: active.length }; + } + + const rows = await tx + .insert(watches) + .values({ + id: params.id ?? generateWatchId(), + chatId: params.chatId, + identity: params.identity, + spec: params.spec, + organizationId: params.organizationId, + projectId: params.projectId, + projectRef: params.projectRef ?? null, + environmentId: params.environmentId, + userId: params.userId, + expiresAt: params.expiresAt, + investigateOnAttention: params.investigateOnAttention ?? false, + }) + .returning(); + + return { ok: true, watch: rows[0]! }; + }); + } catch (error) { + if (!isUniqueViolation(error)) throw error; + // Lost the dedup race. Null if the winner went terminal in the meantime. + const existing = await findActiveWatchByIdentity(db, params); + return { ok: false, error: "duplicate", existingId: existing?.id ?? null }; + } +} + +/** + * Advisory only and not race-proof. {@link createWatch} re-applies both guardrails + * atomically and remains the authority. + */ +export async function precheckWatchCreation( + db: DashboardAgentDb, + params: { chatId: string; projectId: string; environmentId: string; identity: string } +): Promise< + | { ok: true } + | { ok: false; error: "limit_reached"; activeCount: number } + | { ok: false; error: "duplicate"; existingId: string } +> { + const active = await db + .select({ + id: watches.id, + identity: watches.identity, + projectId: watches.projectId, + environmentId: watches.environmentId, + }) + .from(watches) + .where(and(eq(watches.chatId, params.chatId), eq(watches.status, "active"))); + + const duplicate = active.find( + (w) => + w.identity === params.identity && + w.projectId === params.projectId && + w.environmentId === params.environmentId + ); + if (duplicate) return { ok: false, error: "duplicate", existingId: duplicate.id }; + + if (active.length >= MAX_ACTIVE_WATCHES_PER_CHAT) { + return { ok: false, error: "limit_reached", activeCount: active.length }; + } + + return { ok: true }; +} + +/** Covered by `watches_chat_active_identity_key`. */ +export async function findActiveWatchByIdentity( + db: DashboardAgentDb, + params: { chatId: string; projectId: string; environmentId: string; identity: string } +): Promise { + const rows = await db + .select() + .from(watches) + .where( + and( + eq(watches.chatId, params.chatId), + eq(watches.projectId, params.projectId), + eq(watches.environmentId, params.environmentId), + eq(watches.identity, params.identity), + eq(watches.status, "active") + ) + ) + .limit(1); + return rows[0] ?? null; +} + +export async function getWatch( + db: DashboardAgentDb, + params: { id: string } +): Promise { + const rows = await db.select().from(watches).where(eq(watches.id, params.id)).limit(1); + return rows[0] ?? null; +} + +/* ------------------------------------------------------------------ * + * The submission ledger + * ------------------------------------------------------------------ */ + +export interface WatchSubmissionClaim { + submission: WatchSubmission; + /** This call inserted the row, so no earlier attempt exists. */ + claimed: boolean; +} + +/** + * Reserve the ledger row for one submission, or return the row an earlier attempt left. + * The insert is the mutual exclusion: `(chat_id, client_request_id)` is the primary key, + * so exactly one attempt is ever the first, and the rest read its outcome. + */ +export async function claimWatchSubmission( + db: DashboardAgentDb, + params: { + chatId: string; + clientRequestId: string; + organizationId: string; + userId: string; + projectId: string; + environmentId: string; + draftHash: string; + draft: Record; + /** Reserved up front, so a converging retry finds the watch by id. */ + watchId: string; + } +): Promise { + const inserted = await db + .insert(watchSubmissions) + .values({ ...params, state: "pending" }) + .onConflictDoNothing({ + target: [watchSubmissions.chatId, watchSubmissions.clientRequestId], + }) + .returning(); + + if (inserted[0]) return { submission: inserted[0], claimed: true }; + + const existing = await getWatchSubmission(db, params); + // Only reachable if retention deleted the row between the insert and this read. + if (!existing) throw new Error("Watch submission vanished between insert and read"); + return { submission: existing, claimed: false }; +} + +export async function getWatchSubmission( + db: DashboardAgentDb, + params: { chatId: string; clientRequestId: string } +): Promise { + const rows = await db + .select() + .from(watchSubmissions) + .where( + and( + eq(watchSubmissions.chatId, params.chatId), + eq(watchSubmissions.clientRequestId, params.clientRequestId) + ) + ) + .limit(1); + return rows[0] ?? null; +} + +/** + * Re-open a refused submission for another attempt, with a fresh reserved watch id: the + * previous one may already name a cancelled row. A refusal has no side effect to repeat, + * which is what makes this safe; `created` and `immediate` are never re-opened. + */ +export async function reopenWatchSubmission( + db: DashboardAgentDb, + params: { chatId: string; clientRequestId: string; watchId: string } +): Promise { + const rows = await db + .update(watchSubmissions) + .set({ + state: "pending", + watchId: params.watchId, + unavailable: false, + externalNotificationStatus: "not_requested", + externalNotificationReason: null, + immediateResult: null, + refusalCode: null, + refusalError: null, + refusalExistingId: null, + updatedAt: sql`now()`, + }) + .where( + and( + eq(watchSubmissions.chatId, params.chatId), + eq(watchSubmissions.clientRequestId, params.clientRequestId), + eq(watchSubmissions.state, "refused") + ) + ) + .returning(); + return rows[0] ?? null; +} + +export interface WatchSubmissionOutcome { + state: Exclude; + watchId?: string | null; + unavailable?: boolean; + /** What became of the external consent. Replayed verbatim, never re-decided. */ + external?: WatchExternalNotification; + immediateResult?: string | null; + refusalCode?: string | null; + refusalError?: string | null; + refusalExistingId?: string | null; +} + +/** + * Write the outcome, guarded on `pending`, so the first attempt to finish wins and a + * concurrent one reads its record instead of overwriting it. `null` means it lost. + */ +export async function recordWatchSubmissionOutcome( + db: DashboardAgentDb, + params: { chatId: string; clientRequestId: string } & WatchSubmissionOutcome +): Promise { + const rows = await db + .update(watchSubmissions) + .set({ + state: params.state, + ...(params.watchId !== undefined ? { watchId: params.watchId } : {}), + unavailable: params.unavailable ?? false, + externalNotificationStatus: params.external?.status ?? "not_requested", + externalNotificationReason: + params.external?.status === "unavailable" ? params.external.reason : null, + immediateResult: params.immediateResult ?? null, + refusalCode: params.refusalCode ?? null, + refusalError: params.refusalError ?? null, + refusalExistingId: params.refusalExistingId ?? null, + updatedAt: sql`now()`, + }) + .where( + and( + eq(watchSubmissions.chatId, params.chatId), + eq(watchSubmissions.clientRequestId, params.clientRequestId), + eq(watchSubmissions.state, "pending") + ) + ) + .returning(); + return rows[0] ?? null; +} + +/** Retention. A submission outlives its watch only as the key that stops a re-create. */ +export async function deleteWatchSubmissionsOlderThan( + db: DashboardAgentDb, + params: { before: Date; limit?: number } +): Promise { + const eligible = db + .select({ ctid: sql`ctid` }) + .from(watchSubmissions) + .where(sql`${watchSubmissions.createdAt} <= ${params.before.toISOString()}::timestamptz`) + .limit(params.limit ?? 500); + + const deleted = await db + .delete(watchSubmissions) + .where(sql`ctid in ${eligible}`) + .returning({ chatId: watchSubmissions.chatId }); + + return deleted.length; +} + +/** Covered by `watches_chat_active_identity_key`, which leads with `chat_id`. */ +export async function listActiveWatchesForChat( + db: DashboardAgentDb, + params: { chatId: string } +): Promise { + return db + .select() + .from(watches) + .where(and(eq(watches.chatId, params.chatId), eq(watches.status, "active"))) + .orderBy(desc(watches.createdAt)); +} + +export interface ActiveWatchSummary { + id: string; + chatId: string; + identity: string; + status: WatchStatus; + kind: string; + note: string; + checkEveryMinutes: number; + expiresAt: Date; + /** The last check's reason: tells `terminal_unsatisfied` apart from a timeout. */ + endedReason: string | null; + /** NULL while active and for every cancellation. */ + resolution: WatchResolution | null; + observedOutcome: WatchObservedOutcome | null; +} + +/** + * Returns every non-cancelled watch, not only the active ones: the wake banner needs + * an already-fired watch's kind. Tenancy floor is the join, not the caller's chat ids. + */ +export async function listActiveWatchesForChats( + db: DashboardAgentDb, + params: { chatIds: string[]; organizationId: string; userId: string } +): Promise> { + if (params.chatIds.length === 0) return {}; + + const rows = await db + .select({ + id: watches.id, + chatId: watches.chatId, + identity: watches.identity, + status: watches.status, + spec: watches.spec, + expiresAt: watches.expiresAt, + lastResult: watches.lastResult, + resolution: watches.resolution, + observedOutcome: watches.observedOutcome, + }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where( + and( + inArray(watches.chatId, params.chatIds), + inArray(watches.status, ["active", "fired", "expired"]), + eq(chats.organizationId, params.organizationId), + eq(chats.userId, params.userId), + isNull(chats.deletedAt) + ) + ) + .orderBy(desc(watches.createdAt)); + + const byChat: Record = {}; + for (const row of rows) { + (byChat[row.chatId] ??= []).push({ + id: row.id, + chatId: row.chatId, + identity: row.identity, + status: row.status, + kind: row.spec.kind, + note: row.spec.note, + checkEveryMinutes: row.spec.checkEveryMinutes, + expiresAt: row.expiresAt, + endedReason: typeof row.lastResult?.reason === "string" ? row.lastResult.reason : null, + resolution: row.resolution, + observedOutcome: row.observedOutcome, + }); + } + return byChat; +} + +/** When the watch resolved: `fired_at` for a fire, `last_checked_at` for an expiry. */ +const wakeResolvedAt = sql`coalesce(${watches.firedAt}, ${watches.lastCheckedAt})`; + +/** The wake landed after the chat was last read. Never-read chats count as unread. */ +const unreadWake = sql`(${chats.lastReadAt} is null or ${wakeResolvedAt} > ${chats.lastReadAt})`; + +/** + * Shared by the three wake queries so they can't drift. Org and user are asserted on + * the watch row too, so `watches_org_user_wake_idx` narrows before the join runs. + */ +function deliveredWakeScope(params: { organizationId: string; userId: string }) { + return [ + inArray(watches.status, ["fired", "expired"]), + eq(watches.deliveryStatus, "delivered"), + eq(watches.organizationId, params.organizationId), + eq(watches.userId, params.userId), + eq(chats.organizationId, params.organizationId), + eq(chats.userId, params.userId), + isNull(chats.deletedAt), + ]; +} + +/** A wake is a `fired` or `expired` watch; a cancelled one is never narrated. */ +export async function countUnreadWatchWakes( + db: DashboardAgentDb, + params: { organizationId: string; userId: string } +): Promise { + const rows = await db + .select({ count: sql`count(*)::int` }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where(and(...deliveredWakeScope(params), unreadWake)); + return rows[0]?.count ?? 0; +} + +/** + * Whether this user has a watch that can still wake them here. Covered by + * `watches_org_user_active_idx`; a chat deletion cancels its watches, so `active` is enough. + */ +export async function hasActiveWatches( + db: DashboardAgentDb, + params: { organizationId: string; userId: string } +): Promise { + const rows = await db + .select({ one: sql`1` }) + .from(watches) + .where( + and( + eq(watches.status, "active"), + eq(watches.organizationId, params.organizationId), + eq(watches.userId, params.userId) + ) + ) + .limit(1); + return rows.length > 0; +} + +export interface DashboardAgentWakeActivity { + unreadWakes: number; + /** A watch is still running, so a wake can arrive in a tab that has never seen one. */ + hasActiveWatches: boolean; +} + +/** + * The page load's whole wake signal. Both halves are needed: a fresh browser with an active + * watch and no wake yet must still start polling, or its first wake only lands on a reload. + */ +export async function readDashboardAgentWakeActivity( + db: DashboardAgentDb, + params: { organizationId: string; userId: string } +): Promise { + const [unreadWakes, active] = await Promise.all([ + countUnreadWatchWakes(db, params), + hasActiveWatches(db, params), + ]); + return { unreadWakes, hasActiveWatches: active }; +} + +export interface UnreadWatchWake { + watchId: string; + chatId: string; + outcome: "fired" | "expired"; + /** The watch's note, or its identity when the note is blank. */ + note: string; + /** `fired_at` for a fire, `last_checked_at` for an expiry. */ + firedAt: Date; + kind: string; + identity: string; + /** Null on a row written before the resolution model. The surface falls back. */ + resolution: WatchResolution | null; + observedOutcome: WatchObservedOutcome | null; + /** Landed after the chat's read marker. Only the dot cares. */ + unread: boolean; +} + +const UNREAD_WAKE_LIST_LIMIT = 10; + +/** Same wake definition and scoping as {@link countUnreadWatchWakes}. */ +export async function listRecentWatchWakes( + db: DashboardAgentDb, + params: { organizationId: string; userId: string; deliveredAfter: Date } +): Promise { + const rows = await db + .select({ + watchId: watches.id, + chatId: watches.chatId, + status: watches.status, + identity: watches.identity, + spec: watches.spec, + resolution: watches.resolution, + observedOutcome: watches.observedOutcome, + resolvedAt: wakeResolvedAt, + unread: sql`${unreadWake}`, + }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where( + and( + ...deliveredWakeScope(params), + sql`${wakeResolvedAt} > ${params.deliveredAfter.toISOString()}::timestamptz` + ) + ) + .orderBy(desc(wakeResolvedAt)) + .limit(UNREAD_WAKE_LIST_LIMIT); + + return rows.map(toUnreadWatchWake); +} + +/** The wake row shape both wake readers select. */ +type WakeRow = { + watchId: string; + chatId: string; + status: WatchStatus; + identity: string; + spec: PersistedWatchSpec; + resolution: WatchResolution | null; + observedOutcome: WatchObservedOutcome | null; + resolvedAt: Date; + unread: boolean; +}; + +function toUnreadWatchWake(row: WakeRow): UnreadWatchWake { + return { + watchId: row.watchId, + chatId: row.chatId, + // Narrowed by the status `in` clause in the wake scope. + outcome: row.status as "fired" | "expired", + note: row.spec.note?.trim() || row.identity, + firedAt: new Date(row.resolvedAt), + kind: row.spec.kind, + identity: row.identity, + resolution: row.resolution, + observedOutcome: row.observedOutcome, + unread: row.unread, + }; +} + +/** + * The wake feed the dashboard polls: the recent wakes plus the unread total, in one query. The + * window count is evaluated before the limit, so it covers unread wakes older than the window. + */ +export async function readWatchWakeFeed( + db: DashboardAgentDb, + params: { organizationId: string; userId: string; deliveredAfter: Date } +): Promise<{ unreadWakes: number; wakes: UnreadWatchWake[] }> { + const rows = await db + .select({ + watchId: watches.id, + chatId: watches.chatId, + status: watches.status, + identity: watches.identity, + spec: watches.spec, + resolution: watches.resolution, + observedOutcome: watches.observedOutcome, + resolvedAt: wakeResolvedAt, + unread: sql`${unreadWake}`, + recent: sql`${wakeResolvedAt} > ${params.deliveredAfter.toISOString()}::timestamptz`, + unreadTotal: sql`(count(*) filter (where ${unreadWake}) over ())::int`, + }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where( + and( + ...deliveredWakeScope(params), + or(unreadWake, sql`${wakeResolvedAt} > ${params.deliveredAfter.toISOString()}::timestamptz`) + ) + ) + .orderBy(desc(wakeResolvedAt)) + .limit(UNREAD_WAKE_LIST_LIMIT); + + return { + unreadWakes: rows[0]?.unreadTotal ?? 0, + // Newest first, so the windowed rows are a prefix of the ordered result. + wakes: rows.filter((row) => row.recent).map(toUnreadWatchWake), + }; +} + +/** Same wake definition and scoping as {@link countUnreadWatchWakes}, grouped. */ +export async function listChatIdsWithUnreadWakes( + db: DashboardAgentDb, + params: { organizationId: string; userId: string } +): Promise> { + const rows = await db + .selectDistinct({ chatId: watches.chatId }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where(and(...deliveredWakeScope(params), unreadWake)); + return new Set(rows.map((row) => row.chatId)); +} + +export interface ChatWatchContext { + organizationId: string; +} + +/** + * Deliberately returns no project or environment: a watch is bound to the requesting + * turn's environment. The org is the immutable tenancy floor its watches can't leave. + */ +export async function getChatWatchContext( + db: DashboardAgentDb, + params: { chatId: string; userId: string } +): Promise { + const rows = await db + .select({ organizationId: chats.organizationId }) + .from(chats) + .where( + and(eq(chats.id, params.chatId), eq(chats.userId, params.userId), isNull(chats.deletedAt)) + ) + .limit(1); + + const chat = rows[0]; + if (!chat) return null; + + return { organizationId: chat.organizationId }; +} + +/** + * Only an `active` row transitions, so a check racing the sweeper yields one winner. + * `status` is derived, never passed, so it can't disagree with the resolution. + */ +export async function transitionWatchCondition( + db: DashboardAgentDb, + params: { + id: string; + resolution: WatchResolution; + observedOutcome?: WatchObservedOutcome | null; + lastResult?: Record | null; + } +): Promise { + const status = watchResolutionToWireStatus(params.resolution); + const rows = await db + .update(watches) + .set({ + status, + resolution: params.resolution, + deliveryStatus: "pending", + lastCheckedAt: sql`now()`, + firedAt: status === "fired" ? sql`now()` : null, + ...(params.observedOutcome !== undefined ? { observedOutcome: params.observedOutcome } : {}), + ...(params.lastResult !== undefined ? { lastResult: params.lastResult } : {}), + }) + .where(and(eq(watches.id, params.id), eq(watches.status, "active"))) + .returning(); + return rows[0] ?? null; +} + +/** + * Cancellation is never notified. Guarded on `active`, so a watch that already fired + * keeps its outcome and its pending notification, and this returns `null`. + */ +export async function cancelWatch( + db: DashboardAgentDb, + params: { id: string; reason: WatchCancelReason } +): Promise { + const rows = await db + .update(watches) + .set({ + status: "cancelled", + cancelReason: params.reason, + cancelledAt: sql`now()`, + deliveryStatus: "not_required", + }) + .where(and(eq(watches.id, params.id), eq(watches.status, "active"))) + .returning(); + return rows[0] ?? null; +} + +/** For chat deletion, or the user losing access to the watched project. */ +export async function cancelActiveWatchesForChat( + db: DashboardAgentDbOrTx, + params: { chatId: string; reason: WatchCancelReason } +): Promise { + return db + .update(watches) + .set({ + status: "cancelled", + cancelReason: params.reason, + cancelledAt: sql`now()`, + deliveryStatus: "not_required", + }) + .where(and(eq(watches.chatId, params.chatId), eq(watches.status, "active"))) + .returning(); +} + +/** + * How long a `delivering` claim is respected before the wake may be claimed again. + * Much longer than a delivery takes, so it only releases rows whose deliverer died. + */ +export const WATCH_DELIVERY_CLAIM_STALE_MS = 5 * 60 * 1000; + +export interface WatchDeliveryClaim { + watch: Watch; + /** + * The fencing token. {@link releaseWatchDelivery} and {@link markWatchDelivered} + * only act while the row still carries it. + */ + claimId: string; +} + +/** + * The gate that keeps "exactly one wake" true: only the row this returns may append. + * Every claim writes a fresh `deliveryClaimId`, which is what makes takeover safe. + */ +export async function claimWatchDelivery( + db: DashboardAgentDb, + params: { id: string; staleBefore: Date } +): Promise { + const claimId = generateWatchDeliveryClaimId(); + const rows = await db + .update(watches) + .set({ deliveryStatus: "delivering", deliveryClaimedAt: sql`now()`, deliveryClaimId: claimId }) + .where( + and( + eq(watches.id, params.id), + sql`(${watches.deliveryStatus} = 'pending' or (${watches.deliveryStatus} = 'delivering' and coalesce(${watches.deliveryClaimedAt}, ${watches.createdAt}) <= ${params.staleBefore.toISOString()}::timestamptz))` + ) + ) + .returning(); + const watch = rows[0]; + return watch ? { watch, claimId } : null; +} + +/** + * Fenced on `claimId`, so a late release can't hand somebody else's in-flight claim + * back to `pending`. Guarded on `delivering`, so it can't un-deliver a landed wake. + */ +export async function releaseWatchDelivery( + db: DashboardAgentDb, + params: { id: string; claimId: string } +): Promise { + const rows = await db + .update(watches) + .set({ deliveryStatus: "pending", deliveryClaimedAt: null, deliveryClaimId: null }) + .where( + and( + eq(watches.id, params.id), + eq(watches.deliveryClaimId, params.claimId), + eq(watches.deliveryStatus, "delivering") + ) + ) + .returning(); + return rows[0] ?? null; +} + +/** + * With a `claimId` the mark is fenced on it. Without one it marks a `pending` row, + * never a `delivering` one: an unfenced mark must not finish a claim it doesn't own. + */ +export async function markWatchDelivered( + db: DashboardAgentDb, + params: { id: string; claimId?: string } +): Promise { + const rows = await db + .update(watches) + .set({ deliveryStatus: "delivered", deliveredAt: sql`now()` }) + .where( + and( + eq(watches.id, params.id), + params.claimId + ? and( + eq(watches.deliveryClaimId, params.claimId), + eq(watches.deliveryStatus, "delivering") + ) + : eq(watches.deliveryStatus, "pending") + ) + ) + .returning(); + return rows[0] ?? null; +} + +/** + * The only writer of `tickCount`, and resumable on purpose: it lands on the previous + * or current generation, never a later one, so a crashed tick can re-run. + */ +export async function claimWatchTick( + db: DashboardAgentDb, + params: { id: string; generation: number } +): Promise { + const rows = await db + .update(watches) + .set({ tickCount: params.generation }) + .where( + and( + eq(watches.id, params.id), + eq(watches.status, "active"), + inArray(watches.tickCount, [params.generation - 1, params.generation]) + ) + ) + .returning(); + return rows[0] ?? null; +} + +/** + * Deliberately does not touch `tickCount`, keeping {@link claimWatchTick} its single + * writer. Guarded on `active`, so a concurrent fire or expire wins and this no-ops. + */ +export async function recordWatchCheck( + db: DashboardAgentDb, + params: { + id: string; + lastResult?: Record | null; + /** Override the check timestamp; defaults to `now()`. */ + lastCheckedAt?: Date; + } +): Promise<{ tickCount: number; lastCheckedAt: Date | null } | null> { + const rows = await db + .update(watches) + .set({ + lastCheckedAt: params.lastCheckedAt ?? sql`now()`, + // A check is also a look, so the fairness key moves with it. + lastAttemptedAt: params.lastCheckedAt ?? sql`now()`, + ...(params.lastResult !== undefined ? { lastResult: params.lastResult } : {}), + }) + .where(and(eq(watches.id, params.id), eq(watches.status, "active"))) + .returning({ tickCount: watches.tickCount, lastCheckedAt: watches.lastCheckedAt }); + return rows[0] ?? null; +} + +/** + * A look that read nothing. Moves the batch's fairness key only, so a permanently broken + * reader rotates out of its group's head while its dueness and streak facts stay untouched. + * Guarded on `active`, like {@link recordWatchCheck}. + */ +export async function recordWatchAttempt( + db: DashboardAgentDb, + params: { id: string; lastAttemptedAt?: Date } +): Promise { + await db + .update(watches) + .set({ lastAttemptedAt: params.lastAttemptedAt ?? sql`now()` }) + .where(and(eq(watches.id, params.id), eq(watches.status, "active"))); +} + +/** + * Sweep for terminal watches `listExpiredActiveWatches` cannot see. `olderThan` is a + * grace window, so recovery can't race a path that is still mid-delivery. + */ +export async function listWatchesAwaitingDelivery( + db: DashboardAgentDb, + params: { olderThan: Date; limit?: number } +): Promise { + const olderThan = sql`${params.olderThan.toISOString()}::timestamptz`; + const rows = await db + .select({ watch: watches }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where( + and( + inArray(watches.status, ["fired", "expired"]), + sql`(${watches.deliveryStatus} = 'pending' or (${watches.deliveryStatus} = 'delivering' and coalesce(${watches.deliveryClaimedAt}, ${watches.createdAt}) <= ${olderThan}))`, + isNull(chats.deletedAt), + sql`coalesce(${watches.firedAt}, ${watches.lastCheckedAt}) <= ${olderThan}` + ) + ) + .orderBy(sql`coalesce(${watches.firedAt}, ${watches.lastCheckedAt})`) + .limit(params.limit ?? 100); + + return rows.map((row) => row.watch); +} + +/** The marker one terminal outcome's alert is dispatched under. */ +export function watchAlertDispatchKey(params: { id: string; terminalStatus: string }): string { + return `watch:${params.id}:alert:${params.terminalStatus}`; +} + +/** + * Claim the right to alert for this watch's terminal outcome, exactly once. The marker is + * on the row, so a repeated callback — a retried task, a replayed token — sends nothing. + * `false` means the alert was already dispatched; the caller answers success anyway. + */ +export async function claimWatchAlertDispatch( + db: DashboardAgentDb, + params: { id: string; terminalStatus: WatchStatus } +): Promise { + const key = watchAlertDispatchKey({ id: params.id, terminalStatus: params.terminalStatus }); + const rows = await db + .update(watches) + .set({ alertDispatchKey: key }) + .where( + and( + eq(watches.id, params.id), + // The row must still be in the outcome the caller is alerting for. + eq(watches.status, params.terminalStatus), + isNull(watches.alertDispatchKey) + ) + ) + .returning({ id: watches.id }); + return rows.length > 0; +} + +/** + * Hand the claim back when the dispatch it was taken for could not be queued, so the + * caller's retry can alert. Fenced on the key, so it can't clear a later claim. + */ +export async function releaseWatchAlertDispatch( + db: DashboardAgentDb, + params: { id: string; terminalStatus: WatchStatus } +): Promise { + const key = watchAlertDispatchKey({ id: params.id, terminalStatus: params.terminalStatus }); + await db + .update(watches) + .set({ alertDispatchKey: null }) + .where(and(eq(watches.id, params.id), eq(watches.alertDispatchKey, key))); +} + +/** + * Retention sweep. Guarded on settled delivery, so a row that still owes a wake is + * never deleted from under the delivery sweep. Only `watches` rows, never history. + */ +export async function deleteTerminalWatchesOlderThan( + db: DashboardAgentDb, + params: { before: Date; limit?: number } +): Promise { + const eligible = db + .select({ id: watches.id }) + .from(watches) + .where( + and( + inArray(watches.status, ["fired", "expired", "cancelled"]), + inArray(watches.deliveryStatus, ["not_required", "delivered"]), + // The materialized clock, so this is an index range scan on + // `watches_retention_idx` rather than a seq scan over an expression. + sql`${watches.retentionAt} <= ${params.before.toISOString()}::timestamptz` + ) + ) + .limit(params.limit ?? 500); + + const deleted = await db + .delete(watches) + .where(inArray(watches.id, eligible)) + .returning({ id: watches.id }); + + return deleted.length; +} + +/** + * Callers run the final boundary evaluation and resolve these via + * `transitionWatchCondition`, which may still be `condition_met`. + */ +export async function listExpiredActiveWatches( + db: DashboardAgentDb, + params: { now?: Date; limit?: number } = {} +): Promise { + return db + .select() + .from(watches) + .where( + and( + eq(watches.status, "active"), + // A string bind: postgres-js won't serialize a Date into a raw fragment. + params.now + ? sql`${watches.expiresAt} <= ${params.now.toISOString()}::timestamptz` + : sql`${watches.expiresAt} <= now()` + ) + ) + .orderBy(watches.expiresAt) + .limit(params.limit ?? 100); +} + +// Generated from `spec`, so it can't disagree with it, and indexed by +// `watches_active_env_cadence_idx`. +const watchCadenceMinutes = watches.cadenceMinutes; + +/** A group larger than this is checked across several ticks, oldest check first. */ +const BATCH_GROUP_LIMIT = 500; + +/** + * How long ago this watch was last looked at, a look that read nothing included. A + * never-looked-at watch sorts by creation. Dueness reads `lastCheckedAt` instead. + */ +const watchLastLookedAt = sql`coalesce(${watches.lastAttemptedAt}, ${watches.lastCheckedAt}, ${watches.createdAt})`; + +/** + * Which of these are due is the caller's decision, from the tick's own clock. + * + * Least-recently-looked-at first is the fairness invariant: a group over the cap rotates, so + * every watch is reached within `ceil(group / cap)` ticks instead of the same prefix winning + * every tick. Watches whose window closes within a cadence still sort first, so a group over + * the cap never defers a final evaluation. + */ +export async function listActiveWatchesForBatch( + db: DashboardAgentDb, + params: { environmentId: string; cadenceMinutes: number; limit?: number } +): Promise { + const closingSoon = sql`(${watches.expiresAt} <= now() + make_interval(mins => ${params.cadenceMinutes})) desc`; + + return db + .select() + .from(watches) + .where( + and( + eq(watches.status, "active"), + eq(watches.environmentId, params.environmentId), + eq(watchCadenceMinutes, params.cadenceMinutes) + ) + ) + .orderBy(closingSoon, watchLastLookedAt, watches.expiresAt) + .limit(params.limit ?? BATCH_GROUP_LIMIT); +} + +/** + * The batch's half of the delivery backstop: a retried run can't see a watch that left + * the `active` set mid-run. No grace window; the fenced claim settles racing deliverers. + */ +export async function listWatchesAwaitingDeliveryForBatch( + db: DashboardAgentDb, + params: { + environmentId: string; + cadenceMinutes: number; + claimStaleBefore: Date; + limit?: number; + } +): Promise { + const rows = await db + .select({ watch: watches }) + .from(watches) + .innerJoin(chats, eq(chats.id, watches.chatId)) + .where( + and( + eq(watches.environmentId, params.environmentId), + eq(watchCadenceMinutes, params.cadenceMinutes), + inArray(watches.status, ["fired", "expired"]), + sql`(${watches.deliveryStatus} = 'pending' or (${watches.deliveryStatus} = 'delivering' and coalesce(${watches.deliveryClaimedAt}, ${watches.createdAt}) <= ${params.claimStaleBefore.toISOString()}::timestamptz))`, + isNull(chats.deletedAt) + ) + ) + .orderBy(sql`coalesce(${watches.firedAt}, ${watches.lastCheckedAt})`) + .limit(params.limit ?? 100); + + return rows.map((row) => row.watch); +} + +/** + * A returned row means this call armed the chain, so the caller must trigger the run + * owning `epoch` / `generation + 1`. `null` means a live chain already covers the group. + */ +export async function armWatchBatch( + db: DashboardAgentDb, + params: { environmentId: string; cadenceMinutes: number; staleBefore: Date } +): Promise { + const rows = await db + .insert(watchBatches) + .values({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + epoch: 1, + generation: 0, + status: "running", + }) + .onConflictDoUpdate({ + target: [watchBatches.environmentId, watchBatches.cadenceMinutes], + set: { + epoch: sql`${watchBatches.epoch} + 1`, + generation: 0, + status: "running", + armedAt: sql`now()`, + lastTickAt: null, + }, + setWhere: sql`(${watchBatches.status} = 'stopped' or coalesce(${watchBatches.lastTickAt}, ${watchBatches.armedAt}) <= ${params.staleBefore.toISOString()}::timestamptz)`, + }) + .returning(); + return rows[0] ?? null; +} + +/** + * The batch twin of {@link claimWatchTick}, plus an epoch fence so a zombie chain from + * before a re-arm exits. Also the heartbeat the re-arm backstop reads. + */ +export async function claimWatchBatchTick( + db: DashboardAgentDb, + params: { + environmentId: string; + cadenceMinutes: number; + epoch: number; + generation: number; + } +): Promise { + const rows = await db + .update(watchBatches) + .set({ generation: params.generation, lastTickAt: sql`now()` }) + .where( + and( + eq(watchBatches.environmentId, params.environmentId), + eq(watchBatches.cadenceMinutes, params.cadenceMinutes), + eq(watchBatches.epoch, params.epoch), + eq(watchBatches.status, "running"), + inArray(watchBatches.generation, [params.generation - 1, params.generation]) + ) + ) + .returning(); + return rows[0] ?? null; +} + +/** + * Fenced on the epoch, so a stop decided by one epoch's run can't end the chain a + * later arm started. + */ +export async function stopWatchBatch( + db: DashboardAgentDb, + params: { environmentId: string; cadenceMinutes: number; epoch: number } +): Promise { + const rows = await db + .update(watchBatches) + .set({ status: "stopped" }) + .where( + and( + eq(watchBatches.environmentId, params.environmentId), + eq(watchBatches.cadenceMinutes, params.cadenceMinutes), + eq(watchBatches.epoch, params.epoch), + eq(watchBatches.status, "running") + ) + ) + .returning(); + return rows[0] ?? null; +} + +export interface WatchBatchGroup { + environmentId: string; + cadenceMinutes: number; +} + +/** + * Must stay in step with `watchBatchStaleMs`, which applies the same formula in + * TypeScript, or this listing and {@link armWatchBatch} disagree about dead chains. + */ +const batchHeartbeatDeadline = sql`make_interval(mins => ${watchCadenceMinutes} * 3 + 2)`; + +/** Input to the re-arm backstop. A chain ticking normally never appears here. */ +export async function listWatchBatchGroupsToArm( + db: DashboardAgentDb, + params: { now?: Date; limit?: number } = {} +): Promise { + const now = params.now ? sql`${params.now.toISOString()}::timestamptz` : sql`now()`; + + return db + .selectDistinct({ + environmentId: watches.environmentId, + cadenceMinutes: sql`${watchCadenceMinutes}`, + }) + .from(watches) + .leftJoin( + watchBatches, + and( + eq(watchBatches.environmentId, watches.environmentId), + sql`${watchBatches.cadenceMinutes} = ${watchCadenceMinutes}` + ) + ) + .where( + and( + eq(watches.status, "active"), + sql`(${watchBatches.environmentId} is null or ${watchBatches.status} = 'stopped' or coalesce(${watchBatches.lastTickAt}, ${watchBatches.armedAt}) <= ${now} - ${batchHeartbeatDeadline})` + ) + ) + .limit(params.limit ?? 200); +} diff --git a/internal-packages/dashboard-agent-db/src/watch-schema.ts b/internal-packages/dashboard-agent-db/src/watch-schema.ts index afa711d2a00..29202e06942 100644 --- a/internal-packages/dashboard-agent-db/src/watch-schema.ts +++ b/internal-packages/dashboard-agent-db/src/watch-schema.ts @@ -9,6 +9,12 @@ import { timestamp, uniqueIndex, } from "drizzle-orm/pg-core"; +import type { + WatchExternalNotificationStatus, + WatchObservedOutcome, + WatchResolution, + WatchSpec, +} from "@internal/dashboard-agent-contracts"; import { dashboardAgentSchema } from "./schema-base.js"; // The watch tables. Re-exported by `schema.ts`, which is what drizzle-kit reads. @@ -30,7 +36,7 @@ export type WatchCancelReason = | "superseded"; /** `since` is server-set at creation, so `error_recurrence` can't match older errors. */ -export type PersistedWatchSpec = Record & { since?: string }; +export type PersistedWatchSpec = WatchSpec & { since?: string }; /** * The initiating identity is snapshotted at creation, so a membership change can only @@ -53,9 +59,9 @@ export const watches = dashboardAgentSchema.table( * The meaning; `status` above stays the two-value transport encoding so persisted * wake ids and dedup keys remain valid. NULL while active and on cancellation. */ - resolution: text("resolution").$type(), + resolution: text("resolution").$type(), /** Written in the same statement as `resolution` and `lastResult`. */ - observedOutcome: jsonb("observed_outcome").$type>(), + observedOutcome: jsonb("observed_outcome").$type(), /** Consent given at creation. Never part of `identity`. */ investigateOnAttention: boolean("investigate_on_attention").notNull().default(false), // Immutable initiating identity, snapshotted at creation. @@ -223,7 +229,7 @@ export const watchSubmissions = dashboardAgentSchema.table( * is kept so a replay reproduces the same confirmation instead of guessing again. */ externalNotificationStatus: text("external_notification_status") - .$type() + .$type() .notNull() .default("not_requested"), externalNotificationReason: text("external_notification_reason"), diff --git a/internal-packages/dashboard-agent/GUIDEBOOK.md b/internal-packages/dashboard-agent/GUIDEBOOK.md new file mode 100644 index 00000000000..1648d9983c2 --- /dev/null +++ b/internal-packages/dashboard-agent/GUIDEBOOK.md @@ -0,0 +1,461 @@ +# Dashboard agent — guidebook + +An AI assistant in a side panel on every dashboard page. It reads your runs, +errors, queues, deploys and health through the same APIs you use, answers in +place, renders rich cards, and can keep watching things after the conversation +ends. Read-only by design: the only things it ever creates are its own watches +and, with your explicit yes, an email alert subscription. + +Branch: `feat/dashboard-agent-flows`. + +This is a reference of **conditions**: what makes each thing happen, and where +that is decided. It is written so you can predict the behaviour without running +anything. + +--- + +## What the agent does + +**Ask about your project** — runs, errors, queues, tasks, deploys, health. +Every number comes from a tool call; it never invents ids or figures. + +**Investigate** (flagship) — "why did this run fail?", "investigate this +error". It gathers evidence, poses falsifiable hypotheses, tests them, and +concludes on a live card with cited evidence — or says honestly that it's +inconclusive and what to check next. With a connected GitHub repo it reads your +actual source at the deployed commit and cites file:line. + +**Watch** — a durable condition the platform checks on a schedule (no LLM in +the checks), which wakes the chat with the outcome. Ten kinds, listed below. It +answers **once**, then stops. + +**Alerts** — when a watch is created (or fires) without a subscription, the +agent offers an email alert — one line, created only if you say yes. Standing +subscription on the standard alert channels: shows up on the project's Alerts +page, fires for every watch fire, one-click unsubscribe in every email. + +**Reports** — "is anything wrong right now?" renders the deterministic health +report as a card: severity, metric grid with sparklines, who owns the problem, +and a *Next steps* row of buttons. Stale telemetry is flagged and never trusted +for advice. + +**Navigate & query** — filtered page navigation, TRQL data questions with live +charts, deploy correlation, docs answers with source links. + +Every card in every state is browsable at `/storybook/agent-ui` (admin only), +with no LLM and no data. + +--- + +## When the agent is available at all + +`canAccessDashboardAgent` (`apps/webapp/app/v3/canAccessDashboardAgent.server.ts`) +grants access when either holds: + +- the viewer is an admin or impersonating **and** `DASHBOARD_AGENT_ADMIN_PREVIEW=1`; +- the `hasDashboardAgentAccess` feature flag resolves true for the org, whose + default is `DASHBOARD_AGENT_ENABLED=1`. + +Without access the panel provider is never mounted, so every Investigate and +Watch button returns `null` on its own — callers need no gate of their own +(`InvestigateButton.tsx`, `WatchButton.tsx`). + +Scheduling a watch needs one more thing: `DASHBOARD_AGENT_SECRET_KEY`. Without +it creation refuses with "The dashboard agent is not configured, so watches +can't be scheduled." (`isDashboardAgentConfigured`, `dashboardAgent.server.ts`). +The same gate stops the sweep delivering wakes — it still finalizes the rows. + +--- + +## Where the buttons are + +Two separate mechanisms, decided differently. + +### Buttons on the page itself + +| Page | Button | Shown when | +| --- | --- | --- | +| Queue | Investigate | the queue is degraded (see below) | +| Queue | Watch… | the queue is not paused | +| Error group | Investigate this error | always | +| Error group | Watch… | always | +| Run (span panel) | Watch… | the run is not in a final status | +| Run (span panel) | Investigate | the run has an error block **and** a failed status | +| Run, waiting block | Investigate | whenever that block is on screen, which is what "still waiting" means | +| Report card, *Next steps* | Watch recovery | the report is `health` and its severity is `warn` or `crit` | + +**Degraded**, for the Investigate button, is `isQueueDegraded` +(`apps/webapp/app/components/queues/queue-thresholds.ts`) — one predicate now +shared by the queue detail page, the queues list badge and the agent's page +mapper: + +1. a paused queue is never degraded — paused is a state, not a fault; +2. otherwise it is degraded if it is **at capacity**: `running >= limit` with a + non-empty queue, where `limit` is the queue's own concurrency limit else the + environment's. A limit of `0`, `null` or `undefined` is **never** at + capacity — zero capacity is not saturation, and `running >= 0` holds for + every queue; +3. otherwise it is degraded if the oldest run has waited `>= 5 min` + (`OLDEST_WAIT_WARNING_MS`). + +So a zero-concurrency queue with a backlog shows Investigate only once its +head-of-line wait passes 5 minutes, never from saturation. A queue full of runs +with nothing executing shows only `Watch…`. + +**What the queue's Watch button pre-fills** (`watch-recommendations.ts`): the +oldest wait `>= 5 min` gives `backlog_drain`; anything else, including an +unknown wait, gives `queue_oldest_age` at 5 minutes. A watch is for what happens +next — offering the SLA watch when the SLA is already breached would one-shot +instead of watching. + +The other pre-fills: error group → `error_recurrence`, 5 min / 6 h; run panel → +`run_finished`, 1 min / 1 h; report card → `health_recovery` carrying the +current severity as `fromSeverity`, 5 min / 6 h. + +### Chips in the agent panel + +An empty chat offers up to five. They come from *two* places, resolved into the +same slots. + +*The page registry* (`suggested-prompts/page-prompts.ts`) answers by page kind +and its fields. Only `error` and `queue` contribute a watch chip: + +| Page kind | Chip | Offered when | +| --- | --- | --- | +| `queue` | investigate | not paused **and** health is `warn` or `crit` | +| `queue` | watch | not paused | +| `error` | investigate, watch | always | +| `run` | investigate | always | + +A backed-up queue therefore offers **both** chips. A paused queue offers +neither: its computed health is `warn` (paused counts), so the registry carries +its own `paused` guard on top of health. + +*The page's live signals* (`suggested-prompts/signal-prompts.ts`) — the complete +list: + +| Signal | Chip slot | +| --- | --- | +| `fresh_failure` | investigate | +| `slow_run` | investigate | +| `waiting_run` | watch | +| `concurrency_saturation` | watch | + +`concurrency_saturation` is raised only when the queue is at capacity **and** +not paused (`page-mappers.ts`), using the same `isQueueAtCapacity` guard, so the +zero-limit rule holds here too. + +**How five are picked** (`suggested-prompts/resolver.ts`): slots are +`promoted, investigate, watch, status, explain, docs`; each slot takes the first +non-dismissed candidate, signals before the page-kind default, so at most one +chip per slot. Over the cap of five, whole slots are dropped in the order +`status, watch, investigate` — `promoted`, `explain` and `docs` never yield. + +--- + +## The ten watch kinds, and what makes each fire + +The spec union is `internal-packages/dashboard-agent-contracts/src/watch.ts`. +Every check is deterministic and runs without an LLM. + +A check returns one of four results, and only two of them are verdicts: + +| Result | Meaning | +| --- | --- | +| `satisfied` | the condition is true now | +| `terminal_unsatisfied` | it can never become true, so stop checking | +| `pending` | not true yet; keep checking | +| `unavailable` | the check itself couldn't run — never true, never false | + +### Run conditions — `dashboardAgentWatchRunChecks.ts` + +All three read one run row, scoped to the watch's environment. + +| Kind | Satisfied when | Impossible when | +| --- | --- | --- | +| `run_start` | `startedAt` is set, whatever the current status | the run reached a final status without ever starting, or the row is gone | +| `run_finished` | the status is any final status — including cancelled and failed | the row is gone | +| `run_failed` | the status is a final **failing** one | the run completed successfully or was cancelled, or the row is gone | + +The failing set is `COMPLETED_WITH_ERRORS`, `SYSTEM_FAILURE`, `CRASHED`, +`EXPIRED`, `TIMED_OUT`, `INTERRUPTED`. `CANCELED` is neither a success nor a +failure and gets its own presentation. + +`run_finished` fires on a failure too — the resolution alone can't tell the two +apart, so the observed final status decides the headline: a failed run reads +"Run x failed" in error tone, a cancelled one reads neutrally. + +`run_failed` on a successful run resolves *impossible*, and that is presented as +good news — "Run x succeeded" — not as an error. + +### Queue conditions — `dashboardAgentWatchQueueChecks.ts` + +| Kind | Satisfied when | +| --- | --- | +| `backlog_drain` | the current pending depth is `0` | +| `queue_depth_above` | depth `> threshold` | +| `queue_depth_below` | depth `<= threshold` | +| `queue_stalled` | depth `> 0` and the depth failed to decrease for `ticks` consecutive checks | +| `queue_oldest_age` | the oldest still-waiting run has waited `> thresholdMinutes` | + +For all five, the **only** impossible outcome is the queue no longer existing. A +live queue is never impossible, however far it is from the threshold. + +**The freshness fence.** Depth comes from the live queue counter; if that read +fails, from the newest 60-second ClickHouse bucket, which counts as current only +if its end is within 60 s of now (`dashboardAgentWatchChecks.server.ts`). A +non-current reading at or below the *quiet line* is refused as `unavailable` +rather than believed — the quiet line is `0` for drain and stall, and the +threshold for the two depth kinds. A stale empty bucket is never read as +drained. `queue_stalled` is stricter still: it refuses **any** non-current +reading, because a phantom sample would enter the streak as if it had been +observed now. + +**The stall streak** is the one piece of state, carried in the previous check's +facts. `ticks` defaults to 3 (min 2, max 12) and is not offered by the card. The +first check has nothing to compare against, so it scores 0; each later check +whose depth is `>= the previous` adds one; a depth of `0` resets it; a check +that couldn't read a current depth *freezes* it rather than breaking it. With +the default of 3 and the 5-minute cadence floor, the earliest a stall can fire +is the fourth check. + +**Oldest age** takes the worst wait across concurrency keys with a live backlog +(capped at 50 keys), falling back to the queue's oldest message. If either read +fails the whole reading is `unavailable` — a partial read would under-report the +wait and silently miss the SLA. Nothing waiting is `pending`, and is only +terminal if the queue is also gone. + +### `error_recurrence` — `dashboardAgentWatchErrorChecks.ts` + +Satisfied on the first occurrence proven to be after `since`. `since` is +**server-set when the row is persisted** and is deliberately absent from the +spec, so nothing can backdate the window: an occurrence written before the watch +existed is invisible to it. + +The proof is `errors_v1`'s `last_seen` at millisecond precision, because the +per-minute rollup can't separate the prompting error from a recurrence in the +same minute. The rollup then supplies the count, which is a lower bound when the +creation minute itself has hits. + +A fingerprint never seen in the environment is `pending`, not impossible. This +kind has no terminal outcome of its own. + +### `health_recovery` — `dashboardAgentWatchHealthChecks.ts` + +Satisfied only when the health report is **trustworthy and `ok`**. An +untrustworthy report is `pending` and records no severity — it is not an +observation. A report that can't be produced or carries an unknown severity is +`unavailable`. + +### When a check throws + +Any exception inside any check is caught in one place (`checkWatch`) and becomes +`unavailable` with an unverified observation. A check failure is never a verdict. + +--- + +## From a check to an answer + +`watchResolutionForCheck` (`watch.ts`) turns the result into a resolution: + +| Result | Before the deadline | On the deadline | +| --- | --- | --- | +| `satisfied` | `condition_met` | `condition_met` | +| `terminal_unsatisfied` | `condition_impossible` | `condition_impossible` | +| `pending` | keep checking | `window_completed` | +| `unavailable` | keep checking | `window_completed` | + +A check landing exactly on the deadline can still fire or refuse — only an +unfinished or unreadable one becomes a completed window. The claimed row's +`expiresAt` decides that, not the clock the check ran on +(`watch-lifecycle.ts`). + +**A completed window is an answer, not a failure.** Whether it is good or bad +news is declared per kind, never inferred: + +| Kind | `condition_met` | `window_completed` | +| --- | --- | --- | +| `run_start` | good — started | attention — hasn't started yet | +| `run_finished` | good, unless the final status was a failure | attention — still running | +| `run_failed` | attention — failed | **good** — hasn't failed | +| `backlog_drain` | good — drained | attention — still hasn't drained | +| `queue_depth_above` | attention — above the threshold | **good** — stayed below | +| `queue_depth_below` | good — back below | attention — still above | +| `queue_stalled` | attention — stuck | **good** — kept moving | +| `queue_oldest_age` | attention — over the SLA | **good** — stayed under | +| `error_recurrence` | attention — happened again | **good** — stayed quiet | +| `health_recovery` | good — recovered | attention — hasn't recovered | + +`condition_impossible` is neutral for every kind, with two refinements: a +`run_failed` watch on a run that succeeded reads as good news, and on a +cancelled run as neutral. + +One rule overrides the whole table: if the window completed on an **unverified** +observation, the answer is neutral and says only "The watch ended without a +confirmed answer" — an unreadable source is never reported as "it didn't +happen". + +The final English lives in one place, +`dashboard-agent-contracts/src/watch-wording.ts`, and the card, banner, toast, +email, Slack message, webhook and the agent's own narration all read it. Numbers +come from the frozen observation, never a fresh read, so a retry produces the +same sentence. + +### Which wakes cost a model call + +`planWatchNarration` (`watch-narration.ts`): + +- a consented investigation → Sonnet, with the conversation, so the promise and + the findings read as one voice; +- any other **attention** outcome → Haiku, given the wake alone; +- everything else, good or merely factual → **no model call at all**; the + sentence is composed from the contracts' wording. + +--- + +## Creating a watch + +`createDashboardAgentWatch` (`apps/webapp/app/services/dashboardAgentWatches.server.ts`). +The order is load-bearing. + +1. **Not configured** → "The dashboard agent is not configured, so watches can't + be scheduled." +2. **The target must exist in this environment** → otherwise "That target + doesn't exist in this environment." Run kinds need the run row, queue kinds + the queue row, `health_recovery` a known report key. `error_recurrence` + validates only that the fingerprint is non-empty: zero occurrences so far is + the normal case. These reads go to the **primary**, so a run or queue created + a moment ago is visible; the polling checks use the replica. +3. **Duplicate** → "This chat is already watching that." The identity is the + kind plus its target, with the threshold folded in for the two depth kinds + and the SLA for `queue_oldest_age`; cadence, window, note and `ticks` are + deliberately **not** part of it. It is unique across `(chat, project, + environment)` among **active** rows only, enforced by a partial unique index + rather than by the read-then-insert check — two different chats may watch the + same thing. +4. **Cap** → "This chat already has 3 active watches. Cancel one first." The + count is over every active watch on the chat, whatever project or environment + it points at. +5. **The immediate check**, run before any row is written: + - `satisfied` → "That already happened, so there's nothing left to watch." + - `terminal_unsatisfied` → "That can't happen any more, so there's nothing to + watch." + In both cases **no row is created** — no chip, no wake, nothing to cancel. + The check *is* the delivery, so the answer you get is the answer. + - `unavailable` → the watch **is** created, and the confirmation says so: + "We couldn't check that just now. Watching anyway." +6. **The first tick must schedule.** If it can't, the row is cancelled — not + resolved, because the condition was never evaluated — and cancellation is + silent, so no wake is sent: "The watch couldn't be scheduled. Nothing is + being watched." + +## Cadence, window and expiry + +| | Options | +| --- | --- | +| Cadence, run kinds | 1, 5, 15 or 60 minutes | +| Cadence, everything else | 5, 15 or 60 minutes — 5 is the floor for aggregates | +| Window | 30 min, 1, 2, 6, 12 or 24 hours | + +The ceiling is 24 hours and the schema enforces both. A watch schedules its own +next check — there is no shared cron — so the first answer lands one cadence +after you confirm the card. `expiresAt` is creation time plus the window. + +Due watches of one `(environment, cadence)` group can instead be checked +together in one pass (`dashboardAgentWatchBatch.server.ts`). Every per-watch +check arms that group's chain and is told whether it is running; if it is, the +watch hands over and its own chain stops rescheduling, because the group now +reschedules once for everyone. A tick arriving slightly early still counts as +due, by up to half a cadence and at most 30 seconds, and a watch whose deadline +falls within one cadence is always due, so the final evaluation is never missed. +The group's chain stops only when nothing is active **and** nothing is owed. + +A check that came back `unavailable` in a batch is recorded as an attempt rather +than as a check, so the stall streak and the last-checked time survive it. + +A sweep (`dashboardAgentWatchSweep.server.ts`) is the backstop: + +- a row still active **2 minutes** past `expiresAt` is finalized; +- a resolved row whose wake is still owed **5 minutes** later is redelivered — + the sweep can't tell whether the user was already told, so delivery is + id-deduped rather than conditional; +- terminal rows are kept **7 days**; the outcome also lives in the transcript. + +The sweep re-authorizes each row before reading anything, and carries the +previous check's facts into the final evaluation, so a stall streak survives the +boundary. It finalizes overdue rows even when the agent isn't configured to +deliver — otherwise they would stay active forever — and leaves the wake owed. + +## What ends a watch without an answer + +A watch reaches `fired` or `expired` by resolving, and both deliver a wake. +**Cancellation is the silent ending: no resolution, no wake, nothing to read.** +The five reasons (`watch-schema.ts`): + +| Reason | When | +| --- | --- | +| `user` | the chip's cancel, scoped through the chat: the watch must belong to that chat and the chat to this user in this org. A no-op if it already resolved | +| `chat_deleted` | deleting a chat cancels its watches in the same transaction, so live watches can't outlive a chat the user can no longer see | +| `access_revoked` | the creator's access no longer holds — checked before any read | +| `scheduling_failed` | the first tick couldn't be scheduled | +| `superseded` | a concurrent submission recorded a different outcome, so the orphan row is cancelled before the replay | + +**Access is re-authorized on every check, on the primary** — replica lag would +extend access the user has already lost. It requires the row's frozen project +and organization to still match, the environment not archived, the project and +org not deleted, membership to still exist, a development environment to still +belong to this user, and `canAccessDashboardAgent` to still pass. Any partial +pass is a full revoke. + +A watch that has already fired is **not** cancelled when its creator loses +access — it is terminal already. It just gets no alert. + +## The two follow-ups + +They are independent opt-ins, never a radio group. In-chat delivery is always on +and is not one of them. + +**Investigate attention outcomes.** An investigation opens only when the +resolved outcome's category is `attention`, per the table above — both the +webapp's kick (`watchWantsInvestigation`) and the agent's own wake call the same +contracts function, so the two can't disagree. Good news and neutral news never +start one, however the watch was configured. Two consequences worth stating: +every `condition_impossible` is neutral, so an impossible watch never +investigates; and a window that completed on an unverified observation is +neutral too, so an unreadable source never starts one either. + +The kick is best-effort and happens only after the wake is in the transcript: +the wake is the delivery that matters, and a failed kick never retries or +invalidates it. + +**Email me as well.** Attached only when the box is ticked, only after the watch +exists, and never able to fail creation. It goes to the user's own account +email, never an address from the request. + +Two gates, and **neither is a plan check** — billing gates that separately: + +- the same agent-access gate as everything else; +- an email transport must be configured (`ALERT_FROM_EMAIL` and + `ALERT_EMAIL_TRANSPORT`). + +When either refuses, the card reports the subscription as `unavailable` and says +"I couldn't add email notifications, so updates will appear in the dashboard +only." The watch still runs, and that outcome is frozen on the ledger row and +replayed on a retry rather than decided again — the confirmation in the +transcript is append-once, so a second decision would contradict it forever. + +**Only a fired watch emails.** An expiry is narrated in the chat and nothing +else. The access gate is re-checked at delivery time as well as at subscribe +time, so a revoked flag stops the mail without anyone cleaning up channels. + +## What it will not do + +- Write anything beyond its own watches and an alert you explicitly approved. +- Invent numbers or claim something doesn't exist beyond a truncated page. +- Trust a report whose telemetry is stale. +- Report an unreadable source as a negative answer. + +Feedback → #dashboard-agent-feedback, or just tell the agent — a sample of turns +is scored automatically and capability gaps are collected from it (a tenth by +default, none for a turn that read source, none for an org that opted out; see +[README.md](./README.md#turn-evals)). diff --git a/internal-packages/dashboard-agent/README.md b/internal-packages/dashboard-agent/README.md index f268e0fd8c3..b59580137c7 100644 --- a/internal-packages/dashboard-agent/README.md +++ b/internal-packages/dashboard-agent/README.md @@ -4,6 +4,9 @@ The in-dashboard agent, built on `chat.agent` and deployed as its own Trigger project. This is the launch-week dogfood: we run our own product on the primitive we ship. +Running it locally, what it does, and a walkthrough per flow: +[GUIDEBOOK.md](./GUIDEBOOK.md). + ## Why a separate package (not inside apps/webapp) The agent has **no access to the main database, ClickHouse, or webapp @@ -53,9 +56,9 @@ conversation that rides on top of it. accept the snapshot diff (`vitest -u`) — that is the whole point of the numbers. - **The conversation** is compacted in `src/compaction.ts`: above 60k tokens of conversation (on top of the ~21k prefix) the older part becomes a Haiku-written summary. The UI - transcript is never compacted, and an open investigation is pinned back onto the model's - history verbatim, so a summary can never cost the agent the `investigationId` it has to - keep revising. + transcript is never compacted, and an open investigation, a live watch and an already + delivered wake are pinned back onto the model's history verbatim, so a summary can never + cost the agent the `investigationId` it has to keep revising. ## Turn evals diff --git a/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap b/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap index 293c37efdfc..2e293e548f8 100644 --- a/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap +++ b/internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap @@ -4,34 +4,34 @@ exports[`the prefix stays inside its budget > matches the committed measurement { "assistant": { "prompt": { - "chars": 19285, - "estimatedTokens": 4821, + "chars": 25965, + "estimatedTokens": 6491, }, "tools": { - "chars": 38555, - "count": 20, - "estimatedTokens": 9639, + "chars": 38922, + "count": 24, + "estimatedTokens": 9731, }, "total": { - "chars": 57841, - "estimatedTokens": 14460, - "fingerprint": "52bee164", + "chars": 64888, + "estimatedTokens": 16222, + "fingerprint": "9a65830e", }, }, "code": { "prompt": { - "chars": 22040, - "estimatedTokens": 5510, + "chars": 28720, + "estimatedTokens": 7180, }, "tools": { - "chars": 41564, - "count": 24, - "estimatedTokens": 10391, + "chars": 41931, + "count": 28, + "estimatedTokens": 10483, }, "total": { - "chars": 63605, - "estimatedTokens": 15901, - "fingerprint": "037bcfc3", + "chars": 70652, + "estimatedTokens": 17663, + "fingerprint": "76fb3bc9", }, }, } diff --git a/internal-packages/dashboard-agent/src/agent-runtime.ts b/internal-packages/dashboard-agent/src/agent-runtime.ts index 52de221538a..d8d20ba2e20 100644 --- a/internal-packages/dashboard-agent/src/agent-runtime.ts +++ b/internal-packages/dashboard-agent/src/agent-runtime.ts @@ -3,17 +3,18 @@ import { appendChatMessageOnce, createDashboardAgentDb, ensureChat, - findOpenInvestigationForChat, investigationSettlementMessage, persistMessages, persistTurn, setChatTitleIfDefault, + seedInvestigation, settleInvestigationStateAndCloseCard, upsertInvestigationRevision, type ClosedInvestigationCard, type DashboardAgentDbClient, type PendingInvestigationSettlement, type PersistTurnResult, + type SeedInvestigationResult, type UpsertInvestigationResult, } from "@internal/dashboard-agent-db"; import { locals, logger } from "@trigger.dev/sdk"; @@ -39,8 +40,8 @@ import { buildDashboardAgentTools } from "./tools"; * The agent's runtime: its datastore, the investigation bookkeeping every lane * shares, and the model, prompt and tool plumbing a turn is assembled from. * - * Split out of `dashboard-agent.ts` so the turn lanes that are not the agent's own - * hooks can reach it without importing the agent back. + * Split out of `dashboard-agent.ts` so the turn lanes that are not the agent's + * own hooks — the watch actions — can reach it without importing the agent back. */ // One connection pool per worker process, established in onBoot (which fires on @@ -71,9 +72,9 @@ export interface DashboardAgentStore { ensureChat(args: Parameters[1]): Promise; persistMessages(args: Parameters[1]): Promise; /** - * Id-deduped single-message append, for a lane that runs without a client: the - * session's view can miss host-appended blocks, and a wholesale write would drop - * them. + * Id-deduped single-message append. The wake narration writes through this + * rather than `persistMessages`: a wake runs without a client, so the session's + * view can miss host-appended blocks and a wholesale write would drop them. */ appendMessage(args: Parameters[1]): Promise; /** @@ -96,12 +97,13 @@ export interface DashboardAgentStore { args: Parameters[1] ): Promise; /** - * The freshest card this chat still has open, so a turn that continues an - * investigation revises that row instead of opening a second one. + * Open an investigation under a caller-chosen id, or report it already open. A + * consented watch's two lanes both name the row this way, so the second one revises + * what the first opened instead of looking for the freshest open card. */ - findOpenInvestigation( - args: Parameters[1] - ): Promise<{ id: string; projectRef: string; environmentRef: string } | null>; + seedInvestigation( + args: Parameters[1] + ): Promise; } export const dashboardAgentStoreKey = locals.create("dashboard-agent.store"); @@ -259,7 +261,7 @@ export function getStore(): DashboardAgentStore { setChatTitleIfDefault: (args) => setChatTitleIfDefault(db, args), upsertInvestigationRevision: (args) => upsertInvestigationRevision(db, args), settleInvestigationCard: (args) => settleInvestigationStateAndCloseCard(db, args), - findOpenInvestigation: (args) => findOpenInvestigationForChat(db, args), + seedInvestigation: (args) => seedInvestigation(db, args), }); } diff --git a/internal-packages/dashboard-agent/src/compaction.test.ts b/internal-packages/dashboard-agent/src/compaction.test.ts index c03a15f7db3..88d64d1bfab 100644 --- a/internal-packages/dashboard-agent/src/compaction.test.ts +++ b/internal-packages/dashboard-agent/src/compaction.test.ts @@ -19,6 +19,7 @@ import { safeTail, shouldCompactConversation, STATIC_PREFIX_TOKENS, + SUMMARY_INSTRUCTION, withDurableState, } from "./compaction"; @@ -93,6 +94,44 @@ function hostInvestigationMessage(args: { }; } +function watchConfirmationMessage(args: { + watchId: string; + headline: string; + lifetime?: string; +}): UIMessage { + return { + id: `watch-card:${args.watchId}`, + role: "assistant", + parts: [ + { + type: "data-view", + data: { + blocks: [ + { + type: "watch_result", + id: `watch:${args.watchId}`, + revision: 0, + version: 1, + outcome: "watching", + watchId: args.watchId, + headline: args.headline, + lifetime: args.lifetime ?? null, + }, + ], + }, + } as never, + ], + }; +} + +function wakeMessage(actionId: string, body: string): UIMessage { + return { + id: `wake:${actionId}`, + role: "assistant", + parts: [{ type: "text", text: body }], + }; +} + describe("when the conversation is compacted", () => { it("stays under the budget for an ordinary conversation", () => { expect(shouldCompactConversation({ messages: bulk(20, 500), inputTokens: 22_000 })).toBe(false); @@ -243,6 +282,19 @@ describe("the state a summary may not swallow", () => { expect(collectDurableState(mixed).investigations.map((i) => i.id)).toEqual(["inv_2"]); }); + it("pins no watch, live or otherwise — a watch's lifecycle is server-side", () => { + const note = describeDurableState([ + watchConfirmationMessage({ + watchId: "watch_9", + headline: "Watching orders queue until it drains.", + lifetime: "Checking every 15 min for up to 6 hours. It reports once, then stops.", + }), + wakeMessage("watch_9:fired", "orders queue drained — 0 pending after 42 minutes."), + ]); + expect(note).toBeUndefined(); + expect(describeDurableState([])).toBeUndefined(); + }); + it("pins the same state onto the between-steps rebuild path", () => { const rebuilt: ModelMessage[] = [ text("user", "[Conversation summary]\n\nsome summary"), @@ -338,6 +390,31 @@ describe("the summariser's input", () => { }); }); +/** + * A watch's lifecycle is server-side: it can expire or be cancelled with nothing written back + * into the transcript. So the summary can only ever say what the transcript RECORDED — asking + * for what is running turns an old confirmation into a claim that it still is, and the next + * answer tells the user a watch is on that ended hours ago. The property, not the sentence: + * the watch line asks for a record and never for present state. + */ +describe("the summary instruction never asks for present state", () => { + const watchLine = SUMMARY_INSTRUCTION.split("\n").find((line) => /watch/i.test(line)); + + it("has a line about watches at all", () => { + expect(watchLine).toBeDefined(); + }); + + it("asks what the transcript recorded, not what is true now", () => { + expect(watchLine).toMatch(/record/i); + // The transcript cannot know, so the instruction has to say why. + expect(watchLine).toMatch(/expire|cancel/i); + }); + + it("never asks for a watch that is running, scheduled or active", () => { + expect(watchLine).not.toMatch(/(?:that is|still|currently)\s+(?:running|active|scheduled)/i); + }); +}); + /** Records what each model call was actually given, and summarises predictably. */ function capturingModel(prompts: string[], summarized: string[] = []) { return new MockLanguageModelV3({ @@ -423,4 +500,35 @@ describe("dashboardAgent compaction (mock harness)", () => { expect(prompts.at(-1)!).toContain("inv_abc123"); expect(prompts.at(-1)!).toContain("never open a second card"); }); + + it("hands a watch to the summariser instead of pinning it as live", async () => { + const prompts: string[] = []; + const summarized: string[] = []; + harness = runOverBudget({ + chatId: "chat_compaction_watch", + prompts, + summarized, + seeded: [ + userMessage("tell me when the orders queue drains", "u0"), + watchConfirmationMessage({ + watchId: "watch_9", + headline: "Watching orders queue until it drains.", + lifetime: "Checking every 15 min for up to 6 hours. It reports once, then stops.", + }), + wakeMessage("watch_9:fired", "orders queue drained — 0 pending after 42 minutes."), + ], + }); + + await harness.sendMessage(userMessage("what happened with that?", "u1")); + await harness.sendMessage(userMessage("and now?", "u2")); + + const after = prompts.at(-1)!; + expect(after.length).toBeLessThan(FILLER.length); + expect(after).toContain("SUMMARY-OF-THE-CHAT"); + // The watch is the summary's job. Pinning the old confirmation would state a watch + // is running when it may have fired, expired or been cancelled since. + expect(after).not.toContain("It reports once, then stops."); + // But the summariser did see what the watch reported. + expect(summarized.join("\n")).toContain("0 pending after 42 minutes"); + }); }); diff --git a/internal-packages/dashboard-agent/src/compaction.ts b/internal-packages/dashboard-agent/src/compaction.ts index 49b2a483d87..cb178a14405 100644 --- a/internal-packages/dashboard-agent/src/compaction.ts +++ b/internal-packages/dashboard-agent/src/compaction.ts @@ -19,8 +19,8 @@ import { * If the model loses an `investigationId` it opens a SECOND card for the same * question, which is the failure this module exists to prevent. * - * Nothing else is pinned. Finished work is the summary's job: a pin has to be exact, - * and only a live card is both exact and needed verbatim. + * Nothing else is pinned. Finished work and watches are the summary's job: a pin has + * to be exact, and only a live card is both exact and needed verbatim. */ /** @@ -61,15 +61,16 @@ const SUMMARY_MODEL = "anthropic:claude-haiku-4-5" as const; */ const SUMMARY_MAX_OUTPUT_TOKENS = 1_000; -const SUMMARY_INSTRUCTION = `You are compacting a support conversation between a user and an agent that reads a Trigger.dev dashboard, so the agent can keep going with a shorter history. +export const SUMMARY_INSTRUCTION = `You are compacting a support conversation between a user and an agent that reads a Trigger.dev dashboard, so the agent can keep going with a shorter history. Write a summary in under 400 words, as notes rather than prose. Keep, in this order: 1. What the user is trying to do, in their own terms, and anything they asked to be remembered. 2. Facts already established, with the run ids, queue names, task identifiers, error fingerprints and numbers they rest on. Never restate a number you cannot see. 3. Any investigation that is open: its investigationId, its title and its current outcome. -4. What was asked most recently and what is still unanswered. +4. Any watch the transcript records — what it was set up to watch, and what it said if it reported. Write it as what the transcript recorded, never as what is true now: a watch can expire or be cancelled without saying so here, so never present one as current. +5. What was asked most recently and what is still unanswered. -Drop tool mechanics, retries, and anything already superseded. Do not add advice, and do not invent anything that is not in the transcript.`; +Drop tool mechanics, retries, and anything already superseded. Do not add advice, and do not invent anything that is not in the transcript. Everything you write is a record of what the transcript said, not a claim about the present.`; /** A summary that reads as a summary, and never as the user's next question. */ export function summaryMessage(summary: string, durableState?: string): ModelMessage { @@ -134,6 +135,12 @@ export type DurableState = { * Only an `in_progress` card is state: a concluded or inconclusive one is finished * work the summary already covers, and pinning it would grow the note forever and * invite the model to keep revising a card that closed long ago. + * + * Watches are deliberately not read from here. A watch's lifecycle is server-side — + * it can fire, expire, or be cancelled with nothing written back into the transcript + * — so an old confirmation block cannot tell us whether it is still running. There is + * no watch state on the store either, and nothing about a watch depends on the model + * remembering it, so the summary is where a watch belongs. */ export function collectDurableState(uiMessages: UIMessage[]): DurableState { const investigations: PinnedInvestigation[] = []; diff --git a/internal-packages/dashboard-agent/src/dashboard-agent.eval.ts b/internal-packages/dashboard-agent/src/dashboard-agent.eval.ts index aa4415fb7aa..351b8a9815c 100644 --- a/internal-packages/dashboard-agent/src/dashboard-agent.eval.ts +++ b/internal-packages/dashboard-agent/src/dashboard-agent.eval.ts @@ -53,7 +53,7 @@ const NOOP_STORE: DashboardAgentStore = { card: { id: args.messageId, role: "assistant", parts: [] }, closed: true, }), - findOpenInvestigation: async () => null, + seedInvestigation: async (args) => ({ ok: true, id: args.id, created: true }), }; const FIXTURES: Record = { @@ -278,6 +278,9 @@ function makeFixtureTools( execute: async (input: unknown) => { const output = ((): unknown => { if (name === "navigate_to") return input; + if (name === "schedule_watch") { + return { intent: { kind: "watch", spec: (input as { watch?: unknown }).watch } }; + } if (name === "render_view") { const spec = input as { blocks?: Array<{ type?: string }> }; const hasInvestigation = (spec.blocks ?? []).some((b) => b?.type === "investigation"); @@ -550,6 +553,7 @@ const TOOL_CASES: Array<{ question: string; expect: string | string[] }> = [ { question: "How do I use batchTrigger?", expect: "search_docs" }, { question: "How deep is the email queue?", expect: "get_queue" }, { question: "What was deployed recently?", expect: "list_deploys" }, + { question: "Tell me when run run_a1 finishes.", expect: "schedule_watch" }, ]; const TOOL_SELECTION_THRESHOLD = 0.83; diff --git a/internal-packages/dashboard-agent/src/dashboard-agent.test.ts b/internal-packages/dashboard-agent/src/dashboard-agent.test.ts index bb9f89a11a7..491493c1915 100644 --- a/internal-packages/dashboard-agent/src/dashboard-agent.test.ts +++ b/internal-packages/dashboard-agent/src/dashboard-agent.test.ts @@ -119,6 +119,10 @@ describe("dashboardAgent (mock harness)", () => { it("does not rename on a later exchange", () => { expect(isFirstUserExchange([user("u1"), assistant("a1"), user("u2")])).toBe(false); }); + + it("ignores a watch consent record, which the user never typed", () => { + expect(isFirstUserExchange([user("watch-request:watch_1"), user("u1")])).toBe(true); + }); }); it("names the chat once, not on every turn", async () => { @@ -1020,7 +1024,7 @@ describe("a turn that ends in an error", () => { card: { id: args.messageId, role: "assistant", parts: [] }, closed: true, }), - findOpenInvestigation: async () => null, + seedInvestigation: async (args) => ({ ok: true, id: args.id, created: true }), }; return { store, history: () => [...rows.values()] }; } @@ -1131,6 +1135,8 @@ describe("buildDashboardAgentTools", () => { [ "ask_support", "correlate_version", + "create_alert", + "delete_alert", "get_current_page", "get_deploy", "get_error", @@ -1139,6 +1145,7 @@ describe("buildDashboardAgentTools", () => { "get_report", "get_run", "get_run_trace", + "list_alerts", "list_deploys", "list_environments", "list_errors", @@ -1148,6 +1155,7 @@ describe("buildDashboardAgentTools", () => { "navigate_to", "run_query", "render_view", + "schedule_watch", "search_docs", ].sort() ); @@ -1544,7 +1552,11 @@ describe("buildDashboardAgentTools", () => { const grounded = await renderInvestigation(tools, concludedWithSource); const actions = grounded.blocks[0].capabilities.actions; - expect(actions.map((a: { kind: string }) => a.kind)).toEqual(["show_code", "view_similar"]); + expect(actions.map((a: { kind: string }) => a.kind)).toEqual([ + "show_code", + "watch_recurrence", + "view_similar", + ]); expect(actions[0].intent.kind).toBe("ask"); // The ask proposes a change rather than another explanation. const prompt: string = actions[0].intent.prompt; @@ -1554,12 +1566,38 @@ describe("buildDashboardAgentTools", () => { expect(prompt).toMatch(/dirty tree|branch head/i); expect(prompt).toMatch(/don't restate the investigation/i); + // "Watch for a repeat" carries the kind and the subject, so the Watch card is + // pre-filled without another model turn. expect(actions[1].intent).toEqual({ + kind: "watch", + spec: { + kind: "error_recurrence", + fingerprint: "c4b4a797397a9c43", + checkEveryMinutes: 15, + maxHours: 24, + note: `A repeat of: ${concludedWithSource.title}`, + }, + }); + + expect(actions[2].intent).toEqual({ kind: "navigate", target: "trigger://proj_abc/env_abc/error/c4b4a797397a9c43", }); }); + // A concluded card citing no error group has no subject to pre-fill from, so the + // button is left off rather than offered empty. + it("offers no repeat watch when the card cites no error group", async () => { + const { capability } = fakeInvestigations(); + const tools = buildDashboardAgentTools({ ...SCOPE, investigations: capability }); + + const output = await renderInvestigation(tools, concludedState); + const kinds = (output.blocks[0].capabilities?.actions ?? []).map( + (a: { kind: string }) => a.kind + ); + expect(kinds).not.toContain("watch_recurrence"); + }); + it("offers no actions while an investigation is still in progress", async () => { const { capability } = fakeInvestigations(); const tools = buildDashboardAgentTools({ ...SCOPE, investigations: capability }); @@ -1567,7 +1605,9 @@ describe("buildDashboardAgentTools", () => { expect(output.blocks[0].capabilities).toBeUndefined(); }); - it("offers a keep-digging follow-up, and never Show code, on an inconclusive card", async () => { + // An inconclusive card has no cause to watch for a repeat of, so the handoff stays + // off it. + it("offers a keep-digging follow-up, and never Show code or a repeat watch, on an inconclusive card", async () => { await seedWorkspace(); const { capability } = fakeInvestigations(); const tools = buildDashboardAgentTools({ @@ -1661,6 +1701,35 @@ describe("buildDashboardAgentTools", () => { expect(upserts[1]).toMatchObject({ id: first.investigationId }); }); + it("render_view refuses two investigations in one view and writes neither", async () => { + const { capability, rows, upserts } = fakeInvestigations(); + const tools = buildDashboardAgentTools({ ...SCOPE, investigations: capability }); + const renderView = tools.render_view as { + inputSchema: { parse: (input: unknown) => unknown }; + execute: (input: unknown, opts: unknown) => Promise; + }; + + const output = await renderView.execute( + renderView.inputSchema.parse({ + blocks: [ + { type: "investigation", investigation: investigationState }, + { + type: "investigation", + investigation: { ...concludedState, title: "A different question entirely" }, + }, + ], + }), + {} + ); + + // One id is assigned per call, so committing both would file the second subject as + // the first's next revision. + expect(typeof output.error).toBe("string"); + expect(output.blocks).toBeUndefined(); + expect(upserts).toEqual([]); + expect(rows.size).toBe(0); + }); + it("render_view errors on an unknown investigationId and writes nothing", async () => { const { capability, rows } = fakeInvestigations(); const tools = buildDashboardAgentTools({ ...SCOPE, investigations: capability }); @@ -1754,6 +1823,82 @@ describe("buildDashboardAgentTools", () => { await expect(renderView.execute({ blocks: [chart] }, {})).resolves.toEqual({ blocks: [chart] }); }); + const WATCH_CTX = { + userActorToken: "uat_token", + apiOrigin: "http://localhost:3030", + chatId: "chat_1", + }; + + const RUN_WATCH = { + kind: "run_finished" as const, + runId: "run_a1", + checkEveryMinutes: 1 as const, + maxHours: 2, + note: "tell me when the receipt run finishes", + }; + + // Records any request schedule_watch would have made. It must make none: the card + // the intent opens is the only thing that creates a watch. + async function scheduleWatch( + input: unknown = { watch: RUN_WATCH }, + ctx: Record = WATCH_CTX + ) { + const requests: string[] = []; + const original = globalThis.fetch; + globalThis.fetch = (async (url: Parameters[0]) => { + requests.push(String(url)); + return new Response("{}", { status: 200, headers: { "Content-Type": "application/json" } }); + }) as typeof fetch; + try { + const tools = buildDashboardAgentTools(ctx); + const scheduleTool = tools.schedule_watch as { + inputSchema: { parse: (input: unknown) => unknown }; + execute: (input: unknown, opts: unknown) => Promise; + }; + const result = await scheduleTool.execute(input, {}); + return { result, requests }; + } finally { + globalThis.fetch = original; + } + } + + it("schedule_watch returns a watch intent and creates nothing", async () => { + const { result, requests } = await scheduleWatch(); + + expect(result).toEqual({ intent: { kind: "watch", spec: RUN_WATCH } }); + expect(requests).toEqual([]); + }); + + it("schedule_watch has no consent parameter — the card owns the opt-ins", () => { + const tools = buildDashboardAgentTools(WATCH_CTX); + const scheduleTool = tools.schedule_watch as { + inputSchema: { parse: (input: unknown) => unknown }; + }; + const parsed = scheduleTool.inputSchema.parse({ + watch: RUN_WATCH, + investigateOnAttention: true, + }); + expect(parsed).toEqual({ watch: RUN_WATCH }); + }); + + it("schedule_watch rejects a spec the contract won't accept", async () => { + // Aggregate conditions are floored at 5 minutes by the contract's schema. + const floored = await scheduleWatch({ + watch: { + kind: "backlog_drain", + queue: "task/x", + checkEveryMinutes: 1, + maxHours: 1, + note: "n", + }, + }); + expect(typeof floored.result.error).toBe("string"); + expect(floored.requests).toEqual([]); + + const nonsense = await scheduleWatch({ watch: { kind: "run_finished" } }); + expect(typeof nonsense.result.error).toBe("string"); + }); + // The env-JWT exchange is a webapp request plus DB work, so it is paid for once per // tool set, and so once per turn, however many env-scoped tools run. const ENV_CTX = { @@ -1926,19 +2071,21 @@ describe("buildDashboardAgentTools", () => { }); it("render_view commits the chart when its query runs, validating it once", async () => { + // Collected, never asserted here: render_view swallows a throw out of the stub, as + // the sibling test below relies on. + const queryBodies: unknown[] = []; const fetchStub = stubFetch((url, init) => { if (url.endsWith("/jwt")) return { body: { token: "jwt_1" } }; - // The validation runs the same window the panel will render. - expect(JSON.parse(String(init?.body))).toMatchObject({ - scope: "environment", - period: "24h", - }); + queryBodies.push(JSON.parse(String(init?.body))); return { body: { results: [{ bucket: "2026-01-01T00:00:00Z", runs: 1 }] } }; }); try { // The rows aren't embedded in the block — the panel stays the runner. await expect(renderView(ENV_CTX, CHART_SPEC)).resolves.toEqual({ blocks: CHART_SPEC.blocks }); expect(queryRequests(fetchStub.requests)).toHaveLength(1); + // The validation runs the same window the panel will render. + expect(queryBodies).toHaveLength(1); + expect(queryBodies[0]).toMatchObject({ scope: "environment", period: "24h" }); } finally { fetchStub.restore(); } @@ -2037,6 +2184,157 @@ describe("buildDashboardAgentTools", () => { }); }); +describe("watch alert tools", () => { + const ALERT_CTX = { + userActorToken: "uat_token", + apiOrigin: "http://localhost:3030", + projectRef: "proj_abc", + environmentName: "prod", + chatId: "chat_alerts", + }; + + // Hands back the tool's result and the request the webapp would have received. + async function callAlertTool( + name: string, + input: unknown, + response: { status?: number; body: unknown }, + ctx: Record = ALERT_CTX + ) { + const requests: Array<{ url: string; init: RequestInit | undefined }> = []; + const original = globalThis.fetch; + globalThis.fetch = (async (url: Parameters[0], init?: RequestInit) => { + requests.push({ url: String(url), init }); + return new Response(JSON.stringify(response.body), { + status: response.status ?? 200, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof fetch; + try { + const tools = buildDashboardAgentTools(ctx); + const tool = tools[name] as { + inputSchema: { parse: (input: unknown) => unknown }; + execute: (input: unknown, opts: unknown) => Promise; + }; + const result = await tool.execute(tool.inputSchema.parse(input), {}); + return { result, requests }; + } finally { + globalThis.fetch = original; + } + } + + it("list_alerts reads the project's subscriptions as the user", async () => { + const alerts = [{ id: "alert_1", type: "EMAIL", label: "k***@trigger.dev", enabled: true }]; + const { result, requests } = await callAlertTool("list_alerts", {}, { body: { alerts } }); + + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toBe( + "http://localhost:3030/api/v1/dashboard-agent/alerts?chatId=chat_alerts" + ); + expect(requests[0]?.init?.method).toBe("GET"); + expect((requests[0]?.init?.headers as Record | undefined)?.Authorization).toBe( + "Bearer uat_token" + ); + expect(result).toEqual({ alerts }); + }); + + // The body below is exactly what `api.v1.dashboard-agent.alerts.ts` returns on success: + // the channel flat, with no envelope around it. + const CREATED_CHANNEL = { + id: "alert_2", + type: "EMAIL", + target: "so…@example.com", + enabled: true, + }; + + it("create_alert posts the email channel and reports the created alert", async () => { + const { result, requests } = await callAlertTool( + "create_alert", + { email: "someone@example.com" }, + { body: CREATED_CHANNEL } + ); + + expect(requests[0]?.url).toBe("http://localhost:3030/api/v1/dashboard-agent/alerts"); + expect(requests[0]?.init?.method).toBe("POST"); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + chatId: "chat_alerts", + channel: "email", + email: "someone@example.com", + }); + expect(result).toEqual({ created: true, alert: CREATED_CHANNEL }); + + // With no email the host defaults to the user's account email, so the body carries + // only the chat scope and the channel. + const noEmail = await callAlertTool("create_alert", {}, { body: CREATED_CHANNEL }); + expect(JSON.parse(String(noEmail.requests[0]?.init?.body))).toEqual({ + chatId: "chat_alerts", + channel: "email", + }); + }); + + // `code` is the key both alert routes send a 403 refusal under. + it("create_alert relays a 403 with the reason the host gave", async () => { + const noEmailSetup = await callAlertTool( + "create_alert", + {}, + { status: 403, body: { error: "denied", code: "email_alerts_not_configured" } } + ); + expect(noEmailSetup.result.error).toContain("isn't set up on this instance"); + expect(noEmailSetup.result.error).toContain("dashboard"); + + const flag = await callAlertTool( + "create_alert", + {}, + { status: 403, body: { error: "denied", code: "dashboard_agent_disabled" } } + ); + expect(flag.result.error).toContain("aren't enabled here"); + }); + + it("create_alert relays the address refusal verbatim", async () => { + const refused = await callAlertTool( + "create_alert", + { email: "someone@else.com" }, + { + status: 400, + body: { + code: "email_not_allowed", + error: "Alerts can only be sent to your own account email.", + }, + } + ); + expect(refused.result.error).toBe("Alerts can only be sent to your own account email."); + }); + + it("delete_alert deletes by id and surfaces a failure as text", async () => { + const { result, requests } = await callAlertTool( + "delete_alert", + { alertId: "alert_1" }, + { body: { ok: true } } + ); + expect(requests[0]?.url).toBe("http://localhost:3030/api/v1/dashboard-agent/alerts/alert_1"); + expect(requests[0]?.init?.method).toBe("DELETE"); + expect(result).toEqual({ deleted: true, alertId: "alert_1" }); + + const missing = await callAlertTool( + "delete_alert", + { alertId: "alert_gone" }, + { status: 404, body: { error: "No such alert." } } + ); + expect(missing.result.error).toBe("No such alert."); + }); + + it("the alert tools fail closed with no delegated token, without hitting the network", async () => { + for (const [name, input] of [ + ["list_alerts", {}], + ["create_alert", {}], + ["delete_alert", { alertId: "alert_1" }], + ] as const) { + const { result, requests } = await callAlertTool(name, input, { body: {} }, {}); + expect(typeof result.error).toBe("string"); + expect(requests).toHaveLength(0); + } + }); +}); + /** * The two tools whose output IS the panel's view model. `toModelOutput` is the seam: * the client keeps the full payload, the model gets a small one. The end-to-end diff --git a/internal-packages/dashboard-agent/src/dashboard-agent.ts b/internal-packages/dashboard-agent/src/dashboard-agent.ts index 779938b1564..8e1d9082309 100644 --- a/internal-packages/dashboard-agent/src/dashboard-agent.ts +++ b/internal-packages/dashboard-agent/src/dashboard-agent.ts @@ -1,3 +1,4 @@ +import { isWatchRequestMessageId } from "@internal/dashboard-agent-contracts"; import { chat } from "@trigger.dev/sdk/ai"; import { locals, logger, tasks } from "@trigger.dev/sdk"; import { generateText, stepCountIs, streamText, type ModelMessage, type UIMessage } from "ai"; @@ -26,10 +27,11 @@ import { import { titlePrompt } from "./prompts"; import { PROMPT_CACHE_CONTROL } from "./prompt-prefix"; import { recordPromptCacheUsage, stepCachePrepareStep } from "./step-cache"; +import { dashboardAgentActionSchema, handleWatchAction } from "./watch-actions"; import { dashboardAgentCompaction, withDurableState } from "./compaction"; -// The runtime lives in its own module; re-exported here so every existing import -// path still resolves. +// The runtime and the watch lanes live in their own modules; re-exported here so +// every existing import path still resolves. export { clientDataSchema, dashboardAgentModelKey, @@ -50,8 +52,8 @@ export { shouldEvalTurn, turnReadSource, } from "./eval-policy"; -// The rolling step cache lives in `step-cache.ts`; re-exported so every existing -// import path still resolves. +// The rolling step cache lives in `step-cache.ts`, shared with the watch lane; +// re-exported so every existing import path still resolves. export { markStepCacheBreakpoint, MIN_STEP_CACHE_CHARS, @@ -59,6 +61,15 @@ export { STEP_CACHE_CONTROL, withStepCacheBreakpoint, } from "./step-cache"; +export { + dashboardAgentActionSchema, + wakeStartsInvestigation, + watchInvestigateActionSchema, + watchWakeActionSchema, + type DashboardAgentAction, + type WatchInvestigateAction, + type WatchWakeAction, +} from "./watch-actions"; /** * The in-dashboard agent, built on chat.agent and deployed as an internal task @@ -276,10 +287,13 @@ const pendingTitles = new Map>(); * Whether this turn is the one that names the chat. Counted in user messages, not in * transcript length: a head-started turn arrives with the warm first step already in * `uiMessages`, so a length gate would see two messages on the very first exchange and - * never name the chat at all. + * never name the chat at all. A watch's consent record is a user message the user did + * not type, so it doesn't count as an exchange either. */ export function isFirstUserExchange(uiMessages: { role: string; id?: string }[]): boolean { - const typed = uiMessages.filter((message) => message.role === "user"); + const typed = uiMessages.filter( + (message) => message.role === "user" && !isWatchRequestMessageId(message.id) + ); return typed.length <= 1; } @@ -322,8 +336,8 @@ export type { * message so the growing conversation prefix is read back cheaply. * * The between-steps compaction path rebuilds history as the summary alone and never - * reaches `compactModelMessages`, so the live investigation state is pinned back - * here instead. + * reaches `compactModelMessages`, so the live investigation and watch state is pinned + * back here instead. */ export function prepareTurnMessages(args: { messages: ModelMessage[]; @@ -340,6 +354,8 @@ export function prepareTurnMessages(args: { export const dashboardAgent = chat.agent({ id: "dashboard-agent", clientDataSchema, + // Actions are not turns — see `narrateWatchWake`. + actionSchema: dashboardAgentActionSchema, // Short idle window so suspended runs release their DB pool. idleTimeoutInSeconds: 60, @@ -384,6 +400,11 @@ export const dashboardAgent = chat.agent({ }); }, + // Every action is a watch action, handled in `watch-actions.ts`: one message + // deduped on the action id, piped inside so it reaches the history and read-model. + onAction: async ({ action, chatId, clientData, uiMessages, messages }) => + handleWatchAction({ action, chatId, clientData, uiMessages, messages }), + onTurnStart: async ({ chatId, uiMessages, clientData }) => { locals.set(turnErroredKey, false); diff --git a/internal-packages/dashboard-agent/src/eval-policy.ts b/internal-packages/dashboard-agent/src/eval-policy.ts index 6e6e0c9699f..3bae1c440c7 100644 --- a/internal-packages/dashboard-agent/src/eval-policy.ts +++ b/internal-packages/dashboard-agent/src/eval-policy.ts @@ -78,6 +78,7 @@ export function turnReadSource(toolActivity: Array<{ toolName: string }>): boole */ const STRUCTURAL_KEYS = new Set([ // Containers the walk descends into. + "alerts", "data", "deploys", "environments", @@ -90,6 +91,7 @@ const STRUCTURAL_KEYS = new Set([ "runs", "schedules", "tasks", + "watches", // Identity. "batchId", "deployId", @@ -106,6 +108,7 @@ const STRUCTURAL_KEYS = new Set([ "taskId", "taskIdentifier", "traceId", + "watchId", "friendlyId", // Names and kinds — a task, queue, error class or environment name is a fact. "environment", @@ -170,6 +173,7 @@ const TOOL_STRUCTURAL_KEYS: Record = { run_query: ["columns"], get_query_schema: ["columns", "tables", "table", "column"], get_report: ["sections", "section", "title", "metric", "metrics"], + list_alerts: ["channel", "enabled"], get_queue: ["concurrencyLimit", "paused"], correlate_version: ["versions", "before", "after"], }; diff --git a/internal-packages/dashboard-agent/src/index.ts b/internal-packages/dashboard-agent/src/index.ts index ba47d622411..2035f24daab 100644 --- a/internal-packages/dashboard-agent/src/index.ts +++ b/internal-packages/dashboard-agent/src/index.ts @@ -2,4 +2,13 @@ // the webapp bundle and registers the task in the wrong context. export * from "./dashboard-agent.js"; +export type { + WatchBatchCheckEntry, + WatchBatchCheckResponse, + WatchBatchTickPayload, + watchBatchTick, + WatchTickPayload, + watchTick, +} from "./watch-tick.js"; + export type { ChartBlock, DiagnosisBlock, ViewBlock } from "@internal/dashboard-agent-contracts"; diff --git a/internal-packages/dashboard-agent/src/repo-tools.test.ts b/internal-packages/dashboard-agent/src/repo-tools.test.ts index f1061f116b4..5d9579b40fe 100644 --- a/internal-packages/dashboard-agent/src/repo-tools.test.ts +++ b/internal-packages/dashboard-agent/src/repo-tools.test.ts @@ -139,6 +139,19 @@ describe("repo-tools", () => { expect(res.startLine).toBe(3000); }); + it("read_file reports the last line it actually served when the cap cuts a range short", async () => { + const res: any = await call(tools.read_file, { + path: "src/trigger/narrow.ts", + startLine: 1, + endLine: 4000, + }); + expect(res.truncated).toBe(true); + const served = res.content.split("\n"); + expect(served).toHaveLength(MAX_READ_LINES); + expect(res.startLine).toBe(1); + expect(res.endLine).toBe(MAX_READ_LINES); + }); + it("read_file leaves a small file untruncated", async () => { const res: any = await call(tools.read_file, { path: "src/trigger/order.ts" }); expect(res.truncated).toBe(false); diff --git a/internal-packages/dashboard-agent/src/repo-tools.ts b/internal-packages/dashboard-agent/src/repo-tools.ts index c39ac07dd6e..96b44095174 100644 --- a/internal-packages/dashboard-agent/src/repo-tools.ts +++ b/internal-packages/dashboard-agent/src/repo-tools.ts @@ -268,11 +268,14 @@ export function buildRepoTools( const from = Math.max(1, startLine ?? 1); const to = Math.min(lines.length, endLine ?? lines.length); const range = capRead(lines.slice(from - 1, to).join("\n")); + // The cap can cut the range short, and a line number that outruns the + // content is a citation anchored to a line the model never saw. + const served = Math.min(to, from + range.content.split("\n").length - 1); return { path, content: range.content, startLine: from, - endLine: to, + endLine: served, ...(range.truncated ? { truncated: true, notice: READ_TRUNCATION_NOTICE } : {}), }; } diff --git a/internal-packages/dashboard-agent/src/step-cache.ts b/internal-packages/dashboard-agent/src/step-cache.ts index 3aaa2dd4198..56465a0ee9a 100644 --- a/internal-packages/dashboard-agent/src/step-cache.ts +++ b/internal-packages/dashboard-agent/src/step-cache.ts @@ -10,8 +10,9 @@ import { * The rolling within-turn cache breakpoint, and the per-step telemetry that shows * whether the provider honoured it. * - * Shared by every multi-step `streamText` the agent runs, so a ten-step - * investigation doesn't re-send its accumulated tool outputs uncached on every step. + * Shared by every multi-step `streamText` the agent runs — the user's own turn and + * the consented watch investigation — so a ten-step investigation doesn't re-send + * its accumulated tool outputs uncached on every step. */ export const STEP_CACHE_CONTROL = { type: "ephemeral", ttl: "5m" } as const; diff --git a/internal-packages/dashboard-agent/src/test-support.ts b/internal-packages/dashboard-agent/src/test-support.ts index fa3ebdad679..f91ae05201b 100644 --- a/internal-packages/dashboard-agent/src/test-support.ts +++ b/internal-packages/dashboard-agent/src/test-support.ts @@ -85,14 +85,25 @@ export type StoreCalls = { setChatTitleIfDefault: unknown[]; upsertInvestigationRevision: unknown[]; settleInvestigationCard: unknown[]; - findOpenInvestigation: unknown[]; + seedInvestigation: unknown[]; /** Every write in the order it happened, for the tests that assert ordering. */ order: (keyof Omit)[]; }; -export function fakeStore( - options: { openInvestigation?: { id: string; projectRef: string; environmentRef: string } } = {} -): { store: DashboardAgentStore; calls: StoreCalls } { +/** An investigation as the fake store holds it: enough to tell whose card it is. */ +export type FakeInvestigation = { + chatId: string; + projectRef: string; + environmentRef: string; + state: { outcome?: string } & Record; +}; + +export function fakeStore(options: { investigations?: Map } = {}): { + store: DashboardAgentStore; + calls: StoreCalls; + /** The rows, so a test can assert which cards a lane touched and which it left alone. */ + investigations: Map; +} { const calls: StoreCalls = { ensureChat: [], persistMessages: [], @@ -101,7 +112,7 @@ export function fakeStore( setChatTitleIfDefault: [], upsertInvestigationRevision: [], settleInvestigationCard: [], - findOpenInvestigation: [], + seedInvestigation: [], order: [], }; const record = >(kind: K, args: unknown) => { @@ -112,6 +123,11 @@ export function fakeStore( // testable if a later revision is actually a higher number. const revisions = new Map(); const closedCards = new Set(); + const investigations = options.investigations ?? new Map(); + const writeState = (id: string, state: unknown) => { + const row = investigations.get(id); + if (row) row.state = state as FakeInvestigation["state"]; + }; const store: DashboardAgentStore = { ensureChat: async (args) => record("ensureChat", args), persistMessages: async (args) => record("persistMessages", args), @@ -147,6 +163,7 @@ export function fakeStore( const id = args.id ?? "inv_fake"; const revision = args.id ? (revisions.get(id) ?? 0) + 1 : 0; revisions.set(id, revision); + writeState(id, args.state); return { ok: true, id, revision, created: !args.id }; }, // Mirrors the real query: the terminal revision and its closing card are one @@ -162,16 +179,37 @@ export function fakeStore( }); if (!card) throw new Error(`${args.id} settled to a state that isn't renderable`); revisions.set(args.id, revision); + writeState(args.id, args.state); const closed = !closedCards.has(args.messageId); closedCards.add(args.messageId); return { ok: true, id: args.id, revision, card, closed }; }, - findOpenInvestigation: async (args) => { - record("findOpenInvestigation", args); - return options.openInvestigation ?? null; + // Mirrors the real query: insert under the caller's id, or hand back the row that + // is already there — unless it belongs to another chat or environment. + seedInvestigation: async (args) => { + record("seedInvestigation", args); + const existing = investigations.get(args.id); + if (existing) { + if ( + existing.chatId !== args.chatId || + existing.projectRef !== args.projectRef || + existing.environmentRef !== args.environmentRef + ) { + return { ok: false, error: "context_mismatch" }; + } + return { ok: true, id: args.id, created: false }; + } + investigations.set(args.id, { + chatId: args.chatId, + projectRef: args.projectRef, + environmentRef: args.environmentRef, + state: args.state as FakeInvestigation["state"], + }); + revisions.set(args.id, 0); + return { ok: true, id: args.id, created: true }; }, }; - return { store, calls }; + return { store, calls, investigations }; } // Records the eval enqueues, in place of tasks.trigger. diff --git a/internal-packages/dashboard-agent/src/tool-alerts.ts b/internal-packages/dashboard-agent/src/tool-alerts.ts new file mode 100644 index 00000000000..0247e8a28b5 --- /dev/null +++ b/internal-packages/dashboard-agent/src/tool-alerts.ts @@ -0,0 +1,109 @@ +import { tool, type ToolSet } from "ai"; +import { createAlertSchema, deleteAlertSchema, listAlertsSchema } from "./tool-schemas"; +import { NO_AUTH, type DashboardAgentApiClient } from "./tool-api-client"; +import type { DashboardAgentToolContext } from "./tool-context"; + +/** + * The alert tools and the request helper they share. Project-level, so these use the + * delegated token and never the env JWT, and a 403 is a capability refusal to explain. + */ +export function buildAlertTools(args: { + ctx: DashboardAgentToolContext; + client: DashboardAgentApiClient; +}): ToolSet { + const { ctx, client } = args; + const { userActorToken } = ctx; + const { origin, hasAuth } = client; + + async function alertsRequest( + method: "GET" | "POST" | "DELETE", + path: string, + body?: unknown + ): Promise<{ data: unknown } | { error: string }> { + let res: Response; + try { + res = await fetch(`${origin}${path}`, { + method, + headers: { + Authorization: `Bearer ${userActorToken!}`, + Accept: "application/json", + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + } catch (error) { + return { error: `Couldn't reach the alerts API: ${(error as Error).message}` }; + } + + const data = (await res.json().catch(() => undefined)) as + | { error?: string; code?: string } + | undefined; + + // 403 is a capability refusal and `code` says which one. + if (res.status === 403) { + return { + error: + data?.code === "email_alerts_not_configured" + ? "Email delivery isn't set up on this instance, so an email alert can't be created. Tell the user that, and that watch results still show in the dashboard." + : "Email alerts aren't enabled here. Tell the user that, and that watch results still show in the dashboard.", + }; + } + if (res.status === 400 && data?.code === "email_not_allowed") { + return { error: data.error ?? "Alerts can only go to the user's own account email." }; + } + if (!res.ok) { + return { error: data?.error ?? `The alerts API failed (status ${res.status}).` }; + } + return { data }; + } + + return { + // Project-level, so these use the delegated token, not the env JWT. Every call + // carries the chat id, which is what the API scopes its authorization through. + list_alerts: tool({ + ...listAlertsSchema, + execute: async () => { + if (!hasAuth) return NO_AUTH; + if (!ctx.chatId) return { error: "No chat is available to read alerts from." }; + const result = await alertsRequest( + "GET", + `/api/v1/dashboard-agent/alerts?chatId=${encodeURIComponent(ctx.chatId)}` + ); + if ("error" in result) return result; + const alerts = (result.data as { alerts?: unknown } | undefined)?.alerts; + return { alerts: Array.isArray(alerts) ? alerts : [] }; + }, + }), + + create_alert: tool({ + ...createAlertSchema, + execute: async ({ email }) => { + if (!hasAuth) return NO_AUTH; + if (!ctx.chatId) return { error: "No chat is available to create an alert from." }; + const result = await alertsRequest("POST", "/api/v1/dashboard-agent/alerts", { + chatId: ctx.chatId, + channel: "email", + ...(email ? { email } : {}), + }); + if ("error" in result) return result; + // The route's body is the channel itself, not an envelope around one. + return { created: true, alert: result.data }; + }, + }), + + delete_alert: tool({ + ...deleteAlertSchema, + execute: async ({ alertId }) => { + if (!hasAuth) return NO_AUTH; + if (!ctx.chatId) return { error: "No chat is available to change alerts from." }; + const result = await alertsRequest( + "DELETE", + `/api/v1/dashboard-agent/alerts/${encodeURIComponent(alertId)}`, + { chatId: ctx.chatId } + ); + if ("error" in result) return result; + return { deleted: true, alertId }; + }, + }), + }; +} diff --git a/internal-packages/dashboard-agent/src/tool-api-client.ts b/internal-packages/dashboard-agent/src/tool-api-client.ts index eb31b248887..d7c32071845 100644 --- a/internal-packages/dashboard-agent/src/tool-api-client.ts +++ b/internal-packages/dashboard-agent/src/tool-api-client.ts @@ -1,4 +1,5 @@ import { logger } from "@trigger.dev/sdk"; +import { DASHBOARD_AGENT_ENV_JWT_SCOPES } from "./tool-schemas.js"; /** * The agent's HTTP surface: the delegated-token GET, the env-JWT exchange and its @@ -68,9 +69,7 @@ async function exchangeEnvJwt( res = await fetch(`${origin}/api/v1/projects/${projectRef}/${environmentName}/jwt`, { method: "POST", headers, - body: JSON.stringify({ - claims: { scopes: ["read:runs", "read:deployments", "read:errors", "read:query"] }, - }), + body: JSON.stringify({ claims: { scopes: [...DASHBOARD_AGENT_ENV_JWT_SCOPES] } }), }); } catch { return { ok: false, envUnavailable: "unknown" }; diff --git a/internal-packages/dashboard-agent/src/tool-api.ts b/internal-packages/dashboard-agent/src/tool-api.ts index d7f19587566..c70f725ffd3 100644 --- a/internal-packages/dashboard-agent/src/tool-api.ts +++ b/internal-packages/dashboard-agent/src/tool-api.ts @@ -25,6 +25,7 @@ import { isEnvUnavailable, NO_AUTH, type DashboardAgentApiClient, + type EnvFetchResult, type EnvUnavailable, } from "./tool-api-client"; import type { DashboardAgentToolContext } from "./tool-context"; @@ -63,6 +64,116 @@ export function envUnavailableError(result: EnvUnavailable, action: string): { e * The API read tools, in the frozen key order `dashboardAgentToolSchemas` declares: * a different order is a different cached prompt prefix. */ +/** + * Whether a metrics answer carries no evidence the queue exists. Zeroes across the board + * are what the route returns for a name it has never seen, and also what a genuinely idle + * queue looks like — so this only decides whether to try the other queue kind, never what + * to tell the user. + */ +export function queueMetricsAreEmpty(data: unknown): boolean { + const d = data as { + peakQueued?: number; + startedCount?: number; + throttledCount?: number; + depthTrend?: unknown[]; + waitMs?: { p50?: number | null; p95?: number | null }; + } | null; + if (!d) return true; + return ( + (d.peakQueued ?? 0) === 0 && + (d.startedCount ?? 0) === 0 && + (d.throttledCount ?? 0) === 0 && + (d.depthTrend ?? []).length === 0 && + d.waitMs?.p50 == null && + d.waitMs?.p95 == null + ); +} + +/** + * The deployed tasks whose `queueConfig` points at this queue, from the current worker's + * task list. A custom queue's name is unrelated to any task id, so who consumes it can + * only be read off the tasks — never guessed from the name. + */ +export function consumerTasksForQueue(workers: unknown, queueName: string): string[] { + const tasks = (workers as { worker?: { tasks?: unknown } } | null)?.worker?.tasks; + if (!Array.isArray(tasks)) return []; + const slugs = new Set(); + for (const task of tasks as Array<{ slug?: unknown; queueConfig?: { name?: unknown } | null }>) { + if (typeof task?.slug !== "string") continue; + if (task.queueConfig?.name === queueName) slugs.add(task.slug); + } + return [...slugs].sort(); +} + +/** + * What the queue's live row read came back with. "The route said 404" and "the read never + * landed" are different answers: only the first is evidence about the queue, and collapsing + * them turns an expired token or a 5xx into "that queue doesn't exist". + */ +export type QueueLiveRead = + | { kind: "row"; row: Record } + | { kind: "missing" } + | { kind: "unknown"; status?: number }; + +/** Reads the live-row response into those three states. */ +export function readQueueLiveState(result: EnvFetchResult | null): QueueLiveRead { + // No current environment, or a read that never landed: nothing is known either way. + if (!result) return { kind: "unknown" }; + if (isEnvUnavailable(result)) { + return { + kind: "unknown", + status: result.envUnavailable === "unknown" ? result.status : undefined, + }; + } + if (!result.ok) { + return result.status === 404 ? { kind: "missing" } : { kind: "unknown", status: result.status }; + } + const row = (result.data as { data?: Record })?.data ?? result.data; + if (!row || typeof row !== "object") return { kind: "unknown" }; + return { kind: "row", row: row as Record }; +} + +/** + * The better of two live reads of the same queue name under either kind. A row wins; failing + * that a failed read wins over a 404, since one 404 with the other read broken is not proof. + */ +export function pickQueueLiveState(first: QueueLiveRead, second: QueueLiveRead): QueueLiveRead { + if (first.kind === "row") return first; + if (second.kind === "row") return second; + if (first.kind === "unknown") return first; + return second; +} + +/** + * Metrics plus the queue's live row. `paused` is the part the model must lead with: a queue + * someone stopped explains its own emptiness, and every metric below it is a consequence + * rather than a finding. When the read failed, `exists` is `"unknown"` rather than `false`, + * because an unreachable queue is not an absent one. + */ +export function withLiveState(metrics: unknown, queueType: "task" | "custom", live: QueueLiveRead) { + if (live.kind === "missing") return { ...(metrics as object), queueType, exists: false }; + if (live.kind === "unknown") { + return { + ...(metrics as object), + queueType, + exists: "unknown" as const, + liveStateError: live.status + ? `Couldn't read the queue's live row (status ${live.status}).` + : "Couldn't read the queue's live row.", + }; + } + const { row } = live; + return { + ...(metrics as object), + queueType: (row.type as string) ?? queueType, + exists: true, + paused: Boolean(row.paused), + queuedNow: row.queued ?? null, + runningNow: row.running ?? null, + concurrencyLimit: row.concurrencyLimit ?? null, + }; +} + export function buildApiTools(args: { ctx: DashboardAgentToolContext; client: DashboardAgentApiClient; @@ -338,19 +449,69 @@ export function buildApiTools(args: { get_queue: tool({ ...getQueueSchema, execute: async ({ queue, type, period }) => { - const sp = new URLSearchParams({ type: type ?? "task" }); - if (period) sp.append("period", period); - // Queue names may contain `/`; encode them as a single path segment. - const result = await envApiGet( - `/api/v1/queues/${encodeURIComponent(queue)}/metrics?${sp.toString()}` - ); - if (isEnvUnavailable(result)) return envUnavailableError(result, "read queues from"); - if (!result.ok) { + // The metrics route answers an unknown queue with zeroes rather than a 404, so a + // wrong `type` reads exactly like an idle queue — and the wrong half of that pair + // is easy to pick, since a named queue and a task's own queue look alike. Try the + // other kind before believing the zeroes, and say which one answered. + const read = async (kind: "task" | "custom") => { + const sp = new URLSearchParams({ type: kind }); + if (period) sp.append("period", period); + // Queue names may contain `/`; encode them as a single path segment. + const result = await envApiGet( + `/api/v1/queues/${encodeURIComponent(queue)}/metrics?${sp.toString()}` + ); + return result; + }; + + // Live state first: metrics are a window, and a window can't say "paused" or show a + // backlog that arrived after it. A queue nobody is running is not the same as a queue + // someone stopped, and the answer has to lead with which one it is. + const live = async (kind: "task" | "custom") => { + const result = await envApiGet( + `/api/v1/queues/${encodeURIComponent(queue)}?type=${kind}` + ); + return readQueueLiveState(result); + }; + + // Only a custom queue needs this read: a task queue's consumer is the task it is + // named after, while a custom queue's name says nothing about who writes to it. + const answer = async (metrics: unknown, kind: "task" | "custom", state: QueueLiveRead) => { + const base = withLiveState(metrics, kind, state); + if (base.queueType !== "custom" || !hasAuth || !projectRef || !environmentName) { + return base; + } + const workers = await apiGet( + origin, + `/api/v1/projects/${projectRef}/${environmentName}/workers/current`, + userActorToken! + ); + if (!workers.ok) return base; + return { ...base, consumerTasks: consumerTasksForQueue(workers.data, queue) }; + }; + + const first = await read(type ?? "task"); + if (isEnvUnavailable(first)) return envUnavailableError(first, "read queues from"); + if (!first.ok) { return { - error: `Couldn't get metrics for the ${queue} queue (status ${result.status}).`, + error: `Couldn't get metrics for the ${queue} queue (status ${first.status}).`, }; } - return result.data; + if (queueMetricsAreEmpty(first.data)) { + const otherKind = type === "custom" ? "task" : "custom"; + const other = await read(otherKind); + if (!isEnvUnavailable(other) && other.ok && !queueMetricsAreEmpty(other.data)) { + return await answer(other.data, otherKind, await live(otherKind)); + } + // Neither kind has metrics, so the live row is the only thing that can tell them + // apart: a paused or empty queue that exists, against a name that doesn't. + const kind = type ?? "task"; + const primary = await live(kind); + const state = + primary.kind === "row" ? primary : pickQueueLiveState(primary, await live(otherKind)); + return await answer(first.data, kind, state); + } + const kind = type ?? "task"; + return await answer(first.data, kind, await live(kind)); }, }), diff --git a/internal-packages/dashboard-agent/src/tool-investigations.ts b/internal-packages/dashboard-agent/src/tool-investigations.ts index bd5a8b42642..43c8dd4589d 100644 --- a/internal-packages/dashboard-agent/src/tool-investigations.ts +++ b/internal-packages/dashboard-agent/src/tool-investigations.ts @@ -3,6 +3,7 @@ import { investigationBlockSchema, safeParseTriggerUri, VIEW_BLOCK_VERSION, + WATCH_MAX_HOURS, type InvestigationAction, type InvestigationBlockBodyInput, type InvestigationCapabilities, @@ -39,6 +40,9 @@ export function showCodeAskPrompt(args: { path: string; line: number; sha: strin ); } +/** Defaults the "Watch for a repeat" card shows, which the user can change. */ +const RECURRENCE_WATCH = { checkEveryMinutes: 15, maxHours: WATCH_MAX_HOURS } as const; + /** * The card's typed next actions, decided here and never by the model. "Show code" * needs a concluded card, a cited source line, and a read at that commit this turn. @@ -82,6 +86,27 @@ export function investigationCapabilities( const errorUri = cited.find((evidence) => evidence.kind === "error")?.uri; + // "Watch for a repeat" needs a cited error fingerprint to pre-fill from, so without + // one it is left off. + const parsedError = errorUri ? safeParseTriggerUri(errorUri) : undefined; + if (state.outcome === "concluded" && parsedError?.success && parsedError.data.kind === "error") { + actions.push({ + kind: "watch_recurrence", + label: "Watch for a repeat", + intent: { + kind: "watch", + // A pre-fill only: emitting the spec creates nothing. + spec: { + kind: "error_recurrence", + fingerprint: parsedError.data.fingerprint, + checkEveryMinutes: RECURRENCE_WATCH.checkEveryMinutes, + maxHours: RECURRENCE_WATCH.maxHours, + note: `A repeat of: ${state.title}`, + }, + }, + }); + } + if (errorUri) { actions.push({ kind: "view_similar", @@ -124,7 +149,17 @@ export function createInvestigationRenderer( * back. `continueId` is only a pointer; the turn's own closure wins when set. */ return async function renderInvestigations(blocks: ViewBlockInput[], continueId?: string) { - if (!blocks.some((block) => block.type === "investigation")) return { blocks }; + const investigationBlocks = blocks.filter((block) => block.type === "investigation").length; + if (investigationBlocks === 0) return { blocks }; + + // One id is assigned per call, so a second block in the same view would be written + // as the next revision of the first: one card carrying two subjects. + if (investigationBlocks > 1) { + return { + error: + "A view holds at most one investigation block, and this one has more than one. Render one investigation per call, passing its own investigationId back each time.", + }; + } if (!ctx.investigations) { return { error: "Investigations aren't available on this turn, so I can't render one." }; diff --git a/internal-packages/dashboard-agent/src/tool-queue.test.ts b/internal-packages/dashboard-agent/src/tool-queue.test.ts new file mode 100644 index 00000000000..1cbcd5f6742 --- /dev/null +++ b/internal-packages/dashboard-agent/src/tool-queue.test.ts @@ -0,0 +1,191 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildApiTools, + consumerTasksForQueue, + pickQueueLiveState, + queueMetricsAreEmpty, + readQueueLiveState, + withLiveState, +} from "./tool-api"; +import { createApiClient } from "./tool-api-client"; + +/** + * The metrics route answers an unknown queue with zeroes rather than a 404, so asking for + * the wrong queue kind reads exactly like an idle queue. `get_queue` retries with the other + * kind before believing that, which is what stops "no queue named email-sends exists" being + * said about a queue holding thousands of runs. + */ +describe("queueMetricsAreEmpty", () => { + const zeroes = { + peakQueued: 0, + startedCount: 0, + throttledCount: 0, + depthTrend: [], + waitMs: { p50: null, p95: null }, + }; + + it("treats an all-zero answer as no evidence the queue exists", () => { + expect(queueMetricsAreEmpty(zeroes)).toBe(true); + expect(queueMetricsAreEmpty(null)).toBe(true); + }); + + it("takes any single sign of life as evidence", () => { + expect(queueMetricsAreEmpty({ ...zeroes, peakQueued: 4800 })).toBe(false); + expect(queueMetricsAreEmpty({ ...zeroes, startedCount: 3 })).toBe(false); + expect(queueMetricsAreEmpty({ ...zeroes, throttledCount: 1 })).toBe(false); + expect(queueMetricsAreEmpty({ ...zeroes, depthTrend: [0, 0] })).toBe(false); + expect(queueMetricsAreEmpty({ ...zeroes, waitMs: { p50: 0, p95: null } })).toBe(false); + }); +}); + +/** + * The environment that produced the bug: `email-sends` is a custom queue two deployed tasks + * write to, and no task is named after it. Reading the deployed task list for a task called + * `email-sends` finds nothing, which is what let the agent invent a deleted task. + */ +describe("consumerTasksForQueue", () => { + const workers = { + worker: { + tasks: [ + { slug: "send-order-receipt", queueConfig: { name: "email-sends" } }, + { slug: "send-welcome-email", queueConfig: { name: "email-sends" } }, + { slug: "generate-monthly-report", queueConfig: { name: "reports-heavy" } }, + { slug: "sync-inventory", queueConfig: { name: "webhooks" } }, + { slug: "email-sends-audit", queueConfig: null }, + ], + }, + }; + + it("names the tasks that write to a custom queue nothing is named after", () => { + expect(consumerTasksForQueue(workers, "email-sends")).toEqual([ + "send-order-receipt", + "send-welcome-email", + ]); + expect(consumerTasksForQueue(workers, "reports-heavy")).toEqual(["generate-monthly-report"]); + }); + + it("matches the queue config's name, not the task slug", () => { + // `email-sends-audit` has no queue config, so it is on its own task queue. + expect(consumerTasksForQueue(workers, "email-sends-audit")).toEqual([]); + expect(consumerTasksForQueue(workers, "send-order-receipt")).toEqual([]); + }); + + it("says nothing rather than something wrong when the task list is missing", () => { + expect(consumerTasksForQueue(null, "email-sends")).toEqual([]); + expect(consumerTasksForQueue({ worker: {} }, "email-sends")).toEqual([]); + expect(consumerTasksForQueue({ worker: { tasks: [{}] } }, "email-sends")).toEqual([]); + }); +}); + +/** + * A queue nobody can read is not a queue that isn't there. Only the route answering 404 is + * evidence of absence; a 401, a 429 or a 5xx is evidence of nothing, and reporting one as + * `exists: false` tells the model a queue holding thousands of runs was deleted. + */ +describe("the queue's live row has three answers, not two", () => { + const metrics = { peakQueued: 4800, startedCount: 12 }; + + it("reads a row, a 404 and a failed read apart", () => { + expect(readQueueLiveState({ ok: true, data: { paused: true } })).toEqual({ + kind: "row", + row: { paused: true }, + }); + expect(readQueueLiveState({ ok: false, status: 404 })).toEqual({ kind: "missing" }); + for (const status of [401, 403, 429, 500, 503]) { + expect(readQueueLiveState({ ok: false, status })).toEqual({ kind: "unknown", status }); + } + // No current environment: nothing was asked, so nothing is known. + expect(readQueueLiveState(null)).toEqual({ kind: "unknown" }); + }); + + it("says unknown rather than absent when the read failed", () => { + expect(withLiveState(metrics, "custom", { kind: "unknown", status: 503 })).toMatchObject({ + exists: "unknown", + liveStateError: "Couldn't read the queue's live row (status 503).", + }); + expect(withLiveState(metrics, "custom", { kind: "missing" })).toMatchObject({ exists: false }); + expect( + withLiveState(metrics, "custom", { kind: "row", row: { paused: true, queued: 9 } }) + ).toMatchObject({ exists: true, paused: true, queuedNow: 9 }); + }); + + it("prefers a row, then a failed read, over a single 404", () => { + const row = { kind: "row", row: { paused: false } } as const; + const missing = { kind: "missing" } as const; + const unknown = { kind: "unknown", status: 500 } as const; + + expect(pickQueueLiveState(missing, row)).toEqual(row); + expect(pickQueueLiveState(unknown, row)).toEqual(row); + // One kind 404s while the other read broke: that is not proof the name is free. + expect(pickQueueLiveState(missing, unknown)).toEqual(unknown); + expect(pickQueueLiveState(unknown, missing)).toEqual(unknown); + expect(pickQueueLiveState(missing, missing)).toEqual(missing); + }); +}); + +/** The same three cases through `get_queue`, since the tool output is what the model reads. */ +describe("get_queue reports the live read it actually got", () => { + const ORIGIN = "https://api.example.com"; + + function stubFetch(liveResponse: () => Response) { + vi.stubGlobal( + "fetch", + vi.fn(async (input: any) => { + const url = typeof input === "string" ? input : input.url; + if (url.endsWith("/jwt")) { + return new Response(JSON.stringify({ token: "env-jwt" }), { status: 200 }); + } + if (url.includes("/metrics")) { + return new Response(JSON.stringify({ peakQueued: 4800, startedCount: 12 }), { + status: 200, + }); + } + return liveResponse(); + }) + ); + } + + function getQueue() { + const ctx = { + userActorToken: "uat", + apiOrigin: ORIGIN, + projectRef: "proj_ref", + environmentName: "dev", + }; + const tools = buildApiTools({ + ctx, + client: createApiClient(ctx), + renderInvestigations: (() => []) as any, + }); + return (input: any) => (tools.get_queue as any).execute(input, {} as any); + } + + afterEach(() => vi.unstubAllGlobals()); + + it("reports a healthy row as the queue that exists", async () => { + stubFetch( + () => + new Response(JSON.stringify({ type: "custom", paused: true, queued: 31 }), { status: 200 }) + ); + await expect(getQueue()({ queue: "email-sends", type: "custom" })).resolves.toMatchObject({ + exists: true, + paused: true, + queuedNow: 31, + }); + }); + + it("reports a 404 as the queue that isn't there", async () => { + stubFetch(() => new Response("", { status: 404 })); + await expect(getQueue()({ queue: "email-sends", type: "custom" })).resolves.toMatchObject({ + exists: false, + }); + }); + + it("reports a failed read as unknown, never as absent", async () => { + stubFetch(() => new Response("", { status: 503 })); + const answer = await getQueue()({ queue: "email-sends", type: "custom" }); + expect(answer).toMatchObject({ exists: "unknown" }); + expect(answer.exists).not.toBe(false); + expect(answer.liveStateError).toContain("503"); + }); +}); diff --git a/internal-packages/dashboard-agent/src/tool-schemas.ts b/internal-packages/dashboard-agent/src/tool-schemas.ts index b0f47db1b21..16e07bf5e88 100644 --- a/internal-packages/dashboard-agent/src/tool-schemas.ts +++ b/internal-packages/dashboard-agent/src/tool-schemas.ts @@ -2,10 +2,29 @@ * This file runs in the webapp bundle too. Only `ai`, `zod`, type-only AI SDK and * `@internal/dashboard-agent-contracts` may be imported here. */ -import { runFiltersSchema, viewBlockInputSchema } from "@internal/dashboard-agent-contracts"; +import { + runFiltersSchema, + viewBlockInputSchema, + watchSpecSchema, +} from "@internal/dashboard-agent-contracts"; import { tool } from "ai"; import { z } from "zod"; +/** + * What the environment JWT is minted with. Every tool that goes through `envApiGet` is + * authorized by this list and nothing else — the delegated token's own cap only caps it. + * A route whose resource is missing here answers 403, which reaches the model as absent + * data rather than as a permission problem, so it must match what the tools actually call. + */ +export const DASHBOARD_AGENT_ENV_JWT_SCOPES = [ + "read:runs", + "read:deployments", + "read:errors", + "read:query", + // A queue's own row — paused, depth, limit — is a `queues` read; its metrics are not. + "read:queues", +] as const; + export const listProjectsSchema = tool({ description: "List the Trigger.dev projects the user can access, with each project's ref, name, slug, and organization. Only for answering a question about which projects exist — your other tools already target the current project, so this is never a context lookup to prepare another call.", @@ -171,7 +190,7 @@ export const getReportSchema = tool({ export const getQueueSchema = tool({ description: - "Get one queue's metrics over a window: wait latency (p50/p95), peak depth, how many runs started (throughput), and how often the queue was throttled by its concurrency limit. Use this for 'how deep is the X queue', 'is X backed up', or 'why are runs waiting'.", + "Get one queue's metrics over a window: wait latency (p50/p95), peak depth, how many runs started (throughput), and how often the queue was throttled by its concurrency limit. Use this for 'how deep is the X queue', 'is X backed up', or 'why are runs waiting'. The answer also carries the queue's live row: `paused`, `queuedNow`, `runningNow`, `concurrencyLimit`, and `exists: false` when no queue of that name is there at all. When that read fails rather than answers, `exists` is `\"unknown\"` with a `liveStateError`: the queue's state is unknown, not missing. For a custom queue it also carries `consumerTasks`: the deployed tasks whose queue config names this queue.", inputSchema: z.object({ queue: z .string() @@ -293,7 +312,7 @@ export const navigateToSchema = tool({ export const renderViewSchema = tool({ description: - "Render a structured view in the dashboard panel: a stack of catalog blocks, instead of plain prose. The catalog has four blocks: `diagnosis` (the 'why did this run fail?' failure card, after gathering evidence with the read/source tools), `chart` (a line/bar chart of run_query results), `actions` (a row of 1-3 buttons offering next steps — an `ask` intent sends the labelled question as the user's next message, a `navigate` intent takes the user to a page), and `investigation` (a live card for a hypothesis-driven investigation: report the state and the tool assigns and keeps its identity, so re-rendering it updates the same card). The result carries the `investigationId` it assigned — pass that back as `investigationId` when you render the same investigation again, including on a later turn. An investigation is rendered at least TWICE: once as `in_progress` when you open it, then again with the same `investigationId` carrying the final outcome (`concluded` or `inconclusive`), as the last tool call of the turn. A card left at `in_progress` is an unfinished answer whatever your prose says: the user is left watching a spinner. Keep any accompanying message to a one-line lead-in.", + "Render a structured view in the dashboard panel: a stack of catalog blocks, instead of plain prose. The catalog has four blocks: `diagnosis` (the 'why did this run fail?' failure card, after gathering evidence with the read/source tools), `chart` (a line/bar chart of run_query results), `actions` (a row of 1-3 buttons offering next steps — a `watch` intent opens the watch configuration card pre-filled with the spec you composed, an `ask` intent sends the labelled question as the user's next message), and `investigation` (a live card for a hypothesis-driven investigation: report the state and the tool assigns and keeps its identity, so re-rendering it updates the same card). The result carries the `investigationId` it assigned — pass that back as `investigationId` when you render the same investigation again, including on a later turn. An investigation is rendered at least TWICE: once as `in_progress` when you open it, then again with the same `investigationId` carrying the final outcome (`concluded` or `inconclusive`), as the last tool call of the turn. A card left at `in_progress` is an unfinished answer whatever your prose says: the user is left watching a spinner. Keep any accompanying message to a one-line lead-in.", inputSchema: z.object({ blocks: z.array(viewBlockInputSchema).min(1).describe("The blocks to render, top to bottom."), investigationId: z @@ -305,6 +324,46 @@ export const renderViewSchema = tool({ }), }); +// `watchSpecSchema` enforces the cadence floors and 24h ceiling. `since` for error +// recurrence is server-set on persist, so it is absent here. + +export const scheduleWatchSchema = tool({ + description: + "Fill in a watch for the user to confirm. Use this whenever they want to be told about a future event: a run starting or finishing, a queue draining, growing past a threshold or coming back below one, a queue that stops moving at all, runs waiting in a queue longer than a limit, an error recurring, the health report recovering. This is the ONLY way to answer that — never poll by calling read tools over and over. It does NOT start the watch: it opens a configuration card pre-filled with what you composed, and the user confirming that card is what starts it. So never say a watch is running, scheduled, or that you'll tell them later — say you've filled one in for them to review. A watch checks on its own cadence and reports ONCE; it stops within 24 hours either way. `note` is why the watch exists in the user's own words — it is shown with the result.", + inputSchema: z.object({ + watch: watchSpecSchema.describe( + "What to watch, how often to check, and how long to keep watching. `note` is why the watch exists in the user's own words — it is shown when it fires." + ), + }), +}); + +export const listAlertsSchema = tool({ + description: + 'List this project\'s alert subscriptions for watch results — who gets notified when a watch resolves, and whether each one is enabled. Use this to answer "what alerts do I have?".', + inputSchema: z.object({}), +}); + +export const createAlertSchema = tool({ + description: + "Subscribe to an email alert for every watch that resolves in this project. It always goes to the user's own account email. ONLY call this when the user explicitly asked for an alert — never as a helpful extra. If it comes back denied, relay that honestly and offer the dashboard notification, which is always on, instead.", + inputSchema: z.object({ + email: z + .string() + .optional() + .describe( + "Omit this. Alerts can only go to the user's own account email; any other address is rejected." + ), + }), +}); + +export const deleteAlertSchema = tool({ + description: + "Turn one alert subscription off, by its id from list_alerts. Watch results still show in the dashboard.", + inputSchema: z.object({ + alertId: z.string().describe("The alert id returned by list_alerts."), + }), +}); + // Code-mode tools, present only when the project has a connected GitHub repo. const runIdField = z .string() @@ -385,6 +444,10 @@ export const dashboardAgentToolSchemas = { search_docs: searchDocsSchema, get_current_page: getCurrentPageSchema, navigate_to: navigateToSchema, + schedule_watch: scheduleWatchSchema, + list_alerts: listAlertsSchema, + create_alert: createAlertSchema, + delete_alert: deleteAlertSchema, }; // Code mode adds the source tools. Same key order `buildDashboardAgentTools` @@ -418,15 +481,19 @@ You have read-only tools that act as the user against their own account: - get_query_schema: discover the analytics tables and columns you can query with TRQL (runs, metrics, llm_metrics, llm_models). - run_query: run a read-only TRQL query (SQL-style over ClickHouse) against the current environment's analytics data. - ask_support: ask the Trigger.dev support assistant about how Trigger.dev works (docs, concepts, features, configuration, how-tos). -- render_view: render a structured view in the panel from the block catalog. The catalog has the "diagnosis" block (a failure card for a single run), the "chart" block (a line/bar chart of run_query results), the "actions" block (a row of 1-3 buttons offering next steps — an ask intent sends the labelled question as the user's next message, a navigate intent takes the user to a page), and the "investigation" block (a live card for a hypothesis-driven investigation). +- render_view: render a structured view in the panel from the block catalog. The catalog has the "diagnosis" block (a failure card for a single run), the "chart" block (a line/bar chart of run_query results), the "actions" block (a row of 1-3 buttons offering next steps — a watch intent opens the watch card pre-filled, an ask intent sends the labelled question as the user's next message), and the "investigation" block (a live card for a hypothesis-driven investigation). - get_report: the composed health report for the current environment (flow, execution, liveness), with a severity and the metrics behind each. -- get_queue: one queue's wait latency, peak depth, throughput, and throttling over a window. +- get_queue: one queue's wait latency, peak depth, throughput, and throttling over a window, plus its live row. Lead with paused when it is true: a paused queue explains its own emptiness, so say it is paused and only then the numbers. queuedNow is what is waiting right now, which a window of metrics cannot show; exists:false is the only thing that means the queue isn't there, never zeroed metrics, and exists:"unknown" means the live read failed — unknown, never missing. A custom queue's name is not a task id, so no task being named after it is not evidence about it — never conclude from list_tasks or a deployment that it is unconsumed, deleted, or renamed. consumerTasks is the answer to "who feeds this queue": empty means nothing deployed writes to it, and absent means you did not ask a custom queue. - list_deploys: recent deployments (versions) in the current environment, with status and commit message. - get_deploy: one deployment's detail, or the current promoted one when you omit the version. - correlate_version: the version, commit, and pull request a specific run actually ran. - search_docs: search the Trigger.dev documentation. - get_current_page: the page the user is on right now, and what the dashboard already noticed on it. - navigate_to: take the user to a run, error, queue, deployment, or a filtered runs list. +- schedule_watch: fill in a watch — for something to happen (a run finishing, a queue draining, crossing a depth threshold either way, stalling, or its runs waiting past an SLA, an error recurring, health recovering) — and show it to the user to confirm. +- list_alerts: the project's alert subscriptions for watch fires. +- create_alert: subscribe the user to an email alert for watch fires in this project. +- delete_alert: turn one alert subscription off. Guidelines: - Be concise and direct. A short, correct answer beats a long one. Default to 2-4 sentences; go longer only when the user asked for detail or the answer genuinely needs it. @@ -435,10 +502,13 @@ Guidelines: - Never state the same fact or number twice in one turn. If it's on a card you rendered, don't repeat it in prose; if you said it in a sentence, don't restate it in a list. - Never narrate the UI. Don't say a card "is rendered above", announce "here's the short version", or restate what a card you just rendered already shows. A card speaks for itself; add at most one short line, and only if it says something the card doesn't (a next step, a caveat, an answer to the exact question asked). - Prefer reading live data with your tools over guessing. When a run id, task, project, or environment is in question, look it up. +- A state that explains the data comes before the data. A paused queue, a resolved or ignored error, a task with no deployed version, a run someone cancelled: say that first, then the numbers, because every number under it is a consequence rather than a finding. "This queue is paused, so nothing has started" is the answer; "throughput is 0" alone is a fact that misleads. +- Empty is not the same as absent, and neither is the same as never. A window with no rows means nothing happened IN THAT WINDOW — widen it or say which window you looked at, rather than concluding the thing does not exist. A 404 on a trace usually means retention, not a missing run. Zeroed metrics are never proof a queue, task or error is gone. - Do the work — never hand it back. If a tool can fetch it, fetch it in THIS turn: "want me to drill into the queues?", "I can pull the metrics if you'd like" and every variant are banned when the drill-down is one tool call away. Offering to look is answering with homework. - "How do I check X?" about THEIR project means two things at once: the short how-to AND the actual check, done. Answer "how do I check queue health?" with their queues' health, then one line on where it lives in the dashboard. - The user does only what your tools genuinely cannot reach: their own infra, their code, external pages. When a next step really is theirs, separate it clearly ("on your side: …") — and never put a step there that you could have taken yourself. - For "what's broken" or "why is X failing" questions, start with list_errors to find the error groups, get_error for the detail, then list_runs with that error id to drill into the actual failing runs (and get_run_trace for one of them). +- An answer whose headline is an UNRESOLVED, recurring error ENDS with the watch offer — one line, "Want me to set up a watch so you're told if it hits again?", plus the render_view "actions" block that makes it a button — not with generic advice alone. This is the rule from the Watches section applied to its most common case; it is not optional there, and neither is the button. - Your tools are read-only and scoped to the current environment for run and task lookups. You can't change anything; for actions, point the user to where in the dashboard they can do it. - Never invent run IDs, task identifiers, metrics, or features. If a tool returns an error or nothing, say so plainly. - A truncated or paged result supports what you saw, never what you didn't. When a result is truncated or returns a nextCursor, you may not claim an absence — "only send-receipt failed", "nothing else is failing", "there are no others" are all out, even hedged with "in what I saw". Say what the page showed and that the list is incomplete, or read a source that can answer completeness (list_errors groups every error in the window) before you answer. @@ -462,6 +532,21 @@ Is anything wrong?: - When the report points at flow (runs not starting), follow up with get_queue on the queue it names to see depth, wait time, and throttling. When it points at execution, follow up with list_errors / get_run_trace. - When something started failing at a particular time, check list_deploys for a deploy in that window, and correlate_version on a failing run to see the exact commit and pull request it ran. +Watches — telling the user later: +- When the user wants to be told when something happens ("tell me when this run finishes", "let me know when the backlog drains", "tell me when it's back under 100", "tell me if that queue stops moving", "ping me if runs start waiting more than 5 minutes", "ping me if that error comes back", "tell me when prod is healthy again"), call schedule_watch. Never poll: repeating a read tool until the thing happens is not a watch, and you cannot wait inside a turn. +- Offer a watch whenever your answer points at something worth monitoring that you can't resolve now: a recurring or unresolved error, a queue trending toward trouble, a condition the user would want to hear about the moment it changes. The offer is two things together: one short line ("Want me to set up a watch so you're told if it hits again?") AND a render_view "actions" block with one button — label it like "Set up a watch", intent {"kind":"watch","spec":{…}} carrying the same spec schedule_watch would compose. Clicking it opens the configuration card pre-filled, so the user answers with a click instead of typing "yeah". One offer per answer at most; skip it when the news is good, when the user is clearly just browsing, or when this answer's investigation card already carries a watch button — the card is the offer, and repeating it puts two watch buttons on one answer. schedule_watch is still how you answer a user who asks for a watch in their own words. +- schedule_watch does not start anything. It opens a configuration card pre-filled with what you composed, and the user confirming that card is what starts the watch. So say what you filled in — what is being watched, how often it checks, and when it gives up (the maxHours you set) — and that confirming starts it. Never say it's running, scheduled, or that you'll tell them later: "I've filled in a watch for you to review — confirm to start it", never "I'll let you know when it finishes". Pick the longest cadence that still answers in time — 1 minute only for a run's state, 5 minutes or more for backlog, error recurrence, and health. +- The card settles everything after the user confirms: whether this chat can hold another watch, whether the same thing is already watched, and whether the condition is already true (in which case they get the answer instead of a watch). Never promise, predict, or pre-explain any of those. +- A watch wake is a message you send unprompted, and it is narrated ONCE, briefly: what the outcome was, the numbers from the facts you were given, and one suggested next step. Nothing else — no new investigation, no fresh reads, no recap of the conversation. +- The ONE exception to "no new investigation": the user consented on the card ("investigate attention outcomes"). That opt-in is the card's, it starts off, and you cannot set it — if they asked for it ("watch it and dig in if it goes wrong"), say it's there to tick before they confirm. +- A consented investigation applies only to outcomes that need attention: a run that failed, a queue that stayed backed up, an error that came back. Good news and neutral news end the watch and nothing else happens. When the wake tells you the investigation has already started, say so in one short clause and stop: you conduct it yourself straight after, and the findings land in your next message with the card. The user never has to ask for them. +- On an expiry, say which of the two happened: it didn't happen in the window, or the condition couldn't be verified at expiry (then give the last observation and don't claim either way). +- Only call a wait "queue wait" when the facts measured it from when the run was queued. If the facts only have time from creation to start, call it that. +- Being notified outside the chat is the card's other opt-in, also off by default. Don't offer an email after filling in a card — the card is where that's chosen. +- After a wake that fired, and only if no alert is subscribed yet, your ONE suggested next step may be that same offer — one short line. Never create an alert unprompted. +- Call create_alert only after the user confirms. If it comes back denied (plan or feature flag), say so plainly and add that the dashboard still shows the notification badge for every fire. +- "What alerts do I have?" is list_alerts. Turning one off is delete_alert — if which one is ambiguous, list them and ask which. + Product questions: - For "how do I …" questions about Trigger.dev itself, use search_docs and answer from what it returns, citing the doc. ask_support is for longer, composed troubleshooting answers. Never invent an API or option that isn't in either. - When the answer sends the user to a specific URL — the contact page, the status page, a docs page — write it as a markdown link, never as bare text they have to retype. diff --git a/internal-packages/dashboard-agent/src/tools.ts b/internal-packages/dashboard-agent/src/tools.ts index b0b61b2e3b5..24050a3f9ea 100644 --- a/internal-packages/dashboard-agent/src/tools.ts +++ b/internal-packages/dashboard-agent/src/tools.ts @@ -1,10 +1,12 @@ import type { ToolSet } from "ai"; import { buildRepoTools } from "./repo-tools"; +import { buildAlertTools } from "./tool-alerts"; import { buildApiTools } from "./tool-api"; import { createApiClient } from "./tool-api-client"; import { createInvestigationRenderer } from "./tool-investigations"; import { buildNavigationTools } from "./tool-navigation"; import { createSourceReadLedger } from "./tool-source-ledger"; +import { buildWatchTools } from "./watch-tools"; import type { DashboardAgentToolContext } from "./tool-context"; export type { DashboardAgentToolContext } from "./tool-context"; @@ -41,6 +43,8 @@ export function buildDashboardAgentTools(ctx: DashboardAgentToolContext): ToolSe const apiTools: ToolSet = { ...buildApiTools({ ctx, client, renderInvestigations }), ...buildNavigationTools(ctx), + ...buildWatchTools(), + ...buildAlertTools({ ctx, client }), }; // Code mode: when the project has a connected repo, add the source tools. diff --git a/internal-packages/dashboard-agent/src/watch-actions.test.ts b/internal-packages/dashboard-agent/src/watch-actions.test.ts new file mode 100644 index 00000000000..597ec6421f4 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-actions.test.ts @@ -0,0 +1,1216 @@ +// `@trigger.dev/sdk/ai/test` MUST be imported before the agent module so the +// resource catalog is installed before `chat.agent({ id })` / `prompts.define` +// register at module load. +import { mockChatAgent, type MockChatAgentHarness } from "@trigger.dev/sdk/ai/test"; + +import type { LanguageModelV3StreamPart } from "@ai-sdk/provider"; +import { simulateReadableStream, type UIMessage } from "ai"; +import { MockLanguageModelV3 } from "ai/test"; +import { afterEach, describe, expect, it } from "vitest"; + +import { investigationSettlementMessage, watchInvestigationId } from "@internal/dashboard-agent-db"; + +import { + dashboardAgent, + dashboardAgentModelKey, + dashboardAgentStoreKey, + type DashboardAgentStore, +} from "./dashboard-agent"; +import { + CLIENT_DATA, + collectText, + executedTool, + fakeStore, + finish, + type FakeInvestigation, + mockModel, + textStep, + toolCallStep, + USAGE, +} from "./test-support"; + +/** + * Stands in for `chats` + `chat_messages`, as `appendOneMessage`'s upsert sees them: the + * insert is scoped to the owning user and — when the caller passes one — the owning + * organization, and keyed on (chat_id, message_id) with nothing done on conflict. So a + * repeat is never a second row and a foreign tenancy is no row at all. Row counts are + * what the History panel reads, which is why these tests assert those, not call counts. + */ +function transcriptTable(owner: { userId: string; organizationId: string }) { + const rows: { chatId: string; messageId: string }[] = []; + const countOf = (chatId: string, messageId: string) => + rows.filter((row) => row.chatId === chatId && row.messageId === messageId).length; + return { + countOf, + insert(args: { chatId: string; userId: string; organizationId?: string; message: UIMessage }) { + if (args.userId !== owner.userId) return false; + if (args.organizationId !== undefined && args.organizationId !== owner.organizationId) { + return false; + } + if (countOf(args.chatId, args.message.id) > 0) return false; + rows.push({ chatId: args.chatId, messageId: args.message.id }); + return true; + }, + }; +} + +// A store that writes into `table`, failing the appends `failWhen` selects. +function appendingStore( + table: ReturnType, + failWhen: (message: UIMessage) => boolean, + options?: Parameters[0] +) { + const { store, calls } = fakeStore(options); + const wrapped: DashboardAgentStore = { + ...store, + appendMessage: async (args) => { + await store.appendMessage(args); + const message = args.message as UIMessage; + if (failWhen(message)) throw new Error("the append lost the connection"); + return table.insert({ ...args, message }); + }, + }; + return { store: wrapped, calls }; +} + +// Every organization a path's appends were scoped to, in order. +function scopedTo(...stores: { calls: { appendMessage: unknown[] } }[]) { + return stores.flatMap((store) => + store.calls.appendMessage.map((call) => (call as { organizationId?: string }).organizationId) + ); +} + +describe("watch wake narration", () => { + let harness: MockChatAgentHarness | undefined; + + afterEach(async () => { + await harness?.close(); + harness = undefined; + }); + + const WAKE = { + type: "watch.fired" as const, + id: "watch:watch_1:fired", + watchId: "watch_1", + identity: "backlog_drain:task/send-receipt", + spec: { + kind: "backlog_drain", + queue: "task/send-receipt", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when the backlog drains", + }, + facts: { pending: 0, peakPending: 412, drainedAt: "2026-01-01T12:40:00.000Z" }, + }; + + it("narrates the wake once and persists it, and a redelivered wake narrates nothing", async () => { + const { store, calls } = fakeStore(); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake", + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([textStep("never asked for")])); + }, + }); + + const first = await harness.sendAction(WAKE); + // A drained queue is a fact the check already established, so the sentence is the + // dashboard's own wording and no model is called for it. + expect(collectText(first.chunks)).toBe( + "task/send-receipt queue drained\n\nNothing to do — I've stopped watching it." + ); + + // The streamed message must carry the same id the read-model copy is persisted + // under, or the panel renders the narration twice. + const startChunk = first.chunks.find( + (chunk) => (chunk as { type?: string }).type === "start" + ) as { messageId?: string } | undefined; + expect(startChunk?.messageId).toBe("wake:watch:watch_1:fired"); + + // An action is not a turn, so no turn persistence ran. The narration lands in the + // read-model as an id-deduped append, never a wholesale write: a card-born chat's + // transcript holds host blocks the session view can't see. + expect(calls.persistTurn).toHaveLength(0); + expect(calls.persistMessages).toHaveLength(0); + expect(calls.appendMessage).toHaveLength(1); + const appended = calls.appendMessage[0] as { userId: string; message: UIMessage }; + expect(appended.userId).toBe(CLIENT_DATA.userId); + expect(appended.message).toMatchObject({ id: "wake:watch:watch_1:fired", role: "assistant" }); + + // Same action id again (the watcher retried after appending): nothing is narrated, + // and the only write is the id-deduped repair of the same message. + const second = await harness.sendAction(WAKE); + expect(collectText(second.chunks)).toBe(""); + expect(calls.appendMessage.map((call) => (call as { message: UIMessage }).message.id)).toEqual([ + "wake:watch:watch_1:fired", + "wake:watch:watch_1:fired", + ]); + }); + + // Records the prompt it was asked with, so the wake's framing can be asserted. + function recordingModel(text: string) { + const prompts: unknown[] = []; + const model = new MockLanguageModelV3({ + doStream: async (options) => { + prompts.push(options.prompt); + return { stream: simulateReadableStream({ chunks: textStep(text) }) }; + }, + doGenerate: async () => ({ + content: [{ type: "text", text }], + finishReason: { unified: "stop", raw: "stop" }, + usage: USAGE, + warnings: [], + }), + }); + return { model, prompts }; + } + + function wakeText(prompts: unknown[]): string { + return JSON.stringify(prompts); + } + + // A completed window is an answer, never "the watch expired with nothing to say". + it("frames a completed window as the answer the user asked for", async () => { + const { store } = fakeStore(); + const { model, prompts } = recordingModel("The backlog still hasn't drained — 42 pending."); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_window", + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction({ + ...WAKE, + type: "watch.expired" as const, + id: "watch:watch_1:expired", + resolution: "window_completed" as const, + observed: { kind: "backlog_drain", verified: true, depth: 42 }, + facts: { verified: true, reason: "not_met_by_expiry", depth: 42 }, + }); + + const prompt = wakeText(prompts); + expect(prompt).toContain("window_completed"); + expect(prompt).toContain("this is the answer the user asked for"); + expect(prompt).toContain("reports once"); + // The wire encoding is transport, not vocabulary. + expect(prompt).not.toContain("the watch ended without firing"); + }); + + it("hands the observed outcome to the narration, not just the resolution", async () => { + const { store } = fakeStore(); + const { model, prompts } = recordingModel("Run run_abc123 failed after 4.2s."); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_failed", + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction({ + ...WAKE, + identity: "run_finished:run_abc123", + spec: { ...WAKE.spec, kind: "run_finished", runId: "run_abc123" }, + resolution: "condition_met" as const, + observed: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 4200, + }, + facts: { outcome: "COMPLETED_WITH_ERRORS", durationMs: 4200 }, + }); + + const prompt = wakeText(prompts); + expect(prompt).toContain("What the final check observed"); + expect(prompt).toContain("COMPLETED_WITH_ERRORS"); + }); + + // A wake from a watcher predating the resolution model still narrates. + it("falls back to the transport encoding when a wake carries no resolution", async () => { + const { store } = fakeStore(); + const { model, prompts } = recordingModel("never asked for"); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_legacy", + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + const wake = await harness.sendAction({ + ...WAKE, + type: "watch.expired" as const, + id: "watch:watch_1:expired", + facts: { reason: "terminal_unsatisfied" }, + }); + + // Read as `condition_impossible`: only that resolution says the queue is gone. + expect(collectText(wake.chunks)).toContain("task/send-receipt queue no longer exists"); + expect(wakeText(prompts)).toBe("[]"); + }); + + // A wake needs the project's external ref to scope the investigation the way a turn + // would; the watcher puts it in the wake's metadata. + const WAKE_CLIENT_DATA = { + ...CLIENT_DATA, + projectRef: "proj_abc", + environmentId: "env_abc", + }; + + const FAILED_RUN_WAKE = { + ...WAKE, + identity: "run_finished:run_abc123", + spec: { ...WAKE.spec, kind: "run_finished", runId: "run_abc123" }, + resolution: "condition_met" as const, + observed: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 4200, + }, + facts: { outcome: "COMPLETED_WITH_ERRORS", durationMs: 4200 }, + }; + + it("opens the pre-approved investigation on an attention outcome, in the same wake turn", async () => { + const { store, calls } = fakeStore(); + const { model, prompts } = recordingModel( + "Run run_abc123 failed — I've started looking into why." + ); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_investigate", + clientData: WAKE_CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction({ ...FAILED_RUN_WAKE, investigateOnAttention: true }); + + // The wake lands first and says the investigation has started. + expect(calls.appendMessage).toHaveLength(1); + expect(wakeText(prompts)).toContain("ALREADY been started"); + + // Opened, not concluded: the wake has no token to read with, so the findings come + // later in their own message. + expect(calls.seedInvestigation).toHaveLength(1); + const opened = calls.seedInvestigation[0] as { + id: string; + chatId: string; + projectRef: string; + environmentRef: string; + state: { outcome: string; runId?: string }; + }; + // The watch's own id, so the investigating lane can name the same row later. + expect(opened.id).toBe(watchInvestigationId("watch_1")); + expect(opened.chatId).toBe("chat_wake_investigate"); + expect(opened.projectRef).toBe("proj_abc"); + expect(opened.environmentRef).toBe("env_abc"); + expect(opened.state.outcome).toBe("in_progress"); + expect(opened.state.runId).toBe("run_abc123"); + }); + + // Consent is for bad news, and the category comes from the contracts mapping rather + // than the flag or the resolution alone. + it("starts nothing on a positive outcome, consent or not", async () => { + const { store, calls } = fakeStore(); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_positive", + clientData: WAKE_CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([textStep("The backlog drained.")])); + }, + }); + + await harness.sendAction({ + ...WAKE, + resolution: "condition_met" as const, + observed: { kind: "backlog_drain", verified: true, depth: 0 }, + investigateOnAttention: true, + }); + + expect(calls.appendMessage).toHaveLength(1); + expect(calls.seedInvestigation).toHaveLength(0); + }); + + it("starts nothing on an attention outcome without consent", async () => { + const { store, calls } = fakeStore(); + const { model, prompts } = recordingModel("Run run_abc123 failed after 4.2s."); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_no_consent", + clientData: WAKE_CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(FAILED_RUN_WAKE); + + expect(calls.appendMessage).toHaveLength(1); + expect(calls.seedInvestigation).toHaveLength(0); + expect(wakeText(prompts)).not.toContain("ALREADY been started"); + }); + + // Opening the investigation must never delay, retry or invalidate the wake. The + // watcher has already marked the delivery by the time the agent runs, so the only + // thing this can break is the turn. + it("delivers the wake even when opening the investigation fails", async () => { + const { store, calls } = fakeStore(); + const failing: DashboardAgentStore = { + ...store, + seedInvestigation: async () => { + throw new Error("investigations are down"); + }, + }; + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_inv_fails", + clientData: WAKE_CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, failing); + set(dashboardAgentModelKey, mockModel([textStep("Run run_abc123 failed.")])); + }, + }); + + const wake = await harness.sendAction({ ...FAILED_RUN_WAKE, investigateOnAttention: true }); + + expect(collectText(wake.chunks)).toBe("Run run_abc123 failed."); + expect(calls.appendMessage).toHaveLength(1); + }); + + it("writes nothing and fails the action when the narration comes back empty", async () => { + const { store, calls } = fakeStore(); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_empty", + clientData: WAKE_CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([[finish("stop")]])); + }, + }); + + const turn = await harness.sendAction({ ...FAILED_RUN_WAKE, investigateOnAttention: true }); + + expect( + turn.chunks.some( + (chunk) => + (chunk as { type?: string; errorText?: string }).type === "error" && + /produced no text/.test((chunk as { errorText?: string }).errorText ?? "") + ) + ).toBe(true); + expect(collectText(turn.chunks)).toBe(""); + expect(calls.appendMessage).toHaveLength(0); + expect(calls.persistMessages).toHaveLength(0); + expect(calls.seedInvestigation).toHaveLength(0); + }); + + it("a different outcome on the same watch is a different wake", async () => { + const { store, calls } = fakeStore(); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_wake_two", + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([textStep("first"), textStep("second")])); + }, + }); + + await harness.sendAction(WAKE); + await harness.sendAction({ + ...WAKE, + type: "watch.expired", + id: "watch:watch_2:expired", + watchId: "watch_2", + facts: { verified: false, reason: "unverified_at_expiry" }, + }); + + expect(calls.appendMessage).toHaveLength(2); + expect(calls.appendMessage.map((call) => (call as { message: UIMessage }).message.id)).toEqual([ + "wake:watch:watch_1:fired", + "wake:watch:watch_2:expired", + ]); + }); + + /** + * The wake is durable on `session.out` the moment it streams, which is before the + * display copy is written. So an append that fails leaves the model seeing a message + * the History panel doesn't have — and the retry boots with that message already in + * its history. Converging on the row is the retry's job. + */ + it("appends the display copy on a retry that finds the wake already narrated", async () => { + const table = transcriptTable(CLIENT_DATA); + const chatId = "chat_wake_retry"; + const wakeId = "wake:watch:watch_1:fired"; + + const failing = appendingStore(table, (message) => message.id === wakeId); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, failing.store); + set(dashboardAgentModelKey, mockModel([textStep("never asked for")])); + }, + }); + + const first = await harness.sendAction(WAKE); + // Streamed — so it is on `session.out` and in the next boot's history — while the + // row it was supposed to land alongside never arrived. + expect(collectText(first.chunks)).toContain("queue drained"); + expect(failing.calls.appendMessage).toHaveLength(1); + expect(table.countOf(chatId, wakeId)).toBe(0); + const durable = first.chunks; + await harness.close(); + + // The retry is a new run picking up the session, booting its history from the + // chunks the failed one left on `session.out`. + const repairing = appendingStore(table, () => false); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA, + continuation: true, + previousRunId: "run_wake_failed", + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, repairing.store); + set(dashboardAgentModelKey, mockModel([textStep("never asked for")])); + }, + }); + harness.seedSessionOutTail(durable); + + const retry = await harness.sendAction(WAKE); + + // Nothing narrated twice, and the display copy converged on exactly one row. + expect(collectText(retry.chunks)).toBe(""); + expect(table.countOf(chatId, wakeId)).toBe(1); + + // A third delivery repairs nothing, because there is nothing left to repair. + await harness.sendAction(WAKE); + expect(table.countOf(chatId, wakeId)).toBe(1); + + // Every write on this path is scoped to the organization the append verifies — the + // repair included, or the repair would be the one write that skips the check. + expect(scopedTo(failing, repairing)).toEqual([ + CLIENT_DATA.organizationId, + CLIENT_DATA.organizationId, + CLIENT_DATA.organizationId, + ]); + }); + + /** + * The chat id comes from the watch record and the tenancy from the session's + * `clientData`. If those ever disagree the append has to write nothing, rather than put + * a message in another organization's transcript. + */ + it("writes nothing when the wake's tenancy doesn't own the chat", async () => { + const table = transcriptTable(CLIENT_DATA); + const chatId = "chat_wake_other_org"; + const { store, calls } = appendingStore(table, () => false); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: { ...CLIENT_DATA, organizationId: "org_other" }, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([textStep("never asked for")])); + }, + }); + + await harness.sendAction(WAKE); + + // Attempted, and refused by the scope the append carries. + expect(calls.appendMessage).toHaveLength(1); + expect(table.countOf(chatId, "wake:watch:watch_1:fired")).toBe(0); + }); +}); + +describe("watch investigation", () => { + let harness: MockChatAgentHarness | undefined; + + afterEach(async () => { + await harness?.close(); + harness = undefined; + }); + + const INVESTIGATE = { + type: "watch.investigate" as const, + id: "watch:watch_1:fired:investigate", + watchId: "watch_1", + identity: "run_finished:run_abc123", + spec: { + kind: "run_finished", + runId: "run_abc123", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when the receipt run finishes", + }, + facts: { outcome: "COMPLETED_WITH_ERRORS", durationMs: 4200 }, + resolution: "condition_met" as const, + observed: { + kind: "run_finished", + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 4200, + }, + }; + + // The card the wake seeded, named the way both lanes name it: off the watch. + const SEEDED = watchInvestigationId("watch_1"); + + const CLIENT_DATA_WITH_TOKEN = { + ...CLIENT_DATA, + projectRef: "proj_abc", + environmentId: "env_abc", + environmentName: "prod", + apiOrigin: "https://api.example.com", + // The delegated token the kick minted, arriving the way a turn's does. + userActorToken: "uat_investigate", + }; + + const inProgress = { + outcome: "in_progress", + severity: "warn", + confidence: "low", + runId: "run_abc123", + title: "Investigating run_abc123", + headline: "The run finished with errors. Looking into why.", + hypotheses: [], + evidence: [], + }; + + const concluded = { + ...inProgress, + outcome: "concluded", + confidence: "high", + headline: "The receipt task threw on every attempt: the payload lost `order.total`.", + remediation: "Restore the field on the producer, or guard the read.", + hypotheses: [ + { + id: "hyp_payload", + statement: "The new payload no longer carries order.total.", + verdict: "validated", + finding: "Every attempt failed with the same TypeError.", + evidence: [], + }, + ], + }; + + // Records the prompts it was called with, and plays one step per call. + function recordingModel(steps: LanguageModelV3StreamPart[][]) { + const prompts: unknown[] = []; + let call = 0; + const model = new MockLanguageModelV3({ + doStream: async (options) => { + prompts.push(options.prompt); + const chunks = steps[Math.min(call, steps.length - 1)] ?? []; + call++; + return { stream: simulateReadableStream({ chunks }) }; + }, + doGenerate: async () => ({ + content: [{ type: "text", text: "" }], + finishReason: { unified: "stop", raw: "stop" }, + usage: USAGE, + warnings: [], + }), + }); + return { model, prompts }; + } + + const renderStep = ( + investigation: Record, + investigationId: string, + toolCallId: string + ) => + toolCallStep( + "render_view", + { blocks: [{ type: "investigation", investigation }], investigationId }, + toolCallId + ); + + it("revises the card the wake seeded, answers in its own message, and dedupes a replay", async () => { + const { store, calls } = fakeStore(); + const { model, prompts } = recordingModel([ + renderStep(concluded, SEEDED, "tc_verdict"), + textStep("The payload lost order.total — every attempt threw on the same line."), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate", + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + const turn = await harness.sendAction(INVESTIGATE); + + // A real investigating turn: the model called a tool and it executed. + expect(executedTool(turn.chunks)).toBe(true); + expect(collectText(turn.chunks)).toContain("order.total"); + + // The card the wake opened is the one this revises: no second investigation for + // the same news, and no id the model got to choose. + expect(calls.seedInvestigation).toHaveLength(1); + expect(calls.upsertInvestigationRevision).toHaveLength(1); + const revision = calls.upsertInvestigationRevision[0] as { + id?: string; + chatId: string; + state: { outcome: string }; + }; + expect(revision.id).toBe(SEEDED); + expect(revision.chatId).toBe("chat_investigate"); + expect(revision.state.outcome).toBe("concluded"); + + // The prompt names that card and frames the findings as their own message. + const prompt = JSON.stringify(prompts); + expect(prompt).toContain(SEEDED); + expect(prompt).toContain("pre-approved"); + expect(prompt).toContain("its own message"); + + // Findings appended once and whole: the render_view part is what the panel rebuilds + // the card from. + expect(calls.appendMessage).toHaveLength(1); + const appended = calls.appendMessage[0] as { userId: string; message: UIMessage }; + expect(appended.userId).toBe(CLIENT_DATA.userId); + expect(appended.message.id).toBe("investigate:watch:watch_1:fired:investigate"); + expect(appended.message.parts.some((part) => part.type === "tool-render_view")).toBe(true); + + // The same kick again: nothing runs, and the only write is the id-deduped repair of + // the findings message. + await harness.sendAction(INVESTIGATE); + expect(calls.appendMessage.map((call) => (call as { message: UIMessage }).message.id)).toEqual([ + "investigate:watch:watch_1:fired:investigate", + "investigate:watch:watch_1:fired:investigate", + ]); + expect(calls.upsertInvestigationRevision).toHaveLength(1); + }); + + /** + * The consented investigation gets the same ten-step budget a turn does, so without a + * rolling breakpoint every step re-sends the accumulated tool output at full price. + */ + it("rolls a step cache breakpoint across the investigation's steps", async () => { + const bulky = { + ...concluded, + // Past the provider's minimum cacheable prefix, so a breakpoint is worth setting. + headline: `${concluded.headline} ${"the same TypeError on order.total. ".repeat(200)}`, + }; + const { store } = fakeStore(); + const { model, prompts } = recordingModel([ + renderStep(bulky, SEEDED, "tc_verdict"), + textStep("The payload lost order.total."), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate_step_cache", + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(INVESTIGATE); + + const ttlOf = (message: unknown) => + (message as { providerOptions?: { anthropic?: { cacheControl?: { ttl?: unknown } } } }) + ?.providerOptions?.anthropic?.cacheControl?.ttl; + + expect(prompts.length).toBeGreaterThan(1); + // Step two's last message is the accumulated tool output, and it carries the short-lived + // breakpoint — the one the next step reads back instead of re-sending. + const second = prompts[1] as unknown[]; + expect(ttlOf(second.at(-1))).toBe("5m"); + // Never more than one: Anthropic allows four, and the prefix breakpoints take two. + expect(second.filter((message) => ttlOf(message) === "5m")).toHaveLength(1); + // Step one has nothing accumulated yet, so nothing short-lived is marked. + expect((prompts[0] as unknown[]).filter((m) => ttlOf(m) === "5m")).toHaveLength(0); + }); + + it("opens the watch's card itself when the wake's seed never landed", async () => { + const { store, calls } = fakeStore(); + const { model, prompts } = recordingModel([textStep("Couldn't get far — the trace is gone.")]); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate_seed", + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(INVESTIGATE); + + // The same id the wake would have used, so a late seed can never make a second card. + expect(calls.seedInvestigation).toMatchObject([{ id: SEEDED }]); + expect(calls.upsertInvestigationRevision).toHaveLength(0); + expect(calls.settleInvestigationCard).toMatchObject([ + { id: SEEDED, state: { outcome: "inconclusive" } }, + ]); + expect(JSON.stringify(prompts)).toContain(SEEDED); + expect(calls.appendMessage).toHaveLength(1); + }); + + /** + * One settle, not two. The row used to be settled once on its own and then again + * with the card, which bumped the revision twice for one outcome. + */ + it("settles a card the investigating turn left in progress, exactly once", async () => { + const { store, calls } = fakeStore(); + const { model } = recordingModel([ + renderStep(inProgress, SEEDED, "tc_open"), + textStep("still looking"), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate_unsettled", + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(INVESTIGATE); + + // Only the turn's own render_view revision; the settle is the atomic close below. + expect(calls.upsertInvestigationRevision).toHaveLength(1); + expect(calls.settleInvestigationCard).toHaveLength(1); + const settle = calls.settleInvestigationCard[0] as { + id: string; + messageId: string; + state: { outcome: string }; + }; + expect(settle.id).toBe(SEEDED); + expect(settle.state.outcome).toBe("inconclusive"); + expect(settle.messageId).toBe("investigate:watch:watch_1:fired:investigate:settled"); + }); + + function revisioningStore(options: { failClosingCard?: boolean } = {}) { + const { store, calls } = fakeStore(); + const closedCards: UIMessage[] = []; + let revision = 0; + const wrapped: DashboardAgentStore = { + ...store, + // The closing write, however the lane makes it: whether it is one atomic call or + // a bare append, the transcript half fails here. + appendMessage: async (args) => { + if (options.failClosingCard && args.message.id.endsWith(":settled")) { + throw new Error("the append lost the connection"); + } + return store.appendMessage(args); + }, + // The real query commits the revision and the card together, so a card that + // can't be delivered leaves the row exactly as it was. + settleInvestigationCard: async (args) => { + if (options.failClosingCard) { + calls.settleInvestigationCard.push(args); + throw new Error("the append lost the connection"); + } + const result = await store.settleInvestigationCard(args); + if (result.ok) closedCards.push(result.card as UIMessage); + return result; + }, + upsertInvestigationRevision: async (args) => { + await store.upsertInvestigationRevision(args); + return { + ok: true as const, + id: args.id ?? "inv_fake", + revision: revision++, + created: !args.id, + }; + }, + }; + return { store: wrapped, calls, closedCards }; + } + + function cardsIn(message: UIMessage) { + return (message.parts ?? []).flatMap((part) => { + const typed = part as { type?: string; output?: { blocks?: unknown[] } }; + if (typed.type !== "tool-render_view" || !Array.isArray(typed.output?.blocks)) return []; + return typed.output.blocks as Array<{ + type?: string; + id?: string; + revision?: number; + investigation?: { outcome?: string; progress?: string }; + }>; + }); + } + + it("puts the settled card in the transcript, once, without opening a second investigation", async () => { + const { store, calls, closedCards } = revisioningStore(); + const { model } = recordingModel([ + renderStep({ ...inProgress, progress: "Reading the trace" }, SEEDED, "tc_open"), + textStep("still looking"), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate_card", + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(INVESTIGATE); + + // The findings message is the only ordinary append; the closing card lands with the + // terminal revision, in one operation. + expect(calls.appendMessage).toHaveLength(1); + expect(closedCards).toHaveLength(1); + const closing = closedCards[0]!; + expect(closing.id).toBe("investigate:watch:watch_1:fired:investigate:settled"); + + const [card] = cardsIn(closing); + expect(card?.id).toBe(SEEDED); + expect(card?.investigation?.outcome).toBe("inconclusive"); + const [opened] = cardsIn((calls.appendMessage[0] as { message: UIMessage }).message); + expect(card!.revision!).toBeGreaterThan(opened!.revision!); + expect(card?.investigation?.progress).toBeUndefined(); + + const revisions = calls.upsertInvestigationRevision.length; + await harness.sendAction(INVESTIGATE); + expect(calls.appendMessage.map((call) => (call as { message: UIMessage }).message.id)).toEqual([ + "investigate:watch:watch_1:fired:investigate", + "investigate:watch:watch_1:fired:investigate", + ]); + expect(calls.upsertInvestigationRevision).toHaveLength(revisions); + }); + + /** + * The failure window this lane used to have: the row settled, the closing append + * failed, the error was logged and swallowed, and the action reported success. The + * row was then terminal, so the stale sweep no longer selected it and the panel span + * forever. Nothing in production calls the action again on its own — only a thrown + * error gets it retried. + */ + it("fails the action when the closing card can't be written, instead of reporting success", async () => { + const { store, calls } = revisioningStore({ failClosingCard: true }); + const { model } = recordingModel([ + renderStep(inProgress, SEEDED, "tc_open"), + textStep("still looking"), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate_close_fails", + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + const turn = await harness.sendAction(INVESTIGATE); + + expect( + turn.chunks.some( + (chunk) => + (chunk as { type?: string }).type === "error" && + /lost the connection/.test((chunk as { errorText?: string }).errorText ?? "") + ) + ).toBe(true); + + // The close was attempted as one operation, so no separate settle could have made + // the row terminal ahead of the card. + expect(calls.settleInvestigationCard).toHaveLength(1); + expect( + calls.upsertInvestigationRevision.filter( + (call) => (call as { state: { outcome: string } }).state.outcome !== "in_progress" + ) + ).toEqual([]); + }); + + /** A store whose atomic close refuses rather than throws, with the reason it refuses for. */ + function refusingStore(error: "not_found" | "context_mismatch" | "chat_missing") { + const { store, calls } = fakeStore(); + const wrapped: DashboardAgentStore = { + ...store, + settleInvestigationCard: async (args) => { + calls.settleInvestigationCard.push(args); + return { ok: false as const, error }; + }, + }; + return { store: wrapped, calls }; + } + + async function investigateAgainst(store: DashboardAgentStore, chatId: string) { + const { model } = recordingModel([ + renderStep(inProgress, SEEDED, "tc_open"), + textStep("still looking"), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + return harness.sendAction(INVESTIGATE); + } + + function erroredWith(turn: { chunks: unknown[] }, pattern: RegExp) { + return turn.chunks.some( + (chunk) => + (chunk as { type?: string }).type === "error" && + pattern.test((chunk as { errorText?: string }).errorText ?? "") + ); + } + + /** + * A refused close is the same failure as a thrown one: the card never landed, so the + * panel spins until the action is retried, and only a thrown error gets it retried. + */ + it.each(["not_found", "context_mismatch"] as const)( + "fails the action when the close is refused with %s", + async (error) => { + const { store, calls } = refusingStore(error); + const turn = await investigateAgainst(store, `chat_investigate_refused_${error}`); + + expect(calls.settleInvestigationCard).toHaveLength(1); + expect(erroredWith(turn, new RegExp(error))).toBe(true); + } + ); + + it("reports success when the close is refused because the chat is gone", async () => { + const { store, calls } = refusingStore("chat_missing"); + const turn = await investigateAgainst(store, "chat_investigate_refused_chat_missing"); + + expect(calls.settleInvestigationCard).toHaveLength(1); + expect(erroredWith(turn, /chat_missing|couldn't close/)).toBe(false); + }); + + it("says nothing when the kick carries no tenancy to scope a card with", async () => { + const { store, calls } = fakeStore(); + harness = mockChatAgent(dashboardAgent, { + chatId: "chat_investigate_unscoped", + clientData: CLIENT_DATA, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([textStep("should never run")])); + }, + }); + + await harness.sendAction(INVESTIGATE); + + expect(calls.seedInvestigation).toHaveLength(0); + expect(calls.upsertInvestigationRevision).toHaveLength(0); + expect(calls.appendMessage).toHaveLength(0); + }); + + /** + * The same window the wake has: the findings stream to `session.out` before the display + * copy is appended, so an append that fails leaves the model holding a message the + * History panel lost. The retry finds it already answered and must still land the row. + */ + it("appends the display copy on a retry that finds the investigation already answered", async () => { + const table = transcriptTable(CLIENT_DATA); + const chatId = "chat_investigate_retry"; + const findingsId = "investigate:watch:watch_1:fired:investigate"; + const steps = [ + renderStep(concluded, SEEDED, "tc_verdict"), + textStep("The payload lost order.total."), + ]; + + const failing = appendingStore(table, (message) => message.id === findingsId); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, failing.store); + set(dashboardAgentModelKey, recordingModel(steps).model); + }, + }); + + const first = await harness.sendAction(INVESTIGATE); + // Streamed whole, card part and all, while the row it belongs to never landed. + expect(executedTool(first.chunks)).toBe(true); + expect(failing.calls.appendMessage).toHaveLength(1); + expect(table.countOf(chatId, findingsId)).toBe(0); + const durable = first.chunks; + await harness.close(); + + // The retry is a new run picking up the session, booting its history from the chunks + // the failed one left on `session.out`. + const repairing = appendingStore(table, () => false); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA_WITH_TOKEN, + continuation: true, + previousRunId: "run_investigate_failed", + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, repairing.store); + set(dashboardAgentModelKey, recordingModel(steps).model); + }, + }); + harness.seedSessionOutTail(durable); + + const retry = await harness.sendAction(INVESTIGATE); + + // Nothing investigated a second time, and the display copy converged on one row. + expect(collectText(retry.chunks)).toBe(""); + expect(table.countOf(chatId, findingsId)).toBe(1); + + // A third delivery repairs nothing, because there is nothing left to repair. + await harness.sendAction(INVESTIGATE); + expect(table.countOf(chatId, findingsId)).toBe(1); + + // Every write on this path is scoped to the organization the append verifies — the + // repair included, or the repair would be the one write that skips the check. + expect(scopedTo(failing, repairing)).toEqual([ + CLIENT_DATA.organizationId, + CLIENT_DATA.organizationId, + CLIENT_DATA.organizationId, + ]); + }); + + /** + * The card a watch may settle is its own, and only its own. Anything else still + * running in this chat belongs to the user or to another watch, and settling it + * answers a question nobody asked here while overwriting the one they did. + */ + const MANUAL = "inv_manual"; + + function tenanted(chatId: string, state: Record): FakeInvestigation { + return { + chatId, + projectRef: "proj_abc", + environmentRef: "env_abc", + state: state as FakeInvestigation["state"], + }; + } + + it("settles its own card and leaves the user's open investigation alone", async () => { + const chatId = "chat_investigate_beside_manual"; + // The user's card is opened after the watch's, so "the freshest card still open" + // is theirs. + const investigations = new Map([ + [SEEDED, tenanted(chatId, inProgress)], + [MANUAL, tenanted(chatId, { ...inProgress, title: "Why is checkout slow?" })], + ]); + const { store, calls } = fakeStore({ investigations }); + const { model } = recordingModel([ + renderStep(inProgress, SEEDED, "tc_open"), + textStep("still looking"), + ]); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(INVESTIGATE); + + expect(calls.settleInvestigationCard).toMatchObject([{ id: SEEDED }]); + expect(investigations.get(SEEDED)?.state.outcome).toBe("inconclusive"); + expect(investigations.get(MANUAL)?.state.outcome).toBe("in_progress"); + }); + + /** The redelivery path had the same reach: it closed whatever card was still open. */ + it("closes only its own card when a redelivered kick finds the user's still open", async () => { + const chatId = "chat_investigate_redelivered_beside_manual"; + const investigations = new Map([ + [SEEDED, tenanted(chatId, inProgress)], + [MANUAL, tenanted(chatId, { ...inProgress, title: "Why is checkout slow?" })], + ]); + const { store, calls } = fakeStore({ investigations }); + const card = (id: string, title: string) => + investigationSettlementMessage({ + investigationId: id, + revision: 0, + state: { ...inProgress, title }, + messageId: `msg_${id}`, + }) as UIMessage; + + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA_WITH_TOKEN, + continuation: true, + // The findings already landed, so this kick is a repair; the user's card is the + // first one still open in the transcript. + snapshot: { + version: 1, + savedAt: Date.now(), + messages: [ + card(MANUAL, "Why is checkout slow?"), + card(SEEDED, "Investigating run_abc123"), + { + id: "investigate:watch:watch_1:fired:investigate", + role: "assistant", + parts: [{ type: "text", text: "The payload lost order.total." }], + }, + ], + }, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, mockModel([textStep("should never run")])); + }, + }); + + await harness.sendAction(INVESTIGATE); + + expect(calls.settleInvestigationCard).toMatchObject([{ id: SEEDED }]); + expect(investigations.get(MANUAL)?.state.outcome).toBe("in_progress"); + }); + + it("gives two watches resolving in one chat a card each", async () => { + const chatId = "chat_investigate_two_watches"; + const second = watchInvestigationId("watch_2"); + const { store, calls, investigations } = fakeStore(); + const { model } = recordingModel([textStep("still looking")]); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: CLIENT_DATA_WITH_TOKEN, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set(dashboardAgentModelKey, model); + }, + }); + + await harness.sendAction(INVESTIGATE); + await harness.sendAction({ + ...INVESTIGATE, + id: "watch:watch_2:fired:investigate", + watchId: "watch_2", + }); + + expect(calls.settleInvestigationCard.map((call) => (call as { id: string }).id)).toEqual([ + SEEDED, + second, + ]); + expect(investigations.get(SEEDED)?.state.outcome).toBe("inconclusive"); + expect(investigations.get(second)?.state.outcome).toBe("inconclusive"); + }); + + // Same tenancy crossing as the wake's: the kick names the chat, the session names the + // organization, and a disagreement must not write into another organization's chat. + it("writes nothing when the kick's tenancy doesn't own the chat", async () => { + const table = transcriptTable(CLIENT_DATA); + const chatId = "chat_investigate_other_org"; + const { store, calls } = appendingStore(table, () => false); + harness = mockChatAgent(dashboardAgent, { + chatId, + clientData: { ...CLIENT_DATA_WITH_TOKEN, organizationId: "org_other" }, + setupLocals: ({ set }) => { + set(dashboardAgentStoreKey, store); + set( + dashboardAgentModelKey, + recordingModel([ + renderStep(concluded, SEEDED, "tc_verdict"), + textStep("The payload lost order.total."), + ]).model + ); + }, + }); + + await harness.sendAction(INVESTIGATE); + + expect(calls.appendMessage).toHaveLength(1); + expect(table.countOf(chatId, "investigate:watch:watch_1:fired:investigate")).toBe(0); + }); +}); diff --git a/internal-packages/dashboard-agent/src/watch-actions.ts b/internal-packages/dashboard-agent/src/watch-actions.ts new file mode 100644 index 00000000000..ddb46e0f4d1 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-actions.ts @@ -0,0 +1,881 @@ +import { chat } from "@trigger.dev/sdk/ai"; +import { locals, logger } from "@trigger.dev/sdk"; +import { + readUIMessageStream, + stepCountIs, + streamText, + type ModelMessage, + type UIMessage, + type UIMessageChunk, +} from "ai"; +import { z } from "zod"; +import { + forceSettledInvestigationState, + formatTriggerUri, + watchResolutions, + watchResultNeedsAttention, + type InvestigationState, + type WatchObservedOutcome, + type WatchResolution, + type WatchSpec, +} from "@internal/dashboard-agent-contracts"; +import { watchInvestigationId } from "@internal/dashboard-agent-db"; +import { + buildTurnTools, + type clientDataSchema, + type DashboardAgentStore, + dashboardAgentModelKey, + getStore, + getSystemPrompt, + modeFor, + registry, + latestCards, + sanitizeReplayedToolInputs, + clearOpenInvestigations, + withCacheBreakpointOnLast, +} from "./agent-runtime"; +import { planWatchNarration, type WatchNarrationPlan } from "./watch-narration"; +import { recordPromptCacheUsage, stepCachePrepareStep } from "./step-cache"; + +/** + * The watch lanes: the wake narration and the consented investigation that can + * follow it. Both arrive as `.in` action records rather than turns, which is why + * each does its own model call and its own persistence. + */ + +/** + * The wake, as the agent receives it. + * + * A watch resolves long after the turn that scheduled it, so `watch-tick.ts` + * appends one record to the chat's `in` stream with `trigger: "action"`. That + * fires `onAction` only — no `onTurnStart`, `run()` or `onTurnComplete`, and the + * turn counter doesn't move — which is why the narration below does its own model + * call and its own persistence. + * + * `id` is stable per (watch, outcome) and becomes the narration message's id, so a + * redelivered wake finds its message in the history and narrates nothing. + * + * `type` keeps the fired/expired encoding as the stable TRANSPORT only. How the + * watch ended travels in `resolution`, what was seen in `observed`. + */ +export type WatchWakeAction = { + type: "watch.fired" | "watch.expired"; + /** `watch:{watchId}:{status}` — stable, so a redelivery is a no-op. */ + id: string; + watchId: string; + /** The watched thing, as the contracts' dedup string. */ + identity: string; + spec: WatchSpec & { since?: string }; + /** What the final check observed. The numbers the narration must use. */ + facts: Record; + /** How the watch ended: met, window completed, or impossible. */ + resolution?: WatchResolution; + /** What was true when it ended: the run's final status, the depth, the count. */ + observed?: WatchObservedOutcome; + /** Why the watch exists, in the user's words. */ + note?: string; + /** + * The user consented at creation to an investigation after an ATTENTION + * outcome. It relaxes one rule, "never a new investigation unprompted", and + * only for that outcome. + */ + investigateOnAttention?: boolean; +}; + +// Deliberately lenient on `spec`: a wake must never be lost to a validation error +// because the host persisted a field this version doesn't know about. +export const watchWakeActionSchema = z.object({ + type: z.enum(["watch.fired", "watch.expired"]), + id: z.string(), + watchId: z.string(), + identity: z.string().default(""), + spec: z + .object({ + kind: z.string(), + note: z.string().optional(), + checkEveryMinutes: z.number().optional(), + }) + .passthrough(), + facts: z.record(z.unknown()).default({}), + // Optional for the same reason: an older watcher predating the resolution model + // sends neither, and the narration falls back to the transport encoding. + resolution: z.enum(watchResolutions).optional(), + observed: z.record(z.unknown()).optional(), + note: z.string().optional(), + investigateOnAttention: z.boolean().optional(), +}); + +/** + * The second half of a consented watch: conduct the investigation the wake opened. + * + * Sent by the webapp, never by the watcher or a client, right after a delivered + * wake on an attention outcome the creator consented to. It carries a freshly + * minted delegated token in the record's metadata, the same way the `in` proxy + * injects a turn's token, so this turn can read like any other. + * + * It is an action rather than a turn because nobody asked a question: the wake + * landed as its own message and the findings arrive as another one. + */ +export type WatchInvestigateAction = { + type: "watch.investigate"; + /** `watch:{watchId}:{status}:investigate` — stable, so a redelivery is a no-op. */ + id: string; + watchId: string; + identity: string; + spec: WatchSpec & { since?: string }; + facts?: Record; + resolution?: WatchResolution; + observed?: WatchObservedOutcome; + note?: string; + /** + * The card to revise, when the sender knows it. Usually absent: the wake seeds + * the row inside the agent, so the id is resolved by `resolveInvestigationId`. + */ + investigationId?: string; +}; + +// Same leniency as the wake schema, for the same reason. +export const watchInvestigateActionSchema = z.object({ + type: z.literal("watch.investigate"), + id: z.string(), + watchId: z.string(), + identity: z.string().default(""), + spec: z + .object({ + kind: z.string(), + note: z.string().optional(), + checkEveryMinutes: z.number().optional(), + }) + .passthrough(), + facts: z.record(z.unknown()).default({}), + resolution: z.enum(watchResolutions).optional(), + observed: z.record(z.unknown()).optional(), + note: z.string().optional(), + investigationId: z.string().optional(), +}); + +/** + * Every action the agent accepts. The union is the whole vocabulary: anything + * else fails to parse and never reaches a handler. + * + * The trust boundary is the STREAM, not the schema. `.in` records are written + * with an environment secret key or from the dashboard's own server-side hop, + * and the `in` proxy refuses to forward a browser-supplied `trigger: "action"`. + * So the model can describe an action but never place one, and a forged record + * would carry no valid delegated token, leaving every read tool failed closed. + */ +export const dashboardAgentActionSchema = z.union([ + watchWakeActionSchema, + watchInvestigateActionSchema, +]); + +export type DashboardAgentAction = WatchWakeAction | WatchInvestigateAction; + +// The per-wake framing only. How a wake is narrated lives in the managed system +// prompt's Watches section, which is the cached block. +const WAKE_INSTRUCTION = + 'A watch you set up earlier has resolved and reports once, right now — this is not a question, and nobody is waiting on a reply. Write ONE short message: what the watch found, the numbers from the facts below, and one suggested next step. Say what happened; never say the watch "fired" or "expired". A window that ran out with the condition still not true is an answer, not a failure. No tools, no new investigation, no recap.'; + +/** + * How the watch ended. The narration speaks resolution and observed outcome, + * never "fired"/"expired" — those are the wire encoding, and a watch that ran its + * whole window and found nothing has an answer to give, not a failure. + * + * Falls back to the transport when a wake predates the resolution model. + */ +function wakeResolution(action: WatchWakeAction): WatchResolution { + if (action.resolution) return action.resolution; + if (action.type === "watch.fired") return "condition_met"; + return (action.facts as { reason?: string } | undefined)?.reason === "terminal_unsatisfied" + ? "condition_impossible" + : "window_completed"; +} + +function wakeOutcome(action: WatchWakeAction): string { + switch (wakeResolution(action)) { + case "condition_met": + return "the condition became true inside the window"; + case "condition_impossible": + return "the condition can no longer become true — that is the answer, not a timeout"; + case "window_completed": + // Deliberately not "nothing happened": "it didn't drain in an hour" is what + // the user asked to be told. + return "the window ran out with the condition still not true — this is the answer the user asked for, so report it plainly"; + } +} + +/** + * Whether this wake is the one the consent covers. + * + * Consent is for the ATTENTION outcomes only, and the contracts' resolved-result + * mapping decides which those are per kind — no surface may substitute its own + * judgement. Good news never starts anything, however the watch was configured. + */ +export function wakeStartsInvestigation(action: WatchWakeAction): boolean { + if (action.investigateOnAttention !== true) return false; + return watchResultNeedsAttention({ + kind: action.spec.kind, + resolution: wakeResolution(action), + outcome: action.observed as WatchObservedOutcome | undefined, + }); +} + +/** The watched thing, as either action carries it. */ +type WatchedSubject = { spec: WatchWakeAction["spec"]; identity: string }; + +/** The thing being watched, for the seeded investigation's own words. */ +function wakeSubject(action: WatchedSubject): string { + const spec = action.spec as Record; + for (const key of ["runId", "queue", "fingerprint", "report"]) { + const value = spec[key]; + if (typeof value === "string" && value.length > 0) return value; + } + return action.identity || String(spec.kind ?? "this"); +} + +// The wake's line when the investigation is pre-approved. Phrased as a fact about +// this turn so the model can't turn it into an offer. +function investigationInstruction(action: WatchWakeAction): string { + return `The user pre-approved an investigation for an outcome like this when they created the watch, and it has ALREADY been started for them — say so in one short clause, in the past tense, as part of your single message ("…I've started looking into why"). Never offer it, never ask, and don't describe what you'll check: you are conducting it right now and the findings land in your very next message, with the investigation card. Subject: ${wakeSubject( + action + )}.`; +} + +/** + * The watched object as a `trigger://` markdown link. The wake runs with no tools, + * so the link has to be handed to it ready-made, which needs the tenancy from the + * wake's metadata. + */ +function wakeSubjectLink( + action: WatchedSubject, + tenancy: { projectRef?: string; environmentId?: string } | undefined +): string | undefined { + const projectRef = tenancy?.projectRef; + const environmentId = tenancy?.environmentId; + if (!projectRef || !environmentId) return undefined; + + const spec = action.spec; + const target = + "queue" in spec && spec.queue + ? { kind: "queue" as const, projectRef, environmentId, name: spec.queue } + : "runId" in spec && spec.runId + ? { kind: "run" as const, projectRef, environmentId, runId: spec.runId } + : "fingerprint" in spec && spec.fingerprint + ? { kind: "error" as const, projectRef, environmentId, fingerprint: spec.fingerprint } + : "report" in spec && spec.report + ? { kind: "report" as const, projectRef, environmentId, key: spec.report } + : undefined; + if (!target) return undefined; + + const label = + target.kind === "queue" + ? target.name + : target.kind === "run" + ? target.runId + : target.kind === "error" + ? "this error" + : "the report"; + return `[${label}](${formatTriggerUri(target)})`; +} + +function wakePrompt( + action: WatchWakeAction, + tenancy?: { projectRef?: string; environmentId?: string } +): string { + const subjectLink = wakeSubjectLink(action, tenancy); + return [ + WAKE_INSTRUCTION, + `Resolution: ${wakeResolution(action)} — ${wakeOutcome(action)}.`, + `Watching: ${action.spec.kind}${action.identity ? ` (${action.identity})` : ""}.`, + action.observed + ? `What the final check observed:\n${JSON.stringify(action.observed, null, 2)}` + : undefined, + action.note ? `Why the user asked for it: ${action.note}` : undefined, + `Facts from the check:\n${JSON.stringify(action.facts, null, 2)}`, + subjectLink + ? `When you point at the watched object, link it: ${subjectLink} — use this exact markdown link, not a bare name.` + : undefined, + wakeStartsInvestigation(action) ? investigationInstruction(action) : undefined, + ] + .filter(Boolean) + .join("\n\n"); +} + +/** + * Open the pre-approved investigation, the one relaxation of "never a new + * investigation unprompted". + * + * Deliberately a seeded `in_progress` state and nothing more: the wake turn has no + * delegated token to read with, so the findings arrive later in their own message. + * + * Runs after the narration is in the transcript and never throws, so a failure + * here cannot delay, retry or invalidate the wake. + */ +async function openConsentedInvestigation(args: { + action: WatchWakeAction; + chatId: string; + clientData: z.infer | undefined; +}): Promise { + const { action, chatId, clientData } = args; + const projectRef = clientData?.projectRef; + const environmentRef = clientData?.environmentId; + if (!projectRef || !environmentRef) { + // A watch created before the row carried the project's external ref. Scoping + // it by the wrong identifier would strand the investigation, so skip it. + logger.warn("dashboard-agent watch wake can't scope a consented investigation", { + chatId, + watchId: action.watchId, + }); + return; + } + + const subject = wakeSubject(action); + const spec = action.spec as { runId?: unknown }; + try { + const result = await getStore().seedInvestigation({ + id: watchInvestigationId(action.watchId), + chatId, + projectRef, + environmentRef, + state: { + outcome: "in_progress", + severity: "warn", + confidence: "low", + title: `Investigating ${subject}`, + headline: `The watch on ${subject} resolved to something that needs attention${ + action.note ? ` (${action.note})` : "" + }. Looking into why.`, + hypotheses: [], + evidence: [], + ...(typeof spec.runId === "string" ? { runId: spec.runId } : {}), + startedAt: new Date().toISOString(), + }, + }); + logger.info("dashboard-agent watch wake opened a consented investigation", { + chatId, + watchId: action.watchId, + investigationId: result.ok ? result.id : undefined, + error: result.ok ? undefined : result.error, + }); + } catch (error) { + // The wake is the delivery that matters; an investigation that couldn't be + // opened is a lost follow-up, never a lost wake. + logger.error("dashboard-agent watch wake failed to open its investigation", { + chatId, + watchId: action.watchId, + error: (error as Error).message, + }); + } +} + +/** + * The narrating model, when there is one. + * + * Haiku gets the wake and nothing else: every number the sentence may use is in the + * facts, and the deterministic headline is handed over as the opening fact so the + * wording matches every other surface. Sonnet keeps the consented-investigation + * wake, where the promise and the findings that follow must read as one voice — and + * only there does the whole conversation come along. + */ +const HAIKU_WAKE_BRIEF = + 'You are the Trigger.dev dashboard agent, reporting on a watch the user asked you to keep. Write ONE short message, two sentences at most: the fact as given, then what it means for them and the single most useful next step. Never say the watch "fired" or "expired", never invent a number that isn\'t given, and don\'t recap the conversation — nobody is waiting on a reply.'; + +/** + * A hard ceiling on that message, because "two sentences at most" is an instruction and + * not a budget. Two sentences are well under 100 tokens, so this leaves plenty of room. + */ +const HAIKU_WAKE_MAX_OUTPUT_TOKENS = 300; + +/** The fixed narration, as the panel's stream sees it. Same shape a model would emit. */ +async function* fixedNarrationChunks( + messageId: string, + text: string +): AsyncGenerator { + yield { type: "start", messageId }; + yield { type: "text-start", id: "wake" }; + yield { type: "text-delta", id: "wake", delta: text }; + yield { type: "text-end", id: "wake" }; + yield { type: "finish" }; +} + +/** + * Stream the wake and return its final text. + * + * The streamed message must carry the SAME id the persisted copy uses: the panel + * merges live stream and loaded history by message id, so two ids render it twice. + */ +async function narrateWithPlan(input: { + plan: WatchNarrationPlan; + action: WatchWakeAction; + messageId: string; + tenancy: { projectRef?: string; environmentId?: string }; + args: { + clientData: z.infer | undefined; + messages: ModelMessage[]; + }; +}): Promise { + const { plan, action, messageId, tenancy, args } = input; + + if (plan.model === "none") { + await chat.pipe(fixedNarrationChunks(messageId, plan.text)); + return plan.text; + } + + const resolved = await getSystemPrompt(modeFor(args.clientData)); + const wake = wakePrompt(action, tenancy); + const result = + plan.model === "haiku" + ? streamText({ + model: + locals.get(dashboardAgentModelKey) ?? + registry.languageModel("anthropic:claude-haiku-4-5"), + system: HAIKU_WAKE_BRIEF, + // Bounded on purpose: the wake alone, no conversation and no tools. + messages: [ + { + role: "user" as const, + content: `${wake}\n\nOpen with this fact, in these words: ${plan.presentation.headline}.`, + }, + ], + maxOutputTokens: HAIKU_WAKE_MAX_OUTPUT_TOKENS, + }) + : streamText({ + model: + locals.get(dashboardAgentModelKey) ?? + registry.languageModel( + (resolved.model ?? "anthropic:claude-sonnet-4-6") as `anthropic:${string}` + ), + system: resolved.text, + // No tools: a wake reports what the check already established, and carries no + // delegated token to read with. The breakpoint goes on the last message of the + // existing prefix, not on the unique wake, so the wake reads back the same + // cached prefix a normal turn would instead of only writing cache. + messages: [ + ...withCacheBreakpointOnLast(sanitizeReplayedToolInputs(args.messages)), + { role: "user" as const, content: wake }, + ], + ...resolved.toAISDKTelemetry(), + }); + + await chat.pipe(result.toUIMessageStream({ generateMessageId: () => messageId })); + return (await result.text).trim(); +} + +/** + * Narrate one wake, exactly once. + * + * Streams so the panel shows it arriving live, then writes it to both `chat.history` + * (the transcript the model sees next turn) and the display read-model. Both must + * happen before `onAction` returns: `chat.history` mutations are only picked up + * immediately after the hook. + */ +async function narrateWatchWake(args: { + action: WatchWakeAction; + chatId: string; + clientData: z.infer | undefined; + uiMessages: UIMessage[]; + /** The same history in model form, as the action event supplies it. */ + messages: ModelMessage[]; +}): Promise { + const { action, chatId, uiMessages } = args; + const messageId = `wake:${action.id}`; + + // Dedup on the action id. Durable, because the history it checks is the + // snapshot the SDK reseeds on every boot — not per-process state. + const narrated = uiMessages.find((message) => message.id === messageId); + if (narrated) { + logger.info("dashboard-agent watch wake already narrated; repairing the display copy", { + chatId, + watchId: action.watchId, + actionId: action.id, + }); + // The streamed message is durable before the append is, so a retry can find the + // wake narrated and the display copy still owed. The append is id-deduped, so + // repairing when nothing is broken writes nothing. + const userId = args.clientData?.userId; + const organizationId = args.clientData?.organizationId; + if (userId) { + await getStore().appendMessage({ chatId, userId, organizationId, message: narrated }); + } + return; + } + + const tenancy = { + projectRef: args.clientData?.projectRef, + environmentId: args.clientData?.environmentId, + }; + const plan: WatchNarrationPlan = planWatchNarration({ + kind: action.spec.kind, + identity: action.identity, + resolution: wakeResolution(action), + observed: action.observed, + note: action.note, + subjectLink: wakeSubjectLink(action, tenancy), + startsInvestigation: wakeStartsInvestigation(action), + }); + logger.info("dashboard-agent watch wake narration lane", { + chatId, + watchId: action.watchId, + kind: action.spec.kind, + model: plan.model, + }); + + const text = await narrateWithPlan({ plan, action, messageId, tenancy, args }); + // Must throw: an unnarrated wake stays owed, so the retry says it again. + if (!text) { + throw new Error(`the wake narration for ${action.watchId} produced no text`); + } + + const message: UIMessage = { + id: messageId, + role: "assistant", + parts: [{ type: "text", text }], + }; + chat.history.set([...uiMessages, message]); + // The display copy is an id-deduped append, never a wholesale write: a wake has + // no client to carry the stored transcript, so the session view can miss + // host-appended blocks (a card-born chat starts with only those) and + // `persistMessages` would drop them. + const userId = args.clientData?.userId; + const organizationId = args.clientData?.organizationId; + if (userId) { + await getStore().appendMessage({ chatId, userId, organizationId, message }); + } else { + // A wake always carries its watch's tenancy, so reaching this means the + // metadata contract broke. Deliver anyway: losing blocks beats losing the wake. + logger.error("dashboard-agent watch wake has no userId; falling back to persistMessages", { + chatId, + }); + await getStore().persistMessages({ chatId, messages: [...uiMessages, message] }); + } + + // Only once the wake is in the transcript, so the investigation can never hold + // the wake up. + if (wakeStartsInvestigation(action)) { + await openConsentedInvestigation({ action, chatId, clientData: args.clientData }); + } +} + +/** + * The card this turn must revise: the sender's id when it has one, else the watch's + * own card, which the wake seeded under the same derived id. Seeding again is how a + * wake whose seed failed still gets a card, and it can only ever open this watch's. + */ +async function resolveInvestigationId(args: { + action: WatchInvestigateAction; + chatId: string; + projectRef: string; + environmentRef: string; +}): Promise { + const { action, chatId, projectRef, environmentRef } = args; + if (action.investigationId) return action.investigationId; + + const seeded = await getStore().seedInvestigation({ + id: watchInvestigationId(action.watchId), + chatId, + projectRef, + environmentRef, + state: { + outcome: "in_progress", + severity: "warn", + confidence: "low", + title: `Investigating ${wakeSubject(action)}`, + headline: `The watch on ${wakeSubject(action)} resolved to something that needs attention. Looking into why.`, + hypotheses: [], + evidence: [], + startedAt: new Date().toISOString(), + }, + }); + return seeded.ok ? seeded.id : undefined; +} + +/** + * Close the card this lane opened: its terminal revision and the closing card in one + * transaction, so a terminal row whose card never landed cannot exist. + * + * Only a deleted chat is swallowed. The row settles only if the card lands, and every other + * failure has to reach the action for the task's retry to be a real retry. + */ +async function closeCardInTranscript(args: { + store: DashboardAgentStore; + chatId: string; + investigationId: string; + projectRef: string; + environmentRef: string; + messageId: string; + uiMessages: UIMessage[]; + fallback: () => InvestigationState; +}): Promise { + const { store, chatId, investigationId, uiMessages } = args; + + if (uiMessages.some((message) => message.id === args.messageId)) return; + + const card = latestCards(uiMessages).get(investigationId); + if (card && card.state && card.state.outcome !== "in_progress") return; + + // The lane's own message id, not the revision-stable one: a redelivered kick must + // dedupe on the action, and this lane already checked for it above. + const result = await store.settleInvestigationCard({ + id: investigationId, + chatId, + projectRef: args.projectRef, + environmentRef: args.environmentRef, + state: forceSettledInvestigationState(card?.state ?? args.fallback()), + messageId: args.messageId, + }); + if (!result.ok) { + const message = "dashboard-agent watch investigation couldn't close its card"; + const details = { chatId, investigationId, error: result.error }; + // A chat deleted mid-investigation is a race, not a fault: nothing settled, and there + // is no transcript left to close the card in. + if (result.error === "chat_missing") { + logger.warn(message, details); + return; + } + logger.error(message, details); + throw new Error(`${message}: ${result.error}`); + } + + chat.history.set([...uiMessages, result.card as UIMessage]); +} + +// The investigating turn's framing only. The protocol itself lives in the managed +// system prompt's Investigations section, which is the cached block. +function investigatePrompt(args: { + action: WatchInvestigateAction; + investigationId: string; + tenancy: { projectRef?: string; environmentId?: string }; +}): string { + const { action, investigationId } = args; + const subjectLink = wakeSubjectLink(action, args.tenancy); + return [ + `Conduct the investigation the user pre-approved when they created this watch, right now, and finish it in this message. Nobody asked a question and nobody is waiting on a reply: your wake message has already told them the watch resolved and that you started looking into why, so this is the follow-up you promised — write it as its own message, and never re-narrate the wake.`, + `The investigation is ALREADY OPEN as \`${investigationId}\`. Pass that exact investigationId to every render_view you make, so you revise that one card instead of opening a second. Your last tool call must be a render_view of it carrying a terminal outcome (concluded or inconclusive) — an investigation left at in_progress is an unfinished answer.`, + `Subject: ${wakeSubject(action)} (${action.spec.kind}${ + action.identity ? `, ${action.identity}` : "" + }).`, + action.note ? `Why the user asked to be told: ${action.note}` : undefined, + action.observed + ? `What the resolving check observed:\n${JSON.stringify(action.observed, null, 2)}` + : undefined, + action.facts && Object.keys(action.facts).length > 0 + ? `Facts from that check — start from these rather than re-reading them:\n${JSON.stringify( + action.facts, + null, + 2 + )}` + : undefined, + subjectLink + ? `When you point at the watched object, link it: ${subjectLink} — use this exact markdown link, not a bare name.` + : undefined, + ] + .filter(Boolean) + .join("\n\n"); +} + +/** + * Conduct the consented investigation, exactly once, as the agent's own message. + * + * A real turn in everything but name: same tools, same protocol, same step budget + * `run()` gives. It is NOT a turn in the SDK's sense, so no `onTurnComplete` fires + * — the settle guard runs here by hand and the message is appended id-deduped. + * + * The wake was delivered long before this, so everything here is best-effort and + * nothing it does can retry or invalidate the wake. + */ +async function conductWatchInvestigation(args: { + action: WatchInvestigateAction; + chatId: string; + clientData: z.infer | undefined; + uiMessages: UIMessage[]; + messages: ModelMessage[]; +}): Promise { + const { action, chatId, clientData, uiMessages } = args; + const messageId = `investigate:${action.id}`; + const closingMessageId = `${messageId}:settled`; + const projectRef = clientData?.projectRef; + const environmentRef = clientData?.environmentId; + + const seedState = (): InvestigationState => ({ + outcome: "in_progress", + severity: "warn", + confidence: "low", + title: `Investigating ${wakeSubject(action)}`, + headline: `The watch on ${wakeSubject(action)} resolved to something that needs attention.`, + hypotheses: [], + evidence: [], + }); + + const closeCard = async (investigationId: string, messages: UIMessage[]) => { + if (!projectRef || !environmentRef) return; + await closeCardInTranscript({ + store: getStore(), + chatId, + investigationId, + projectRef, + environmentRef, + messageId: closingMessageId, + uiMessages: messages, + fallback: seedState, + }); + }; + + // Dedup on the action id, against the durable transcript — a redelivered kick + // must not investigate (or answer) twice. + const alreadyAnswered = uiMessages.find((message) => message.id === messageId); + if (alreadyAnswered) { + logger.info("dashboard-agent watch investigation already ran; repairing the display copy", { + chatId, + watchId: action.watchId, + actionId: action.id, + }); + // Same window as the wake's: the findings streamed durably before the append, so a + // retry can owe only the display copy. Id-deduped, so a repeat writes nothing. + const userId = clientData?.userId; + if (userId) { + await getStore().appendMessage({ + chatId, + userId, + organizationId: clientData?.organizationId, + message: alreadyAnswered, + }); + } + // This watch's card only: any other card still running belongs to the user or to + // another watch, and closing it would answer a question nobody asked here. + const cardId = action.investigationId ?? watchInvestigationId(action.watchId); + const open = latestCards(uiMessages).get(cardId); + if (open && (open.state === null || open.state.outcome === "in_progress")) { + await closeCard(cardId, uiMessages); + } + return; + } + + if (!projectRef || !environmentRef) { + // A card can't be scoped without the tenancy, and saying nothing beats a + // findings message with nowhere to render. + logger.error("dashboard-agent watch investigation can't be scoped; skipping", { + chatId, + watchId: action.watchId, + }); + return; + } + + const investigationId = await resolveInvestigationId({ + action, + chatId, + projectRef, + environmentRef, + }); + if (!investigationId) { + logger.error("dashboard-agent watch investigation has no card to revise; skipping", { + chatId, + watchId: action.watchId, + }); + return; + } + + const store = getStore(); + let answered: UIMessage | undefined; + const resolved = await getSystemPrompt(modeFor(clientData)); + const tools = buildTurnTools(chatId, clientData); + let step = 0; + const result = streamText({ + model: + locals.get(dashboardAgentModelKey) ?? + registry.languageModel( + (resolved.model ?? "anthropic:claude-sonnet-4-6") as `anthropic:${string}` + ), + system: resolved.text, + tools, + // Ten steps of accumulating tool output is exactly what the rolling breakpoint + // is for; without it every step re-sends the lot uncached. + prepareStep: stepCachePrepareStep(undefined) as never, + onStepFinish: (finished) => + recordPromptCacheUsage({ + source: "watch-investigation", + usage: finished.usage, + system: resolved.text, + tools, + step: step++, + providerMetadata: finished.providerMetadata, + }), + messages: [ + ...withCacheBreakpointOnLast(sanitizeReplayedToolInputs(args.messages)), + { + role: "user" as const, + content: investigatePrompt({ + action, + investigationId, + tenancy: { projectRef, environmentId: environmentRef }, + }), + }, + ], + // The same budget a turn gets: four tool phases plus the answer. + stopWhen: stepCountIs(10), + ...resolved.toAISDKTelemetry(), + }); + + try { + // Tee'd because the findings message has to be persisted whole: the panel + // renders the card from the `render_view` tool part, so a text-only copy would + // lose it on the next page load. One branch streams to the panel, the other + // reduces the same chunks back into a UIMessage. + const [toPanel, toTranscript] = result + .toUIMessageStream({ generateMessageId: () => messageId }) + .tee(); + let response: UIMessage | undefined; + const reduced = (async () => { + for await (const snapshot of readUIMessageStream({ stream: toTranscript })) { + response = snapshot; + } + })(); + await chat.pipe(toPanel); + await reduced; + + if (response) { + const message: UIMessage = { ...response, id: messageId, role: "assistant" }; + chat.history.set([...uiMessages, message]); + answered = message; + const userId = clientData?.userId; + if (userId) { + await store.appendMessage({ + chatId, + userId, + organizationId: clientData?.organizationId, + message, + }); + } else { + logger.error("dashboard-agent watch investigation has no userId; skipping the append", { + chatId, + }); + } + } + } finally { + // No `onTurnComplete` on an action, so the guard runs here: a card left at + // in_progress is a spinner nothing else will ever stop. `closeCard` is the settle, + // so nothing settles the row separately first. + await closeCard(investigationId, answered ? [...uiMessages, answered] : uiMessages); + // Only once the close committed: a retry needs the tracked entry to still be there. + clearOpenInvestigations(chatId); + } +} + +/** + * The agent's `onAction` lane, whole: narrate the outcome, then conduct the + * investigation it opened when the user consented. + */ +export async function handleWatchAction(args: { + action: unknown; + chatId: string; + clientData: z.infer | undefined; + uiMessages: UIMessage[]; + messages: ModelMessage[]; +}): Promise { + const { chatId, clientData, uiMessages, messages } = args; + const typed = args.action as DashboardAgentAction; + if (typed.type === "watch.investigate") { + await conductWatchInvestigation({ action: typed, chatId, clientData, uiMessages, messages }); + return; + } + await narrateWatchWake({ action: typed, chatId, clientData, uiMessages, messages }); +} diff --git a/internal-packages/dashboard-agent/src/watch-batch.ts b/internal-packages/dashboard-agent/src/watch-batch.ts new file mode 100644 index 00000000000..bcb04fb7588 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-batch.ts @@ -0,0 +1,187 @@ +import type { WatchCheckResult, WatchObservedOutcome } from "@internal/dashboard-agent-contracts"; +import { logger } from "@trigger.dev/sdk"; +import type { WatchDeliveryDeps, WatchTickOutcome, WatchTickStore } from "./watch-delivery"; +import { REVOKED_CODES, runWatchLifecycle, type CheckOutcome } from "./watch-lifecycle"; + +// The batch tick: one run per (environment, cadence), for every watch in it. + +export type WatchBatchTickPayload = { + environmentId: string; + cadenceMinutes: number; + apiOrigin: string; + /** + * Names the (environment, cadence) and carries no authority: the batch check + * re-authorizes every watch's initiating user against that watch's own scope. + */ + token: string; + /** The chain incarnation this run belongs to. A mismatch means it owns nothing. */ + epoch: number; + /** The tick generation this run owns inside `epoch`, starting at 1. */ + tick: number; +}; + +export type WatchBatchCheckEntry = { + watchId: string; + token: string; + /** The generation to claim for this watch (its `tickCount + 1` when listed). */ + tick: number; + /** The row is already resolved and its wake is owed, so the group recovers it. */ + deliverOnly?: boolean; + result?: WatchCheckResult; + facts?: Record; + observed?: WatchObservedOutcome; + /** `access_revoked` / `cancelled` / `not_found`, instead of a result. */ + code?: string; + error?: string; +}; + +export type WatchBatchCheckResponse = { + /** This run's epoch/generation is not the chain's, so it owns nothing and exits. */ + stale?: boolean; + watches?: WatchBatchCheckEntry[]; + /** Whether the group still has active watches, i.e. whether to tick again. */ + continues?: boolean; +}; + +export type WatchBatchTickDeps = { + store: WatchTickStore; + checkBatch: (payload: WatchBatchTickPayload) => Promise; + deliver: WatchDeliveryDeps["deliver"]; + notifyFired: (target: { watchId: string; token: string }) => Promise; + notifyInvestigate: (target: { watchId: string; token: string }) => Promise; + reschedule: ( + payload: WatchBatchTickPayload, + options: { delay: string; idempotencyKey: string } + ) => Promise; + now?: () => Date; + /** How many watches are resolved at once. Defaults to {@link BATCH_CONCURRENCY}. */ + concurrency?: number; +}; + +export type WatchBatchTickResult = { + outcome: "ticked" | "stale"; + results: Array<{ watchId: string; outcome?: WatchTickOutcome; error?: string }>; + rescheduled: boolean; +}; + +const BATCH_CONCURRENCY = 8; + +/** `mapper` over `items`, at most `limit` in flight. Order is preserved. */ +async function mapWithConcurrency( + items: T[], + limit: number, + mapper: (item: T) => Promise +): Promise { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await mapper(items[index]!); + } + }); + await Promise.all(workers); + return results; +} + +/** + * One tick of a whole (environment, cadence) group. Nothing here guards against a + * double fire: the terminal transition and the delivery claim do. + */ +export async function runWatchBatchTick( + payload: WatchBatchTickPayload, + deps: WatchBatchTickDeps +): Promise { + const response = await deps.checkBatch(payload); + + if (response.stale) { + logger.info("dashboard-agent watch batch is stale; exiting", { + environmentId: payload.environmentId, + cadenceMinutes: payload.cadenceMinutes, + epoch: payload.epoch, + tick: payload.tick, + }); + return { outcome: "stale", results: [], rescheduled: false }; + } + + const entries = response.watches ?? []; + const results = await mapWithConcurrency( + entries, + deps.concurrency ?? BATCH_CONCURRENCY, + (entry) => resolveBatchEntry(payload, deps, entry) + ); + + // Before the rethrow below, so the chain survives a watch that keeps failing. + let rescheduled = false; + if (response.continues) { + const next = payload.tick + 1; + await deps.reschedule( + { ...payload, tick: next }, + { + delay: `${payload.cadenceMinutes}m`, + // Keyed on the epoch too, so a re-armed chain can't collide with its + // predecessor's keys. + idempotencyKey: `watch-batch:${payload.environmentId}:${payload.cadenceMinutes}:${payload.epoch}:tick:${next}`, + } + ); + rescheduled = true; + } + + const failed = results.filter((result) => result.error !== undefined); + if (failed.length > 0) { + // Safe to retry the whole batch: the chain's claim is resumable and the check hands + // back the owed wakes again. + throw new Error( + `${failed.length} of ${entries.length} watches failed their tick (${failed + .map((result) => result.watchId) + .join(", ")})` + ); + } + + return { outcome: "ticked", results, rescheduled }; +} + +/** One watch of a batch: each resolves in its own try, so a failure isolates. */ +async function resolveBatchEntry( + payload: WatchBatchTickPayload, + deps: WatchBatchTickDeps, + entry: WatchBatchCheckEntry +): Promise<{ watchId: string; outcome?: WatchTickOutcome; error?: string }> { + const target = { apiOrigin: payload.apiOrigin, watchId: entry.watchId, token: entry.token }; + try { + const result = await runWatchLifecycle( + { watchId: entry.watchId, tick: entry.tick, deliverOnly: entry.deliverOnly }, + { + store: deps.store, + deliver: deps.deliver, + notifyFired: () => deps.notifyFired(target), + notifyInvestigate: () => deps.notifyInvestigate(target), + now: deps.now, + check: async () => batchCheckOutcome(entry), + // The group's single reschedule covers every watch in it. + onPending: async () => {}, + } + ); + return { watchId: entry.watchId, outcome: result.outcome }; + } catch (error) { + logger.error("dashboard-agent watch batch: a watch failed its tick", { + watchId: entry.watchId, + environmentId: payload.environmentId, + error: (error as Error).message, + }); + return { watchId: entry.watchId, error: (error as Error).message }; + } +} + +function batchCheckOutcome(entry: WatchBatchCheckEntry): CheckOutcome { + if (entry.code && REVOKED_CODES.has(entry.code)) return { kind: "revoked", code: entry.code }; + if (!entry.result || entry.result === "unavailable") { + return { kind: "unavailable", detail: entry.error, observed: entry.observed }; + } + return { + kind: "result", + result: entry.result, + facts: entry.facts, + observed: entry.observed, + }; +} diff --git a/internal-packages/dashboard-agent/src/watch-delivery.ts b/internal-packages/dashboard-agent/src/watch-delivery.ts new file mode 100644 index 00000000000..1343abf83b3 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-delivery.ts @@ -0,0 +1,237 @@ +import { + isTerminalWatchStatus, + isWatchDeliveryOwed, + WATCH_DELIVERY_CLAIM_STALE_MS, + type PersistedWatchSpec, + type Watch, + type WatchDeliveryClaim, +} from "@internal/dashboard-agent-db"; +import { + watchResolutionToWireStatus, + type WatchObservedOutcome, + type WatchResolution, +} from "@internal/dashboard-agent-contracts"; +import { logger } from "@trigger.dev/sdk"; +import type { WatchWakeAction } from "./dashboard-agent"; + +/** + * The delivery half of a watch: the terminal transition, the claim that makes exactly + * one deliverer append the wake, and the facts each outcome carries. + */ + +export type WatchTickStore = { + getWatch(params: { id: string }): Promise; + claimWatchTick(params: { id: string; generation: number }): Promise; + transitionWatchCondition(params: { + id: string; + resolution: WatchResolution; + observedOutcome?: WatchObservedOutcome | null; + lastResult?: Record | null; + }): Promise; + /** + * Take the wake. Only the returned row may be appended, and only while its + * `claimId` is still the row's: that token fences the two writes below. + */ + claimWatchDelivery(params: { id: string; staleBefore: Date }): Promise; + /** Hand the wake back after a failed append, so the retry can re-claim it. */ + releaseWatchDelivery(params: { id: string; claimId: string }): Promise; + markWatchDelivered(params: { id: string; claimId: string }): Promise; + recordWatchCheck(params: { + id: string; + lastResult?: Record | null; + }): Promise<{ tickCount: number; lastCheckedAt: Date | null } | null>; +}; + +export type WatchWakeAck = { appended: boolean }; + +export type WatchDeliveryDeps = { + store: Pick< + WatchTickStore, + | "getWatch" + | "transitionWatchCondition" + | "claimWatchDelivery" + | "releaseWatchDelivery" + | "markWatchDelivered" + >; + /** Must throw, or report `{ appended: false }`, if the append fails. */ + deliver: (args: { + chatId: string; + action: WatchWakeAction; + watch: Watch; + }) => Promise; + /** Send the user's alerts. Best-effort: a failure here must never fail the tick. */ + notifyFired: (watchId: string) => Promise; + /** Send the agent off to investigate. Best-effort, like `notifyFired`. */ + notifyInvestigate: (watchId: string) => Promise; + now?: () => Date; +}; + +export type WatchTickOutcome = + | "missing" + | "already_terminal" + | "delivered_only" + // The wake is owed but another invocation holds the delivery claim, so this one + // must not append: exactly one wake reaches the chat. + | "already_delivering" + // A late duplicate: the row has moved past this invocation's generation. + | "stale" + // A `deliverOnly` invocation on a row that isn't terminal yet. It must not decide + // an outcome. + | "nothing_to_deliver" + | "revoked" + | "unavailable" + | "pending" + // A batch chain now polls this watch's group, so the per-watch chain stops here + // instead of rescheduling itself. + | "handed_off" + | "fired" + | "expired"; + +export type WatchTickResult = { outcome: WatchTickOutcome; tickCount?: number }; + +export function firedFacts(facts: Record | undefined): Record { + return { verified: true, ...(facts ?? {}) }; +} + +// An unverified expiry must not claim the thing didn't happen, so it carries the row's +// last observation instead. +export function expiredFacts( + watch: Watch, + args: { + verified: boolean; + reason: string; + facts?: Record; + } +): Record { + return { + verified: args.verified, + reason: args.reason, + expiredAt: watch.expiresAt.toISOString(), + checks: watch.tickCount, + ...(args.verified + ? (args.facts ?? {}) + : { + lastObservedAt: watch.lastCheckedAt?.toISOString(), + lastObservation: watch.lastResult, + }), + }; +} + +// `type` and `id` keep the two-value `fired`/`expired` encoding, which persisted wakes +// and dedup keys depend on. The meaning travels in `resolution` and `observed`. +function wakeAction(watch: Watch, facts: Record): WatchWakeAction { + const spec = watch.spec as PersistedWatchSpec; + return { + type: watch.status === "fired" ? "watch.fired" : "watch.expired", + // Stable per (watch, outcome): a redelivered wake never narrates twice. + id: `watch:${watch.id}:${watch.status}`, + watchId: watch.id, + identity: watch.identity, + spec, + facts, + resolution: watch.resolution ?? undefined, + observed: watch.observedOutcome ?? undefined, + note: spec.note, + investigateOnAttention: watch.investigateOnAttention, + }; +} + +/** + * The claim is the gate: `pending → delivering` in one statement, so exactly one of + * two racing deliverers appends. The action id only dedups read-then-write. + */ +export async function deliverWake(deps: WatchDeliveryDeps, watch: Watch): Promise { + const now = deps.now?.() ?? new Date(); + const claim = await deps.store.claimWatchDelivery({ + id: watch.id, + staleBefore: new Date(now.getTime() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + + if (!claim) { + logger.info("dashboard-agent watch wake is already being delivered; skipping", { + watchId: watch.id, + }); + return false; + } + + const { watch: claimed, claimId } = claim; + const facts = (claimed.lastResult ?? {}) as Record; + let ack: void | WatchWakeAck; + try { + ack = await deps.deliver({ + chatId: claimed.chatId, + action: wakeAction(claimed, facts), + watch: claimed, + }); + } catch (error) { + await deps.store.releaseWatchDelivery({ id: claimed.id, claimId }); + throw error; + } + + if (ack && ack.appended === false) { + await deps.store.releaseWatchDelivery({ id: claimed.id, claimId }); + throw new Error(`the wake for watch ${claimed.id} wasn't appended`); + } + + await deps.store.markWatchDelivered({ id: claimed.id, claimId }); + + // Outside the wake's failure path: an alert must never fail the delivery. + if (claimed.status === "fired") { + try { + await deps.notifyFired(claimed.id); + } catch (error) { + logger.warn("dashboard-agent watch: the fired notification failed", { + watchId: claimed.id, + error: (error as Error).message, + }); + } + } + + // Fires on any resolved outcome: whether it warrants attention is the webapp's call. + if (claimed.investigateOnAttention) { + try { + await deps.notifyInvestigate(claimed.id); + } catch (error) { + logger.warn("dashboard-agent watch: the investigate kick failed", { + watchId: claimed.id, + error: (error as Error).message, + }); + } + } + + return true; +} + +// Only an `active` row transitions, so a check racing the sweeper yields one winner. +// The loser re-reads the row and delivers what the winner decided, if still owed. +export async function resolveAndDeliver( + deps: WatchDeliveryDeps, + watch: Watch, + resolution: WatchResolution, + facts: Record, + observed?: WatchObservedOutcome +): Promise { + const transitioned = await deps.store.transitionWatchCondition({ + id: watch.id, + resolution, + observedOutcome: observed ?? null, + lastResult: facts, + }); + + if (!transitioned) { + // Someone else resolved it. Deliver only if that outcome is still owed. + const current = await deps.store.getWatch({ id: watch.id }); + if ( + current && + isTerminalWatchStatus(current.status) && + isWatchDeliveryOwed(current.deliveryStatus) + ) { + const delivered = await deliverWake(deps, current); + return { outcome: delivered ? "delivered_only" : "already_delivering" }; + } + return { outcome: "already_terminal" }; + } + + await deliverWake(deps, transitioned); + return { outcome: watchResolutionToWireStatus(resolution) }; +} diff --git a/internal-packages/dashboard-agent/src/watch-lifecycle.ts b/internal-packages/dashboard-agent/src/watch-lifecycle.ts new file mode 100644 index 00000000000..2e55dc38018 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-lifecycle.ts @@ -0,0 +1,209 @@ +import { + isTerminalWatchStatus, + isWatchDeliveryOwed, + type Watch, +} from "@internal/dashboard-agent-db"; +import { + watchResolutionForCheck, + type WatchCheckResult, + type WatchObservedOutcome, +} from "@internal/dashboard-agent-contracts"; +import { logger } from "@trigger.dev/sdk"; +import { + deliverWake, + expiredFacts, + firedFacts, + resolveAndDeliver, + type WatchDeliveryDeps, + type WatchTickResult, + type WatchTickStore, +} from "./watch-delivery"; + +/** + * The condition half of a watch: one generation's claim, check and verdict. The webapp + * evaluates conditions; this decides what the verdict means for the row. + */ + +export type WatchLifecycleDeps = WatchDeliveryDeps & { + store: WatchTickStore; + /** `final` is decided by the claimed row, never by the implementation. */ + check: (args: { watch: Watch; final: boolean }) => Promise; + /** Keep this watch's chain alive. A no-op in the batch, which reschedules once. */ + onPending: (watch: Watch, tick: number) => Promise; +}; + +// `handOff` is orthogonal to the verdict: the group's batch chain now polls this +// watch, so the caller stops keeping its own chain alive. +export type CheckOutcome = + | { + kind: "result"; + result: WatchCheckResult; + facts?: Record; + observed?: WatchObservedOutcome; + handOff?: boolean; + } + // The row is already over and the webapp knows it, so the tick exits without + // transitioning or delivering. + | { kind: "revoked"; code?: string } + // The check itself couldn't run. Never true, never false. + | { + kind: "unavailable"; + detail?: string; + observed?: WatchObservedOutcome; + handOff?: boolean; + }; + +// The row is no longer active, so nothing is left to transition or deliver. Any other +// non-2xx is a failed check and the tick keeps watching to the row's own deadline. +export const REVOKED_CODES = new Set(["access_revoked", "cancelled", "not_found"]); + +/** + * The last result the check actually produced. A failure record is unwrapped, so a run of + * failures replaces one another instead of nesting — the row's `lastResult` reaches the + * wake facts, the alert and the webhook body. + */ +export function lastObservedResult(lastResult: unknown): Record | undefined { + let current = lastResult; + while (isCheckFailure(current)) current = current.previous; + return current !== null && typeof current === "object" && !Array.isArray(current) + ? (current as Record) + : undefined; +} + +function isCheckFailure(value: unknown): value is { previous?: unknown } { + return ( + typeof value === "object" && + value !== null && + (value as { checkFailed?: unknown }).checkFailed === true + ); +} + +// One watch's tick, shared by the per-watch task and the batch. The order of the +// branches below is the algorithm. +export async function runWatchLifecycle( + args: { watchId: string; tick: number; deliverOnly?: boolean }, + deps: WatchLifecycleDeps +): Promise { + const now = deps.now?.() ?? new Date(); + const watch = await deps.store.getWatch({ id: args.watchId }); + + if (!watch) return { outcome: "missing" }; + + // Terminal already, so only the delivery may be owed. Runs before the claim so a + // retry after a crash between the transition and the append still delivers. + if (isTerminalWatchStatus(watch.status)) { + if (isWatchDeliveryOwed(watch.deliveryStatus)) { + const delivered = await deliverWake(deps, watch); + return { outcome: delivered ? "delivered_only" : "already_delivering" }; + } + return { outcome: "already_terminal" }; + } + + // Delivery-only invocations never decide anything, so no generation is claimed. + if (args.deliverOnly) { + logger.info("dashboard-agent watch delivery has nothing to deliver", { + watchId: watch.id, + status: watch.status, + }); + return { outcome: "nothing_to_deliver" }; + } + + // The claim is resumable (previous generation or this one) and refuses only once the + // row has moved past it. A resumed tick re-runs the generation; every write is guarded. + const claimed = await deps.store.claimWatchTick({ + id: watch.id, + generation: args.tick, + }); + + if (!claimed) { + logger.info("dashboard-agent watch tick is stale; exiting", { + watchId: watch.id, + tick: args.tick, + tickCount: watch.tickCount, + }); + return { outcome: "stale" }; + } + + // The claimed row is the authority on expiry from here on, not the clock the check + // ran on. + const final = claimed.expiresAt.getTime() <= now.getTime(); + const check = await deps.check({ watch: claimed, final }); + + if (check.kind === "revoked") { + logger.info("dashboard-agent watch check refused; exiting", { + watchId: claimed.id, + code: check.code, + }); + return { outcome: "revoked" }; + } + + if (check.kind === "unavailable") { + if (!final) { + // The generation is spent and the result isn't trusted, so keep watching. + await deps.store.recordWatchCheck({ + id: claimed.id, + lastResult: { + checkFailed: true, + detail: check.detail, + previous: lastObservedResult(claimed.lastResult), + }, + }); + if (check.handOff) return { outcome: "handed_off", tickCount: args.tick }; + await deps.onPending(claimed, args.tick); + return { outcome: "unavailable", tickCount: args.tick }; + } + // The deadline passed with the final check unable to run, so the window completed + // on an unverified observation. + return resolveAndDeliver( + deps, + claimed, + "window_completed", + expiredFacts(claimed, { verified: false, reason: "unverified_at_expiry" }), + check.observed + ); + } + + // At the boundary only `pending` and `unavailable` become `window_completed`; + // `satisfied` and `terminal_unsatisfied` resolve as they would before it. + const resolution = watchResolutionForCheck(check.result, final); + + if (resolution === "condition_met") { + return resolveAndDeliver( + deps, + claimed, + "condition_met", + firedFacts(check.facts), + check.observed + ); + } + + if (resolution === "condition_impossible") { + return resolveAndDeliver( + deps, + claimed, + "condition_impossible", + expiredFacts(claimed, { + verified: true, + reason: "terminal_unsatisfied", + facts: check.facts, + }), + check.observed + ); + } + + if (resolution === "window_completed") { + return resolveAndDeliver( + deps, + claimed, + "window_completed", + expiredFacts(claimed, { verified: true, reason: "not_met_by_expiry", facts: check.facts }), + check.observed + ); + } + + await deps.store.recordWatchCheck({ id: claimed.id, lastResult: check.facts ?? {} }); + // A hand-off means the group's batch chain keeps looking instead of this one. + if (check.handOff) return { outcome: "handed_off", tickCount: args.tick }; + await deps.onPending(claimed, args.tick); + return { outcome: "pending", tickCount: args.tick }; +} diff --git a/internal-packages/dashboard-agent/src/watch-narration.test.ts b/internal-packages/dashboard-agent/src/watch-narration.test.ts new file mode 100644 index 00000000000..f53ea15c34a --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-narration.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { + deterministicWakeNarration, + planWatchNarration, + type NarratableWake, +} from "./watch-narration"; + +const DRAINED: NarratableWake = { + kind: "backlog_drain", + identity: "backlog_drain:task/send-receipt", + resolution: "condition_met", + observed: { kind: "backlog_drain", verified: true, depth: 0 }, + note: "tell me when the backlog drains", + startsInvestigation: false, +}; + +const STALLED: NarratableWake = { + kind: "queue_stalled", + identity: "queue_stalled:task/send-receipt", + resolution: "condition_met", + observed: { + kind: "queue_stalled", + verified: true, + depth: 412, + ticks: 4, + notDecreasingStreak: 4, + }, + startsInvestigation: false, +}; + +describe("which model narrates a wake", () => { + it("needs no model for a condition that simply became true", () => { + const plan = planWatchNarration(DRAINED); + expect(plan.model).toBe("none"); + if (plan.model !== "none") throw new Error("unreachable"); + // The dashboard's own sentence, the user's reason, then what to do. + expect(plan.text).toContain("task/send-receipt queue drained"); + expect(plan.text).toContain("You asked to be told when: tell me when the backlog drains"); + expect(plan.text).toContain("Nothing to do"); + }); + + it("needs no model when the answer is that the watched thing is gone", () => { + const plan = planWatchNarration({ ...DRAINED, resolution: "condition_impossible" }); + expect(plan.model).toBe("none"); + if (plan.model !== "none") throw new Error("unreachable"); + expect(plan.text).toContain("no longer exists"); + }); + + it("uses Haiku when the fact has to be turned into what to do", () => { + expect(planWatchNarration(STALLED).model).toBe("haiku"); + // A window that ran out with the queue still backed up is the same judgement. + expect(planWatchNarration({ ...DRAINED, resolution: "window_completed" }).model).toBe("haiku"); + }); + + it("keeps Sonnet for the consented investigation, whatever the outcome", () => { + expect(planWatchNarration({ ...STALLED, startsInvestigation: true }).model).toBe("sonnet"); + expect(planWatchNarration({ ...DRAINED, startsInvestigation: true }).model).toBe("sonnet"); + }); + + it("links the watched object when the wake carries the tenancy for one", () => { + const { text } = deterministicWakeNarration({ + ...DRAINED, + subjectLink: "[task/send-receipt](trigger://queue/proj_abc/env_abc/task%2Fsend-receipt)", + }); + expect(text).toContain("(trigger://queue/"); + expect(text).toContain("I've stopped watching [task/send-receipt]"); + }); + + it("never says fired or expired", () => { + for (const resolution of [ + "condition_met", + "window_completed", + "condition_impossible", + ] as const) { + const { text } = deterministicWakeNarration({ ...DRAINED, resolution }); + expect(text).not.toMatch(/fired|expired/); + } + }); +}); diff --git a/internal-packages/dashboard-agent/src/watch-narration.ts b/internal-packages/dashboard-agent/src/watch-narration.ts new file mode 100644 index 00000000000..03594565ad8 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-narration.ts @@ -0,0 +1,99 @@ +import { + presentResolvedWatch, + watchNoteLine, + type WatchObservedOutcome, + type WatchPresentation, + type WatchResolution, +} from "@internal/dashboard-agent-contracts"; + +/** + * How a resolved watch is said. + * + * "Queue drained: 0 pending after 42 minutes" is a fact the check already + * established, and the sentence for it is already written — in the contracts' + * watch vocabulary, which every other surface (card, banner, toast, email) reads. + * So a good or merely factual outcome is narrated here, with no model call at all. + * + * A model earns its call only where the fact has to be turned into what to do about + * it. That is Haiku with the wake alone, never Sonnet with the whole conversation: + * the wake carries every number the sentence may use. + */ + +/** Which lane a wake takes. */ +export type WatchNarrationPlan = + /** Said from the contracts' wording. No model call. */ + | { model: "none"; text: string; presentation: WatchPresentation } + /** One or two sentences from Haiku, given the wake and nothing else. */ + | { model: "haiku"; presentation: WatchPresentation } + /** The consented-investigation wake: the wake and the findings must read as one. */ + | { model: "sonnet" }; + +export type NarratableWake = { + kind: string; + identity: string; + resolution: WatchResolution; + observed?: WatchObservedOutcome; + note?: string; + /** The watched thing as a `trigger://` markdown link, when the tenancy allows one. */ + subjectLink?: string; + /** The user pre-approved an investigation and it has already been started. */ + startsInvestigation: boolean; +}; + +/** + * The stock next step per category. Deliberately generic: a specific suggestion is + * exactly the judgement a model is for, and this lane is the one where there is + * nothing to judge. + */ +function nextStep(presentation: WatchPresentation, subjectLink?: string): string { + const subject = subjectLink ?? "it"; + switch (presentation.category) { + case "positive": + return `Nothing to do — I've stopped watching ${subject}.`; + // A neutral outcome is an answer without a problem in it: the watched thing is + // gone, cancelled, or was never readable. + case "neutral": + return `I've stopped watching ${subject}. Ask me if you want another watch set up.`; + case "attention": + return `Ask me to look into ${subject} if you want the why.`; + } +} + +/** The whole wake message, when no model is needed. */ +export function deterministicWakeNarration(wake: NarratableWake): { + text: string; + presentation: WatchPresentation; +} { + const presentation = presentResolvedWatch({ + kind: wake.kind, + identity: wake.identity, + resolution: wake.resolution, + observed: wake.observed ?? null, + }); + const lines = [ + // The headline is already a complete fact, and every surface states it this way. + wake.subjectLink ? `${presentation.headline} (${wake.subjectLink})` : presentation.headline, + wake.note ? watchNoteLine(wake.note) : null, + nextStep(presentation, wake.subjectLink), + ].filter((line): line is string => Boolean(line)); + + return { text: lines.join("\n\n"), presentation }; +} + +/** + * Which lane this wake takes. + * + * - A consented investigation goes to Sonnet: the wake has to promise the findings + * that its own next message delivers, and the two must read as one voice. + * - An outcome that needs attention goes to Haiku: the fact is known, but saying + * what it means for the user is a judgement. + * - Everything else — the condition became true, or the answer is simply "that's + * gone" — is said from the contracts' wording. + */ +export function planWatchNarration(wake: NarratableWake): WatchNarrationPlan { + if (wake.startsInvestigation) return { model: "sonnet" }; + + const { text, presentation } = deterministicWakeNarration(wake); + if (presentation.category === "attention") return { model: "haiku", presentation }; + return { model: "none", text, presentation }; +} diff --git a/internal-packages/dashboard-agent/src/watch-task-adapters.ts b/internal-packages/dashboard-agent/src/watch-task-adapters.ts new file mode 100644 index 00000000000..45222b6fc45 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-task-adapters.ts @@ -0,0 +1,167 @@ +import { + claimWatchDelivery, + claimWatchTick, + createDashboardAgentDb, + getWatch, + markWatchDelivered, + recordWatchCheck, + releaseWatchDelivery, + transitionWatchCondition, + type DashboardAgentDbClient, + type Watch, +} from "@internal/dashboard-agent-db"; +import { sessions } from "@trigger.dev/sdk"; +import type { WatchWakeAction } from "./dashboard-agent"; +import type { WatchTickStore } from "./watch-delivery"; +import type { WatchBatchCheckResponse, WatchBatchTickPayload } from "./watch-batch"; + +/** What the two watch tasks plug into the lifecycle: the db, the wake append, the callbacks. */ + +// One connection pool per worker process. +let dbClient: DashboardAgentDbClient | undefined; +export function getWatchDb(): DashboardAgentDbClient { + if (!dbClient) { + const connectionString = process.env.DASHBOARD_AGENT_DATABASE_URL ?? process.env.DATABASE_URL; + if (!connectionString) { + throw new Error( + "DASHBOARD_AGENT_DATABASE_URL (or DATABASE_URL) must be set for the watch task" + ); + } + dbClient = createDashboardAgentDb(connectionString, { max: 2 }); + } + return dbClient; +} + +export function watchStore(db: DashboardAgentDbClient["db"]): WatchTickStore { + return { + getWatch: (params) => getWatch(db, params), + claimWatchTick: (params) => claimWatchTick(db, params), + transitionWatchCondition: (params) => transitionWatchCondition(db, params), + claimWatchDelivery: (params) => claimWatchDelivery(db, params), + releaseWatchDelivery: (params) => releaseWatchDelivery(db, params), + markWatchDelivered: (params) => markWatchDelivered(db, params), + recordWatchCheck: (params) => recordWatchCheck(db, params), + }; +} + +export type WatchCallbackTarget = { apiOrigin: string; watchId: string; token: string }; + +// No retry loop: the endpoint dedupes on the watch, and losing an alert beats losing +// the wake. +export async function postFired(target: WatchCallbackTarget): Promise { + await postWatchCallback(target, "fired"); +} + +// Says only that the wake is delivered; the endpoint decides whether the outcome is +// one the consent covers. +export async function postInvestigate(target: WatchCallbackTarget): Promise { + await postWatchCallback(target, "investigate"); +} + +async function postWatchCallback( + target: WatchCallbackTarget, + path: "fired" | "investigate" +): Promise { + const origin = target.apiOrigin.replace(/\/$/, ""); + const response = await fetch( + `${origin}/api/v1/dashboard-agent/watches/${encodeURIComponent(target.watchId)}/${path}`, + { + method: "POST", + headers: { + Authorization: `Bearer ${target.token}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: "{}", + } + ); + + if (!response.ok) { + throw new Error(`the ${path} callback returned ${response.status}`); + } +} + +export async function postBatchCheck( + payload: WatchBatchTickPayload +): Promise { + const origin = payload.apiOrigin.replace(/\/$/, ""); + const response = await fetch(`${origin}/api/v1/dashboard-agent/watches/batch-check`, { + method: "POST", + headers: { + Authorization: `Bearer ${payload.token}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + environmentId: payload.environmentId, + cadenceMinutes: payload.cadenceMinutes, + epoch: payload.epoch, + tick: payload.tick, + }), + }); + + const body = (await response.json().catch(() => undefined)) as + | (WatchBatchCheckResponse & { error?: string }) + | undefined; + + if (!response.ok) { + // A throw, not an empty batch: rescheduling as if nothing was due would skip the + // whole group. + throw new Error(body?.error ?? `the batch check returned ${response.status}`); + } + + return body ?? {}; +} + +/** + * Appends a `trigger: "action"` record, which fires `onAction` and nothing else. + * `metadata` is the agent's `clientData` and carries no delegated token. + */ +export async function appendWakeToSession(args: { + chatId: string; + action: WatchWakeAction; + watch: Watch; +}): Promise { + const metadata = { + userId: args.watch.userId, + organizationId: args.watch.organizationId, + projectId: args.watch.projectId, + environmentId: args.watch.environmentId, + // The external ref a consented investigation is scoped by. + ...(args.watch.projectRef ? { projectRef: args.watch.projectRef } : {}), + }; + + const send = () => + sessions.open(args.chatId).in.send({ + kind: "message", + payload: { + chatId: args.chatId, + trigger: "action", + action: args.action, + metadata, + }, + }); + + try { + await send(); + } catch (error) { + // A chat born from the configuration card has no session yet, so create it + // (idempotent on externalId) and retry once. + if (!isSessionNotFound(error)) throw error; + await sessions.start({ + type: "chat.agent", + externalId: args.chatId, + taskIdentifier: "dashboard-agent", + triggerConfig: { + basePayload: { trigger: "preload", chatId: args.chatId, metadata }, + }, + }); + await send(); + } +} + +function isSessionNotFound(error: unknown): boolean { + if (error === null || typeof error !== "object") return false; + const e = error as { name?: string; status?: number }; + return e.name === "TriggerApiError" && e.status === 404; +} diff --git a/internal-packages/dashboard-agent/src/watch-tick.test.ts b/internal-packages/dashboard-agent/src/watch-tick.test.ts new file mode 100644 index 00000000000..0223ea14fd7 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-tick.test.ts @@ -0,0 +1,1455 @@ +import { watchResolutionToWireStatus } from "@internal/dashboard-agent-contracts"; +import type { Watch } from "@internal/dashboard-agent-db"; +import { describe, expect, it } from "vitest"; + +import { + runWatchBatchTick, + runWatchTick, + type WatchBatchCheckEntry, + type WatchBatchCheckResponse, + type WatchBatchTickDeps, + type WatchBatchTickPayload, + type WatchTickDeps, + type WatchTickPayload, + type WatchTickStore, +} from "./watch-tick"; +import type { WatchWakeAction } from "./dashboard-agent"; + +function payloadFor(tick: number): WatchTickPayload { + return { + watchId: "watch_1", + token: "watch_token", + apiOrigin: "http://localhost:3030", + tick, + }; +} + +const PAYLOAD = payloadFor(1); + +const NOW = new Date("2026-01-01T12:00:00.000Z"); + +function watchRow(overrides: Partial = {}): Watch { + return { + id: "watch_1", + chatId: "chat_1", + identity: "run_finished:run_a1", + spec: { + kind: "run_finished", + runId: "run_a1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when the receipt run finishes", + }, + status: "active", + deliveryStatus: "not_required", + cancelReason: null, + investigateOnAttention: false, + organizationId: "org_1", + projectId: "proj_1", + projectRef: "proj_abc", + environmentId: "env_1", + userId: "user_1", + createdAt: new Date("2026-01-01T11:00:00.000Z"), + expiresAt: new Date("2026-01-01T13:00:00.000Z"), + lastCheckedAt: null, + firedAt: null, + deliveryClaimedAt: null, + deliveryClaimId: null, + deliveredAt: null, + cancelledAt: null, + lastResult: null, + tickCount: 0, + ...overrides, + } as Watch; +} + +// Guarded exactly like the real queries. Several rows, because a batch tick's +// isolation is only real if its watches are separate rows. +function fakeStore(first: Watch, ...rest: Watch[]) { + const rows = [first, ...rest]; + const byId = new Map(rows.map((row) => [row.id, row])); + let claimSeq = 0; + const calls = { + claims: [] as unknown[], + transition: [] as unknown[], + deliveryClaims: [] as unknown[], + released: [] as unknown[], + delivered: [] as unknown[], + checks: [] as unknown[], + }; + const store: WatchTickStore = { + getWatch: async ({ id }) => { + const row = byId.get(id); + return row ? { ...row } : null; + }, + claimWatchTick: async (params) => { + calls.claims.push(params); + const row = byId.get(params.id); + if (!row || row.status !== "active") return null; + // Resumable like the real query: the previous generation or this one. + if (row.tickCount !== params.generation - 1 && row.tickCount !== params.generation) { + return null; + } + row.tickCount = params.generation; + // Deliberately NOT lastCheckedAt: a claim is not an observation. + return { ...row }; + }, + claimWatchDelivery: async (params) => { + calls.deliveryClaims.push(params); + const row = byId.get(params.id); + if (!row) return null; + const stale = + row.deliveryStatus === "delivering" && + (row.deliveryClaimedAt ?? row.createdAt).getTime() <= params.staleBefore.getTime(); + if (row.deliveryStatus !== "pending" && !stale) return null; + const claimId = `wdc_${++claimSeq}`; + row.deliveryStatus = "delivering"; + row.deliveryClaimedAt = NOW; + row.deliveryClaimId = claimId; + return { watch: { ...row }, claimId }; + }, + releaseWatchDelivery: async (params) => { + calls.released.push(params); + const row = byId.get(params.id); + if (!row) return null; + // Fenced: only the deliverer whose token the row still holds may release it. + if (row.deliveryStatus !== "delivering" || row.deliveryClaimId !== params.claimId) + return null; + row.deliveryStatus = "pending"; + row.deliveryClaimedAt = null; + row.deliveryClaimId = null; + return { ...row }; + }, + transitionWatchCondition: async (params) => { + calls.transition.push(params); + const row = byId.get(params.id); + if (!row || row.status !== "active") return null; + // Mirrors the query layer: status is derived from the resolution. + const status = watchResolutionToWireStatus(params.resolution); + row.status = status; + row.resolution = params.resolution; + row.deliveryStatus = "pending"; + row.lastCheckedAt = NOW; + if (status === "fired") row.firedAt = NOW; + if (params.observedOutcome !== undefined) row.observedOutcome = params.observedOutcome; + if (params.lastResult !== undefined) row.lastResult = params.lastResult; + return { ...row }; + }, + markWatchDelivered: async (params) => { + calls.delivered.push(params); + const row = byId.get(params.id); + if (!row) return null; + // Same fence: a mark from a taken-over deliverer completes nothing. + if (row.deliveryStatus !== "delivering" || row.deliveryClaimId !== params.claimId) + return null; + row.deliveryStatus = "delivered"; + row.deliveredAt = NOW; + return { ...row }; + }, + recordWatchCheck: async (params) => { + calls.checks.push(params); + const row = byId.get(params.id); + if (!row || row.status !== "active") return null; + row.lastCheckedAt = NOW; + if (params.lastResult !== undefined) row.lastResult = params.lastResult; + return { tickCount: row.tickCount, lastCheckedAt: row.lastCheckedAt }; + }, + }; + return { store, calls, row: first, rows, byId }; +} + +type FetchCall = { url: string; init: RequestInit | undefined }; + +function fakeFetch(responder: (call: FetchCall) => { status?: number; body: unknown }): { + fetch: typeof fetch; + calls: FetchCall[]; +} { + const calls: FetchCall[] = []; + const fetchImpl = (async (input: Parameters[0], init?: RequestInit) => { + const call = { url: String(input), init }; + calls.push(call); + const { status = 200, body } = responder(call); + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof fetch; + return { fetch: fetchImpl, calls }; +} + +function fakeDeliver(options: { throwOnce?: boolean } = {}) { + const appends: Array<{ chatId: string; action: WatchWakeAction }> = []; + let thrown = false; + return { + appends, + deliver: async ({ chatId, action }: { chatId: string; action: WatchWakeAction }) => { + if (options.throwOnce && !thrown) { + thrown = true; + throw new Error("session append failed"); + } + appends.push({ chatId, action }); + }, + }; +} + +function fakeNotifyFired(options: { throws?: boolean } = {}) { + const notified: string[] = []; + return { + notified, + notifyFired: async (watchId: string) => { + notified.push(watchId); + if (options.throws) throw new Error("the fired callback returned 500"); + }, + }; +} + +function fakeReschedule() { + const triggers: Array<{ payload: WatchTickPayload; options: Record }> = []; + return { + triggers, + reschedule: async (payload: WatchTickPayload, options: Record) => { + triggers.push({ payload, options }); + }, + }; +} + +function deps(parts: { + store: WatchTickStore; + fetch: typeof fetch; + deliver: WatchTickDeps["deliver"]; + reschedule: WatchTickDeps["reschedule"]; + notifyFired?: WatchTickDeps["notifyFired"]; + notifyInvestigate?: WatchTickDeps["notifyInvestigate"]; + now?: Date; +}): WatchTickDeps { + return { + store: parts.store, + fetch: parts.fetch, + deliver: parts.deliver, + reschedule: parts.reschedule, + notifyFired: parts.notifyFired ?? (async () => {}), + notifyInvestigate: parts.notifyInvestigate ?? (async () => {}), + now: () => parts.now ?? NOW, + }; +} + +describe("runWatchTick", () => { + it("pending: claims its generation, records the check, and reschedules the next generation", async () => { + const { store, calls, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: { result: "pending" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(4), deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "pending", tickCount: 4 }); + expect(calls.claims).toEqual([{ id: "watch_1", generation: 4 }]); + expect(row.tickCount).toBe(4); + expect(fetchCalls[0]?.url).toBe( + "http://localhost:3030/api/v1/dashboard-agent/watches/watch_1/check" + ); + expect( + (fetchCalls[0]?.init?.headers as Record | undefined)?.Authorization + ).toBe("Bearer watch_token"); + expect(JSON.parse(String(fetchCalls[0]?.init?.body))).toEqual({}); + + expect(calls.checks).toEqual([{ id: "watch_1", lastResult: {} }]); + expect(triggers).toEqual([ + { + payload: payloadFor(5), + options: { delay: "1m", idempotencyKey: "watch:watch_1:tick:5" }, + }, + ]); + + expect(row.status).toBe("active"); + expect(appends).toHaveLength(0); + }); + + it("satisfied: transitions to fired, appends the wake, then marks it delivered", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ + body: { result: "satisfied", facts: { status: "COMPLETED", durationMs: 4200 } }, + })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + const { notified, notifyFired } = fakeNotifyFired(); + + const result = await runWatchTick( + PAYLOAD, + deps({ store, fetch, deliver, reschedule, notifyFired }) + ); + + expect(result).toEqual({ outcome: "fired" }); + expect(row.status).toBe("fired"); + expect(row.deliveryStatus).toBe("delivered"); + expect(triggers).toHaveLength(0); + + expect(appends).toHaveLength(1); + expect(appends[0]?.chatId).toBe("chat_1"); + expect(appends[0]?.action).toMatchObject({ + type: "watch.fired", + id: "watch:watch_1:fired", + watchId: "watch_1", + identity: "run_finished:run_a1", + facts: { verified: true, status: "COMPLETED", durationMs: 4200 }, + note: "tell me when the receipt run finishes", + }); + expect(calls.delivered).toEqual([{ id: "watch_1", claimId: "wdc_1" }]); + expect(notified).toEqual(["watch_1"]); + }); + + it("the wake carries the row's investigate-on-attention consent, and kicks the investigation", async () => { + const { store } = fakeStore(watchRow({ investigateOnAttention: true })); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const kicked: string[] = []; + + await runWatchTick( + PAYLOAD, + deps({ + store, + fetch, + deliver, + reschedule, + notifyInvestigate: async (watchId) => void kicked.push(watchId), + }) + ); + + expect(appends[0]?.action.investigateOnAttention).toBe(true); + expect(kicked).toEqual(["watch_1"]); + }); + + it("kicks no investigation without the consent", async () => { + const { store } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const kicked: string[] = []; + + await runWatchTick( + PAYLOAD, + deps({ + store, + fetch, + deliver, + reschedule, + notifyInvestigate: async (watchId) => void kicked.push(watchId), + }) + ); + + expect(kicked).toEqual([]); + }); + + it("a failing investigate kick does not fail the tick", async () => { + const { store, row } = fakeStore(watchRow({ investigateOnAttention: true })); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + const result = await runWatchTick( + PAYLOAD, + deps({ + store, + fetch, + deliver, + reschedule, + notifyInvestigate: async () => { + throw new Error("the investigate callback returned 500"); + }, + }) + ); + + expect(result).toEqual({ outcome: "fired" }); + expect(appends).toHaveLength(1); + expect(row.deliveryStatus).toBe("delivered"); + }); + + it("kicks the investigation on an expiry as well", async () => { + const { store } = fakeStore(watchRow({ tickCount: 3, investigateOnAttention: true })); + const { fetch } = fakeFetch(() => ({ body: { result: "pending" } })); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const kicked: string[] = []; + + const result = await runWatchTick( + payloadFor(4), + deps({ + store, + fetch, + deliver, + reschedule, + notifyInvestigate: async (watchId) => void kicked.push(watchId), + now: new Date("2026-01-01T13:00:01.000Z"), + }) + ); + + expect(result.outcome).toBe("expired"); + expect(kicked).toEqual(["watch_1"]); + }); + + it("a failing fired notification does not fail the tick", async () => { + const { store, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const { notified, notifyFired } = fakeNotifyFired({ throws: true }); + + const result = await runWatchTick( + PAYLOAD, + deps({ store, fetch, deliver, reschedule, notifyFired }) + ); + + expect(result).toEqual({ outcome: "fired" }); + expect(notified).toEqual(["watch_1"]); + expect(appends).toHaveLength(1); + expect(row.deliveryStatus).toBe("delivered"); + }); + + it("an expiry does not notify: only a fired watch alerts", async () => { + const { store, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch } = fakeFetch(() => ({ body: { result: "pending" } })); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const { notified, notifyFired } = fakeNotifyFired(); + + const result = await runWatchTick( + payloadFor(4), + deps({ + store, + fetch, + deliver, + reschedule, + notifyFired, + now: new Date("2026-01-01T13:00:01.000Z"), + }) + ); + + expect(result.outcome).toBe("expired"); + expect(row.status).toBe("expired"); + expect(notified).toEqual([]); + }); + + it("a run that started and finished between two ticks fires, it does not go terminal_unsatisfied", async () => { + const { store, row } = fakeStore( + watchRow({ tickCount: 1, spec: { ...watchRow().spec, kind: "run_start" } as Watch["spec"] }) + ); + const { fetch } = fakeFetch(() => ({ + body: { result: "satisfied", facts: { startedAt: "2026-01-01T11:59:00.000Z" } }, + })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(2), deps({ store, fetch, deliver, reschedule })); + + expect(result.outcome).toBe("fired"); + expect(row.status).toBe("fired"); + expect(appends[0]?.action.type).toBe("watch.fired"); + }); + + it("crash between the transition and the append: the invocation throws, and the retry delivers only once", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied", facts: { runs: 0 } } })); + const { appends, deliver } = fakeDeliver({ throwOnce: true }); + const { reschedule } = fakeReschedule(); + const d = deps({ store, fetch, deliver, reschedule }); + + await expect(runWatchTick(PAYLOAD, d)).rejects.toThrow("session append failed"); + + expect(row.status).toBe("fired"); + expect(row.deliveryStatus).toBe("pending"); + expect(calls.released).toEqual([{ id: "watch_1", claimId: "wdc_1" }]); + expect(calls.delivered).toHaveLength(0); + expect(appends).toHaveLength(0); + + const retry = await runWatchTick(PAYLOAD, d); + expect(retry).toEqual({ outcome: "delivered_only" }); + expect(calls.transition).toHaveLength(1); + expect(appends).toHaveLength(1); + expect(row.deliveryStatus).toBe("delivered"); + }); + + it("two concurrent invocations of the same generation wake the chat exactly once", async () => { + const { store, calls, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied", facts: { runs: 1 } } })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const { notified, notifyFired } = fakeNotifyFired(); + const d = deps({ store, fetch, deliver, reschedule, notifyFired }); + + const outcomes = ( + await Promise.all([runWatchTick(payloadFor(4), d), runWatchTick(payloadFor(4), d)]) + ).map((result) => result.outcome); + + expect(calls.deliveryClaims).toHaveLength(2); + expect(appends).toHaveLength(1); + expect(calls.delivered).toEqual([{ id: "watch_1", claimId: "wdc_1" }]); + expect(notified).toEqual(["watch_1"]); + expect(calls.transition).toHaveLength(2); + expect(outcomes).toContain("fired"); + expect(outcomes).toContain("already_delivering"); + expect(row).toMatchObject({ status: "fired", deliveryStatus: "delivered" }); + }); + + it("a live delivery claim is left alone, and a dead one is recovered", async () => { + const fresh = fakeStore( + watchRow({ + status: "fired", + deliveryStatus: "delivering", + deliveryClaimedAt: NOW, + firedAt: NOW, + }) + ); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { reschedule } = fakeReschedule(); + const live = fakeDeliver(); + + expect( + await runWatchTick( + { ...payloadFor(0), deliverOnly: true }, + deps({ store: fresh.store, fetch, deliver: live.deliver, reschedule }) + ) + ).toEqual({ outcome: "already_delivering" }); + expect(live.appends).toHaveLength(0); + expect(fresh.row.deliveryStatus).toBe("delivering"); + + const dead = fakeStore( + watchRow({ + status: "fired", + deliveryStatus: "delivering", + deliveryClaimedAt: new Date(NOW.getTime() - 60 * 60 * 1000), + firedAt: NOW, + }) + ); + const recovered = fakeDeliver(); + + expect( + await runWatchTick( + { ...payloadFor(0), deliverOnly: true }, + deps({ store: dead.store, fetch, deliver: recovered.deliver, reschedule }) + ) + ).toEqual({ outcome: "delivered_only" }); + expect(recovered.appends).toHaveLength(1); + expect(dead.row.deliveryStatus).toBe("delivered"); + }); + + it("a terminal, already-delivered watch does nothing at all", async () => { + const { store, calls } = fakeStore(watchRow({ status: "fired", deliveryStatus: "delivered" })); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "already_terminal" }); + expect(fetchCalls).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(triggers).toHaveLength(0); + expect(calls.transition).toHaveLength(0); + }); + + it("unavailable: a failed tick, never a fire and never a miss", async () => { + const { store, calls, row } = fakeStore( + watchRow({ tickCount: 2, lastResult: { pending: 12 } }) + ); + const { fetch } = fakeFetch(() => ({ status: 503, body: { error: "clickhouse is down" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(3), deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "unavailable", tickCount: 3 }); + expect(row.status).toBe("active"); + expect(calls.transition).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(triggers[0]?.options.idempotencyKey).toBe("watch:watch_1:tick:4"); + expect(row.lastResult).toMatchObject({ checkFailed: true, previous: { pending: 12 } }); + }); + + it("a run of failed checks does not nest: `previous` stays the last real observation", async () => { + const { store, row } = fakeStore( + watchRow({ + tickCount: 2, + lastCheckedAt: new Date("2026-01-01T12:50:00.000Z"), + lastResult: { pending: 12 }, + }) + ); + const { fetch } = fakeFetch(() => ({ status: 503, body: { error: "clickhouse is down" } })); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const d = deps({ store, fetch, deliver, reschedule }); + + for (const tick of [3, 4, 5, 6]) await runWatchTick(payloadFor(tick), d); + + // One level, not four: the row is serialised into the wake, the alert and the webhook. + expect((row.lastResult as { previous?: unknown }).previous).toEqual({ pending: 12 }); + }); + + it("the facts an unverified expiry carries are bounded by the same unwrap", async () => { + const { store, row } = fakeStore( + watchRow({ + tickCount: 28, + lastCheckedAt: new Date("2026-01-01T12:50:00.000Z"), + lastResult: { pending: 41 }, + }) + ); + const { fetch } = fakeFetch(() => ({ status: 500, body: { error: "metrics unavailable" } })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + for (const tick of [29, 30, 31]) { + await runWatchTick(payloadFor(tick), deps({ store, fetch, deliver, reschedule })); + } + await runWatchTick( + payloadFor(32), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:00:01.000Z") }) + ); + + expect(row.status).toBe("expired"); + const facts = (appends[0]!.action as { facts: Record }).facts; + expect(facts.reason).toBe("unverified_at_expiry"); + const observation = facts.lastObservation as { checkFailed?: boolean; previous?: unknown }; + expect(observation.checkFailed).toBe(true); + expect(observation.previous).toEqual({ pending: 41 }); + }); + + it("access_revoked: exits without resolving, delivering, or rescheduling", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ + status: 403, + body: { error: "no access", code: "access_revoked" }, + })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "revoked" }); + expect(calls.transition).toHaveLength(0); + expect(calls.checks).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(triggers).toHaveLength(0); + expect(row.tickCount).toBe(1); + }); + + it("an unrecognized 403 is a failed check, not a silent exit: it reschedules", async () => { + // Anything but access_revoked/cancelled/not_found leaves the row active. + const { store, calls, row } = fakeStore(watchRow({ tickCount: 1 })); + const { fetch } = fakeFetch(() => ({ status: 403, body: { error: "nope" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(2), deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "unavailable", tickCount: 2 }); + expect(row.status).toBe("active"); + expect(calls.transition).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(triggers[0]?.options.idempotencyKey).toBe("watch:watch_1:tick:3"); + expect(row.lastResult).toMatchObject({ checkFailed: true }); + }); + + it("a late duplicate of an old generation claims nothing: the chain can't fork", async () => { + const { store, calls, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: { result: "pending" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + const d = deps({ store, fetch, deliver, reschedule }); + + expect(await runWatchTick(payloadFor(4), d)).toEqual({ outcome: "pending", tickCount: 4 }); + expect(await runWatchTick(payloadFor(5), d)).toEqual({ outcome: "pending", tickCount: 5 }); + + // A duplicate of generation 4, arriving after its successor ran. + const late = await runWatchTick(payloadFor(4), d); + + expect(late).toEqual({ outcome: "stale" }); + expect(fetchCalls).toHaveLength(2); + expect(calls.checks).toHaveLength(2); + expect(triggers.map((trigger) => trigger.options.idempotencyKey)).toEqual([ + "watch:watch_1:tick:5", + "watch:watch_1:tick:6", + ]); + expect(calls.transition).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(row.tickCount).toBe(5); + }); + + it("a retry of a generation that crashed before its successor was accepted resumes it", async () => { + const { store, calls, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: { result: "pending" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + let failNextTrigger = true; + const d = deps({ + store, + fetch, + deliver, + reschedule: async (next, options) => { + if (failNextTrigger) { + failNextTrigger = false; + throw new Error("the trigger failed"); + } + return reschedule(next, options); + }, + }); + + await expect(runWatchTick(payloadFor(4), d)).rejects.toThrow("the trigger failed"); + expect(row.tickCount).toBe(4); + expect(triggers).toHaveLength(0); + + // Refusing the claim would leave the chain with no successor at all. + const retry = await runWatchTick(payloadFor(4), d); + + expect(retry).toEqual({ outcome: "pending", tickCount: 4 }); + expect(calls.claims).toEqual([ + { id: "watch_1", generation: 4 }, + { id: "watch_1", generation: 4 }, + ]); + expect(fetchCalls).toHaveLength(2); + expect(triggers).toHaveLength(1); + expect(triggers[0]?.payload).toEqual(payloadFor(5)); + expect(triggers[0]?.options.idempotencyKey).toBe("watch:watch_1:tick:5"); + expect(row.status).toBe("active"); + expect(row.tickCount).toBe(4); + expect(appends).toHaveLength(0); + }); + + it("a resumed generation past the deadline still resolves exactly once", async () => { + const { store, calls, row } = fakeStore(watchRow({ tickCount: 12 })); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied", facts: { runs: 1 } } })); + const { appends, deliver } = fakeDeliver({ throwOnce: true }); + const { reschedule } = fakeReschedule(); + const d = deps({ store, fetch, deliver, reschedule }); + + await expect(runWatchTick(payloadFor(13), d)).rejects.toThrow("session append failed"); + const retry = await runWatchTick(payloadFor(13), d); + + expect(retry).toEqual({ outcome: "delivered_only" }); + expect(calls.transition).toHaveLength(1); + expect(appends).toHaveLength(1); + expect(row.status).toBe("fired"); + expect(row.deliveryStatus).toBe("delivered"); + }); + + it("deliverOnly: wakes a resolved watch without claiming, checking, or rescheduling", async () => { + const { store, calls, row } = fakeStore( + watchRow({ + status: "fired", + deliveryStatus: "pending", + firedAt: NOW, + lastResult: { runs: 2 }, + }) + ); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick( + { ...payloadFor(0), deliverOnly: true }, + deps({ store, fetch, deliver, reschedule }) + ); + + expect(result).toEqual({ outcome: "delivered_only" }); + expect(calls.claims).toHaveLength(0); + expect(fetchCalls).toHaveLength(0); + expect(triggers).toHaveLength(0); + expect(appends).toHaveLength(1); + expect(appends[0]?.action).toMatchObject({ type: "watch.fired", id: "watch:watch_1:fired" }); + expect(row.deliveryStatus).toBe("delivered"); + }); + + it("deliverOnly on a watch that is still active decides nothing", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick( + { ...payloadFor(0), deliverOnly: true }, + deps({ store, fetch, deliver, reschedule }) + ); + + expect(result).toEqual({ outcome: "nothing_to_deliver" }); + expect(calls.claims).toHaveLength(0); + expect(calls.transition).toHaveLength(0); + expect(fetchCalls).toHaveLength(0); + expect(triggers).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(row.status).toBe("active"); + expect(row.tickCount).toBe(0); + }); + + it("expiry with an unavailable final check: the watch still expires, and the facts say it couldn't be verified", async () => { + const { store, row } = fakeStore( + watchRow({ + tickCount: 30, + lastCheckedAt: new Date("2026-01-01T12:50:00.000Z"), + lastResult: { pending: 41 }, + }) + ); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ + status: 500, + body: { error: "metrics unavailable" }, + })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick( + payloadFor(31), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:00:01.000Z") }) + ); + + expect(JSON.parse(String(fetchCalls[0]?.init?.body))).toEqual({ final: true }); + expect(result).toEqual({ outcome: "expired" }); + expect(row.status).toBe("expired"); + expect(triggers).toHaveLength(0); + expect(appends[0]?.action).toMatchObject({ + type: "watch.expired", + id: "watch:watch_1:expired", + facts: { + verified: false, + reason: "unverified_at_expiry", + lastObservedAt: "2026-01-01T12:50:00.000Z", + lastObservation: { pending: 41 }, + }, + }); + }); + + it("expiry with a pending final check: it expires as not met, verified", async () => { + const { store, row } = fakeStore(watchRow({ tickCount: 12 })); + const { fetch } = fakeFetch(() => ({ body: { result: "pending", facts: { pending: 7 } } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick( + payloadFor(13), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:30:00.000Z") }) + ); + + expect(result).toEqual({ outcome: "expired" }); + expect(row.status).toBe("expired"); + expect(triggers).toHaveLength(0); + expect(appends[0]?.action.facts).toMatchObject({ + verified: true, + reason: "not_met_by_expiry", + pending: 7, + }); + }); + + it("terminal_unsatisfied: stops as an expiry that says it can never happen now", async () => { + const { store, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ + body: { result: "terminal_unsatisfied", facts: { status: "CANCELED" } }, + })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "expired" }); + expect(row.status).toBe("expired"); + expect(triggers).toHaveLength(0); + expect(appends[0]?.action).toMatchObject({ + type: "watch.expired", + facts: { verified: true, reason: "terminal_unsatisfied", status: "CANCELED" }, + }); + }); + + it("a watch that no longer exists is a no-op", async () => { + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ body: {} })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + const store: WatchTickStore = { + getWatch: async () => null, + claimWatchTick: async () => null, + transitionWatchCondition: async () => null, + claimWatchDelivery: async () => null, + releaseWatchDelivery: async () => null, + markWatchDelivered: async () => null, + recordWatchCheck: async () => null, + }; + + const result = await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "missing" }); + expect(fetchCalls).toHaveLength(0); + expect(appends).toHaveLength(0); + }); +}); + +describe("the resolution model", () => { + const OBSERVED = { + kind: "run_finished" as const, + verified: true, + finalStatus: "COMPLETED_WITH_ERRORS", + durationMs: 4200, + }; + + it("records condition_met with the observation, and keeps the wire encoding", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ + body: { + result: "satisfied", + facts: { outcome: "COMPLETED_WITH_ERRORS" }, + observed: OBSERVED, + }, + })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(calls.transition).toEqual([ + { + id: "watch_1", + resolution: "condition_met", + observedOutcome: OBSERVED, + lastResult: { verified: true, outcome: "COMPLETED_WITH_ERRORS" }, + }, + ]); + expect(row.resolution).toBe("condition_met"); + + expect(appends[0]?.action).toMatchObject({ + type: "watch.fired", + id: "watch:watch_1:fired", + resolution: "condition_met", + observed: OBSERVED, + }); + }); + + it("records condition_impossible, not a plain expiry", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ + body: { result: "terminal_unsatisfied", facts: { status: "CANCELED" } }, + })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(calls.transition[0]).toMatchObject({ resolution: "condition_impossible" }); + expect(row.resolution).toBe("condition_impossible"); + expect(appends[0]?.action).toMatchObject({ + id: "watch:watch_1:expired", + resolution: "condition_impossible", + }); + }); + + // A condition true exactly at the deadline resolves `condition_met`. + it("lets the boundary check still resolve condition_met", async () => { + const { store, calls } = fakeStore(watchRow({ tickCount: 12 })); + const { fetch, calls: fetchCalls } = fakeFetch(() => ({ + body: { result: "satisfied", facts: { pending: 0 } }, + })); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + const result = await runWatchTick( + payloadFor(13), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:30:00.000Z") }) + ); + + expect(JSON.parse(String(fetchCalls[0]?.init?.body))).toEqual({ final: true }); + expect(calls.transition[0]).toMatchObject({ resolution: "condition_met" }); + expect(result).toEqual({ outcome: "fired" }); + }); + + it("lets the boundary check still resolve condition_impossible", async () => { + const { store, calls } = fakeStore(watchRow({ tickCount: 12 })); + const { fetch } = fakeFetch(() => ({ + body: { result: "terminal_unsatisfied", facts: { status: "CANCELED" } }, + })); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + await runWatchTick( + payloadFor(13), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:30:00.000Z") }) + ); + + expect(calls.transition[0]).toMatchObject({ resolution: "condition_impossible" }); + }); + + it("only a pending or unavailable boundary check becomes window_completed", async () => { + for (const body of [{ result: "pending" as const, facts: { pending: 7 } }, undefined]) { + const { store, calls } = fakeStore(watchRow({ tickCount: 12 })); + const { fetch } = fakeFetch(() => + body ? { body } : { status: 500, body: { error: "clickhouse is down" } } + ); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + await runWatchTick( + payloadFor(13), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:30:00.000Z") }) + ); + + expect(calls.transition[0]).toMatchObject({ resolution: "window_completed" }); + } + }); + + it("resolves nothing on a pending or unavailable check inside the window", async () => { + for (const body of [{ result: "pending" as const, facts: {} }, undefined]) { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => + body ? { body } : { status: 503, body: { error: "down" } } + ); + const { deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(calls.transition).toHaveLength(0); + expect(row.status).toBe("active"); + expect(row.resolution ?? null).toBeNull(); + } + }); + + it("carries an unverified observation through a window that could not be confirmed", async () => { + const { store, calls } = fakeStore(watchRow({ tickCount: 12 })); + const { fetch } = fakeFetch(() => ({ + body: { + result: "unavailable", + error: "metrics unavailable", + observed: { kind: "backlog_drain", verified: false, depth: null }, + }, + })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + await runWatchTick( + payloadFor(13), + deps({ store, fetch, deliver, reschedule, now: new Date("2026-01-01T13:30:00.000Z") }) + ); + + expect(calls.transition[0]).toMatchObject({ + resolution: "window_completed", + observedOutcome: { kind: "backlog_drain", verified: false }, + }); + expect(appends[0]?.action.facts).toMatchObject({ + verified: false, + reason: "unverified_at_expiry", + }); + }); +}); + +describe("runWatchBatchTick", () => { + const ENVIRONMENT = "env_1"; + const CADENCE = 5; + + function batchPayload( + tick: number, + overrides: Partial = {} + ): WatchBatchTickPayload { + return { + environmentId: ENVIRONMENT, + cadenceMinutes: CADENCE, + apiOrigin: "http://localhost:3030", + token: "batch_token", + epoch: 3, + tick, + ...overrides, + }; + } + + function group(count: number, overrides: Partial = {}): Watch[] { + return Array.from({ length: count }, (_, index) => + watchRow({ + id: `watch_${index + 1}`, + chatId: `chat_${index + 1}`, + environmentId: ENVIRONMENT, + spec: { ...watchRow().spec, checkEveryMinutes: CADENCE } as Watch["spec"], + ...overrides, + }) + ); + } + + function entry( + watch: Watch, + overrides: Partial = {} + ): WatchBatchCheckEntry { + return { + watchId: watch.id, + token: `token_${watch.id}`, + tick: watch.tickCount + 1, + result: "satisfied", + ...overrides, + }; + } + + function batchDeps(parts: { + store: WatchTickStore; + response: WatchBatchCheckResponse | (() => Promise); + deliver: WatchBatchTickDeps["deliver"]; + notifyFired?: WatchBatchTickDeps["notifyFired"]; + reschedule?: WatchBatchTickDeps["reschedule"]; + now?: Date; + }): WatchBatchTickDeps { + return { + store: parts.store, + checkBatch: async () => + typeof parts.response === "function" ? parts.response() : parts.response, + deliver: parts.deliver, + notifyFired: parts.notifyFired ?? (async () => {}), + notifyInvestigate: async () => {}, + reschedule: parts.reschedule ?? (async () => {}), + now: () => parts.now ?? NOW, + }; + } + + it("resolves every watch of the group from ONE check call, and reschedules once", async () => { + const rows = group(3); + const { store, calls } = fakeStore(rows[0]!, rows[1]!, rows[2]!); + const { appends, deliver } = fakeDeliver(); + const triggers: Array<{ payload: WatchBatchTickPayload; options: Record }> = + []; + let checkCalls = 0; + + const result = await runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: async () => { + checkCalls++; + return { watches: rows.map((row) => entry(row)), continues: true }; + }, + deliver, + reschedule: async (payload, options) => void triggers.push({ payload, options }), + }) + ); + + expect(checkCalls).toBe(1); + expect(result.outcome).toBe("ticked"); + expect(result.results.map((one) => one.outcome)).toEqual(["fired", "fired", "fired"]); + expect(appends.map((append) => append.chatId)).toEqual(["chat_1", "chat_2", "chat_3"]); + expect(calls.delivered).toHaveLength(3); + expect(rows.map((row) => row.status)).toEqual(["fired", "fired", "fired"]); + + expect(result.rescheduled).toBe(true); + expect(triggers).toHaveLength(1); + expect(triggers[0]?.payload).toEqual(batchPayload(8)); + expect(triggers[0]?.options).toEqual({ + delay: "5m", + idempotencyKey: "watch-batch:env_1:5:3:tick:8", + }); + }); + + it("one watch failing costs only that watch, and the chain still ticks on", async () => { + const rows = group(3); + const { store } = fakeStore(rows[0]!, rows[1]!, rows[2]!); + const appends: string[] = []; + const triggers: unknown[] = []; + + const batch = batchDeps({ + store, + response: { watches: rows.map((row) => entry(row)), continues: true }, + deliver: async ({ chatId }) => { + if (chatId === "chat_2") throw new Error("session append failed"); + appends.push(chatId); + }, + reschedule: async (payload, options) => void triggers.push({ payload, options }), + }); + + await expect(runWatchBatchTick(batchPayload(7), batch)).rejects.toThrow( + "1 of 3 watches failed their tick" + ); + + expect(appends).toEqual(["chat_1", "chat_3"]); + expect(rows[0]).toMatchObject({ status: "fired", deliveryStatus: "delivered" }); + expect(rows[2]).toMatchObject({ status: "fired", deliveryStatus: "delivered" }); + + expect(rows[1]).toMatchObject({ status: "fired", deliveryStatus: "pending" }); + + expect(triggers).toHaveLength(1); + }); + + it("a watch whose wake is owed is redelivered by the group's own tick", async () => { + const [owed] = group(1, { + status: "fired", + deliveryStatus: "pending", + firedAt: NOW, + lastResult: { runs: 2 }, + }); + const { store, calls } = fakeStore(owed!); + const { appends, deliver } = fakeDeliver(); + + const result = await runWatchBatchTick( + batchPayload(8), + batchDeps({ + store, + response: { + watches: [{ watchId: owed!.id, token: "t", tick: 0, deliverOnly: true }], + continues: true, + }, + deliver, + }) + ); + + expect(result.results).toEqual([{ watchId: "watch_1", outcome: "delivered_only" }]); + expect(calls.claims).toHaveLength(0); + expect(calls.transition).toHaveLength(0); + expect(appends).toHaveLength(1); + expect(owed!.deliveryStatus).toBe("delivered"); + }); + + it("evaluates the window boundary inside the batch: a pending final check expires the watch", async () => { + const rows = group(2); + rows[0]!.expiresAt = new Date(NOW.getTime() - 1000); + const { store, calls } = fakeStore(rows[0]!, rows[1]!); + const { appends, deliver } = fakeDeliver(); + + const result = await runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: { + watches: [ + entry(rows[0]!, { result: "pending", facts: { pending: 7 } }), + entry(rows[1]!, { result: "pending" }), + ], + continues: true, + }, + deliver, + }) + ); + + expect(result.results.map((one) => one.outcome)).toEqual(["expired", "pending"]); + expect(rows[0]?.status).toBe("expired"); + expect(rows[1]?.status).toBe("active"); + expect(calls.transition).toHaveLength(1); + expect(calls.transition[0]).toMatchObject({ resolution: "window_completed" }); + expect(appends[0]?.action.facts).toMatchObject({ + verified: true, + reason: "not_met_by_expiry", + pending: 7, + }); + }); + + it("a boundary check that is satisfied still fires, inside a batch too", async () => { + const [watch] = group(1); + watch!.expiresAt = new Date(NOW.getTime() - 1000); + const { store, calls } = fakeStore(watch!); + const { appends, deliver } = fakeDeliver(); + + const result = await runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: { watches: [entry(watch!, { facts: { pending: 0 } })], continues: true }, + deliver, + }) + ); + + expect(result.results[0]?.outcome).toBe("fired"); + expect(calls.transition[0]).toMatchObject({ resolution: "condition_met" }); + expect(appends[0]?.action.type).toBe("watch.fired"); + }); + + it("two overlapping batch runs wake each chat exactly once", async () => { + const rows = group(2); + const { store, calls } = fakeStore(rows[0]!, rows[1]!); + const { appends, deliver } = fakeDeliver(); + const fired: string[] = []; + const batch = batchDeps({ + store, + response: { watches: rows.map((row) => entry(row)), continues: true }, + deliver, + notifyFired: async ({ watchId }) => void fired.push(watchId), + }); + + const [first, second] = await Promise.all([ + runWatchBatchTick(batchPayload(7), batch), + runWatchBatchTick(batchPayload(7), batch), + ]); + + expect(calls.deliveryClaims).toHaveLength(4); + expect(calls.transition).toHaveLength(4); + expect(appends.map((append) => append.chatId).sort()).toEqual(["chat_1", "chat_2"]); + expect(calls.delivered).toHaveLength(2); + expect(fired.sort()).toEqual(["watch_1", "watch_2"]); + expect(rows.map((row) => row.deliveryStatus)).toEqual(["delivered", "delivered"]); + + const outcomes = [...first!.results, ...second!.results].map((one) => one.outcome); + expect(outcomes.filter((outcome) => outcome === "fired")).toHaveLength(2); + expect(outcomes.filter((outcome) => outcome === "already_delivering")).toHaveLength(2); + }); + + it("a stale run owns nothing: no checks, no wakes, no reschedule", async () => { + const rows = group(2); + const { store, calls } = fakeStore(rows[0]!, rows[1]!); + const { appends, deliver } = fakeDeliver(); + const triggers: unknown[] = []; + + const result = await runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: { stale: true }, + deliver, + reschedule: async () => void triggers.push(1), + }) + ); + + expect(result).toEqual({ outcome: "stale", results: [], rescheduled: false }); + expect(calls.claims).toHaveLength(0); + expect(appends).toHaveLength(0); + expect(triggers).toHaveLength(0); + expect(rows.map((row) => row.status)).toEqual(["active", "active"]); + }); + + it("stops the chain when the group has nothing left to watch", async () => { + const [watch] = group(1); + const { store } = fakeStore(watch!); + const { deliver } = fakeDeliver(); + const triggers: unknown[] = []; + + const result = await runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: { watches: [entry(watch!)], continues: false }, + deliver, + reschedule: async () => void triggers.push(1), + }) + ); + + expect(result.rescheduled).toBe(false); + expect(triggers).toHaveLength(0); + expect(watch!.status).toBe("fired"); + }); + + it("a revoked watch inside a batch is skipped, and its neighbours are not", async () => { + const rows = group(2); + const { store, calls } = fakeStore(rows[0]!, rows[1]!); + const { appends, deliver } = fakeDeliver(); + + const result = await runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: { + watches: [ + { ...entry(rows[0]!), result: undefined, code: "access_revoked" }, + entry(rows[1]!), + ], + continues: true, + }, + deliver, + }) + ); + + expect(result.results.map((one) => one.outcome)).toEqual(["revoked", "fired"]); + expect(calls.transition).toHaveLength(1); + expect(appends.map((append) => append.chatId)).toEqual(["chat_2"]); + }); + + it("throws when the batch check itself can't be read, before anything is scheduled", async () => { + const rows = group(2); + const { store, calls } = fakeStore(rows[0]!, rows[1]!); + const { deliver } = fakeDeliver(); + const triggers: unknown[] = []; + + await expect( + runWatchBatchTick( + batchPayload(7), + batchDeps({ + store, + response: async () => { + throw new Error("the batch check returned 500"); + }, + deliver, + reschedule: async () => void triggers.push(1), + }) + ) + ).rejects.toThrow("the batch check returned 500"); + + expect(calls.checks).toHaveLength(0); + expect(triggers).toHaveLength(0); + }); +}); + +describe("the hand-off to a batch chain", () => { + it("a pending check that reports a chain stops rescheduling the per-watch tick", async () => { + const { store, calls, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch } = fakeFetch(() => ({ body: { result: "pending", batched: true } })); + const { appends, deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(4), deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "handed_off", tickCount: 4 }); + expect(calls.checks).toHaveLength(1); + expect(triggers).toHaveLength(0); + expect(row.status).toBe("active"); + expect(appends).toHaveLength(0); + }); + + it("a failed check that reports a chain hands over too", async () => { + const { store, row } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch } = fakeFetch(() => ({ + status: 503, + body: { error: "clickhouse is down", batched: true }, + })); + const { deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(4), deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "handed_off", tickCount: 4 }); + expect(triggers).toHaveLength(0); + expect(row.lastResult).toMatchObject({ checkFailed: true }); + }); + + it("keeps its own chain when no chain is polling the group yet", async () => { + const { store } = fakeStore(watchRow({ tickCount: 3 })); + const { fetch } = fakeFetch(() => ({ body: { result: "pending", batched: false } })); + const { deliver } = fakeDeliver(); + const { triggers, reschedule } = fakeReschedule(); + + const result = await runWatchTick(payloadFor(4), deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "pending", tickCount: 4 }); + expect(triggers[0]?.options.idempotencyKey).toBe("watch:watch_1:tick:5"); + }); + + it("resolves as usual even when the answer reports a chain", async () => { + const { store, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied", batched: true } })); + const { appends, deliver } = fakeDeliver(); + const { reschedule } = fakeReschedule(); + + const result = await runWatchTick(PAYLOAD, deps({ store, fetch, deliver, reschedule })); + + expect(result).toEqual({ outcome: "fired" }); + expect(row.status).toBe("fired"); + expect(appends).toHaveLength(1); + }); +}); + +describe("the wake's delivery acknowledgement", () => { + it("leaves the wake owed when the append isn't acknowledged, and delivers it on the retry", async () => { + const { store, calls, row } = fakeStore(watchRow()); + const { fetch } = fakeFetch(() => ({ body: { result: "satisfied" } })); + const { reschedule } = fakeReschedule(); + + await expect( + runWatchTick( + PAYLOAD, + deps({ store, fetch, reschedule, deliver: async () => ({ appended: false }) }) + ) + ).rejects.toThrow(/wasn't appended/); + + expect(row.status).toBe("fired"); + expect(row.deliveryStatus).toBe("pending"); + expect(calls.delivered).toHaveLength(0); + expect(calls.released).toEqual([{ id: "watch_1", claimId: "wdc_1" }]); + + const { appends, deliver } = fakeDeliver(); + const retry = await runWatchTick(PAYLOAD, deps({ store, fetch, reschedule, deliver })); + + expect(retry).toEqual({ outcome: "delivered_only" }); + expect(appends).toHaveLength(1); + expect(row.deliveryStatus).toBe("delivered"); + }); +}); diff --git a/internal-packages/dashboard-agent/src/watch-tick.ts b/internal-packages/dashboard-agent/src/watch-tick.ts new file mode 100644 index 00000000000..e4821de205d --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-tick.ts @@ -0,0 +1,221 @@ +import type { PersistedWatchSpec, Watch } from "@internal/dashboard-agent-db"; +import type { WatchCheckResult, WatchObservedOutcome } from "@internal/dashboard-agent-contracts"; +import { logger, task, tasks } from "@trigger.dev/sdk"; +import { + runWatchBatchTick, + type WatchBatchTickPayload, + type WatchBatchTickResult, +} from "./watch-batch"; +import type { WatchDeliveryDeps, WatchTickResult, WatchTickStore } from "./watch-delivery"; +import { REVOKED_CODES, runWatchLifecycle, type CheckOutcome } from "./watch-lifecycle"; +import { + appendWakeToSession, + getWatchDb, + postBatchCheck, + postFired, + postInvestigate, + watchStore, +} from "./watch-task-adapters"; + +/** + * Two tasks over one lifecycle (`runWatchLifecycle`): `watchTick` for one watch, + * `watchBatchTick` for a group. The webapp evaluates conditions; a tick records them. + */ + +export type { + WatchDeliveryDeps, + WatchTickOutcome, + WatchTickResult, + WatchTickStore, +} from "./watch-delivery"; +export { expiredFacts, resolveAndDeliver } from "./watch-delivery"; +export type { CheckOutcome, WatchLifecycleDeps } from "./watch-lifecycle"; +export { runWatchLifecycle } from "./watch-lifecycle"; +export type { + WatchBatchCheckEntry, + WatchBatchCheckResponse, + WatchBatchTickDeps, + WatchBatchTickPayload, + WatchBatchTickResult, +} from "./watch-batch"; +export { runWatchBatchTick } from "./watch-batch"; +export { appendWakeToSession, getWatchDb } from "./watch-task-adapters"; + +export type WatchTickPayload = { + watchId: string; + /** The watch's own token, minted by the webapp. Authorizes the check endpoint. */ + token: string; + apiOrigin: string; + /** The tick generation this invocation owns, starting at 1. */ + tick: number; + /** Wake and mark only: no claim, no check, no reschedule, and `tick` is ignored. */ + deliverOnly?: boolean; +}; + +export type WatchTickDeps = WatchDeliveryDeps & { + store: WatchTickStore; + /** Injected so tests can assert the request the check endpoint receives. */ + fetch: typeof fetch; + reschedule: ( + payload: WatchTickPayload, + options: { delay: string; idempotencyKey: string } + ) => Promise; +}; + +async function postCheck( + deps: WatchTickDeps, + payload: WatchTickPayload, + final: boolean +): Promise { + const origin = payload.apiOrigin.replace(/\/$/, ""); + let response: Response; + try { + response = await deps.fetch( + `${origin}/api/v1/dashboard-agent/watches/${encodeURIComponent(payload.watchId)}/check`, + { + method: "POST", + headers: { + Authorization: `Bearer ${payload.token}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(final ? { final: true } : {}), + } + ); + } catch (error) { + return { kind: "unavailable", detail: (error as Error).message }; + } + + const body = (await response.json().catch(() => undefined)) as + | { + result?: WatchCheckResult; + facts?: Record; + observed?: WatchObservedOutcome; + code?: string; + error?: string; + batched?: boolean; + } + | undefined; + + if (!response.ok) { + if (body?.code && REVOKED_CODES.has(body.code)) return { kind: "revoked", code: body.code }; + return { + kind: "unavailable", + detail: body?.error ?? `status ${response.status}${body?.code ? ` (${body.code})` : ""}`, + observed: body?.observed, + handOff: body?.batched === true, + }; + } + + if (!body?.result) return { kind: "unavailable", detail: "the check returned no result" }; + if (body.result === "unavailable") { + return { + kind: "unavailable", + detail: body.error, + observed: body.observed, + handOff: body.batched === true, + }; + } + return { + kind: "result", + result: body.result, + facts: body.facts, + observed: body.observed, + handOff: body.batched === true, + }; +} + +export function runWatchTick( + payload: WatchTickPayload, + deps: WatchTickDeps +): Promise { + return runWatchLifecycle( + { watchId: payload.watchId, tick: payload.tick, deliverOnly: payload.deliverOnly }, + { + store: deps.store, + deliver: deps.deliver, + notifyFired: deps.notifyFired, + notifyInvestigate: deps.notifyInvestigate, + now: deps.now, + check: ({ final }) => postCheck(deps, payload, final), + onPending: (watch) => scheduleNextTick(deps, payload, watch), + } + ); +} + +// The successor's generation comes from the claimed generation, never the row's +// counter, and rides in the idempotency key, so a retry can't fork the chain. +async function scheduleNextTick( + deps: WatchTickDeps, + payload: WatchTickPayload, + watch: Watch +): Promise { + const spec = watch.spec as PersistedWatchSpec; + const next = payload.tick + 1; + await deps.reschedule( + { ...payload, tick: next }, + { + delay: `${spec.checkEveryMinutes}m`, + idempotencyKey: `watch:${watch.id}:tick:${next}`, + } + ); +} + +export const watchTick = task({ + id: "dashboard-agent-watch", + // Every write is idempotent or guarded and the failure modes are transient, so + // retry rather than lose the wake. + retry: { maxAttempts: 5 }, + run: async (payload: WatchTickPayload): Promise => { + const { db } = getWatchDb(); + const result = await runWatchTick(payload, { + store: watchStore(db), + fetch: (input, init) => fetch(input, init), + deliver: appendWakeToSession, + notifyFired: () => postFired(payload), + notifyInvestigate: () => postInvestigate(payload), + reschedule: (next, options) => + tasks.trigger("dashboard-agent-watch", next, options), + }); + + logger.info("dashboard-agent watch ticked", { + watchId: payload.watchId, + tick: payload.tick, + outcome: result.outcome, + tickCount: result.tickCount, + }); + + return result; + }, +}); + +export const watchBatchTick = task({ + id: "dashboard-agent-watch-batch", + // The reschedule happens before the failures are rethrown, so a retry can't end the + // chain. + retry: { maxAttempts: 5 }, + run: async (payload: WatchBatchTickPayload): Promise => { + const { db } = getWatchDb(); + const result = await runWatchBatchTick(payload, { + store: watchStore(db), + checkBatch: postBatchCheck, + deliver: appendWakeToSession, + notifyFired: (target) => postFired({ apiOrigin: payload.apiOrigin, ...target }), + notifyInvestigate: (target) => postInvestigate({ apiOrigin: payload.apiOrigin, ...target }), + reschedule: (next, options) => + tasks.trigger("dashboard-agent-watch-batch", next, options), + }); + + logger.info("dashboard-agent watch batch ticked", { + environmentId: payload.environmentId, + cadenceMinutes: payload.cadenceMinutes, + epoch: payload.epoch, + tick: payload.tick, + outcome: result.outcome, + watches: result.results.length, + rescheduled: result.rescheduled, + }); + + return result; + }, +}); diff --git a/internal-packages/dashboard-agent/src/watch-tools.ts b/internal-packages/dashboard-agent/src/watch-tools.ts new file mode 100644 index 00000000000..4d1a99bc764 --- /dev/null +++ b/internal-packages/dashboard-agent/src/watch-tools.ts @@ -0,0 +1,23 @@ +import { agentIntentSchema } from "@internal/dashboard-agent-contracts"; +import { tool, type ToolSet } from "ai"; +import { scheduleWatchSchema } from "./tool-schemas"; + +/** The watch-facing tool set. Everything watch-specific the agent can call lives here. */ +export function buildWatchTools(): ToolSet { + return { + // Proposes a watch, never creates one: the user confirming the card is what starts + // it, so the card owns consent, the cap and dedup. + schedule_watch: tool({ + ...scheduleWatchSchema, + execute: async ({ watch }) => { + // Re-validated through the intent schema, so a rejected spec becomes a tool + // error rather than an intent the host drops. + try { + return { intent: agentIntentSchema.parse({ kind: "watch", spec: watch }) }; + } catch (error) { + return { error: `Couldn't build that watch: ${(error as Error).message}` }; + } + }, + }), + }; +} diff --git a/internal-packages/database/prisma/migrations/20260729120000_add_dashboard_agent_watch_alert_type/migration.sql b/internal-packages/database/prisma/migrations/20260729120000_add_dashboard_agent_watch_alert_type/migration.sql new file mode 100644 index 00000000000..e659c8d0a2c --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260729120000_add_dashboard_agent_watch_alert_type/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "public"."ProjectAlertType" ADD VALUE IF NOT EXISTS 'DASHBOARD_AGENT_WATCH'; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 8ea3ea2fc3a..78c1c8b5af4 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2421,6 +2421,7 @@ enum ProjectAlertType { DEPLOYMENT_FAILURE DEPLOYMENT_SUCCESS ERROR_GROUP + DASHBOARD_AGENT_WATCH } enum ProjectAlertStatus { diff --git a/internal-packages/emails/emails/alert-dashboard-agent-watch.tsx b/internal-packages/emails/emails/alert-dashboard-agent-watch.tsx new file mode 100644 index 00000000000..f60070c483b --- /dev/null +++ b/internal-packages/emails/emails/alert-dashboard-agent-watch.tsx @@ -0,0 +1,171 @@ +import { + Body, + Button, + Container, + Head, + Heading, + Html, + Link, + Preview, + Section, + Tailwind, + Text, +} from "@react-email/components"; +import React from "react"; +import { z } from "zod"; +import { Footer } from "./components/Footer"; +import { Image } from "./components/Image"; +import { footerAnchor, footerItalic } from "./components/styles"; + +export const AlertDashboardAgentWatchEmailSchema = z.object({ + email: z.literal("alert-dashboard-agent-watch"), + /** The watched condition, as the agent names it (e.g. `run_finished:run_abc`). */ + identity: z.string(), + /** The watch kind, e.g. `run_finished`. */ + kind: z.string(), + /** Rendered by the webapp's `watch-presentation.ts`. Optional so an older enqueue still renders. */ + headline: z.string().optional(), + /** Colours the accent only. */ + tone: z.enum(["success", "warning", "error", "neutral"]).optional(), + /** Why the watch exists, in the user's own words. */ + note: z.string(), + /** + * The sentence that quotes the note, rendered by the webapp's presenter so this + * email and the Slack message say it identically. Optional so an older enqueue + * still renders. + */ + noteLine: z.string().optional(), + firedAt: z.string(), + /** What the check observed, already flattened to label/value pairs. */ + facts: z.array(z.object({ label: z.string(), value: z.string() })), + dashboardLink: z.string().url(), + unsubscribeLink: z.string().url().optional(), + organization: z.string(), + project: z.string(), + environment: z.string(), +}); + +type AlertDashboardAgentWatchEmailProps = z.infer; + +const previewDefaults: AlertDashboardAgentWatchEmailProps = { + email: "alert-dashboard-agent-watch", + identity: "run_finished:run_abc123", + kind: "run_finished", + headline: "Run run_abc123 finished", + tone: "success", + note: "tell me when the nightly invoice run finishes", + noteLine: "You asked to be told when: tell me when the nightly invoice run finishes", + firedAt: "2026-07-29T12:00:00.000Z", + facts: [ + { label: "Status", value: "COMPLETED" }, + { label: "Duration", value: "4.2s" }, + ], + dashboardLink: "https://cloud.trigger.dev", + unsubscribeLink: "https://cloud.trigger.dev/unsubscribe", + organization: "my-organization", + project: "my-project", + environment: "Production", +}; + +function formatFiredAt(firedAt: string) { + const date = new Date(firedAt); + + if (Number.isNaN(date.getTime())) { + return firedAt; + } + + return `${date.toISOString().slice(0, 16).replace("T", " ")} UTC`; +} + +// Only the accent colour is chosen here: the headline and note line arrive already rendered. +const TONE_COLOR: Record = { + success: "#A8FF53", + warning: "#FBBF24", + error: "#F87171", + neutral: "#D7D9DD", +}; + +/** Fallback for a payload enqueued before `headline` existed. */ +function fallbackHeadline(identity: string): string { + return `Your watch has an answer — ${identity}`; +} + +export default function Email(props: AlertDashboardAgentWatchEmailProps) { + const { + identity, + headline, + tone, + note, + noteLine, + firedAt, + facts, + dashboardLink, + unsubscribeLink, + organization, + project, + environment, + } = { ...previewDefaults, ...props }; + + const details = [identity, ...facts.slice(0, 3).map((fact) => `${fact.label}: ${fact.value}`)]; + const accentColor = TONE_COLOR[tone ?? "neutral"] ?? TONE_COLOR.neutral; + + return ( + + + {`${organization}: ${headline ?? fallbackHeadline(identity)}`} + + + +
+ Trigger.dev +
+
+ {/* Fact first, as in the panel: the micro-label says it's a watch, the + headline carries the fact. */} + + Watch update + + + {headline ?? fallbackHeadline(identity)} + + + I was keeping an eye on {project} ({environment}) for you, and this is the answer, + as of {formatFiredAt(firedAt)}. {noteLine ?? `You asked to be told when: ${note}`} + + + {details.join(" · ")} + +
+
+ +
+ + {unsubscribeLink && ( + + + Turn off these alerts + + + )} + +