From fee3bdea9cdd7a16c4eac14fb0f58116a68801cd Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 09:46:41 -0300 Subject: [PATCH 01/45] Take on a cron parser, the one piece of time arithmetic not worth owning --- bun.lock | 5 +++++ server/package.json | 1 + 2 files changed, 6 insertions(+) diff --git a/bun.lock b/bun.lock index 56d7b45c..2911131c 100644 --- a/bun.lock +++ b/bun.lock @@ -67,6 +67,7 @@ "@modelcontextprotocol/sdk": "^1.30.0", "better-auth": "^1.7.1", "cel-js": "^0.8.2", + "cron-parser": "^5", "drizzle-orm": "^0.45.2", "hono": "^4.10.0", "postgres": "^3.4.9", @@ -1022,6 +1023,8 @@ "cosmiconfig": ["cosmiconfig@9.0.2", "", { "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", "parse-json": "^5.2.0" }, "peerDependencies": { "typescript": ">=4.9.5" }, "optionalPeers": ["typescript"] }, "sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg=="], + "cron-parser": ["cron-parser@5.10.0", "", { "dependencies": { "luxon": "^3.7.2" } }, "sha512-izNAxJyRWUP8ljBoDSub5WyrVOUlT4SLGShswE7eoRBpp6QUsSycYxLBMJlbshgPBMcPT/nrfgjNY2918ayv2A=="], + "cross-inspect": ["cross-inspect@1.0.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-Pcw1JTvZLSJH83iiGWt6fRcT+BjZlCDRVwYLbUcHzv/CRpB7r0MlSrGbIyQvVSNyGnbt7G4AXuyCiDR3POvZ1A=="], "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], @@ -1566,6 +1569,8 @@ "lucide-react": ["lucide-react@0.525.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Tm1txJ2OkymCGkvwoHt33Y2JpN5xucVq1slHcgE6Lk0WjDfjgKWor5CdVER8U6DvcfMwh4M8XxmpTiyzfmfDYQ=="], + "luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], + "lz-string": ["lz-string@1.5.0", "", { "bin": { "lz-string": "bin/bin.js" } }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], diff --git a/server/package.json b/server/package.json index 84d1f35a..4b6fa4c1 100644 --- a/server/package.json +++ b/server/package.json @@ -19,6 +19,7 @@ "@modelcontextprotocol/sdk": "^1.30.0", "better-auth": "^1.7.1", "cel-js": "^0.8.2", + "cron-parser": "^5", "drizzle-orm": "^0.45.2", "hono": "^4.10.0", "postgres": "^3.4.9", From 0ede96dc53321f6718a5e271b1cf6354dfc1a3aa Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 10:07:25 -0300 Subject: [PATCH 02/45] Give a deployment somewhere to keep a standing instruction --- server/drizzle/0020_routines.sql | 30 +++++++++++++++ server/src/db/schema/coworker.ts | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 server/drizzle/0020_routines.sql diff --git a/server/drizzle/0020_routines.sql b/server/drizzle/0020_routines.sql new file mode 100644 index 00000000..ed22352a --- /dev/null +++ b/server/drizzle/0020_routines.sql @@ -0,0 +1,30 @@ +CREATE TYPE "public"."routine_run_status" AS ENUM('succeeded', 'failed', 'skipped');--> statement-breakpoint +CREATE TABLE "routine_runs" ( + "id" text PRIMARY KEY NOT NULL, + "routine_id" text NOT NULL, + "started_at" timestamp with time zone DEFAULT now() NOT NULL, + "finished_at" timestamp with time zone, + "status" "routine_run_status", + "error" text +); +--> statement-breakpoint +CREATE TABLE "routines" ( + "id" text PRIMARY KEY NOT NULL, + "owner_user_id" text NOT NULL, + "agent_id" text NOT NULL, + "channel_id" text NOT NULL, + "instruction" text NOT NULL, + "cron" text NOT NULL, + "timezone" text DEFAULT 'UTC' NOT NULL, + "enabled" boolean DEFAULT true NOT NULL, + "next_run_at" timestamp with time zone NOT NULL, + "last_run_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "routine_runs" ADD CONSTRAINT "routine_runs_routine_id_routines_id_fk" FOREIGN KEY ("routine_id") REFERENCES "public"."routines"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "routines" ADD CONSTRAINT "routines_owner_user_id_users_id_fk" FOREIGN KEY ("owner_user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "routines" ADD CONSTRAINT "routines_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "routine_runs_by_routine_idx" ON "routine_runs" USING btree ("routine_id","started_at");--> statement-breakpoint +CREATE INDEX "routines_due_idx" ON "routines" USING btree ("enabled","next_run_at"); \ No newline at end of file diff --git a/server/src/db/schema/coworker.ts b/server/src/db/schema/coworker.ts index 73c06a8e..c3c860b6 100644 --- a/server/src/db/schema/coworker.ts +++ b/server/src/db/schema/coworker.ts @@ -5,6 +5,7 @@ * here; never edit core.ts or computer.ts to do it. */ import { + boolean, index, pgEnum, pgTable, @@ -76,3 +77,67 @@ export const agentPreferences = pgTable( }, (table) => [primaryKey({ columns: [table.userId, table.agentId] })], ); + +export const routineRunStatus = pgEnum("routine_run_status", [ + "succeeded", + "failed", + "skipped", +]); + +/** + * A standing instruction one person gave one Bot, on a schedule. + * + * Owned rows all the way down: the owner is who the headless turn runs as, so the routine can do + * exactly what its owner could do in chat and nothing more. The channel is where the reply lands. + */ +export const routines = pgTable( + "routines", + { + id: text("id").primaryKey(), + ownerUserId: text("owner_user_id") + .notNull() + .references(() => users.id, { onDelete: "cascade" }), + agentId: text("agent_id") + .notNull() + .references(() => agents.id, { onDelete: "cascade" }), + /** + * Not a foreign key. Channels soft-delete (`channels.deletedAt`), and a routine pointing at a + * deleted channel must survive to be shown as broken rather than vanish in a cascade. + */ + channelId: text("channel_id").notNull(), + instruction: text("instruction").notNull(), + /** Five-field cron. Validated at the tool boundary; never parsed by the client. */ + cron: text("cron").notNull(), + /** IANA zone the cron is read in. UTC when the person never said otherwise. */ + timezone: text("timezone").notNull().default("UTC"), + enabled: boolean("enabled").notNull().default(true), + /** The sweep's read target. Recomputed on every write and CAS-advanced by the sweep. */ + nextRunAt: timestamp("next_run_at", { withTimezone: true }).notNull(), + lastRunAt: timestamp("last_run_at", { withTimezone: true }), + createdAt: createdAt(), + updatedAt: updatedAt(), + }, + (table) => [index("routines_due_idx").on(table.enabled, table.nextRunAt)], +); + +/** One row per firing, which is what the page's "last ran" and the fatigue rule read. */ +export const routineRuns = pgTable( + "routine_runs", + { + id: text("id").primaryKey(), + routineId: text("routine_id") + .notNull() + .references(() => routines.id, { onDelete: "cascade" }), + startedAt: timestamp("started_at", { withTimezone: true }) + .notNull() + .defaultNow(), + finishedAt: timestamp("finished_at", { withTimezone: true }), + /** Null means the firing is still in flight; only a finished run has succeeded/failed/skipped. */ + status: routineRunStatus("status"), + /** The refusal or the throw, capped like audit payloads. Never shown raw to a person. */ + error: text("error"), + }, + (table) => [ + index("routine_runs_by_routine_idx").on(table.routineId, table.startedAt), + ], +); From 08eb43d4a97a6abd536ebfcb563a125c14790c79 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 10:16:32 -0300 Subject: [PATCH 03/45] Read a cron in somebody's timezone, and say it back in words --- server/src/routines/schedule.ts | 183 ++++++++++++++++++++++++++ server/tests/routine-schedule.test.ts | 118 +++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 server/src/routines/schedule.ts create mode 100644 server/tests/routine-schedule.test.ts diff --git a/server/src/routines/schedule.ts b/server/src/routines/schedule.ts new file mode 100644 index 00000000..e3dc98fd --- /dev/null +++ b/server/src/routines/schedule.ts @@ -0,0 +1,183 @@ +import { CronExpressionParser } from "cron-parser"; + +/** Routines may run at most this often. A model can be talked into anything; the floor cannot. */ +export const MINIMUM_INTERVAL_MS = 15 * 60 * 1000; + +export class ScheduleRefusedError extends Error {} + +const UNREADABLE_MESSAGE = "That schedule could not be read."; +const UNKNOWN_TIMEZONE_MESSAGE = "That is not a timezone I know."; +const TOO_FREQUENT_MESSAGE = "Routines may run at most every 15 minutes."; + +/** + * cron-parser is lenient about field count on purpose (it also accepts a leading + * seconds field or a trailing year field), so it will happily "parse" a four-field + * or six-field string by filling in defaults. We want a hard five-field contract, + * so that check happens here, before the string ever reaches the parser. + */ +function hasFiveFields(cron: string): boolean { + return cron.trim().split(/\s+/).length === 5; +} + +/** + * The only reliable way to validate an IANA zone name in plain JS/TS: ask Intl to + * build a formatter for it and see whether it throws. cron-parser (via luxon) + * accepts an invalid zone silently at parse() time and only blows up later, with + * a message ("unhandled timestamp: Invalid Date") that says nothing about + * timezones — so we check this ourselves, up front, to give a sentence that means + * something. + */ +function isKnownTimeZone(timezone: string): boolean { + try { + new Intl.DateTimeFormat(undefined, { timeZone: timezone }); + return true; + } catch { + return false; + } +} + +/** + * Parse, validate against the floor, and return the next occurrence after `after`. + * + * One function owns both acceptance and scheduling, so what was accepted is always schedulable: + * the floor is checked on the gap between the next two occurrences, not on a guess about the + * expression's shape. + */ +export function nextOccurrence( + cron: string, + timezone: string, + after: Date, +): Date { + if (!hasFiveFields(cron)) { + throw new ScheduleRefusedError(UNREADABLE_MESSAGE); + } + if (!isKnownTimeZone(timezone)) { + throw new ScheduleRefusedError(UNKNOWN_TIMEZONE_MESSAGE); + } + + let first: Date; + let second: Date; + try { + const expression = CronExpressionParser.parse(cron, { + tz: timezone, + currentDate: after, + }); + // next() is strictly after currentDate, including when currentDate itself + // lands exactly on an occurrence. + first = expression.next().toDate(); + second = expression.next().toDate(); + } catch { + throw new ScheduleRefusedError(UNREADABLE_MESSAGE); + } + + if (second.getTime() - first.getTime() < MINIMUM_INTERVAL_MS) { + throw new ScheduleRefusedError(TOO_FREQUENT_MESSAGE); + } + + return first; +} + +const WEEKDAY_NAMES = [ + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", +]; + +function pad2(value: number): string { + return value.toString().padStart(2, "0"); +} + +function ordinal(day: number): string { + const remainder100 = day % 100; + if (remainder100 >= 11 && remainder100 <= 13) { + return `${day}th`; + } + switch (day % 10) { + case 1: + return `${day}st`; + case 2: + return `${day}nd`; + case 3: + return `${day}rd`; + default: + return `${day}th`; + } +} + +/** Joins names the way a person would say them: "A", "A and B", "A, B and C". */ +function joinWords(words: string[]): string { + if (words.length === 1) return words[0]; + const last = words[words.length - 1]; + const rest = words.slice(0, -1); + return `${rest.join(", ")} and ${last}`; +} + +function parsePlainInt(field: string, min: number, max: number): number | null { + if (!/^\d{1,2}$/.test(field)) return null; + const value = Number.parseInt(field, 10); + if (value < min || value > max) return null; + return value; +} + +/** + * "Weekdays at 09:00", "Every day at 18:30", or the raw expression when it is stranger than + * words. + * + * This string gets read by a model mid-conversation and rendered on a page. A formatter that + * throws on a weird expression would take a page down over a schedule nobody could read anyway, + * so unrecognized shapes fall back to the raw expression rather than raising anything, and the + * whole body is wrapped in a belt-and-suspenders try/catch to guarantee it. + */ +export function describeCron(cron: string): string { + try { + const fields = cron.trim().split(/\s+/); + if (fields.length !== 5) return cron; + + const [ + minuteField, + hourField, + dayOfMonthField, + monthField, + dayOfWeekField, + ] = fields; + const minute = parsePlainInt(minuteField, 0, 59); + const hour = parsePlainInt(hourField, 0, 23); + if (minute === null || hour === null) return cron; + + const time = `${pad2(hour)}:${pad2(minute)}`; + + if (dayOfMonthField === "*" && monthField === "*") { + if (dayOfWeekField === "*") { + return `Every day at ${time}`; + } + if (dayOfWeekField === "1-5") { + return `Weekdays at ${time}`; + } + if (/^[0-6]$/.test(dayOfWeekField)) { + const dayIndex = Number.parseInt(dayOfWeekField, 10); + return `${WEEKDAY_NAMES[dayIndex]}s at ${time}`; + } + if (/^[0-6](,[0-6])+$/.test(dayOfWeekField)) { + const names = dayOfWeekField + .split(",") + .map((digit) => `${WEEKDAY_NAMES[Number.parseInt(digit, 10)]}s`); + return `${joinWords(names)} at ${time}`; + } + } + + if (monthField === "*" && dayOfWeekField === "*") { + const day = parsePlainInt(dayOfMonthField, 1, 31); + if (day !== null) { + return `On the ${ordinal(day)} of the month at ${time}`; + } + } + + return cron; + } catch { + return cron; + } +} diff --git a/server/tests/routine-schedule.test.ts b/server/tests/routine-schedule.test.ts new file mode 100644 index 00000000..55f53cfb --- /dev/null +++ b/server/tests/routine-schedule.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { + describeCron, + MINIMUM_INTERVAL_MS, + nextOccurrence, + ScheduleRefusedError, +} from "../src/routines/schedule"; + +describe("nextOccurrence", () => { + test("crosses the US spring-forward DST boundary at 09:30 local on both sides", () => { + // 2026-03-08 is when America/New_York springs forward (02:00 -> 03:00 local). + // A naive "+24h in UTC" implementation would land at 08:30 or 10:30 local on + // one side of the transition; a correct implementation stays pinned at 09:30 + // local because it re-derives wall-clock time in the target zone each day. + const before = nextOccurrence( + "30 9 * * *", + "America/New_York", + new Date("2026-03-07T00:00:00Z"), + ); + // 2026-03-07 09:30 EST (UTC-5) -> 14:30 UTC. Before the transition. + expect(before.toISOString()).toBe("2026-03-07T14:30:00.000Z"); + + const after = nextOccurrence("30 9 * * *", "America/New_York", before); + // 2026-03-08 09:30 EDT (UTC-4) -> 13:30 UTC. After the transition. + expect(after.toISOString()).toBe("2026-03-08T13:30:00.000Z"); + }); + + test("refuses every-minute schedules with the floor sentence", () => { + expect(() => + nextOccurrence("* * * * *", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow(ScheduleRefusedError); + expect(() => + nextOccurrence("* * * * *", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow("Routines may run at most every 15 minutes."); + }); + + test("refuses every-5-minute schedules with the floor sentence", () => { + expect(() => + nextOccurrence("*/5 * * * *", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow("Routines may run at most every 15 minutes."); + }); + + test("accepts a 15-minute schedule, exactly at the floor", () => { + expect(MINIMUM_INTERVAL_MS).toBe(15 * 60 * 1000); + const result = nextOccurrence( + "*/15 * * * *", + "UTC", + new Date("2026-01-01T00:00:00Z"), + ); + expect(result).toBeInstanceOf(Date); + }); + + test("refuses an unknown IANA timezone", () => { + expect(() => + nextOccurrence( + "0 9 * * *", + "Mars/Olympus", + new Date("2026-01-01T00:00:00Z"), + ), + ).toThrow("That is not a timezone I know."); + }); + + test("refuses an expression that is not five whitespace-separated fields", () => { + expect(() => + nextOccurrence("0 9 * *", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow("That schedule could not be read."); + expect(() => + nextOccurrence("0 9 * * * *", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow("That schedule could not be read."); + }); + + test("refuses garbage that is not a cron expression at all", () => { + expect(() => + nextOccurrence("every morning", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow("That schedule could not be read."); + }); + + test("is strictly after `after`, even when `after` sits exactly on an occurrence", () => { + // Midnight UTC is itself a "0 0 * * *" occurrence; the next one must be the + // following midnight, not the same instant handed in. + const after = new Date("2026-01-01T00:00:00Z"); + const result = nextOccurrence("0 0 * * *", "UTC", after); + expect(result.getTime()).toBeGreaterThan(after.getTime()); + expect(result.toISOString()).toBe("2026-01-02T00:00:00.000Z"); + }); +}); + +describe("describeCron", () => { + test("every day", () => { + expect(describeCron("30 18 * * *")).toBe("Every day at 18:30"); + }); + + test("weekdays", () => { + expect(describeCron("0 9 * * 1-5")).toBe("Weekdays at 09:00"); + }); + + test("a single weekday", () => { + expect(describeCron("0 9 * * 3")).toBe("Wednesdays at 09:00"); + }); + + test("a listed set of weekdays", () => { + expect(describeCron("0 9 * * 1,3,5")).toBe( + "Mondays, Wednesdays and Fridays at 09:00", + ); + }); + + test("monthly on a day", () => { + expect(describeCron("0 9 1 * *")).toBe("On the 1st of the month at 09:00"); + }); + + test("falls through to the raw expression when it is stranger than words", () => { + expect(describeCron("*/7 3,4 * * *")).toBe("*/7 3,4 * * *"); + }); + + test("never throws, even on garbage", () => { + expect(describeCron("not a cron expression")).toBe("not a cron expression"); + }); +}); From f61f074adc8eac181f2a1c7501338a26c9a534ff Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 10:27:55 -0300 Subject: [PATCH 04/45] Keep and guard a person's standing instructions --- server/src/routines/store.ts | 455 ++++++++++++++ .../tests/routines-store.integration.test.ts | 553 ++++++++++++++++++ 2 files changed, 1008 insertions(+) create mode 100644 server/src/routines/store.ts create mode 100644 server/tests/routines-store.integration.test.ts diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts new file mode 100644 index 00000000..1002c63e --- /dev/null +++ b/server/src/routines/store.ts @@ -0,0 +1,455 @@ +/** + * A person's standing instructions: keeping them, and guarding them. + * + * THIS FILE HAS TWO HALVES, AND THEY ARE SEPARABLE ON PURPOSE. The half below is the one a person + * touches: create, list, change, remove, switch off. It is asked questions through a Bot, so every + * refusal is a sentence a model can act on, every statement is guarded by the owner, and the owner is + * never taken from an argument a model supplied. Its failures are one person's failures — a bad cron, + * a channel they are not in — and none of them are about two things happening at once. + * + * The other half is the sweep's: the ledger read on a clock (which routines are due, advancing the + * next run, opening and closing a run row). Nothing there is asked a question by a person, and + * everything there is about concurrency — several replicas reading the same due row in the same + * second. That is why the halves are worth telling apart: only the second one has anything to do with + * concurrency, and only the second one needs to be reasoned about as a race. It lands in the next + * commit, and this file deliberately does not contain it. + * + * NO CLAIM OR LEASE MACHINERY, EVER. Firing mechanics belong to the shared `work_items` queue in + * `server/src/work/queue.ts`, which already owns `for update skip locked`, leases on the database's + * clock, and an attempt count. A second lease grown on the routines table — a `claimed_by`, a + * `locked_until` — is exactly the duplicated firing mechanism #235 exists to prevent: two half-right + * implementations of the same hard thing, one of which nobody tests. + */ +import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; +import type { Database } from "../db/client"; +import { + channelAgents, + channelMemberships, + channels, + routineRuns, + routines, +} from "../db/schema"; +import { ScheduleRefusedError, describeCron, nextOccurrence } from "./schedule"; + +export class RoutineNotFoundError extends Error { + constructor(message = "That routine does not exist.") { + super(message); + this.name = "RoutineNotFoundError"; + } +} + +/** The floor, the cap, a bad zone, a channel that is not the caller's. Carries the sentence verbatim. */ +export class RoutineRefusedError extends Error { + constructor(message: string) { + super(message); + this.name = "RoutineRefusedError"; + } +} + +/** A person may keep this many routines switched on. A constant with a reason, not a setting. */ +export const MAX_ENABLED_ROUTINES = 20; +/** Same code-point cap discipline as channel activity. */ +export const MAX_INSTRUCTION_CODE_POINTS = 2000; + +const NO_SHARED_CHANNEL = + "I can only post into a channel you and I are both in."; +const NO_CHANNEL_AT_ALL = + "We have no channel for me to post into. Start one and ask again."; +const INSTRUCTION_EMPTY = "A routine needs an instruction to carry out."; +const INSTRUCTION_TOO_LONG = `An instruction can be at most ${MAX_INSTRUCTION_CODE_POINTS} characters.`; +const TOO_MANY_ENABLED = `You already have ${MAX_ENABLED_ROUTINES} routines switched on. Switch one off before adding another.`; + +/** How many names an ambiguity refusal reads out before it gives up and says "and others". */ +const MAX_NAMED_CHANNELS = 5; + +export type RoutineRunOutcome = "succeeded" | "failed" | "skipped"; + +export type Routine = { + id: string; + ownerUserId: string; + agentId: string; + channelId: string; + instruction: string; + cron: string; + timezone: string; + enabled: boolean; + nextRunAt: Date; + lastRunAt: Date | null; + createdAt: Date; +}; + +/** One row of the routines page: everything it draws, and nothing it would have to parse. */ +export type RoutineSummary = { + id: string; + agentId: string; + instruction: string; + /** + * The schedule in words — "Weekdays at 09:00" — never the cron expression. + * + * The client never parses a schedule. A cron string on the wire is an invitation for the browser + * to grow a second, disagreeing parser, and for the page to render one thing while the sweep does + * another. + */ + schedule: string; + timezone: string; + enabled: boolean; + nextRunAt: Date; + channelId: string; + /** The target channel's name, or null when the row is gone entirely rather than soft-deleted. */ + channelName: string | null; + /** Whether the target channel was deleted. A broken routine is shown, not hidden. */ + channelDeleted: boolean; + /** The most recent firing, or null when it has never fired. */ + lastRun: { status: RoutineRunOutcome | null; finishedAt: Date | null } | null; +}; + +export type RoutineInput = { + ownerUserId: string; + agentId: string; + channelId?: string; + instruction: string; + cron: string; + timezone?: string; +}; + +export type RoutinePatch = Partial<{ + instruction: string; + cron: string; + timezone: string; + channelId: string; + enabled: boolean; +}>; + +export type RoutineStore = { + create(input: RoutineInput): Promise; + listFor(ownerUserId: string): Promise; + update( + ownerUserId: string, + id: string, + patch: RoutinePatch, + ): Promise; + remove(ownerUserId: string, id: string): Promise; + setEnabled(ownerUserId: string, id: string, enabled: boolean): Promise; +}; + +type RoutineRow = typeof routines.$inferSelect; + +function toRoutine(row: RoutineRow): Routine { + return { + id: row.id, + ownerUserId: row.ownerUserId, + agentId: row.agentId, + channelId: row.channelId, + instruction: row.instruction, + cron: row.cron, + timezone: row.timezone, + enabled: row.enabled, + nextRunAt: row.nextRunAt, + lastRunAt: row.lastRunAt, + createdAt: row.createdAt, + }; +} + +/** Trim, then measure in code points like channel activity does — not in UTF-16 units. */ +function validInstruction(instruction: string): string { + const trimmed = instruction.trim(); + if (trimmed.length === 0) throw new RoutineRefusedError(INSTRUCTION_EMPTY); + if (Array.from(trimmed).length > MAX_INSTRUCTION_CODE_POINTS) { + throw new RoutineRefusedError(INSTRUCTION_TOO_LONG); + } + return trimmed; +} + +/** + * The schedule module owns both acceptance and the next occurrence, so its refusal sentence is the + * one a person should read. It is carried through verbatim rather than reworded here: a model that + * gets "Routines may run at most every 15 minutes" can propose a schedule that works, and a model + * that gets "invalid cron" cannot. + */ +function nextRunFor(cron: string, timezone: string, after: Date): Date { + try { + return nextOccurrence(cron, timezone, after); + } catch (error) { + if (error instanceof ScheduleRefusedError) { + throw new RoutineRefusedError(error.message); + } + throw error; + } +} + +/** "A", "A, B", or five names and "and others" — a sentence, not a list a client renders. */ +function nameThem(names: string[]): string { + if (names.length <= MAX_NAMED_CHANNELS) return names.join(", "); + return [...names.slice(0, MAX_NAMED_CHANNELS), "and others"].join(", "); +} + +export function createRoutineStore(database: Database): RoutineStore { + /** + * Where the reply lands, decided here rather than at the tool boundary. + * + * Discovery confirmed the conversation's own channel is not reachable at the tool layer, so a model + * asked to make a routine either names a channel or names nothing. Both are resolved against what + * the owner and this agent actually share, which is the only check that stops a routine posting + * one person's summary into another person's conversation. + */ + async function resolveChannel( + ownerUserId: string, + agentId: string, + channelId?: string, + ): Promise { + if (channelId !== undefined) { + // One query for all four conditions: the channel exists, it is not deleted, the owner is a + // member, and this agent is in it. Any miss is the same refusal, because telling them apart + // would tell a caller which channel ids exist. + const rows = await database + .select({ id: channels.id }) + .from(channels) + .innerJoin( + channelMemberships, + and( + eq(channelMemberships.channelId, channels.id), + eq(channelMemberships.userId, ownerUserId), + ), + ) + .innerJoin( + channelAgents, + and( + eq(channelAgents.channelId, channels.id), + eq(channelAgents.agentId, agentId), + ), + ) + .where(and(eq(channels.id, channelId), isNull(channels.deletedAt))) + .limit(1); + const found = rows[0]; + if (!found) throw new RoutineRefusedError(NO_SHARED_CHANNEL); + return found.id; + } + + // Six rows is enough to answer the question: one resolves, more than one refuses, and the + // sentence names at most five before it says "and others". + const candidates = await database + .select({ id: channels.id, name: channels.name }) + .from(channels) + .innerJoin( + channelMemberships, + and( + eq(channelMemberships.channelId, channels.id), + eq(channelMemberships.userId, ownerUserId), + ), + ) + .innerJoin( + channelAgents, + and( + eq(channelAgents.channelId, channels.id), + eq(channelAgents.agentId, agentId), + ), + ) + .where(isNull(channels.deletedAt)) + .orderBy(desc(channels.createdAt), desc(channels.id)) + .limit(MAX_NAMED_CHANNELS + 1); + + const only = candidates[0]; + if (!only) throw new RoutineRefusedError(NO_CHANNEL_AT_ALL); + if (candidates.length > 1) { + // Named, because this refusal goes back to the model: a sentence listing the channels is a + // question it can put to the person, and "be more specific" is not. + throw new RoutineRefusedError( + `You are in more than one channel with me — ${nameThem( + candidates.map((candidate) => candidate.name), + )}. Say which one.`, + ); + } + return only.id; + } + + /** How many of this person's routines are switched on. The cap counts these, not rows. */ + async function countEnabled(ownerUserId: string): Promise { + const [row] = await database + .select({ total: sql`count(*)::int` }) + .from(routines) + .where( + and(eq(routines.ownerUserId, ownerUserId), eq(routines.enabled, true)), + ); + return row?.total ?? 0; + } + + async function loadOwned( + ownerUserId: string, + id: string, + ): Promise { + const [row] = await database + .select() + .from(routines) + .where(and(eq(routines.id, id), eq(routines.ownerUserId, ownerUserId))) + .limit(1); + // A routine that is not yours is a routine that does not exist: the `setPinned` rule. + if (!row) throw new RoutineNotFoundError(); + return row; + } + + async function update( + ownerUserId: string, + id: string, + patch: RoutinePatch, + ): Promise { + const existing = await loadOwned(ownerUserId, id); + + const values: Partial = { + updatedAt: new Date(), + }; + if (patch.instruction !== undefined) { + values.instruction = validInstruction(patch.instruction); + } + if (patch.channelId !== undefined) { + values.channelId = await resolveChannel( + ownerUserId, + existing.agentId, + patch.channelId, + ); + } + if (patch.enabled !== undefined) values.enabled = patch.enabled; + + const enabling = patch.enabled === true && !existing.enabled; + if (enabling && (await countEnabled(ownerUserId)) >= MAX_ENABLED_ROUTINES) { + throw new RoutineRefusedError(TOO_MANY_ENABLED); + } + + const cron = patch.cron ?? existing.cron; + const timezone = patch.timezone ?? existing.timezone; + if (patch.cron !== undefined) values.cron = patch.cron; + if (patch.timezone !== undefined) values.timezone = timezone; + /* + * Recomputed for a new cron, a new zone, and for switching back on. That last one is the subtle + * case: a routine switched off in June still holds June's `next_run_at`, and enabling it without + * recomputing hands the sweep a firing that was due months ago. + */ + if (patch.cron !== undefined || patch.timezone !== undefined || enabling) { + values.nextRunAt = nextRunFor(cron, timezone, new Date()); + } + + const [row] = await database + .update(routines) + .set(values) + .where(and(eq(routines.id, id), eq(routines.ownerUserId, ownerUserId))) + .returning(); + if (!row) throw new RoutineNotFoundError(); + return toRoutine(row); + } + + return { + update, + + async create(input) { + const instruction = validInstruction(input.instruction); + const timezone = input.timezone ?? "UTC"; + const channelId = await resolveChannel( + input.ownerUserId, + input.agentId, + input.channelId, + ); + const nextRunAt = nextRunFor(input.cron, timezone, new Date()); + + if ((await countEnabled(input.ownerUserId)) >= MAX_ENABLED_ROUTINES) { + throw new RoutineRefusedError(TOO_MANY_ENABLED); + } + + const [row] = await database + .insert(routines) + .values({ + id: `routine_${crypto.randomUUID()}`, + ownerUserId: input.ownerUserId, + agentId: input.agentId, + channelId, + instruction, + cron: input.cron, + timezone, + nextRunAt, + }) + .returning(); + // An insert that returned nothing is not a missing routine, it is a broken database: loud + // rather than folded into the not-found sentence a caller is meant to be able to trust. + if (!row) throw new Error("inserting a routine returned no row"); + return toRoutine(row); + }, + + async listFor(ownerUserId) { + /* + * The last-run join reads `routine_runs`, which migration 0020 already created even though + * nothing in THIS commit writes a row to it — the sweep's half does. The join is not dead code; + * it is the half of the page that stays empty until the sweep lands. + * + * `distinct on (routine_id) ... order by started_at desc` gives one row per routine, the most + * recent, in a single index scan of `routine_runs_by_routine_idx`. Reading the runs in a + * separate statement rather than a lateral keeps the main query one flat join. + */ + const rows = await database + .select({ + routine: routines, + channelName: channels.name, + channelDeletedAt: channels.deletedAt, + // A left join, and the id is selected to tell "no channel row at all" (a hard delete + // somewhere else) from "soft-deleted": both mean the target is unusable, and neither is + // allowed to hide the routine, which is why `channel_id` is not a foreign key. + channelExists: channels.id, + }) + .from(routines) + .leftJoin(channels, eq(channels.id, routines.channelId)) + .where(eq(routines.ownerUserId, ownerUserId)) + .orderBy(desc(routines.createdAt), desc(routines.id)); + + const routineIds = rows.map((row) => row.routine.id); + const lastRuns = new Map< + string, + { status: RoutineRunOutcome | null; finishedAt: Date | null } + >(); + if (routineIds.length > 0) { + const runRows = await database + .selectDistinctOn([routineRuns.routineId], { + routineId: routineRuns.routineId, + status: routineRuns.status, + finishedAt: routineRuns.finishedAt, + }) + .from(routineRuns) + .where(inArray(routineRuns.routineId, routineIds)) + .orderBy(routineRuns.routineId, desc(routineRuns.startedAt)); + for (const run of runRows) { + lastRuns.set(run.routineId, { + status: run.status, + finishedAt: run.finishedAt, + }); + } + } + + return rows.map( + ({ routine, channelName, channelDeletedAt, channelExists }) => ({ + id: routine.id, + agentId: routine.agentId, + instruction: routine.instruction, + schedule: describeCron(routine.cron), + timezone: routine.timezone, + enabled: routine.enabled, + nextRunAt: routine.nextRunAt, + channelId: routine.channelId, + channelName, + channelDeleted: channelExists === null || channelDeletedAt !== null, + lastRun: lastRuns.get(routine.id) ?? null, + }), + ); + }, + + async remove(ownerUserId, id) { + // Hard, unlike a channel: nothing reads a routine that was deleted, and its runs cascade. + const deleted = await database + .delete(routines) + .where(and(eq(routines.id, id), eq(routines.ownerUserId, ownerUserId))) + .returning({ id: routines.id }); + if (deleted.length === 0) throw new RoutineNotFoundError(); + }, + + async setEnabled(ownerUserId, id, enabled) { + // One field through the same path, so enabling re-checks the cap and recomputes the next run + // rather than having a second, quieter version of those rules. + await update(ownerUserId, id, { enabled }); + }, + }; +} diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts new file mode 100644 index 00000000..2120b9f8 --- /dev/null +++ b/server/tests/routines-store.integration.test.ts @@ -0,0 +1,553 @@ +import { afterAll, afterEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { createAgentProfileStore } from "../src/agents/profile-store"; +import type { AgentActor } from "../src/agents/profile-types"; +import { createChannelStore } from "../src/channels/routes"; +import { createThreadIdentity } from "../src/channels/thread-identity"; +import { createDatabase } from "../src/db/client"; +import { + agentProfiles, + agents, + channels, + intelligenceChannelMappings, + routines, + users, +} from "../src/db/schema"; +import { + MAX_ENABLED_ROUTINES, + MAX_INSTRUCTION_CODE_POINTS, + RoutineNotFoundError, + RoutineRefusedError, + createRoutineStore, +} from "../src/routines/store"; +import { TEST_POOL } from "./support/database"; + +const databaseUrl = + process.env.DATABASE_URL ?? + "postgres://openbot:openbot@localhost:5432/openbot"; +const database = createDatabase(databaseUrl, TEST_POOL); +const profileStore = createAgentProfileStore( + database, + new URL("https://managed.example.test/ag-ui"), +); +const channelStore = createChannelStore( + database, + profileStore, + createThreadIdentity("test-deployment"), +); +const store = createRoutineStore(database); + +const testPrefix = `routines-store-${randomUUID()}`; +const createdUserIds: string[] = []; +const createdAgentIds: string[] = []; +const createdChannelIds: string[] = []; + +/** Every day at 09:00 UTC: comfortably above the floor and stable to describe. */ +const DAILY = "0 9 * * *"; + +afterEach(async () => { + // Routines cascade from both the owner and the agent, but they are cleared first so a failure + // part-way through cleanup leaves nothing pointing at rows this file is about to delete. + for (const userId of createdUserIds) { + await database.delete(routines).where(eq(routines.ownerUserId, userId)); + } + for (const channelId of createdChannelIds.splice(0)) { + await database + .delete(intelligenceChannelMappings) + .where(eq(intelligenceChannelMappings.channelId, channelId)); + await database.delete(channels).where(eq(channels.id, channelId)); + } + for (const agentId of createdAgentIds.splice(0)) { + await database + .delete(agentProfiles) + .where(eq(agentProfiles.agentId, agentId)); + await database.delete(agents).where(eq(agents.id, agentId)); + } + for (const userId of createdUserIds.splice(0)) { + await database.delete(users).where(eq(users.id, userId)); + } +}); + +afterAll(async () => { + await database.$client.close(); +}); + +async function createUser(): Promise { + const id = `${testPrefix}-user-${randomUUID()}`; + await database.insert(users).values({ + id, + email: `${id}@example.test`, + name: "Routine Store Test User", + }); + createdUserIds.push(id); + return { id, role: "user" }; +} + +async function createAgent(owner: AgentActor, name = "Expense Manager") { + const profile = await profileStore.create(owner, { + name, + title: "Finance Operations", + roleDescription: "Review receipts.", + visibility: "private", + }); + createdAgentIds.push(profile.id); + return profile.id; +} + +async function createChannel(owner: AgentActor, agentIds: string[]) { + const channel = await channelStore.create(owner, agentIds); + createdChannelIds.push(channel.id); + return channel; +} + +/** A person, a Bot and the one channel they share: the ordinary starting point. */ +async function setUp() { + const owner = await createUser(); + const agentId = await createAgent(owner); + const channel = await createChannel(owner, [agentId]); + return { owner, agentId, channel }; +} + +/** + * A routine is one person's standing instruction, so every method takes the owner and every + * statement is guarded by it. A routine belonging to somebody else has to be indistinguishable from + * one that was never created — the `setPinned` rule — because anything else tells a stranger which + * ids exist. + */ +describe("owner-guarding", () => { + test("another person cannot update, remove or switch off a routine", async () => { + const { owner, agentId, channel } = await setUp(); + const stranger = await createUser(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise yesterday's receipts.", + cron: DAILY, + }); + + await expect( + store.update(stranger.id, routine.id, { instruction: "Do my bidding." }), + ).rejects.toBeInstanceOf(RoutineNotFoundError); + await expect(store.remove(stranger.id, routine.id)).rejects.toBeInstanceOf( + RoutineNotFoundError, + ); + await expect( + store.setEnabled(stranger.id, routine.id, false), + ).rejects.toBeInstanceOf(RoutineNotFoundError); + + const [mine] = await store.listFor(owner.id); + expect(mine?.instruction).toBe("Summarise yesterday's receipts."); + expect(mine?.enabled).toBe(true); + }); + + test("a stranger's list does not contain the routine", async () => { + const { owner, agentId, channel } = await setUp(); + const stranger = await createUser(); + await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise yesterday's receipts.", + cron: DAILY, + }); + + expect(await store.listFor(stranger.id)).toEqual([]); + }); +}); + +describe("what the schedule has to be", () => { + test("refuses a cron under the floor, in the schedule's own words", async () => { + const { owner, agentId, channel } = await setUp(); + + // Every minute. The sentence is the schedule module's; the store only passes it along, because + // a model recovers from prose and this is the prose that says what to try instead. + await expect( + store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Check my inbox.", + cron: "* * * * *", + }), + ).rejects.toThrow(/15 minutes/); + await expect( + store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Check my inbox.", + cron: "* * * * *", + }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + }); + + test("refuses an instruction over the cap and accepts one at it", async () => { + const { owner, agentId, channel } = await setUp(); + + const atTheCap = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "x".repeat(MAX_INSTRUCTION_CODE_POINTS), + cron: DAILY, + }); + expect(Array.from(atTheCap.instruction)).toHaveLength( + MAX_INSTRUCTION_CODE_POINTS, + ); + + await expect( + store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "x".repeat(MAX_INSTRUCTION_CODE_POINTS + 1), + cron: DAILY, + }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + }); +}); + +/** + * The cap is a constant with a reason: every enabled routine is a headless turn somebody's Bot will + * take without being watched, and a model that can be talked into creating them one at a time can be + * talked into creating a hundred. It counts what is switched ON, so switching one off makes room. + */ +describe("how many a person may have switched on", () => { + test("refuses the one past the cap, and accepts it once another is off", async () => { + const { owner, agentId, channel } = await setUp(); + const created: string[] = []; + for (let index = 0; index < MAX_ENABLED_ROUTINES; index += 1) { + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: `Routine ${index}`, + cron: DAILY, + }); + created.push(routine.id); + } + + await expect( + store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "One too many.", + cron: DAILY, + }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + + await store.setEnabled(owner.id, created[0] as string, false); + + const room = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Now there is room.", + cron: DAILY, + }); + expect(room.enabled).toBe(true); + + // And switching the disabled one back on is refused for the same reason. + await expect( + store.setEnabled(owner.id, created[0] as string, true), + ).rejects.toBeInstanceOf(RoutineRefusedError); + }); +}); + +/** + * The conversation's channel is not reachable at the tool layer, so the store resolves it. All four + * branches matter: the wrong channel is a place a Bot could post something private, and the two + * ambiguous branches are refusals a model has to be able to recover from in language. + */ +describe("resolving the channel to post into", () => { + test("accepts a channel the owner and the agent share", async () => { + const { owner, agentId, channel } = await setUp(); + + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Post the summary here.", + cron: DAILY, + }); + + expect(routine.channelId).toBe(channel.id); + }); + + test("refuses a channel the owner is not in", async () => { + const { owner, agentId } = await setUp(); + const stranger = await createUser(); + const strangerAgentId = await createAgent(stranger, "Their Bot"); + const theirChannel = await createChannel(stranger, [strangerAgentId]); + + await expect( + store.create({ + ownerUserId: owner.id, + agentId, + channelId: theirChannel.id, + instruction: "Post into somebody else's channel.", + cron: DAILY, + }), + ).rejects.toThrow(/both in/); + }); + + test("refuses a channel this agent is not in", async () => { + const { owner, channel } = await setUp(); + const otherAgentId = await createAgent(owner, "Unrelated Bot"); + + await expect( + store.create({ + ownerUserId: owner.id, + agentId: otherAgentId, + channelId: channel.id, + instruction: "Post where I do not live.", + cron: DAILY, + }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + }); + + test("uses the only shared channel when none was named", async () => { + const { owner, agentId, channel } = await setUp(); + + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + instruction: "Wherever we talk.", + cron: DAILY, + }); + + expect(routine.channelId).toBe(channel.id); + }); + + test("refuses with no channel at all, and says to start one", async () => { + const owner = await createUser(); + const agentId = await createAgent(owner); + + await expect( + store.create({ + ownerUserId: owner.id, + agentId, + instruction: "Nowhere to put this.", + cron: DAILY, + }), + ).rejects.toThrow(/Start one/); + }); + + test("refuses ambiguity by naming the channels, so a model can ask", async () => { + const { owner, agentId, channel } = await setUp(); + const secondAgentId = await createAgent(owner, "Second Bot"); + const other = await createChannel(owner, [agentId, secondAgentId]); + + const failure = await store + .create({ + ownerUserId: owner.id, + agentId, + instruction: "Which one?", + cron: DAILY, + }) + .then( + () => null, + (error: unknown) => error, + ); + + expect(failure).toBeInstanceOf(RoutineRefusedError); + const message = (failure as Error).message; + expect(message).toContain(channel.name); + expect(message).toContain(other.name); + expect(message).toContain("Say which one."); + }); +}); + +describe("reading a person's routines", () => { + test("says the schedule in words and never in cron", async () => { + const { owner, agentId, channel } = await setUp(); + await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + timezone: "Europe/Madrid", + }); + + const [summary] = await store.listFor(owner.id); + + expect(summary?.schedule).toBe("Every day at 09:00"); + expect(summary?.schedule).not.toContain("*"); + expect(summary?.timezone).toBe("Europe/Madrid"); + expect(summary?.channelName).toBe(channel.name); + expect(summary?.channelDeleted).toBe(false); + expect(summary?.nextRunAt).toBeInstanceOf(Date); + // Nothing writes routine_runs yet; the join is here for the commit that does. + expect(summary?.lastRun).toBeNull(); + }); + + test("puts the newest first", async () => { + const { owner, agentId, channel } = await setUp(); + const first = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "First.", + cron: DAILY, + }); + const second = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Second.", + cron: DAILY, + }); + + expect((await store.listFor(owner.id)).map((row) => row.id)).toEqual([ + second.id, + first.id, + ]); + }); + + test("keeps a routine whose channel was deleted and reports it as gone", async () => { + // The whole reason `routines.channel_id` is not a foreign key: a cascade here would delete the + // person's standing instruction silently, and they would find out by it never running again. + const { owner, agentId, channel } = await setUp(); + await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Post into a channel that is about to go.", + cron: DAILY, + }); + + await channelStore.softDelete(owner, channel.id); + + const [summary] = await store.listFor(owner.id); + expect(summary?.channelId).toBe(channel.id); + expect(summary?.channelDeleted).toBe(true); + }); +}); + +describe("changing a routine", () => { + test("recomputes the next run when the cron changes", async () => { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + + const updated = await store.update(owner.id, routine.id, { + cron: "30 21 * * *", + }); + + expect(updated.cron).toBe("30 21 * * *"); + expect(updated.nextRunAt.getTime()).not.toBe(routine.nextRunAt.getTime()); + }); + + test("leaves the next run alone when only the instruction changes", async () => { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + + const updated = await store.update(owner.id, routine.id, { + instruction: "Summarise the week.", + }); + + expect(updated.instruction).toBe("Summarise the week."); + expect(updated.nextRunAt.getTime()).toBe(routine.nextRunAt.getTime()); + }); + + test("re-validates only what was supplied", async () => { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + + await expect( + store.update(owner.id, routine.id, { cron: "* * * * *" }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + await expect( + store.update(owner.id, routine.id, { + instruction: "x".repeat(MAX_INSTRUCTION_CODE_POINTS + 1), + }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + // Refused writes changed nothing. + const [summary] = await store.listFor(owner.id); + expect(summary?.instruction).toBe("Summarise the day."); + expect(summary?.schedule).toBe("Every day at 09:00"); + }); + + test("re-resolves the channel when a new one is named", async () => { + const { owner, agentId, channel } = await setUp(); + const secondAgentId = await createAgent(owner, "Second Bot"); + const other = await createChannel(owner, [agentId, secondAgentId]); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + + const moved = await store.update(owner.id, routine.id, { + channelId: other.id, + }); + expect(moved.channelId).toBe(other.id); + + const stranger = await createUser(); + const strangerAgentId = await createAgent(stranger, "Their Bot"); + const theirChannel = await createChannel(stranger, [strangerAgentId]); + await expect( + store.update(owner.id, routine.id, { channelId: theirChannel.id }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + }); + + test("switching off and on again recomputes the next run", async () => { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + // A routine switched off for a month has a next run a month in the past. Enabling it must not + // hand the sweep a firing that was due in June. + await store.setEnabled(owner.id, routine.id, false); + + const enabled = await store.update(owner.id, routine.id, { enabled: true }); + + expect(enabled.enabled).toBe(true); + expect(enabled.nextRunAt.getTime()).toBeGreaterThan(Date.now()); + }); + + test("removing one is a hard delete, and only once", async () => { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + + await store.remove(owner.id, routine.id); + + expect(await store.listFor(owner.id)).toEqual([]); + await expect(store.remove(owner.id, routine.id)).rejects.toBeInstanceOf( + RoutineNotFoundError, + ); + }); +}); From c0497d5d448de971299588acb2d9d63078747d22 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 10:37:39 -0300 Subject: [PATCH 05/45] Say every accepted schedule in words, and pin the recompute rules --- server/src/routines/schedule.ts | 40 ++++++- server/tests/routine-schedule.test.ts | 8 ++ .../tests/routines-store.integration.test.ts | 111 +++++++++++++++++- 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/server/src/routines/schedule.ts b/server/src/routines/schedule.ts index e3dc98fd..de9989d9 100644 --- a/server/src/routines/schedule.ts +++ b/server/src/routines/schedule.ts @@ -3,7 +3,12 @@ import { CronExpressionParser } from "cron-parser"; /** Routines may run at most this often. A model can be talked into anything; the floor cannot. */ export const MINIMUM_INTERVAL_MS = 15 * 60 * 1000; -export class ScheduleRefusedError extends Error {} +export class ScheduleRefusedError extends Error { + constructor(message: string) { + super(message); + this.name = "ScheduleRefusedError"; + } +} const UNREADABLE_MESSAGE = "That schedule could not be read."; const UNKNOWN_TIMEZONE_MESSAGE = "That is not a timezone I know."; @@ -144,8 +149,41 @@ export function describeCron(cron: string): string { monthField, dayOfWeekField, ] = fields; + + const wideOpen = + dayOfMonthField === "*" && monthField === "*" && dayOfWeekField === "*"; + + // "*/20 * * * *": a step under the floor's shape but at or above the floor itself, e.g. */15 + // and up. Only when hour/day/month/weekday are all "*" — a step on any other field is stranger + // than this narrow rendering is meant to cover. + if (wideOpen && hourField === "*") { + const stepMatch = /^\*\/(\d{1,2})$/.exec(minuteField); + if (stepMatch) { + const step = Number.parseInt(stepMatch[1] as string, 10); + if (Number.isInteger(step) && step > 0) { + return `Every ${step} minutes`; + } + } + } + const minute = parsePlainInt(minuteField, 0, 59); const hour = parsePlainInt(hourField, 0, 23); + + // "0,30 9 * * *": a handful of plain minutes within one hour, every day. Only when the hour is + // a single plain value and day/month/weekday are all "*" — anything with its own weekday or + // day-of-month shape stays out of this narrow rendering. + if (wideOpen && hour !== null && /^\d{1,2}(,\d{1,2})+$/.test(minuteField)) { + const minutes = minuteField + .split(",") + .map((part) => parsePlainInt(part, 0, 59)); + if (minutes.every((value): value is number => value !== null)) { + const times = [...minutes] + .sort((a, b) => a - b) + .map((value) => `${pad2(hour)}:${pad2(value)}`); + return `Every day at ${joinWords(times)}`; + } + } + if (minute === null || hour === null) return cron; const time = `${pad2(hour)}:${pad2(minute)}`; diff --git a/server/tests/routine-schedule.test.ts b/server/tests/routine-schedule.test.ts index 55f53cfb..3117bc05 100644 --- a/server/tests/routine-schedule.test.ts +++ b/server/tests/routine-schedule.test.ts @@ -108,6 +108,14 @@ describe("describeCron", () => { expect(describeCron("0 9 1 * *")).toBe("On the 1st of the month at 09:00"); }); + test("step minutes", () => { + expect(describeCron("*/20 * * * *")).toBe("Every 20 minutes"); + }); + + test("a comma list of plain minutes on one hour", () => { + expect(describeCron("0,30 9 * * *")).toBe("Every day at 09:00 and 09:30"); + }); + test("falls through to the raw expression when it is stranger than words", () => { expect(describeCron("*/7 3,4 * * *")).toBe("*/7 3,4 * * *"); }); diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 2120b9f8..6acc53d1 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -207,6 +207,28 @@ describe("what the schedule has to be", () => { }), ).rejects.toBeInstanceOf(RoutineRefusedError); }); + + test("refuses a blank instruction with the sentence a model is meant to read", async () => { + const { owner, agentId, channel } = await setUp(); + + const failure = await store + .create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: " ", + cron: DAILY, + }) + .then( + () => null, + (error: unknown) => error, + ); + + expect(failure).toBeInstanceOf(RoutineRefusedError); + expect((failure as Error).message).toBe( + "A routine needs an instruction to carry out.", + ); + }); }); /** @@ -359,6 +381,36 @@ describe("resolving the channel to post into", () => { expect(message).toContain(other.name); expect(message).toContain("Say which one."); }); + + test("names at most five channels before it gives up and says 'and others'", async () => { + const owner = await createUser(); + const agentId = await createAgent(owner); + const sharedChannels = []; + for (let index = 0; index < 6; index += 1) { + sharedChannels.push(await createChannel(owner, [agentId])); + } + + const failure = await store + .create({ + ownerUserId: owner.id, + agentId, + instruction: "Which one?", + cron: DAILY, + }) + .then( + () => null, + (error: unknown) => error, + ); + + expect(failure).toBeInstanceOf(RoutineRefusedError); + const message = (failure as Error).message; + expect(message).toContain(", and others"); + // All six channels share one agent, so they share one name; the cap names five of them, not + // six, so that one name appears exactly five times rather than six. + const name = sharedChannels[0]?.name as string; + const occurrences = message.split(name).length - 1; + expect(occurrences).toBe(5); + }); }); describe("reading a person's routines", () => { @@ -385,6 +437,21 @@ describe("reading a person's routines", () => { expect(summary?.lastRun).toBeNull(); }); + test("says a step schedule in words too, never in cron", async () => { + const { owner, agentId, channel } = await setUp(); + await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Check in often.", + cron: "*/20 * * * *", + }); + + const [summary] = await store.listFor(owner.id); + + expect(summary?.schedule).not.toContain("*"); + }); + test("puts the newest first", async () => { const { owner, agentId, channel } = await setUp(); const first = await store.create({ @@ -447,6 +514,28 @@ describe("changing a routine", () => { expect(updated.nextRunAt.getTime()).not.toBe(routine.nextRunAt.getTime()); }); + test("recomputes the next run when only the timezone changes, and refuses an unknown zone", async () => { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Summarise the day.", + cron: DAILY, + }); + + const updated = await store.update(owner.id, routine.id, { + timezone: "Asia/Tokyo", + }); + + expect(updated.timezone).toBe("Asia/Tokyo"); + expect(updated.nextRunAt.getTime()).not.toBe(routine.nextRunAt.getTime()); + + await expect( + store.update(owner.id, routine.id, { timezone: "Mars/Olympus" }), + ).rejects.toBeInstanceOf(RoutineRefusedError); + }); + test("leaves the next run alone when only the instruction changes", async () => { const { owner, agentId, channel } = await setUp(); const routine = await store.create({ @@ -524,13 +613,33 @@ describe("changing a routine", () => { cron: DAILY, }); // A routine switched off for a month has a next run a month in the past. Enabling it must not - // hand the sweep a firing that was due in June. + // hand the sweep a firing that was due in June. Forcing the stamp into the past makes that + // stale state real rather than assumed: without this, `nextRunAt > now` would already have + // been true before the disable, and deleting the recompute branch would leave this green. await store.setEnabled(owner.id, routine.id, false); + const stale = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + await database + .update(routines) + .set({ nextRunAt: stale }) + .where(eq(routines.id, routine.id)); const enabled = await store.update(owner.id, routine.id, { enabled: true }); expect(enabled.enabled).toBe(true); expect(enabled.nextRunAt.getTime()).toBeGreaterThan(Date.now()); + + // The mirror image: an instruction-only change on an already-enabled routine must not recompute, + // even when the stored stamp is one that recomputing would obviously move. + await database + .update(routines) + .set({ nextRunAt: stale }) + .where(eq(routines.id, routine.id)); + + const untouched = await store.update(owner.id, routine.id, { + instruction: "Summarise the week instead.", + }); + + expect(untouched.nextRunAt.getTime()).toBe(stale.getTime()); }); test("removing one is a hard delete, and only once", async () => { From 40e4d685268f07a894efe9fb706ec1d304741c44 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 10:42:48 -0300 Subject: [PATCH 06/45] Promise only the prose the renderer can keep --- server/src/routines/schedule.ts | 9 +++++++- server/src/routines/store.ts | 11 +++++---- server/tests/routine-schedule.test.ts | 23 +++++++++++++++++++ .../tests/routines-store.integration.test.ts | 5 ++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/server/src/routines/schedule.ts b/server/src/routines/schedule.ts index de9989d9..fd4be6c4 100644 --- a/server/src/routines/schedule.ts +++ b/server/src/routines/schedule.ts @@ -160,7 +160,14 @@ export function describeCron(cron: string): string { const stepMatch = /^\*\/(\d{1,2})$/.exec(minuteField); if (stepMatch) { const step = Number.parseInt(stepMatch[1] as string, 10); - if (Number.isInteger(step) && step > 0) { + // A minute step restarts every hour rather than counting continuously from the first + // firing, so it only actually recurs every N minutes when N divides the hour evenly + // (e.g. */20 fires at :00, :20, :40 — a steady 20-minute gap). A step like */40 fires at + // :00 and :40 and then wraps, a 20-minute gap the second time round, so "Every 40 minutes" + // would be false. `step > 1` also rules out */1, which is reachable here even though the + // create-time floor refuses it, and would otherwise render the ungrammatical "Every 1 + // minutes". Anything that fails this falls back to the raw expression. + if (Number.isInteger(step) && step > 1 && 60 % step === 0) { return `Every ${step} minutes`; } } diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index 1002c63e..bb43d3ca 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -84,11 +84,14 @@ export type RoutineSummary = { agentId: string; instruction: string; /** - * The schedule in words — "Weekdays at 09:00" — never the cron expression. + * The schedule in words — "Weekdays at 09:00" — for the shapes `describeCron` recognizes, and + * the raw five-field expression for everything else. Prose where prose is possible; never a + * value the client is expected to parse. * - * The client never parses a schedule. A cron string on the wire is an invitation for the browser - * to grow a second, disagreeing parser, and for the page to render one thing while the sweep does - * another. + * Rendering cron exhaustively in English needs a real cron-description library, which this does + * not carry. So a consumer must treat this as opaque display text: show it, never parse it, and + * never compute a time from it. The authoritative next firing is `nextRunAt`, which the store + * and the sweep both derive from the expression itself. */ schedule: string; timezone: string; diff --git a/server/tests/routine-schedule.test.ts b/server/tests/routine-schedule.test.ts index 3117bc05..bf101fc0 100644 --- a/server/tests/routine-schedule.test.ts +++ b/server/tests/routine-schedule.test.ts @@ -110,6 +110,21 @@ describe("describeCron", () => { test("step minutes", () => { expect(describeCron("*/20 * * * *")).toBe("Every 20 minutes"); + expect(describeCron("*/15 * * * *")).toBe("Every 15 minutes"); + }); + + // Cron minute steps restart every hour, so */40 does not actually fire every 40 minutes: it + // fires at :00 and :40 past each hour, a 20-minute gap the second time round. "Every 40 minutes" + // would be false prose, so a step that does not divide the hour evenly falls back to the raw + // expression rather than claim a cadence the schedule does not keep. + test("a step that does not evenly divide the hour falls through to the raw expression", () => { + expect(describeCron("*/40 * * * *")).toBe("*/40 * * * *"); + }); + + // */1 is reachable through the exported function even though the create-time floor refuses it, + // and "Every 1 minutes" is bad grammar besides. A step of 1 falls back to the raw expression. + test("a step of exactly 1 minute falls through to the raw expression", () => { + expect(describeCron("*/1 * * * *")).toBe("*/1 * * * *"); }); test("a comma list of plain minutes on one hour", () => { @@ -120,6 +135,14 @@ describe("describeCron", () => { expect(describeCron("*/7 3,4 * * *")).toBe("*/7 3,4 * * *"); }); + // A comma list on the hour field (twice a day) has no narrow rendering here, and that is the + // contract, not a gap: cron cannot be rendered exhaustively in prose without a dedicated + // library, so this shape is expected to stay on the raw-expression fallback. If someone later + // teaches `describeCron` this shape, that is a deliberate extension, not a bug fix. + test("falls through to the raw expression for an hour list, by design", () => { + expect(describeCron("0 9,17 * * *")).toBe("0 9,17 * * *"); + }); + test("never throws, even on garbage", () => { expect(describeCron("not a cron expression")).toBe("not a cron expression"); }); diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 6acc53d1..f8253245 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -534,6 +534,11 @@ describe("changing a routine", () => { await expect( store.update(owner.id, routine.id, { timezone: "Mars/Olympus" }), ).rejects.toBeInstanceOf(RoutineRefusedError); + + // The refusal must not have partially applied: the row still carries the timezone from the + // update that succeeded, not the rejected "Mars/Olympus". + const [summary] = await store.listFor(owner.id); + expect(summary?.timezone).toBe("Asia/Tokyo"); }); test("leaves the next run alone when only the instruction changes", async () => { From 7233527c0075baa6d7ed33254fab8e438497edcb Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 10:50:14 -0300 Subject: [PATCH 07/45] Move a routine's clock exactly once, however many sweeps race --- server/src/routines/store.ts | 203 ++++++++++++- .../tests/routines-store.integration.test.ts | 275 +++++++++++++++++- 2 files changed, 466 insertions(+), 12 deletions(-) diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index bb43d3ca..0371314f 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -7,12 +7,12 @@ * never taken from an argument a model supplied. Its failures are one person's failures — a bad cron, * a channel they are not in — and none of them are about two things happening at once. * - * The other half is the sweep's: the ledger read on a clock (which routines are due, advancing the - * next run, opening and closing a run row). Nothing there is asked a question by a person, and - * everything there is about concurrency — several replicas reading the same due row in the same - * second. That is why the halves are worth telling apart: only the second one has anything to do with - * concurrency, and only the second one needs to be reasoned about as a race. It lands in the next - * commit, and this file deliberately does not contain it. + * The other half is the sweep's, and it starts at the marked boundary further down: the ledger read + * on a clock (which routines are due, advancing the next run, opening and closing a run row). + * Nothing there is asked a question by a person, and everything there is about concurrency — several + * replicas reading the same due row in the same second. That is why the halves are worth telling + * apart: only the second one has anything to do with concurrency, and only the second one needs to be + * reasoned about as a race. * * NO CLAIM OR LEASE MACHINERY, EVER. Firing mechanics belong to the shared `work_items` queue in * `server/src/work/queue.ts`, which already owns `for update skip locked`, leases on the database's @@ -20,7 +20,17 @@ * `locked_until` — is exactly the duplicated firing mechanism #235 exists to prevent: two half-right * implementations of the same hard thing, one of which nobody tests. */ -import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; +import { + and, + asc, + desc, + eq, + inArray, + isNotNull, + isNull, + lte, + sql, +} from "drizzle-orm"; import type { Database } from "../db/client"; import { channelAgents, @@ -50,6 +60,17 @@ export class RoutineRefusedError extends Error { export const MAX_ENABLED_ROUTINES = 20; /** Same code-point cap discipline as channel activity. */ export const MAX_INSTRUCTION_CODE_POINTS = 2000; +/** Capped like audit payloads, because a failure is not a promise about length. */ +export const MAX_RUN_ERROR = 400; +/** + * How far back the failure count looks. + * + * Twice what the fatigue rule can act on, so the answer is never truncated where it matters, and + * bounded because an unbounded read of a routine's whole history is the wrong shape for something + * called on every failed firing, for ever: a routine that has failed nightly for a year would make + * the count get slower exactly as the routine got worse. + */ +const FAILURE_SCAN_LIMIT = 20; const NO_SHARED_CHANNEL = "I can only post into a channel you and I are both in."; @@ -133,6 +154,23 @@ export type RoutineStore = { ): Promise; remove(ownerUserId: string, id: string): Promise; setEnabled(ownerUserId: string, id: string, enabled: boolean): Promise; + + /* The sweep's half. Deliberately not owner-scoped — see the boundary comment below. */ + + /** Enabled routines whose next run has arrived, oldest due first. */ + dueRoutines(limit: number): Promise<{ id: string; nextRunAt: Date }[]>; + /** Compare-and-set the clock forward. False means another sweep got there first. */ + advanceNextRun(id: string, from: Date): Promise; + /** Open a run row. Its status stays null until something finishes it. */ + insertRun(routineId: string): Promise<{ runId: string }>; + /** Close a run row with its outcome, and the capped error when there was one. */ + finishRun( + runId: string, + status: RoutineRunOutcome, + error?: string, + ): Promise; + /** How many failures the routine has at the tail, for the fatigue rule to read. */ + consecutiveFailures(routineId: string): Promise; }; type RoutineRow = typeof routines.$inferSelect; @@ -377,9 +415,8 @@ export function createRoutineStore(database: Database): RoutineStore { async listFor(ownerUserId) { /* - * The last-run join reads `routine_runs`, which migration 0020 already created even though - * nothing in THIS commit writes a row to it — the sweep's half does. The join is not dead code; - * it is the half of the page that stays empty until the sweep lands. + * The last-run join reads `routine_runs`, which only the sweep's half of this file writes to: + * the page stays empty here until a routine has actually fired. * * `distinct on (routine_id) ... order by started_at desc` gives one row per routine, the most * recent, in a single index scan of `routine_runs_by_routine_idx`. Reading the runs in a @@ -414,7 +451,15 @@ export function createRoutineStore(database: Database): RoutineStore { }) .from(routineRuns) .where(inArray(routineRuns.routineId, routineIds)) - .orderBy(routineRuns.routineId, desc(routineRuns.startedAt)); + .orderBy( + routineRuns.routineId, + desc(routineRuns.startedAt), + // The id breaks the tie. Two runs of one routine can share a `started_at` — a retry in + // the same instant, a clock with coarse resolution — and without a tiebreak `distinct + // on` picks whichever of them the scan reached first, so the page's "last ran" would + // flip between two rows for no reason a person could see. + desc(routineRuns.id), + ); for (const run of runRows) { lastRuns.set(run.routineId, { status: run.status, @@ -454,5 +499,141 @@ export function createRoutineStore(database: Database): RoutineStore { // rather than having a second, quieter version of those rules. await update(ownerUserId, id, { enabled }); }, + + /* ========================================================================================= + * THE SWEEP'S HALF STARTS HERE. + * + * One store, because there is one table and one owner-guarding discipline to keep straight + * about it; but the two halves read nothing alike. Above, every method takes an owner and + * every failure is one person's — a bad cron, a channel they are not in — and none of them is + * about two things happening at once. Below, nobody asks a question: a sweep on a clock reads + * the ledger, and every method has to be read as a race between replicas. + * + * WHOSE CLOCK. The database's, for every moment these methods compare — the same discipline + * `server/src/work/queue.ts` states in its header, for the same reason: a node ninety seconds + * behind once wrote a lease Postgres considered expired on arrival, and two replicas ran the + * same item. `now()` in SQL, never `Date.now()`. + * ========================================================================================= */ + + async dueRoutines(limit) { + /* + * NOT OWNER-SCOPED, ON PURPOSE. Every other method in this file is guarded by the owner, so + * an unguarded one reads like an oversight; this one is the sweep's read, and the sweep has + * no owner. It runs as no person and looks at every person's routines, which is exactly why + * it returns ids and stamps and nothing a person wrote. + */ + return await database + .select({ id: routines.id, nextRunAt: routines.nextRunAt }) + .from(routines) + .where( + and( + eq(routines.enabled, true), + // The comparison Postgres makes against its own clock. A replica's `Date.now()` here + // would decide what is due from a clock the row was never written by. + lte(routines.nextRunAt, sql`now()`), + ), + ) + // Oldest due first, so a backlog drains in the order it built up. The id is a tiebreak, so + // two routines due in the same instant come back in a fixed order rather than whichever + // the index happened to hand over. + .orderBy(asc(routines.nextRunAt), asc(routines.id)) + .limit(limit); + }, + + async advanceNextRun(id, from) { + const [row] = await database + .select({ cron: routines.cron, timezone: routines.timezone }) + .from(routines) + .where(eq(routines.id, id)) + .limit(1); + if (!row) return false; + + const next = nextRunFor(row.cron, row.timezone, from); + + /* + * THE COMPARE-AND-SET IS THE WHOLE MECHANISM. `where next_run_at = from` means the row only + * moves for the sweep that read that exact stamp: however many replicas see the same due + * routine in the same second, exactly one update matches and the rest change nothing. False + * means another sweep won, which is fine either way — the firing still happens once. + * + * And it happens BEFORE the run is offered. A crash between advancing and running skips that + * firing; advancing afterwards would mean a crash re-offers it, and a routine that spends + * money or sends mail would double-fire. A skipped firing is recoverable by waiting for the + * next one; a doubled one is not. + */ + const moved = await database + .update(routines) + .set({ nextRunAt: next, lastRunAt: from, updatedAt: sql`now()` }) + .where(and(eq(routines.id, id), eq(routines.nextRunAt, from))) + .returning({ id: routines.id }); + return moved.length > 0; + }, + + async insertRun(routineId) { + const runId = `routine_run_${crypto.randomUUID()}`; + // `startedAt` defaults to the database's now, and `status` stays null: null is the in-flight + // state, which is the reason that column is nullable rather than defaulted to something. + const [row] = await database + .insert(routineRuns) + .values({ id: runId, routineId }) + .returning({ id: routineRuns.id }); + if (!row) throw new Error("inserting a routine run returned no row"); + return { runId: row.id }; + }, + + async finishRun(runId, status, error) { + await database + .update(routineRuns) + .set({ + status, + // The database's clock closes the row, the same as it opened it. + finishedAt: sql`now()`, + // Left alone rather than nulled when there was no error, so finishing a run twice cannot + // erase what the first finish recorded. + ...(error === undefined + ? {} + : { error: error.slice(0, MAX_RUN_ERROR) }), + }) + .where(eq(routineRuns.id, runId)); + }, + + async consecutiveFailures(routineId) { + /* + * Bounded, then counted here. The bound is the point: this is read on every failed firing, + * for ever, and `select ... where routine_id = $1` with no limit gets slower for exactly the + * routines that fail most. The rule only acts on the first handful, so reading twice that + * many and stopping is the whole answer. + * + * Finished runs only — an in-flight run has no outcome yet and must not end the streak. + */ + const rows = await database + .select({ status: routineRuns.status }) + .from(routineRuns) + .where( + and( + eq(routineRuns.routineId, routineId), + isNotNull(routineRuns.status), + ), + ) + .orderBy(desc(routineRuns.startedAt), desc(routineRuns.id)) + .limit(FAILURE_SCAN_LIMIT); + + let failures = 0; + for (const run of rows) { + if (run.status === "failed") { + failures += 1; + continue; + } + /* + * A SKIP IS NOT A FAILURE, AND DOES NOT BREAK THE STREAK. It means the channel was gone, + * not that the turn failed, so it is not counted; and it does not reset the count either, + * because a routine whose channel flaps would otherwise never reach the fatigue rule — it + * would disable itself over ten missing channels, or never at all. + */ + if (run.status === "skipped") continue; + break; + } + return failures; + }, }; } diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index f8253245..1d34f727 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -11,12 +11,14 @@ import { agents, channels, intelligenceChannelMappings, + routineRuns, routines, users, } from "../src/db/schema"; import { MAX_ENABLED_ROUTINES, MAX_INSTRUCTION_CODE_POINTS, + MAX_RUN_ERROR, RoutineNotFoundError, RoutineRefusedError, createRoutineStore, @@ -433,7 +435,7 @@ describe("reading a person's routines", () => { expect(summary?.channelName).toBe(channel.name); expect(summary?.channelDeleted).toBe(false); expect(summary?.nextRunAt).toBeInstanceOf(Date); - // Nothing writes routine_runs yet; the join is here for the commit that does. + // A routine that has never fired has no run row, so the join has nothing to report. expect(summary?.lastRun).toBeNull(); }); @@ -665,3 +667,274 @@ describe("changing a routine", () => { ); }); }); + +/* + * --------------------------------------------------------------------------------------------- + * The sweep's half. Nothing below is asked a question by a person, and everything below is about + * two things happening at once: several replicas reading the same due row in the same second. + * + * NO CLAIM OR LEASE SEMANTICS ARE TESTED HERE. Firing mechanics live in the shared `work_items` + * queue, and `server/tests/work-queue.integration.test.ts` owns `for update skip locked`, leases + * on the database's clock, and the attempt count. What this file tests is the routines table's own + * compare-and-set on `next_run_at`, which is a different guarantee: the clock moves once. + */ + +/** + * The create path always computes a future stamp, so a due-in-the-past row has to be written + * directly — the same direct write the staleness test above uses. The stamps are deliberately + * ancient: `dueRoutines` is not owner-scoped, so a test that asserts on ordering has to be sure + * its own rows sort ahead of anything else in the database. + */ +async function makeDueAt(routineId: string, nextRunAt: Date): Promise { + await database + .update(routines) + .set({ nextRunAt }) + .where(eq(routines.id, routineId)); + const [row] = await database + .select({ nextRunAt: routines.nextRunAt }) + .from(routines) + .where(eq(routines.id, routineId)) + .limit(1); + // Read back rather than trusting the Date we wrote: the comparison the CAS makes is Postgres's. + return row?.nextRunAt as Date; +} + +async function readRoutine(routineId: string) { + const [row] = await database + .select() + .from(routines) + .where(eq(routines.id, routineId)) + .limit(1); + return row; +} + +async function makeRoutine(instruction = "Summarise the day.") { + const { owner, agentId, channel } = await setUp(); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction, + cron: DAILY, + }); + return { owner, agentId, channel, routine }; +} + +describe("moving a routine's clock", () => { + test("two sweeps racing on the same stamp advance it exactly once", async () => { + const { routine } = await makeRoutine(); + const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z")); + + const outcomes = await Promise.all([ + store.advanceNextRun(routine.id, from), + store.advanceNextRun(routine.id, from), + ]); + + expect(outcomes.filter((won) => won)).toHaveLength(1); + expect(outcomes.filter((won) => !won)).toHaveLength(1); + + // The loser's re-read sees the advanced value: whichever call lost, the clock has moved on and + // nothing is left holding the stamp it was asked to move from. + const after = await readRoutine(routine.id); + expect(after?.nextRunAt.getTime()).toBeGreaterThan(from.getTime()); + }); + + test("a stale `from` returns false and moves nothing", async () => { + const { routine } = await makeRoutine(); + const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z")); + + const moved = await store.advanceNextRun( + routine.id, + new Date("2001-03-03T09:00:00Z"), + ); + + expect(moved).toBe(false); + const after = await readRoutine(routine.id); + expect(after?.nextRunAt.getTime()).toBe(from.getTime()); + expect(after?.lastRunAt).toBeNull(); + }); + + test("advancing stamps last_run_at with the `from` it was given", async () => { + const { routine } = await makeRoutine(); + const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z")); + + expect(await store.advanceNextRun(routine.id, from)).toBe(true); + + const after = await readRoutine(routine.id); + // The moment the routine was due, not the moment the sweep happened to look at it. + expect(after?.lastRunAt?.getTime()).toBe(from.getTime()); + expect(after?.nextRunAt.getTime()).toBeGreaterThan(from.getTime()); + }); +}); + +describe("reading which routines are due", () => { + test("returns the due ones oldest first and respects the limit", async () => { + const { owner, agentId, channel } = await setUp(); + const oldest = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Oldest.", + cron: DAILY, + }); + const middle = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Middle.", + cron: DAILY, + }); + const newest = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Newest.", + cron: DAILY, + }); + await makeDueAt(oldest.id, new Date("2001-01-01T09:00:00Z")); + await makeDueAt(middle.id, new Date("2001-01-02T09:00:00Z")); + await makeDueAt(newest.id, new Date("2001-01-03T09:00:00Z")); + + const all = await store.dueRoutines(10); + expect(all.map((row) => row.id).slice(0, 3)).toEqual([ + oldest.id, + middle.id, + newest.id, + ]); + expect(all[0]?.nextRunAt).toBeInstanceOf(Date); + + const limited = await store.dueRoutines(2); + expect(limited.map((row) => row.id)).toEqual([oldest.id, middle.id]); + }); + + test("excludes a routine that is switched off", async () => { + const { owner, routine } = await makeRoutine(); + await store.setEnabled(owner.id, routine.id, false); + await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z")); + + const due = await store.dueRoutines(50); + expect(due.map((row) => row.id)).not.toContain(routine.id); + }); + + test("excludes a routine whose next run is still ahead", async () => { + const { routine } = await makeRoutine(); + // The create path already put this in the future; the point is that nothing rounds it down. + const due = await store.dueRoutines(50); + expect(due.map((row) => row.id)).not.toContain(routine.id); + + await makeDueAt(routine.id, new Date(Date.now() + 60 * 60 * 1000)); + const stillAhead = await store.dueRoutines(50); + expect(stillAhead.map((row) => row.id)).not.toContain(routine.id); + }); +}); + +describe("opening and closing a run", () => { + test("a finished run stamps finished_at and leaves the error null", async () => { + const { routine } = await makeRoutine(); + + const { runId } = await store.insertRun(routine.id); + const [inFlight] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.id, runId)); + // Null status is the in-flight state, which is why the column is nullable. + expect(inFlight?.status).toBeNull(); + expect(inFlight?.finishedAt).toBeNull(); + expect(inFlight?.startedAt).toBeInstanceOf(Date); + + await store.finishRun(runId, "succeeded"); + + const [finished] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.id, runId)); + expect(finished?.status).toBe("succeeded"); + expect(finished?.finishedAt).toBeInstanceOf(Date); + expect(finished?.error).toBeNull(); + }); + + test("caps a long error rather than storing whatever a failure produced", async () => { + const { routine } = await makeRoutine(); + const { runId } = await store.insertRun(routine.id); + + await store.finishRun(runId, "failed", "x".repeat(600)); + + const [finished] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.id, runId)); + expect(finished?.status).toBe("failed"); + expect(finished?.error).toHaveLength(MAX_RUN_ERROR); + expect(MAX_RUN_ERROR).toBe(400); + }); + + test("the page's last run reads the newest of several", async () => { + const { owner, routine } = await makeRoutine(); + const first = await store.insertRun(routine.id); + await store.finishRun(first.runId, "failed", "the first one"); + const second = await store.insertRun(routine.id); + await store.finishRun(second.runId, "succeeded"); + + const [summary] = await store.listFor(owner.id); + expect(summary?.lastRun?.status).toBe("succeeded"); + expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date); + }); +}); + +/** + * The fatigue rule counts the failures at the tail, and this file pins what a `skipped` run does to + * that tail: A SKIP IS NOT A FAILURE AND DOES NOT BREAK THE STREAK. A skip means the channel was + * gone, not that the turn failed, so it is not counted; and it does not reset the count either, + * because a routine whose channel flaps must not be able to escape the fatigue rule by skipping + * between its failures. + */ +describe("counting the failures at the tail", () => { + async function finish( + routineId: string, + status: "succeeded" | "failed" | "skipped", + ) { + const { runId } = await store.insertRun(routineId); + await store.finishRun( + runId, + status, + status === "failed" ? "it threw" : undefined, + ); + } + + test("counts up, resets on a success, and is not broken by a skip", async () => { + const { routine } = await makeRoutine(); + + expect(await store.consecutiveFailures(routine.id)).toBe(0); + + await finish(routine.id, "succeeded"); + expect(await store.consecutiveFailures(routine.id)).toBe(0); + + await finish(routine.id, "failed"); + expect(await store.consecutiveFailures(routine.id)).toBe(1); + + await finish(routine.id, "failed"); + await finish(routine.id, "failed"); + expect(await store.consecutiveFailures(routine.id)).toBe(3); + + await finish(routine.id, "succeeded"); + expect(await store.consecutiveFailures(routine.id)).toBe(0); + }); + + test("a skip between failures neither counts nor resets", async () => { + const { routine } = await makeRoutine(); + + await finish(routine.id, "failed"); + await finish(routine.id, "skipped"); + await finish(routine.id, "failed"); + + expect(await store.consecutiveFailures(routine.id)).toBe(2); + }); + + test("an in-flight run is not an outcome", async () => { + const { routine } = await makeRoutine(); + await finish(routine.id, "failed"); + await store.insertRun(routine.id); + + expect(await store.consecutiveFailures(routine.id)).toBe(1); + }); +}); From a159419947e8ccfb58523139ef045595c1902a76 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:02:20 -0300 Subject: [PATCH 08/45] Count only the runs the fatigue rule can act on --- server/src/routines/store.ts | 47 +++++++++++------ .../tests/routines-store.integration.test.ts | 52 +++++++++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index 0371314f..f5e50af8 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -29,6 +29,7 @@ import { isNotNull, isNull, lte, + ne, sql, } from "drizzle-orm"; import type { Database } from "../db/client"; @@ -560,6 +561,13 @@ export function createRoutineStore(database: Database): RoutineStore { * firing; advancing afterwards would mean a crash re-offers it, and a routine that spends * money or sends mail would double-fire. A skipped firing is recoverable by waiting for the * next one; a doubled one is not. + * + * The equality is on a stamp that round-trips: every writer of `next_run_at` computes it from + * `nextOccurrence`, which lands on a cron boundary with no sub-second part, and the driver + * binds a `Date` at millisecond precision. The column is microsecond-precision, so + * `next_run_at = now()` written anywhere in SQL would put microseconds in a value this + * comparison reads back truncated — and the CAS would stop matching, silently, for ever. This + * one moment is the database's clock via a value the database gave us, not via `now()`. */ const moved = await database .update(routines) @@ -589,12 +597,20 @@ export function createRoutineStore(database: Database): RoutineStore { // The database's clock closes the row, the same as it opened it. finishedAt: sql`now()`, // Left alone rather than nulled when there was no error, so finishing a run twice cannot - // erase what the first finish recorded. + // erase what the first finish recorded — and the `status is null` guard below is what + // makes that true. ...(error === undefined ? {} - : { error: error.slice(0, MAX_RUN_ERROR) }), + : { + // Measured in code points, like `validInstruction`, so an emoji-bearing error + // cannot be cut mid-surrogate-pair. + error: Array.from(error).slice(0, MAX_RUN_ERROR).join(""), + }), }) - .where(eq(routineRuns.id, runId)); + // A run finishes once. The second call — succeeded, then a downstream throw whose catch + // calls finishRun("failed") — matches no row here and is a silent no-op, rather than + // relabeling what the first finish already recorded. + .where(and(eq(routineRuns.id, runId), isNull(routineRuns.status))); }, async consecutiveFailures(routineId) { @@ -613,6 +629,17 @@ export function createRoutineStore(database: Database): RoutineStore { and( eq(routineRuns.routineId, routineId), isNotNull(routineRuns.status), + /* + * A SKIP IS NOT A FAILURE, AND DOES NOT BREAK THE STREAK. It means the channel was + * gone, not that the turn failed, so it is not counted; and it does not reset the + * count either, because a routine whose channel flaps would otherwise never reach the + * fatigue rule — it would disable itself over ten missing channels, or never at all. + * Excluding it here, rather than reading it and skipping over it below, keeps it from + * consuming a slot in the bounded window: a routine that skips twice for every + * failure must still be able to count past ten failures, not top out around six or + * seven because skips ate two-thirds of the rows the window could hold. + */ + ne(routineRuns.status, "skipped"), ), ) .orderBy(desc(routineRuns.startedAt), desc(routineRuns.id)) @@ -620,18 +647,8 @@ export function createRoutineStore(database: Database): RoutineStore { let failures = 0; for (const run of rows) { - if (run.status === "failed") { - failures += 1; - continue; - } - /* - * A SKIP IS NOT A FAILURE, AND DOES NOT BREAK THE STREAK. It means the channel was gone, - * not that the turn failed, so it is not counted; and it does not reset the count either, - * because a routine whose channel flaps would otherwise never reach the fatigue rule — it - * would disable itself over ten missing channels, or never at all. - */ - if (run.status === "skipped") continue; - break; + if (run.status !== "failed") break; + failures += 1; } return failures; }, diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 1d34f727..e38f4b1d 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -868,6 +868,32 @@ describe("opening and closing a run", () => { expect(MAX_RUN_ERROR).toBe(400); }); + test("finishing a run twice cannot relabel what the first finish recorded", async () => { + const { routine } = await makeRoutine(); + const { runId } = await store.insertRun(routine.id); + + await store.finishRun(runId, "succeeded"); + const [firstFinish] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.id, runId)); + + // The natural next-task implementation: finish "succeeded", then something downstream throws, + // and the catch calls finishRun("failed", ...). Without a finish-once guard this would + // relabel a run that had already succeeded. + await store.finishRun(runId, "failed", "late"); + + const [secondFinish] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.id, runId)); + expect(secondFinish?.status).toBe("succeeded"); + expect(secondFinish?.error).toBeNull(); + expect(secondFinish?.finishedAt?.getTime()).toBe( + firstFinish?.finishedAt?.getTime(), + ); + }); + test("the page's last run reads the newest of several", async () => { const { owner, routine } = await makeRoutine(); const first = await store.insertRun(routine.id); @@ -937,4 +963,30 @@ describe("counting the failures at the tail", () => { expect(await store.consecutiveFailures(routine.id)).toBe(1); }); + + /** + * A routine with a flapping channel skips far more often than it fails. If the bounded window + * is filled with every finished run — skips included — the skips eat slots a failure needed, + * and a routine that has genuinely failed a dozen times in a row can look like it has failed + * only six or seven. That falsifies the fatigue rule for exactly the scenario the skip rule was + * written for: it would never reach the disable threshold. The window has to be filled with the + * rows the fatigue rule can act on, not with every finished row. + */ + test("skips do not dilute the bounded window the fatigue rule reads", async () => { + const { routine } = await makeRoutine(); + + // Twelve failures, each trailed by two skips, and nothing after them: a flapping channel's + // shape. Post-fix, all twelve failures are read (well under the 20-row limit) because skips + // never occupy a slot. Pre-fix, the 20-row window is filled with the interleaved skips too, + // so it only reaches back a handful of failures — short of the >=10 disable threshold. + for (let i = 0; i < 12; i++) { + await finish(routine.id, "failed"); + await finish(routine.id, "skipped"); + await finish(routine.id, "skipped"); + } + + expect(await store.consecutiveFailures(routine.id)).toBeGreaterThanOrEqual( + 10, + ); + }); }); From 5aa3e03b508a0ca858dfd6a29b59439aafd54b70 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:07:32 -0300 Subject: [PATCH 09/45] Put Routines in the catalogue, the first entry that never leaves the building --- app/src/lib/plugins/queries.ts | 5 +-- server/src/plugins/builtin-routines.ts | 26 ++++++++++++++ server/src/plugins/catalogue.ts | 33 +++++++++++++++++ server/src/plugins/store.ts | 20 +++++++++-- server/src/plugins/transport.ts | 36 +++++++++++++++++-- server/tests/plugin-catalogue.test.ts | 49 +++++++++++++++++++++++++- 6 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 server/src/plugins/builtin-routines.ts diff --git a/app/src/lib/plugins/queries.ts b/app/src/lib/plugins/queries.ts index 9530b33b..f74f2a27 100644 --- a/app/src/lib/plugins/queries.ts +++ b/app/src/lib/plugins/queries.ts @@ -85,9 +85,10 @@ export type CatalogueItem = { * * `deployment-bearer` is a token an administrator holds for everybody, and the only one this page * can collect. `user-oauth` is reached as whoever is asking, so each person connects their own - * account and there is no token to type here. + * account and there is no token to type here. `builtin` is a first-party capability that runs + * inside this deployment — there is nothing to connect and nothing to type. */ - auth: "none" | "deployment-bearer" | "user-oauth"; + auth: "none" | "deployment-bearer" | "user-oauth" | "builtin"; /** True for a vendor that gives every customer their own hostname. */ perInstance: boolean; }; diff --git a/server/src/plugins/builtin-routines.ts b/server/src/plugins/builtin-routines.ts new file mode 100644 index 00000000..4d51c901 --- /dev/null +++ b/server/src/plugins/builtin-routines.ts @@ -0,0 +1,26 @@ +import type { McpCallResult, McpTool } from "./mcp"; + +/** + * The builtin transport for Routines — a refusing stub, this commit only. + * + * The four tools (`create_routine`, `update_routine`, `delete_routine`, `list_routines`) arrive in a + * later commit. This stub exists so the closed union in {@link ./transport} is honest per commit: a + * `TransportKind` member with no registry entry would not typecheck, and a registry entry that + * throws at import would break every deployment that merely loads the module graph. Registering a + * transport that refuses every call, instead, keeps the union closed and every deployment booting + * while the actual tools are still being built. + */ + +export const listNeedsCredential = false; + +export async function listTools(): Promise { + return []; +} + +export async function callTool(): Promise { + return { + text: "Routines is not wired up in this build.", + isError: true, + truncated: false, + }; +} diff --git a/server/src/plugins/catalogue.ts b/server/src/plugins/catalogue.ts index f4fd2d3e..47e59e7c 100644 --- a/server/src/plugins/catalogue.ts +++ b/server/src/plugins/catalogue.ts @@ -37,6 +37,11 @@ export type CatalogueAuth = | { kind: "none" } /** One token, held by the deployment, used for everybody. */ | { kind: "deployment-bearer" } + /** + * First-party and in-process. There is no credential, because there is nothing to authenticate + * to: the call runs against this deployment's own tables, as the person whose turn it is. + */ + | { kind: "builtin" } /** * The asker's own grant. The deployment registers an OAuth client; each person consents once and * the call runs on their token, so the vendor decides what comes back. @@ -137,6 +142,12 @@ export type CatalogueEntry = { * `deployment-bearer` therefore has no entry using it. The shape stays because the call path still * needs it: a server an administrator added by URL has no catalogue entry at all, and that is the * branch it falls into. + * + * Routines is the first entry here that is not a remote vendor at all — no host to dial, nothing + * outside this process to trust. It is in the catalogue anyway, on purpose rather than by oversight: + * the catalogue is where a deployment decides which Bots may do what, and a Bot that can schedule its + * own future runs is a capability worth that same deliberate grant, even though there is no vendor on + * the other end of it. */ export const CATALOGUE: readonly CatalogueEntry[] = Object.freeze([ { @@ -254,6 +265,28 @@ export const CATALOGUE: readonly CatalogueEntry[] = Object.freeze([ ]), docsUrl: "https://developers.notion.com/guides/mcp/build-mcp-client", }, + { + key: "routines", + title: "Routines", + vendor: "OpenBot", + summary: + "Standing instructions a Bot runs on a schedule, as whoever scheduled them.", + /* + * First-party and in-process: no host to dial, no credential to hold. In the catalogue anyway, + * because the catalogue is where a deployment decides WHICH Bots may do WHAT — and scheduling + * future work is a capability an administrator should grant as deliberately as a vendor. + */ + host: "builtin://routines", + path: "/", + transport: "builtin-routines", + auth: Object.freeze({ kind: "builtin" }), + writeTools: Object.freeze([ + "create_routine", + "update_routine", + "delete_routine", + ]), + docsUrl: "https://github.com/CopilotKit/OpenBot/blob/main/docs/routines.md", + }, ]); const BY_KEY = new Map(CATALOGUE.map((entry) => [entry.key, entry])); diff --git a/server/src/plugins/store.ts b/server/src/plugins/store.ts index 958c29f9..7b49a897 100644 --- a/server/src/plugins/store.ts +++ b/server/src/plugins/store.ts @@ -291,9 +291,13 @@ const iso = (value: Date | string | null): string | null => * the row a per-person connector exists to be able to trust. * * `deployment` for a shared token; the asker's own id for a server reached as the person asking. + * `builtin` is the third case and the only one with no credential at all — the actor is not whose + * token was used, it is whose rows were touched. */ const reachedAsFor = (entry: CatalogueEntry | null, actorId: string): string => - entry?.auth.kind === "user-oauth" ? actorId : "deployment"; + entry?.auth.kind === "user-oauth" || entry?.auth.kind === "builtin" + ? actorId + : "deployment"; /** * Where this server actually is, when the stored row and the catalogue disagree. @@ -501,7 +505,12 @@ export type PluginStoreOptions = { * reachable, which means the property most worth testing would be the one thing never tested. */ callVendor?: ( - connection: { url: string; token?: string }, + connection: { + url: string; + token?: string; + actorId?: string; + botId?: string; + }, toolName: string, args: Record, ) => Promise<{ text: string; isError: boolean }>; @@ -2802,7 +2811,12 @@ export function createPluginStore(options: PluginStoreOptions) { const { token } = await connectionTokenFor(row, entry, input.actorId); const vendor = injectedVendor ?? transportFor(entry).callTool; const result = await vendor( - { url: effectiveUrl(row, entry), token }, + { + url: effectiveUrl(row, entry), + token, + actorId: input.actorId, + botId: input.botId, + }, toolName, args, ); diff --git a/server/src/plugins/transport.ts b/server/src/plugins/transport.ts index 1a024f09..0918826a 100644 --- a/server/src/plugins/transport.ts +++ b/server/src/plugins/transport.ts @@ -1,4 +1,5 @@ import type { CatalogueEntry } from "./catalogue"; +import * as builtinRoutines from "./builtin-routines"; import * as driveRest from "./google-drive-rest"; import * as mcp from "./mcp"; import type { McpCallResult, McpTool } from "./mcp"; @@ -36,9 +37,37 @@ export type VendorTransport = { * sequence hinted that the middle step was doing no work. */ listNeedsCredential: boolean; - listTools(connection: { url: string; token?: string }): Promise; + listTools(connection: { + url: string; + token?: string; + /** + * Who this call is for, and which Bot is making it. + * + * Ignored by every transport that dials a vendor: MCP and Drive answer to a credential, and who + * holds it is already decided by the time the connection is built. The builtin transport has no + * credential and no vendor — it acts on this deployment's own tables — so the actor is not + * context, it is the authorization, and it refuses without one. A routine is somebody's. + */ + actorId?: string; + /** The Bot the run belongs to. A routine runs as its Bot, which is never a name a model supplies. */ + botId?: string; + }): Promise; callTool( - connection: { url: string; token?: string }, + connection: { + url: string; + token?: string; + /** + * Who this call is for, and which Bot is making it. + * + * Ignored by every transport that dials a vendor: MCP and Drive answer to a credential, and who + * holds it is already decided by the time the connection is built. The builtin transport has no + * credential and no vendor — it acts on this deployment's own tables — so the actor is not + * context, it is the authorization, and it refuses without one. A routine is somebody's. + */ + actorId?: string; + /** The Bot the run belongs to. A routine runs as its Bot, which is never a name a model supplies. */ + botId?: string; + }, toolName: string, args: Record, ): Promise; @@ -50,11 +79,12 @@ export type VendorTransport = { * A closed union rather than a string, so adding one is a change to this file and to the registry * below together. An entry naming a transport that does not exist should not typecheck. */ -export type TransportKind = "mcp" | "google-drive-rest"; +export type TransportKind = "mcp" | "google-drive-rest" | "builtin-routines"; const TRANSPORTS: Record = { mcp, "google-drive-rest": driveRest, + "builtin-routines": builtinRoutines, }; /** diff --git a/server/tests/plugin-catalogue.test.ts b/server/tests/plugin-catalogue.test.ts index 9ba4dfd6..d6b533a4 100644 --- a/server/tests/plugin-catalogue.test.ts +++ b/server/tests/plugin-catalogue.test.ts @@ -92,6 +92,11 @@ describe("which servers this deployment will talk to", () => { // Anchored at both ends or the pattern is decoration. expect(entry.hostPattern?.startsWith("^")).toBe(true); expect(entry.hostPattern?.endsWith("$")).toBe(true); + } else if (entry.auth.kind === "builtin") { + // First-party and in-process: there is no host outside this process to reach, so the + // https requirement below does not apply. Asserted positively instead, so this branch + // cannot quietly become a loophole for a future entry that DOES dial a real host. + expect(entry.host).toBe("builtin://routines"); } else { expect(entry.host.startsWith("https://")).toBe(true); } @@ -105,7 +110,7 @@ describe("whose credential a server uses", () => { // whose, and a reader who guessed would guess the deployment's, which for a user-oauth vendor // is the one answer that breaks the promise the connector exists to keep. for (const entry of CATALOGUE) { - expect(["none", "deployment-bearer", "user-oauth"]).toContain( + expect(["none", "deployment-bearer", "user-oauth", "builtin"]).toContain( entry.auth.kind, ); } @@ -238,6 +243,48 @@ describe("Notion", () => { }); }); +describe("Routines", () => { + const entry = catalogueEntry("routines"); + + test("is in the catalogue and resolves to its own builtin address", () => { + expect(entry).not.toBeNull(); + expect(resolveServerUrl("routines")?.url).toBe("builtin://routines"); + }); + + test("has no credential, because there is nothing to authenticate to", () => { + expect(entry?.auth.kind).toBe("builtin"); + }); + + test("is reached through the builtin transport, not a vendor", () => { + expect(entry?.transport).toBe("builtin-routines"); + }); + + test("pins the exact write list, so a dropped or renamed entry fails here", () => { + expect(entry?.writeTools).toEqual([ + "create_routine", + "update_routine", + "delete_routine", + ]); + }); + + test("classifies its tools the same way every other vendor's are classified", () => { + for (const name of entry?.writeTools ?? []) { + expect(classifyTool(entry, name, true)).toBe("write"); + } + expect(classifyTool(entry, "list_routines", true)).toBe("read"); + // A name nothing here has vouched for is a write, the same as for any other vendor. + expect(classifyTool(entry, "brand-new-tool", false)).toBe("write"); + // Every tool, advertised or not, is a write when the server never said it was advertised. + for (const name of [ + ...(entry?.writeTools ?? []), + "list_routines", + "brand-new-tool", + ]) { + expect(classifyTool(entry, name, false)).toBe("write"); + } + }); +}); + describe("what a tool does", () => { const drive = catalogueEntry("google-drive")!; From 2cf7d358b071e930f2849c2ced94764cb4ee4779 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:14:23 -0300 Subject: [PATCH 10/45] Say what a built-in connector needs, which is nothing --- app/src/routes/_authed/admin/plugins/$key.tsx | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/app/src/routes/_authed/admin/plugins/$key.tsx b/app/src/routes/_authed/admin/plugins/$key.tsx index 6c6f96d4..5740bd78 100644 --- a/app/src/routes/_authed/admin/plugins/$key.tsx +++ b/app/src/routes/_authed/admin/plugins/$key.tsx @@ -320,22 +320,51 @@ function RouteComponent() { description={ auth === "user-oauth" ? "This vendor answers as whoever is asking. The deployment registers an OAuth client, and each person connects their own account, so a Bot only ever sees what that person can see." - : "What this deployment presents to the vendor. One credential, used for everybody." + : auth === "builtin" + ? "Built into this deployment. There is no vendor to reach and no credential to hold — a call runs as whoever asked." + : "What this deployment presents to the vendor. One credential, used for everybody." } title="Connection" > {/* - * Rows that DO something, and nothing else — with one admitted exception. The layout + * Rows that DO something, and nothing else — with two admitted exceptions. The layout * skill's third row kind — a value with no chevron and nothing to click — earns its * place on a screen full of them, but among four actionable rows a dead one reads as a * control that has stopped working. The redirect URI is prose under the card instead. * - * The exception is the OAuth client row for a vendor with a dynamic client: there is a - * real fact to state — this deployment registers itself, nobody configures it — right - * where the actionable client row would otherwise sit. Leaving that slot empty would - * read as a missing setup step, not as nothing to do. + * The first exception is the OAuth client row for a vendor with a dynamic client: there + * is a real fact to state — this deployment registers itself, nobody configures it — + * right where the actionable client row would otherwise sit. Leaving that slot empty + * would read as a missing setup step, not as nothing to do. + * + * The second is the whole Connection card for a builtin server: there is nothing to + * configure, but a card of nothing under a "Connection" heading reads as a missing setup + * step rather than as the answer. The row states that plainly instead of leaving the + * card empty — and being first, it also gives the docsUrl row below something other than + * the card's own top border to sit its leading separator against. */} + {auth === "builtin" ? ( + /* + * Nothing to click. A builtin server runs inside this deployment, on tables it + * already owns — there is no vendor to authenticate to and no credential to store. + */ + + + Connection + + Nothing to configure. These tools run inside this + deployment, on the tables it already owns. + + + + + Built in + + + + ) : null} + {auth === "deployment-bearer" ? ( Date: Wed, 26 Aug 2026 11:24:11 -0300 Subject: [PATCH 11/45] Let a Bot keep, change and drop routines by calling tools --- server/src/index.ts | 15 + server/src/plugins/builtin-routines.ts | 452 ++++++++++++++++++++++++- server/src/plugins/catalogue.ts | 3 +- server/tests/builtin-routines.test.ts | 394 +++++++++++++++++++++ 4 files changed, 849 insertions(+), 15 deletions(-) create mode 100644 server/tests/builtin-routines.test.ts diff --git a/server/src/index.ts b/server/src/index.ts index 4bef6cc0..6347288a 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -47,9 +47,11 @@ import { import { createAttentionStore } from "./attention/store"; import { createDatabase } from "./db/client"; import { createPeopleStore } from "./people/store"; +import { useRoutineTools } from "./plugins/builtin-routines"; import { redirectUriFor } from "./plugins/oauth"; import { createPluginStore } from "./plugins/store"; import { grantedSkills, grantedTools } from "./plugins/tools"; +import { createRoutineStore } from "./routines/store"; import { createIntentRouter } from "./routing/classify"; import { createModelCompleter } from "./routing/model"; import { @@ -291,6 +293,19 @@ const pluginStore = createPluginStore({ redirectUri: config.publicUrl ? redirectUriFor(config.publicUrl) : undefined, }); +/** + * Routines, and the one moment its tools are told what to act on. + * + * The builtin transport is reached as a MODULE — `transportFor` maps a kind to one — so there is no + * constructor to hand a store to and no request-time seam either: the transport registry is built at + * import time, long before there is a database. So the store is installed here, once, from the place + * that already owns building stores. Without this call the four tools are advertised and every one of + * them refuses, which is the honest behaviour for a deployment that never wired it, and would be a + * silent outage for this one. + */ +const routineStore = createRoutineStore(database); +useRoutineTools(routineStore); + void recordAuditEvent(bootAuditStore, { eventType: "computer.policy_loaded", targetType: "policy", diff --git a/server/src/plugins/builtin-routines.ts b/server/src/plugins/builtin-routines.ts index 4d51c901..cd225bcc 100644 --- a/server/src/plugins/builtin-routines.ts +++ b/server/src/plugins/builtin-routines.ts @@ -1,26 +1,450 @@ -import type { McpCallResult, McpTool } from "./mcp"; +import { + RoutineNotFoundError, + RoutineRefusedError, + type Routine, + type RoutinePatch, + type RoutineSummary, +} from "../routines/store"; +import { MAX_RESULT_CHARS, type McpCallResult, type McpTool } from "./mcp"; /** - * The builtin transport for Routines — a refusing stub, this commit only. + * The builtin transport for Routines: a Bot keeping, changing and dropping a person's standing + * instructions, without leaving the building. * - * The four tools (`create_routine`, `update_routine`, `delete_routine`, `list_routines`) arrive in a - * later commit. This stub exists so the closed union in {@link ./transport} is honest per commit: a - * `TransportKind` member with no registry entry would not typecheck, and a registry entry that - * throws at import would break every deployment that merely loads the module graph. Registering a - * transport that refuses every call, instead, keeps the union closed and every deployment booting - * while the actual tools are still being built. + * WHAT MAKES THIS DIFFERENT FROM EVERY OTHER TRANSPORT. There is no vendor. `mcp` dials somebody + * else's server and `google-drive-rest` dials Google; both answer to a credential, and who holds it + * is settled before the connection is built. Here there is no credential at all — the call runs + * against this deployment's own tables — so the ACTOR is not context, it is the authorization. That + * is why {@link callTool} refuses a run that is not attributed to a person, and why the owner and + * the Bot are read off the connection and never out of the arguments. + * + * It implements the same interface as the other two, as module-level exports, because that is the + * shape {@link ./transport} resolves: a `TransportKind` maps to a MODULE. Which is also why the store + * arrives through {@link useRoutineTools} rather than through a constructor — see the comment there. */ -export const listNeedsCredential = false; +/** No fetch, no vendor, no third party: the store's own refusal cap, in code points. */ +const MAX_FAILURE_CODE_POINTS = 400; + +/** + * What the tools act on. + * + * A narrow projection of `RoutineStore` — the five sweep methods and `setEnabled` are deliberately + * absent, because nothing a model calls has any business advancing a clock or opening a run row. A + * store satisfies this structurally, so wiring it is one call and no adapter. + */ +export type RoutineTools = { + create(input: { + ownerUserId: string; + agentId: string; + channelId?: string; + instruction: string; + cron: string; + timezone?: string; + }): Promise; + listFor(ownerUserId: string): Promise; + update( + ownerUserId: string, + id: string, + patch: RoutinePatch, + ): Promise; + remove(ownerUserId: string, id: string): Promise; +}; + +let installed: RoutineTools | null = null; + +/** + * Hand this module the store, once, from the place that builds stores. + * + * A module-level binding rather than a constructor argument, because `transportFor` resolves a kind + * to a MODULE and there is no seam to pass anything through: the registry is built at import time, + * long before `index.ts` has a database. Mutable and set once, so a test can install a recording + * stub and the real boot can install the store, and neither has to know about the other. + * + * `null` is a supported argument, and not only for symmetry: the suite is one process, so a test + * that installs a stub has to be able to take it back out again. + */ +export function useRoutineTools(tools: RoutineTools | null): void { + installed = tools; +} + +/** + * The four tools, as the same shape a server would have answered `tools/list` with. + * + * THE DESCRIPTIONS ARE THE PRODUCT HERE. Every other field is mechanical; these are the only thing + * standing between "remind me at nine" and a routine that fires at nine UTC for ever, or every + * minute, or into somebody else's channel. A tool a model misuses is not a failed call — it is a + * wrong schedule that keeps being wrong on a timer. So the cron contract is written out in full, + * with worked examples, rather than left to a field named `cron`. + */ +const TOOLS: readonly McpTool[] = Object.freeze([ + { + name: "create_routine", + description: [ + "Set up a standing instruction that you carry out on a schedule for the person you are talking to.", + "", + "The schedule is a five-field cron expression, in the order `minute hour day-of-month month day-of-week`.", + "`0 9 * * 1-5` is weekdays at nine in the morning. `30 18 * * *` is every day at half past six in the", + "evening. `0 9 1 * *` is the first of the month at nine. A routine may run at most every 15 minutes;", + "anything more frequent is refused.", + "", + "Pass the person's IANA timezone (`America/New_York`, `Europe/Madrid`) whenever they speak in local", + "time — the default is UTC and there is no deployment timezone to inherit, so a schedule set without", + "one fires at a UTC hour rather than at theirs.", + "", + "The routine posts its reply into a channel. Leave `channelId` out and it goes to the one channel you", + "share with this person; when you share more than one you will be told so and asked to pick, and the", + "answer to that is to ask them which.", + "", + "It takes no owner and no Bot. The routine belongs to the person you are talking to and runs as you;", + "both are taken from the run itself, and there is no field for either.", + ].join("\n"), + inputSchema: { + type: "object", + properties: { + instruction: { + type: "string", + description: + "What to do each time it runs, written as an instruction to yourself.", + }, + cron: { + type: "string", + description: + "Five fields: `minute hour day-of-month month day-of-week`. At most every 15 minutes.", + }, + timezone: { + type: "string", + description: + "The person's IANA timezone, such as `Europe/Madrid`. Omit only when they meant UTC; the default is UTC.", + }, + channelId: { + type: "string", + description: + "Where the reply is posted. Omit for the one channel you share with this person.", + }, + }, + required: ["instruction", "cron"], + }, + }, + { + name: "list_routines", + description: [ + "List the standing instructions this person has: each one's id, its schedule in words, its timezone,", + "the channel it posts into, when it next runs and how the last run went.", + "", + "Take the id from here before changing or deleting one. It lists only this person's own routines.", + ].join("\n"), + inputSchema: { type: "object", properties: {} }, + }, + { + name: "update_routine", + description: [ + "Change one of this person's routines: its instruction, its schedule, its timezone, the channel it", + "posts into, or whether it is switched on.", + "", + "Give the id from `list_routines` and only the fields that change; anything left out stays as it is. A", + "new `cron` follows exactly the same five-field rules as `create_routine`, and a routine switched back", + "on is scheduled from now rather than from wherever it left off.", + ].join("\n"), + inputSchema: { + type: "object", + properties: { + id: { + type: "string", + description: "The routine's id, from `list_routines`.", + }, + instruction: { + type: "string", + description: "A new instruction, replacing the old one entirely.", + }, + cron: { + type: "string", + description: + "A new schedule. Five fields: `minute hour day-of-month month day-of-week`. At most every 15 minutes.", + }, + timezone: { + type: "string", + description: "A new IANA timezone, such as `America/New_York`.", + }, + channelId: { + type: "string", + description: + "A different channel to post into. Must be one you and this person share.", + }, + enabled: { + type: "boolean", + description: + "False switches it off without deleting it; true switches it back on.", + }, + }, + required: ["id"], + }, + }, + { + name: "delete_routine", + description: [ + "Delete one of this person's routines, by the id from `list_routines`. It stops for good and there is", + "nothing to undo. When they might want it back, switch it off with `update_routine` instead.", + ].join("\n"), + inputSchema: { + type: "object", + properties: { + id: { + type: "string", + description: "The routine's id, from `list_routines`.", + }, + }, + required: ["id"], + }, + }, +]); + +/** + * Who this call is for, and which Bot is making it. + * + * `url` and `token` are the shared connection shape and are both unused here: there is no host to + * dial and no credential to send. The two that matter are the two a model never supplies. + */ +type Connection = { + url: string; + token?: string; + actorId?: string; + botId?: string; +}; +/** + * The list is static, actor-free and needs no store. + * + * The four definitions are schemas in this file: nothing to discover, nobody to ask, no credential to + * hold. It takes no argument at all, which is honest about that — and load-bearing, because the only + * call site is `refreshTools`, which passes `{url, token}` and never an actor. A list that insisted + * on one would store zero tools and Routines would advertise nothing to anybody. The refusal that + * belongs to the actor is {@link callTool}'s, where the actor is what authorizes the change. + */ export async function listTools(): Promise { - return []; + return TOOLS.map((tool) => ({ ...tool })); } -export async function callTool(): Promise { +export const listNeedsCredential = false; + +const failure = (message: string): McpCallResult => ({ + text: message, + isError: true, + truncated: false, +}); + +/** Success as a result, with the same visible cap the vendor transports use. */ +function asResult(text: string): McpCallResult { + if (text.length <= MAX_RESULT_CHARS) { + return { text, isError: false, truncated: false }; + } return { - text: "Routines is not wired up in this build.", - isError: true, - truncated: false, + text: `${text.slice(0, MAX_RESULT_CHARS)}\n\n[truncated: the answer was ${text.length} characters]`, + isError: false, + truncated: true, }; } + +/** A string argument that was actually given, or nothing. Blank is not a value. */ +function stringArg( + args: Record, + key: string, +): string | undefined { + const value = args[key]; + return typeof value === "string" && value.trim() !== "" ? value : undefined; +} + +/** The target channel as something a person would recognise. A broken one is named as broken. */ +function channelOf(summary: RoutineSummary): string { + if (summary.channelDeleted) { + return `${summary.channelName ?? "a channel"} (which no longer exists, so it cannot post)`; + } + return summary.channelName ?? summary.channelId; +} + +/** The last firing, in the plainest true words. Null status with a null finish means in flight. */ +function lastRunOf(summary: RoutineSummary): string { + const run = summary.lastRun; + if (!run) return "never run"; + if (!run.finishedAt) return "running now"; + return `last ran ${run.finishedAt.toISOString()}${run.status ? ` (${run.status})` : ""}`; +} + +/** + * One routine in words. + * + * `schedule` is OPAQUE DISPLAY TEXT from the store — prose for the shapes it can render, the raw + * expression otherwise. Shown verbatim either way, never parsed and never used to work out a time: + * `nextRunAt` is the authoritative firing, and it comes from the store too. The whole point of + * answering in words is that the model confirms a schedule in language a person can check, rather + * than reading five fields back to somebody who did not write them. + */ +function inWords(summary: RoutineSummary): string { + const parts = [ + `${summary.schedule} (${summary.timezone})`, + `in ${channelOf(summary)}`, + `next ${summary.nextRunAt.toISOString()}`, + lastRunOf(summary), + // The model needs this to change or delete the routine later, and it cannot derive it. + `id: ${summary.id}`, + ]; + if (!summary.enabled) parts.push("switched off"); + return `"${summary.instruction}" — ${parts.join(" · ")}`; +} + +/** + * The routine that was just written, read back as a summary so it can be described. + * + * `create` and `update` return a `Routine`, which carries a cron expression and a channel id: the + * two things this answer must not be. The summary carries the words and the channel's name, so the + * confirmation is a sentence rather than a row. Absent only if it vanished between the two calls, + * where naming the id is the whole of what is still true. + */ +async function describeWritten( + tools: RoutineTools, + routine: Routine, + opening: string, +): Promise { + const summaries = await tools.listFor(routine.ownerUserId); + const summary = summaries.find((candidate) => candidate.id === routine.id); + return asResult( + summary + ? `${opening} ${inWords(summary)}` + : `${opening} Its id is ${routine.id}.`, + ); +} + +/** + * Call one tool. + * + * WHOSE ROUTINE IT IS COMES FROM THE CONNECTION, NEVER FROM `args`. A model that could name another + * Bot's id could schedule work as another Bot; a model that could name another owner could schedule + * work as another person, into a channel it was never in, on a timer nobody else can see. So + * `ownerUserId` is `connection.actorId` and `agentId` is `connection.botId`, and an `ownerUserId` or + * `agentId` in the arguments is simply never read — there is no field for either in the schemas + * above, and a model that invents one is ignored rather than trusted. + * + * Nothing thrown escapes: a refusal from the store is carried through verbatim, because its sentence + * is the one a model can act on ("at most every 15 minutes" proposes a fix; "invalid cron" does + * not). These come back as `isError` results rather than as throws, which is what the vendor + * transports do and what `plugins/tools.ts` expects — it prefixes them with "The vendor reported an + * error: " and the sentence survives intact, which is the part that matters. + */ +export async function callTool( + connection: Connection, + toolName: string, + args: Record, +): Promise { + const ownerUserId = connection.actorId?.trim(); + if (!ownerUserId) { + return failure( + "A routine belongs to somebody, and this run is not attributed to anybody.", + ); + } + const agentId = connection.botId?.trim(); + if (!agentId) { + return failure("A routine runs as a Bot, and this run does not name one."); + } + const tools = installed; + if (!tools) { + return failure("Routines is not available in this deployment."); + } + + try { + if (toolName === "create_routine") { + const instruction = stringArg(args, "instruction"); + if (!instruction) { + return failure("A routine needs an instruction to carry out."); + } + const cron = stringArg(args, "cron"); + if (!cron) { + return failure( + "A routine needs a schedule: five cron fields, `minute hour day-of-month month day-of-week`.", + ); + } + const routine = await tools.create({ + ownerUserId, + agentId, + instruction, + cron, + // Absent means absent. The store owns the UTC default, so guessing a zone here would put a + // zone nobody chose on a routine that then fires at the wrong hour for ever. + timezone: stringArg(args, "timezone"), + channelId: stringArg(args, "channelId"), + }); + return await describeWritten(tools, routine, "That routine is set:"); + } + + if (toolName === "list_routines") { + const summaries = await tools.listFor(ownerUserId); + if (summaries.length === 0) { + // Said in words, not returned as an empty string: an empty result reads to a model as "the + // tool had nothing to say" and gets filled in from memory. + return asResult("You have no routines set up."); + } + return asResult(summaries.map((one) => `- ${inWords(one)}`).join("\n")); + } + + if (toolName === "update_routine") { + const id = stringArg(args, "id"); + if (!id) { + return failure( + "Say which routine to change, by the id from list_routines.", + ); + } + const patch: RoutinePatch = {}; + const instruction = stringArg(args, "instruction"); + if (instruction !== undefined) patch.instruction = instruction; + const cron = stringArg(args, "cron"); + if (cron !== undefined) patch.cron = cron; + const timezone = stringArg(args, "timezone"); + if (timezone !== undefined) patch.timezone = timezone; + const channelId = stringArg(args, "channelId"); + if (channelId !== undefined) patch.channelId = channelId; + if (typeof args.enabled === "boolean") patch.enabled = args.enabled; + + // An empty patch is not a change, and sending it would answer "changed" having changed + // nothing — which a model will report to somebody as done. + if (Object.keys(patch).length === 0) { + return failure("Say what to change about that routine."); + } + + const routine = await tools.update(ownerUserId, id, patch); + return await describeWritten(tools, routine, "That routine now reads:"); + } + + if (toolName === "delete_routine") { + const id = stringArg(args, "id"); + if (!id) { + return failure( + "Say which routine to delete, by the id from list_routines.", + ); + } + await tools.remove(ownerUserId, id); + return asResult("That routine is deleted. It will not run again."); + } + + return failure( + `${toolName} is not a tool Routines implements. The stored tool list is out of date; refresh it on the Plugins page.`, + ); + } catch (error) { + /* + * The store's sentence, unchanged and unprefixed. It is written for a model to act on — the + * frequency floor, the channel it is not in, the several channels it must choose between — and + * rewording it here would turn an actionable refusal into a vague one. + */ + if (error instanceof RoutineRefusedError) return failure(error.message); + // A routine that is not yours is a routine that does not exist, which is the store's rule; this + // is the same fact said to somebody who is holding an id. + if (error instanceof RoutineNotFoundError) { + return failure("There is no routine of yours with that id."); + } + // Anything else is a bug or a broken database, and it still has to come back as a sentence + // rather than as a thrown error mid-turn. Capped in code points, like the store caps a run's + // error, so a message carrying an emoji cannot be cut mid-surrogate-pair. + const message = error instanceof Error ? error.message : String(error); + return failure( + Array.from(message).slice(0, MAX_FAILURE_CODE_POINTS).join(""), + ); + } +} diff --git a/server/src/plugins/catalogue.ts b/server/src/plugins/catalogue.ts index 47e59e7c..b8445bfc 100644 --- a/server/src/plugins/catalogue.ts +++ b/server/src/plugins/catalogue.ts @@ -104,7 +104,8 @@ export type CatalogueEntry = { * * `deployment-bearer` is a token an administrator holds on behalf of everybody. `user-oauth` is * the person's own grant, where the deployment holds only the OAuth client and each person - * consents for themselves. + * consents for themselves. `builtin` is neither: there is nothing to authenticate to, because the + * call runs in this process against this deployment's own tables as the person whose turn it is. */ auth: CatalogueAuth; /** diff --git a/server/tests/builtin-routines.test.ts b/server/tests/builtin-routines.test.ts new file mode 100644 index 00000000..56d4f245 --- /dev/null +++ b/server/tests/builtin-routines.test.ts @@ -0,0 +1,394 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + callTool, + listTools, + useRoutineTools, + type RoutineTools, +} from "../src/plugins/builtin-routines"; +import { + RoutineNotFoundError, + RoutineRefusedError, + type Routine, + type RoutinePatch, + type RoutineSummary, +} from "../src/routines/store"; + +/** + * The builtin Routines transport, asserted without a database. + * + * What is under test is the boundary, not the store: which method a tool name reaches, whose id the + * call is attributed to, and what a refusal reads as. A recording stub is installed through + * {@link useRoutineTools}, which is the only seam the module has — `transportFor` resolves a kind to + * a MODULE, so there is no constructor to pass a store to. + * + * The security property this file exists for is the attribution one: the owner comes from the + * connection's actor and the Bot from the connection's Bot, never from the arguments a model + * produced. A model that could name an owner could schedule work as somebody else. + */ + +const CONNECTION = { + url: "builtin://routines/", + actorId: "user_asker", + botId: "bot_helper", +}; + +const ROUTINE: Routine = { + id: "routine_1", + ownerUserId: "user_asker", + agentId: "bot_helper", + channelId: "channel_1", + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + timezone: "Europe/Madrid", + enabled: true, + nextRunAt: new Date("2026-01-05T08:00:00.000Z"), + lastRunAt: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), +}; + +const SUMMARY: RoutineSummary = { + id: "routine_1", + agentId: "bot_helper", + instruction: "Post the standup summary.", + schedule: "Weekdays at 09:00", + timezone: "Europe/Madrid", + enabled: true, + nextRunAt: new Date("2026-01-05T08:00:00.000Z"), + channelId: "channel_1", + channelName: "Standup", + channelDeleted: false, + lastRun: { + status: "succeeded", + finishedAt: new Date("2026-01-02T08:00:03.000Z"), + }, +}; + +type Recorded = + | { method: "create"; input: Parameters[0] } + | { method: "listFor"; ownerUserId: string } + | { + method: "update"; + ownerUserId: string; + id: string; + patch: RoutinePatch; + } + | { method: "remove"; ownerUserId: string; id: string }; + +/** Installs a stub that records every call, and answers with the fixtures above. */ +function recordingTools(overrides: Partial = {}): Recorded[] { + const calls: Recorded[] = []; + useRoutineTools({ + async create(input) { + calls.push({ method: "create", input }); + return ROUTINE; + }, + async listFor(ownerUserId) { + calls.push({ method: "listFor", ownerUserId }); + return [SUMMARY]; + }, + async update(ownerUserId, id, patch) { + calls.push({ method: "update", ownerUserId, id, patch }); + return ROUTINE; + }, + async remove(ownerUserId, id) { + calls.push({ method: "remove", ownerUserId, id }); + }, + ...overrides, + }); + return calls; +} + +// The binding is module-level and the suite is one process, so a stub left installed here would be +// the store some other file's test unexpectedly reaches. +afterEach(() => { + useRoutineTools(null); +}); + +describe("the tool list", () => { + test("is the four routine tools, named exactly", async () => { + const tools = await listTools(); + expect(tools.map((tool) => tool.name)).toEqual([ + "create_routine", + "list_routines", + "update_routine", + "delete_routine", + ]); + for (const tool of tools) { + expect(tool.description.length).toBeGreaterThan(0); + expect(tool.inputSchema).toBeDefined(); + } + }); + + test("carries the cron contract in create_routine's description", async () => { + const tools = await listTools(); + const create = tools.find((tool) => tool.name === "create_routine"); + expect(create).toBeDefined(); + const description = create?.description ?? ""; + expect(description).toContain("15"); + expect(description).toContain("timezone"); + expect(description).toContain("minute hour day-of-month month day-of-week"); + expect(description).toContain("0 9 * * 1-5"); + }); + + test("needs no actor, no arguments and no store", async () => { + // The only call site is `refreshTools`, which passes `{url, token}` and never an actor. A list + // that refused without one would store zero tools and Routines would advertise nothing. + useRoutineTools(null); + const tools = await listTools(); + expect(tools).toHaveLength(4); + }); +}); + +describe("dispatch", () => { + test("create_routine reaches create, attributed to the connection", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "create_routine", { + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + timezone: "Europe/Madrid", + channelId: "channel_1", + }); + + expect(result.isError).toBe(false); + const created = calls.find((call) => call.method === "create"); + expect(created).toBeDefined(); + expect(created).toMatchObject({ + method: "create", + input: { + ownerUserId: "user_asker", + agentId: "bot_helper", + channelId: "channel_1", + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + timezone: "Europe/Madrid", + }, + }); + }); + + test("create_routine answers in words, not in cron", async () => { + recordingTools(); + const result = await callTool(CONNECTION, "create_routine", { + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + }); + + expect(result.isError).toBe(false); + expect(result.text).toContain("Weekdays at 09:00"); + expect(result.text).toContain("Europe/Madrid"); + expect(result.text).toContain("Standup"); + expect(result.text).toContain("routine_1"); + expect(result.text).not.toContain("0 9 * * 1-5"); + }); + + test("create_routine without a timezone passes none, inventing nothing", async () => { + const calls = recordingTools(); + await callTool(CONNECTION, "create_routine", { + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + }); + + const created = calls.find((call) => call.method === "create"); + // The STORE owns the UTC default. A zone guessed here would be a zone nobody chose. + expect(created?.method === "create" && created.input.timezone).toBe( + undefined, + ); + }); + + test("list_routines reaches listFor, and renders id, words, channel and last run", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "list_routines", {}); + + expect(result.isError).toBe(false); + expect(calls).toContainEqual({ + method: "listFor", + ownerUserId: "user_asker", + }); + expect(result.text).toContain("routine_1"); + expect(result.text).toContain("Weekdays at 09:00"); + expect(result.text).toContain("Standup"); + expect(result.text).toContain("succeeded"); + }); + + test("update_routine reaches update with only the fields given", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "update_routine", { + id: "routine_1", + cron: "30 18 * * *", + enabled: false, + }); + + expect(result.isError).toBe(false); + expect(calls).toContainEqual({ + method: "update", + ownerUserId: "user_asker", + id: "routine_1", + patch: { cron: "30 18 * * *", enabled: false }, + }); + }); + + test("delete_routine reaches remove", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "delete_routine", { + id: "routine_1", + }); + + expect(result.isError).toBe(false); + expect(calls).toContainEqual({ + method: "remove", + ownerUserId: "user_asker", + id: "routine_1", + }); + }); + + test("an unknown tool refuses without touching the store", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "pause_routine", { + id: "routine_1", + }); + + expect(result.isError).toBe(true); + expect(result.text).toContain("pause_routine"); + expect(calls).toHaveLength(0); + }); +}); + +describe("attribution", () => { + test("an owner named in the arguments is ignored", async () => { + const calls = recordingTools(); + await callTool(CONNECTION, "create_routine", { + ownerUserId: "someone-else", + agentId: "another-bot", + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + }); + + const created = calls.find((call) => call.method === "create"); + expect(created?.method === "create" && created.input.ownerUserId).toBe( + "user_asker", + ); + expect(created?.method === "create" && created.input.agentId).toBe( + "bot_helper", + ); + }); + + test("an owner named in the arguments is ignored by list, update and delete", async () => { + const calls = recordingTools(); + const spoofed = { ownerUserId: "someone-else", agentId: "another-bot" }; + await callTool(CONNECTION, "list_routines", { ...spoofed }); + await callTool(CONNECTION, "update_routine", { + ...spoofed, + id: "routine_1", + instruction: "Something else.", + }); + await callTool(CONNECTION, "delete_routine", { + ...spoofed, + id: "routine_1", + }); + + for (const call of calls) { + expect(call).toMatchObject({ ownerUserId: "user_asker" }); + } + }); + + test("a run attributed to nobody is refused", async () => { + const calls = recordingTools(); + const result = await callTool( + { url: CONNECTION.url, botId: "bot_helper" }, + "list_routines", + {}, + ); + + expect(result.isError).toBe(true); + expect(result.text).toBe( + "A routine belongs to somebody, and this run is not attributed to anybody.", + ); + expect(calls).toHaveLength(0); + }); + + test("an empty actor is not an actor", async () => { + const calls = recordingTools(); + const result = await callTool( + { url: CONNECTION.url, actorId: " ", botId: "bot_helper" }, + "list_routines", + {}, + ); + + expect(result.isError).toBe(true); + expect(result.text).toBe( + "A routine belongs to somebody, and this run is not attributed to anybody.", + ); + expect(calls).toHaveLength(0); + }); + + test("a run that names no Bot is refused", async () => { + const calls = recordingTools(); + const result = await callTool( + { url: CONNECTION.url, actorId: "user_asker" }, + "create_routine", + { instruction: "Post it.", cron: "0 9 * * 1-5" }, + ); + + expect(result.isError).toBe(true); + expect(result.text).toBe( + "A routine runs as a Bot, and this run does not name one.", + ); + expect(calls).toHaveLength(0); + }); + + test("a deployment with no routine store refuses", async () => { + useRoutineTools(null); + const result = await callTool(CONNECTION, "list_routines", {}); + + expect(result.isError).toBe(true); + expect(result.text).toBe("Routines is not available in this deployment."); + }); +}); + +describe("what the store said", () => { + test("a refusal is carried through verbatim", async () => { + recordingTools({ + async create() { + throw new RoutineRefusedError( + "Routines may run at most every 15 minutes.", + ); + }, + }); + + const result = await callTool(CONNECTION, "create_routine", { + instruction: "Post it.", + cron: "* * * * *", + }); + + expect(result.isError).toBe(true); + expect(result.text).toBe("Routines may run at most every 15 minutes."); + }); + + test("a missing routine is a sentence about ownership", async () => { + recordingTools({ + async remove() { + throw new RoutineNotFoundError(); + }, + }); + + const result = await callTool(CONNECTION, "delete_routine", { + id: "routine_nope", + }); + + expect(result.isError).toBe(true); + expect(result.text).toBe("There is no routine of yours with that id."); + }); + + test("anything else is capped in code points and never escapes", async () => { + recordingTools({ + async listFor() { + throw new Error(`${"é".repeat(600)}!`); + }, + }); + + const result = await callTool(CONNECTION, "list_routines", {}); + + expect(result.isError).toBe(true); + expect(Array.from(result.text)).toHaveLength(400); + expect(result.text).toBe("é".repeat(400)); + }); +}); From 85a8cda7957a031e55649eefab47ac3dda244a1e Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:32:55 -0300 Subject: [PATCH 12/45] Tell the truth about a write that landed and a routine that sleeps --- server/src/plugins/builtin-routines.ts | 48 ++++++-- server/tests/builtin-routines.test.ts | 147 +++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 8 deletions(-) diff --git a/server/src/plugins/builtin-routines.ts b/server/src/plugins/builtin-routines.ts index cd225bcc..d16b7f2d 100644 --- a/server/src/plugins/builtin-routines.ts +++ b/server/src/plugins/builtin-routines.ts @@ -283,7 +283,12 @@ function inWords(summary: RoutineSummary): string { const parts = [ `${summary.schedule} (${summary.timezone})`, `in ${channelOf(summary)}`, - `next ${summary.nextRunAt.toISOString()}`, + // The store does not recompute `nextRunAt` on disable, so a switched-off routine's stored + // value is a stale firing time, not a fact — stating it as "next" would let a model repeat a + // time that will never come until somebody switches it back on. + summary.enabled + ? `next ${summary.nextRunAt.toISOString()}` + : "next when switched back on", lastRunOf(summary), // The model needs this to change or delete the routine later, and it cannot derive it. `id: ${summary.id}`, @@ -297,20 +302,35 @@ function inWords(summary: RoutineSummary): string { * * `create` and `update` return a `Routine`, which carries a cron expression and a channel id: the * two things this answer must not be. The summary carries the words and the channel's name, so the - * confirmation is a sentence rather than a row. Absent only if it vanished between the two calls, - * where naming the id is the whole of what is still true. + * confirmation is a sentence rather than a row. `ownerUserId` is the connection-derived value the + * caller already validated — not `routine.ownerUserId` — so "identity always from the connection" + * holds at this call site too, not just at the write. + * + * The re-read is wrapped on its own: the write already landed by the time this runs, so a `listFor` + * that throws (the routine vanished, or the read itself failed) is not a write that failed, and must + * not be reported as one — that would tell the model to retry and duplicate the routine. Either way + * the id is still true, so `fallbackOpening` composes a full sentence around it rather than reusing + * `opening`, which does not grammatically continue into "Its id is …" for every verb. */ async function describeWritten( tools: RoutineTools, routine: Routine, + ownerUserId: string, opening: string, + fallbackOpening: string, ): Promise { - const summaries = await tools.listFor(routine.ownerUserId); - const summary = summaries.find((candidate) => candidate.id === routine.id); + let summary: RoutineSummary | undefined; + try { + summary = (await tools.listFor(ownerUserId)).find( + (candidate) => candidate.id === routine.id, + ); + } catch { + // The write landed. A confirmation that could not be read is not a write that failed. + } return asResult( summary ? `${opening} ${inWords(summary)}` - : `${opening} Its id is ${routine.id}.`, + : `${fallbackOpening} Its id is ${routine.id}.`, ); } @@ -372,7 +392,13 @@ export async function callTool( timezone: stringArg(args, "timezone"), channelId: stringArg(args, "channelId"), }); - return await describeWritten(tools, routine, "That routine is set:"); + return await describeWritten( + tools, + routine, + ownerUserId, + "That routine is set:", + "That routine is set.", + ); } if (toolName === "list_routines") { @@ -410,7 +436,13 @@ export async function callTool( } const routine = await tools.update(ownerUserId, id, patch); - return await describeWritten(tools, routine, "That routine now reads:"); + return await describeWritten( + tools, + routine, + ownerUserId, + "That routine now reads:", + "That routine is updated.", + ); } if (toolName === "delete_routine") { diff --git a/server/tests/builtin-routines.test.ts b/server/tests/builtin-routines.test.ts index 56d4f245..c0111f5e 100644 --- a/server/tests/builtin-routines.test.ts +++ b/server/tests/builtin-routines.test.ts @@ -344,6 +344,153 @@ describe("attribution", () => { }); }); +describe("a write that landed but could not be read back", () => { + test("create: a listFor that throws after a successful write is not reported as a failure", async () => { + recordingTools({ + async listFor() { + throw new Error("connection reset"); + }, + }); + + const result = await callTool(CONNECTION, "create_routine", { + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + }); + + expect(result.isError).toBe(false); + expect(result.text).toBe("That routine is set. Its id is routine_1."); + }); + + test("update: a listFor that throws after a successful write is not reported as a failure", async () => { + recordingTools({ + async listFor() { + throw new Error("connection reset"); + }, + }); + + const result = await callTool(CONNECTION, "update_routine", { + id: "routine_1", + instruction: "Something else.", + }); + + expect(result.isError).toBe(false); + expect(result.text).toBe("That routine is updated. Its id is routine_1."); + }); + + test("create: the vanished-routine fallback reads as a sentence and names the id", async () => { + recordingTools({ + async listFor() { + return []; + }, + }); + + const result = await callTool(CONNECTION, "create_routine", { + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + }); + + expect(result.isError).toBe(false); + expect(result.text).toBe("That routine is set. Its id is routine_1."); + }); + + test("update: the vanished-routine fallback reads as a sentence and names the id", async () => { + recordingTools({ + async listFor() { + return []; + }, + }); + + const result = await callTool(CONNECTION, "update_routine", { + id: "routine_1", + instruction: "Something else.", + }); + + expect(result.isError).toBe(false); + expect(result.text).toBe("That routine is updated. Its id is routine_1."); + }); + + test("the confirmation read is attributed to the connection's actor, not the routine's owner", async () => { + const calls = recordingTools(); + await callTool(CONNECTION, "create_routine", { + instruction: "Post the standup summary.", + cron: "0 9 * * 1-5", + }); + + const reads = calls.filter((call) => call.method === "listFor"); + expect(reads).toContainEqual({ + method: "listFor", + ownerUserId: "user_asker", + }); + }); +}); + +describe("a disabled routine's rendered next-run", () => { + test('omits an ISO timestamp after "next" and keeps the switched-off wording', async () => { + recordingTools({ + async listFor() { + return [{ ...SUMMARY, enabled: false }]; + }, + }); + + const result = await callTool(CONNECTION, "list_routines", {}); + + expect(result.isError).toBe(false); + expect(result.text).not.toContain(SUMMARY.nextRunAt.toISOString()); + expect(result.text).toContain("switched off"); + expect(result.text).toMatch(/next when switched back on/); + }); +}); + +describe("beyond-spec validation refusals", () => { + test("create_routine without an instruction refuses without touching the store", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "create_routine", { + cron: "0 9 * * 1-5", + }); + + expect(result.isError).toBe(true); + expect(result.text).toBe("A routine needs an instruction to carry out."); + expect(calls).toHaveLength(0); + }); + + test("create_routine without a cron refuses without touching the store", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "create_routine", { + instruction: "Post it.", + }); + + expect(result.isError).toBe(true); + expect(result.text).toBe( + "A routine needs a schedule: five cron fields, `minute hour day-of-month month day-of-week`.", + ); + expect(calls).toHaveLength(0); + }); + + test("update_routine without an id refuses without touching the store", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "update_routine", { + instruction: "Something else.", + }); + + expect(result.isError).toBe(true); + expect(result.text).toBe( + "Say which routine to change, by the id from list_routines.", + ); + expect(calls).toHaveLength(0); + }); + + test("update_routine with an empty patch refuses without touching the store", async () => { + const calls = recordingTools(); + const result = await callTool(CONNECTION, "update_routine", { + id: "routine_1", + }); + + expect(result.isError).toBe(true); + expect(result.text).toBe("Say what to change about that routine."); + expect(calls).toHaveLength(0); + }); +}); + describe("what the store said", () => { test("a refusal is carried through verbatim", async () => { recordingTools({ From 593df68673777255182b3d30829cc5d8dfc0119b Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:40:39 -0300 Subject: [PATCH 13/45] Run a routine's turn with nobody's browser open --- server/src/routines/runner.ts | 207 +++++++++++++ server/src/routines/store.ts | 45 +++ server/tests/routine-runner.test.ts | 282 ++++++++++++++++++ .../tests/routines-store.integration.test.ts | 34 +++ 4 files changed, 568 insertions(+) create mode 100644 server/src/routines/runner.ts create mode 100644 server/tests/routine-runner.test.ts diff --git a/server/src/routines/runner.ts b/server/src/routines/runner.ts new file mode 100644 index 00000000..99775aa0 --- /dev/null +++ b/server/src/routines/runner.ts @@ -0,0 +1,207 @@ +/** + * Running one firing of a routine with nobody's browser open. + * + * A turn normally has a person in front of it: their browser runs the agent, sees the reply, and + * tells the server what was said, which is what moves the channel to the top of their roster and + * lights the unread dot. A routine has none of that, so this file does that last step itself — + * `recordActivity` — and it is the only place a headless turn does, which is why there is exactly + * one of those calls per successful firing here and why the tests count them. + * + * TWO LEDGERS, AND THEY ARE NOT THE SAME LEDGER. + * + * The queue's attempt count (`server/src/work/queue.ts`) bounds retries of ONE FIRING: a consumer + * that died mid-turn, a runner that is flapping, a lease that stopped being renewed. It answers + * "how many times have we handed this one due moment out?". + * + * The fatigue rule below counts CONSECUTIVE FIRINGS THAT FAILED, in `routine_runs`, across days. + * It answers "is this routine still worth firing at all?" — a Notion token that expired in March + * fails every night, and no number of retries of any one night's firing will fix it. + * + * Conflating them is how a broken routine either spams a channel — a per-attempt notification + * posting three times for one bad night — or retries for ever, because a rule counting attempts + * within a firing never sees the routine that fails cleanly, once, every single night. + */ +import type { AgentActor } from "../agents/profile-types"; +import type { ChannelStore } from "../channels/routes"; +import type { RoutineStore } from "./store"; + +/** Everything a headless turn needs, injectable so tests never dial a model. */ +export type TurnRunner = (input: { + ownerUserId: string; // the actor the run asserts — grants and connections resolve to them + agentId: string; + threadId: string; // the owner's thread for the routine's channel + instruction: string; // the user message of this turn +}) => Promise<{ replyText: string }>; + +export type RoutineRunner = { run(routineRunId: string): Promise }; + +/** + * How much of a failure's reason the channel notification carries. + * + * `recordActivity` caps its preview at 200 code points on its own, so this is not about safety, it + * is about the sentence surviving: the prefix plus this much reason still fits, so the roster shows + * a failure that says what failed rather than a truncated "This routine failed: Error: could not…". + * The whole message is on the run row for anybody who wants all of it. + */ +const MAX_NOTIFIED_REASON = 160; + +/** Consecutive failed firings after which a routine stops being fired at all. */ +const FATIGUE_LIMIT = 10; + +const SWITCHED_OFF = + "This routine has failed ten times in a row, so I have switched it off. Ask me to turn it back on when whatever it needs is working."; + +/** Measured in code points, like every other cap in this area, so nothing is cut mid-pair. */ +function shorten(reason: string): string { + const codePoints = Array.from(reason); + if (codePoints.length <= MAX_NOTIFIED_REASON) return reason; + return `${codePoints.slice(0, MAX_NOTIFIED_REASON - 1).join("")}…`; +} + +function reasonOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function createRoutineRunner(options: { + routineStore: RoutineStore; + channelStore: ChannelStore; + runTurn: TurnRunner; +}): RoutineRunner { + const { routineStore, channelStore, runTurn } = options; + + async function runOnce(routineRunId: string): Promise { + const context = await routineStore.runContext(routineRunId); + /* + * A deleted routine's queued run is nobody's problem. The routine is gone, its runs cascaded + * with it, and there is no row left to finish — so there is nothing to record and nothing to + * say to anybody. The firing is dropped on the floor deliberately. + */ + if (!context) return; + + const { routineId, ownerUserId, agentId, channelId, instruction } = context; + // Everything below is done AS the owner: their channel, their thread, their grants. + const owner: AgentActor = { id: ownerUserId, role: "user" }; + + /** One activity record, from the routine's Bot, or a logged miss. Never a throw. */ + async function say(text: string): Promise { + try { + await channelStore.recordActivity(owner, channelId, { + text, + agentId, + at: new Date(), + }); + } catch (error) { + /* + * A channel that cannot be written to must not turn a recorded outcome into an unrecorded + * one. The run row is the ledger; this call is the courtesy of saying so where a person + * will see it, and losing the courtesy is not worth losing the ledger. + */ + console.error( + JSON.stringify({ + type: "routine-activity-unrecorded", + routineId, + routineRunId, + reason: reasonOf(error), + }), + ); + } + } + + /* + * `get` as the owner already filters soft-deleted channels and non-members, so null covers all + * three ways this can be over: the channel was deleted, the row is gone, or the owner is no + * longer in it. No second check, and no turn — a reply with nowhere to land is model spend for + * nothing. + * + * This is why `routines.channel_id` is not a foreign key: the routine survives its channel so + * the routines page can show it as broken, and the person can point it somewhere else. + */ + const channel = await channelStore.get(owner, channelId); + if (!channel) { + await routineStore.finishRun( + routineRunId, + "skipped", + "the channel is gone", + ); + return; + } + + let replyText: string; + try { + ({ replyText } = await runTurn({ + ownerUserId, + agentId, + threadId: channel.threadId, + instruction, + })); + } catch (error) { + const reason = reasonOf(error); + await routineStore.finishRun(routineRunId, "failed", reason); + + /* + * THE FATIGUE RULE, read after the failure is recorded — the count has to include this + * firing, or the tenth failure in a row reads as the ninth and the routine keeps going. + * + * Its own try/catch, around the read and the switching-off as well as the message: this all + * happens after `finishRun`, and nothing in it is allowed to throw its way out of a firing + * whose failure is already on the row. + */ + try { + const failures = await routineStore.consecutiveFailures(routineId); + if (failures === 1) { + // Only the first failure after a success. A routine that fails every night at three has + // one line in the channel, not one line a night for a month. + await say(`This routine failed: ${shorten(reason)}`); + } else if (failures >= FATIGUE_LIMIT) { + // A dead integration should not burn model spend for ever. Switched off, and said out + // loud, because a routine that goes quiet without explaining itself is worse than one + // that fails. + await routineStore.setEnabled(ownerUserId, routineId, false); + await say(SWITCHED_OFF); + } + // In between, nothing is said. The run rows carry it, and the routines page reads them. + } catch (fatigueError) { + console.error( + JSON.stringify({ + type: "routine-fatigue-rule-failed", + routineId, + routineRunId, + reason: reasonOf(fatigueError), + }), + ); + } + return; + } + + /* + * The reply, then the outcome. `say` cannot throw, and `finishRun` is finish-once, so neither + * order can lose the run row — this one is the browser's order: what was said lands in the + * channel, and then the firing is closed. + */ + await say(replyText); + await routineStore.finishRun(routineRunId, "succeeded"); + } + + return { + async run(routineRunId) { + /* + * `run` never throws. Every outcome a routine can have — a gone channel, a refused model, a + * routine deleted underneath the firing — is recorded on the run row above rather than raised + * at whoever is draining the queue. This backstop is for the ones that are not outcomes at + * all, a store that cannot be reached being the honest example: said out loud, because the + * alternative is a firing that vanishes without a line anywhere. + */ + try { + await runOnce(routineRunId); + } catch (error) { + console.error( + JSON.stringify({ + type: "routine-run-crashed", + routineRunId, + reason: reasonOf(error), + }), + ); + } + }, + }; +} diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index f5e50af8..c163e429 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -137,6 +137,21 @@ export type RoutineInput = { timezone?: string; }; +/** + * One firing, and everything running it headlessly needs — the runner's read, and nobody else's. + * + * The owner comes back because the runner acts AS the owner: the channel it posts into, the thread + * it continues and the grants the turn resolves are all that person's, and a runner that had to be + * told whose they were would be a runner that could be told the wrong answer. + */ +export type RoutineRunContext = { + routineId: string; + ownerUserId: string; + agentId: string; + channelId: string; + instruction: string; +}; + export type RoutinePatch = Partial<{ instruction: string; cron: string; @@ -164,6 +179,15 @@ export type RoutineStore = { advanceNextRun(id: string, from: Date): Promise; /** Open a run row. Its status stays null until something finishes it. */ insertRun(routineId: string): Promise<{ runId: string }>; + /** + * The runner's read: an opened run row, joined to the routine it fires. + * + * Not owner-scoped, like everything else in this half — a run id is not something a person names, + * it is something the queue hands back — and deliberately out of RoutineTools' reach: nothing a + * model can call resolves a run id, so nothing a model can call gets an owner out of one. Null + * means the routine was deleted between queueing and running, which takes its runs with it. + */ + runContext(runId: string): Promise; /** Close a run row with its outcome, and the capped error when there was one. */ finishRun( runId: string, @@ -589,6 +613,27 @@ export function createRoutineStore(database: Database): RoutineStore { return { runId: row.id }; }, + async runContext(runId) { + /* + * An inner join, so a run whose routine is gone reads as no row rather than as a firing with + * nothing to say. `remove` is a hard delete and the runs cascade with it, so in practice both + * sides disappear together; the join is what makes that one absence instead of two. + */ + const [row] = await database + .select({ + routineId: routines.id, + ownerUserId: routines.ownerUserId, + agentId: routines.agentId, + channelId: routines.channelId, + instruction: routines.instruction, + }) + .from(routineRuns) + .innerJoin(routines, eq(routines.id, routineRuns.routineId)) + .where(eq(routineRuns.id, runId)) + .limit(1); + return row ?? null; + }, + async finishRun(runId, status, error) { await database .update(routineRuns) diff --git a/server/tests/routine-runner.test.ts b/server/tests/routine-runner.test.ts new file mode 100644 index 00000000..4a6bef9b --- /dev/null +++ b/server/tests/routine-runner.test.ts @@ -0,0 +1,282 @@ +import { describe, expect, test } from "bun:test"; +import type { AgentActor } from "../src/agents/profile-types"; +import type { + AgentChannel, + ChannelActivity, + ChannelStore, +} from "../src/channels/routes"; +import { createRoutineRunner, type TurnRunner } from "../src/routines/runner"; +import type { + RoutineRunContext, + RoutineRunOutcome, + RoutineStore, +} from "../src/routines/store"; + +/** + * A routine's turn with nobody's browser open, asserted without a database and without a model. + * + * The runner is the one place a headless firing does what a browser normally does — record the + * reply as channel activity — so what is under test is the sequence and the counts: exactly one + * activity per successful turn, exactly one run row closed, and the fatigue rule speaking once and + * then shutting up. `runTurn` is injected, which is why nothing here dials a model, and the store + * halves are recording stubs, which is why nothing here needs Postgres. + * + * The person-facing half of RoutineStore throws on contact: the runner acts as the owner but is not + * the owner asking a question, and reaching for `create` or `listFor` from here would be a bug that + * a test asserting only outcomes would not see. + */ + +const OWNER: AgentActor = { id: "user_owner", role: "user" }; + +const CONTEXT: RoutineRunContext = { + routineId: "routine_1", + ownerUserId: "user_owner", + agentId: "bot_helper", + channelId: "channel_1", + instruction: "Post the standup summary.", +}; + +const CHANNEL: AgentChannel = { + id: "channel_1", + name: "Standup", + agentIds: ["bot_helper"], + threadId: "thread_owner_channel_1", + active: true, +}; + +const RUN_ID = "routine_run_1"; + +type Recorded = { + finished: { runId: string; status: RoutineRunOutcome; error?: string }[]; + activity: { + actor: AgentActor; + channelId: string; + activity: ChannelActivity; + }[]; + enabled: { ownerUserId: string; id: string; enabled: boolean }[]; + turns: Parameters[0][]; +}; + +function unreachable(method: string): never { + throw new Error(`the runner must not call ${method}`); +} + +function harness(options: { + context?: RoutineRunContext | null; + channel?: AgentChannel | null; + failures?: number; + runTurn?: TurnRunner; + recordActivity?: () => Promise; +}) { + const recorded: Recorded = { + finished: [], + activity: [], + enabled: [], + turns: [], + }; + + const routineStore: RoutineStore = { + create: () => unreachable("create"), + listFor: () => unreachable("listFor"), + update: () => unreachable("update"), + remove: () => unreachable("remove"), + dueRoutines: () => unreachable("dueRoutines"), + advanceNextRun: () => unreachable("advanceNextRun"), + insertRun: () => unreachable("insertRun"), + + async runContext(runId) { + expect(runId).toBe(RUN_ID); + return options.context === undefined ? CONTEXT : options.context; + }, + async finishRun(runId, status, error) { + recorded.finished.push({ runId, status, error }); + }, + async consecutiveFailures(routineId) { + expect(routineId).toBe(CONTEXT.routineId); + return options.failures ?? 0; + }, + async setEnabled(ownerUserId, id, enabled) { + recorded.enabled.push({ ownerUserId, id, enabled }); + }, + }; + + const channelStore: ChannelStore = { + create: () => unreachable("channels.create"), + list: () => unreachable("channels.list"), + setPinned: () => unreachable("channels.setPinned"), + markRead: () => unreachable("channels.markRead"), + softDelete: () => unreachable("channels.softDelete"), + + async get(actor, channelId) { + expect(actor).toEqual(OWNER); + expect(channelId).toBe(CONTEXT.channelId); + return options.channel === undefined ? CHANNEL : options.channel; + }, + async recordActivity(actor, channelId, activity) { + recorded.activity.push({ actor, channelId, activity }); + if (options.recordActivity) await options.recordActivity(); + }, + }; + + const runTurn: TurnRunner = async (input) => { + recorded.turns.push(input); + if (options.runTurn) return await options.runTurn(input); + return { replyText: "Three people are blocked." }; + }; + + return { + recorded, + runner: createRoutineRunner({ routineStore, channelStore, runTurn }), + }; +} + +const throwingTurn: TurnRunner = async () => { + throw new Error("the model refused"); +}; + +describe("createRoutineRunner", () => { + test("runs the owner's thread and records the reply exactly once", async () => { + const { runner, recorded } = harness({}); + + await runner.run(RUN_ID); + + expect(recorded.turns).toEqual([ + { + ownerUserId: CONTEXT.ownerUserId, + agentId: CONTEXT.agentId, + threadId: CHANNEL.threadId, + instruction: CONTEXT.instruction, + }, + ]); + // The count, not merely that one happened: a second record would ring a second unread dot. + expect(recorded.activity).toHaveLength(1); + expect(recorded.activity[0]?.actor).toEqual(OWNER); + expect(recorded.activity[0]?.channelId).toBe(CONTEXT.channelId); + expect(recorded.activity[0]?.activity.text).toBe( + "Three people are blocked.", + ); + expect(recorded.activity[0]?.activity.agentId).toBe(CONTEXT.agentId); + expect(recorded.finished).toEqual([ + { runId: RUN_ID, status: "succeeded", error: undefined }, + ]); + expect(recorded.enabled).toEqual([]); + }); + + test("records a thrown turn as a failure carrying its message", async () => { + const { runner, recorded } = harness({ + runTurn: throwingTurn, + failures: 1, + }); + + await runner.run(RUN_ID); + + expect(recorded.finished).toEqual([ + { runId: RUN_ID, status: "failed", error: "the model refused" }, + ]); + }); + + test("says so once on the first failure after a success", async () => { + const { runner, recorded } = harness({ + runTurn: throwingTurn, + failures: 1, + }); + + await runner.run(RUN_ID); + + expect(recorded.activity).toHaveLength(1); + expect(recorded.activity[0]?.activity.text).toBe( + "This routine failed: the model refused", + ); + expect(recorded.activity[0]?.activity.agentId).toBe(CONTEXT.agentId); + expect(recorded.enabled).toEqual([]); + }); + + test("says nothing on the second consecutive failure", async () => { + const { runner, recorded } = harness({ + runTurn: throwingTurn, + failures: 2, + }); + + await runner.run(RUN_ID); + + expect(recorded.activity).toEqual([]); + expect(recorded.enabled).toEqual([]); + expect(recorded.finished).toHaveLength(1); + }); + + test("switches the routine off after ten consecutive failures", async () => { + const { runner, recorded } = harness({ + runTurn: throwingTurn, + failures: 10, + }); + + await runner.run(RUN_ID); + + expect(recorded.enabled).toEqual([ + { + ownerUserId: CONTEXT.ownerUserId, + id: CONTEXT.routineId, + enabled: false, + }, + ]); + expect(recorded.activity).toHaveLength(1); + expect(recorded.activity[0]?.activity.text).toBe( + "This routine has failed ten times in a row, so I have switched it off. Ask me to turn it back on when whatever it needs is working.", + ); + }); + + test("skips a firing whose channel is gone without running a turn", async () => { + const { runner, recorded } = harness({ channel: null }); + + await runner.run(RUN_ID); + + expect(recorded.turns).toEqual([]); + expect(recorded.activity).toEqual([]); + expect(recorded.finished).toEqual([ + { runId: RUN_ID, status: "skipped", error: "the channel is gone" }, + ]); + }); + + test("does nothing at all for a run row that is gone", async () => { + const { runner, recorded } = harness({ context: null }); + + await runner.run(RUN_ID); + + expect(recorded).toEqual({ + finished: [], + activity: [], + enabled: [], + turns: [], + }); + }); + + test("keeps a succeeded run recorded when the channel cannot be written to", async () => { + const { runner, recorded } = harness({ + recordActivity: async () => { + throw new Error("that channel is gone"); + }, + }); + + await runner.run(RUN_ID); + + expect(recorded.finished).toEqual([ + { runId: RUN_ID, status: "succeeded", error: undefined }, + ]); + }); + + test("keeps a failed run recorded when the notification cannot be posted", async () => { + const { runner, recorded } = harness({ + runTurn: throwingTurn, + failures: 1, + recordActivity: async () => { + throw new Error("that channel is gone"); + }, + }); + + await runner.run(RUN_ID); + + expect(recorded.finished).toEqual([ + { runId: RUN_ID, status: "failed", error: "the model refused" }, + ]); + }); +}); diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index e38f4b1d..8be016d9 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -907,6 +907,40 @@ describe("opening and closing a run", () => { }); }); +describe("the runner's read of one firing", () => { + test("joins the run to its routine, owner included", async () => { + const { owner, agentId, channel, routine } = await makeRoutine( + "Post the standup summary.", + ); + const { runId } = await store.insertRun(routine.id); + + // The owner comes back because the runner acts AS the owner: the channel it posts into and the + // thread it continues are that person's, and nothing else in the run row says who that is. + expect(await store.runContext(runId)).toEqual({ + routineId: routine.id, + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Post the standup summary.", + }); + }); + + test("a run id nothing wrote reads as no firing", async () => { + expect(await store.runContext("routine_run_missing")).toBeNull(); + }); + + test("a deleted routine takes its queued firing with it", async () => { + const { owner, routine } = await makeRoutine(); + const { runId } = await store.insertRun(routine.id); + + await store.remove(owner.id, routine.id); + + // Nobody's problem: the delete cascades, so there is no row left to finish and the runner has + // nothing to say to anybody. + expect(await store.runContext(runId)).toBeNull(); + }); +}); + /** * The fatigue rule counts the failures at the tail, and this file pins what a `skipped` run does to * that tail: A SKIP IS NOT A FAILURE AND DOES NOT BREAK THE STREAK. A skip means the channel was From 0425dc6b3dab5cb5910f580b1ded9f9721c490c4 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:48:21 -0300 Subject: [PATCH 14/45] Pin the order the fatigue rule reads in, and the cut it makes --- server/tests/routine-runner.test.ts | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/server/tests/routine-runner.test.ts b/server/tests/routine-runner.test.ts index 4a6bef9b..2013419a 100644 --- a/server/tests/routine-runner.test.ts +++ b/server/tests/routine-runner.test.ts @@ -93,6 +93,13 @@ function harness(options: { }, async consecutiveFailures(routineId) { expect(routineId).toBe(CONTEXT.routineId); + // Read AFTER the failure is on the row, or the tenth failure reads as the ninth: hoisting + // this read above `finishRun` in the runner would keep every other assertion here green. + expect(recorded.finished).toHaveLength(1); + expect(recorded.finished[0]).toMatchObject({ + runId: RUN_ID, + status: "failed", + }); return options.failures ?? 0; }, async setEnabled(ownerUserId, id, enabled) { @@ -279,4 +286,53 @@ describe("createRoutineRunner", () => { { runId: RUN_ID, status: "failed", error: "the model refused" }, ]); }); + + test("truncates a failure reason at the code-point cap without splitting an emoji", async () => { + // 170 code points: an ASCII run long enough to push the cut (at code point 159) into the + // middle of the run of astral emoji that follows, so the cut has to land between code points, + // never inside one of their surrogate pairs. + const longReason = `${"a".repeat(150)}${"\u{1F600}".repeat(20)}`; + const { runner, recorded } = harness({ + runTurn: async () => { + throw new Error(longReason); + }, + failures: 1, + }); + + await runner.run(RUN_ID); + + expect(recorded.activity).toHaveLength(1); + const posted = recorded.activity[0]?.activity.text ?? ""; + expect(posted.startsWith("This routine failed: ")).toBe(true); + const reasonPart = posted.slice("This routine failed: ".length); + + // Capped at MAX_NOTIFIED_REASON (160) code points: 159 kept, plus the implementation's own + // ellipsis character — read runner.ts's `shorten` rather than assume a format here. + expect(Array.from(reasonPart)).toHaveLength(160); + expect(reasonPart).toBe(`${"a".repeat(150)}${"\u{1F600}".repeat(9)}…`); + + // The cut is measured in code points, not UTF-16 units, so no emoji is split into a lone + // surrogate: the string stays well-formed and a code-point split round-trips cleanly. + expect(reasonPart.isWellFormed()).toBe(true); + expect(Array.from(reasonPart).join("")).toBe(reasonPart); + }); + + test("posts a thrown non-Error value with String(error)", async () => { + const { runner, recorded } = harness({ + runTurn: async () => { + throw "not an Error object"; + }, + failures: 1, + }); + + await runner.run(RUN_ID); + + expect(recorded.finished).toEqual([ + { runId: RUN_ID, status: "failed", error: "not an Error object" }, + ]); + expect(recorded.activity).toHaveLength(1); + expect(recorded.activity[0]?.activity.text).toBe( + "This routine failed: not an Error object", + ); + }); }); From 894342701918638dfaef22a6ce73c8ed80f743b5 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 11:53:39 -0300 Subject: [PATCH 15/45] Open one door for the worker, and compare the secret behind it --- .env.example | 9 ++ server/src/app.ts | 55 ++++++- server/src/config.ts | 10 ++ server/tests/routine-endpoint.test.ts | 208 ++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 server/tests/routine-endpoint.test.ts diff --git a/.env.example b/.env.example index 706b402a..dad529df 100644 --- a/.env.example +++ b/.env.example @@ -295,3 +295,12 @@ COMPUTER_RUNTIME= # may call tools back and it is told so rather than being quietly allowed. AGENT_TOOL_TOKEN= +# The secret the routine worker presents when it hands a run back to this server. It authenticates +# the handoff of one routine run id, and nothing else: the server re-reads the routine, the Bot and +# everything else it needs from the database, so the worker cannot use this to inject an instruction. +# +# Left empty, the internal endpoint refuses every call and no routine ever fires — the correct state +# for a deployment that has not stood up a worker. Set for one that has: openssl rand -base64 32. +# Do not accept a default in production. +WORKER_SHARED_SECRET= + diff --git a/server/src/app.ts b/server/src/app.ts index 2e30c769..d7e4fffa 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,7 +1,7 @@ import type { Hono as HonoApp, MiddlewareHandler } from "hono"; import { Hono } from "hono"; import { serveStatic } from "hono/bun"; -import { authoriseAgentCall } from "./agents/callback-token"; +import { authoriseAgentCall, sameToken } from "./agents/callback-token"; import type { BotAccessCheck } from "./agents/profile-policy"; import type { AgentProfileStore } from "./agents/profile-store"; import { createAgentRoutes } from "./agents/routes"; @@ -44,6 +44,7 @@ import type { PluginStore } from "./plugins/store"; import { REFUSAL_MARKER } from "./plugins/tools"; import type { IntentRouter } from "./routing/classify"; import { createRoutingRoutes } from "./routing/routes"; +import type { RoutineRunner } from "./routines/runner"; import type { PackageStatusReader } from "./tenant-package"; /** @@ -175,6 +176,16 @@ export function createApp( * the wrong thing. */ pageFrames?: PageFrameStore, + /** + * Fires one routine run with nobody's browser open, when the worker hands one back. + * + * Appended last, like `pageFrames`: these are positional, so inserting one anywhere else silently + * shifts every existing call site's arguments by one. + * + * Absent leaves the internal `/internal/routines/run` route unmounted rather than mounted and + * refusing every call: a deployment that never built a worker has no door for it, not a locked one. + */ + routineRunner?: RoutineRunner, ) { const app = new Hono<{ Variables: AppVariables }>(); @@ -613,6 +624,48 @@ export function createApp( } return context.json({ package: await packageStatusReader.active() }); }); + /* + * Where the worker hands a routine run back. Not under /api and not behind requireUser: the + * worker is not a person with a session, it is another process on this deployment's own network, + * and the shared secret below is its whole credential. + * + * Mounted only when a runner was built, so a deployment that never stood up a worker has no door + * for this at all, rather than one that is mounted and permanently refuses. + */ + if (routineRunner) { + app.post("/internal/routines/run", async (context) => { + const offered = context.req.header("authorization"); + const expected = config.workerSharedSecret + ? `Bearer ${config.workerSharedSecret}` + : null; + /* + * The no-secret-configured case is refused here, before any comparison, and with the exact + * same response as a wrong secret. A deployment with no worker must not answer a guess any + * differently than a deployment with a worker and a wrong key would. + */ + if (!expected || !offered || !sameToken(offered, expected)) { + return context.json({ error: "This endpoint is the worker's." }, 401); + } + const body = await context.req.json().catch(() => null); + if ( + typeof (body as { routineRunId?: unknown } | null)?.routineRunId !== + "string" + ) { + return context.json({ error: "A routineRunId is required." }, 400); + } + /* + * Fire and answer: the consumer needs "accepted", not the outcome. The run row in + * `routine_runs` carries the outcome, and the consumer finishes the work item on this 202. + * Queue retries exist for DISPATCH failures only — a failed turn is final for this firing, and + * the fatigue rule owns it. `run()` never throws by contract; this swallow only guards against + * that contract being wrong without turning a bug there into an unhandled rejection here. + */ + void routineRunner + .run((body as { routineRunId: string }).routineRunId) + .catch(() => {}); + return context.json({ accepted: true }, 202); + }); + } // The CopilotKit runtime, behind the same session guard as every other API route. Mounted last so // its own routing under /api/copilotkit cannot shadow an OpenBot route declared above. if (copilotHandler) { diff --git a/server/src/config.ts b/server/src/config.ts index 3dced0e6..ba419afe 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -227,6 +227,14 @@ export type DeploymentConfig = { * than an open door. */ agentToolToken?: string; + /** + * The secret the worker presents when it hands a routine run back to this server. + * + * Absent means the internal routines endpoint refuses everything, which is the correct state of a + * deployment with no worker — a deployment that has not asked for scheduled turns should not have a + * door for them standing open. + */ + workerSharedSecret?: string; }; type Environment = Record; @@ -759,6 +767,7 @@ export function loadConfig( const google = oauthClient(environment, "GOOGLE"); const auth = authConfig(environment, google); const managedAgent = managedAgentConfig(environment); + const workerSharedSecret = optional(environment, "WORKER_SHARED_SECRET"); return { databaseUrl: required(environment, "DATABASE_URL"), @@ -794,5 +803,6 @@ export function loadConfig( ...(optional(environment, "AGENT_TOOL_TOKEN") ? { agentToolToken: optional(environment, "AGENT_TOOL_TOKEN") as string } : {}), + ...(workerSharedSecret ? { workerSharedSecret } : {}), }; } diff --git a/server/tests/routine-endpoint.test.ts b/server/tests/routine-endpoint.test.ts new file mode 100644 index 00000000..ba80e589 --- /dev/null +++ b/server/tests/routine-endpoint.test.ts @@ -0,0 +1,208 @@ +import { describe, expect, test } from "bun:test"; +import { createApp } from "../src/app"; +import { loadConfig } from "../src/config"; +import type { RoutineRunner } from "../src/routines/runner"; +import { testEnvironment } from "./support/environment"; + +/** + * `/internal/routines/run` is the one door the worker gets: no session, no cookie, one bearer + * secret, and a route that does not exist at all unless a runner was actually built. See + * server/src/app.ts and server/src/config.ts. + */ + +const SECRET = "worker-shared-secret"; + +function stubRunner(): { runner: RoutineRunner; calls: string[] } { + const calls: string[] = []; + return { + runner: { + run: (id: string) => { + calls.push(id); + return Promise.resolve(); + }, + }, + calls, + }; +} + +/** + * `createApp` is an 18-parameter-and-growing positional function; everything after `config` is + * optional. Building the argument list explicitly, once, keeps every call site here honest about + * which slot `routineRunner` (the last one) actually lands in. + */ +function buildApp( + environment: Record, + runner: RoutineRunner | undefined, +) { + const args: Parameters = [ + loadConfig(environment), + undefined, // auth + undefined, // roleRepository + undefined, // auditReader + undefined, // credentialService + undefined, // packageStatusReader + undefined, // copilotHandler + undefined, // computerGateway + undefined, // computerPolicy + undefined, // agentProfileStore + undefined, // channelStore + undefined, // channelEvents + undefined, // auditStore + undefined, // componentStore + undefined, // pluginStore + undefined, // sandboxedStore + undefined, // threadIdentity + undefined, // peopleStore + undefined, // identityProviders + undefined, // intentRouter + undefined, // pageFrames + runner, + ]; + return createApp(...args); +} + +function appWithSecret(runner?: RoutineRunner) { + return buildApp( + { ...testEnvironment(), WORKER_SHARED_SECRET: SECRET }, + runner, + ); +} + +function appWithoutSecret(runner?: RoutineRunner) { + return buildApp( + { ...testEnvironment(), WORKER_SHARED_SECRET: undefined }, + runner, + ); +} + +async function post( + app: ReturnType, + init: RequestInit = {}, +) { + return app.request("http://openbot.local/internal/routines/run", { + method: "POST", + ...init, + }); +} + +describe("POST /internal/routines/run", () => { + test("401s with no authorization header", async () => { + const { runner } = stubRunner(); + const response = await post(appWithSecret(runner), { + body: JSON.stringify({ routineRunId: "run-1" }), + headers: { "content-type": "application/json" }, + }); + + expect(response.status).toBe(401); + }); + + test("401s with the wrong bearer secret", async () => { + const { runner } = stubRunner(); + const response = await post(appWithSecret(runner), { + body: JSON.stringify({ routineRunId: "run-1" }), + headers: { + "content-type": "application/json", + authorization: "Bearer wrong", + }, + }); + + expect(response.status).toBe(401); + }); + + test( + "401s, byte-identically to a wrong secret, when no secret is configured " + + "even with a correct-looking header", + async () => { + const { runner } = stubRunner(); + + const wrongSecretResponse = await post(appWithSecret(runner), { + body: JSON.stringify({ routineRunId: "run-1" }), + headers: { + "content-type": "application/json", + authorization: "Bearer wrong", + }, + }); + const noSecretResponse = await post(appWithoutSecret(runner), { + body: JSON.stringify({ routineRunId: "run-1" }), + headers: { + "content-type": "application/json", + authorization: `Bearer ${SECRET}`, + }, + }); + + expect(noSecretResponse.status).toBe(401); + expect(noSecretResponse.status).toBe(wrongSecretResponse.status); + await expect(noSecretResponse.json()).resolves.toEqual( + await wrongSecretResponse.json(), + ); + }, + ); + + test("400s with the right secret and no routineRunId", async () => { + const { runner } = stubRunner(); + const response = await post(appWithSecret(runner), { + body: JSON.stringify({}), + headers: { + "content-type": "application/json", + authorization: `Bearer ${SECRET}`, + }, + }); + + expect(response.status).toBe(400); + }); + + test("400s with the right secret and a non-string routineRunId", async () => { + const { runner } = stubRunner(); + const response = await post(appWithSecret(runner), { + body: JSON.stringify({ routineRunId: 12345 }), + headers: { + "content-type": "application/json", + authorization: `Bearer ${SECRET}`, + }, + }); + + expect(response.status).toBe(400); + }); + + test("202s with the right secret and a routineRunId, running it exactly once", async () => { + const { runner, calls } = stubRunner(); + const response = await post(appWithSecret(runner), { + body: JSON.stringify({ routineRunId: "run-42" }), + headers: { + "content-type": "application/json", + authorization: `Bearer ${SECRET}`, + }, + }); + + expect(response.status).toBe(202); + await expect(response.json()).resolves.toEqual({ accepted: true }); + // The response is 202 before the turn runs, so give the fire-and-forget call a tick to land. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(calls).toEqual(["run-42"]); + }); + + test("requires no session or cookie: a bearer header alone is accepted", async () => { + const { runner } = stubRunner(); + const response = await post(appWithSecret(runner), { + body: JSON.stringify({ routineRunId: "run-1" }), + headers: { + "content-type": "application/json", + authorization: `Bearer ${SECRET}`, + }, + }); + + expect(response.status).toBe(202); + }); + + test("the route does not exist at all when no runner was built", async () => { + const response = await post(appWithSecret(undefined), { + body: JSON.stringify({ routineRunId: "run-1" }), + headers: { + "content-type": "application/json", + authorization: `Bearer ${SECRET}`, + }, + }); + + expect(response.status).toBe(404); + }); +}); From c348d81693842d99a4cc2230ddaae9a23701f400 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 12:02:40 -0300 Subject: [PATCH 16/45] Leave a trace when the worker is turned away --- server/src/app.ts | 36 +++++++++++++++++++ server/src/audit.ts | 11 ++++++ server/tests/routine-endpoint.test.ts | 51 ++++++++++++++++++++++----- 3 files changed, 89 insertions(+), 9 deletions(-) diff --git a/server/src/app.ts b/server/src/app.ts index d7e4fffa..29a3ca78 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -644,6 +644,42 @@ export function createApp( * differently than a deployment with a worker and a wrong key would. */ if (!expected || !offered || !sameToken(offered, expected)) { + /* + * Recorded, because this is a boundary being held and every other one here leaves a row. A + * worker with a stale or missing secret used to fail here in total silence: every routine + * stopped firing and nothing anywhere said why, the same false negative `mcp.callback_refused` + * exists to catch on the sibling unauthenticated boundary above. + * + * The reason is short and lives only in the row, never on the wire: the response below stays + * byte-identical across all three causes on purpose (see the comment above), so this is the + * one place the distinction is allowed to exist. The offered credential itself is never + * recorded, not even a fragment of it. + */ + if (auditStore) { + try { + await recordAuditEvent(auditStore, { + eventType: "routines.dispatch_refused", + targetType: "worker", + payload: { + reason: !expected + ? "unconfigured" + : !offered + ? "missing-header" + : "mismatch", + note: "A worker's bearer secret did not check out, so no routine run was dispatched.", + }, + }); + } catch (error) { + // Never fatal: the 401 above is already decided and sent. A trail that is briefly + // unavailable is not a reason to turn a refusal into a 500. + console.error( + JSON.stringify({ + type: "routine-dispatch-audit-write-failed", + error: String(error), + }), + ); + } + } return context.json({ error: "This endpoint is the worker's." }, 401); } const body = await context.req.json().catch(() => null); diff --git a/server/src/audit.ts b/server/src/audit.ts index b7100e12..c69fdc77 100644 --- a/server/src/audit.ts +++ b/server/src/audit.ts @@ -337,6 +337,17 @@ export const auditEventTypes = [ "bot.deleted", "bot.callback_token_issued", "bot.callback_token_revoked", + /* + * A worker's bearer secret did not check out at `/internal/routines/run`, and every routine this + * deployment has stopped firing until somebody notices. + * + * Recorded because that route answers a refusal with the exact same 401 for a missing header, a + * wrong secret, and a deployment that never configured one — deliberately, so a caller cannot tell + * those apart from the wire. Which means the wire is also the only place this trail could otherwise + * be read from, and it was told nothing. `payload.reason` carries the distinction the response + * withholds; the offered credential never does. + */ + "routines.dispatch_refused", ] as const; export type AuditEventType = (typeof auditEventTypes)[number]; diff --git a/server/tests/routine-endpoint.test.ts b/server/tests/routine-endpoint.test.ts index ba80e589..91ccc06e 100644 --- a/server/tests/routine-endpoint.test.ts +++ b/server/tests/routine-endpoint.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { createApp } from "../src/app"; +import type { AuditEventInput, AuditStore } from "../src/audit"; import { loadConfig } from "../src/config"; import type { RoutineRunner } from "../src/routines/runner"; import { testEnvironment } from "./support/environment"; @@ -25,6 +26,15 @@ function stubRunner(): { runner: RoutineRunner; calls: string[] } { }; } +/** Captures every row written, so a test can assert on the reason without touching a database. */ +function recordingAuditStore(): { + store: AuditStore; + rows: AuditEventInput[]; +} { + const rows: AuditEventInput[] = []; + return { store: { insert: async (event) => void rows.push(event) }, rows }; +} + /** * `createApp` is an 18-parameter-and-growing positional function; everything after `config` is * optional. Building the argument list explicitly, once, keeps every call site here honest about @@ -33,6 +43,7 @@ function stubRunner(): { runner: RoutineRunner; calls: string[] } { function buildApp( environment: Record, runner: RoutineRunner | undefined, + auditStore?: AuditStore, ) { const args: Parameters = [ loadConfig(environment), @@ -47,7 +58,7 @@ function buildApp( undefined, // agentProfileStore undefined, // channelStore undefined, // channelEvents - undefined, // auditStore + auditStore, // auditStore undefined, // componentStore undefined, // pluginStore undefined, // sandboxedStore @@ -61,17 +72,19 @@ function buildApp( return createApp(...args); } -function appWithSecret(runner?: RoutineRunner) { +function appWithSecret(runner?: RoutineRunner, auditStore?: AuditStore) { return buildApp( { ...testEnvironment(), WORKER_SHARED_SECRET: SECRET }, runner, + auditStore, ); } -function appWithoutSecret(runner?: RoutineRunner) { +function appWithoutSecret(runner?: RoutineRunner, auditStore?: AuditStore) { return buildApp( { ...testEnvironment(), WORKER_SHARED_SECRET: undefined }, runner, + auditStore, ); } @@ -88,17 +101,22 @@ async function post( describe("POST /internal/routines/run", () => { test("401s with no authorization header", async () => { const { runner } = stubRunner(); - const response = await post(appWithSecret(runner), { + const { store, rows } = recordingAuditStore(); + const response = await post(appWithSecret(runner, store), { body: JSON.stringify({ routineRunId: "run-1" }), headers: { "content-type": "application/json" }, }); expect(response.status).toBe(401); + expect(rows).toHaveLength(1); + expect(rows[0]?.eventType).toBe("routines.dispatch_refused"); + expect(rows[0]?.payload.reason).toBe("missing-header"); }); test("401s with the wrong bearer secret", async () => { const { runner } = stubRunner(); - const response = await post(appWithSecret(runner), { + const { store, rows } = recordingAuditStore(); + const response = await post(appWithSecret(runner, store), { body: JSON.stringify({ routineRunId: "run-1" }), headers: { "content-type": "application/json", @@ -107,6 +125,9 @@ describe("POST /internal/routines/run", () => { }); expect(response.status).toBe(401); + expect(rows).toHaveLength(1); + expect(rows[0]?.eventType).toBe("routines.dispatch_refused"); + expect(rows[0]?.payload.reason).toBe("mismatch"); }); test( @@ -114,6 +135,7 @@ describe("POST /internal/routines/run", () => { "even with a correct-looking header", async () => { const { runner } = stubRunner(); + const { store, rows } = recordingAuditStore(); const wrongSecretResponse = await post(appWithSecret(runner), { body: JSON.stringify({ routineRunId: "run-1" }), @@ -122,7 +144,7 @@ describe("POST /internal/routines/run", () => { authorization: "Bearer wrong", }, }); - const noSecretResponse = await post(appWithoutSecret(runner), { + const noSecretResponse = await post(appWithoutSecret(runner, store), { body: JSON.stringify({ routineRunId: "run-1" }), headers: { "content-type": "application/json", @@ -132,9 +154,16 @@ describe("POST /internal/routines/run", () => { expect(noSecretResponse.status).toBe(401); expect(noSecretResponse.status).toBe(wrongSecretResponse.status); - await expect(noSecretResponse.json()).resolves.toEqual( - await wrongSecretResponse.json(), + // The distinction between "nobody configured a secret" and "somebody guessed wrong" must + // live in the audit row, never on the wire: the raw bodies (not just their parsed shape) + // have to match byte for byte. + expect(await noSecretResponse.text()).toBe( + await wrongSecretResponse.text(), ); + + expect(rows).toHaveLength(1); + expect(rows[0]?.eventType).toBe("routines.dispatch_refused"); + expect(rows[0]?.payload.reason).toBe("unconfigured"); }, ); @@ -166,7 +195,8 @@ describe("POST /internal/routines/run", () => { test("202s with the right secret and a routineRunId, running it exactly once", async () => { const { runner, calls } = stubRunner(); - const response = await post(appWithSecret(runner), { + const { store, rows } = recordingAuditStore(); + const response = await post(appWithSecret(runner, store), { body: JSON.stringify({ routineRunId: "run-42" }), headers: { "content-type": "application/json", @@ -179,6 +209,9 @@ describe("POST /internal/routines/run", () => { // The response is 202 before the turn runs, so give the fire-and-forget call a tick to land. await new Promise((resolve) => setTimeout(resolve, 0)); expect(calls).toEqual(["run-42"]); + // The 202 is already the record, via `routine_runs`. A dispatch that succeeded writes no + // refusal row. + expect(rows).toHaveLength(0); }); test("requires no session or cookie: a bearer header alone is accepted", async () => { From 1b7e2ae14b31993b8d86bc3feb3374aac43700a0 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 12:20:41 -0300 Subject: [PATCH 17/45] Run a routine's turn into the thread the person will read --- server/src/index.ts | 344 +++++++++++----- server/src/routines/run-turn.ts | 497 +++++++++++++++++++++++ server/tests/routine-run-turn.test.ts | 544 ++++++++++++++++++++++++++ 3 files changed, 1284 insertions(+), 101 deletions(-) create mode 100644 server/src/routines/run-turn.ts create mode 100644 server/tests/routine-run-turn.test.ts diff --git a/server/src/index.ts b/server/src/index.ts index 6347288a..9085685d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,7 +1,13 @@ +import { + CopilotKitIntelligence, + IntelligenceAgentRunner, +} from "@copilotkit/runtime/v2"; import { serve } from "bun"; +import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; import { mintRunAssertion } from "./agents/callback-token"; import { createAgentFetch } from "./agents/endpoint"; import { createAgentProfileStore } from "./agents/profile-store"; +import type { AgentActor } from "./agents/profile-types"; import { createRuntimeAgentLoader } from "./agents/runtime-agents"; import { createApp } from "./app"; import { createAuditReader, createAuditStore, recordAuditEvent } from "./audit"; @@ -38,6 +44,8 @@ import { type IdentifyActor, type IdentifyUser, mountCopilotRuntime, + resolveRuntimeAgents, + type ToolSelection, } from "./copilot"; import { createCredentialAdminService, @@ -51,6 +59,8 @@ import { useRoutineTools } from "./plugins/builtin-routines"; import { redirectUriFor } from "./plugins/oauth"; import { createPluginStore } from "./plugins/store"; import { grantedSkills, grantedTools } from "./plugins/tools"; +import { createTurnRunner } from "./routines/run-turn"; +import { createRoutineRunner } from "./routines/runner"; import { createRoutineStore } from "./routines/store"; import { createIntentRouter } from "./routing/classify"; import { createModelCompleter } from "./routing/model"; @@ -423,6 +433,231 @@ const chooseSkills = createModelCompleter({ }), }); +/* + * WHY THESE ARE NAMED CONSTANTS RATHER THAN ARGUMENTS WRITTEN INLINE. + * + * Two callers now build a Bot: a person's chat request, through `mountCopilotRuntime` below, and a + * routine's headless turn, through `buildAgentFor` further down. They have to build the SAME Bot. A + * routine that resolved its tools, its run assertion or its endpoint dialling through a second, + * slightly different set of collaborators would be a Bot that behaves one way when a person asks and + * another way at three in the morning, with nothing to point at. So each of these is written once and + * passed to both. + */ + +/** The deployment's model key, resolved per call so a credential rotated a moment ago is used next. */ +const resolveRuntimeModelApiKey = () => + resolveModelApiKey({ + encryptionKey: config.keyEncryptionKey, + reader: credentialStore, + provider: tenantPackage.model.provider, + keyId: tenantPackage.model.credentialSecretRef, + environment: process.env, + }); + +// Tools run here, not in the browser. Each one still executes through the plugin store, so the +// grant, the policy and the audit row are exactly where they were. +const loadToolsForActor = (actorId: string) => (botId: string) => + grantedTools({ store: pluginStore, botId, actorId }); + +/* + * What the deployment tells a remote Bot about the run it is starting. + * + * Signed here, where the encryption key lives, so the runtime module never holds a secret. The Bot + * hands this back when it calls a tool, and it is where the Bot id and the person's name come + * from: its own token proves which agent is calling, this proves who it is calling for, and + * neither is read out of the request body any more. + */ +const signRunForActor = (actorId: string) => (botId: string, runId: string) => + mintRunAssertion({ botId, actorId, runId }, config.keyEncryptionKey); + +/* + * Which vendors this deployment connects to, held by a Bot or not. + * + * A Bot holding no grants used to be told nothing about connectors at all, so it treated a + * connected vendor as an ordinary website and browsed to it: a Bot with no Drive grant opened + * Google's sign-in page and asked a person to sign in to an account the deployment had already + * connected. Naming them lets it say which one it has not been granted instead. + * + * Read per request rather than held, because a connector added a minute ago has to count, and + * failing is the same as having none: a Bot that cannot be told loses a sentence, not a run. + */ +const loadVendors = async () => { + try { + return (await pluginStore.listServers()).map((server) => server.id); + } catch { + return []; + } +}; + +/* + * How a run's tools are narrowed to the ones it is about. + * + * A model picks the right tool reliably out of about ten, and a deployment of this template + * clears that as soon as it connects a second vendor. Past it the wrong tool gets called, or + * none does and the answer comes from memory, and neither says so. So a Bot holding more than a + * handful is offered the tools of the skills that match the message rather than everything at + * once. See `plugins/selection.ts`. + * + * This narrows the offer and nothing else. What a Bot may call is the grant, checked in + * `callTool` with the policy and the audit row exactly as before, so every path through here can + * be wrong without a Bot gaining anything. That is also why every failure below is silent and + * lands on the whole catalogue: the narrowing is worth an accuracy point, never a capability. + */ +const selectionForActor = (actorId: string): ToolSelection => ({ + loadSkills: (botId) => grantedSkills({ store: pluginStore, botId }), + // The deployment's own model and key, the same pair the intent router uses, so selection is + // never a second thing to configure. It throws on a missing key, which reads as "could not + // choose" and leaves the whole catalogue offered. + choose: chooseSkills, + record: async (botId, selection) => { + await recordAuditEvent(bootAuditStore, { + eventType: "mcp.tools_discovered", + targetType: "bot", + targetId: botId, + actorUserId: actorId, + payload: { + bot: botId, + reason: selection.reason, + granted: selection.granted, + offered: selection.offered.length, + skills: selection.skills, + }, + }); + }, +}); + +// Every run dials the stored endpoint again, so the check that was applied when it was +// registered has to be applied to wherever it redirects now. +// Absent computer configuration means nothing opted into private hosts, which is the safe +// reading and the same one `createApp` takes. +const agentFetch = createAgentFetch({ + allowPrivateHosts: config.computer?.allowPrivateHosts === true, + // Named addresses are reachable on every hop, not only the one that was registered. + allowedHosts: config.agentEndpointAllowedHosts, + // The refusal is what the run already knows; this is what the deployment knows. Written here + // rather than in `endpoint.ts` so that file keeps deciding and nothing else, the way the + // target check it reuses does. + onRefusal: ({ address, reason }) => { + void recordAuditEvent(bootAuditStore, { + eventType: "agent.dial_refused", + targetType: "agent_endpoint", + targetId: address, + payload: { address, reason }, + }).catch((error) => { + // A trail that cannot be written must not take a refusal down with it: the request is + // already refused by the time this runs, and the alternative to a logged failure here is + // an unhandled rejection. + console.error("Could not record a refused agent dial.", error); + }); + }, +}); + +/** + * Who a routine acts as, resolved the way {@link resolveRequestActor} resolves it. + * + * THE ROLE IS READ, NOT ASSUMED. Which coworkers exist is decided per person and an administrator + * sees Bots a user does not, so hardcoding `role: "user"` here would hide an administrator's own Bots + * from their own routine — the routine would fail with "that Bot is no longer registered" for a Bot + * sitting in front of them in chat. This asks the same repository the request path asks, so a routine + * sees exactly the coworkers its owner sees. + */ +const actorFor = async (ownerUserId: string): Promise => { + // One person, and they are an administrator. The id stays the routine owner's rather than being + // rewritten to DEV_ACTOR's: in this mode they are the same person, and if they ever were not, + // silently borrowing the dev actor's identity would be worse than finding nothing. + if (config.singleUser) return { id: ownerUserId, role: DEV_ACTOR.role }; + const roles = await roleRepository.rolesForUser(ownerUserId); + if (!roles.includes("admin") && !roles.includes("user")) { + throw new Error("A routine requires an authorized owner."); + } + return { + id: ownerUserId, + role: roles.includes("admin") ? "admin" : "user", + }; +}; + +/** + * One Bot, built for a routine's turn, as its owner. + * + * Per turn rather than per boot, for the same reason the request path rebuilds: a Bot registered or + * edited since the last firing has to count, and a private coworker must be absent for everybody but + * its owner. No header and no request are involved — the owner is asserted by construction, from the + * routine row — which is the whole point of doing it here rather than adding an impersonation path to + * a public route. + */ +const buildAgentFor = async ({ + ownerUserId, + agentId, +}: { + ownerUserId: string; + agentId: string; +}) => { + const actor = await actorFor(ownerUserId); + const agents = await resolveRuntimeAgents( + () => loadAgentsForActor(actor), + tenantPackage.model, + resolveRuntimeModelApiKey, + stallGuard, + loadToolsForActor(actor.id), + signRunForActor(actor.id), + config.computer ? COMPUTER_GUIDANCE : undefined, + loadVendors, + selectionForActor(actor.id), + agentFetch, + ); + const agent = agents[agentId]; + if (!agent) { + /* + * Named, and raised rather than swallowed. The routine's Bot was deleted, or made private by + * somebody else, or the owner lost the role that could see it. The runner turns this into a + * failed run row with this sentence on it, the first failure is said once in the channel, and + * the fatigue rule switches the routine off after ten — which is exactly the right handling for + * a routine pointed at something that is not coming back. + */ + const error = new Error( + `That Bot is no longer registered, so this routine has nothing to run: ${agentId}.`, + ); + error.name = "RoutineBotNotRegistered"; + throw error; + } + return agent; +}; + +/* + * The pair a headless turn is driven through, built ONCE. + * + * Not the runtime's own pair: `mountCopilotRuntime` keeps its client and its runner inside + * `CopilotRuntime` and hands neither back, and reaching into that object would be a worse seam than + * building our own from the same three settings. Built from `config.runtime.intelligence`, which is + * required and not optional — `RuntimeCapabilities` has exactly one mode and every Intelligence field + * with it (`config.ts:10-22`), and `loadConfig` refuses to boot without them — so there is no + * not-in-Intelligence-mode branch to write here. If a second mode is ever added, THIS is the line that + * has to grow a guard, and the routine runner must then be left off `createApp` entirely. + * + * One runner for the process, reused across firings: it opens a socket per run and holds no idle + * connection, but its `threads` map is per instance, and a runner per turn would fragment the + * already-running check that keeps two turns off one thread. See `routines/run-turn.ts`. + */ +const routineIntelligence = new CopilotKitIntelligence({ + apiUrl: config.runtime.intelligence.apiUrl, + wsUrl: config.runtime.intelligence.gatewayWsUrl, + apiKey: config.runtime.intelligence.apiKey, +}); +const routineAgentRunner = new IntelligenceAgentRunner({ + url: routineIntelligence.ɵgetRunnerWsUrl(), + authToken: routineIntelligence.ɵgetRunnerAuthToken(), +}); + +const routineRunner = createRoutineRunner({ + routineStore, + channelStore, + runTurn: createTurnRunner({ + intelligence: routineIntelligence, + runner: routineAgentRunner, + buildAgentFor, + }), +}); + const app = createApp( config, auth, @@ -440,111 +675,16 @@ const app = createApp( config, tenantPackage.model, loadAgentsForActor, - () => - resolveModelApiKey({ - encryptionKey: config.keyEncryptionKey, - reader: credentialStore, - provider: tenantPackage.model.provider, - keyId: tenantPackage.model.credentialSecretRef, - environment: process.env, - }), + resolveRuntimeModelApiKey, identifyUser, identifyActor, stallGuard, - // Tools run here, not in the browser. Each one still executes through the plugin store, so the - // grant, the policy and the audit row are exactly where they were. - (actorId) => (botId) => - grantedTools({ store: pluginStore, botId, actorId }), - /* - * What the deployment tells a remote Bot about the run it is starting. - * - * Signed here, where the encryption key lives, so the runtime module never holds a secret. The Bot - * hands this back when it calls a tool, and it is where the Bot id and the person's name come - * from: its own token proves which agent is calling, this proves who it is calling for, and - * neither is read out of the request body any more. - */ - (actorId) => (botId, runId) => - mintRunAssertion({ botId, actorId, runId }, config.keyEncryptionKey), + loadToolsForActor, + signRunForActor, undefined, - /* - * Which vendors this deployment connects to, held by a Bot or not. - * - * A Bot holding no grants used to be told nothing about connectors at all, so it treated a - * connected vendor as an ordinary website and browsed to it: a Bot with no Drive grant opened - * Google's sign-in page and asked a person to sign in to an account the deployment had already - * connected. Naming them lets it say which one it has not been granted instead. - * - * Read per request rather than held, because a connector added a minute ago has to count, and - * failing is the same as having none: a Bot that cannot be told loses a sentence, not a run. - */ - async () => { - try { - return (await pluginStore.listServers()).map((server) => server.id); - } catch { - return []; - } - }, - /* - * How a run's tools are narrowed to the ones it is about. - * - * A model picks the right tool reliably out of about ten, and a deployment of this template - * clears that as soon as it connects a second vendor. Past it the wrong tool gets called, or - * none does and the answer comes from memory, and neither says so. So a Bot holding more than a - * handful is offered the tools of the skills that match the message rather than everything at - * once. See `plugins/selection.ts`. - * - * This narrows the offer and nothing else. What a Bot may call is the grant, checked in - * `callTool` with the policy and the audit row exactly as before, so every path through here can - * be wrong without a Bot gaining anything. That is also why every failure below is silent and - * lands on the whole catalogue: the narrowing is worth an accuracy point, never a capability. - */ - (actorId) => ({ - loadSkills: (botId) => grantedSkills({ store: pluginStore, botId }), - // The deployment's own model and key, the same pair the intent router uses, so selection is - // never a second thing to configure. It throws on a missing key, which reads as "could not - // choose" and leaves the whole catalogue offered. - choose: chooseSkills, - record: async (botId, selection) => { - await recordAuditEvent(bootAuditStore, { - eventType: "mcp.tools_discovered", - targetType: "bot", - targetId: botId, - actorUserId: actorId, - payload: { - bot: botId, - reason: selection.reason, - granted: selection.granted, - offered: selection.offered.length, - skills: selection.skills, - }, - }); - }, - }), - // Every run dials the stored endpoint again, so the check that was applied when it was - // registered has to be applied to wherever it redirects now. - // Absent computer configuration means nothing opted into private hosts, which is the safe - // reading and the same one `createApp` takes. - createAgentFetch({ - allowPrivateHosts: config.computer?.allowPrivateHosts === true, - // Named addresses are reachable on every hop, not only the one that was registered. - allowedHosts: config.agentEndpointAllowedHosts, - // The refusal is what the run already knows; this is what the deployment knows. Written here - // rather than in `endpoint.ts` so that file keeps deciding and nothing else, the way the - // target check it reuses does. - onRefusal: ({ address, reason }) => { - void recordAuditEvent(bootAuditStore, { - eventType: "agent.dial_refused", - targetType: "agent_endpoint", - targetId: address, - payload: { address, reason }, - }).catch((error) => { - // A trail that cannot be written must not take a refusal down with it: the request is - // already refused by the time this runs, and the alternative to a logged failure here is - // an unhandled rejection. - console.error("Could not record a refused agent dial.", error); - }); - }, - }), + loadVendors, + selectionForActor, + agentFetch, ), // The only path to an acting call. computerGateway, @@ -575,6 +715,8 @@ const app = createApp( createAttentionStore(database), // What a browsing turn's screen looked like when it finished, so the transcript can show it later. createPageFrameStore(database), + // What a due routine actually does: a turn, run as its owner, into the thread they will open. + routineRunner, ); /** diff --git a/server/src/routines/run-turn.ts b/server/src/routines/run-turn.ts new file mode 100644 index 00000000..2861d61c --- /dev/null +++ b/server/src/routines/run-turn.ts @@ -0,0 +1,497 @@ +/** + * One headless turn, run into the Intelligence thread the person will open. + * + * WRITTEN AGAINST `@copilotkit/runtime` 1.69.0, MIRRORING + * `node_modules/@copilotkit/runtime/dist/v2/runtime/core/channel-manager.mjs:189-316` + * (`runCanonicalChannelAgent`, the package's own module-private headless-turn engine) and + * `dist/v2/runtime/handlers/intelligence/run.mjs:114-127` for `persistedInputMessages`. That engine is + * not exported, so this is a hand copy of it with one addition — `getOrCreateThread` first — and it + * has to be re-read against the package whenever the runtime is upgraded. + * + * WHY NOT A SECOND MOUNTED HANDLER. A loopback that mounts a second `mountCopilotRuntime` and POSTs + * to its own run route is viable on identity grounds: `identifyUser` and `identifyActor` are both + * injectable there (`copilot.ts:915-916`), so a routine's request could assert its owner without a + * header. It was rejected on information, not on identity. The run route answers at gateway-JOIN + * rather than at completion (`run.mjs:229-247` returns `{threadId, runId, joinToken, realtime}` as + * soon as the runner has joined), so a caller learns that a turn STARTED and nothing else: no + * completion signal, no reply text, and no failure. Recovering either would need a second transport — + * a websocket back into the gateway — which is strictly more moving parts for strictly less + * information than driving the runner in-process. + * + * WHAT WE ARE REACHING INTO. Five `ɵ`-prefixed methods: `ɵgetRunnerWsUrl` and `ɵgetRunnerAuthToken` + * (at wiring time, in `index.ts`), and `ɵacquireThreadLock`, `ɵrenewThreadLock`, + * `ɵcleanupThreadLock` here. They typecheck today and are how the package's own handlers do this, but + * the `ɵ` prefix is the package saying it may change them without a major. Their request and response + * interfaces — `AcquireThreadLockRequest`, `RenewThreadLockRequest`, `CleanupThreadLockRequest` — are + * declared in `dist/v2/runtime/intelligence-platform/client.d.mts:339-367` and are NOT exported from + * `@copilotkit/runtime/v2`, so the shapes in `IntelligenceLike` below are RESTATED BY HAND. Nothing + * fails loudly when the package changes them: a renamed field would typecheck against our own + * restatement and be silently dropped on the wire. That is what the test file is for. + * + * THE LOCK LIFECYCLE IS NOW OURS TO KEEP CORRECT. In the browser path the runtime holds the lock and + * releases it; here we do. A bug in it is not a failed routine, it is a thread the person cannot chat + * in — see the `finally` block, which is the single most important thing in this file. + * + * GATEWAY AVAILABILITY IS NOW ON THE CRON RUN'S CRITICAL PATH. Driving the runner means the turn goes + * through the Intelligence gateway's Phoenix channel: it can answer `CHANNEL_JOIN_ERROR` or time out + * joining (`runner/intelligence.mjs:194-229`), and events must be durably acknowledged within + * `EVENT_DURABILITY_DEADLINE_MS = 60_000` (`intelligence.mjs:16, 505-511`) or the run fails. So a + * routine firing during an Intelligence incident fails HERE, where a turn that only called the model + * and never persisted anything would have succeeded. That trade was made deliberately: a reply nobody + * can find in the channel is not a reply, and the transcript is the whole point of a routine. + * + * WHY A SECOND RUNNER INSTANCE IS SAFE. The thread lock is a platform resource, not a process one — + * `POST /api/threads/:id/lock`, Redis-backed, keyed by thread — so a lock taken by this runner is seen + * by the runtime's runner and by every other replica. `IntelligenceAgentRunner.threads` is a local + * fast path (`intelligence.mjs:105`, "Thread already running") and nothing else, which is why the + * runner is built ONCE at wiring time and reused: one instance per turn would fragment that map, and + * two concurrent turns on one thread would then race past the local check and collide at the platform + * lock instead of failing cheaply here. + */ +import type { + AbstractAgent, + BaseEvent, + Message, + RunAgentInput, +} from "@ag-ui/client"; +import { EventType } from "@ag-ui/client"; +import { historyOrEmpty } from "../copilot"; +import type { TurnRunner } from "./runner"; + +/** + * The gap between stopping a turn and giving up on it. + * + * `abortRun` on `RunSelectedAgent` reaches the agent the run turned into, and that agent does not + * exist until `build()` resolves (`copilot.ts:649, 664-673`): during that window the wrapper has no + * `inner`, so abort is a no-op and the deadline cannot actually stop anything. This is the backstop + * that settles the promise anyway, so a firing cannot hang for ever on a build that never finishes. + * + * Injectable only so the test can exercise the backstop without waiting five real seconds for it. + */ +const DEFAULT_ABORT_GRACE_MS = 5_000; + +/** How long one headless turn may take before it is stopped. */ +const DEFAULT_TURN_TIMEOUT_MS = 5 * 60_000; + +/** + * The lock TTL and how often it is renewed. + * + * The same relationship the runtime's own handler uses: renew comfortably inside the TTL so one slow + * request does not drop a lock we still hold. The TTL matters to a person: while it is held, their + * browser's next message is refused with 409 "Thread lock denied" (`run.mjs:91`), so a lock leaked by + * a failed routine locks them out of their own conversation for exactly this long. + */ +const DEFAULT_LOCK_TTL_SECONDS = 20; +const DEFAULT_HEARTBEAT_MS = 15_000; + +/** + * One row of Intelligence history, as `ThreadMessagesResponse` declares it + * (`client.d.mts:280-302`). Restated because it is not exported. + */ +type ThreadHistoryMessage = { + id: string; + role: string; + content?: unknown; + activityType?: string; + toolCalls?: { id: string; name: string; args: string }[]; + toolCallId?: string; +}; + +/** + * The platform client, named by the methods this file calls and nothing else. + * + * Narrow and structural on purpose. It is what lets the tests drive every exit path without a + * gateway, and it is the honest documentation of how much of `CopilotKitIntelligence` a headless turn + * depends on. The real client satisfies it; see the seam note above about the `ɵ` shapes being + * restatements rather than imports. + */ +export type IntelligenceLike = { + getOrCreateThread(params: { + threadId: string; + userId: string; + agentId: string; + }): Promise; + getThreadMessages(params: { + threadId: string; + userId: string; + }): Promise<{ messages: ThreadHistoryMessage[] }>; + ɵacquireThreadLock(params: { + threadId: string; + runId: string; + userId: string; + agentId: string; + ttlSeconds?: number; + }): Promise; + /** NOTE: no `userId` and no `agentId` — renew is identified by the thread and the run alone. */ + ɵrenewThreadLock(params: { + threadId: string; + runId: string; + ttlSeconds: number; + }): Promise; + ɵcleanupThreadLock(params: { + threadId: string; + runId: string; + }): Promise; +}; + +/** + * What we subscribe to. Declared rather than imported as `Observable` so a fake is a plain + * object; the real observable satisfies it. + */ +type EventStream = { + subscribe(observer: { + next: (event: BaseEvent) => void; + error: (error: unknown) => void; + complete: () => void; + }): unknown; +}; + +/** The `IntelligenceAgentRunner`, named by the two methods this file calls. */ +export type RunnerLike = { + run(request: { + threadId: string; + agent: AbstractAgent; + input: RunAgentInput; + persistedInputMessages?: Message[]; + }): EventStream; + stop(request: { + threadId: string; + runId?: string; + }): Promise; +}; + +/** + * Convert one canonical Intelligence row into an AG-UI message. + * + * The shape at `channel-manager.mjs:337-353`, minus the managed-asset hydration that only a Slack or + * Teams attachment needs. `content ?? ""` because the platform omits content on a tool-call-only + * assistant row and AG-UI requires the field; `toolCalls` are re-nested into AG-UI's + * `{ id, type: "function", function: { name, arguments } }`; `toolCallId` is carried so a tool result + * in history still points at the call it answers. + * + * Rows with `role: "activity"` are seeded as they are. `prepareRunAgentInput` filters them out of the + * input it hands the agent (`@ag-ui/client` 0.0.57), so there is no filter to write here. + * + * Cast at the end because the platform types `role` as `string` and `content` as `unknown`, while + * `Message` is a union discriminated on `role`. There is nothing to narrow against at this boundary: + * the platform is the authority on its own history. + */ +function toAgentMessage(message: ThreadHistoryMessage): Message { + return { + id: message.id, + role: message.role, + content: message.content ?? "", + ...(message.activityType ? { activityType: message.activityType } : {}), + ...(message.toolCalls + ? { + toolCalls: message.toolCalls.map((call) => ({ + id: call.id, + type: "function", + function: { name: call.name, arguments: call.args }, + })), + } + : {}), + ...(message.toolCallId ? { toolCallId: message.toolCallId } : {}), + } as Message; +} + +/** What a message said out loud, or nothing if it did not say anything. */ +function assistantText(message: Message): string | undefined { + if (message.role !== "assistant") return undefined; + const { content } = message; + return typeof content === "string" && content.length > 0 + ? content + : undefined; +} + +export function createTurnRunner(options: { + intelligence: IntelligenceLike; + runner: RunnerLike; + /** The owner's coworkers, resolved as the owner. Built per turn, keyed by registry id. */ + buildAgentFor: (input: { + ownerUserId: string; + agentId: string; + }) => Promise; + /** How long one headless turn may take before it is stopped. */ + turnTimeoutMs?: number; + lockTtlSeconds?: number; + heartbeatMs?: number; + /** See {@link DEFAULT_ABORT_GRACE_MS}. */ + abortGraceMs?: number; +}): TurnRunner { + const { + intelligence, + runner, + buildAgentFor, + turnTimeoutMs = DEFAULT_TURN_TIMEOUT_MS, + lockTtlSeconds = DEFAULT_LOCK_TTL_SECONDS, + heartbeatMs = DEFAULT_HEARTBEAT_MS, + abortGraceMs = DEFAULT_ABORT_GRACE_MS, + } = options; + + return async ({ ownerUserId, agentId, threadId, instruction }) => { + /* + * One id for this turn, minted once. + * + * The same value goes to the lock acquire, to every renew, to `runner.stop`, and to the cleanup. + * `ɵacquireThreadLock` does echo back a canonical `threadId` and `runId` — and the Channels path + * adopts them, because a Slack thread id is not a platform one — but ours already IS the canonical + * pair: the thread was just created through `getOrCreateThread` below, and the run id is minted + * here and nowhere else. Re-minting or re-reading it is how a renew keeps a different lock alive + * than the one the cleanup releases. + */ + const runId = crypto.randomUUID(); + + /* + * THE ONE ADDITION over `runCanonicalChannelAgent`. + * + * A routine may be the very first thing to touch this (person, channel) thread. In the browser + * path the thread is created by the first message anybody sends; here there is no browser, and + * every call below — history, the lock, the run — is about a thread the platform has never heard + * of. `getOrCreateThread` is public API, idempotent, and already handles the 409 create-race + * (`client.d.mts:603-621`), so it is safe on the thousandth firing as well as the first. + */ + await intelligence.getOrCreateThread({ + threadId, + userId: ownerUserId, + agentId, + }); + + /* + * History, seeded by us because nobody else will. + * + * The browser path takes history from the request body (`handle-run.mjs:44`) and the Channels path + * loads its own; a headless turn has neither, so a routine that did not do this would ask its Bot + * the same question every night with no memory of the last answer. `historyOrEmpty` is the + * 404-on-a-fresh-thread case: `getOrCreateThread` above makes that rare, not impossible, since a + * concurrent delete is still a thing that can happen between the two calls. + */ + const history = await historyOrEmpty( + () => intelligence.getThreadMessages({ threadId, userId: ownerUserId }), + { messages: [] as ThreadHistoryMessage[] }, + ); + + const seeded = history.messages.map(toAgentMessage); + const turn = { + id: crypto.randomUUID(), + role: "user", + content: instruction, + } as Message; + const messages = [...seeded, turn]; + + /* + * WHAT THIS RUN IS ALLOWED TO PERSIST, and it is mandatory. + * + * `run.mjs:117-127`: the set subtraction on ids, not on positions. The runner defaults it to the + * WHOLE input (`intelligence.mjs:283`), so omitting it re-persists every message in the thread on + * every firing — a transcript that doubles in size every night until the person's channel is + * unreadable. + */ + const historicIds = new Set(history.messages.map((message) => message.id)); + const persistedInputMessages = messages.filter( + (message) => !historicIds.has(message.id), + ); + + /* + * The Bot, resolved as its owner, and pointed at this thread. + * + * `threadId` and the messages are assigned ON THE AGENT because that is where the runner reads + * them from: it calls `agent.runAgent(input, …)` (`intelligence.mjs:309`) and `runAgent` rebuilds + * its own `RunAgentInput` from `this.threadId`, `this.messages` and `this.state` through + * `prepareRunAgentInput`, taking only `runId`, `tools`, `context` and `forwardedProps` from what + * is passed. So an input object alone would run the right id against an empty conversation. + * + * `agent.run` is never called from here. The runner owns the run: it is what stamps canonical + * ownership on every event and pushes them to the gateway, which is the whole reason this file + * exists rather than a bare `runAgent`. + */ + const agent = await buildAgentFor({ ownerUserId, agentId }); + agent.threadId = threadId; + agent.setMessages(messages); + + const input: RunAgentInput = { + threadId, + runId, + messages, + state: agent.state, + // Empty because a headless turn has no browser to register frontend tools. What the Bot itself + // may call is decided where it is built, not here. + tools: [], + context: [], + forwardedProps: undefined, + }; + + /* + * The reply is recovered by diffing the agent, because the runner throws away what `runAgent` + * returns (`intelligence.mjs:309` awaits it and discards the `RunAgentResult`), so `newMessages` + * is unreachable from out here. This is the before-picture. + */ + const before = new Set(agent.messages.map((message) => message.id)); + const chunks: string[] = []; + const spoken = agent.subscribe({ + onTextMessageEndEvent: ({ textMessageBuffer }) => { + if (textMessageBuffer.length > 0) chunks.push(textMessageBuffer); + }, + }); + + await intelligence.ɵacquireThreadLock({ + threadId, + runId, + userId: ownerUserId, + agentId, + ttlSeconds: lockTtlSeconds, + }); + + let heartbeat: ReturnType | undefined; + let deadline: ReturnType | undefined; + let backstop: ReturnType | undefined; + let heartbeatError: unknown; + /** Whether the deadline stopped this turn. See the throw below the `finally`. */ + let stopped = false; + + const clearHeartbeat = () => { + if (heartbeat === undefined) return; + clearInterval(heartbeat); + heartbeat = undefined; + }; + + /** Stop this exact run, both ends: the agent's own abort and the runner's stop flag. */ + const stopTurn = () => { + try { + agent.abortRun(); + } catch { + // An agent that cannot be aborted must not stop us telling the runner to give up. The + // reason it refused is not actionable here and `runner.stop` is the half that matters: + // it sets `stopRequested`, which is what makes `finalizeRunEvents` close the run as + // stopped rather than leaving it open for ever on the platform. + } + void runner.stop({ threadId, runId }).catch(() => undefined); + }; + + heartbeat = setInterval(() => { + void intelligence + .ɵrenewThreadLock({ threadId, runId, ttlSeconds: lockTtlSeconds }) + .catch((error: unknown) => { + if (heartbeat === undefined) return; + /* + * A lock we no longer hold means somebody else is in this thread — the person, most + * likely, having just typed something. Continuing would write this turn's events into + * their run, so the turn is stopped and the failure is raised rather than recovered. + */ + clearHeartbeat(); + heartbeatError = error; + stopTurn(); + }); + }, heartbeatMs); + // So a heartbeat that is still pending cannot hold a one-shot process open. + heartbeat.unref?.(); + + try { + const completed = new Promise((resolve, reject) => { + let terminal: Error | undefined; + runner + .run({ threadId, agent, input, persistedInputMessages }) + .subscribe({ + /* + * RUN_ERROR THROUGH `next` IS TERMINAL. The Intelligence runner reports a failed run by + * emitting RUN_ERROR and then COMPLETING the observable (`intelligence.mjs:317-340`) — + * `error` is only for a socket or durability failure. A RUN_ERROR not caught here would + * therefore arrive as a successful completion, and the turn would look like a Bot that + * answered with nothing. + */ + next: (event) => { + if (event.type !== EventType.RUN_ERROR || terminal) return; + const message = + "message" in event && typeof event.message === "string" + ? event.message + : "The routine's turn failed."; + terminal = new Error(message); + terminal.name = "RoutineTurnRunError"; + }, + error: reject, + complete: () => { + if (terminal) reject(terminal); + else resolve(); + }, + }); + }); + + const timeout = new Promise((_resolve, reject) => { + deadline = setTimeout(() => { + stopped = true; + stopTurn(); + }, turnTimeoutMs); + deadline.unref?.(); + backstop = setTimeout(() => { + reject( + new Error( + `The routine's turn did not finish within ${Math.round(turnTimeoutMs / 1000)}s and could not be stopped.`, + ), + ); + }, turnTimeoutMs + abortGraceMs); + backstop.unref?.(); + }); + + await Promise.race([completed, timeout]); + } finally { + /* + * THE SINGLE MOST IMPORTANT LINES IN THIS FILE, on every exit path — success, a thrown run, the + * deadline, a failed heartbeat. + * + * While this lock is held, the person's next browser message is refused with 409 "Thread lock + * denied" (`run.mjs:85-92`) for the whole TTL. A routine that fails quietly and leaks its lock + * does not just fail: it locks somebody out of their own conversation, at three in the morning, + * for a reason no screen explains. `.catch` because a cleanup that cannot be reached must not + * replace the real failure with a second one — the TTL is the backstop for that case. + */ + clearHeartbeat(); + if (deadline !== undefined) clearTimeout(deadline); + if (backstop !== undefined) clearTimeout(backstop); + spoken.unsubscribe(); + await intelligence + .ɵcleanupThreadLock({ threadId, runId }) + .catch(() => undefined); + } + + // Raised after the lock is released, and ahead of any reply: a turn that lost its lock partway + // through is not a turn that answered, however much text it produced first. + if (heartbeatError !== undefined) throw heartbeatError; + + /* + * And the same for a turn the deadline stopped, even when the abort worked and the run then + * completed inside the grace window. A stopped run is a truncated one: whatever text it had + * reached is half a sentence, and returning it here would post it into the channel as the answer + * and close the firing as a success. + */ + if (stopped) { + throw new Error( + `The routine's turn was stopped after ${Math.round(turnTimeoutMs / 1000)}s.`, + ); + } + + const said = agent.messages + .filter((message) => !before.has(message.id)) + .map(assistantText) + .filter((text): text is string => text !== undefined); + // The diff first, the streamed chunks as the fallback: the diff is what was persisted, which is + // what the person will read in the channel, and the chunks are only what went past. + const replyText = (said.length > 0 ? said : chunks).join("\n\n"); + if (replyText.length === 0) { + throw new Error("The turn finished without saying anything."); + } + /* + * An interrupt is an unfinished turn with nobody to ask. + * + * The Bot stopped to put a question to a person who is not there, so whatever it said first is + * half of an exchange. Posting it as the answer would be the worst of the options: the routine + * would read as successful and the channel would carry a reply that is waiting on something. + */ + if (agent.pendingInterrupts.length > 0) { + throw new Error( + "The turn stopped to ask a question, and a routine has nobody to ask.", + ); + } + + return { replyText }; + }; +} diff --git a/server/tests/routine-run-turn.test.ts b/server/tests/routine-run-turn.test.ts new file mode 100644 index 00000000..05a5bae5 --- /dev/null +++ b/server/tests/routine-run-turn.test.ts @@ -0,0 +1,544 @@ +import { AbstractAgent, EventType } from "@ag-ui/client"; +import { describe, expect, test } from "bun:test"; +import { EMPTY } from "rxjs"; +import { createTurnRunner } from "../src/routines/run-turn"; + +/** + * A headless turn, asserted without a gateway, without a database and without a model. + * + * This file exists because `run-turn.ts` RESTATES BY HAND five `ɵ`-prefixed request shapes that + * `@copilotkit/runtime` does not export. Nothing else in the repository can catch a lock that is + * acquired and never released, a renew that keeps a different lock alive than the one the cleanup + * releases, or a `persistedInputMessages` that quietly re-persists a whole transcript — and every one + * of those is felt by a person rather than by a test: a leaked lock refuses their next browser message + * with 409 for the whole TTL, so a routine that fails at three in the morning locks them out of their + * own conversation. + * + * So the properties here are the lifecycle ones: cleanup exactly once on every exit path, one run id + * everywhere, the subtraction, and the order the three platform calls happen in. + */ + +const OWNER = "user_owner"; +const AGENT_ID = "bot_helper"; +const THREAD_ID = "thread_owner_channel_1"; +const INSTRUCTION = "Post the standup summary."; + +type HistoryRow = { + id: string; + role: string; + content?: unknown; + activityType?: string; + toolCalls?: { id: string; name: string; args: string }[]; + toolCallId?: string; +}; + +/** A `PlatformRequestError` as `isMissingThread` matches it: the name and the status, nothing else. */ +function threadNotFound(): Error { + const error = new Error("THREAD_NOT_FOUND"); + error.name = "PlatformRequestError"; + (error as Error & { status?: number }).status = 404; + return error; +} + +class FakeAgent extends AbstractAgent { + aborts = 0; + /** Set by a driver that wants the run to end when the turn is stopped. */ + onAbort?: () => void; + + run() { + return EMPTY; + } + + override abortRun(): void { + this.aborts += 1; + this.onAbort?.(); + super.abortRun(); + } +} + +type Observer = { + next: (event: { type: string; message?: string }) => void; + error: (error: unknown) => void; + complete: () => void; +}; + +type Driver = (context: { + agent: FakeAgent; + observer: Observer; + request: { input: { runId: string; threadId: string } }; +}) => void; + +/** The default: the Bot answers, the way the runner leaves the answer on the agent it was passed. */ +const answers: Driver = ({ agent, observer }) => { + agent.messages = [ + ...agent.messages, + { id: "assistant_1", role: "assistant", content: "Three things happened." }, + ] as typeof agent.messages; + observer.complete(); +}; + +function harness(options: { + history?: HistoryRow[]; + historyFails?: () => Error; + drive?: Driver; + /** + * What a renew does. A thunk that THROWS rather than one that returns a rejected promise: a + * pre-rejected promise handed back through the async fake below is briefly handler-less while the + * async function adopts it, which the test runner reports as an uncaught error even though the + * code under test catches it. + */ + renew?: () => unknown; + turnTimeoutMs?: number; + abortGraceMs?: number; + heartbeatMs?: number; + lockTtlSeconds?: number; +}) { + const order: string[] = []; + const calls = { + threads: [] as { threadId: string; userId: string; agentId: string }[], + acquired: [] as { + threadId: string; + runId: string; + userId: string; + agentId: string; + ttlSeconds?: number; + }[], + renewed: [] as { threadId: string; runId: string; ttlSeconds: number }[], + cleaned: [] as { threadId: string; runId: string }[], + runs: [] as { + threadId: string; + input: { runId: string; messages: { id: string }[] }; + persistedInputMessages?: { id: string; content?: unknown }[]; + }[], + stops: [] as { threadId: string; runId?: string }[], + }; + + const agent = new FakeAgent({ agentId: AGENT_ID }); + const drive = options.drive ?? answers; + + const intelligence = { + getOrCreateThread: async (params: { + threadId: string; + userId: string; + agentId: string; + }) => { + order.push("getOrCreateThread"); + calls.threads.push(params); + return { thread: { id: params.threadId }, created: false }; + }, + getThreadMessages: async () => { + order.push("getThreadMessages"); + if (options.historyFails) throw options.historyFails(); + return { messages: options.history ?? [] }; + }, + ɵacquireThreadLock: async (params: { + threadId: string; + runId: string; + userId: string; + agentId: string; + ttlSeconds?: number; + }) => { + order.push("acquire"); + calls.acquired.push(params); + return { threadId: params.threadId, runId: params.runId, joinToken: "t" }; + }, + ɵrenewThreadLock: async (params: { + threadId: string; + runId: string; + ttlSeconds: number; + }) => { + calls.renewed.push(params); + if (options.renew) return options.renew(); + return { ttlSeconds: params.ttlSeconds }; + }, + ɵcleanupThreadLock: async (params: { threadId: string; runId: string }) => { + order.push("cleanup"); + calls.cleaned.push(params); + }, + }; + + const runner = { + run: (request: { + threadId: string; + agent: unknown; + input: { runId: string; threadId: string; messages: { id: string }[] }; + persistedInputMessages?: { id: string; content?: unknown }[]; + }) => { + order.push("run"); + calls.runs.push(request); + return { + subscribe(observer: Observer) { + drive({ agent: request.agent as FakeAgent, observer, request }); + return { unsubscribe: () => undefined }; + }, + }; + }, + stop: async (request: { threadId: string; runId?: string }) => { + order.push("stop"); + calls.stops.push(request); + return true; + }, + }; + + const runTurn = createTurnRunner({ + // biome-ignore lint/suspicious/noExplicitAny: narrow structural fakes, on purpose. + intelligence: intelligence as any, + // biome-ignore lint/suspicious/noExplicitAny: narrow structural fakes, on purpose. + runner: runner as any, + buildAgentFor: async () => agent, + ...(options.turnTimeoutMs === undefined + ? {} + : { turnTimeoutMs: options.turnTimeoutMs }), + ...(options.abortGraceMs === undefined + ? {} + : { abortGraceMs: options.abortGraceMs }), + ...(options.heartbeatMs === undefined + ? {} + : { heartbeatMs: options.heartbeatMs }), + ...(options.lockTtlSeconds === undefined + ? {} + : { lockTtlSeconds: options.lockTtlSeconds }), + }); + + const run = () => + runTurn({ + ownerUserId: OWNER, + agentId: AGENT_ID, + threadId: THREAD_ID, + instruction: INSTRUCTION, + }); + + return { run, agent, calls, order }; +} + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const THREE_ROWS: HistoryRow[] = [ + { id: "m1", role: "user", content: "Hello." }, + { id: "m2", role: "assistant", content: "Hello back." }, + { id: "m3", role: "user", content: "Again." }, +]; + +describe("a routine's headless turn", () => { + test("creates the thread, takes the lock, then runs — in that order", async () => { + const { run, order, calls } = harness({}); + + await run(); + + expect(order.indexOf("getOrCreateThread")).toBeLessThan( + order.indexOf("acquire"), + ); + expect(order.indexOf("acquire")).toBeLessThan(order.indexOf("run")); + expect(calls.threads).toEqual([ + { threadId: THREAD_ID, userId: OWNER, agentId: AGENT_ID }, + ]); + }); + + test("returns what the Bot said, taken from the agent the runner was handed", async () => { + const { run } = harness({}); + + expect(await run()).toEqual({ replyText: "Three things happened." }); + }); + + test("seeds the thread's history and the turn onto the agent", async () => { + const { run, agent } = harness({ + history: [ + ...THREE_ROWS, + { + id: "m4", + role: "assistant", + toolCalls: [{ id: "call_1", name: "search", args: '{"q":"x"}' }], + }, + ], + }); + + await run(); + + expect(agent.threadId).toBe(THREAD_ID); + // The four history rows, then this turn's instruction, then what the run added. + expect(agent.messages.map((message) => message.id).slice(0, 4)).toEqual([ + "m1", + "m2", + "m3", + "m4", + ]); + // A tool-call-only row has no content on the platform, and AG-UI requires the field. + expect(agent.messages[3]).toMatchObject({ + content: "", + toolCalls: [ + { + id: "call_1", + type: "function", + function: { name: "search", arguments: '{"q":"x"}' }, + }, + ], + }); + expect(agent.messages[4]).toMatchObject({ + role: "user", + content: INSTRUCTION, + }); + }); + + test("a thread the platform has never heard of reads as no history", async () => { + const { run, calls } = harness({ historyFails: threadNotFound }); + + await run(); + + expect(calls.runs[0]?.input.messages).toHaveLength(1); + }); +}); + +describe("persistedInputMessages is the subtraction", () => { + test("a history of three plus one new message persists exactly the new one", async () => { + const { run, calls } = harness({ history: THREE_ROWS }); + + await run(); + + const [request] = calls.runs; + expect(request?.input.messages).toHaveLength(4); + expect(request?.persistedInputMessages).toHaveLength(1); + expect(request?.persistedInputMessages?.[0]?.content).toBe(INSTRUCTION); + // Identified by id, not by position: none of the history's ids may appear. + const historic = new Set(THREE_ROWS.map((row) => row.id)); + for (const message of request?.persistedInputMessages ?? []) { + expect(historic.has(message.id)).toBe(false); + } + }); + + test("an empty history persists everything", async () => { + const { run, calls } = harness({ history: [] }); + + await run(); + + const [request] = calls.runs; + expect(request?.persistedInputMessages).toHaveLength(1); + expect(request?.persistedInputMessages?.length).toBe( + request?.input.messages.length, + ); + }); +}); + +describe("the lock is released on every exit path", () => { + test("on success", async () => { + const { run, calls } = harness({}); + + await run(); + + expect(calls.cleaned).toEqual([ + { threadId: THREAD_ID, runId: calls.acquired[0]?.runId ?? "" }, + ]); + }); + + test("when the run rejects", async () => { + const { run, calls } = harness({ + drive: ({ observer }) => observer.error(new Error("the socket died")), + }); + + await expect(run()).rejects.toThrow("the socket died"); + + expect(calls.cleaned).toEqual([ + { threadId: THREAD_ID, runId: calls.acquired[0]?.runId ?? "" }, + ]); + }); + + test("when the deadline fires", async () => { + const { run, calls, agent } = harness({ + // Never finishes and never notices the abort: the backstop is what settles this. + drive: () => undefined, + turnTimeoutMs: 5, + abortGraceMs: 5, + }); + + await expect(run()).rejects.toThrow("could not be stopped"); + + expect(agent.aborts).toBe(1); + expect(calls.cleaned).toEqual([ + { threadId: THREAD_ID, runId: calls.acquired[0]?.runId ?? "" }, + ]); + }); + + test("when a heartbeat renew rejects", async () => { + const { run, calls } = harness({ + drive: ({ agent, observer }) => { + agent.onAbort = () => observer.complete(); + }, + heartbeatMs: 2, + renew: () => { + throw new Error("somebody else holds this lock"); + }, + }); + + await expect(run()).rejects.toThrow("somebody else holds this lock"); + + expect(calls.cleaned).toEqual([ + { threadId: THREAD_ID, runId: calls.acquired[0]?.runId ?? "" }, + ]); + // And the timer really was cleared: no second renew, however long we wait. + const renews = calls.renewed.length; + await wait(20); + expect(calls.renewed.length).toBe(renews); + }); + + test("when the deadline fires and the run then finishes inside the grace", async () => { + const { run, calls } = harness({ + // The abort works: the run ends, with an answer on the agent. It is still a stopped turn, and + // half a sentence must not be posted into the channel as if it were the reply. + drive: (context) => { + context.agent.onAbort = () => answers(context); + }, + turnTimeoutMs: 5, + abortGraceMs: 50, + }); + + await expect(run()).rejects.toThrow("was stopped after"); + + expect(calls.cleaned).toEqual([ + { threadId: THREAD_ID, runId: calls.acquired[0]?.runId ?? "" }, + ]); + }); +}); + +describe("one run id, everywhere", () => { + test("reaches the acquire, every renew and the cleanup", async () => { + const { run, calls } = harness({ + heartbeatMs: 2, + drive: ({ observer, agent }) => { + setTimeout( + () => answers({ observer, agent, request: null as never }), + 20, + ); + }, + }); + + await run(); + + const runId = calls.acquired[0]?.runId; + expect(typeof runId).toBe("string"); + expect(calls.renewed.length).toBeGreaterThan(1); + for (const renew of calls.renewed) { + expect(renew).toEqual({ threadId: THREAD_ID, runId, ttlSeconds: 20 }); + } + expect(calls.cleaned).toEqual([{ threadId: THREAD_ID, runId }]); + expect(calls.runs[0]?.input.runId).toBe(runId); + }); + + test("reaches runner.stop when the turn is stopped", async () => { + const { run, calls } = harness({ + drive: () => undefined, + turnTimeoutMs: 5, + abortGraceMs: 5, + }); + + await expect(run()).rejects.toThrow("could not be stopped"); + + const runId = calls.acquired[0]?.runId; + expect(calls.stops).toEqual([{ threadId: THREAD_ID, runId }]); + }); +}); + +describe("a failed heartbeat stops the turn", () => { + test("aborts the agent, stops the run, and rethrows", async () => { + const { run, calls, agent } = harness({ + drive: ({ agent: driven, observer }) => { + driven.onAbort = () => observer.complete(); + }, + heartbeatMs: 2, + renew: () => { + throw new Error("lock lost"); + }, + }); + + await expect(run()).rejects.toThrow("lock lost"); + + expect(agent.aborts).toBe(1); + expect(calls.stops).toEqual([ + { threadId: THREAD_ID, runId: calls.acquired[0]?.runId }, + ]); + }); + + test("does not return a reply, even when the Bot had already answered", async () => { + const { run } = harness({ + drive: ({ agent, observer }) => { + agent.onAbort = () => { + answers({ agent, observer, request: null as never }); + }; + }, + heartbeatMs: 2, + renew: () => { + throw new Error("lock lost"); + }, + }); + + await expect(run()).rejects.toThrow("lock lost"); + }); +}); + +describe("recovering what was said", () => { + test("falls back to the streamed chunks when no message was added", async () => { + const { run } = harness({ + drive: ({ agent, observer }) => { + for (const subscriber of agent.subscribers) { + void subscriber.onTextMessageEndEvent?.({ + event: { + type: EventType.TEXT_MESSAGE_END, + messageId: "streamed_1", + }, + textMessageBuffer: "Said out loud but never persisted.", + messages: agent.messages, + state: agent.state, + agent, + // biome-ignore lint/suspicious/noExplicitAny: the subscriber params are not the subject. + } as any); + } + observer.complete(); + }, + }); + + expect(await run()).toEqual({ + replyText: "Said out loud but never persisted.", + }); + }); + + test("throws when the turn finished without saying anything", async () => { + const { run, calls } = harness({ + drive: ({ observer }) => observer.complete(), + }); + + await expect(run()).rejects.toThrow( + "The turn finished without saying anything.", + ); + expect(calls.cleaned).toHaveLength(1); + }); + + test("throws when the turn stopped to ask a question", async () => { + const { run, calls } = harness({ + drive: (context) => { + context.agent.pendingInterrupts = [ + // biome-ignore lint/suspicious/noExplicitAny: the interrupt's shape is not the subject. + { id: "interrupt_1" } as any, + ]; + answers(context); + }, + }); + + await expect(run()).rejects.toThrow("nobody to ask"); + expect(calls.cleaned).toHaveLength(1); + }); +}); + +describe("a RUN_ERROR through next", () => { + test("rejects rather than hanging, and does not read as an empty answer", async () => { + const { run, calls } = harness({ + drive: ({ observer }) => { + observer.next({ + type: EventType.RUN_ERROR, + message: "the model refused", + }); + observer.complete(); + }, + }); + + await expect(run()).rejects.toThrow("the model refused"); + expect(calls.cleaned).toHaveLength(1); + }); +}); From 5117d3910fbfcc0f5fad73a65877e07d46926470 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 12:34:25 -0300 Subject: [PATCH 18/45] Stop once, clean only what was taken, and name the real reason --- server/src/routines/run-turn.ts | 40 ++++++++++---- server/tests/routine-run-turn.test.ts | 75 +++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 11 deletions(-) diff --git a/server/src/routines/run-turn.ts b/server/src/routines/run-turn.ts index 2861d61c..137d2e9f 100644 --- a/server/src/routines/run-turn.ts +++ b/server/src/routines/run-turn.ts @@ -348,6 +348,15 @@ export function createTurnRunner(options: { let heartbeatError: unknown; /** Whether the deadline stopped this turn. See the throw below the `finally`. */ let stopped = false; + /** + * `stopCanonicalRun`'s shape (`channel-manager.mjs:222-229`): one promise for the whole turn, + * not one per caller. Both the heartbeat-reject path and the deadline path call `stopTurn`, and + * without the `??=` each would issue its own `runner.stop`, which is two stops racing each other + * for one run id. The seam note above about the acquire echo applies here too: if this file ever + * adopts the acquired `threadId`/`runId` instead of minting its own, it must guard the echo the + * way `run.mjs:94` does — `lock.threadId || threadId` — before trusting it, not use it bare. + */ + let stopPromise: Promise | undefined; const clearHeartbeat = () => { if (heartbeat === undefined) return; @@ -365,7 +374,7 @@ export function createTurnRunner(options: { // it sets `stopRequested`, which is what makes `finalizeRunEvents` close the run as // stopped rather than leaving it open for ever on the platform. } - void runner.stop({ threadId, runId }).catch(() => undefined); + stopPromise ??= runner.stop({ threadId, runId }).catch(() => undefined); }; heartbeat = setInterval(() => { @@ -454,8 +463,13 @@ export function createTurnRunner(options: { } // Raised after the lock is released, and ahead of any reply: a turn that lost its lock partway - // through is not a turn that answered, however much text it produced first. - if (heartbeatError !== undefined) throw heartbeatError; + // through is not a turn that answered, however much text it produced first. `stopPromise` is + // awaited first — the reference's own order (`channel-manager.mjs:311-313`) — so a stop this + // path itself requested has actually settled before we report on it, not just been requested. + if (heartbeatError !== undefined) { + await stopPromise; + throw heartbeatError; + } /* * And the same for a turn the deadline stopped, even when the abort worked and the run then @@ -464,6 +478,7 @@ export function createTurnRunner(options: { * and close the firing as a success. */ if (stopped) { + await stopPromise; throw new Error( `The routine's turn was stopped after ${Math.round(turnTimeoutMs / 1000)}s.`, ); @@ -476,21 +491,24 @@ export function createTurnRunner(options: { // The diff first, the streamed chunks as the fallback: the diff is what was persisted, which is // what the person will read in the channel, and the chunks are only what went past. const replyText = (said.length > 0 ? said : chunks).join("\n\n"); - if (replyText.length === 0) { - throw new Error("The turn finished without saying anything."); - } + /* - * An interrupt is an unfinished turn with nobody to ask. - * - * The Bot stopped to put a question to a person who is not there, so whatever it said first is - * half of an exchange. Posting it as the answer would be the worst of the options: the routine - * would read as successful and the channel would carry a reply that is waiting on something. + * An interrupt is an unfinished turn with nobody to ask, and it is checked BEFORE the empty-reply + * case below. A turn that interrupted before saying anything has both conditions true at once, + * and only one sentence can go on the run row and into the channel: "finished without saying + * anything" would be a lie about a turn that in fact stopped to ask a question. The Bot stopped + * to put a question to a person who is not there, so whatever it said first is half of an + * exchange. Posting it as the answer would be the worst of the options: the routine would read as + * successful and the channel would carry a reply that is waiting on something. */ if (agent.pendingInterrupts.length > 0) { throw new Error( "The turn stopped to ask a question, and a routine has nobody to ask.", ); } + if (replyText.length === 0) { + throw new Error("The turn finished without saying anything."); + } return { replyText }; }; diff --git a/server/tests/routine-run-turn.test.ts b/server/tests/routine-run-turn.test.ts index 05a5bae5..187c459d 100644 --- a/server/tests/routine-run-turn.test.ts +++ b/server/tests/routine-run-turn.test.ts @@ -80,6 +80,8 @@ const answers: Driver = ({ agent, observer }) => { function harness(options: { history?: HistoryRow[]; historyFails?: () => Error; + /** What the acquire call does. A thunk that throws, for the same reason `renew` is one. */ + acquireFails?: () => Error; drive?: Driver; /** * What a renew does. A thunk that THROWS rather than one that returns a rejected promise: a @@ -139,6 +141,7 @@ function harness(options: { ttlSeconds?: number; }) => { order.push("acquire"); + if (options.acquireFails) throw options.acquireFails(); calls.acquired.push(params); return { threadId: params.threadId, runId: params.runId, joinToken: "t" }; }, @@ -240,6 +243,17 @@ describe("a routine's headless turn", () => { expect(await run()).toEqual({ replyText: "Three things happened." }); }); + test("does not leak history into the reply: the before-set must be taken after seeding, not before", async () => { + // A non-empty history containing an existing assistant row ("Hello back.") is the regression + // guard the empty-history tests above cannot provide: if `before` were ever taken ABOVE + // `agent.setMessages(messages)` instead of below it, that seeded row would read as "new" too, + // and the reply would come back as "Hello back.\n\nThree things happened." instead of just the + // one line the run actually added. + const { run } = harness({ history: THREE_ROWS }); + + expect(await run()).toEqual({ replyText: "Three things happened." }); + }); + test("seeds the thread's history and the turn onto the agent", async () => { const { run, agent } = harness({ history: [ @@ -398,6 +412,27 @@ describe("the lock is released on every exit path", () => { }); }); +describe("cleanup only runs for a lock that was actually taken", () => { + test("a rejected acquire never cleans up, renews, or starts the heartbeat", async () => { + // `ɵcleanupThreadLock` is DELETE on the platform's lock endpoint. If cleanup ran when the + // acquire itself failed, it would delete whoever DOES hold the lock — the person's own browser + // session, most likely. So this is not just "no cleanup call happened to be made", it is "no + // cleanup call may ever be made when we never held anything to begin with". + const { run, calls } = harness({ + acquireFails: () => new Error("lock service unavailable"), + heartbeatMs: 2, + }); + + await expect(run()).rejects.toThrow("lock service unavailable"); + + expect(calls.cleaned).toEqual([]); + expect(calls.renewed).toEqual([]); + // And no heartbeat was ever scheduled: still nothing, even after it would have ticked. + await wait(20); + expect(calls.renewed).toEqual([]); + }); +}); + describe("one run id, everywhere", () => { test("reaches the acquire, every renew and the cleanup", async () => { const { run, calls } = harness({ @@ -434,6 +469,27 @@ describe("one run id, everywhere", () => { const runId = calls.acquired[0]?.runId; expect(calls.stops).toEqual([{ threadId: THREAD_ID, runId }]); }); + + test("stops the run exactly once when the heartbeat rejects and the deadline also fires", async () => { + // Two independent callers of `stopTurn` — the heartbeat-reject path and the deadline path — can + // both fire in the same run. `runner.stop` deletes the platform's stop-requested flag work for + // one run id; issuing it twice is not double-safe the way the lock cleanup's `.catch` is, it is + // just two racing calls. The `stopPromise ??=` dedup (mirroring `channel-manager.mjs:222-229`) + // is what keeps this to exactly one call regardless of which path got there first. + const { run, calls } = harness({ + drive: () => undefined, + heartbeatMs: 2, + turnTimeoutMs: 20, + abortGraceMs: 50, + renew: () => { + throw new Error("somebody else holds this lock"); + }, + }); + + await expect(run()).rejects.toThrow("could not be stopped"); + + expect(calls.stops).toHaveLength(1); + }); }); describe("a failed heartbeat stops the turn", () => { @@ -524,6 +580,25 @@ describe("recovering what was said", () => { await expect(run()).rejects.toThrow("nobody to ask"); expect(calls.cleaned).toHaveLength(1); }); + + test("throws the interrupt sentence, not the empty-reply one, when the turn interrupted before saying anything", async () => { + // Both conditions are true at once here: no new assistant message AND a pending interrupt. Only + // one sentence can go on the run row and into the channel, and "finished without saying + // anything" would be a lie about a turn that in fact stopped to ask a question. + const { run } = harness({ + drive: ({ observer, agent }) => { + agent.pendingInterrupts = [ + // biome-ignore lint/suspicious/noExplicitAny: the interrupt's shape is not the subject. + { id: "interrupt_1" } as any, + ]; + observer.complete(); + }, + }); + + await expect(run()).rejects.toThrow( + "The turn stopped to ask a question, and a routine has nobody to ask.", + ); + }); }); describe("a RUN_ERROR through next", () => { From b48820238e70fa0f03c3692c248b0267b25fa8ea Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 12:43:10 -0300 Subject: [PATCH 19/45] Offer every due routine to the shared queue, once --- server/src/routines/sweep.ts | 180 +++++++ .../tests/routine-sweep.integration.test.ts | 467 ++++++++++++++++++ 2 files changed, 647 insertions(+) create mode 100644 server/src/routines/sweep.ts create mode 100644 server/tests/routine-sweep.integration.test.ts diff --git a/server/src/routines/sweep.ts b/server/src/routines/sweep.ts new file mode 100644 index 00000000..aaa4dbed --- /dev/null +++ b/server/src/routines/sweep.ts @@ -0,0 +1,180 @@ +/** + * Turning "this routine is due" into exactly one firing, on a clock, across replicas. + * + * TWO HALVES, LIKE `server/src/work/culler.ts`, and separated for its reason: deciding what should + * fire is not the same act as firing it. This half reads the ledger and puts an item on the shared + * queue; the next one claims those items and dispatches them. So whichever replica noticed does not + * have to be the one that carries it out, and a dispatch that dies halfway is picked up by whoever + * claims it next rather than lost with the process that saw it was due. + * + * Two free functions over one options type rather than a factory, again like the culler, so the two + * halves cannot drift about what a lease, an owner or a limit means: there is one description of the + * things they share, and both read it. + * + * THE OFFER KEY IS THE WHOLE IDEMPOTENCE STORY. It carries the minute the firing was due, so three + * replicas waking at 09:00 produce one work item and one run. That holds only while every replica + * renders that minute identically, which is why the format is pinned in one function below and + * asserted literally in `server/tests/routine-sweep.integration.test.ts`. + * + * NO CLAIM OR LEASE MACHINERY HERE, and none on the routines table. `server/src/work/queue.ts` + * already owns `for update skip locked`, leases named on the database's clock and an attempt count. + * A second half-right copy grown next to it is the duplicated firing mechanism #235 exists to + * prevent. + */ +import type { WorkQueue } from "../work/queue"; +import type { RoutineStore } from "./store"; + +export const ROUTINE_FIRE_KIND = "routine.fire"; + +/** How many due routines one pass will look at. Bounded, because a pass has to end. */ +const DEFAULT_LIMIT = 50; + +/** + * How late a firing may be and still be worth having. + * + * Ten minutes: several sweep intervals plus a slow pass, so a firing delayed by a deploy, a restart + * or a busy queue is still delivered rather than silently dropped — and comfortably under the + * fifteen-minute floor a routine's schedule may have (`MINIMUM_INTERVAL_MS` in `./schedule`), so the + * window can never call two consecutive occurrences of one routine current at the same time. + */ +const DEFAULT_GRACE_MS = 10 * 60_000; + +export type RoutineSweepOptions = { + routineStore: RoutineStore; + /** The shared `work_items` queue. Not a second queue, and not a timer. */ + queue: WorkQueue; + /** POST /internal/routines/run. Throws on anything that is not a 202. */ + dispatch: (routineRunId: string) => Promise; + /** Who this process is, for the lease. A name, so a stuck claim traces back to a pod. */ + owner: string; + /** Lease for a claimed firing; phase two is what applies it. Default 60_000. */ + leaseMs?: number; + /** How many goes one firing gets before it stops being offered. Default `DEFAULT_MAX_ATTEMPTS`. */ + maxAttempts?: number; + /** How many due routines one pass considers. Default 50. */ + limit?: number; + /** How late a firing may be and still be offered. Default ten minutes; see the policy below. */ + graceMs?: number; + now?: () => Date; +}; + +/** + * What one pass of phase two did. + * + * Declared here with the options it shares because PHASE TWO ARRIVES NEXT — the half that claims + * these items, opens a run row and dispatches it. Nothing in this file returns one yet. + */ +export type RoutineSweepReport = { + considered: number; + fired: string[]; + skipped: { routineId: string; reason: string }[]; +}; + +/** + * The minute a firing was due, rendered the same way by every replica. + * + * `2026-08-26T09:30Z`: ISO, truncated to the minute, always UTC, no seconds and no fractional part. + * This string is half of the offer key, so THE FORMAT IS THE IDEMPOTENCE — two sweeps that render one + * due moment differently offer two items and a person gets two runs of the same routine. Changing it + * also orphans every key already in `work_items`, which is why the tests assert it literally rather + * than recomputing it. + * + * Truncated rather than rounded, so a stamp never renders as a minute it is not in, and UTC by + * construction: `toISOString` has no local component, so a replica in another zone cannot name the + * same moment differently. + */ +function minuteKey(due: Date): string { + // "2026-08-26T09:30:00.000Z" -> "2026-08-26T09:30" -> back with the zone it never left. + return `${due.toISOString().slice(0, 16)}Z`; +} + +/** + * Phase one: due routines become idempotent work items, and `next_run_at` moves exactly once. + * + * WHICH CLOCK, given that everything around this names its moments in SQL. Both moments that decide + * anything come from the database: `dueRoutines` compares `next_run_at <= now()` inside Postgres, so + * what is due is Postgres's judgement, and the stamp this keys the offer on and hands the + * compare-and-set is the value Postgres gave back. The one process-clock reading is `now` here, used + * only to measure the width of the grace window below — a window minutes wide, against a stamp the + * database chose, so sub-second skew cannot change an answer. A badly skewed node could admit or + * suppress a firing near the boundary; it cannot double-fire one, because the key and the CAS are + * both the database's. That is the difference between this and a lease, which is why a lease is + * never computed here. The option exists so tests can put a stamp anywhere they like. + */ +export async function offerDueRoutines( + options: RoutineSweepOptions, +): Promise<{ offered: string[] }> { + const now = options.now?.() ?? new Date(); + const graceMs = options.graceMs ?? DEFAULT_GRACE_MS; + const due = await options.routineStore.dueRoutines( + options.limit ?? DEFAULT_LIMIT, + ); + + const offered: string[] = []; + for (const routine of due) { + /* + * ONE ROUTINE'S FAILURE IS ONE ROUTINE'S FAILURE. A cron the parser cannot read — a row written + * before a validation existed, a hand-edited value — makes `advanceNextRun` throw, and an + * unguarded loop would take the whole pass down with it: everybody else's routines, on every + * pass, for as long as the bad row exists. So each routine is its own attempt, and the pass + * carries on. + */ + try { + /* + * A STALE STAMP IS NOT A BACKLOG TO REPLAY. `advanceNextRun` moves the clock one occurrence on + * from the stamp it was given, so a routine whose stamp is a month behind — a deployment that + * ran with no worker, a worker that was down — comes back due on the next pass, and the pass + * after that, once per missed occurrence, each with its own offer key and its own real firing. + * Turn the worker on after a quiet month and a person gets thirty summaries of thirty days ago. + * + * So this loop offers only firings that are still worth having: a stamp within GRACE of now. + * For anything later than that, advance WITHOUT offering and say nothing — the occurrence is + * past and nobody wants it now — and let successive passes drain the stamp silently until it is + * current. The store deliberately does not decide this: it moves the clock one step and reports + * whether it won, and which steps are worth firing is this file's policy. + * + * Draining costs a slot of `limit` per pass per stale routine, which is the price of a bounded + * pass; the routines still current are read on the same passes, because the ordering is by due + * stamp and a stale one leaves the list as soon as its clock catches up. + */ + const lateBy = now.getTime() - routine.nextRunAt.getTime(); + if (lateBy <= graceMs) { + /* + * OFFERED BEFORE THE CLOCK MOVES. A crash between the two leaves the stamp where it was, so + * the next pass reads the same stamp, renders the same key and collides: the firing happens + * once and nothing is lost. Advancing first and offering second loses that firing outright — + * the stamp is gone and nothing remembers what it was for. + */ + await options.queue.offer({ + kind: ROUTINE_FIRE_KIND, + key: `${routine.id}:${minuteKey(routine.nextRunAt)}`, + payload: { + routineId: routine.id, + scheduledFor: routine.nextRunAt.toISOString(), + }, + }); + offered.push(routine.id); + } + // False means another sweep advanced it first, which is fine either way: the firing was offered + // under the same key by both, so it still happens once. + await options.routineStore.advanceNextRun(routine.id, routine.nextRunAt); + } catch (error) { + /* + * Said out loud, with the routine in it. A pass that swallowed this would look clean while one + * routine's clock never moved again: it would be read as due, warned about and re-offered under + * the same key on every pass thereafter — harmless, but invisible to anybody not reading logs. + */ + console.warn( + JSON.stringify({ + type: "routine-sweep-offer-failed", + routineId: routine.id, + scheduledFor: routine.nextRunAt.toISOString(), + reason: + error instanceof Error ? error.message : "could not be offered", + }), + ); + } + } + + return { offered }; +} diff --git a/server/tests/routine-sweep.integration.test.ts b/server/tests/routine-sweep.integration.test.ts new file mode 100644 index 00000000..144d8f2f --- /dev/null +++ b/server/tests/routine-sweep.integration.test.ts @@ -0,0 +1,467 @@ +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + spyOn, + test, +} from "bun:test"; +import { randomUUID } from "node:crypto"; +import { and, eq, like } from "drizzle-orm"; +import { createAgentProfileStore } from "../src/agents/profile-store"; +import type { AgentActor } from "../src/agents/profile-types"; +import { createChannelStore } from "../src/channels/routes"; +import { createThreadIdentity } from "../src/channels/thread-identity"; +import { createDatabase } from "../src/db/client"; +import { + agentProfiles, + agents, + channels, + intelligenceChannelMappings, + routines, + users, + workItems, +} from "../src/db/schema"; +import { createRoutineStore } from "../src/routines/store"; +import { ROUTINE_FIRE_KIND, offerDueRoutines } from "../src/routines/sweep"; +import { createWorkQueue } from "../src/work/queue"; +import { TEST_POOL } from "./support/database"; + +/** + * The sweep's first half against a real PostgreSQL and the real `work_items` queue. + * + * A fake queue cannot answer the only question worth asking here. The whole reason a routine fires + * once when three replicas wake at 09:00 is a primary key on `(kind, key)` and + * `on conflict do nothing`, which is a promise the database makes; a stub that remembered what it + * had been offered would pass every test below while production produced three runs. + * + * THIS FILE OWNS `ROUTINE_FIRE_KIND`. Unlike `work-queue.integration.test.ts`, which invents a + * per-file kind, the kind under test here is a shared constant, so the rows cannot be namespaced + * away — the cleanup below deletes every row of that kind. A second test file that offers this kind + * would race this one and both would flake. Put such tests here. + * + * NO CLAIM, LEASE OR OWNERSHIP INTERNALS ARE TESTED HERE. `for update skip locked`, leases named on + * the database's clock and the attempt count belong to `work-queue.integration.test.ts`. What this + * file tests is which firings the sweep decides are worth offering, and that the key it offers them + * under makes a repeat harmless. + */ +const databaseUrl = + process.env.DATABASE_URL ?? + "postgres://openbot:openbot@localhost:5432/openbot"; +const database = createDatabase(databaseUrl, TEST_POOL); +const profileStore = createAgentProfileStore( + database, + new URL("https://managed.example.test/ag-ui"), +); +const channelStore = createChannelStore( + database, + profileStore, + createThreadIdentity("test-deployment"), +); +const store = createRoutineStore(database); +const queue = createWorkQueue(database); + +const testPrefix = `routine-sweep-${randomUUID()}`; +const createdUserIds: string[] = []; +const createdAgentIds: string[] = []; +const createdChannelIds: string[] = []; + +/** Every day at 09:00 UTC: comfortably above the floor, and one occurrence is one day. */ +const DAILY = "0 9 * * *"; + +/** + * The dispatch half of the options, recorded rather than dialled. + * + * Nothing in this commit calls it — `offerDueRoutines` only puts work on the queue — but the option + * is wired so the file is ready for the half that claims those items, and so a dispatch that + * started happening in this phase would show up as a recorded call rather than as nothing. + */ +const dispatched: string[] = []; +const dispatch = async (routineRunId: string) => { + dispatched.push(routineRunId); +}; + +function sweepOptions(overrides: Record = {}) { + return { + routineStore: store, + queue, + dispatch, + owner: "sweep-test", + ...overrides, + } as Parameters[0]; +} + +beforeEach(async () => { + dispatched.length = 0; + await database.delete(workItems).where(eq(workItems.kind, ROUTINE_FIRE_KIND)); +}); + +afterEach(async () => { + // Routines first, so a failure part-way through cleanup leaves nothing pointing at rows this file + // is about to delete. + for (const userId of createdUserIds) { + await database.delete(routines).where(eq(routines.ownerUserId, userId)); + } + for (const channelId of createdChannelIds.splice(0)) { + await database + .delete(intelligenceChannelMappings) + .where(eq(intelligenceChannelMappings.channelId, channelId)); + await database.delete(channels).where(eq(channels.id, channelId)); + } + for (const agentId of createdAgentIds.splice(0)) { + await database + .delete(agentProfiles) + .where(eq(agentProfiles.agentId, agentId)); + await database.delete(agents).where(eq(agents.id, agentId)); + } + for (const userId of createdUserIds.splice(0)) { + await database.delete(users).where(eq(users.id, userId)); + } +}); + +afterAll(async () => { + await database.delete(workItems).where(eq(workItems.kind, ROUTINE_FIRE_KIND)); + await database.$client.close(); +}); + +async function createUser(): Promise { + const id = `${testPrefix}-user-${randomUUID()}`; + await database.insert(users).values({ + id, + email: `${id}@example.test`, + name: "Routine Sweep Test User", + }); + createdUserIds.push(id); + return { id, role: "user" }; +} + +async function createAgent(owner: AgentActor, name = "Expense Manager") { + const profile = await profileStore.create(owner, { + name, + title: "Finance Operations", + roleDescription: "Review receipts.", + visibility: "private", + }); + createdAgentIds.push(profile.id); + return profile.id; +} + +async function createChannel(owner: AgentActor, agentIds: string[]) { + const channel = await channelStore.create(owner, agentIds); + createdChannelIds.push(channel.id); + return channel; +} + +/** A person, a Bot, the one channel they share, and a routine on it. */ +async function makeRoutine(instruction = "Summarise the day.") { + const owner = await createUser(); + const agentId = await createAgent(owner); + const channel = await createChannel(owner, [agentId]); + const routine = await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction, + cron: DAILY, + }); + return { owner, agentId, channel, routine }; +} + +/** + * The create path always computes a future stamp, so a due-in-the-past row is written directly. + * + * THE STAMPS IN THIS FILE ARE DELIBERATELY ANCIENT, and the `now` option is moved to match them. + * `dueRoutines` is not owner-scoped and orders oldest-due first, so a test that asserts on ordering + * or on a limit has to be sure its own rows sort ahead of anything else in the database; and the + * grace policy is measured against `now`, so the injected clock is what makes "two minutes late" and + * "a month late" mean anything in a row stamped in 2001. + */ +async function makeDueAt(routineId: string, nextRunAt: Date): Promise { + await database + .update(routines) + .set({ nextRunAt }) + .where(eq(routines.id, routineId)); + const [row] = await database + .select({ nextRunAt: routines.nextRunAt }) + .from(routines) + .where(eq(routines.id, routineId)) + .limit(1); + // Read back rather than trusting the Date we wrote: the stamp the sweep keys on is Postgres's. + return row?.nextRunAt as Date; +} + +async function readRoutine(routineId: string) { + const [row] = await database + .select() + .from(routines) + .where(eq(routines.id, routineId)) + .limit(1); + return row; +} + +/** Every queued firing of one routine, whatever minute it was keyed on. */ +async function firingsFor(routineId: string) { + return await database + .select() + .from(workItems) + .where( + and( + eq(workItems.kind, ROUTINE_FIRE_KIND), + like(workItems.key, `${routineId}:%`), + ), + ); +} + +describe("offering the firings that are due", () => { + /** + * THE TEST THIS FILE EXISTS FOR. Three replicas wake on the same minute and read the same due + * row; the person gets one run. Nothing in the sweep coordinates that — the offer key carries the + * minute, so the second and third offers collide on the primary key, and the compare-and-set on + * `next_run_at` means only one of them moves the clock. + */ + test("two sweeps racing on one due routine offer it exactly once", async () => { + const { routine } = await makeRoutine(); + const from = await makeDueAt(routine.id, new Date("2001-01-01T09:25:00Z")); + const now = () => new Date("2001-01-01T09:26:00Z"); + + const outcomes = await Promise.all([ + offerDueRoutines(sweepOptions({ now })), + offerDueRoutines(sweepOptions({ now })), + ]); + + // Both sweeps saw it as due and both offered it, which is the honest report: each did put the + // work on the queue. What must be single is the row, and the clock. + for (const outcome of outcomes) { + expect(outcome.offered).toContain(routine.id); + } + expect(await firingsFor(routine.id)).toHaveLength(1); + + const after = await readRoutine(routine.id); + expect(after?.nextRunAt.getTime()).toBeGreaterThan(from.getTime()); + // One occurrence on from the stamp it was given, not two: a second advance would have moved it + // to the 3rd of January. + expect(after?.nextRunAt.toISOString()).toBe("2001-01-02T09:00:00.000Z"); + }); + + /** + * The key format is a compatibility surface, not an implementation detail: it is the identity of a + * firing, already written into rows in `work_items`. Asserted literally so a later change to the + * truncation is a failing test here rather than a routine that fires twice in production. + */ + test("the offer key is the routine and the minute it was due", async () => { + const { routine } = await makeRoutine(); + await makeDueAt(routine.id, new Date("2001-01-01T09:25:00Z")); + + await offerDueRoutines( + sweepOptions({ now: () => new Date("2001-01-01T09:26:00Z") }), + ); + + const [firing] = await firingsFor(routine.id); + expect(firing?.kind).toBe(ROUTINE_FIRE_KIND); + expect(firing?.key).toBe(`${routine.id}:2001-01-01T09:25Z`); + expect(firing?.payload).toEqual({ + routineId: routine.id, + scheduledFor: "2001-01-01T09:25:00.000Z", + }); + }); + + /** + * The crash between the offer and the advance, which is why the offer comes first. + * + * The stamp is put back by hand to stand for the sweep that died before it could move the clock — + * or the replica that lost the compare-and-set. Either way the next pass reads the same stamp, + * renders the same key, and adds nothing: the firing is not lost and it is not doubled. + */ + test("a second pass over the same stamp adds no row, before or after the run", async () => { + const { routine } = await makeRoutine(); + const from = await makeDueAt(routine.id, new Date("2001-01-01T09:25:00Z")); + const now = () => new Date("2001-01-01T09:26:00Z"); + + await offerDueRoutines(sweepOptions({ now })); + expect(await firingsFor(routine.id)).toHaveLength(1); + + await makeDueAt(routine.id, from); + await offerDueRoutines(sweepOptions({ now })); + expect(await firingsFor(routine.id)).toHaveLength(1); + + /* + * AND AFTER THE FIRING HAS HAPPENED, which is the half that is easy to lose. `finish` marks the + * row rather than deleting it, so a finished row still counts as a conflict + * (`server/src/work/queue.ts:120-133`) — deleting it would hand the key back and make the + * recovery path a duplicate-run path. The claim here is setup for that state, not a test of + * claiming. + */ + const [claimed] = await queue.claim({ + kind: ROUTINE_FIRE_KIND, + owner: "sweep-test", + leaseMs: 30_000, + }); + expect( + await queue.finish({ + kind: ROUTINE_FIRE_KIND, + key: claimed?.key as string, + owner: "sweep-test", + }), + ).toBe(true); + + await makeDueAt(routine.id, from); + await offerDueRoutines(sweepOptions({ now })); + + const firings = await firingsFor(routine.id); + expect(firings).toHaveLength(1); + expect(firings[0]?.finishedAt).not.toBeNull(); + }); + + test("a routine that is switched off, or not due yet, is not offered", async () => { + const { owner: offOwner, routine: off } = + await makeRoutine("Switched off."); + await store.setEnabled(offOwner.id, off.id, false); + await makeDueAt(off.id, new Date("2001-01-01T09:25:00Z")); + + const { routine: ahead } = await makeRoutine("Still ahead."); + // The create path already put this in the future; nothing rounds it down. + const aheadStamp = (await readRoutine(ahead.id))?.nextRunAt as Date; + + const { offered } = await offerDueRoutines( + sweepOptions({ now: () => new Date("2001-01-01T09:26:00Z") }), + ); + + expect(offered).not.toContain(off.id); + expect(offered).not.toContain(ahead.id); + expect(await firingsFor(off.id)).toHaveLength(0); + expect(await firingsFor(ahead.id)).toHaveLength(0); + // And neither clock moved: a routine nobody offered is a routine nobody advanced. + expect((await readRoutine(off.id))?.nextRunAt.getTime()).toBe( + new Date("2001-01-01T09:25:00Z").getTime(), + ); + expect((await readRoutine(ahead.id))?.nextRunAt.getTime()).toBe( + aheadStamp.getTime(), + ); + }); + + test("the limit bounds one pass, and the rest wait for the next", async () => { + const first = (await makeRoutine("First.")).routine; + const second = (await makeRoutine("Second.")).routine; + const third = (await makeRoutine("Third.")).routine; + await makeDueAt(first.id, new Date("2001-01-01T09:25:00Z")); + await makeDueAt(second.id, new Date("2001-01-01T09:26:00Z")); + await makeDueAt(third.id, new Date("2001-01-01T09:27:00Z")); + + const { offered } = await offerDueRoutines( + sweepOptions({ limit: 2, now: () => new Date("2001-01-01T09:28:00Z") }), + ); + + // Oldest due first, so a backlog drains in the order it built up. + expect(offered).toEqual([first.id, second.id]); + expect(await firingsFor(third.id)).toHaveLength(0); + expect((await readRoutine(third.id))?.nextRunAt.getTime()).toBe( + new Date("2001-01-01T09:27:00Z").getTime(), + ); + }); +}); + +/** + * A STALE STAMP IS NOT A BACKLOG TO REPLAY. + * + * `advanceNextRun` moves the clock one occurrence on from the stamp it was given, so a routine whose + * stamp is a month behind comes back due pass after pass, once per missed occurrence. Turn the + * worker on after a quiet month and a person gets thirty summaries of thirty days ago; a 15-minute + * routine idle a year would be some thirty-five thousand firings. The stamp still has to drain, so + * the pass advances it — it just says nothing while it does. + */ +describe("draining a stale stamp instead of replaying it", () => { + test("a month-old firing is advanced without being offered, and a two-minute-old one fires", async () => { + const { routine: stale } = await makeRoutine("A month behind."); + const { routine: fresh } = await makeRoutine("Two minutes late."); + const staleFrom = await makeDueAt( + stale.id, + new Date("2001-01-01T09:00:00Z"), + ); + await makeDueAt(fresh.id, new Date("2001-02-01T08:58:00Z")); + + const { offered } = await offerDueRoutines( + sweepOptions({ now: () => new Date("2001-02-01T09:00:00Z") }), + ); + + expect(offered).toEqual([fresh.id]); + expect(await firingsFor(stale.id)).toHaveLength(0); + expect(await firingsFor(fresh.id)).toHaveLength(1); + + // Advanced all the same, one occurrence on, so successive passes drain it silently rather than + // reading it as due for ever. + const after = await readRoutine(stale.id); + expect(after?.nextRunAt.getTime()).toBeGreaterThan(staleFrom.getTime()); + expect(after?.nextRunAt.toISOString()).toBe("2001-01-02T09:00:00.000Z"); + }); + + test("the grace window is a setting, so a caller can say what counts as worth having", async () => { + const { routine } = await makeRoutine(); + await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z")); + + // Twenty minutes late. Outside the default window, inside a thirty-minute one. + const now = () => new Date("2001-01-01T09:20:00Z"); + await offerDueRoutines(sweepOptions({ now })); + expect(await firingsFor(routine.id)).toHaveLength(0); + + await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z")); + const { offered } = await offerDueRoutines( + sweepOptions({ now, graceMs: 30 * 60_000 }), + ); + expect(offered).toContain(routine.id); + expect(await firingsFor(routine.id)).toHaveLength(1); + }); +}); + +/** + * One poisoned routine is one person's problem, not everybody's. + * + * A cron the parser cannot read makes `advanceNextRun` throw, and an unguarded loop would take the + * whole pass down with it — for every other person's routine too, on every pass, for as long as the + * bad row exists. The one routine nobody can schedule must not be able to stop the sweep. + */ +describe("surviving a routine that cannot be scheduled", () => { + test("a poisoned cron is warned about and the next routine is still offered", async () => { + const { routine: poisoned } = await makeRoutine("Unschedulable."); + const { routine: healthy } = await makeRoutine("Perfectly fine."); + // Only a direct write can make this row: `create` and `update` both refuse a cron the schedule + // module cannot read, which is exactly why the bad row has to be simulated rather than created. + await database + .update(routines) + .set({ cron: "not a cron" }) + .where(eq(routines.id, poisoned.id)); + // The poisoned one is due FIRST, so a pass that dies on it never reaches the healthy one. + await makeDueAt(poisoned.id, new Date("2001-01-01T09:25:00Z")); + await makeDueAt(healthy.id, new Date("2001-01-01T09:26:00Z")); + + // Captured as it is written rather than read off the spy afterwards, so restoring the real + // `console.warn` cannot take the evidence with it. + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((...args) => { + lines.push(String(args[0])); + }); + let offered: string[] = []; + try { + ({ offered } = await offerDueRoutines( + sweepOptions({ now: () => new Date("2001-01-01T09:27:00Z") }), + )); + } finally { + warn.mockRestore(); + } + + expect(offered).toContain(healthy.id); + expect(await firingsFor(healthy.id)).toHaveLength(1); + + // Said out loud, with the routine in it: a sweep that swallowed this would look clean while one + // routine never advanced again. + const complaint = lines.find((line) => line.includes(poisoned.id)); + expect(complaint).toBeDefined(); + expect(JSON.parse(complaint as string).routineId).toBe(poisoned.id); + + // The poisoned routine's clock could not move, so it stays due and stays being warned about — + // loudly and harmlessly, because the offer key is the same one every pass. + expect((await readRoutine(poisoned.id))?.nextRunAt.getTime()).toBe( + new Date("2001-01-01T09:25:00Z").getTime(), + ); + }); +}); From 494049701faa59ddbe72dbc60246e1cd5712f74a Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 12:52:04 -0300 Subject: [PATCH 20/45] Say which order the clock and the queue really move in --- server/src/routines/store.ts | 32 ++++++++++++++--- server/src/routines/sweep.ts | 11 +++--- .../tests/routine-sweep.integration.test.ts | 25 ++++++++++++-- .../tests/routines-store.integration.test.ts | 34 +++++++++++++++++++ 4 files changed, 91 insertions(+), 11 deletions(-) diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index c163e429..bd7d3c93 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -188,6 +188,15 @@ export type RoutineStore = { * means the routine was deleted between queueing and running, which takes its runs with it. */ runContext(runId: string): Promise; + /** + * The sweep's read, like `dueRoutines` — not owner-scoped. A routine id here comes from a work + * item's own payload, not from a person, so there is no owner to check it against. Answers exactly + * what the consumer's re-read needs before firing: has the routine been deleted or switched off + * since the offer. Null means deleted; otherwise `enabled` says the rest. + */ + routineForFiring( + id: string, + ): Promise<{ id: string; enabled: boolean } | null>; /** Close a run row with its outcome, and the capped error when there was one. */ finishRun( runId: string, @@ -581,10 +590,14 @@ export function createRoutineStore(database: Database): RoutineStore { * routine in the same second, exactly one update matches and the rest change nothing. False * means another sweep won, which is fine either way — the firing still happens once. * - * And it happens BEFORE the run is offered. A crash between advancing and running skips that - * firing; advancing afterwards would mean a crash re-offers it, and a routine that spends - * money or sends mail would double-fire. A skipped firing is recoverable by waiting for the - * next one; a doubled one is not. + * And it happens AFTER the run is offered — `sweep.ts`'s ordering comment is the one to read + * for why. The old fear here was that advancing after offering would double-fire on a crash + * between the two; it doesn't, because the offer key carries the minute the firing was due, so + * a re-offer of the same due stamp collides on `work_items`' `(kind, key)` primary key rather + * than queueing a second run. That holds even for a firing that already finished: `finish` + * marks the row rather than deleting it, so a finished row still conflicts. The compare-and-set + * below is what makes the advance itself safe under the same race, which is a separate claim + * from the ordering. * * The equality is on a stamp that round-trips: every writer of `next_run_at` computes it from * `nextOccurrence`, which lands on a cron boundary with no sub-second part, and the driver @@ -634,6 +647,17 @@ export function createRoutineStore(database: Database): RoutineStore { return row ?? null; }, + async routineForFiring(id) { + // A single select, not owner-scoped — the sweep's read, like `dueRoutines`. A routine id here + // comes from a work item's own payload, not from a person, so there is no owner to check. + const [row] = await database + .select({ id: routines.id, enabled: routines.enabled }) + .from(routines) + .where(eq(routines.id, id)) + .limit(1); + return row ?? null; + }, + async finishRun(runId, status, error) { await database .update(routineRuns) diff --git a/server/src/routines/sweep.ts b/server/src/routines/sweep.ts index aaa4dbed..45be5f69 100644 --- a/server/src/routines/sweep.ts +++ b/server/src/routines/sweep.ts @@ -37,7 +37,7 @@ const DEFAULT_LIMIT = 50; * fifteen-minute floor a routine's schedule may have (`MINIMUM_INTERVAL_MS` in `./schedule`), so the * window can never call two consecutive occurrences of one routine current at the same time. */ -const DEFAULT_GRACE_MS = 10 * 60_000; +export const DEFAULT_GRACE_MS = 10 * 60_000; export type RoutineSweepOptions = { routineStore: RoutineStore; @@ -49,7 +49,7 @@ export type RoutineSweepOptions = { owner: string; /** Lease for a claimed firing; phase two is what applies it. Default 60_000. */ leaseMs?: number; - /** How many goes one firing gets before it stops being offered. Default `DEFAULT_MAX_ATTEMPTS`. */ + /** How many goes one firing gets before it stops being offered. Default the queue's default attempt cap. */ maxAttempts?: number; /** How many due routines one pass considers. Default 50. */ limit?: number; @@ -161,8 +161,11 @@ export async function offerDueRoutines( } catch (error) { /* * Said out loud, with the routine in it. A pass that swallowed this would look clean while one - * routine's clock never moved again: it would be read as due, warned about and re-offered under - * the same key on every pass thereafter — harmless, but invisible to anybody not reading logs. + * routine's clock never moved again: it would be read as due and warned about on every pass + * thereafter, but OFFERED only while its stamp is still inside `graceMs` — once the stamp ages + * past the grace window, the guard above skips the offer before this throw is ever reached. So + * the grace policy (the window worth having, above) is what bounds this failure mode to one + * firing: harmless and loud, but invisible to anybody not reading logs. */ console.warn( JSON.stringify({ diff --git a/server/tests/routine-sweep.integration.test.ts b/server/tests/routine-sweep.integration.test.ts index 144d8f2f..11746e5d 100644 --- a/server/tests/routine-sweep.integration.test.ts +++ b/server/tests/routine-sweep.integration.test.ts @@ -23,8 +23,13 @@ import { users, workItems, } from "../src/db/schema"; +import { MINIMUM_INTERVAL_MS } from "../src/routines/schedule"; import { createRoutineStore } from "../src/routines/store"; -import { ROUTINE_FIRE_KIND, offerDueRoutines } from "../src/routines/sweep"; +import { + DEFAULT_GRACE_MS, + ROUTINE_FIRE_KIND, + offerDueRoutines, +} from "../src/routines/sweep"; import { createWorkQueue } from "../src/work/queue"; import { TEST_POOL } from "./support/database"; @@ -213,6 +218,13 @@ async function firingsFor(routineId: string) { ); } +test("the grace window stays under the schedule floor, as a compile/test-time fact rather than prose", () => { + // Two consecutive occurrences of one routine must never both be inside the grace window — that + // would make a routine's own next firing look like a re-offer of a stale one. The floor between + // two occurrences is `MINIMUM_INTERVAL_MS`, so the grace window has to stay strictly under it. + expect(DEFAULT_GRACE_MS).toBeLessThan(MINIMUM_INTERVAL_MS); +}); + describe("offering the firings that are due", () => { /** * THE TEST THIS FILE EXISTS FOR. Three replicas wake on the same minute and read the same due @@ -458,10 +470,17 @@ describe("surviving a routine that cannot be scheduled", () => { expect(complaint).toBeDefined(); expect(JSON.parse(complaint as string).routineId).toBe(poisoned.id); - // The poisoned routine's clock could not move, so it stays due and stays being warned about — - // loudly and harmlessly, because the offer key is the same one every pass. + // The poisoned routine's clock could not move, so it stays due and stays being warned about on + // every pass thereafter. It is not re-offered forever, though: this pass fired it because its + // stamp was still inside the grace window (two minutes late). Once the stamp ages past graceMs, + // the grace guard skips the offer before this throw is ever reached, so the grace policy is what + // bounds this failure mode to a single firing rather than an unbounded stream of re-offers. expect((await readRoutine(poisoned.id))?.nextRunAt.getTime()).toBe( new Date("2001-01-01T09:25:00Z").getTime(), ); + + // That single firing is the entire blast radius: pin it as exactly one `work_items` row, the + // offer that preceded the throw, rather than leaving it inferred from the warning alone. + expect(await firingsFor(poisoned.id)).toHaveLength(1); }); }); diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 8be016d9..4bc558f2 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -941,6 +941,40 @@ describe("the runner's read of one firing", () => { }); }); +/** + * The consumer's re-read: "gone or disabled since the offer -> finish without dispatching" needs a + * by-id read that is not owner-scoped, because the id it is given comes from a work item's own + * payload, not from a person calling in with their own identity. `runContext` cannot answer this — + * it needs a run row to exist and does not report `enabled` — so this is a second, narrower read. + */ +describe("the sweep's by-id read of a routine before firing", () => { + test("an existing enabled routine reads back enabled", async () => { + const { routine } = await makeRoutine(); + + expect(await store.routineForFiring(routine.id)).toEqual({ + id: routine.id, + enabled: true, + }); + }); + + test("a disabled routine reads back disabled, not missing", async () => { + const { owner, routine } = await makeRoutine(); + await store.setEnabled(owner.id, routine.id, false); + + expect(await store.routineForFiring(routine.id)).toEqual({ + id: routine.id, + enabled: false, + }); + }); + + test("a deleted routine reads as no routine at all", async () => { + const { owner, routine } = await makeRoutine(); + await store.remove(owner.id, routine.id); + + expect(await store.routineForFiring(routine.id)).toBeNull(); + }); +}); + /** * The fatigue rule counts the failures at the tail, and this file pins what a `skipped` run does to * that tail: A SKIP IS NOT A FAILURE AND DOES NOT BREAK THE STREAK. A skip means the channel was From 5cee46b5fd4f025ed3dfc903e40d2a0f3419a759 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 13:15:40 -0300 Subject: [PATCH 21/45] Consume a claimed firing honestly, and reap what is done with --- server/scripts/fire-routines.ts | 129 +++++ server/src/routines/sweep.ts | 294 +++++++++- .../tests/routine-sweep.integration.test.ts | 512 +++++++++++++++++- 3 files changed, 925 insertions(+), 10 deletions(-) create mode 100644 server/scripts/fire-routines.ts diff --git a/server/scripts/fire-routines.ts b/server/scripts/fire-routines.ts new file mode 100644 index 00000000..08e52f33 --- /dev/null +++ b/server/scripts/fire-routines.ts @@ -0,0 +1,129 @@ +/** + * One sweep: notice which routines are due, and fire whatever this pod can claim. + * + * Run from a CronJob rather than from a timer inside the API, for the reason the culler states: every + * replica would fire its own timer and each would independently decide that the 09:00 summary is due. + * Deleting old audit rows twice is harmless, which is why the retention sweep may work that way; + * posting somebody's morning summary three times is not, and neither is spending three turns' worth + * of tokens on it. + * + * Exits non-zero only when a phase itself could not run — no database, no secret, no server to hand a + * run to. A single firing that failed is reported and left on the queue for the next sweep, because a + * routine that fires a minute late has lost nothing, and a failing CronJob that pages somebody at 3am + * should mean something worse than that. + */ +import { randomUUID } from "node:crypto"; +import { loadConfig } from "../src/config"; +import { createDatabase } from "../src/db/client"; +import { createRoutineStore } from "../src/routines/store"; +import { + ROUTINE_FIRE_KIND, + dispatchClaimedRoutines, + offerDueRoutines, +} from "../src/routines/sweep"; +import { createWorkQueue } from "../src/work/queue"; + +const config = loadConfig(process.env); + +/* + * Refused up front rather than at the first dispatch. + * + * A sweep that cannot authenticate its handoff will open a run row for every due routine and collect + * a 401 for each one: every firing recorded as attempted, none of them carried out, and the only + * evidence a line in the server's audit trail. Saying so once, loudly, before anything is claimed is + * the difference between a CronJob that failed and a deployment where routines quietly do nothing. + */ +const workerSharedSecret = config.workerSharedSecret; +if (!workerSharedSecret) { + throw new Error( + "WORKER_SHARED_SECRET is not set, so this sweep cannot authenticate itself to /internal/routines/run and no routine could be fired.", + ); +} + +/* + * Read from the environment rather than from `DeploymentConfig`, on purpose. + * + * Where this process can reach its own API server is a fact about where this process runs — a pod in + * a namespace, a laptop on localhost — not a fact about the deployment, which is what that config + * describes. A CronJob in another namespace and a developer's shell want different values for the + * same deployment, so it belongs to the process's environment. + */ +const serverInternalUrl = process.env.SERVER_INTERNAL_URL; +if (!serverInternalUrl) { + throw new Error( + "SERVER_INTERNAL_URL is not set, so this sweep does not know where to hand a routine run.", + ); +} + +const database = createDatabase(config.databaseUrl); +const queue = createWorkQueue(database); +const routineStore = createRoutineStore(database); + +// A name for the lease, so a stuck claim can be traced back to the pod that took it. +const owner = `routines/${process.env.HOSTNAME ?? randomUUID().slice(0, 8)}`; + +/** + * Hand one opened run to the server, which owns everything about running it. + * + * The run id is all that crosses: the server resolves the routine, the owner and the channel from it, + * so nothing a caller says here can decide whose routine gets run. A 202 means accepted, not + * finished — the run row carries the outcome — and anything else throws, which is what puts the work + * item back on the queue for another go. + */ +async function dispatch(routineRunId: string): Promise { + const response = await fetch(`${serverInternalUrl}/internal/routines/run`, { + method: "POST", + headers: { + // The whole header string is what the server compares, so the casing and the one space are the + // credential's format rather than a style choice. + authorization: `Bearer ${workerSharedSecret}`, + "content-type": "application/json", + }, + body: JSON.stringify({ routineRunId }), + }); + if (response.status !== 202) { + // The status is in the sentence, because it is the whole diagnosis: 401 is the secret, 404 is a + // server with no runner mounted, 5xx is the server itself. That sentence ends up on the work + // item's `last_error`, which is where somebody looks when a routine stopped firing. + throw new Error( + `the server answered ${response.status} rather than 202 when handed a routine run`, + ); + } +} + +try { + const options = { routineStore, queue, dispatch, owner }; + /* + * Both halves in one pass, offering first: the items this offer puts on the queue are claimable by + * the consume below, so a routine due right now fires in this sweep rather than in the next one. + * Neither half needs the other to have run — another replica's offer is claimed here just the same. + */ + const { offered } = await offerDueRoutines(options); + const report = await dispatchClaimedRoutines(options); + /* + * Sweep what is done with for a day. NOT OPTIONAL. + * + * A finished row is what stops a key being run twice, so it has to outlive the run by long enough + * for a late replica to collide with it — and no longer, because a queue is not an archive. The + * half that is easy to forget is the other one: an item at its attempt cap is not finished and is + * reaped by nothing else, so without this its key stays wedged for ever and that routine can never + * be offered for that minute again. `server/src/work/queue.ts:262-274` documents that bug at + * length; the culler pays the same rent at `cull-idle-computers.ts:71-74`. + */ + const purged = await queue.purge({ + kind: ROUTINE_FIRE_KIND, + olderThanMs: 24 * 60 * 60 * 1000, + }); + console.info( + JSON.stringify({ + type: "routine-sweep", + offered, + considered: report.considered, + fired: report.fired, + skipped: report.skipped, + purged, + }), + ); +} finally { + await database.$client.end({ timeout: 5 }); +} diff --git a/server/src/routines/sweep.ts b/server/src/routines/sweep.ts index 45be5f69..b0f79e80 100644 --- a/server/src/routines/sweep.ts +++ b/server/src/routines/sweep.ts @@ -21,7 +21,7 @@ * A second half-right copy grown next to it is the duplicated firing mechanism #235 exists to * prevent. */ -import type { WorkQueue } from "../work/queue"; +import { DEFAULT_MAX_ATTEMPTS, type WorkQueue } from "../work/queue"; import type { RoutineStore } from "./store"; export const ROUTINE_FIRE_KIND = "routine.fire"; @@ -29,6 +29,23 @@ export const ROUTINE_FIRE_KIND = "routine.fire"; /** How many due routines one pass will look at. Bounded, because a pass has to end. */ const DEFAULT_LIMIT = 50; +/** + * How long a claimed firing is leased for, and renewed by, while it is being dispatched. + * + * A minute: a dispatch is one HTTP call to this deployment's own server, which either answers or + * fails long before that, and a firing whose owner died is worth picking up again quickly. It is + * renewed before every item, so the length bounds one dispatch rather than the whole batch. + */ +const DEFAULT_LEASE_MS = 60_000; + +/** + * How long a firing that could not be dispatched waits before anybody tries again. + * + * A minute, for the culler's reason: whatever refused this will probably refuse it again in the next + * second, and the queue's attempt cap is what stops the waiting going on for ever. + */ +const DISPATCH_RETRY_DELAY_MS = 60_000; + /** * How late a firing may be and still be worth having. * @@ -61,8 +78,9 @@ export type RoutineSweepOptions = { /** * What one pass of phase two did. * - * Declared here with the options it shares because PHASE TWO ARRIVES NEXT — the half that claims - * these items, opens a run row and dispatches it. Nothing in this file returns one yet. + * `fired` is "a run was opened and the dispatch was accepted", not "the routine succeeded": the run + * row in `routine_runs` owns the outcome from the moment the dispatch resolves, and this report is + * the sweep's own account of its pass rather than a summary of anybody's turn. */ export type RoutineSweepReport = { considered: number; @@ -181,3 +199,273 @@ export async function offerDueRoutines( return { offered }; } + +/** + * Which routine a claimed item is about. + * + * The payload is the answer, and the key is the fallback for a row written before the payload was: + * the key is `:` and a routine id carries no colon, so everything up to the first + * one is the routine. A firing whose routine cannot be named at all would be a firing nothing could + * report, which is why this never returns undefined. + */ +function routineIdOf(item: { key: string; payload: Record }) { + const fromPayload = item.payload.routineId; + if (typeof fromPayload === "string" && fromPayload.length > 0) { + return fromPayload; + } + const colon = item.key.indexOf(":"); + return colon === -1 ? item.key : item.key.slice(0, colon); +} + +/** + * Phase two: claimed items become dispatched firings, with the queue's booleans honoured. + * + * EVERY BRANCH HERE IS ABOUT THE GAP BETWEEN THE OFFER AND THE FIRING. Another replica decided this + * should fire, at another time, and by now the routine may be switched off, deleted, or the + * occurrence may have gone stale while the item waited behind a backlog. So the world is re-read at + * the moment of acting — the culler's discipline, for the culler's reason — and the queue's own + * answers are believed: a `renew` that says no means the item is somebody else's, and a `finish` that + * says no means it stopped being ours while we were working. + * + * `finish` and `release` mean different things and are not interchangeable. `release` is for a + * dispatch that could have worked and might work next time; `finish` is for a firing that will never + * be worth having, however many times it comes back. + */ +export async function dispatchClaimedRoutines( + options: RoutineSweepOptions, +): Promise { + const leaseMs = options.leaseMs ?? DEFAULT_LEASE_MS; + const graceMs = options.graceMs ?? DEFAULT_GRACE_MS; + const maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + const claimed = await options.queue.claim({ + kind: ROUTINE_FIRE_KIND, + owner: options.owner, + leaseMs, + limit: options.limit ?? DEFAULT_LIMIT, + // Passed through only when asked for, so the queue's own default stays the one default. + ...(options.maxAttempts === undefined + ? {} + : { maxAttempts: options.maxAttempts }), + }); + + const report: RoutineSweepReport = { + considered: claimed.length, + fired: [], + skipped: [], + }; + + for (const item of claimed) { + const routineId = routineIdOf(item); + + /* + * Renewed before acting, because the batch is many and the lease is one. + * + * This is the lesson at `server/src/work/culler.ts:141-151` verbatim: twenty API calls with + * nothing renewed while they ran, the lease expiring part-way down the list, and another replica + * claiming the tail this one was still working through. A lease nobody renews is a timer, and a + * timer is what this queue exists not to be. + * + * False means it is already somebody else's, and then the only correct answer is to leave it + * entirely alone: no dispatch, because that replica is already running this firing and a second + * dispatch is a second message to a person; and no `finish`, because finishing somebody else's + * item takes the lease away from a run that is still going. + */ + if ( + !(await options.queue.renew({ + kind: ROUTINE_FIRE_KIND, + key: item.key, + owner: options.owner, + leaseMs, + })) + ) { + report.skipped.push({ + routineId, + reason: "the lease went to another replica", + }); + continue; + } + + try { + /* + * RE-READ, because the offer was another replica's judgement at another time. A person who + * switched a routine off a minute after it was offered, or deleted it, has said what they want; + * firing it anyway posts a message they have asked not to receive. + * + * Finished rather than released in both cases: no number of retries will make a deleted routine + * exist, and a switched-off one does not want its queued firing carried out later either. The + * next occurrence is offered afresh if it is switched back on. + */ + const routine = await options.routineStore.routineForFiring(routineId); + if (!routine?.enabled) { + const reason = routine + ? "switched off between the offer and the firing" + : "deleted between the offer and the firing"; + await finishOrSay(options, item.key, routineId, reason); + report.skipped.push({ routineId, reason }); + continue; + } + + /* + * AND THE WINDOW AGAIN, HERE, before any run row exists. + * + * The offer already enforced this window at offer time, and that is not enough: the queue's + * redelivery machinery can outlive it. A backlogged queue, or five releases at a minute each, + * and the item is claimed well after the occurrence it names — "here is your morning summary", + * in the afternoon, which is exactly what the stale-stamp policy above exists to prevent. So it + * is re-checked at the moment of acting, which is the culler's precedent ("Somebody came back", + * `culler.ts:~170`): the decision was made elsewhere and the world has moved since. + * + * BESIDE the deleted/disabled branch and before `insertRun`, so a skipped firing leaves no + * `routine_runs` row: a run opened with no outcome and nothing coming to give it one shows on + * the routines page as a firing that started and never ended. + * + * Finished, not released, for the same reason as above: re-delivery cannot make a past + * occurrence current. A missing or unreadable stamp is not treated as stale — the offer is the + * only writer of this payload and always writes one, so there is no window to enforce rather + * than a window that has passed, and dropping the firing on a payload this file wrote would be + * inventing a reason to lose it. + */ + const now = options.now?.() ?? new Date(); + const stamp = item.payload.scheduledFor; + const scheduledFor = + typeof stamp === "string" ? new Date(stamp) : undefined; + if ( + scheduledFor && + !Number.isNaN(scheduledFor.getTime()) && + now.getTime() - scheduledFor.getTime() > graceMs + ) { + const reason = "claimed too long after the occurrence it was due for"; + await finishOrSay(options, item.key, routineId, reason); + report.skipped.push({ routineId, reason }); + continue; + } + + /* + * The run row first, then the dispatch, because the dispatch is told a run id and nothing else. + * From the moment it resolves the run row owns the outcome: the queue's retries are for + * DISPATCH failures only, and a turn that failed is final for this firing — the fatigue rule + * owns that, not this loop. + */ + const { runId } = await options.routineStore.insertRun(routineId); + await options.dispatch(runId); + if ( + !(await options.queue.finish({ + kind: ROUTINE_FIRE_KIND, + key: item.key, + owner: options.owner, + })) + ) { + /* + * The dispatch happened, so this is `fired` either way; but a `finish` that says no says the + * lease lapsed while the call was in flight, which means another replica may claim this same + * minute and dispatch it again. Said out loud, because it is the shape of a duplicate run and + * nothing else in the system will mention it. + */ + console.warn( + JSON.stringify({ + type: "routine-fire-finish-lost", + routineId, + runId, + reason: + "the lease had gone by the time the dispatch came back, so the firing may be redelivered", + }), + ); + } + report.fired.push(routineId); + } catch (error) { + /* + * ONE FIRING'S FAILURE IS ONE FIRING'S FAILURE, exactly as in the offering half above: an + * unguarded throw here would take everybody else's claimed firings down with it, on every + * pass, for as long as the one bad item exists. + * + * Released rather than finished, and pushed out rather than retried in this pass: a server + * that refused this dispatch will probably refuse it again in the next second, and the queue's + * attempt cap is what bounds the retrying. + */ + const reason = + error instanceof Error ? error.message : "could not be dispatched"; + let released = false; + try { + released = await options.queue.release({ + kind: ROUTINE_FIRE_KIND, + key: item.key, + owner: options.owner, + delayMs: DISPATCH_RETRY_DELAY_MS, + reason, + }); + } catch (releaseError) { + // Best-effort: an item that could not even be released has a lease that will lapse on its + // own, and the pass still has other firings to get through. + console.warn( + JSON.stringify({ + type: "routine-fire-release-failed", + routineId, + reason: String(releaseError), + }), + ); + } + /* + * Said out loud when it gives up, because otherwise it stops silently. + * + * At the cap the item is no longer claimed, so this loop simply never sees that routine again + * and every sweep looks clean while one person's routine never fires. The row carries the count + * and the reason for anybody who queries the table; this is for whoever reads the logs. + */ + if (item.attempts >= maxAttempts) { + console.warn( + JSON.stringify({ + type: "routine-fire-gave-up", + routineId, + key: item.key, + attempts: item.attempts, + reason, + }), + ); + } else if (!released) { + // Not ours any more, which means somebody else holds it: worth a line, because a release that + // did nothing leaves this pass's failure recorded nowhere on the row. + console.warn( + JSON.stringify({ + type: "routine-fire-release-lost", + routineId, + key: item.key, + reason, + }), + ); + } + report.skipped.push({ routineId, reason }); + } + } + + return report; +} + +/** + * Finish a firing nobody wants, and say so if it was not ours to finish. + * + * The boolean is the truth about ownership rather than a formality: false means the lease went while + * this pass was deciding, so the routine was NOT stopped from firing here — whoever holds it now will + * make its own decision, and this one should say what it saw rather than retry into a race. + */ +async function finishOrSay( + options: RoutineSweepOptions, + key: string, + routineId: string, + reason: string, +): Promise { + const finished = await options.queue.finish({ + kind: ROUTINE_FIRE_KIND, + key, + owner: options.owner, + }); + if (!finished) { + console.warn( + JSON.stringify({ + type: "routine-fire-finish-lost", + routineId, + key, + reason, + }), + ); + } +} diff --git a/server/tests/routine-sweep.integration.test.ts b/server/tests/routine-sweep.integration.test.ts index 11746e5d..2c98dadf 100644 --- a/server/tests/routine-sweep.integration.test.ts +++ b/server/tests/routine-sweep.integration.test.ts @@ -19,6 +19,7 @@ import { agents, channels, intelligenceChannelMappings, + routineRuns, routines, users, workItems, @@ -28,13 +29,14 @@ import { createRoutineStore } from "../src/routines/store"; import { DEFAULT_GRACE_MS, ROUTINE_FIRE_KIND, + dispatchClaimedRoutines, offerDueRoutines, } from "../src/routines/sweep"; -import { createWorkQueue } from "../src/work/queue"; +import { DEFAULT_MAX_ATTEMPTS, createWorkQueue } from "../src/work/queue"; import { TEST_POOL } from "./support/database"; /** - * The sweep's first half against a real PostgreSQL and the real `work_items` queue. + * Both halves of the sweep against a real PostgreSQL and the real `work_items` queue. * * A fake queue cannot answer the only question worth asking here. The whole reason a routine fires * once when three replicas wake at 09:00 is a primary key on `(kind, key)` and @@ -48,8 +50,9 @@ import { TEST_POOL } from "./support/database"; * * NO CLAIM, LEASE OR OWNERSHIP INTERNALS ARE TESTED HERE. `for update skip locked`, leases named on * the database's clock and the attempt count belong to `work-queue.integration.test.ts`. What this - * file tests is which firings the sweep decides are worth offering, and that the key it offers them - * under makes a repeat harmless. + * file tests is which firings the sweep decides are worth offering, and — for the consuming half — + * whether the consumer honours the booleans the queue hands back: a `renew` that says the lease has + * gone, a `finish` that says the item was not ours, an attempt count that has reached its cap. */ const databaseUrl = process.env.DATABASE_URL ?? @@ -78,9 +81,9 @@ const DAILY = "0 9 * * *"; /** * The dispatch half of the options, recorded rather than dialled. * - * Nothing in this commit calls it — `offerDueRoutines` only puts work on the queue — but the option - * is wired so the file is ready for the half that claims those items, and so a dispatch that - * started happening in this phase would show up as a recorded call rather than as nothing. + * `offerDueRoutines` never calls it — it only puts work on the queue — so a dispatch that started + * happening in the offering phase shows up here as a recorded call rather than as nothing. What the + * consuming phase hands it is a run id, which is the only thing `/internal/routines/run` is told. */ const dispatched: string[] = []; const dispatch = async (routineRunId: string) => { @@ -218,6 +221,44 @@ async function firingsFor(routineId: string) { ); } +/** Every run row this routine has, in-flight ones included: `status` is null until something ends. */ +async function runsFor(routineId: string) { + return await database + .select() + .from(routineRuns) + .where(eq(routineRuns.routineId, routineId)); +} + +/** + * One routine, due at `due`, offered by a sweep that thinks it is `at`. + * + * The offer is made by the real `offerDueRoutines` rather than by a hand-written insert, so what the + * consuming half claims is the row and the payload production would give it — including + * `scheduledFor`, which the consumer re-checks against the grace window before it fires anything. + */ +async function offerFiring(routineId: string, due: Date, at: Date) { + await makeDueAt(routineId, due); + await offerDueRoutines(sweepOptions({ now: () => at })); +} + +/** + * Age a queued firing by hand. + * + * The queue names every moment of its own in SQL, so a test cannot wait for a lease to lapse or for + * a release delay to pass; it writes the timestamp the queue would have arrived at. That is a test + * driving the clock, not a test reimplementing the queue: what is asserted afterwards is still the + * queue's own answer to `claim`, `renew` and `purge`. + */ +async function backdate( + key: string, + values: Partial, +) { + await database + .update(workItems) + .set(values) + .where(and(eq(workItems.kind, ROUTINE_FIRE_KIND), eq(workItems.key, key))); +} + test("the grace window stays under the schedule floor, as a compile/test-time fact rather than prose", () => { // Two consecutive occurrences of one routine must never both be inside the grace window — that // would make a routine's own next firing look like a re-offer of a stale one. The floor between @@ -484,3 +525,460 @@ describe("surviving a routine that cannot be scheduled", () => { expect(await firingsFor(poisoned.id)).toHaveLength(1); }); }); + +/** + * The consuming half: a claimed item becomes a run, and the queue's booleans are believed. + * + * Every branch below is about the gap between the offer and the firing. The item was put on the + * queue by another replica at another time, and by the time this one claims it the routine may have + * been switched off, deleted, or the occurrence may simply have gone stale while the item sat behind + * a backlog. The queue answers those questions with booleans — `renew`, `finish`, `release` — and a + * consumer that treats them as formalities is a consumer that fires twice. + */ +describe("consuming a claimed firing", () => { + const at = (moment: string) => () => new Date(moment); + + test("a claimed firing is dispatched once, finished, and never claimed again", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + + const report = await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:26:00Z") }), + ); + + expect(report.fired).toEqual([routine.id]); + expect(report.skipped).toEqual([]); + // The run row is what the dispatch is told about, and it owns the outcome from here: opened with + // no status, because null is the in-flight state. + const runs = await runsFor(routine.id); + expect(runs).toHaveLength(1); + expect(runs[0]?.status).toBeNull(); + expect(dispatched).toEqual([runs[0]?.id]); + + // Marked, not deleted: the finished row is what a late replica's re-offer collides with. + const [row] = await firingsFor(routine.id); + expect(row?.finishedAt).not.toBeNull(); + expect(row?.claimedBy).toBeNull(); + + // And the next sweep claims nothing, so the person gets one run rather than one per sweep. + const second = await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:27:00Z") }), + ); + expect(second.considered).toBe(0); + expect(dispatched).toHaveLength(1); + expect(await runsFor(routine.id)).toHaveLength(1); + }); + + test("a dispatch that throws pushes the firing out, and it comes back with its attempts grown", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + + const report = await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + + expect(report.fired).toEqual([]); + expect(report.skipped[0]?.routineId).toBe(routine.id); + expect(report.skipped[0]?.reason).toContain("503"); + + /* + * THE ROW, NOT THE RETURN VALUE. A consumer that reported a failure and finished the item anyway + * would pass any assertion made on the report alone, and the firing would be gone for good. + */ + const [row] = await firingsFor(routine.id); + expect(row?.finishedAt).toBeNull(); + expect(row?.claimedBy).toBeNull(); + expect(row?.attempts).toBe(1); + // The reason stays on the row, so an item that eventually runs out of attempts says why. + expect(row?.lastError).toContain("503"); + // Pushed out rather than retried in the same pass: whatever refused this will probably refuse it + // again in the next second. + expect(row?.runAt.getTime()).toBeGreaterThan(Date.now() + 30_000); + + // Claimable again once the delay has passed, and the second hand-out says it is a second one. + await backdate(row?.key as string, { + runAt: new Date("2001-01-01T09:27:00Z"), + }); + const [again] = await queue.claim({ + kind: ROUTINE_FIRE_KIND, + owner: "sweep-test", + leaseMs: 30_000, + }); + expect(again?.key).toBe(row?.key); + expect(again?.attempts).toBe(2); + }); + + test("a routine switched off between the offer and the firing is finished without dispatching", async () => { + const { owner, routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + await store.setEnabled(owner.id, routine.id, false); + + const report = await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:26:00Z") }), + ); + + expect(report.fired).toEqual([]); + expect(report.skipped[0]?.routineId).toBe(routine.id); + expect(dispatched).toEqual([]); + expect(await runsFor(routine.id)).toHaveLength(0); + // Finished rather than released: re-running cannot make a switched-off routine want to fire. + const [row] = await firingsFor(routine.id); + expect(row?.finishedAt).not.toBeNull(); + }); + + test("a routine deleted between the offer and the firing is finished without dispatching", async () => { + const { owner, routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + // A hard delete, which is what `remove` does — and it takes the routine's runs with it. + await store.remove(owner.id, routine.id); + + const report = await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:26:00Z") }), + ); + + expect(report.fired).toEqual([]); + expect(report.skipped[0]?.routineId).toBe(routine.id); + expect(dispatched).toEqual([]); + // Finished, not released: no number of retries will make a deleted routine exist. + const [row] = await firingsFor(routine.id); + expect(row?.finishedAt).not.toBeNull(); + }); + + /** + * THE WINDOW IS ENFORCED AGAIN AT FIRING TIME, not only at offering time. + * + * The offer refused anything staler than the grace window, but the queue's own machinery can + * outlive that window: a backlogged sweep, or five releases at a minute each, and the item is + * claimed long after the occurrence it names. Firing it then posts "here is your morning summary" + * in the afternoon, which is exactly what the stale-stamp policy exists to prevent. + */ + test("a firing claimed after its window has passed is finished without dispatching, and leaves no run row", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + // Offered one minute late, well inside the window. Nothing here is a stale offer. + new Date("2001-01-01T09:26:00Z"), + ); + + // Claimed twenty minutes after the occurrence, which is outside the default ten. + const report = await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:45:00Z") }), + ); + + expect(report.fired).toEqual([]); + expect(report.skipped[0]?.routineId).toBe(routine.id); + expect(dispatched).toEqual([]); + /* + * NO ORPHAN. The check sits beside the deleted/disabled branch, before `insertRun`, so a skipped + * firing leaves nothing in `routine_runs` — a run row with no outcome and nothing coming to give + * it one would show on the routines page as a firing that started and never ended. + */ + expect(await runsFor(routine.id)).toHaveLength(0); + // Finished rather than released: re-delivery cannot make a past occurrence current. + const [row] = await firingsFor(routine.id); + expect(row?.finishedAt).not.toBeNull(); + + // The window is still a setting, so a caller that wants the late firing can have it. + await makeDueAt(routine.id, new Date("2001-02-01T09:25:00Z")); + await offerDueRoutines( + sweepOptions({ now: () => new Date("2001-02-01T09:26:00Z") }), + ); + const generous = await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-02-01T09:45:00Z"), + graceMs: 30 * 60_000, + }), + ); + expect(generous.fired).toEqual([routine.id]); + }); + + /** + * THE LESSON FROM `culler.ts:141-151`, which is why the renew comes before anything else. + * + * A batch is many and a lease is one. The first consumer claims, is slow, and its lease lapses; + * another replica takes the item and runs it. If the first then dispatched what it was holding, + * the person would get the same summary twice — and worse, the first would `finish` an item that + * belongs to the second, taking the lease away from a run that is still going. + */ + test("a consumer whose lease has gone neither dispatches nor finishes the item that is now somebody else's", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + const [offered] = await firingsFor(routine.id); + const now = at("2001-01-01T09:26:00Z"); + + const dispatchedBySecond: string[] = []; + const finishedByFirst: string[] = []; + let second: Awaited> | undefined; + + /* + * The first consumer's claim succeeds and then it stalls: the lease is written into the past and + * a whole second consumer runs the item to completion before the first gets to its own loop. + * Standing in for a slow pass rather than reimplementing one — everything after this point is + * still the queue's own answer. + */ + const stalling = { + ...queue, + claim: async (input: Parameters[0]) => { + const claimed = await queue.claim(input); + await backdate(offered?.key as string, { + leaseUntil: new Date("2001-01-01T00:00:00Z"), + }); + second = await dispatchClaimedRoutines( + sweepOptions({ + owner: "consumer-second", + now, + dispatch: async (runId: string) => { + dispatchedBySecond.push(runId); + }, + }), + ); + return claimed; + }, + finish: async (input: Parameters[0]) => { + finishedByFirst.push(input.key); + return await queue.finish(input); + }, + }; + + const first = await dispatchClaimedRoutines( + sweepOptions({ + owner: "consumer-first", + queue: stalling, + now, + dispatch: async (runId: string) => { + dispatched.push(`first:${runId}`); + }, + }), + ); + + // The second consumer did the work, exactly once. + expect(second?.fired).toEqual([routine.id]); + expect(dispatchedBySecond).toHaveLength(1); + expect(await runsFor(routine.id)).toHaveLength(1); + + // The first stopped at `renew`, which is not an error: the item is being handled, just not here. + expect(first.considered).toBe(1); + expect(first.fired).toEqual([]); + expect(first.skipped[0]?.routineId).toBe(routine.id); + expect(dispatched).toEqual([]); + // And it never called `finish`, so it could not have taken the item off a run in flight. + expect(finishedByFirst).toEqual([]); + }); + + /** + * At the cap the item stops being claimed, so this loop never sees that routine again: every + * sweep looks clean while one person's routine silently never fires. The row carries the count and + * the reason for anybody who queries the table; the warning is for whoever reads the logs. + */ + test("at the attempt cap the firing stops being claimed, and the giving up is said out loud", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((...args) => { + lines.push(String(args[0])); + }); + try { + await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + // One go, so the first failure is also the last. + maxAttempts: 1, + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + } finally { + warn.mockRestore(); + } + + const complaint = lines.find((line) => line.includes(routine.id)); + expect(complaint).toBeDefined(); + const said = JSON.parse(complaint as string); + expect(said.routineId).toBe(routine.id); + expect(said.attempts).toBe(1); + expect(said.reason).toContain("503"); + + /* + * And it is the cap that stops it, not the release delay: the delay is put in the past first, so + * a claim that still refuses the item is refusing it for having run out of goes. + */ + const [row] = await firingsFor(routine.id); + await backdate(row?.key as string, { + runAt: new Date("2001-01-01T09:27:00Z"), + }); + expect( + await queue.claim({ + kind: ROUTINE_FIRE_KIND, + owner: "sweep-test", + leaseMs: 30_000, + maxAttempts: 1, + }), + ).toEqual([]); + }); + + /** + * BOTH KINDS OF DONE WITH, which is the half `queue.ts:262-274` documents as having been forgotten. + * + * A finished row has to outlive the run long enough for a late replica to collide with it, and + * then go. An item at its attempt cap is not finished and is reaped by nothing else, so without + * this its key stays wedged for ever and that routine can never be offered for that minute again. + */ + test("the purge takes the finished firing and the one that gave up, once each is past the window", async () => { + const { routine: done } = await makeRoutine("Finished cleanly."); + const { routine: gaveUp } = await makeRoutine("Out of attempts."); + await offerFiring( + done.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:26:00Z") }), + ); + /* + * The finished routine's clock is parked out of reach before the second offer. + * + * `dueRoutines` asks Postgres what is due, so a routine whose stamp the first offer advanced to + * another day in 2001 is still due by the real clock, and the second offer would queue a second + * firing of it. That is the offering half behaving as designed against ancient stamps; what this + * test wants is one finished row and one that gave up, so it says which routine each pass is + * about rather than working around the overlap afterwards. + */ + await makeDueAt(done.id, new Date("2999-01-01T09:00:00Z")); + await offerFiring( + gaveUp.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + // The giving-up warning is asserted in its own test above; here it is only noise. + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + maxAttempts: 1, + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + } finally { + warn.mockRestore(); + } + + const window = 24 * 60 * 60 * 1000; + // Inside the window both rows stay: a finished row too young to have been collided with is the + // whole reason `finish` marks rather than deletes. + expect( + await queue.purge({ + kind: ROUTINE_FIRE_KIND, + olderThanMs: window, + maxAttempts: 1, + }), + ).toBe(0); + expect(await firingsFor(done.id)).toHaveLength(1); + expect(await firingsFor(gaveUp.id)).toHaveLength(1); + + const aged = new Date(Date.now() - 2 * window); + const [doneRow] = await firingsFor(done.id); + const [gaveUpRow] = await firingsFor(gaveUp.id); + await backdate(doneRow?.key as string, { finishedAt: aged }); + // The one that gave up has no `finished_at` to age at all — its `updated_at` is what dates it, + // which is precisely why it used to be reaped by nothing. + await backdate(gaveUpRow?.key as string, { updatedAt: aged }); + + expect( + await queue.purge({ + kind: ROUTINE_FIRE_KIND, + olderThanMs: window, + maxAttempts: 1, + }), + ).toBe(2); + expect(await firingsFor(done.id)).toHaveLength(0); + expect(await firingsFor(gaveUp.id)).toHaveLength(0); + }); + + /** + * The cap the consumer warns at has to be the cap the queue stops claiming at. + * + * A consumer with its own number warns on the wrong pass: too low and it complains every pass + * while the item is still being retried, too high and it never complains at all — the item stops + * being claimed and nothing anywhere says so. + */ + test("with no cap given, the consumer gives up on the pass the queue's own default stops handing it out", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + const [offered] = await firingsFor(routine.id); + // One short of the queue's default, so this claim is the last one the queue will allow. + await backdate(offered?.key as string, { + attempts: DEFAULT_MAX_ATTEMPTS - 1, + }); + + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((...args) => { + lines.push(String(args[0])); + }); + try { + await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + } finally { + warn.mockRestore(); + } + + const complaint = lines.find((line) => line.includes(routine.id)); + expect(complaint).toBeDefined(); + expect(JSON.parse(complaint as string).attempts).toBe(DEFAULT_MAX_ATTEMPTS); + + await backdate(offered?.key as string, { + runAt: new Date("2001-01-01T09:27:00Z"), + }); + expect( + await queue.claim({ + kind: ROUTINE_FIRE_KIND, + owner: "sweep-test", + leaseMs: 30_000, + }), + ).toEqual([]); + }); +}); From 85cf58d79e11fccde3c1e9a7932fccf5a7f137c5 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 13:27:19 -0300 Subject: [PATCH 22/45] Loop the sweep on a laptop, the way a cluster schedules it --- scripts/start.sh | 35 ++++++++- worker/src/index.ts | 177 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 2 deletions(-) diff --git a/scripts/start.sh b/scripts/start.sh index 54831826..ceada882 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -35,6 +35,12 @@ ONE_COMPUTER_EACH="${OPENBOT_ONE_COMPUTER_EACH:-true}" export APP_PORT SERVER_PORT SUPERVISOR_TOKEN="$(setting SUPERVISOR_TOKEN openbot-dev-supervisor-token)" COMPUTER_TOKEN="$(setting COMPUTER_TOKEN openbot-dev-computer-token)" +# A fixed default is fine here, unlike `AGENT_TOOL_TOKEN` below: this secret is compared by the +# server on its own loopback-bound port, never by anything a Bot publishes, so a well-known value +# from a public repository is not a boundary anybody outside this machine could reach anyway. That is +# also why it is generated fresh and persisted for AGENT_TOOL_TOKEN (see the SECRETS_ROTATED block) +# but not for this one. +WORKER_SHARED_SECRET="$(setting WORKER_SHARED_SECRET openbot-dev-worker-secret)" # The secret the server sends to a managed Bot, generated and written back on first run. # @@ -210,7 +216,7 @@ for svc in agent-computer agent-bot agent-langgraph; do SERVICES+=("$svc") done -export SUPERVISOR_TOKEN COMPUTER_TOKEN +export SUPERVISOR_TOKEN COMPUTER_TOKEN WORKER_SHARED_SECRET export COMPUTER_PORT BOT_PORT LANGGRAPH_PORT SUPERVISOR_PORT docker compose up -d --build "${SERVICES[@]}" >/dev/null if ! docker compose run --rm --build migrate >"$LOGS/migrate.log" 2>&1; then @@ -264,13 +270,37 @@ if ! identifies_as_openbot "$SERVER_PORT" server; then COMPUTER_SUPERVISOR_URL="http://localhost:$SUPERVISOR_PORT" \ SUPERVISOR_TOKEN="$SUPERVISOR_TOKEN" \ COMPUTER_TOKEN="$COMPUTER_TOKEN" \ + WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 &) else - (cd server && PORT="$SERVER_PORT" bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 &) + (cd server && PORT="$SERVER_PORT" \ + WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ + bun --env-file=../.env src/index.ts >"$LOGS/server.log" 2>&1 &) fi fi wait_for_openbot "$SERVER_PORT" server +# The worker: a local stand-in for the routines CronJob, looping the same sweep +# (`offerDueRoutines`/`dispatchClaimedRoutines`) a cluster would run on a schedule instead. Started +# only now, because dispatching a claimed routine is one HTTP call to this server's own +# /internal/routines/run, and the wait_for above is what confirms that call has somewhere to land. +# +# Guarded by a pgrep check rather than an HTTP health check, the same way the restart guard above +# matches the server's own command line with `pkill -f`: the worker loop has no HTTP endpoint of its +# own to ask, so "is a matching process already running" is the only signal a rerun of this script +# has for "leave it alone." +if ! pgrep -f "bun src/index.ts" >/dev/null 2>&1; then + WORKER_DATABASE_URL="$(setting DATABASE_URL postgres://openbot:openbot@localhost:5432/openbot)" + (cd worker && \ + DATABASE_URL="$WORKER_DATABASE_URL" \ + SERVER_INTERNAL_URL="http://localhost:$SERVER_PORT" \ + WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ + bun src/index.ts >"$LOGS/worker.log" 2>&1 &) + info " worker: started (routine sweep loop)" +else + info " worker: already running" +fi + info "3/4 Runtime health" INFO="$(curl -fsS --max-time 8 "http://localhost:$SERVER_PORT/api/copilotkit/info")" python3 - "$INFO" <<'PY' @@ -316,6 +346,7 @@ Try: 4. Add a deny rule in /admin/boundaries, then retry the same action. Logs: $LOGS + Routine sweep worker: $LOGS/worker.log Stop Docker services: docker compose down A Bot's computer is made by the supervisor rather than by compose, so it keeps running: docker rm -f \$(docker ps -q --filter label=openbot.supervisor=true) diff --git a/worker/src/index.ts b/worker/src/index.ts index aad40e41..9bc7b46b 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -1,3 +1,180 @@ +/** + * The local stand-in for the routines CronJob: `server/scripts/fire-routines.ts`, looped. + * + * That script runs one sweep and exits — a CronJob outside the process is what makes it recurring, + * and a failed run is meant to page somebody. A laptop running the dev stack has no CronJob around + * it, so this file supplies the recurrence itself, in-process, by importing the very same sweep + * (`offerDueRoutines`, `dispatchClaimedRoutines`) and the same stores/queue construction. It never + * spawns the script as a child process — shelling out to run it every 30 seconds would be a second, + * divergent implementation of what a sweep is, with its own bugs to keep in sync with the first. + * + * WHY THIS LOOP MUST NOT DIE ON THE FIRST DB BLIP, unlike the script it wraps: `fire-routines.ts` + * lets a phase's exception propagate so the CronJob's run is marked failed and a person is paged — + * that is correct there, because a fresh pod is one `kubectl` restart away and paging is cheap + * compared to routines silently going stale. This process has no restart policy watching it; it is + * somebody's laptop, left running. A worker that exited because Postgres hiccuped for two seconds + * would need a human to notice and restart it, which is worse than a worker that logs the failure and + * tries again on the next tick. So every phase below gets its own try/catch, and nothing here ever + * lets a phase's error reach the top and take the process down. + */ +import { createDatabase } from "../../server/src/db/client"; +import { createRoutineStore } from "../../server/src/routines/store"; +import { + ROUTINE_FIRE_KIND, + dispatchClaimedRoutines, + offerDueRoutines, + type RoutineSweepOptions, +} from "../../server/src/routines/sweep"; +import { createWorkQueue } from "../../server/src/work/queue"; import { workerStatus } from "./status"; console.info(`OpenBot worker status: ${workerStatus().status}`); + +/* + * Refused up front, for the reason `fire-routines.ts` refuses up front: a loop that started anyway + * would open a run row for every routine it offers itself and collect a 401 on every dispatch, + * forever, with the only evidence a line in the server's audit trail. Said once, loudly, before the + * first tick, is the difference between a worker that failed to start and a deployment where + * routines quietly do nothing. + */ +const workerSharedSecret = process.env.WORKER_SHARED_SECRET; +if (!workerSharedSecret) { + throw new Error( + "WORKER_SHARED_SECRET is not set, so this worker cannot authenticate itself to /internal/routines/run and no routine could be fired.", + ); +} + +/* + * Read from the environment rather than from `DeploymentConfig`/`loadConfig`, and deliberately so. + * + * `loadConfig` demands the whole server deployment's configuration — Intelligence credentials, key + * encryption, auth — because it answers "what can this deployment do". This process is handed exactly + * three settings by `scripts/start.sh` (`DATABASE_URL`, `SERVER_INTERNAL_URL`, + * `WORKER_SHARED_SECRET`); calling `loadConfig(process.env)` here would refuse to start over + * settings this loop has no opinion about and does not need. Where this process can reach its own API + * server is a fact about where this process runs, same as `fire-routines.ts` argues for + * `SERVER_INTERNAL_URL` alone — this file extends that reasoning to the secret and the database too. + */ +const serverInternalUrl = process.env.SERVER_INTERNAL_URL; +if (!serverInternalUrl) { + throw new Error( + "SERVER_INTERNAL_URL is not set, so this worker does not know where to hand a routine run.", + ); +} + +const database = createDatabase(process.env.DATABASE_URL ?? ""); +const queue = createWorkQueue(database); +const routineStore = createRoutineStore(database); + +// A name for the lease, so a stuck claim can be traced back to the process that took it. +const owner = `routines/${process.env.HOSTNAME ?? "laptop"}`; + +/** + * Hand one opened run to the server, which owns everything about running it. + * + * Identical to `fire-routines.ts`'s `dispatch`: the run id is all that crosses, the header string + * (casing and the one space included) is the whole credential the server compares, and anything but + * a 202 throws — naming the status, because that is the whole diagnosis a person reading + * `last_error` needs. + */ +async function dispatch(routineRunId: string): Promise { + const response = await fetch(`${serverInternalUrl}/internal/routines/run`, { + method: "POST", + headers: { + authorization: `Bearer ${workerSharedSecret}`, + "content-type": "application/json", + }, + body: JSON.stringify({ routineRunId }), + }); + if (response.status !== 202) { + throw new Error( + `the server answered ${response.status} rather than 202 when handed a routine run`, + ); + } +} + +const options: RoutineSweepOptions = { routineStore, queue, dispatch, owner }; + +/** How often both sweep phases run. A laptop's clock, standing in for the CronJob's schedule. */ +const TICK_MS = 30_000; + +/* + * How often the queue is purged of finished (and wedged) `routine.fire` items, in ticks rather than + * milliseconds, so the two cadences cannot drift apart by editing one constant and not the other. + * + * Once every 120 ticks — roughly hourly at a 30-second tick — not once a tick. `queue.purge` deletes + * rows older than the 24-hour window it is given below; running that DELETE every 30 seconds is three + * orders of magnitude more query load than the window needs, for a retention job whose whole job is + * to keep a day's worth of history. Hourly still purges comfortably inside the 24h window, with + * enormous room to spare if a tick is ever missed. + */ +const PURGE_EVERY_N_TICKS = 120; +const PURGE_OLDER_THAN_MS = 24 * 60 * 60 * 1000; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +let tick = 0; + +async function runOneTick(): Promise { + tick += 1; + + /* + * Both sweep phases, in one try/catch: this is the phase that runs every tick, and the one + * `fire-routines.ts` lets throw. Here it does not — it is logged and the loop moves on to the next + * tick 30 seconds later, per the file header above. A routine due right now that was missed by a + * failed tick is still due on the next one; nothing about being late loses it (see `DEFAULT_GRACE_MS` + * in `../../server/src/routines/sweep.ts`). + */ + try { + const { offered } = await offerDueRoutines(options); + const report = await dispatchClaimedRoutines(options); + console.info( + JSON.stringify({ + type: "routine-sweep", + offered, + considered: report.considered, + fired: report.fired, + skipped: report.skipped, + }), + ); + } catch (error) { + console.warn( + JSON.stringify({ + type: "routine-sweep-tick-failed", + reason: error instanceof Error ? error.message : String(error), + }), + ); + } + + // The purge phase, on its own much longer cadence and its own try/catch: a purge failure this hour + // is worth logging and retrying next hour, not a reason to stop offering and firing routines. + if (tick % PURGE_EVERY_N_TICKS === 0) { + try { + const purged = await queue.purge({ + kind: ROUTINE_FIRE_KIND, + olderThanMs: PURGE_OLDER_THAN_MS, + }); + console.info(JSON.stringify({ type: "routine-sweep-purge", purged })); + } catch (error) { + console.warn( + JSON.stringify({ + type: "routine-sweep-purge-failed", + reason: error instanceof Error ? error.message : String(error), + }), + ); + } + } +} + +async function main(): Promise { + // Loop for ever, one tick every TICK_MS, awaiting each tick fully before scheduling the next so two + // ticks are never in flight at once. + for (;;) { + await runOneTick(); + await sleep(TICK_MS); + } +} + +void main(); From 90ac96541998e5beb82d69eb76f9f8b803502437 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 13:36:38 -0300 Subject: [PATCH 23/45] Guard the worker start against every look-alike process --- scripts/start.sh | 18 +++++++++++++++--- worker/src/index.ts | 29 +++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/start.sh b/scripts/start.sh index ceada882..15a74e4c 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -289,14 +289,25 @@ wait_for_openbot "$SERVER_PORT" server # matches the server's own command line with `pkill -f`: the worker loop has no HTTP endpoint of its # own to ask, so "is a matching process already running" is the only signal a rerun of this script # has for "leave it alone." -if ! pgrep -f "bun src/index.ts" >/dev/null 2>&1; then +# +# Launched from `$ROOT` with `bun worker/src/index.ts`, not `cd worker && bun src/index.ts`: on a +# Linux host, container processes are visible to `pgrep` too, and `server/Dockerfile`, +# `agent-computer/Dockerfile`, and `supervisor/Dockerfile` all run `bun src/index.ts` as their argv. +# The old pattern matched those containers, the guard false-positived, and the worker silently never +# started. `bun worker/src/index.ts` matches nothing else in the repo. Running from `$ROOT` is safe: +# relative imports resolve from the importing file, not from the process's cwd. +if ! pgrep -f "bun worker/src/index.ts" >/dev/null 2>&1; then WORKER_DATABASE_URL="$(setting DATABASE_URL postgres://openbot:openbot@localhost:5432/openbot)" - (cd worker && \ + (cd "$ROOT" && \ DATABASE_URL="$WORKER_DATABASE_URL" \ SERVER_INTERNAL_URL="http://localhost:$SERVER_PORT" \ WORKER_SHARED_SECRET="$WORKER_SHARED_SECRET" \ - bun src/index.ts >"$LOGS/worker.log" 2>&1 &) + bun worker/src/index.ts >"$LOGS/worker.log" 2>&1 &) info " worker: started (routine sweep loop)" + sleep 1 + if ! pgrep -f "bun worker/src/index.ts" >/dev/null 2>&1; then + red " worker: did not stay up, check $LOGS/worker.log" + fi else info " worker: already running" fi @@ -347,6 +358,7 @@ Try: Logs: $LOGS Routine sweep worker: $LOGS/worker.log +Stop the routine worker: pkill -f 'bun worker/src/index.ts' Stop Docker services: docker compose down A Bot's computer is made by the supervisor rather than by compose, so it keeps running: docker rm -f \$(docker ps -q --filter label=openbot.supervisor=true) diff --git a/worker/src/index.ts b/worker/src/index.ts index 9bc7b46b..aa41ffc2 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -17,6 +17,7 @@ * tries again on the next tick. So every phase below gets its own try/catch, and nothing here ever * lets a phase's error reach the top and take the process down. */ +import { randomUUID } from "node:crypto"; import { createDatabase } from "../../server/src/db/client"; import { createRoutineStore } from "../../server/src/routines/store"; import { @@ -62,12 +63,27 @@ if (!serverInternalUrl) { ); } -const database = createDatabase(process.env.DATABASE_URL ?? ""); +/* + * Refused for the same reason as the two checks above: a loop that started anyway would hand + * `createDatabase` an empty connection string and fail on the first query with no indication of + * what was actually missing. + */ +const databaseUrl = process.env.DATABASE_URL; +if (!databaseUrl) { + throw new Error( + "DATABASE_URL is not set, so this worker has no database to read routines from or claim them in.", + ); +} + +const database = createDatabase(databaseUrl); const queue = createWorkQueue(database); const routineStore = createRoutineStore(database); -// A name for the lease, so a stuck claim can be traced back to the process that took it. -const owner = `routines/${process.env.HOSTNAME ?? "laptop"}`; +// A name for the lease, so a stuck claim can be traced back to the process that took it. Mirrors +// `fire-routines.ts`: `HOSTNAME` is not set by bash, so without the random fallback every worker +// started by `scripts/start.sh` would share the owner "routines/laptop" and `ours()` could no +// longer tell one worker's lease apart from another's. +const owner = `routines/${process.env.HOSTNAME ?? randomUUID().slice(0, 8)}`; /** * Hand one opened run to the server, which owns everything about running it. @@ -85,6 +101,9 @@ async function dispatch(routineRunId: string): Promise { "content-type": "application/json", }, body: JSON.stringify({ routineRunId }), + // The `for(;;)` loop below has no CronJob deadline bounding this call from outside; a wedged + // server must not stall the only thing firing routines. + signal: AbortSignal.timeout(30_000), }); if (response.status !== 202) { throw new Error( @@ -150,7 +169,9 @@ async function runOneTick(): Promise { // The purge phase, on its own much longer cadence and its own try/catch: a purge failure this hour // is worth logging and retrying next hour, not a reason to stop offering and firing routines. - if (tick % PURGE_EVERY_N_TICKS === 0) { + // Also on the very first tick: a laptop restarted every 40 minutes would otherwise never survive + // to tick 120, and would never reap. + if (tick === 1 || tick % PURGE_EVERY_N_TICKS === 0) { try { const purged = await queue.purge({ kind: ROUTINE_FIRE_KIND, From cb8968428c5df38cb5611c63f594013a60bc3034 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 13:39:10 -0300 Subject: [PATCH 24/45] Schedule the routines sweep the way the culler is scheduled --- charts/openbot/ci/self-hosted-values.yaml | 6 ++ charts/openbot/templates/_helpers.tpl | 14 ++++ .../openbot/templates/routines/cronjob.yaml | 67 +++++++++++++++++++ charts/openbot/templates/secret.yaml | 8 +++ charts/openbot/values.yaml | 12 ++++ 5 files changed, 107 insertions(+) create mode 100644 charts/openbot/templates/routines/cronjob.yaml diff --git a/charts/openbot/ci/self-hosted-values.yaml b/charts/openbot/ci/self-hosted-values.yaml index b4f7ea8f..688d4570 100644 --- a/charts/openbot/ci/self-hosted-values.yaml +++ b/charts/openbot/ci/self-hosted-values.yaml @@ -28,6 +28,8 @@ secrets: googleClientSecret: "example-for-rendering-only" computerToken: "example-for-rendering-only" licenseToken: "example-for-rendering-only" + # So the routines CronJob renders in CI at all. Rendering example only. + workerSharedSecret: "example-for-rendering-only" ingress: enabled: true className: nginx @@ -40,6 +42,10 @@ ingress: computers: mode: shared +# So the routines CronJob renders in CI at all. +routines: + enabled: true + # On, so these render in CI at all. They were written, reviewed and shipped without a single target # producing one, which means nothing had ever checked they were valid YAML, let alone right. Off is # still the chart's default: a policy on a cluster whose CNI ignores it does nothing, and on one that diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl index 14d96645..a1869f97 100644 --- a/charts/openbot/templates/_helpers.tpl +++ b/charts/openbot/templates/_helpers.tpl @@ -267,6 +267,20 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon {{- with .Values.config.extraEnv }} {{ toYaml . }} {{- end }} +{{- /* + One definition, for the same reason `openbot.databaseUrlEnv` is one (see its comment above): the + API server needs this value to RECOGNISE the worker, and the routines CronJob needs the same value + to BE the worker. Two definitions could drift; this can't. Gated on `routines.enabled` so a + deployment that never turns routines on gets no env var pointing at a key its secret store may not + hold. +*/}} +{{- if .Values.routines.enabled }} +- name: WORKER_SHARED_SECRET + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" . }} + key: worker-shared-secret +{{- end }} {{- end -}} {{/* diff --git a/charts/openbot/templates/routines/cronjob.yaml b/charts/openbot/templates/routines/cronjob.yaml new file mode 100644 index 00000000..2a360d24 --- /dev/null +++ b/charts/openbot/templates/routines/cronjob.yaml @@ -0,0 +1,67 @@ +{{- if .Values.routines.enabled }} +{{- $component := "routines" -}} +{{/* +Firing the routines a Bot was scheduled to run. + +A CronJob rather than a timer in the API, for the same reason the culler beside it is one: an +interval in the server fires in every replica, so five replicas would each offer the same due +routine to the queue. The queue itself only lets one worker take a given firing, but there is no +reason to make five pods race for it every tick when one CronJob run does the whole sweep. + +`concurrencyPolicy: Forbid` on top, because a schedule that overlaps itself is the cross-replica +problem in one workload rather than across several. +*/}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + schedule: {{ .Values.routines.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 120 + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 12 }} + {{- if .Values.postgresql.enabled }} + {{- /* + The bundled database admits pods carrying this label and nothing else, which this chart pins on. + Anything that opens the database needs it, and only the API server had it: this would have been + refused on any cluster that actually enforces a NetworkPolicy. Not caught by hand, because the + cluster it was driven on ships enforcement switched off. + */}} + {{ .Release.Name }}-postgresql-client: "true" + {{- end }} + spec: + restartPolicy: Never + serviceAccountName: {{ include "openbot.serviceAccountName" . }} + {{- /* This sweep reads the database and posts to the API; it never asks the cluster for anything, so it gets no token. */}} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 12 }} + {{- end }} + containers: + - name: routines + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + workingDir: /app/server + command: ["/usr/local/bin/bun", "scripts/fire-routines.ts"] + env: +{{ include "openbot.databaseUrlEnv" . | indent 16 }} +{{ include "openbot.commonEnv" . | indent 16 }} + - name: SERVER_INTERNAL_URL + value: http://{{ include "openbot.componentName" (dict "root" . "component" "server") }}:{{ .Values.server.service.port }} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi +{{- end }} diff --git a/charts/openbot/templates/secret.yaml b/charts/openbot/templates/secret.yaml index c10439d5..8b7ecf8b 100644 --- a/charts/openbot/templates/secret.yaml +++ b/charts/openbot/templates/secret.yaml @@ -47,4 +47,12 @@ stringData: {{- with .Values.secrets.supervisorToken }} supervisor-token: {{ . | quote }} {{- end }} + {{- /* + `commonEnv` references this key whenever `routines.enabled`, so a silent `with` here would let a + deployment turn routines on with no secret and find out at 03:05 that every firing gets a 401. + `required` fails at `helm install` instead. + */}} + {{- if .Values.routines.enabled }} + worker-shared-secret: {{ required "secrets.workerSharedSecret is required when routines.enabled. Generate one with: openssl rand -base64 32" .Values.secrets.workerSharedSecret | quote }} + {{- end }} {{- end }} diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml index 65f98420..fe47886c 100644 --- a/charts/openbot/values.yaml +++ b/charts/openbot/values.yaml @@ -237,6 +237,15 @@ computers: enabled: true schedule: "*/5 * * * *" +# Standing instructions a Bot runs on a schedule. The sweep offers due routines to the shared work +# queue and hands each firing to the API server, which runs the turn as the person who scheduled it. +routines: + # Off by default, because it needs a secret. Without `secrets.workerSharedSecret` the API server + # refuses every handoff, and a CronJob whose every run is a 401 is worse than no CronJob at all. + enabled: false + # Inside the 15-minute floor the tools enforce, so a firing waits at most one tick. + schedule: "*/5 * * * *" + database: # Used when `postgresql.enabled` is false. A URL, or a secret holding one. # @@ -333,6 +342,9 @@ secrets: licenseToken: "" # Sent to `config.managedAgent.url` on every call. Required when that url is set. managedAgentToken: "" + # Required when `routines.enabled`: what the CronJob presents to the API server to be recognised + # as the worker rather than as an unauthenticated caller. + workerSharedSecret: "" # The client secret for whichever provider is configured above. googleClientSecret: "" microsoftClientSecret: "" From a700643cd11b9aa5cc539276f58084f84c185f71 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 13:50:07 -0300 Subject: [PATCH 25/45] Refuse at install what would crash-loop at midnight --- charts/openbot/templates/_helpers.tpl | 11 ++++++++--- .../openbot/templates/routines/cronjob.yaml | 1 + charts/openbot/templates/validation.yaml | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl index a1869f97..bc75d2d8 100644 --- a/charts/openbot/templates/_helpers.tpl +++ b/charts/openbot/templates/_helpers.tpl @@ -264,15 +264,17 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon name: {{ default (include "openbot.secretName" .) .Values.computers.existingTokenSecret }} key: computer-token optional: {{ eq .Values.computers.mode "external" }} -{{- with .Values.config.extraEnv }} -{{ toYaml . }} -{{- end }} {{- /* One definition, for the same reason `openbot.databaseUrlEnv` is one (see its comment above): the API server needs this value to RECOGNISE the worker, and the routines CronJob needs the same value to BE the worker. Two definitions could drift; this can't. Gated on `routines.enabled` so a deployment that never turns routines on gets no env var pointing at a key its secret store may not hold. + + Above `config.extraEnv`, not below it: Kubernetes takes the last of a duplicate name, and this must + lose to an operator's own value, not win over it. Below it, this chart's own secretKeyRef would + override whatever `extraEnv` set, which turns the escape hatch into a trap for the one variable + someone would need it for. */}} {{- if .Values.routines.enabled }} - name: WORKER_SHARED_SECRET @@ -281,6 +283,9 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon name: {{ include "openbot.secretName" . }} key: worker-shared-secret {{- end }} +{{- with .Values.config.extraEnv }} +{{ toYaml . }} +{{- end }} {{- end -}} {{/* diff --git a/charts/openbot/templates/routines/cronjob.yaml b/charts/openbot/templates/routines/cronjob.yaml index 2a360d24..066ed660 100644 --- a/charts/openbot/templates/routines/cronjob.yaml +++ b/charts/openbot/templates/routines/cronjob.yaml @@ -43,6 +43,7 @@ spec: restartPolicy: Never serviceAccountName: {{ include "openbot.serviceAccountName" . }} {{- /* This sweep reads the database and posts to the API; it never asks the cluster for anything, so it gets no token. */}} + automountServiceAccountToken: false {{- with .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml . | indent 12 }} diff --git a/charts/openbot/templates/validation.yaml b/charts/openbot/templates/validation.yaml index 8d34c137..5b569eca 100644 --- a/charts/openbot/templates/validation.yaml +++ b/charts/openbot/templates/validation.yaml @@ -275,3 +275,22 @@ This template renders nothing. {{- end }} {{- end }} {{- end }} + +{{- /* + The same requirement, for the worker's shared secret. + + `secret.yaml`, where `WORKER_SHARED_SECRET` would otherwise be required into existence, is skipped + entirely when `externalSecrets.enabled` — that Secret is the store's to create, not this chart's. + So a cloud operator who turns on `routines.enabled` without adding this key to + `externalSecrets.data` gets a clean install: the API server and the CronJob both reference a key + that is never coming, and every pod that mounts it fails to start. Checked the same way as + `managed-agent-token` and `better-auth-secret` above: the value is not readable at template time, + but the list of keys is, and a store that never mentions this key cannot be holding one. +*/}} +{{- if and .Values.routines.enabled .Values.externalSecrets.enabled }} +{{- $named := list }} +{{- range .Values.externalSecrets.data }}{{- $named = append $named .secretKey }}{{- end }} +{{- if not (has "worker-shared-secret" $named) }} +{{- fail "routines.enabled but externalSecrets.data does not name worker-shared-secret. The API server reads it to recognise the worker and the CronJob reads it to be one, so every pod that mounts it fails to start." }} +{{- end }} +{{- end }} From b840e58186a7c58a1503c63a0daf28d86b4ba1bd Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 13:55:45 -0300 Subject: [PATCH 26/45] Serve a person their own standing instructions --- server/src/app.ts | 16 ++ server/src/index.ts | 2 + server/src/routines/routes.ts | 126 +++++++++++ server/tests/routine-routes.test.ts | 326 ++++++++++++++++++++++++++++ 4 files changed, 470 insertions(+) create mode 100644 server/src/routines/routes.ts create mode 100644 server/tests/routine-routes.test.ts diff --git a/server/src/app.ts b/server/src/app.ts index 29a3ca78..eeae6a7b 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -45,6 +45,7 @@ import { REFUSAL_MARKER } from "./plugins/tools"; import type { IntentRouter } from "./routing/classify"; import { createRoutingRoutes } from "./routing/routes"; import type { RoutineRunner } from "./routines/runner"; +import { createRoutineRoutes, type RoutineStore } from "./routines/routes"; import type { PackageStatusReader } from "./tenant-package"; /** @@ -186,6 +187,17 @@ export function createApp( * refusing every call: a deployment that never built a worker has no door for it, not a locked one. */ routineRunner?: RoutineRunner, + /** + * A person's own standing instructions: the list, and a switch to stop one. + * + * Appended last, like `routineRunner` beside it: these are positional, so inserting one anywhere + * else silently shifts every existing call site's arguments by one. + * + * Absent leaves the routes unmounted rather than mounted and refusing every call, the same + * degraded shape every other optional store here takes: a deployment that never built the store + * has no door for this at all, not a locked one. + */ + routineStore?: RoutineStore, ) { const app = new Hono<{ Variables: AppVariables }>(); @@ -816,6 +828,10 @@ export function createApp( ); } + if (routineStore) { + app.route("/api/routines", createRoutineRoutes(routineStore, requireUser)); + } + if (componentStore) { app.route( "/api/components", diff --git a/server/src/index.ts b/server/src/index.ts index 9085685d..4ad296c1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -717,6 +717,8 @@ const app = createApp( createPageFrameStore(database), // What a due routine actually does: a turn, run as its owner, into the thread they will open. routineRunner, + // A person's own standing instructions: the list, and a switch to stop one. + routineStore, ); /** diff --git a/server/src/routines/routes.ts b/server/src/routines/routes.ts new file mode 100644 index 00000000..0be1d021 --- /dev/null +++ b/server/src/routines/routes.ts @@ -0,0 +1,126 @@ +import type { Context, MiddlewareHandler } from "hono"; +import { Hono } from "hono"; +import type { AppVariables } from "../auth/guards"; +import { + RoutineNotFoundError, + RoutineRefusedError, + type RoutineStore, + type RoutineSummary, +} from "./store"; + +export type { RoutineStore } from "./store"; + +/** + * The routines page: what a person's standing instructions are, and a switch to stop one. + * + * THERE IS DELIBERATELY NO CREATE AND NO EDIT ENDPOINT HERE. Making a routine and changing one are + * conversational — the four `RoutineTools` a Bot calls mid-chat, in `plugins/builtin-routines.ts` — + * because the hard part of both is turning a sentence into a cron expression and a channel, which + * is exactly what a conversation is for. This surface answers a narrower question: what is standing, + * and does it stay standing. So it shows and it stops; it does not compose. + * + * THERE IS ALSO NO GET-BY-ID. A routine id is never a page's own state — nothing links to one, and + * nothing needs to fetch one in isolation — so the list is the read, the same way the channel + * roster is a list with no companion single-channel screen. + * + * Owner-scoped through the store on every route, never by filtering a broader read afterwards: + * `listFor`, `setEnabled` and `remove` all take the caller's id and answer as if a routine that + * belongs to somebody else does not exist, which is what keeps a wrong id and somebody else's id + * indistinguishable from outside. + */ +export function createRoutineRoutes( + routineStore: RoutineStore, + requireUser: MiddlewareHandler<{ Variables: AppVariables }>, +) { + const routes = new Hono<{ Variables: AppVariables }>(); + + routes.get("/", requireUser, async (context) => { + const routines = await routineStore.listFor(context.var.actor.id); + return context.json({ routines: routines.map(routineDto) }); + }); + + routes.put("/:id/enabled", requireUser, async (context) => { + const body = await context.req.json().catch(() => null); + const enabled = (body as { enabled?: unknown } | null)?.enabled; + if (typeof enabled !== "boolean") { + return context.json({ error: "enabled must be true or false." }, 400); + } + + try { + await routineStore.setEnabled( + context.var.actor.id, + context.req.param("id"), + enabled, + ); + return context.json({ enabled }); + } catch (error) { + return mapStoreError(context, error); + } + }); + + routes.delete("/:id", requireUser, async (context) => { + try { + await routineStore.remove(context.var.actor.id, context.req.param("id")); + return context.body(null, 204); + } catch (error) { + return mapStoreError(context, error); + } + }); + + return routes; +} + +/** + * One row of the routines page. + * + * THE DTO CARRIES THE WORDS, NOT THE CRON. The client never parses a schedule, so words-from-cron is + * computed once, server-side, by the same library that decides when a routine actually fires — there + * is no second implementation of cron-to-English anywhere to drift out of step with it. `schedule` is + * opaque display text, never a value to parse: prose for the shapes `describeCron` recognizes, and + * the raw five-field expression for everything stranger than that. `nextRunAt` is the one place a + * time is computed, and it is computed the same way, by `nextOccurrence`. + */ +type RoutineDto = { + id: string; + schedule: string; + timezone: string; + instruction: string; + channel: { id: string; name: string | null; gone: boolean }; + enabled: boolean; + nextRunAt: string; + lastRun: { status: string | null; at: string | null } | null; +}; + +function routineDto(routine: RoutineSummary): RoutineDto { + return { + id: routine.id, + schedule: routine.schedule, + timezone: routine.timezone, + instruction: routine.instruction, + channel: { + id: routine.channelId, + name: routine.channelName, + gone: routine.channelDeleted, + }, + enabled: routine.enabled, + nextRunAt: routine.nextRunAt.toISOString(), + lastRun: routine.lastRun + ? { + status: routine.lastRun.status, + at: routine.lastRun.finishedAt?.toISOString() ?? null, + } + : null, + }; +} + +function mapStoreError(context: Context, error: unknown): Response { + // The store's own sentence, verbatim: it already reads the same whether the id belongs to nobody + // or to somebody else, which is what keeps ownership unprobeable from out here. + if (error instanceof RoutineNotFoundError) { + return context.json({ error: error.message }, 404); + } + if (error instanceof RoutineRefusedError) { + return context.json({ error: error.message }, 400); + } + throw error; +} diff --git a/server/tests/routine-routes.test.ts b/server/tests/routine-routes.test.ts new file mode 100644 index 00000000..f8211e39 --- /dev/null +++ b/server/tests/routine-routes.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, test } from "bun:test"; +import type { MiddlewareHandler } from "hono"; +import { Hono } from "hono"; +import type { AppVariables } from "../src/auth/guards"; +import { createRoutineRoutes } from "../src/routines/routes"; +import { + RoutineNotFoundError, + RoutineRefusedError, + type RoutineStore, + type RoutineSummary, +} from "../src/routines/store"; + +const actor = { + id: "user-1", + email: "member@openbot.test", + role: "user", +} as const; + +function summary(overrides: Partial = {}): RoutineSummary { + return { + id: "routine-1", + agentId: "agent-1", + instruction: "Post the weather every weekday morning.", + schedule: "Weekdays at 09:00", + timezone: "UTC", + enabled: true, + nextRunAt: new Date("2026-08-27T09:00:00.000Z"), + channelId: "channel-1", + channelName: "Assistant channel", + channelDeleted: false, + lastRun: { + status: "succeeded", + finishedAt: new Date("2026-08-26T09:00:00.000Z"), + }, + ...overrides, + }; +} + +type StoreCall = [method: keyof RoutineStore, ...arguments_: unknown[]]; + +function fakeStore( + overrides: Partial = {}, +): RoutineStore & { calls: StoreCall[] } { + const calls: StoreCall[] = []; + const base: RoutineStore = { + async create() { + throw new Error("not used by these tests"); + }, + async listFor(ownerUserId) { + calls.push(["listFor", ownerUserId]); + return [summary()]; + }, + async update() { + throw new Error("not used by these tests"); + }, + async remove(ownerUserId, id) { + calls.push(["remove", ownerUserId, id]); + }, + async setEnabled(ownerUserId, id, enabled) { + calls.push(["setEnabled", ownerUserId, id, enabled]); + }, + async dueRoutines() { + return []; + }, + async advanceNextRun() { + return false; + }, + async insertRun() { + return { runId: "routine_run-1" }; + }, + async runContext() { + return null; + }, + async routineForFiring() { + return null; + }, + async finishRun() {}, + async consecutiveFailures() { + return 0; + }, + }; + + return Object.assign(base, overrides, { calls }); +} + +const requireUser: MiddlewareHandler<{ Variables: AppVariables }> = async ( + context, + next, +) => { + context.set("actor", actor); + await next(); +}; + +const denied: MiddlewareHandler<{ Variables: AppVariables }> = (context) => + Promise.resolve(context.json({ error: "denied" }, 401)); + +function appFor( + store: RoutineStore, + middleware: MiddlewareHandler<{ Variables: AppVariables }> = requireUser, +) { + const app = new Hono<{ Variables: AppVariables }>(); + app.route("/", createRoutineRoutes(store, middleware)); + return app; +} + +async function json(response: Response) { + return response.json(); +} + +describe("GET /", () => { + test("lists the caller's own routines as words, not cron", async () => { + const store = fakeStore(); + const response = await appFor(store).request("http://openbot.test/"); + + expect(response.status).toBe(200); + expect(await json(response)).toEqual({ + routines: [ + { + id: "routine-1", + schedule: "Weekdays at 09:00", + timezone: "UTC", + instruction: "Post the weather every weekday morning.", + channel: { id: "channel-1", name: "Assistant channel", gone: false }, + enabled: true, + nextRunAt: "2026-08-27T09:00:00.000Z", + lastRun: { status: "succeeded", at: "2026-08-26T09:00:00.000Z" }, + }, + ], + }); + expect(store.calls).toEqual([["listFor", actor.id]]); + }); + + test("carries no lastRun when the routine has never fired", async () => { + const store = fakeStore({ + listFor: async () => [summary({ lastRun: null })], + }); + const response = await appFor(store).request("http://openbot.test/"); + + expect((await json(response)).routines[0].lastRun).toBeNull(); + }); + + test("the DTO carries the schedule as words and never a cron field", async () => { + const store = fakeStore(); + const response = await appFor(store).request("http://openbot.test/"); + const body = await json(response); + + expect(body.routines[0].schedule).toBe("Weekdays at 09:00"); + expect(JSON.stringify(body)).not.toContain("cron"); + }); + + test("refuses without a session, before the store is asked", async () => { + const store = fakeStore(); + const response = await appFor(store, denied).request( + "http://openbot.test/", + ); + + expect(response.status).toBe(401); + expect(store.calls).toEqual([]); + }); +}); + +describe("PUT /:id/enabled", () => { + test("switches a routine on or off through the authenticated actor", async () => { + const store = fakeStore(); + const response = await appFor(store).request( + "http://openbot.test/routine-1/enabled", + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ enabled: false }), + }, + ); + + expect(response.status).toBe(200); + expect(await json(response)).toEqual({ enabled: false }); + expect(store.calls).toEqual([["setEnabled", actor.id, "routine-1", false]]); + }); + + test.each([ + ["{", "enabled must be true or false."], + [JSON.stringify({}), "enabled must be true or false."], + [JSON.stringify({ enabled: "yes" }), "enabled must be true or false."], + [JSON.stringify({ enabled: 1 }), "enabled must be true or false."], + ])("rejects a malformed body: %p", async (body, error) => { + const store = fakeStore(); + const response = await appFor(store).request( + "http://openbot.test/routine-1/enabled", + { + method: "PUT", + headers: { "content-type": "application/json" }, + body, + }, + ); + + expect(response.status).toBe(400); + expect(await json(response)).toEqual({ error }); + expect(store.calls).toEqual([]); + }); + + test("another owner's routine reads exactly like one that does not exist", async () => { + const store = fakeStore({ + setEnabled: async () => { + throw new RoutineNotFoundError(); + }, + }); + const missingStore = fakeStore({ + setEnabled: async () => { + throw new RoutineNotFoundError(); + }, + }); + + const notMine = await appFor(store).request( + "http://openbot.test/somebody-elses-routine/enabled", + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ enabled: true }), + }, + ); + const missing = await appFor(missingStore).request( + "http://openbot.test/no-such-routine/enabled", + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ enabled: true }), + }, + ); + + expect(notMine.status).toBe(404); + expect(missing.status).toBe(404); + const notMineBody = await json(notMine); + expect(notMineBody).toEqual(await json(missing)); + expect(notMineBody).toEqual({ error: "That routine does not exist." }); + }); + + test("carries a store refusal's sentence verbatim as a 400", async () => { + const store = fakeStore({ + setEnabled: async () => { + throw new RoutineRefusedError( + "You already have 20 routines switched on. Switch one off before adding another.", + ); + }, + }); + const response = await appFor(store).request( + "http://openbot.test/routine-1/enabled", + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ enabled: true }), + }, + ); + + expect(response.status).toBe(400); + expect(await json(response)).toEqual({ + error: + "You already have 20 routines switched on. Switch one off before adding another.", + }); + }); + + test("refuses without a session, before the store is asked", async () => { + const store = fakeStore(); + const response = await appFor(store, denied).request( + "http://openbot.test/routine-1/enabled", + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ enabled: true }), + }, + ); + + expect(response.status).toBe(401); + expect(store.calls).toEqual([]); + }); +}); + +describe("DELETE /:id", () => { + test("stops a routine through the authenticated actor", async () => { + const store = fakeStore(); + const response = await appFor(store).request( + "http://openbot.test/routine-1", + { method: "DELETE" }, + ); + + expect(response.status).toBe(204); + expect(store.calls).toEqual([["remove", actor.id, "routine-1"]]); + }); + + test("another owner's routine reads exactly like one that does not exist", async () => { + const notMineStore = fakeStore({ + remove: async () => { + throw new RoutineNotFoundError(); + }, + }); + const missingStore = fakeStore({ + remove: async () => { + throw new RoutineNotFoundError(); + }, + }); + + const notMine = await appFor(notMineStore).request( + "http://openbot.test/somebody-elses-routine", + { method: "DELETE" }, + ); + const missing = await appFor(missingStore).request( + "http://openbot.test/no-such-routine", + { method: "DELETE" }, + ); + + expect(notMine.status).toBe(404); + expect(missing.status).toBe(404); + const notMineBody = await json(notMine); + expect(notMineBody).toEqual(await json(missing)); + expect(notMineBody).toEqual({ error: "That routine does not exist." }); + }); + + test("refuses without a session, before the store is asked", async () => { + const store = fakeStore(); + const response = await appFor(store, denied).request( + "http://openbot.test/routine-1", + { method: "DELETE" }, + ); + + expect(response.status).toBe(401); + expect(store.calls).toEqual([]); + }); +}); From 3217b5d868611a121b9f5319045588c9d08f34a9 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 14:01:53 -0300 Subject: [PATCH 27/45] Show every standing instruction, and let a person stop one --- .../components/app-sidebar/app-sidebar.tsx | 21 ++ app/src/components/routines/routines-list.tsx | 250 ++++++++++++++++++ app/src/lib/routines/mutations.ts | 41 +++ app/src/lib/routines/queries.ts | 51 ++++ app/src/routeTree.gen.ts | 21 ++ app/src/routes/_authed/_app/routines.tsx | 30 +++ app/src/routes/_authed/admin/audit.tsx | 11 +- 7 files changed, 423 insertions(+), 2 deletions(-) create mode 100644 app/src/components/routines/routines-list.tsx create mode 100644 app/src/lib/routines/mutations.ts create mode 100644 app/src/lib/routines/queries.ts create mode 100644 app/src/routes/_authed/_app/routines.tsx diff --git a/app/src/components/app-sidebar/app-sidebar.tsx b/app/src/components/app-sidebar/app-sidebar.tsx index 7666a258..3bf10b10 100644 --- a/app/src/components/app-sidebar/app-sidebar.tsx +++ b/app/src/components/app-sidebar/app-sidebar.tsx @@ -2,6 +2,7 @@ import { IconBellRinging, IconBolt, IconBox, + IconClock, IconLogout, IconPlus, IconSearch, @@ -394,6 +395,26 @@ export function AppSidebar({ ...props }: React.ComponentProps) { Agents + + {/* Beside Skills and Agents rather than inside Admin: a routine is something anybody has. */} + ( + + )} + > +
+ +
+ Routines + + Math.abs(elapsed) < limit) ?? + RELATIVE_UNITS[RELATIVE_UNITS.length - 1]; + return relativeFormat.format( + -Math.round(elapsed / scale.divisor), + scale.unit, + ); +} + +/** + * What the last-run cell says, and in what tone. + * + * `lastRun === null` and `lastRun.status === null` are different facts, and saying the wrong one + * invents news: the first is "this routine has never finished a run," the second is "one is open + * right now" — which is also what a run stuck open after repeated dispatch failures looks like from + * here. Neither is a failure, so neither gets the destructive tone; only `status: "failed"` does. + */ +function lastRunLabel(lastRun: RoutineRecord["lastRun"]): { + text: string; + className: string; +} { + if (lastRun === null) { + return { text: "Never run yet", className: "text-muted-foreground" }; + } + if (lastRun.status === null) { + return { text: "Running…", className: "text-muted-foreground" }; + } + const when = lastRun.at ? relativeTime(lastRun.at) : "recently"; + if (lastRun.status === "failed") { + return { text: `Failed ${when}`, className: "text-destructive" }; + } + if (lastRun.status === "skipped") { + return { + text: `Skipped ${when}`, + className: "text-amber-600 dark:text-amber-500", + }; + } + return { text: `Ran ${when}`, className: "text-muted-foreground" }; +} + +/** + * The one list the Routines page shows: every standing instruction the signed-in person owns, a + * switch to stop one taking effect, and a delete that ends it for good. + */ +export function RoutinesList() { + const queryClient = useQueryClient(); + const routines = useQuery(routinesQueryOptions()); + const setEnabled = useMutation(setRoutineEnabledMutationOptions(queryClient)); + const deleteRoutine = useMutation(deleteRoutineMutationOptions(queryClient)); + /** The routine a delete is being confirmed for, or null. Its own dialog rather than one per row. */ + const [confirmingId, setConfirmingId] = useState(null); + const rows = routines.data ?? []; + const confirming = rows.find((row) => row.id === confirmingId) ?? null; + + return ( + + {setEnabled.error ? ( +

+ {setEnabled.error.message} +

+ ) : null} + + {/* Pending renders nothing: the empty-state sentence would otherwise flash for the fetch. */} + {routines.isPending ? null : routines.error ? ( +

+ Your routines could not be loaded. +

+ ) : rows.length === 0 ? ( + + Nothing scheduled. Ask a Bot — "every weekday at 9, …" — and it will + appear here. + + ) : ( + + {rows.map((routine, index) => { + const { text: lastRunText, className: lastRunClassName } = + lastRunLabel(routine.lastRun); + return ( +
+ + + + {routine.schedule} + + {routine.timezone} + + + + {routine.instruction} + + {/* A set, so it wraps onto its own line rather than crowding the title. */} + +
+ + {routine.channel.gone + ? "This channel is gone" + : (routine.channel.name ?? "Unnamed channel")} + + {lastRunText} +
+ + + + {/* + * Binary and immediate: it takes effect when switched, there is no save. + * Disabled only while its own write is in flight, so switching one routine + * does not freeze the rest of the list — the same idiom the per-tool plugins + * page uses for its per-Bot grant switches. + */} + + setEnabled.mutate({ id: routine.id, enabled: next }) + } + /> + + + + {index !== rows.length - 1 && } +
+ ); + })} + + )} + + {/* + * One dialog for the whole list rather than one per row, keyed by which routine is being + * confirmed. It names the schedule, not the id or the instruction, because the schedule is + * the word a person reads first on the row and the one most likely to tell two routines apart + * at a glance. + */} + { + if (!open) setConfirmingId(null); + }} + open={confirming !== null} + > + + + Delete "{confirming?.schedule}"? + + This standing instruction stops for good. Nothing further runs on + this schedule, and there is no undo. + + + {deleteRoutine.error ? ( +

+ {deleteRoutine.error.message} +

+ ) : null} + + + + + + + + ); +} diff --git a/app/src/lib/routines/mutations.ts b/app/src/lib/routines/mutations.ts new file mode 100644 index 00000000..20b8ee43 --- /dev/null +++ b/app/src/lib/routines/mutations.ts @@ -0,0 +1,41 @@ +import { mutationOptions, type QueryClient } from "@tanstack/react-query"; +import { client } from "@/lib/client"; +import { routineKeys } from "./queries"; + +/** + * Writes against a person's own standing instructions. + * + * THERE IS NO CREATE AND NO EDIT HERE, on purpose: this page only shows and stops. Making a routine + * and changing one are conversational, through the `RoutineTools` a Bot calls mid-chat — see + * `server/src/routines/routes.ts` for the full reasoning. + */ + +const FALLBACK = "That routine could not be changed."; + +function invalidateRoutines(queryClient: QueryClient) { + return queryClient.invalidateQueries({ queryKey: routineKeys.all }); +} + +/** Switch one routine on or off. Immediate; there is no save. */ +export function setRoutineEnabledMutationOptions(queryClient: QueryClient) { + return mutationOptions({ + mutationFn: (variables: { id: string; enabled: boolean }) => + client(`/api/routines/${encodeURIComponent(variables.id)}/enabled`, { + method: "PUT", + body: { enabled: variables.enabled }, + fallback: FALLBACK, + }), + onSuccess: () => invalidateRoutines(queryClient), + }); +} + +export function deleteRoutineMutationOptions(queryClient: QueryClient) { + return mutationOptions({ + mutationFn: (id: string) => + client(`/api/routines/${encodeURIComponent(id)}`, { + method: "DELETE", + fallback: FALLBACK, + }), + onSuccess: () => invalidateRoutines(queryClient), + }); +} diff --git a/app/src/lib/routines/queries.ts b/app/src/lib/routines/queries.ts new file mode 100644 index 00000000..33414c40 --- /dev/null +++ b/app/src/lib/routines/queries.ts @@ -0,0 +1,51 @@ +import { queryOptions } from "@tanstack/react-query"; +import { client } from "@/lib/client"; + +/** + * One standing instruction, as the Routines page sees it. + * + * `schedule` IS OPAQUE DISPLAY TEXT, NEVER A VALUE TO PARSE. The server computes it once, from the + * same library that decides when the routine actually fires — prose for a shape it recognizes, the + * raw five-field cron expression for anything stranger than that. Rendering it verbatim is the only + * correct thing to do with it; a second cron-to-English implementation in the browser would drift + * out of step with the server's the first time either one changes. + */ +export type RoutineRecord = { + id: string; + schedule: string; + timezone: string; + instruction: string; + channel: { id: string; name: string | null; gone: boolean }; + enabled: boolean; + nextRunAt: string; + /** + * Null means no run has ever finished. An object with `status: null` means a run is open — + * started but not yet finished, whether genuinely in flight or stuck there after repeated + * dispatch failures. Neither is a failure; only `status: "failed"` is. + */ + lastRun: { + status: "succeeded" | "failed" | "skipped" | null; + at: string | null; + } | null; +}; + +export const routineKeys = { + all: ["routines"] as const, + list: () => ["routines", "list"] as const, +}; + +/** + * The signed-in person's own routines. + * + * Owner-scoped by the server on every read; there is no version of this that takes an owner id, + * the same way `connectionsQueryOptions` answers only for whoever is asking. + */ +export function routinesQueryOptions() { + return queryOptions({ + queryKey: routineKeys.list(), + queryFn: (): Promise => + client("/api/routines", "routines", { + fallback: "Your routines could not be loaded.", + }), + }); +} diff --git a/app/src/routeTree.gen.ts b/app/src/routeTree.gen.ts index 058da25e..8d25b683 100644 --- a/app/src/routeTree.gen.ts +++ b/app/src/routeTree.gen.ts @@ -17,6 +17,7 @@ import { Route as AuthedSettingsRouteRouteImport } from './routes/_authed/settin import { Route as AuthedAppIndexRouteImport } from './routes/_authed/_app/index' import { Route as AuthedAppAttentionRouteImport } from './routes/_authed/_app/attention' import { Route as AuthedAppBotRouteImport } from './routes/_authed/_app/bot' +import { Route as AuthedAppRoutinesRouteImport } from './routes/_authed/_app/routines' import { Route as AuthedAppSkillsRouteImport } from './routes/_authed/_app/skills' import { Route as AuthedAdminIndexRouteImport } from './routes/_authed/admin/index' import { Route as AuthedAdminAuditRouteImport } from './routes/_authed/admin/audit' @@ -79,6 +80,11 @@ const AuthedAppBotRoute = AuthedAppBotRouteImport.update({ path: '/bot', getParentRoute: () => AuthedAppRoute, } as any) +const AuthedAppRoutinesRoute = AuthedAppRoutinesRouteImport.update({ + id: '/routines', + path: '/routines', + getParentRoute: () => AuthedAppRoute, +} as any) const AuthedAppSkillsRoute = AuthedAppSkillsRouteImport.update({ id: '/skills', path: '/skills', @@ -211,6 +217,7 @@ export interface FileRoutesByFullPath { '/settings': typeof AuthedSettingsRouteRouteWithChildren '/attention': typeof AuthedAppAttentionRoute '/bot': typeof AuthedAppBotRoute + '/routines': typeof AuthedAppRoutinesRoute '/skills': typeof AuthedAppSkillsRoute '/admin/audit': typeof AuthedAdminAuditRoute '/admin/boundaries': typeof AuthedAdminBoundariesRoute @@ -240,6 +247,7 @@ export interface FileRoutesByTo { '/sign': typeof SignRoute '/attention': typeof AuthedAppAttentionRoute '/bot': typeof AuthedAppBotRoute + '/routines': typeof AuthedAppRoutinesRoute '/skills': typeof AuthedAppSkillsRoute '/admin/audit': typeof AuthedAdminAuditRoute '/admin/boundaries': typeof AuthedAdminBoundariesRoute @@ -273,6 +281,7 @@ export interface FileRoutesById { '/_authed/_app': typeof AuthedAppRouteWithChildren '/_authed/_app/attention': typeof AuthedAppAttentionRoute '/_authed/_app/bot': typeof AuthedAppBotRoute + '/_authed/_app/routines': typeof AuthedAppRoutinesRoute '/_authed/_app/skills': typeof AuthedAppSkillsRoute '/_authed/admin/audit': typeof AuthedAdminAuditRoute '/_authed/admin/boundaries': typeof AuthedAdminBoundariesRoute @@ -307,6 +316,7 @@ export interface FileRouteTypes { | '/settings' | '/attention' | '/bot' + | '/routines' | '/skills' | '/admin/audit' | '/admin/boundaries' @@ -336,6 +346,7 @@ export interface FileRouteTypes { | '/sign' | '/attention' | '/bot' + | '/routines' | '/skills' | '/admin/audit' | '/admin/boundaries' @@ -368,6 +379,7 @@ export interface FileRouteTypes { | '/_authed/_app' | '/_authed/_app/attention' | '/_authed/_app/bot' + | '/_authed/_app/routines' | '/_authed/_app/skills' | '/_authed/admin/audit' | '/_authed/admin/boundaries' @@ -457,6 +469,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthedAppBotRouteImport parentRoute: typeof AuthedAppRoute } + '/_authed/_app/routines': { + id: '/_authed/_app/routines' + path: '/routines' + fullPath: '/routines' + preLoaderRoute: typeof AuthedAppRoutinesRouteImport + parentRoute: typeof AuthedAppRoute + } '/_authed/_app/skills': { id: '/_authed/_app/skills' path: '/skills' @@ -684,6 +703,7 @@ const AuthedSettingsRouteRouteWithChildren = interface AuthedAppRouteChildren { AuthedAppAttentionRoute: typeof AuthedAppAttentionRoute AuthedAppBotRoute: typeof AuthedAppBotRoute + AuthedAppRoutinesRoute: typeof AuthedAppRoutinesRoute AuthedAppSkillsRoute: typeof AuthedAppSkillsRoute AuthedAppIndexRoute: typeof AuthedAppIndexRoute AuthedAppChannelChannelIdRoute: typeof AuthedAppChannelChannelIdRoute @@ -694,6 +714,7 @@ interface AuthedAppRouteChildren { const AuthedAppRouteChildren: AuthedAppRouteChildren = { AuthedAppAttentionRoute: AuthedAppAttentionRoute, AuthedAppBotRoute: AuthedAppBotRoute, + AuthedAppRoutinesRoute: AuthedAppRoutinesRoute, AuthedAppSkillsRoute: AuthedAppSkillsRoute, AuthedAppIndexRoute: AuthedAppIndexRoute, AuthedAppChannelChannelIdRoute: AuthedAppChannelChannelIdRoute, diff --git a/app/src/routes/_authed/_app/routines.tsx b/app/src/routes/_authed/_app/routines.tsx new file mode 100644 index 00000000..ba448ed6 --- /dev/null +++ b/app/src/routes/_authed/_app/routines.tsx @@ -0,0 +1,30 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { PageShell } from "@/components/layout/page-shell"; +import { RoutinesList } from "@/components/routines/routines-list"; + +/** + * A person's own standing instructions: what runs on a schedule, and a switch to stop one. + * + * `_authed/_app`, not Admin: a routine is something anybody has, the same way a skill is — it is + * scoped to the signed-in person on every read and write, not to the deployment. + * + * THERE IS DELIBERATELY NO CREATE AND NO EDIT FORM ON THIS PAGE. Turning a sentence into a cron + * expression and a channel is conversational work — ask a Bot in a channel, "every weekday at 9, + * post the standup notes here" — and that is exactly what a conversation is for. This screen answers + * a narrower question: what is standing right now, and does it stay standing. It shows and it stops; + * it does not compose. Absent on purpose, not an omission. + */ +export const Route = createFileRoute("/_authed/_app/routines")({ + component: RoutinesPage, +}); + +function RoutinesPage() { + return ( + + + + ); +} diff --git a/app/src/routes/_authed/admin/audit.tsx b/app/src/routes/_authed/admin/audit.tsx index 801c8629..403294ad 100644 --- a/app/src/routes/_authed/admin/audit.tsx +++ b/app/src/routes/_authed/admin/audit.tsx @@ -42,9 +42,14 @@ const FILTERS = [ * judged: a caller could not prove which Bot it was. Somebody filtering for what this deployment * turned away wants that in the list, and it is the one refusal with no policy behind it, so * leaving it out would hide the only evidence that anything was attempted. + * + * `routines.dispatch_refused` is the same shape one boundary over: the worker, not a Bot, and a + * stale or missing secret rather than a policy decision. The same reasoning that put + * `mcp.callback_refused` here applies unchanged — nobody was judged, something was still turned + * away, and the saved view a person clicks for "what did this deployment block" should show it. */ search: - "?eventType=computer.action_refused,mcp.call_rejected,mcp.callback_refused,component.refused,component.function_refused", + "?eventType=computer.action_refused,mcp.call_rejected,mcp.callback_refused,component.refused,component.function_refused,routines.dispatch_refused", }, { label: "Did not happen", @@ -153,7 +158,9 @@ function Row({ * that way here: the fallback below calls anything it does not recognise "Allowed", which for a * refusal is the one wrong answer. A trail that is confidently wrong is worse than a silent one. */ - event.eventType === "mcp.callback_refused"; + event.eventType === "mcp.callback_refused" || + // The worker turned away at the door, same reasoning as the caller above. + event.eventType === "routines.dispatch_refused"; const stalled = event.eventType === "agent.stream_stalled"; /* * Three different things, and the difference is what somebody comes to this row to find out. From feec808d802a57fc8503d5df8f7d5971eb1d2b8c Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 14:11:05 -0300 Subject: [PATCH 28/45] Say when the next run is, and type what the last one was --- app/src/components/routines/routines-list.tsx | 22 ++++++++++++-- server/src/routines/routes.ts | 3 +- server/tests/routine-routes.test.ts | 29 +++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/app/src/components/routines/routines-list.tsx b/app/src/components/routines/routines-list.tsx index eec158bc..97ff20e2 100644 --- a/app/src/components/routines/routines-list.tsx +++ b/app/src/components/routines/routines-list.tsx @@ -1,5 +1,5 @@ import { IconTrash } from "@tabler/icons-react"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useMutation, useQuery } from "@tanstack/react-query"; import { useState } from "react"; import { PageEmpty, @@ -33,6 +33,7 @@ import { type RoutineRecord, routinesQueryOptions, } from "@/lib/routines/queries"; +import { queryClient } from "@/query-client"; const RELATIVE_UNITS = [ { limit: 60_000, divisor: 1_000, unit: "second" }, @@ -86,7 +87,13 @@ function lastRunLabel(lastRun: RoutineRecord["lastRun"]): { className: "text-amber-600 dark:text-amber-500", }; } - return { text: `Ran ${when}`, className: "text-muted-foreground" }; + if (lastRun.status === "succeeded") { + return { text: `Ran ${when}`, className: "text-muted-foreground" }; + } + // An outcome this DTO doesn't recognise degrades to a neutral label rather than an invented + // success — the contract typing (`RoutineRunOutcome | null`) makes a fourth outcome a build-time + // error, but this is the runtime fallback if that ever slips through. + return { text: `Finished ${when}`, className: "text-muted-foreground" }; } /** @@ -94,7 +101,6 @@ function lastRunLabel(lastRun: RoutineRecord["lastRun"]): { * switch to stop one taking effect, and a delete that ends it for good. */ export function RoutinesList() { - const queryClient = useQueryClient(); const routines = useQuery(routinesQueryOptions()); const setEnabled = useMutation(setRoutineEnabledMutationOptions(queryClient)); const deleteRoutine = useMutation(deleteRoutineMutationOptions(queryClient)); @@ -154,6 +160,16 @@ export function RoutinesList() { : (routine.channel.name ?? "Unnamed channel")} {lastRunText} + {/* + * Enabled only: the store recomputes nextRunAt on cron/timezone change or + * re-enable, so a disabled routine's stamp is frozen in the past — rendering + * it unguarded would announce a stale "3 days ago" as the next run. + */} + {routine.enabled ? ( + + Next {relativeTime(routine.nextRunAt)} + + ) : null} diff --git a/server/src/routines/routes.ts b/server/src/routines/routes.ts index 0be1d021..f723c43d 100644 --- a/server/src/routines/routes.ts +++ b/server/src/routines/routes.ts @@ -4,6 +4,7 @@ import type { AppVariables } from "../auth/guards"; import { RoutineNotFoundError, RoutineRefusedError, + type RoutineRunOutcome, type RoutineStore, type RoutineSummary, } from "./store"; @@ -88,7 +89,7 @@ type RoutineDto = { channel: { id: string; name: string | null; gone: boolean }; enabled: boolean; nextRunAt: string; - lastRun: { status: string | null; at: string | null } | null; + lastRun: { status: RoutineRunOutcome | null; at: string | null } | null; }; function routineDto(routine: RoutineSummary): RoutineDto { diff --git a/server/tests/routine-routes.test.ts b/server/tests/routine-routes.test.ts index f8211e39..c069400f 100644 --- a/server/tests/routine-routes.test.ts +++ b/server/tests/routine-routes.test.ts @@ -139,6 +139,35 @@ describe("GET /", () => { expect((await json(response)).routines[0].lastRun).toBeNull(); }); + test("an open run stays an object with a null status, never collapsed to null", async () => { + const store = fakeStore({ + listFor: async () => [ + summary({ lastRun: { status: null, finishedAt: null } }), + ], + }); + const response = await appFor(store).request("http://openbot.test/"); + + expect((await json(response)).routines[0].lastRun).toEqual({ + status: null, + at: null, + }); + }); + + test("a channel with no name and gone reads as gone with a null name", async () => { + const store = fakeStore({ + listFor: async () => [ + summary({ channelName: null, channelDeleted: true }), + ], + }); + const response = await appFor(store).request("http://openbot.test/"); + + expect((await json(response)).routines[0].channel).toEqual({ + id: "channel-1", + name: null, + gone: true, + }); + }); + test("the DTO carries the schedule as words and never a cron field", async () => { const store = fakeStore(); const response = await appFor(store).request("http://openbot.test/"); From de96a9fd0ee9e3cfdef2b10949a3164327d58baa Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 14:29:35 -0300 Subject: [PATCH 29/45] Say what a routine is, who it runs as, and what it will not do --- CHANGELOG.md | 21 ++++ README.md | 1 + app/src/routes/_authed/admin/plugins/$key.tsx | 10 +- charts/openbot/README.md | 11 +- docs/README.md | 1 + docs/architecture.md | 12 ++ docs/configuration.md | 14 +++ docs/deployment.md | 7 ++ docs/routines.md | 115 ++++++++++++++++++ 9 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 docs/routines.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 06123b91..1edeebe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,26 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### A Bot can be asked to do something on a schedule + +"Every weekday at nine, post the standup notes here" is now something a Bot can be asked rather than +something somebody has to remember. A routine created this way runs under its own creator's grants — +it can do exactly what they could do in chat, and nothing more — and its reply lands in the channel as +an ordinary Bot message: it lights the unread dot the same way any other message does, and it appears +in the transcript rather than anywhere separate. A routine that fails posts one message about its +first failure and, after ten in a row, switches itself off with a final one rather than failing +forever unnoticed. + +The deployment gains two tables, via migration `0021`. + +**This needs a new process.** A worker fires due routines by calling this deployment's own API server, +and a deployment that never starts one schedules nothing — the routine sits on the Routines page with +a next run time like any other, and nothing on the screen says a worker is missing. `WORKER_SHARED_SECRET` +is the credential the worker presents; a deployment without it configured refuses every handoff rather +than accepting one it cannot attribute. `scripts/start.sh` runs the worker locally; the Helm chart +turns it on with `routines.enabled` and takes the secret as `secrets.workerSharedSecret`. No new port +is opened for any of this — the worker only ever calls out to the server it already trusts. + ### A Bot in trouble no longer needs somebody watching A boundary refusal or a stalled run was recorded and then waited for a person to happen to look — at @@ -23,6 +43,7 @@ It is a view over the trail, not a second record of it. Refusals and stalls are transactionally by the gateway and the stall guard, so the inbox cannot miss one and nothing new runs on the action path. The only state it owns is the resolution, held beside the append-only trail rather than in it. The trail itself still keeps everything; the inbox is only what is open now. + ### A channel a Bot has spoken in unseen shows a dot The sidebar marks a channel when a Bot has said something since you last had it open: a dot beside diff --git a/README.md b/README.md index f71a6a5f..61285e37 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ Leave `EMBEDDED_POSTGRES` off and set `DATABASE_URL` to point at a database you - **Credentials encrypted at rest**: stored through `/admin/credentials`, never returned by an API, and redacted from audit events. - **Loopback by default**: computers bind to `127.0.0.1` and require a per-container token, so nothing reaches a logged-in browser by knowing its port. The supervisor binds there too, because it holds the Docker socket and its token is a shared secret rather than a network boundary. - **Durable threads and memory**: conversations survive restarts through CopilotKit Intelligence, and each deployment stamps the threads it owns. +- **Routines**: ask a Bot to do something on a schedule and it does, running as you, in the channel you asked in. A 15-minute floor and a 20-routine cap keep a sentence from scheduling more than a person meant, and ten failures in a row switch a routine off rather than burn model spend forever. Needs a worker process; see [docs/routines.md](docs/routines.md). ## Bring your own agent diff --git a/app/src/routes/_authed/admin/plugins/$key.tsx b/app/src/routes/_authed/admin/plugins/$key.tsx index 5740bd78..2c37ba95 100644 --- a/app/src/routes/_authed/admin/plugins/$key.tsx +++ b/app/src/routes/_authed/admin/plugins/$key.tsx @@ -534,9 +534,15 @@ function RouteComponent() { size="sm" > - Vendor documentation + + {auth === "builtin" + ? "Documentation" + : "Vendor documentation"} + - What this server offers, from the people who maintain it. + {auth === "builtin" + ? "What these tools offer, from the people who maintain them." + : "What this server offers, from the people who maintain it."} diff --git a/charts/openbot/README.md b/charts/openbot/README.md index b43a71c3..b3e02e9d 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -124,7 +124,9 @@ installing on somebody's bare-metal cluster. The chart fails the install, naming the value to change, when: there is no database or two of them; nobody would be an administrator; `singleUser` is combined with a public URL; both an Ingress and an HTTPRoute are enabled; both `externalSecrets` and an existing Secret are named; a Bot endpoint is -named with no token to call it with; or a browser is asked for inside more than one API replica. +named with no token to call it with; a browser is asked for inside more than one API replica; or +`routines.enabled` is set with no `secrets.workerSharedSecret` — and, on `externalSecrets`, no +`worker-shared-secret` key named for it to read instead. ## Your own Bot @@ -197,6 +199,13 @@ PostgreSQL with `select ... for update skip locked`, so whichever pod runs the s nobody else holds, and one that dies mid-suspend hands its work back when the lease expires. The decision is re-checked at the moment of acting, because somebody may have come back in between. +A second CronJob shares that same mechanism for a different job: `routines.enabled` turns on the +sweep that fires standing instructions a Bot was asked to carry out on a schedule, on +`routines.schedule`. It needs `secrets.workerSharedSecret` — the credential it presents to the API +server to be recognised as the worker rather than an arbitrary caller — and is off by default because +turning it on with no secret set is a CronJob whose every run is refused. See the routines refusal +below, and [docs/routines.md](../../docs/routines.md). + ## NetworkPolicy, and whether your cluster enforces one Off by default, because a NetworkPolicy on a cluster whose CNI does not enforce one is a resource diff --git a/docs/README.md b/docs/README.md index e850f9ee..9ef2cf6e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ Start with the root [README](../README.md), then use these references: - [Configuration](configuration.md): environment variables and tenant package YAML. - [Development](development.md): local setup, migrations, ports, and quality checks. - [Coworkers](coworkers.md): durable Bot profiles, channels, visibility, deletion, and external AG-UI registration. +- [Routines](routines.md): standing instructions a Bot runs on a schedule, the worker that fires them, and who they run as. - Plugins, one connector per page — what an administrator registers, what each person consents to, and what the failures mean: - [Google Drive](plugins/google-drive.md) - [Notion](plugins/notion.md) diff --git a/docs/architecture.md b/docs/architecture.md index 46072940..5a2c9403 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -113,6 +113,18 @@ on group membership from the identity provider, not as a control that is running See [coworkers.md](coworkers.md). +## Routines + +A routine is a standing instruction, created by asking a Bot in a channel rather than through a form, +that fires on a schedule and posts its reply into that channel as the person who created it. + +The sweep that notices a routine is due sits beside the computer culler on one shared mechanism: both +write to `work_items`, one PostgreSQL table claimed with `select ... for update skip locked`, leases +timed on the database's own clock, and an attempt cap. Neither runs as a timer inside the API, because +a timer fires in every replica and each would decide independently that the same firing or the same +suspension is due; the queue is what lets exactly one claim it while every other replica's attempt +collides harmlessly with the same row. See [routines.md](routines.md). + ## Components Components are frontend tools a Bot can call instead of answering only in prose. diff --git a/docs/configuration.md b/docs/configuration.md index a8291d3c..618061f7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -55,6 +55,7 @@ at `agent-langgraph` on a laptop. | `AGENT_TOOL_TOKEN` | unset; `start.sh` generates one | The secret a framework Bot presents when it calls a granted tool back through this server. | | `APP_DIST_DIR` | unset | Where the built app is, when this process serves it. Set inside the container image; unset in development, where Vite serves the app. | | `AUDIT_RETENTION_DAYS` | unset | Whole number of days to keep audit rows; older ones are removed. Unset keeps the trail forever. | +| `WORKER_SHARED_SECRET` | unset; `start.sh` generates one | The secret the routines worker presents to fire a due routine. Without it the server refuses every handoff, whether or not a worker exists to send one. | **`AGENT_STALL_TIMEOUT_MS`** watches for the failure a Bot has that nothing else in the trail can show: a stream that stops producing anything. Every other audit row is something that happened, and @@ -78,6 +79,19 @@ itself to a Bot, this is a Bot proving itself to the server. Rotating either mea holding the old one refuses every call, which is why `start.sh` restarts the server and recreates the Bot containers on a run that mints one. +**`WORKER_SHARED_SECRET`** is the same shape of secret for a different pair: it is what the routines +worker presents to `/internal/routines/run` to prove a routine's dispatch actually came from it. The +API server refuses a handoff without one configured, and the worker refuses to start without one at +all. See [routines.md](routines.md) for what a deployment with no worker at all looks like — it is +not obvious from the screen. `start.sh` generates one for a laptop, the same way it does for +`AGENT_TOOL_TOKEN`. + +**`SERVER_INTERNAL_URL`** is read by the worker, not by the API server, so it is not in the table +above: it says where the worker's own process can reach this deployment's API, which is a fact about +where the worker runs rather than a fact about the deployment `loadConfig` describes. `start.sh` points +it at the server's own port on a laptop; the Helm chart's routines CronJob points it at the server's +in-cluster Service address. + ## OpenAI-compatible endpoints `OPENAI_BASE_URL` decides where an OpenAI-shaped request is answered. Unset, that is OpenAI. Set, it is any endpoint speaking the same API: a gateway in front of several providers, a proxy, or a model on hardware you control. diff --git a/docs/deployment.md b/docs/deployment.md index 14164094..84a699b1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -38,6 +38,13 @@ do on a laptop with no supervisor configured. A shared browser means shared logi shared session between Bots, which is fine for a deployment where one team trusts its own Bots and is not fine as a boundary between tenants. +**The routines worker.** `worker/` is not in this image, so nothing here fires a due routine — and +nothing warns you: a routine created against this deployment is stored, its next run time is +computed, and the Routines page shows it sitting there exactly as it would with a worker watching it. +It simply never fires. Firing routines needs a second process, built from the repository rather than +from this image, pointed at the same `DATABASE_URL` and at this deployment's own reachable address +with `WORKER_SHARED_SECRET` and `SERVER_INTERNAL_URL` set. See [routines.md](routines.md). + ## Minimum size Measured on the real image, one Bot, arm64. diff --git a/docs/routines.md b/docs/routines.md new file mode 100644 index 00000000..6089e74a --- /dev/null +++ b/docs/routines.md @@ -0,0 +1,115 @@ +# Routines + +A routine is a standing instruction: something a Bot carries out on a schedule instead of waiting to +be asked. "Every weekday at nine, summarize what changed in this channel overnight" is a routine, not +a message — it fires on its own, for as long as it stays switched on, and its reply lands in a channel +the same way any other message from that Bot does. + +## Creating one + +There is no form for this. Ask a Bot, in a channel: "every weekday at 9, post the standup notes +here." Turning a sentence into a five-field cron expression and a channel is conversational work, and +that is what the conversation is for. The same Bot can list what is standing, change one, or delete +one, all by being asked. + +**The prerequisite:** a Bot can only do this once an administrator has granted it to. Routines is a +catalogue entry like any other — `create_routine`, `update_routine` and `delete_routine` are its write +tools — and enabling the entry does not hand any Bot access to it. Each tool is granted per Bot at +`/admin/plugins/routines`, exactly as a Google Drive or Notion tool would be. An administrator decides +which Bots may schedule future work at all, before deciding what that work is; a Bot with none of the +three tools can still be asked and will say it cannot. + +The routine belongs to whoever asked for it and runs as them. See +[Who a routine runs as](#who-a-routine-runs-as). + +## The 15-minute floor and the 20-enabled cap + +A routine may fire at most every 15 minutes. The floor exists because a model can be talked into +anything a sentence can describe, including "every minute", and the floor is what a sentence cannot +talk its way past. + +A person may have at most 20 routines switched on at once. The cap exists for the same reason as the +floor: a conversation is an easy place to accumulate standing work without noticing, and 20 is where a +person's own list stops being something they can hold in their head. Switching one off frees a slot; +deleting one is not required. + +## The fatigue rule + +A routine that fails posts exactly one message about it — the first failure after a success, not +every failure. Ten consecutive failures switch the routine off and post a second, final message +saying so; nothing further fires until a person turns it back on. + +This is deliberately not a retry policy. A retry policy answers "did this one attempt make it through +a dispatch that failed for a moment" — a busy queue, a server that hiccuped — and that question is +already answered by the shared work queue's own attempt count, quietly, before a routine's turn ever +runs. The fatigue rule answers a different question: is this routine worth firing at all. A Notion +token that expired in March fails cleanly, once, every single night, and no number of retries of any +one night's attempt will fix that — only switching it off, and saying so, does. + +## Missed windows are skipped, not replayed + +A routine's next run is a stamp, not a queue. If nothing was watching the clock — a worker that was +never started, or one that was down for a month — a routine's stamp falls behind, and the deployment +does not owe it every occurrence it missed: catching up is a silent drain, not a burst. A deployment +whose worker comes back after a quiet month drains that backlog by advancing the stamp forward without +firing anything for it, occurrence by occurrence, until it is current again — not by firing thirty +stale summaries of thirty different mornings. + +A firing that is still recent enough to be worth having does still happen. A stopped server pod loses +at most the one occurrence that was in flight when it stopped; the next one fires on schedule, because +the clock had already moved on before that firing was attempted. + +## The worker requirement + +Nothing above happens without a second process. The API server answers `/internal/routines/run` when +it is handed a run, but nothing hands it one on its own — that is a separate worker's whole job, and a +deployment that never started one schedules nothing. + +This fails silently. A routine created in chat is stored, its schedule is computed, and the Routines +page shows it sitting there with a next run time like any other — because as far as that page knows, +it is correct. Nothing on the screen says a worker exists to act on it, so a deployment with no worker +looks identical to one running normally, right up until nobody's standup notes ever arrive. + +Two settings carry this: + +- **`WORKER_SHARED_SECRET`** — the credential the worker presents to `/internal/routines/run`. The API + server refuses every handoff without it configured on both sides, and the worker refuses to start + without it at all, rather than firing routines nobody could ever prove came from it. +- **`SERVER_INTERNAL_URL`** — where the worker reaches this deployment's own API server. It is a fact + about where the worker process runs rather than a fact about the deployment, so it is read from the + environment directly rather than from the rest of the deployment's configuration. + +On Kubernetes, `routines.enabled` turns on a CronJob running `fire-routines.ts` on `routines.schedule`, +the same way the computer culler is a CronJob rather than a timer inside the API — a timer fires in +every replica, and a CronJob's single run does the whole sweep once. On a laptop, `scripts/start.sh` +starts a worker process that runs that same sweep in a loop instead of once and exiting, so the two +shapes are the same code doing the same thing on two different clocks, not two implementations to keep +in sync. + +## Who a routine runs as + +A routine runs as the person who created it, not as the Bot and not as an administrator. Its turn is +built with that person's own grants, so it can do in the middle of the night exactly what they could +do by typing the same instruction in chat themselves, and nothing more — a routine cannot reach a +connector its creator never connected, or post into a channel they are not in. Its reply is posted +into the channel as an ordinary message from that Bot: it lights the recipients' unread dot the same +way any other Bot message does, and it appears in the conversation transcript rather than anywhere +separate, because as far as the channel is concerned, that is exactly what it is. + +## Scope + +This ships the core: creating, listing, changing and deleting routines from chat; the schedule, the +cap and the fatigue rule; the worker that fires them. Four follow-ups are tracked in +[#193](https://github.com/CopilotKit/OpenBot/issues/193) and deliberately not in this pass: audit rows +are not yet marked as unattended, so telling a routine's action apart from the same person's own is a +manual correlation against `routine_runs` timestamps rather than a flag; there is no admin view of +other people's routines, only the owner-scoped page each person sees for their own; there is no +per-deployment or per-Bot cap on how many routines may be running at once beyond the sweep's own claim +limit; and a tenant package cannot yet ship routines the way it ships agents, channels or skills. + +## See also + +- [Architecture](architecture.md) — where the routines sweep sits beside the computer culler on the + shared work queue. +- [Coworkers](coworkers.md) — durable Bot profiles and channels, which a routine posts into. +- [Configuration](configuration.md) — `WORKER_SHARED_SECRET`, `SERVER_INTERNAL_URL`. From 70302e9f6933340b06cdd8565fa2b84f80f6e34f Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 14:39:08 -0300 Subject: [PATCH 30/45] Tell the truth about the secret and the single container --- README.md | 2 +- charts/openbot/README.md | 5 ++++- docs/configuration.md | 13 ++++++++++--- docs/deployment.md | 14 ++++++++------ docs/routines.md | 8 +++++--- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 61285e37..9bb0801c 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Leave `EMBEDDED_POSTGRES` off and set `DATABASE_URL` to point at a database you - **Credentials encrypted at rest**: stored through `/admin/credentials`, never returned by an API, and redacted from audit events. - **Loopback by default**: computers bind to `127.0.0.1` and require a per-container token, so nothing reaches a logged-in browser by knowing its port. The supervisor binds there too, because it holds the Docker socket and its token is a shared secret rather than a network boundary. - **Durable threads and memory**: conversations survive restarts through CopilotKit Intelligence, and each deployment stamps the threads it owns. -- **Routines**: ask a Bot to do something on a schedule and it does, running as you, in the channel you asked in. A 15-minute floor and a 20-routine cap keep a sentence from scheduling more than a person meant, and ten failures in a row switch a routine off rather than burn model spend forever. Needs a worker process; see [docs/routines.md](docs/routines.md). +- **Routines**: ask a Bot to do something on a schedule and it does, running as you, in the channel you asked in. A 15-minute floor and a cap of 20 enabled routines keep a sentence from scheduling more than a person meant, and ten failures in a row switch a routine off rather than burn model spend forever. Needs a worker process; see [docs/routines.md](docs/routines.md). ## Bring your own agent diff --git a/charts/openbot/README.md b/charts/openbot/README.md index b3e02e9d..7e98e33d 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -126,7 +126,10 @@ nobody would be an administrator; `singleUser` is combined with a public URL; bo HTTPRoute are enabled; both `externalSecrets` and an existing Secret are named; a Bot endpoint is named with no token to call it with; a browser is asked for inside more than one API replica; or `routines.enabled` is set with no `secrets.workerSharedSecret` — and, on `externalSecrets`, no -`worker-shared-secret` key named for it to read instead. +`worker-shared-secret` key named for it to read instead. One combination gets no refusal at all: +`secrets.existingSecret` with `routines.enabled`, because the Secret this chart would otherwise +validate is somebody else's to create — put `worker-shared-secret` in it yourself, or the CronJob's +every run is refused with nothing at install time to say so. ## Your own Bot diff --git a/docs/configuration.md b/docs/configuration.md index 618061f7..812d031f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -55,7 +55,7 @@ at `agent-langgraph` on a laptop. | `AGENT_TOOL_TOKEN` | unset; `start.sh` generates one | The secret a framework Bot presents when it calls a granted tool back through this server. | | `APP_DIST_DIR` | unset | Where the built app is, when this process serves it. Set inside the container image; unset in development, where Vite serves the app. | | `AUDIT_RETENTION_DAYS` | unset | Whole number of days to keep audit rows; older ones are removed. Unset keeps the trail forever. | -| `WORKER_SHARED_SECRET` | unset; `start.sh` generates one | The secret the routines worker presents to fire a due routine. Without it the server refuses every handoff, whether or not a worker exists to send one. | +| `WORKER_SHARED_SECRET` | unset; `start.sh` uses a fixed local default | The secret the routines worker presents to fire a due routine. Without it the server refuses every handoff, whether or not a worker exists to send one. | **`AGENT_STALL_TIMEOUT_MS`** watches for the failure a Bot has that nothing else in the trail can show: a stream that stops producing anything. Every other audit row is something that happened, and @@ -83,8 +83,15 @@ Bot containers on a run that mints one. worker presents to `/internal/routines/run` to prove a routine's dispatch actually came from it. The API server refuses a handoff without one configured, and the worker refuses to start without one at all. See [routines.md](routines.md) for what a deployment with no worker at all looks like — it is -not obvious from the screen. `start.sh` generates one for a laptop, the same way it does for -`AGENT_TOOL_TOKEN`. +not obvious from the screen. + +Unlike `AGENT_TOOL_TOKEN`, `start.sh` does not generate and persist this one. It supplies a fixed +local default, `openbot-dev-worker-secret`, the same value every clone of this repository gets. That +is fine here because this secret is only ever compared on this machine's own loopback-bound port, +never by anything a Bot publishes — a well-known value from a public repository is not a boundary +anybody outside this machine could reach anyway. `AGENT_TOOL_TOKEN` is generated fresh and written to +`.env` precisely because it is not that: it is presented by a Bot's own published port, so a fixed +default there would be no boundary at all. **`SERVER_INTERNAL_URL`** is read by the worker, not by the API server, so it is not in the table above: it says where the worker's own process can reach this deployment's API, which is a fact about diff --git a/docs/deployment.md b/docs/deployment.md index 84a699b1..da5b23ee 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -38,12 +38,14 @@ do on a laptop with no supervisor configured. A shared browser means shared logi shared session between Bots, which is fine for a deployment where one team trusts its own Bots and is not fine as a boundary between tenants. -**The routines worker.** `worker/` is not in this image, so nothing here fires a due routine — and -nothing warns you: a routine created against this deployment is stored, its next run time is -computed, and the Routines page shows it sitting there exactly as it would with a worker watching it. -It simply never fires. Firing routines needs a second process, built from the repository rather than -from this image, pointed at the same `DATABASE_URL` and at this deployment's own reachable address -with `WORKER_SHARED_SECRET` and `SERVER_INTERNAL_URL` set. See [routines.md](routines.md). +**The routines schedule.** Nothing in this image is scheduled to fire a routine — there is no +worker service beside the API, and `worker/` (the looping local variant) is not in the image. The +sweep itself is: `bun scripts/fire-routines.ts` from `/app/server`, one pass then exit, which is what +the Helm chart's CronJob runs from this same image. So a one-container deployment needs something +outside the container to run it on a schedule — an external cron, a platform scheduled job, or a +second container of this image with that command — with `DATABASE_URL`, `SERVER_INTERNAL_URL` and +`WORKER_SHARED_SECRET` set. Until something does, a routine is stored, its next run time is computed, +the Routines page shows it, and it never fires. See [routines.md](routines.md). ## Minimum size diff --git a/docs/routines.md b/docs/routines.md index 6089e74a..562d215b 100644 --- a/docs/routines.md +++ b/docs/routines.md @@ -55,9 +55,11 @@ whose worker comes back after a quiet month drains that backlog by advancing the firing anything for it, occurrence by occurrence, until it is current again — not by firing thirty stale summaries of thirty different mornings. -A firing that is still recent enough to be worth having does still happen. A stopped server pod loses -at most the one occurrence that was in flight when it stopped; the next one fires on schedule, because -the clock had already moved on before that firing was attempted. +A firing that is still recent enough to be worth having does still happen. A server pod that restarts +loses at most the one occurrence that was in flight when it stopped; the next one fires on schedule, +because the clock had already moved on before that firing was attempted. A server that stays down +loses more than that: every occurrence whose stamp ages past the grace window while nothing is +running to offer it is skipped, not just the one that was in flight. ## The worker requirement From 2f38233d9082d5b8576b52edcc3c2c403ed35ffe Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 14:41:46 -0300 Subject: [PATCH 31/45] Name the pods that fail when the key is missing --- charts/openbot/README.md | 5 +++-- docs/configuration.md | 5 +++-- docs/routines.md | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/charts/openbot/README.md b/charts/openbot/README.md index 7e98e33d..9b8f99aa 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -128,8 +128,9 @@ named with no token to call it with; a browser is asked for inside more than one `routines.enabled` is set with no `secrets.workerSharedSecret` — and, on `externalSecrets`, no `worker-shared-secret` key named for it to read instead. One combination gets no refusal at all: `secrets.existingSecret` with `routines.enabled`, because the Secret this chart would otherwise -validate is somebody else's to create — put `worker-shared-secret` in it yourself, or the CronJob's -every run is refused with nothing at install time to say so. +validate is somebody else's to create — put `worker-shared-secret` in it yourself, or every pod that +mounts it fails to start — the routines CronJob, the culler, and the API server itself — with nothing +at install time to say so. ## Your own Bot diff --git a/docs/configuration.md b/docs/configuration.md index 812d031f..56405edd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -90,8 +90,9 @@ local default, `openbot-dev-worker-secret`, the same value every clone of this r is fine here because this secret is only ever compared on this machine's own loopback-bound port, never by anything a Bot publishes — a well-known value from a public repository is not a boundary anybody outside this machine could reach anyway. `AGENT_TOOL_TOKEN` is generated fresh and written to -`.env` precisely because it is not that: it is presented by a Bot's own published port, so a fixed -default there would be no boundary at all. +`.env` precisely because it is not that: it is copied into every Bot container, and a framework Bot +holding it may be running on a machine of its own, so a fixed default there would be no boundary at +all. **`SERVER_INTERNAL_URL`** is read by the worker, not by the API server, so it is not in the table above: it says where the worker's own process can reach this deployment's API, which is a fact about diff --git a/docs/routines.md b/docs/routines.md index 562d215b..2c5cd2d5 100644 --- a/docs/routines.md +++ b/docs/routines.md @@ -58,8 +58,8 @@ stale summaries of thirty different mornings. A firing that is still recent enough to be worth having does still happen. A server pod that restarts loses at most the one occurrence that was in flight when it stopped; the next one fires on schedule, because the clock had already moved on before that firing was attempted. A server that stays down -loses more than that: every occurrence whose stamp ages past the grace window while nothing is -running to offer it is skipped, not just the one that was in flight. +loses more than that: every occurrence whose stamp ages past the grace window while nothing can +carry it out is skipped, not just the one that was in flight. ## The worker requirement From d31aa9b7ee1e16a98152012e15826c40ab5dc326 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 14:58:23 -0300 Subject: [PATCH 32/45] Bound every dispatch, and close the runs that never ran --- .../openbot/templates/routines/cronjob.yaml | 4 ++ docs/configuration.md | 15 +++-- scripts/start.sh | 13 ++-- server/scripts/fire-routines.ts | 3 + server/src/plugins/builtin-routines.ts | 2 +- server/src/routines/store.ts | 30 +++++++++ server/src/routines/sweep.ts | 15 ++++- .../tests/routine-sweep.integration.test.ts | 42 ++++++++++++ .../tests/routines-store.integration.test.ts | 66 +++++++++++++++++++ worker/src/index.ts | 5 +- 10 files changed, 180 insertions(+), 15 deletions(-) diff --git a/charts/openbot/templates/routines/cronjob.yaml b/charts/openbot/templates/routines/cronjob.yaml index 066ed660..386e9ea3 100644 --- a/charts/openbot/templates/routines/cronjob.yaml +++ b/charts/openbot/templates/routines/cronjob.yaml @@ -26,6 +26,10 @@ spec: jobTemplate: spec: backoffLimit: 1 + # Under the five-minute schedule period, and load-bearing with `concurrencyPolicy: Forbid` + # above: without it a wedged run is never killed, so Forbid suppresses every later sweep and + # routines stop firing for good, with only a Running job as evidence. + activeDeadlineSeconds: 240 template: metadata: labels: diff --git a/docs/configuration.md b/docs/configuration.md index 56405edd..baf69eef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,12 +87,15 @@ not obvious from the screen. Unlike `AGENT_TOOL_TOKEN`, `start.sh` does not generate and persist this one. It supplies a fixed local default, `openbot-dev-worker-secret`, the same value every clone of this repository gets. That -is fine here because this secret is only ever compared on this machine's own loopback-bound port, -never by anything a Bot publishes — a well-known value from a public repository is not a boundary -anybody outside this machine could reach anyway. `AGENT_TOOL_TOKEN` is generated fresh and written to -`.env` precisely because it is not that: it is copied into every Bot container, and a framework Bot -holding it may be running on a machine of its own, so a fixed default there would be no boundary at -all. +is fine here not because of where the server listens — it binds no hostname, so the port itself is +reachable like any other — but because this is a dev-only default on a machine's own dev stack, and +the endpoint it guards accepts nothing but an unguessable `routine_run_` id: the server +re-reads the routine, the owner and the channel from its own tables rather than trusting anything +else the caller says, so a well-known value from a public repository gates nothing sensitive here. +`AGENT_TOOL_TOKEN` is generated fresh and written to `.env` precisely because it is not that: it is +copied into every Bot container, and a framework Bot holding it may be running on a machine of its +own, so a fixed default there would be no boundary at all. Production deployments must set a real +`WORKER_SHARED_SECRET`. **`SERVER_INTERNAL_URL`** is read by the worker, not by the API server, so it is not in the table above: it says where the worker's own process can reach this deployment's API, which is a fact about diff --git a/scripts/start.sh b/scripts/start.sh index 15a74e4c..024bb818 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -35,11 +35,14 @@ ONE_COMPUTER_EACH="${OPENBOT_ONE_COMPUTER_EACH:-true}" export APP_PORT SERVER_PORT SUPERVISOR_TOKEN="$(setting SUPERVISOR_TOKEN openbot-dev-supervisor-token)" COMPUTER_TOKEN="$(setting COMPUTER_TOKEN openbot-dev-computer-token)" -# A fixed default is fine here, unlike `AGENT_TOOL_TOKEN` below: this secret is compared by the -# server on its own loopback-bound port, never by anything a Bot publishes, so a well-known value -# from a public repository is not a boundary anybody outside this machine could reach anyway. That is -# also why it is generated fresh and persisted for AGENT_TOOL_TOKEN (see the SECRETS_ROTATED block) -# but not for this one. +# A fixed default is fine here, unlike `AGENT_TOOL_TOKEN` below, but not because of where the server +# listens — it binds no hostname, so this port is reachable from the network like any other. It is +# fine because this is a dev-only default on a machine's own dev stack, and the endpoint it guards +# accepts nothing but an unguessable `routine_run_` id: the server re-reads the routine, the +# owner and the channel from its own tables rather than trusting anything else the caller says, so +# the fixed default gates nothing sensitive here. That is also why it is generated fresh and +# persisted for AGENT_TOOL_TOKEN (see the SECRETS_ROTATED block) but not for this one — production +# must set a real WORKER_SHARED_SECRET. WORKER_SHARED_SECRET="$(setting WORKER_SHARED_SECRET openbot-dev-worker-secret)" # The secret the server sends to a managed Bot, generated and written back on first run. diff --git a/server/scripts/fire-routines.ts b/server/scripts/fire-routines.ts index 08e52f33..cbfb0bb4 100644 --- a/server/scripts/fire-routines.ts +++ b/server/scripts/fire-routines.ts @@ -80,6 +80,9 @@ async function dispatch(routineRunId: string): Promise { "content-type": "application/json", }, body: JSON.stringify({ routineRunId }), + // Mirrors worker/src/index.ts's dispatch: a wedged server must not stall the sweep, even under + // the CronJob's own activeDeadlineSeconds. + signal: AbortSignal.timeout(30_000), }); if (response.status !== 202) { // The status is in the sentence, because it is the whole diagnosis: 401 is the secret, 404 is a diff --git a/server/src/plugins/builtin-routines.ts b/server/src/plugins/builtin-routines.ts index d16b7f2d..d660c455 100644 --- a/server/src/plugins/builtin-routines.ts +++ b/server/src/plugins/builtin-routines.ts @@ -239,7 +239,7 @@ function asResult(text: string): McpCallResult { return { text, isError: false, truncated: false }; } return { - text: `${text.slice(0, MAX_RESULT_CHARS)}\n\n[truncated: the answer was ${text.length} characters]`, + text: `${text.slice(0, MAX_RESULT_CHARS)}\n\n[truncated: the tool returned ${text.length} characters]`, isError: false, truncated: true, }; diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index bd7d3c93..3987ef53 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -203,6 +203,18 @@ export type RoutineStore = { status: RoutineRunOutcome, error?: string, ): Promise; + /** + * Close every open (`status is null`) run for one routine as "failed", with the same error on all + * of them. Returns how many rows it closed. + * + * The give-up branch's cleanup, not `finishRun`'s: `insertRun` runs before `dispatch` on every + * attempt, so a dispatch that throws leaves an open run row behind, and the queue's attempt cap + * eventually stops offering that item to anybody — nothing else ever closes those rows. `listFor` + * shows the newest one, so without this a routine that never ran once reads "running now" forever. + * Closing ALL of them, not just the newest, is the more truthful shape: every open row is a real + * dispatch attempt that went nowhere, not just the last one. + */ + failOpenRuns(routineId: string, error: string): Promise; /** How many failures the routine has at the tail, for the fatigue rule to read. */ consecutiveFailures(routineId: string): Promise; }; @@ -682,6 +694,24 @@ export function createRoutineStore(database: Database): RoutineStore { .where(and(eq(routineRuns.id, runId), isNull(routineRuns.status))); }, + async failOpenRuns(routineId, error) { + // One UPDATE, not a select-then-loop: every row this WHERE matches is a leaked attempt, and + // there is nothing to decide per row that `status is null` does not already decide. + const closed = await database + .update(routineRuns) + .set({ + status: "failed", + finishedAt: sql`now()`, + // Same code-point cap as `finishRun`, so a give-up reason cannot be cut mid-surrogate-pair. + error: Array.from(error).slice(0, MAX_RUN_ERROR).join(""), + }) + .where( + and(eq(routineRuns.routineId, routineId), isNull(routineRuns.status)), + ) + .returning({ id: routineRuns.id }); + return closed.length; + }, + async consecutiveFailures(routineId) { /* * Bounded, then counted here. The bound is the point: this is read on every failed firing, diff --git a/server/src/routines/sweep.ts b/server/src/routines/sweep.ts index b0f79e80..d43e2d48 100644 --- a/server/src/routines/sweep.ts +++ b/server/src/routines/sweep.ts @@ -363,7 +363,7 @@ export async function dispatchClaimedRoutines( */ console.warn( JSON.stringify({ - type: "routine-fire-finish-lost", + type: "routine-fire-redelivery-possible", routineId, runId, reason: @@ -412,6 +412,18 @@ export async function dispatchClaimedRoutines( * and the reason for anybody who queries the table; this is for whoever reads the logs. */ if (item.attempts >= maxAttempts) { + /* + * THE ROW THIS GIVE-UP LEAKED. Every attempt opened a run row before it dispatched + * (`insertRun` above), and a dispatch that throws never reaches `finishRun` — so an item at + * the cap is not just off the queue, it is one or more `routine_runs` rows stuck open with no + * status. `listFor` shows the newest one, so without this the routines page reads "running + * now" for a routine that never ran at all, forever. Closed before the warning so the row is + * never left open even if the log line itself fails. + */ + const closed = await options.routineStore.failOpenRuns( + routineId, + reason, + ); console.warn( JSON.stringify({ type: "routine-fire-gave-up", @@ -419,6 +431,7 @@ export async function dispatchClaimedRoutines( key: item.key, attempts: item.attempts, reason, + closedRuns: closed, }), ); } else if (!released) { diff --git a/server/tests/routine-sweep.integration.test.ts b/server/tests/routine-sweep.integration.test.ts index 2c98dadf..2b3a4459 100644 --- a/server/tests/routine-sweep.integration.test.ts +++ b/server/tests/routine-sweep.integration.test.ts @@ -848,6 +848,48 @@ describe("consuming a claimed firing", () => { ).toEqual([]); }); + /** + * THE LEAKED RUN ROW THE ATTEMPT CAP LEFT BEHIND. `insertRun` runs before `dispatch` on every + * attempt, so a dispatch that throws on the very last attempt still leaves an open (`status` null) + * run row nothing else closes: the item stops being claimed at the cap and the queue's own + * machinery has nothing to do with `routine_runs`. Without closing it, `listFor` — the routines + * page's read — keeps showing that open row as the newest run, and a routine that never ran reads + * as "running now" forever. + */ + test("giving up at the attempt cap also closes the run row it leaked, so the page stops reading 'running'", async () => { + const { owner, routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + maxAttempts: 1, + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + } finally { + warn.mockRestore(); + } + + // No run row for this routine is left open: every attempt that opened one also got it closed. + const runs = await runsFor(routine.id); + expect(runs.length).toBeGreaterThan(0); + expect(runs.every((run) => run.status !== null)).toBe(true); + + // And the page-facing read agrees: the newest run reads "failed", not "running now". + const [summary] = await store.listFor(owner.id); + expect(summary?.lastRun?.status).toBe("failed"); + expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date); + }); + /** * BOTH KINDS OF DONE WITH, which is the half `queue.ts:262-274` documents as having been forgotten. * diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 4bc558f2..6ae12a32 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -907,6 +907,72 @@ describe("opening and closing a run", () => { }); }); +/** + * Every dispatch attempt that goes nowhere opens a run row and leaves it open — `insertRun` runs + * before `dispatch`, and a dispatch that throws never reaches `finishRun`. `dueRoutines`/the sweep's + * attempt cap eventually stops retrying, but nothing else closes those rows: `listFor` shows the + * newest one, so the page reads "running now" forever for a routine that never ran at all. + * + * `failOpenRuns` is the sweep's cleanup for exactly that: close every open (`status is null`) run for + * one routine as "failed", not just the newest, because every one of them was a real dispatch attempt + * that went nowhere — closing only the newest would still leave the others open and wrong. + */ +describe("closing the runs a dispatch never got to finish", () => { + test("closes every open run for the routine, leaves a finished one untouched, and reports the count", async () => { + const { routine } = await makeRoutine(); + const firstOpen = await store.insertRun(routine.id); + const secondOpen = await store.insertRun(routine.id); + const finished = await store.insertRun(routine.id); + await store.finishRun(finished.runId, "succeeded"); + + const closed = await store.failOpenRuns( + routine.id, + "the server answered 503", + ); + + expect(closed).toBe(2); + + const rows = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.routineId, routine.id)); + const byId = new Map(rows.map((row) => [row.id, row])); + + for (const { runId } of [firstOpen, secondOpen]) { + const row = byId.get(runId); + expect(row?.status).toBe("failed"); + expect(row?.finishedAt).toBeInstanceOf(Date); + expect(row?.error).toBe("the server answered 503"); + } + + // The finished run's outcome is untouched: this cleanup closes leaked attempts, not runs that + // already have an outcome. + const finishedRow = byId.get(finished.runId); + expect(finishedRow?.status).toBe("succeeded"); + expect(finishedRow?.error).toBeNull(); + }); + + test("caps the error the same way finishRun does", async () => { + const { routine } = await makeRoutine(); + const { runId } = await store.insertRun(routine.id); + + await store.failOpenRuns(routine.id, "x".repeat(600)); + + const [row] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.id, runId)); + expect(row?.error).toHaveLength(MAX_RUN_ERROR); + }); + + test("a routine with nothing open closes nothing", async () => { + const { routine } = await makeRoutine(); + expect(await store.failOpenRuns(routine.id, "no attempts to close")).toBe( + 0, + ); + }); +}); + describe("the runner's read of one firing", () => { test("joins the run to its routine, owner included", async () => { const { owner, agentId, channel, routine } = await makeRoutine( diff --git a/worker/src/index.ts b/worker/src/index.ts index aa41ffc2..38e7dfa8 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -101,8 +101,9 @@ async function dispatch(routineRunId: string): Promise { "content-type": "application/json", }, body: JSON.stringify({ routineRunId }), - // The `for(;;)` loop below has no CronJob deadline bounding this call from outside; a wedged - // server must not stall the only thing firing routines. + // The `for(;;)` loop below has no CronJob around it at all, so nothing bounds this call from + // outside the process the way `activeDeadlineSeconds` bounds the CronJob's job; a wedged server + // must not stall the only thing firing routines. signal: AbortSignal.timeout(30_000), }); if (response.status !== 202) { From 7319a30273c8ad955af520b26330d8dc5bcb8ec3 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 15:05:35 -0300 Subject: [PATCH 33/45] Renumber the routines migration behind the attention inbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main took idx 20 while this branch was built, so 0020_routines becomes 0021_routines: regenerated with drizzle-kit (SQL byte-identical to the original), snapshot chained off main's 0020, journal appended. The last_run_at column comment now says what the sweep actually writes there — the stamp advanced past, fired or drained — rather than implying a run history the routine_runs table owns. --- .../{0020_routines.sql => 0021_routines.sql} | 0 server/drizzle/meta/0021_snapshot.json | 2856 +++++++++++++++++ server/drizzle/meta/_journal.json | 9 +- server/src/db/schema/coworker.ts | 6 + 4 files changed, 2870 insertions(+), 1 deletion(-) rename server/drizzle/{0020_routines.sql => 0021_routines.sql} (100%) create mode 100644 server/drizzle/meta/0021_snapshot.json diff --git a/server/drizzle/0020_routines.sql b/server/drizzle/0021_routines.sql similarity index 100% rename from server/drizzle/0020_routines.sql rename to server/drizzle/0021_routines.sql diff --git a/server/drizzle/meta/0021_snapshot.json b/server/drizzle/meta/0021_snapshot.json new file mode 100644 index 00000000..94f051f6 --- /dev/null +++ b/server/drizzle/meta/0021_snapshot.json @@ -0,0 +1,2856 @@ +{ + "id": "1ac1d8fc-e594-4a52-a87c-047ee1038713", + "prevId": "69d7cc41-1fbf-4da8-b535-c7a0e99435be", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": ["channel_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": ["channel_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": ["last_message_agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": ["user_id", "channel_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": ["provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": ["user_id", "role"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": ["computer_id", "tool_call_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": ["user_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": ["routine_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": ["component_name", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": ["component_name", "function_name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": ["server_id", "name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": ["server_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": ["kind", "ref", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": ["skill_id", "ref"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attention_resolutions": { + "name": "attention_resolutions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "audit_event_id": { + "name": "audit_event_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "attention_resolutions_event_idx": { + "name": "attention_resolutions_event_idx", + "columns": [ + { + "expression": "audit_event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": ["kind", "key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": ["built_in", "remote_ag_ui"] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": ["admin", "user"] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": ["public", "private"] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": ["succeeded", "failed", "skipped"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index b6120507..6e3f5217 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -148,6 +148,13 @@ "when": 1787761381038, "tag": "0020_attention_resolutions", "breakpoints": true + }, + { + "idx": 21, + "version": "7", + "when": 1787767359748, + "tag": "0021_routines", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/server/src/db/schema/coworker.ts b/server/src/db/schema/coworker.ts index c3c860b6..7040f006 100644 --- a/server/src/db/schema/coworker.ts +++ b/server/src/db/schema/coworker.ts @@ -113,6 +113,12 @@ export const routines = pgTable( enabled: boolean("enabled").notNull().default(true), /** The sweep's read target. Recomputed on every write and CAS-advanced by the sweep. */ nextRunAt: timestamp("next_run_at", { withTimezone: true }).notNull(), + /** + * The last occurrence stamp the sweep advanced past — fired OR silently drained as stale. + * Not "when this last ran": the run history lives in routine_runs, and everything a person + * sees reads that table. This is the scheduler's own bookmark, kept because a CAS needs the + * value it compared against recorded somewhere a human can inspect when a clock looks wrong. + */ lastRunAt: timestamp("last_run_at", { withTimezone: true }), createdAt: createdAt(), updatedAt: updatedAt(), From c9ab8b2eea42f5546d3a180e6584da169bc01a78 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 15:06:27 -0300 Subject: [PATCH 34/45] Format what main merged unformatted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit biome 2.5.10 (the version bun.lock has pinned throughout) rejects eight files as merged by #241 and #255 — main's CI is red on both. Formatting them here keeps this branch's whole-tree gate honest; if main fixes itself first, this commit rebases away to nothing. --- app/src/routes/_authed/admin/boundaries.tsx | 1 - server/drizzle/meta/0020_snapshot.json | 323 +++++--------------- server/src/app.ts | 7 +- server/src/attention/view.ts | 4 +- server/src/computer/routes.ts | 9 +- server/src/db/schema/attention.ts | 8 +- server/tests/attention-view.test.ts | 9 +- server/tests/computer-policy.test.ts | 7 +- 8 files changed, 109 insertions(+), 259 deletions(-) diff --git a/app/src/routes/_authed/admin/boundaries.tsx b/app/src/routes/_authed/admin/boundaries.tsx index a4830562..d8bff38c 100644 --- a/app/src/routes/_authed/admin/boundaries.tsx +++ b/app/src/routes/_authed/admin/boundaries.tsx @@ -301,7 +301,6 @@ function BoundariesPage() { ); } - /** * What the tested rule would have done to actions already on the trail. * diff --git a/server/drizzle/meta/0020_snapshot.json b/server/drizzle/meta/0020_snapshot.json index 2977372f..16628782 100644 --- a/server/drizzle/meta/0020_snapshot.json +++ b/server/drizzle/meta/0020_snapshot.json @@ -123,12 +123,8 @@ "name": "accounts_user_id_users_id_fk", "tableFrom": "accounts", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -201,12 +197,8 @@ "name": "agents_package_id_deployment_packages_id_fk", "tableFrom": "agents", "tableTo": "deployment_packages", - "columnsFrom": [ - "package_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["package_id"], + "columnsTo": ["id"], "onDelete": "set null", "onUpdate": "no action" } @@ -407,12 +399,8 @@ "name": "channel_agents_channel_id_channels_id_fk", "tableFrom": "channel_agents", "tableTo": "channels", - "columnsFrom": [ - "channel_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -420,12 +408,8 @@ "name": "channel_agents_agent_id_agents_id_fk", "tableFrom": "channel_agents", "tableTo": "agents", - "columnsFrom": [ - "agent_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -433,10 +417,7 @@ "compositePrimaryKeys": { "channel_agents_channel_id_agent_id_pk": { "name": "channel_agents_channel_id_agent_id_pk", - "columns": [ - "channel_id", - "agent_id" - ] + "columns": ["channel_id", "agent_id"] } }, "uniqueConstraints": {}, @@ -486,12 +467,8 @@ "name": "channel_memberships_channel_id_channels_id_fk", "tableFrom": "channel_memberships", "tableTo": "channels", - "columnsFrom": [ - "channel_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -499,12 +476,8 @@ "name": "channel_memberships_user_id_users_id_fk", "tableFrom": "channel_memberships", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -512,10 +485,7 @@ "compositePrimaryKeys": { "channel_memberships_channel_id_user_id_pk": { "name": "channel_memberships_channel_id_user_id_pk", - "columns": [ - "channel_id", - "user_id" - ] + "columns": ["channel_id", "user_id"] } }, "uniqueConstraints": {}, @@ -632,12 +602,8 @@ "name": "channels_package_id_deployment_packages_id_fk", "tableFrom": "channels", "tableTo": "deployment_packages", - "columnsFrom": [ - "package_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["package_id"], + "columnsTo": ["id"], "onDelete": "set null", "onUpdate": "no action" }, @@ -645,12 +611,8 @@ "name": "channels_last_message_agent_id_agents_id_fk", "tableFrom": "channels", "tableTo": "agents", - "columnsFrom": [ - "last_message_agent_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["last_message_agent_id"], + "columnsTo": ["id"], "onDelete": "set null", "onUpdate": "no action" } @@ -805,9 +767,7 @@ "deployment_packages_tenant_id_unique": { "name": "deployment_packages_tenant_id_unique", "nullsNotDistinct": false, - "columns": [ - "tenant_id" - ] + "columns": ["tenant_id"] } }, "policies": {}, @@ -873,12 +833,8 @@ "name": "intelligence_channel_mappings_user_id_users_id_fk", "tableFrom": "intelligence_channel_mappings", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -886,12 +842,8 @@ "name": "intelligence_channel_mappings_channel_id_channels_id_fk", "tableFrom": "intelligence_channel_mappings", "tableTo": "channels", - "columnsFrom": [ - "channel_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -899,10 +851,7 @@ "compositePrimaryKeys": { "intelligence_channel_mappings_user_id_channel_id_pk": { "name": "intelligence_channel_mappings_user_id_channel_id_pk", - "columns": [ - "user_id", - "channel_id" - ] + "columns": ["user_id", "channel_id"] } }, "uniqueConstraints": {}, @@ -1003,12 +952,8 @@ "name": "sessions_user_id_users_id_fk", "tableFrom": "sessions", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1018,9 +963,7 @@ "sessions_token_unique": { "name": "sessions_token_unique", "nullsNotDistinct": false, - "columns": [ - "token" - ] + "columns": ["token"] } }, "policies": {}, @@ -1086,12 +1029,8 @@ "name": "sso_providers_user_id_users_id_fk", "tableFrom": "sso_providers", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "set null", "onUpdate": "no action" } @@ -1101,9 +1040,7 @@ "sso_providers_provider_id_unique": { "name": "sso_providers_provider_id_unique", "nullsNotDistinct": false, - "columns": [ - "provider_id" - ] + "columns": ["provider_id"] } }, "policies": {}, @@ -1141,12 +1078,8 @@ "name": "user_roles_user_id_users_id_fk", "tableFrom": "user_roles", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1154,10 +1087,7 @@ "compositePrimaryKeys": { "user_roles_user_id_role_pk": { "name": "user_roles_user_id_role_pk", - "columns": [ - "user_id", - "role" - ] + "columns": ["user_id", "role"] } }, "uniqueConstraints": {}, @@ -1229,9 +1159,7 @@ "users_email_unique": { "name": "users_email_unique", "nullsNotDistinct": false, - "columns": [ - "email" - ] + "columns": ["email"] } }, "policies": {}, @@ -1402,10 +1330,7 @@ "compositePrimaryKeys": { "computer_page_frame_computer_id_tool_call_id_pk": { "name": "computer_page_frame_computer_id_tool_call_id_pk", - "columns": [ - "computer_id", - "tool_call_id" - ] + "columns": ["computer_id", "tool_call_id"] } }, "uniqueConstraints": {}, @@ -1492,12 +1417,8 @@ "name": "agent_preferences_user_id_users_id_fk", "tableFrom": "agent_preferences", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -1505,12 +1426,8 @@ "name": "agent_preferences_agent_id_agents_id_fk", "tableFrom": "agent_preferences", "tableTo": "agents", - "columnsFrom": [ - "agent_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1518,10 +1435,7 @@ "compositePrimaryKeys": { "agent_preferences_user_id_agent_id_pk": { "name": "agent_preferences_user_id_agent_id_pk", - "columns": [ - "user_id", - "agent_id" - ] + "columns": ["user_id", "agent_id"] } }, "uniqueConstraints": {}, @@ -1631,12 +1545,8 @@ "name": "agent_profiles_agent_id_agents_id_fk", "tableFrom": "agent_profiles", "tableTo": "agents", - "columnsFrom": [ - "agent_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -1644,12 +1554,8 @@ "name": "agent_profiles_owner_user_id_users_id_fk", "tableFrom": "agent_profiles", "tableTo": "users", - "columnsFrom": [ - "owner_user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], "onDelete": "set null", "onUpdate": "no action" } @@ -1703,12 +1609,8 @@ "name": "component_exclusions_component_name_components_name_fk", "tableFrom": "component_exclusions", "tableTo": "components", - "columnsFrom": [ - "component_name" - ], - "columnsTo": [ - "name" - ], + "columnsFrom": ["component_name"], + "columnsTo": ["name"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -1716,12 +1618,8 @@ "name": "component_exclusions_agent_id_agents_id_fk", "tableFrom": "component_exclusions", "tableTo": "agents", - "columnsFrom": [ - "agent_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1729,10 +1627,7 @@ "compositePrimaryKeys": { "component_exclusions_component_name_agent_id_pk": { "name": "component_exclusions_component_name_agent_id_pk", - "columns": [ - "component_name", - "agent_id" - ] + "columns": ["component_name", "agent_id"] } }, "uniqueConstraints": {}, @@ -1783,12 +1678,8 @@ "name": "component_functions_component_name_components_name_fk", "tableFrom": "component_functions", "tableTo": "components", - "columnsFrom": [ - "component_name" - ], - "columnsTo": [ - "name" - ], + "columnsFrom": ["component_name"], + "columnsTo": ["name"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1796,10 +1687,7 @@ "compositePrimaryKeys": { "component_functions_component_name_function_name_pk": { "name": "component_functions_component_name_function_name_pk", - "columns": [ - "component_name", - "function_name" - ] + "columns": ["component_name", "function_name"] } }, "uniqueConstraints": {}, @@ -1963,12 +1851,8 @@ "name": "mcp_servers_credential_id_credentials_id_fk", "tableFrom": "mcp_servers", "tableTo": "credentials", - "columnsFrom": [ - "credential_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], "onDelete": "restrict", "onUpdate": "no action" } @@ -2023,12 +1907,8 @@ "name": "mcp_tools_server_id_mcp_servers_id_fk", "tableFrom": "mcp_tools", "tableTo": "mcp_servers", - "columnsFrom": [ - "server_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["server_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -2036,10 +1916,7 @@ "compositePrimaryKeys": { "mcp_tools_server_id_name_pk": { "name": "mcp_tools_server_id_name_pk", - "columns": [ - "server_id", - "name" - ] + "columns": ["server_id", "name"] } }, "uniqueConstraints": {}, @@ -2112,12 +1989,8 @@ "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", "tableFrom": "mcp_user_credentials", "tableTo": "mcp_servers", - "columnsFrom": [ - "server_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["server_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -2125,12 +1998,8 @@ "name": "mcp_user_credentials_user_id_users_id_fk", "tableFrom": "mcp_user_credentials", "tableTo": "users", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -2138,12 +2007,8 @@ "name": "mcp_user_credentials_credential_id_credentials_id_fk", "tableFrom": "mcp_user_credentials", "tableTo": "credentials", - "columnsFrom": [ - "credential_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -2151,10 +2016,7 @@ "compositePrimaryKeys": { "mcp_user_credentials_server_id_user_id_pk": { "name": "mcp_user_credentials_server_id_user_id_pk", - "columns": [ - "server_id", - "user_id" - ] + "columns": ["server_id", "user_id"] } }, "uniqueConstraints": {}, @@ -2227,12 +2089,8 @@ "name": "plugin_grants_agent_id_agents_id_fk", "tableFrom": "plugin_grants", "tableTo": "agents", - "columnsFrom": [ - "agent_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -2240,11 +2098,7 @@ "compositePrimaryKeys": { "plugin_grants_kind_ref_agent_id_pk": { "name": "plugin_grants_kind_ref_agent_id_pk", - "columns": [ - "kind", - "ref", - "agent_id" - ] + "columns": ["kind", "ref", "agent_id"] } }, "uniqueConstraints": {}, @@ -2441,12 +2295,8 @@ "name": "skill_tools_skill_id_skills_id_fk", "tableFrom": "skill_tools", "tableTo": "skills", - "columnsFrom": [ - "skill_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -2454,10 +2304,7 @@ "compositePrimaryKeys": { "skill_tools_skill_id_ref_pk": { "name": "skill_tools_skill_id_ref_pk", - "columns": [ - "skill_id", - "ref" - ] + "columns": ["skill_id", "ref"] } }, "uniqueConstraints": {}, @@ -2570,12 +2417,8 @@ "name": "skills_owner_user_id_users_id_fk", "tableFrom": "skills", "tableTo": "users", - "columnsFrom": [ - "owner_user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -2744,10 +2587,7 @@ "compositePrimaryKeys": { "work_items_kind_key_pk": { "name": "work_items_kind_key_pk", - "columns": [ - "kind", - "key" - ] + "columns": ["kind", "key"] } }, "uniqueConstraints": {}, @@ -2760,10 +2600,7 @@ "public.agent_type": { "name": "agent_type", "schema": "public", - "values": [ - "built_in", - "remote_ag_ui" - ] + "values": ["built_in", "remote_ag_ui"] }, "public.credential_kind": { "name": "credential_kind", @@ -2780,18 +2617,12 @@ "public.role": { "name": "role", "schema": "public", - "values": [ - "admin", - "user" - ] + "values": ["admin", "user"] }, "public.agent_visibility": { "name": "agent_visibility", "schema": "public", - "values": [ - "public", - "private" - ] + "values": ["public", "private"] } }, "schemas": {}, @@ -2804,4 +2635,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/server/src/app.ts b/server/src/app.ts index eeae6a7b..e418714d 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -765,7 +765,12 @@ export function createApp( if (auditReader && attentionStore) { app.route( "/api/attention", - createAttentionRoutes(auditReader, attentionStore, requireUser, canUseBot), + createAttentionRoutes( + auditReader, + attentionStore, + requireUser, + canUseBot, + ), ); } diff --git a/server/src/attention/view.ts b/server/src/attention/view.ts index 3a766b9f..3525e975 100644 --- a/server/src/attention/view.ts +++ b/server/src/attention/view.ts @@ -47,7 +47,9 @@ const text = (value: unknown): string => * "google-drive/search_files", which is not a Bot, which `canUseBot` correctly denies, which hid * every tool rejection from exactly the person it was for. */ -export function botOf(event: Pick): string { +export function botOf( + event: Pick, +): string { if (event.targetType === "computer" || event.targetType === "agent") { return event.targetId ?? text(event.payload.bot); } diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.ts index d3fcb02b..e57cdd0c 100644 --- a/server/src/computer/routes.ts +++ b/server/src/computer/routes.ts @@ -19,10 +19,7 @@ import { import type { PageFrameStore } from "./page-frames"; import type { AuditReader } from "../audit"; import { type PolicyStore, parseActionPolicy } from "./policy-store"; -import { - dryRunAgainstHistory, - REPLAYABLE_EVENT_TYPES, -} from "./policy-dry-run"; +import { dryRunAgainstHistory, REPLAYABLE_EVENT_TYPES } from "./policy-dry-run"; /** * The Bot computer's surface, behind the same session guard as every other API route. @@ -672,7 +669,9 @@ export function createComputerRoutes( targetType: "computer", }); - return context.json({ report: dryRunAgainstHistory(parsed.policy, events) }); + return context.json({ + report: dryRunAgainstHistory(parsed.policy, events), + }); }); return routes; diff --git a/server/src/db/schema/attention.ts b/server/src/db/schema/attention.ts index ddd74ecd..0a90e61e 100644 --- a/server/src/db/schema/attention.ts +++ b/server/src/db/schema/attention.ts @@ -1,4 +1,10 @@ -import { pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"; +import { + pgTable, + text, + timestamp, + uniqueIndex, + uuid, +} from "drizzle-orm/pg-core"; /** * A trail row somebody has marked handled. diff --git a/server/tests/attention-view.test.ts b/server/tests/attention-view.test.ts index 38f1830d..4ec901e9 100644 --- a/server/tests/attention-view.test.ts +++ b/server/tests/attention-view.test.ts @@ -13,7 +13,10 @@ function event(overrides: Partial): AuditEvent { actorUserId: null, eventType: overrides.eventType ?? "computer.action_refused", targetType: overrides.targetType ?? "computer", - targetId: overrides.targetId === undefined ? "general-assistant" : overrides.targetId, + targetId: + overrides.targetId === undefined + ? "general-assistant" + : overrides.targetId, payload: overrides.payload ?? {}, createdAt: overrides.createdAt ?? "2026-08-25T00:00:00.000Z", }; @@ -33,7 +36,9 @@ describe("attentionItemsFrom", () => { ); expect(items).toHaveLength(1); expect(items[0]?.kind).toBe("refused"); - expect(items[0]?.sentence).toBe("“Submit order” on shop.example is blocked."); + expect(items[0]?.sentence).toBe( + "“Submit order” on shop.example is blocked.", + ); expect(items[0]?.botId).toBe("general-assistant"); }); diff --git a/server/tests/computer-policy.test.ts b/server/tests/computer-policy.test.ts index 60c2462d..63f886c9 100644 --- a/server/tests/computer-policy.test.ts +++ b/server/tests/computer-policy.test.ts @@ -620,7 +620,6 @@ describe("a rule about one surface does not refuse another", () => { }); }); - describe("refusal wording under the context the gateway actually builds", () => { /* * The gateway attaches a neutral all-empty `mcp` to every browser context so a rule naming @@ -630,7 +629,11 @@ describe("refusal wording under the context the gateway actually builds", () => */ test("a browser refusal names the element, neutral mcp notwithstanding", () => { const decision = evaluateActionPolicy( - { mode: "enforce", deny: ['contains(element.name, "Submit")'], allow: ["true"] }, + { + mode: "enforce", + deny: ['contains(element.name, "Submit")'], + allow: ["true"], + }, { tool: { name: "computer_click" }, bot: { id: "general-assistant" }, From 076fec90030bfdba3fef80b11ee483f3fcbae7bc Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 15:09:51 -0300 Subject: [PATCH 35/45] Give the endpoint test the attention slot main added main inserted attentionStore into createApp's positional signature before pageFrames; the endpoint test's hand-built tuple put the runner one slot early, so the route never mounted and all seven cases 404ed. --- server/tests/routine-endpoint.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/tests/routine-endpoint.test.ts b/server/tests/routine-endpoint.test.ts index 91ccc06e..fbfb96fe 100644 --- a/server/tests/routine-endpoint.test.ts +++ b/server/tests/routine-endpoint.test.ts @@ -66,6 +66,7 @@ function buildApp( undefined, // peopleStore undefined, // identityProviders undefined, // intentRouter + undefined, // attentionStore undefined, // pageFrames runner, ]; From 251d588115ebd96acd7ab6ac690005171c16fdab Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 15:52:19 -0300 Subject: [PATCH 36/45] Name the channel by id when its name cannot name it --- server/src/routines/store.ts | 30 +++++++++-- .../tests/routines-store.integration.test.ts | 51 +++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index 3987ef53..663bc18d 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -264,10 +264,30 @@ function nextRunFor(cron: string, timezone: string, after: Date): Date { } } -/** "A", "A, B", or five names and "and others" — a sentence, not a list a client renders. */ -function nameThem(names: string[]): string { - if (names.length <= MAX_NAMED_CHANNELS) return names.join(", "); - return [...names.slice(0, MAX_NAMED_CHANNELS), "and others"].join(", "); +/** + * "A", "A, B", or five names and "and others" — a sentence, not a list a client renders. + * + * A name alone is not always enough to ask by: a real account turned up six channels all named + * "General Assistant" with the same Bot, and a refusal built from names alone read "General + * Assistant, General Assistant, … and others" — circular, because the person cannot answer it and + * the model cannot map an answer back to a channelId. So every candidate whose name is shared by + * another candidate gets its full id appended in parentheses; the id is the one thing the model can + * pass back as `channelId` when names cannot tell two channels apart, and a person pasting it back + * is ugly but functional. A candidate with a unique name stays bare, so the common case — and the + * sentence's length — is unaffected by a collision elsewhere in the list. + */ +function nameThem(candidates: { id: string; name: string }[]): string { + const counts = new Map(); + for (const candidate of candidates) { + counts.set(candidate.name, (counts.get(candidate.name) ?? 0) + 1); + } + const labels = candidates.map((candidate) => + (counts.get(candidate.name) ?? 0) > 1 + ? `${candidate.name} (${candidate.id})` + : candidate.name, + ); + if (labels.length <= MAX_NAMED_CHANNELS) return labels.join(", "); + return [...labels.slice(0, MAX_NAMED_CHANNELS), "and others"].join(", "); } export function createRoutineStore(database: Database): RoutineStore { @@ -342,7 +362,7 @@ export function createRoutineStore(database: Database): RoutineStore { // question it can put to the person, and "be more specific" is not. throw new RoutineRefusedError( `You are in more than one channel with me — ${nameThem( - candidates.map((candidate) => candidate.name), + candidates, )}. Say which one.`, ); } diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 6ae12a32..33b93d8d 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -413,6 +413,57 @@ describe("resolving the channel to post into", () => { const occurrences = message.split(name).length - 1; expect(occurrences).toBe(5); }); + + test("names a colliding channel by id when its name cannot tell it apart from another", async () => { + // The finding this guards: a real account had six channels all named "General Assistant" with + // the same Bot, and the refusal read "General Assistant, General Assistant, … and others" — + // circular, because the person cannot answer it and the model cannot map an answer back to a + // channelId. Two same-named channels plus one distinctly-named one is the smallest case that + // proves the fix: the colliding pair gets ids, the distinct one stays bare. + const owner = await createUser(); + const agentId = await createAgent(owner); + const twinOne = await createChannel(owner, [agentId]); + const twinTwo = await createChannel(owner, [agentId]); + // channelStore.create names channels itself, so force the collision directly rather than + // trusting default naming to produce two identical names. + await database + .update(channels) + .set({ name: "General Assistant" }) + .where(eq(channels.id, twinOne.id)); + await database + .update(channels) + .set({ name: "General Assistant" }) + .where(eq(channels.id, twinTwo.id)); + const distinct = await createChannel(owner, [agentId]); + await database + .update(channels) + .set({ name: "Marketing Bot" }) + .where(eq(channels.id, distinct.id)); + + const failure = await store + .create({ + ownerUserId: owner.id, + agentId, + instruction: "Which one?", + cron: DAILY, + }) + .then( + () => null, + (error: unknown) => error, + ); + + expect(failure).toBeInstanceOf(RoutineRefusedError); + const message = (failure as Error).message; + // Both same-named channels are named by their full id, because that is the one thing the model + // can pass back as `channelId` when the names alone cannot disambiguate. + expect(message).toContain(twinOne.id); + expect(message).toContain(twinTwo.id); + // The distinctly-named channel is not: its name alone already disambiguates it, and appending + // an id it does not need would make the common, non-colliding case's sentence longer for + // nothing. + expect(message).toContain("Marketing Bot"); + expect(message).not.toContain(`Marketing Bot (${distinct.id})`); + }); }); describe("reading a person's routines", () => { From 1f465049c58309ce2f0a99edfa803461312f4eea Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 16:39:48 -0300 Subject: [PATCH 37/45] Seed a history the model will accept, however a turn once ended --- server/src/routines/run-turn.ts | 105 ++++++++++++++- server/tests/routine-run-turn.test.ts | 186 +++++++++++++++++++++++++- 2 files changed, 286 insertions(+), 5 deletions(-) diff --git a/server/src/routines/run-turn.ts b/server/src/routines/run-turn.ts index 137d2e9f..41ae85d5 100644 --- a/server/src/routines/run-turn.ts +++ b/server/src/routines/run-turn.ts @@ -53,6 +53,7 @@ import type { BaseEvent, Message, RunAgentInput, + ToolCall, } from "@ag-ui/client"; import { EventType } from "@ag-ui/client"; import { historyOrEmpty } from "../copilot"; @@ -195,6 +196,105 @@ function toAgentMessage(message: ThreadHistoryMessage): Message { } as Message; } +/** Whether a message said nothing at all — no text, no parts, nothing to show a person. */ +function isSilent(message: Message): boolean { + const content = (message as { content?: unknown }).content; + if (content === undefined || content === null) return true; + if (typeof content === "string") return content.length === 0; + if (Array.isArray(content)) return content.length === 0; + return false; +} + +/** + * Refuse to re-present a conversation the model API will reject. + * + * FOUND IN PRODUCTION. Two firings of one routine, fifteen minutes apart, both failed with + * `Tool result is missing for tool call call_TTbiXzJVNifQt8ioU1JJmj4S.` — the SAME call id both + * times, so it did not come from the live turn: the channel's Intelligence thread held an assistant + * message carrying a tool call whose result message never landed, because an earlier CHAT turn was + * interrupted mid-call. The seeding below hands the whole converted history to the runner, the model + * provider validates call/result pairing, and it rejects the conversation. One historical dangle + * therefore poisons EVERY future firing in that channel until the fatigue rule disables the routine: + * a permanent failure grown out of transient damage, and nothing the person did wrong. + * + * WHY DROPPING IS THE RIGHT ANSWER, and not repair. History here is CONTEXT for a turn, not a + * transaction to resume. A dangling call is already permanently unanswerable — the tool run that + * would have answered it ended when that chat turn did, and there is no result to invent. The only + * two options are to seed a conversation the API refuses, or to seed the same conversation minus a + * call that never completed. The second one loses a fragment of an interrupted exchange; the first + * one disables a routine forever. + * + * WHAT THIS DOES NOT DO. It does not DELETE anything from the platform. The thread still holds every + * row, the person still sees the interrupted exchange in their channel, and a browser turn is + * unaffected. This is a read-side filter on one turn's input and nothing more. + * + * IDS ARE NEVER CHANGED, which is what keeps `persistedInputMessages`' id-subtraction below correct: + * a message this pass stripped a tool call from keeps its id and is still subtracted out as historic, + * and a message it dropped was never a candidate to persist. So sanitizing cannot turn a firing into + * one that re-persists the transcript. + * + * The rules, in order: + * 1. A tool call is ANSWERED if some later message carries it as `toolCallId`. Later, not merely + * present: a result ahead of its call is not a pairing any provider accepts either. + * 2. An assistant message keeps only its answered calls. If that leaves it with no calls and + * nothing said, the message is dropped — an empty assistant husk is itself invalid for some + * providers, so stripping the call is not enough. + * 3. A tool result whose `toolCallId` matches no surviving call is dropped: the mirror-image dangle, + * which is what an interruption between the two rows leaves behind in the other order. + * + * Order is preserved, the input array is not mutated, and a message the pass does not change is + * returned as the same object — a healthy thread, which is nearly all of them, goes through + * untouched rather than through a re-normalization that could quietly differ. + */ +export function sanitizeSeededHistory(history: Message[]): Message[] { + /** For each answered call id, the earliest position that answers it. */ + const answeredAt = new Map(); + for (const [index, message] of history.entries()) { + const { toolCallId } = message as { toolCallId?: string }; + if (toolCallId === undefined) continue; + if (!answeredAt.has(toolCallId)) answeredAt.set(toolCallId, index); + } + + const surviving = new Set(); + const kept: (Message | undefined)[] = history.map((message, index) => { + const { toolCalls } = message as { toolCalls?: ToolCall[] }; + if (toolCalls === undefined) return message; + + const answered = toolCalls.filter((call) => { + const at = answeredAt.get(call.id); + return at !== undefined && at > index; + }); + for (const call of answered) surviving.add(call.id); + + // The husk check goes FIRST so it also catches a row that arrived with no calls and nothing + // said — the same invalid shape, reached without a dangle. + if (answered.length === 0 && isSilent(message)) return undefined; + // The healthy path, and the only one that returns the very same object. + if (answered.length === toolCalls.length) return message; + + /* + * Cast for the same reason `toAgentMessage` casts: `Message` is a union discriminated on `role`, + * and a spread over the union widens past every branch of it. Neither rewrite here can change + * the role or the shape — one narrows the `toolCalls` array, the other removes the key — so + * there is nothing to narrow against and nothing that could stop being a `Message`. + */ + if (answered.length > 0) { + return { ...message, toolCalls: answered } as Message; + } + // Text it did say, minus a call it cannot complete. + const { toolCalls: _dropped, ...rest } = message as Message & { + toolCalls?: ToolCall[]; + }; + return rest as Message; + }); + + return kept.filter((message): message is Message => { + if (message === undefined) return false; + const { toolCallId } = message as { toolCallId?: string }; + return toolCallId === undefined || surviving.has(toolCallId); + }); +} + /** What a message said out loud, or nothing if it did not say anything. */ function assistantText(message: Message): string | undefined { if (message.role !== "assistant") return undefined; @@ -265,13 +365,16 @@ export function createTurnRunner(options: { * the same question every night with no memory of the last answer. `historyOrEmpty` is the * 404-on-a-fresh-thread case: `getOrCreateThread` above makes that rare, not impossible, since a * concurrent delete is still a thing that can happen between the two calls. + * + * And sanitized on the way in — see {@link sanitizeSeededHistory}, which is the difference + * between a routine that survives one interrupted chat turn and one that never fires again. */ const history = await historyOrEmpty( () => intelligence.getThreadMessages({ threadId, userId: ownerUserId }), { messages: [] as ThreadHistoryMessage[] }, ); - const seeded = history.messages.map(toAgentMessage); + const seeded = sanitizeSeededHistory(history.messages.map(toAgentMessage)); const turn = { id: crypto.randomUUID(), role: "user", diff --git a/server/tests/routine-run-turn.test.ts b/server/tests/routine-run-turn.test.ts index 187c459d..add0cab3 100644 --- a/server/tests/routine-run-turn.test.ts +++ b/server/tests/routine-run-turn.test.ts @@ -1,7 +1,11 @@ import { AbstractAgent, EventType } from "@ag-ui/client"; +import type { Message } from "@ag-ui/client"; import { describe, expect, test } from "bun:test"; import { EMPTY } from "rxjs"; -import { createTurnRunner } from "../src/routines/run-turn"; +import { + createTurnRunner, + sanitizeSeededHistory, +} from "../src/routines/run-turn"; /** * A headless turn, asserted without a gateway, without a database and without a model. @@ -263,18 +267,23 @@ describe("a routine's headless turn", () => { role: "assistant", toolCalls: [{ id: "call_1", name: "search", args: '{"q":"x"}' }], }, + // The result that answers `call_1`. Present because the row above is otherwise a dangling + // call, which `sanitizeSeededHistory` drops — and what this test is about is the conversion, + // not the sanitation, so the fixture has to be a healthy exchange. + { id: "m5", role: "tool", content: "found x", toolCallId: "call_1" }, ], }); await run(); expect(agent.threadId).toBe(THREAD_ID); - // The four history rows, then this turn's instruction, then what the run added. - expect(agent.messages.map((message) => message.id).slice(0, 4)).toEqual([ + // The five history rows, then this turn's instruction, then what the run added. + expect(agent.messages.map((message) => message.id).slice(0, 5)).toEqual([ "m1", "m2", "m3", "m4", + "m5", ]); // A tool-call-only row has no content on the platform, and AG-UI requires the field. expect(agent.messages[3]).toMatchObject({ @@ -287,7 +296,9 @@ describe("a routine's headless turn", () => { }, ], }); - expect(agent.messages[4]).toMatchObject({ + // And a result row still points at the call it answers. + expect(agent.messages[4]).toMatchObject({ toolCallId: "call_1" }); + expect(agent.messages[5]).toMatchObject({ role: "user", content: INSTRUCTION, }); @@ -302,6 +313,148 @@ describe("a routine's headless turn", () => { }); }); +/** + * The seeded history has to be a conversation the model API will ACCEPT, and a thread that was once + * interrupted mid-tool-call is not one. + * + * Found in production: two firings fifteen minutes apart both failed with `Tool result is missing for + * tool call call_TTbiXzJVNifQt8ioU1JJmj4S.` — the SAME call id both times, so it came from persisted + * history rather than from the live turn. One interrupted chat turn therefore poisoned every + * subsequent firing in that channel until the fatigue rule disabled the routine: a permanent failure + * out of transient damage. These are the properties that keep that from happening again. + */ +describe("the seeded history is sanitized of dangling tool calls", () => { + test("an unanswered tool call is dropped and the answered one survives, with all text intact", async () => { + const { run, calls } = harness({ + history: [ + { id: "m1", role: "user", content: "Look two things up." }, + { + id: "m2", + role: "assistant", + content: "Looking them up.", + toolCalls: [ + { id: "call_answered", name: "search", args: '{"q":"x"}' }, + { id: "call_dangling", name: "search", args: '{"q":"y"}' }, + ], + }, + { + id: "m3", + role: "tool", + content: "found x", + toolCallId: "call_answered", + }, + { id: "m4", role: "assistant", content: "Here is x." }, + ], + }); + + await run(); + + const seeded = calls.runs[0]?.input.messages ?? []; + // Every message survives — nothing said out loud is thrown away — and only the call that has no + // answer is gone. + expect(seeded.map((message) => message.id).slice(0, 4)).toEqual([ + "m1", + "m2", + "m3", + "m4", + ]); + expect(seeded).toHaveLength(5); + expect(seeded[1]).toMatchObject({ + content: "Looking them up.", + toolCalls: [ + { + id: "call_answered", + type: "function", + function: { name: "search", arguments: '{"q":"x"}' }, + }, + ], + }); + expect(seeded[2]).toMatchObject({ toolCallId: "call_answered" }); + }); + + test("an assistant message whose only content was a dangling tool call is dropped entirely", async () => { + // An assistant row with neither text nor tool calls is itself invalid for some providers, so + // stripping the call is not enough: the husk has to go too. + const { run, calls } = harness({ + history: [ + { id: "m1", role: "user", content: "Look it up." }, + { + id: "m2", + role: "assistant", + toolCalls: [{ id: "call_dangling", name: "search", args: "{}" }], + }, + { id: "m3", role: "user", content: "Anything?" }, + ], + }); + + await run(); + + const seeded = calls.runs[0]?.input.messages ?? []; + expect(seeded.map((message) => message.id).slice(0, 2)).toEqual([ + "m1", + "m3", + ]); + expect(seeded).toHaveLength(3); + }); + + test("an orphaned tool result is dropped", async () => { + // The mirror-image dangle: a result whose call is not in the history at all. + const { run, calls } = harness({ + history: [ + { id: "m1", role: "user", content: "Hello." }, + { + id: "m2", + role: "tool", + content: "left over", + toolCallId: "call_gone", + }, + { id: "m3", role: "assistant", content: "Hello back." }, + ], + }); + + await run(); + + const seeded = calls.runs[0]?.input.messages ?? []; + expect(seeded.map((message) => message.id).slice(0, 2)).toEqual([ + "m1", + "m3", + ]); + expect(seeded).toHaveLength(3); + }); + + test("a clean history passes through unchanged, object for object", () => { + const clean = [ + { id: "m1", role: "user", content: "Look it up." }, + { + id: "m2", + role: "assistant", + content: "Looking it up.", + toolCalls: [ + { + id: "call_1", + type: "function", + function: { name: "search", arguments: "{}" }, + }, + ], + }, + { id: "m3", role: "tool", content: "found", toolCallId: "call_1" }, + { id: "m4", role: "assistant", content: "Here it is." }, + ] as unknown as Message[]; + const snapshot = structuredClone(clean); + + const sanitized = sanitizeSeededHistory(clean); + + // Nothing reordered, nothing rewritten — and not even reallocated, so there is no room for a + // silent normalization to creep in on the overwhelmingly common healthy-thread path. + expect(sanitized).toEqual(snapshot); + for (const [index, message] of sanitized.entries()) { + expect(message).toBe(clean[index]); + } + // And the caller's array was not mutated underneath it. + expect(clean).toEqual(snapshot); + }); +}); + describe("persistedInputMessages is the subtraction", () => { test("a history of three plus one new message persists exactly the new one", async () => { const { run, calls } = harness({ history: THREE_ROWS }); @@ -319,6 +472,31 @@ describe("persistedInputMessages is the subtraction", () => { } }); + test("a history carrying a dangle persists exactly the new message, and nothing sanitized", async () => { + // The subtraction is over the ids the PLATFORM handed back, and sanitizing changes no id: a + // message the sanitizer stripped a tool call from keeps its id and so is still subtracted out, + // and a message it dropped was never a candidate to persist in the first place. So a dangle + // must not turn this firing into one that re-persists half the transcript. + const { run, calls } = harness({ + history: [ + ...THREE_ROWS, + { + id: "m4", + role: "assistant", + toolCalls: [{ id: "call_dangling", name: "search", args: "{}" }], + }, + ], + }); + + await run(); + + const [request] = calls.runs; + // Three seeded rows survive the sanitation, plus this turn's instruction. + expect(request?.input.messages).toHaveLength(4); + expect(request?.persistedInputMessages).toHaveLength(1); + expect(request?.persistedInputMessages?.[0]?.content).toBe(INSTRUCTION); + }); + test("an empty history persists everything", async () => { const { run, calls } = harness({ history: [] }); From 9da3ea96fadfd9f5e2fb6cd697278de538d6b5ee Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Wed, 26 Aug 2026 17:12:41 -0300 Subject: [PATCH 38/45] Tell a firing turn that it is one --- server/src/plugins/builtin-routines.ts | 21 +++++- server/src/routines/run-turn.ts | 47 ++++++++++++- server/tests/builtin-routines.test.ts | 15 ++++ server/tests/routine-run-turn.test.ts | 97 ++++++++++++++++++++++++-- 4 files changed, 171 insertions(+), 9 deletions(-) diff --git a/server/src/plugins/builtin-routines.ts b/server/src/plugins/builtin-routines.ts index d660c455..c21861da 100644 --- a/server/src/plugins/builtin-routines.ts +++ b/server/src/plugins/builtin-routines.ts @@ -76,6 +76,12 @@ export function useRoutineTools(tools: RoutineTools | null): void { * minute, or into somebody else's channel. A tool a model misuses is not a failed call — it is a * wrong schedule that keeps being wrong on a timer. So the cron contract is written out in full, * with worked examples, rather than left to a field named `cron`. + * + * HOW THE INSTRUCTION IS PHRASED IS PART OF THAT CONTRACT, and it was learned the hard way: a + * routine stored as "Every run, append the current date and time to the Notion page …" fired, and the + * turn read its own schedule-shaped text as a question about SCHEDULING — it checked that the routine + * existed, said so, and appended nothing, successfully. `routines/run-turn.ts` now frames the firing + * turn as a firing; this is the other half, so the sentence is written as work to begin with. */ const TOOLS: readonly McpTool[] = Object.freeze([ { @@ -83,6 +89,12 @@ const TOOLS: readonly McpTool[] = Object.freeze([ description: [ "Set up a standing instruction that you carry out on a schedule for the person you are talking to.", "", + "Write the `instruction` as the work of ONE firing, in the imperative, and do not restate the schedule", + 'inside it: `Append the current UTC time to the page "Log"`, not `Every 15 minutes, append the current', + 'UTC time to the page "Log"`. The schedule belongs in `cron`, and the instruction is handed to a turn', + "that has already fired on it — an instruction that describes a schedule reads as a request to set one", + "up, and gets answered instead of carried out.", + "", "The schedule is a five-field cron expression, in the order `minute hour day-of-month month day-of-week`.", "`0 9 * * 1-5` is weekdays at nine in the morning. `30 18 * * *` is every day at half past six in the", "evening. `0 9 1 * *` is the first of the month at nine. A routine may run at most every 15 minutes;", @@ -105,7 +117,7 @@ const TOOLS: readonly McpTool[] = Object.freeze([ instruction: { type: "string", description: - "What to do each time it runs, written as an instruction to yourself.", + 'The work of one firing, imperative, with no schedule restated in it: `Append the current UTC time to the page "Log"`.', }, cron: { type: "string", @@ -145,6 +157,10 @@ const TOOLS: readonly McpTool[] = Object.freeze([ "Give the id from `list_routines` and only the fields that change; anything left out stays as it is. A", "new `cron` follows exactly the same five-field rules as `create_routine`, and a routine switched back", "on is scheduled from now rather than from wherever it left off.", + "", + "A new `instruction` follows the same rule as `create_routine`'s: the work of one firing, imperative, and", + 'do not restate the schedule inside it — `Append the current UTC time to the page "Log"`, not `Every 15', + 'minutes, append the current UTC time to the page "Log"`.', ].join("\n"), inputSchema: { type: "object", @@ -155,7 +171,8 @@ const TOOLS: readonly McpTool[] = Object.freeze([ }, instruction: { type: "string", - description: "A new instruction, replacing the old one entirely.", + description: + "A new instruction, replacing the old one entirely. The work of one firing, imperative, with no schedule restated in it.", }, cron: { type: "string", diff --git a/server/src/routines/run-turn.ts b/server/src/routines/run-turn.ts index 41ae85d5..3a62953d 100644 --- a/server/src/routines/run-turn.ts +++ b/server/src/routines/run-turn.ts @@ -304,6 +304,46 @@ function assistantText(message: Message): string | undefined { : undefined; } +/** + * The stored instruction, wrapped in the sentences that tell the turn it IS a firing. + * + * FOUND ON A LIVE FIRING, and it recorded `succeeded`. The instruction read "Every run, append the + * current date and time as a new bulleted list item to the Notion page …" and was sent to the model + * verbatim as the turn's user message. The model read it as a question about routine MANAGEMENT + * rather than as work: it called `list_routines`, found a routine that already said exactly that, + * answered that it was already configured, and appended nothing. Nothing failed, so nothing was + * reported — a routine telling somebody it is working while doing nothing at all, which is worse than + * one that breaks. + * + * And the model was not being stupid. Instructions are WRITTEN in schedule-speak — "every run", + * "every 15 minutes", "each morning" — because that is how a person asks for a standing thing, and + * schedule-shaped prose arriving out of nowhere reads as a request to SET UP a schedule. The most + * plausible reading of its own routine's text was "check whether this is set up"; it was, so it did + * nothing, successfully. No wording of the stored instruction fixes that on its own, because the + * sentence a person writes is the sentence that describes the schedule. + * + * So the frame says the three things the instruction cannot say about itself: that this is a + * scheduled firing happening now, that the work belongs in this turn, and that managing routines is + * not what was asked. It is PRESENTATION — which is why it lives here and not in the stored row or in + * {@link TurnRunner}'s signature: the row keeps what the person asked for, and this is how it is put + * to the model. + * + * ONLY THE NEW MESSAGE IS FRAMED, and that matters twice. The framed text is what + * `persistedInputMessages` writes to the transcript — correctly, since the transcript should show + * what the turn was actually asked — so it comes back as HISTORY on the next firing. History is + * converted and seeded exactly as the platform handed it over and nothing re-frames it; a test holds + * that, because the alternative is a message that grows a fresh paragraph of frame every night. + */ +export function frameFiring(instruction: string): string { + return [ + "One of your routines is firing right now, on its schedule, and this is that firing.", + "Carry out the instruction below in this turn: do the work now, then say what happened.", + "Do not create, list or change any routine unless the instruction itself asks you to.", + "", + instruction, + ].join("\n"); +} + export function createTurnRunner(options: { intelligence: IntelligenceLike; runner: RunnerLike; @@ -375,10 +415,15 @@ export function createTurnRunner(options: { ); const seeded = sanitizeSeededHistory(history.messages.map(toAgentMessage)); + /* + * This turn's own message — and the ONLY message that is framed. See {@link frameFiring} for the + * firing it did nothing on. The seeded history above is untouched, which is what keeps a previous + * firing's framed message (it persisted, so it is back here as history) from being framed twice. + */ const turn = { id: crypto.randomUUID(), role: "user", - content: instruction, + content: frameFiring(instruction), } as Message; const messages = [...seeded, turn]; diff --git a/server/tests/builtin-routines.test.ts b/server/tests/builtin-routines.test.ts index c0111f5e..e7e9757f 100644 --- a/server/tests/builtin-routines.test.ts +++ b/server/tests/builtin-routines.test.ts @@ -130,6 +130,21 @@ describe("the tool list", () => { expect(description).toContain("0 9 * * 1-5"); }); + test("tells the model how to phrase the instruction itself", async () => { + // A live firing did nothing and reported success because its instruction was written in + // schedule-speak ("Every run, append …"): the model read it as a question about scheduling, + // checked that the routine existed, and answered "already configured". The firing turn now says + // it is a firing (`routines/run-turn.ts`), and this is the other half — the authoring side, so + // the instruction is written as work in the first place. + const tools = await listTools(); + const create = tools.find((tool) => tool.name === "create_routine"); + const description = create?.description ?? ""; + expect(description).toContain("restate the schedule"); + + const update = tools.find((tool) => tool.name === "update_routine"); + expect(update?.description ?? "").toContain("restate the schedule"); + }); + test("needs no actor, no arguments and no store", async () => { // The only call site is `refreshTools`, which passes `{url, token}` and never an actor. A list // that refused without one would store zero tools and Routines would advertise nothing. diff --git a/server/tests/routine-run-turn.test.ts b/server/tests/routine-run-turn.test.ts index add0cab3..de658158 100644 --- a/server/tests/routine-run-turn.test.ts +++ b/server/tests/routine-run-turn.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test } from "bun:test"; import { EMPTY } from "rxjs"; import { createTurnRunner, + frameFiring, sanitizeSeededHistory, } from "../src/routines/run-turn"; @@ -27,6 +28,13 @@ const AGENT_ID = "bot_helper"; const THREAD_ID = "thread_owner_channel_1"; const INSTRUCTION = "Post the standup summary."; +/** + * A fragment of the firing frame that no instruction a person writes would contain by accident, and + * which every one of the frame's three jobs runs through. Asserted rather than the whole paragraph so + * the wording can be improved without rewriting the suite. + */ +const FRAME_MARK = "firing right now"; + type HistoryRow = { id: string; role: string; @@ -298,10 +306,10 @@ describe("a routine's headless turn", () => { }); // And a result row still points at the call it answers. expect(agent.messages[4]).toMatchObject({ toolCallId: "call_1" }); - expect(agent.messages[5]).toMatchObject({ - role: "user", - content: INSTRUCTION, - }); + expect(agent.messages[5]).toMatchObject({ role: "user" }); + // The turn's own message carries the instruction, framed as a firing — see the describe below. + expect(agent.messages[5]?.content).toContain(INSTRUCTION); + expect(agent.messages[5]?.content).toContain(FRAME_MARK); }); test("a thread the platform has never heard of reads as no history", async () => { @@ -313,6 +321,79 @@ describe("a routine's headless turn", () => { }); }); +/** + * The turn has to know it IS a firing. + * + * FOUND ON A LIVE FIRING. The instruction read "Every run, append the current date and time as a new + * bulleted list item to the Notion page …" and went to the model verbatim. The model read + * schedule-shaped prose as a request about SCHEDULING: it called `list_routines`, found a routine + * that already said that, replied "already configured", and appended nothing. The firing recorded + * `succeeded` having done nothing — a routine reporting that it works while doing nothing at all, + * which is worse than one that fails. + */ +describe("the turn's message is framed as a firing happening now", () => { + test("carries the instruction and the frame around it", async () => { + const { run, calls } = harness({ history: THREE_ROWS }); + + await run(); + + const seeded = calls.runs[0]?.input.messages ?? []; + const turn = seeded[seeded.length - 1] as { content?: unknown }; + expect(typeof turn.content).toBe("string"); + const content = String(turn.content); + // The instruction survives whole — the frame wraps it, it does not rewrite it. + expect(content).toContain(INSTRUCTION); + expect(content).toContain(FRAME_MARK); + // And the three things the bare instruction could not say. + expect(content.toLowerCase()).toContain("schedule"); + expect(content.toLowerCase()).toContain("this turn"); + expect(content).toContain("routine"); + }); + + test("the framed message is what persists, so the transcript shows what was asked", async () => { + const { run, calls } = harness({ history: THREE_ROWS }); + + await run(); + + const [request] = calls.runs; + expect(request?.persistedInputMessages).toHaveLength(1); + const persisted = String(request?.persistedInputMessages?.[0]?.content); + expect(persisted).toContain(FRAME_MARK); + expect(persisted).toContain(INSTRUCTION); + }); + + test("a prior firing's framed message, arriving back as history, is not framed again", async () => { + // The framed text persists, so the NEXT firing reads it back as history. Only the new message is + // framed; history is seeded exactly as the platform handed it over. Without that, an instruction + // would grow a fresh paragraph of frame on every single firing until the turn is mostly frame. + const alreadyFramed = frameFiring( + "Append the current UTC time to the log page.", + ); + const { run, calls } = harness({ + history: [ + { id: "m1", role: "user", content: alreadyFramed }, + { id: "m2", role: "assistant", content: "Appended." }, + ], + }); + + await run(); + + const seeded = calls.runs[0]?.input.messages ?? []; + expect(seeded).toHaveLength(3); + // Byte for byte what came out of the platform. + expect((seeded[0] as { content?: unknown }).content).toBe(alreadyFramed); + // And exactly one frame in it, not two. + const occurrences = + String((seeded[0] as { content?: unknown }).content).split(FRAME_MARK) + .length - 1; + expect(occurrences).toBe(1); + // The new message is the only framed one this turn added. + expect(String((seeded[2] as { content?: unknown }).content)).toBe( + frameFiring(INSTRUCTION), + ); + }); +}); + /** * The seeded history has to be a conversation the model API will ACCEPT, and a thread that was once * interrupted mid-tool-call is not one. @@ -464,7 +545,9 @@ describe("persistedInputMessages is the subtraction", () => { const [request] = calls.runs; expect(request?.input.messages).toHaveLength(4); expect(request?.persistedInputMessages).toHaveLength(1); - expect(request?.persistedInputMessages?.[0]?.content).toBe(INSTRUCTION); + expect(request?.persistedInputMessages?.[0]?.content).toContain( + INSTRUCTION, + ); // Identified by id, not by position: none of the history's ids may appear. const historic = new Set(THREE_ROWS.map((row) => row.id)); for (const message of request?.persistedInputMessages ?? []) { @@ -494,7 +577,9 @@ describe("persistedInputMessages is the subtraction", () => { // Three seeded rows survive the sanitation, plus this turn's instruction. expect(request?.input.messages).toHaveLength(4); expect(request?.persistedInputMessages).toHaveLength(1); - expect(request?.persistedInputMessages?.[0]?.content).toBe(INSTRUCTION); + expect(request?.persistedInputMessages?.[0]?.content).toContain( + INSTRUCTION, + ); }); test("an empty history persists everything", async () => { From bcf1be1224d9658b9b6e750b776864353527462c Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 10:35:40 -0300 Subject: [PATCH 39/45] Take the attention inbox out of this branch, keeping its applied migration Reverts e8aa344 (#255) code on guido/routines only: the API routes, store, view, schema, app wiring, sidebar entry and /attention page go; the 0020 migration, journal entry and snapshot stay, because the local database has already applied them and an unused table is cheaper than a broken chain. The routine endpoint test loses the positional attentionStore slot it had gained for main's signature. --- CHANGELOG.md | 16 --- .../components/app-sidebar/app-sidebar.tsx | 34 ----- app/src/lib/attention/mutations.ts | 27 ---- app/src/lib/attention/queries.ts | 34 ----- app/src/routeTree.gen.ts | 21 --- app/src/routes/_authed/_app/attention.tsx | 130 ------------------ server/drizzle.config.ts | 1 - server/src/app.ts | 25 ---- server/src/attention/routes.ts | 88 ------------ server/src/attention/store.ts | 106 -------------- server/src/attention/view.ts | 110 --------------- server/src/db/schema/attention.ts | 39 ------ server/src/db/schema/index.ts | 1 - server/src/index.ts | 3 - .../tests/attention-store.integration.test.ts | 56 -------- server/tests/attention-view.test.ts | 115 ---------------- server/tests/routine-endpoint.test.ts | 1 - 17 files changed, 807 deletions(-) delete mode 100644 app/src/lib/attention/mutations.ts delete mode 100644 app/src/lib/attention/queries.ts delete mode 100644 app/src/routes/_authed/_app/attention.tsx delete mode 100644 server/src/attention/routes.ts delete mode 100644 server/src/attention/store.ts delete mode 100644 server/src/attention/view.ts delete mode 100644 server/src/db/schema/attention.ts delete mode 100644 server/tests/attention-store.integration.test.ts delete mode 100644 server/tests/attention-view.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1edeebe5..bd28b9a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,22 +28,6 @@ than accepting one it cannot attribute. `scripts/start.sh` runs the worker local turns it on with `routines.enabled` and takes the secret as `secrets.workerSharedSecret`. No new port is opened for any of this — the worker only ever calls out to the server it already trusts. -### A Bot in trouble no longer needs somebody watching - -A boundary refusal or a stalled run was recorded and then waited for a person to happen to look — at -the right channel, or at the audit page an administrator has and nobody else does. The trail knew; -nobody was told. - -**Attention**, in the sidebar for everybody, shows the refusals and stalled runs nobody has handled -yet, scoped to the Bots this person may use, with a badge saying how many. Marking one handled -clears it for everyone and records who did; two people pressing Resolve at once is settled by the -database rather than by luck, and the second is told who got there first. - -It is a view over the trail, not a second record of it. Refusals and stalls are already written -transactionally by the gateway and the stall guard, so the inbox cannot miss one and nothing new -runs on the action path. The only state it owns is the resolution, held beside the append-only trail -rather than in it. The trail itself still keeps everything; the inbox is only what is open now. - ### A channel a Bot has spoken in unseen shows a dot The sidebar marks a channel when a Bot has said something since you last had it open: a dot beside diff --git a/app/src/components/app-sidebar/app-sidebar.tsx b/app/src/components/app-sidebar/app-sidebar.tsx index 3bf10b10..c512ed97 100644 --- a/app/src/components/app-sidebar/app-sidebar.tsx +++ b/app/src/components/app-sidebar/app-sidebar.tsx @@ -1,5 +1,4 @@ import { - IconBellRinging, IconBolt, IconBox, IconClock, @@ -47,7 +46,6 @@ import { SidebarRail, } from "@/components/ui/sidebar"; import { signOutMutationOptions } from "@/lib/auth/mutations"; -import { attentionListQueryOptions } from "@/lib/attention/queries"; import { currentUserQueryOptions } from "@/lib/auth/queries"; import { type ChannelSummary, @@ -207,9 +205,6 @@ function ChannelRow({ export function AppSidebar({ ...props }: React.ComponentProps) { const { data: currentUser } = useQuery(currentUserQueryOptions()); - // Unhandled attention items this person may see; drawn as a badge only when nonzero. - const attentionCount = - useQuery(attentionListQueryOptions()).data?.length ?? 0; const queryClient = useQueryClient(); const navigate = useNavigate(); const signOut = useMutation(signOutMutationOptions(queryClient)); @@ -327,35 +322,6 @@ export function AppSidebar({ ...props }: React.ComponentProps) { - - {/* - * Above Skills because it is the row that can be urgent. The count is the number of - * unhandled items this person may see; zero draws no badge, because an empty inbox - * asking for attention is the boy who cried wolf. - */} - ( - - )} - > -
- -
- Attention - {attentionCount > 0 ? ( - - {attentionCount} - - ) : null} - - {/* Beside Agents rather than inside Admin: writing a skill is something anybody does. */} => - client( - `/api/attention/${encodeURIComponent(eventId)}/resolve`, - "resolution", - { - method: "POST", - fallback: "The item could not be marked handled.", - }, - ), - onSuccess: () => - queryClient.invalidateQueries({ queryKey: attentionKeys.all }), - }); -} diff --git a/app/src/lib/attention/queries.ts b/app/src/lib/attention/queries.ts deleted file mode 100644 index 5f83a325..00000000 --- a/app/src/lib/attention/queries.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { queryOptions } from "@tanstack/react-query"; -import { client } from "@/lib/client"; - -/** One trail row that means a Bot is waiting on a person. */ -export type AttentionItem = { - /** The trail row's id; resolving cites the exact row. */ - id: string; - kind: "refused" | "tool_rejected" | "stalled"; - botId: string; - at: string; - /** One sentence a person can act on, written by whatever recorded the row. */ - sentence: string; -}; - -export const attentionKeys = { - all: ["attention"] as const, - list: () => ["attention", "list"] as const, -}; - -/** - * Polled the way grants are: often enough that a Bot in trouble is noticed inside a minute, and - * refetched on focus so coming back to the tab answers immediately. - */ -export function attentionListQueryOptions() { - return queryOptions({ - queryKey: attentionKeys.list(), - refetchInterval: 15_000, - refetchOnWindowFocus: true, - queryFn: (): Promise => - client("/api/attention", "items", { - fallback: "The attention list could not be loaded.", - }), - }); -} diff --git a/app/src/routeTree.gen.ts b/app/src/routeTree.gen.ts index 8d25b683..2ca64fab 100644 --- a/app/src/routeTree.gen.ts +++ b/app/src/routeTree.gen.ts @@ -15,7 +15,6 @@ import { Route as AuthedAppRouteImport } from './routes/_authed/_app' import { Route as AuthedAdminRouteRouteImport } from './routes/_authed/admin/route' import { Route as AuthedSettingsRouteRouteImport } from './routes/_authed/settings/route' import { Route as AuthedAppIndexRouteImport } from './routes/_authed/_app/index' -import { Route as AuthedAppAttentionRouteImport } from './routes/_authed/_app/attention' import { Route as AuthedAppBotRouteImport } from './routes/_authed/_app/bot' import { Route as AuthedAppRoutinesRouteImport } from './routes/_authed/_app/routines' import { Route as AuthedAppSkillsRouteImport } from './routes/_authed/_app/skills' @@ -70,11 +69,6 @@ const AuthedAppIndexRoute = AuthedAppIndexRouteImport.update({ path: '/', getParentRoute: () => AuthedAppRoute, } as any) -const AuthedAppAttentionRoute = AuthedAppAttentionRouteImport.update({ - id: '/attention', - path: '/attention', - getParentRoute: () => AuthedAppRoute, -} as any) const AuthedAppBotRoute = AuthedAppBotRouteImport.update({ id: '/bot', path: '/bot', @@ -215,7 +209,6 @@ export interface FileRoutesByFullPath { '/sign': typeof SignRoute '/admin': typeof AuthedAdminRouteRouteWithChildren '/settings': typeof AuthedSettingsRouteRouteWithChildren - '/attention': typeof AuthedAppAttentionRoute '/bot': typeof AuthedAppBotRoute '/routines': typeof AuthedAppRoutinesRoute '/skills': typeof AuthedAppSkillsRoute @@ -245,7 +238,6 @@ export interface FileRoutesByFullPath { export interface FileRoutesByTo { '/': typeof AuthedAppIndexRoute '/sign': typeof SignRoute - '/attention': typeof AuthedAppAttentionRoute '/bot': typeof AuthedAppBotRoute '/routines': typeof AuthedAppRoutinesRoute '/skills': typeof AuthedAppSkillsRoute @@ -279,7 +271,6 @@ export interface FileRoutesById { '/_authed/admin': typeof AuthedAdminRouteRouteWithChildren '/_authed/settings': typeof AuthedSettingsRouteRouteWithChildren '/_authed/_app': typeof AuthedAppRouteWithChildren - '/_authed/_app/attention': typeof AuthedAppAttentionRoute '/_authed/_app/bot': typeof AuthedAppBotRoute '/_authed/_app/routines': typeof AuthedAppRoutinesRoute '/_authed/_app/skills': typeof AuthedAppSkillsRoute @@ -314,7 +305,6 @@ export interface FileRouteTypes { | '/sign' | '/admin' | '/settings' - | '/attention' | '/bot' | '/routines' | '/skills' @@ -344,7 +334,6 @@ export interface FileRouteTypes { to: | '/' | '/sign' - | '/attention' | '/bot' | '/routines' | '/skills' @@ -377,7 +366,6 @@ export interface FileRouteTypes { | '/_authed/admin' | '/_authed/settings' | '/_authed/_app' - | '/_authed/_app/attention' | '/_authed/_app/bot' | '/_authed/_app/routines' | '/_authed/_app/skills' @@ -455,13 +443,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthedAppIndexRouteImport parentRoute: typeof AuthedAppRoute } - '/_authed/_app/attention': { - id: '/_authed/_app/attention' - path: '/attention' - fullPath: '/attention' - preLoaderRoute: typeof AuthedAppAttentionRouteImport - parentRoute: typeof AuthedAppRoute - } '/_authed/_app/bot': { id: '/_authed/_app/bot' path: '/bot' @@ -701,7 +682,6 @@ const AuthedSettingsRouteRouteWithChildren = AuthedSettingsRouteRoute._addFileChildren(AuthedSettingsRouteRouteChildren) interface AuthedAppRouteChildren { - AuthedAppAttentionRoute: typeof AuthedAppAttentionRoute AuthedAppBotRoute: typeof AuthedAppBotRoute AuthedAppRoutinesRoute: typeof AuthedAppRoutinesRoute AuthedAppSkillsRoute: typeof AuthedAppSkillsRoute @@ -712,7 +692,6 @@ interface AuthedAppRouteChildren { } const AuthedAppRouteChildren: AuthedAppRouteChildren = { - AuthedAppAttentionRoute: AuthedAppAttentionRoute, AuthedAppBotRoute: AuthedAppBotRoute, AuthedAppRoutinesRoute: AuthedAppRoutinesRoute, AuthedAppSkillsRoute: AuthedAppSkillsRoute, diff --git a/app/src/routes/_authed/_app/attention.tsx b/app/src/routes/_authed/_app/attention.tsx deleted file mode 100644 index f8fba2dd..00000000 --- a/app/src/routes/_authed/_app/attention.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import { useMutation, useQuery } from "@tanstack/react-query"; -import { createFileRoute, Link } from "@tanstack/react-router"; -import { - IconAlertTriangle, - IconHandStop, - IconPlugOff, -} from "@tabler/icons-react"; -import { - PageRows, - PageSection, - PageShell, -} from "@/components/layout/page-shell"; -import { Button } from "@/components/ui/button"; -import { - Item, - ItemActions, - ItemContent, - ItemDescription, - ItemMedia, - ItemTitle, -} from "@/components/ui/item"; -import { Separator } from "@/components/ui/separator"; -import type { AttentionItem } from "@/lib/attention/queries"; -import { attentionListQueryOptions } from "@/lib/attention/queries"; -import { resolveAttentionMutationOptions } from "@/lib/attention/mutations"; -import { queryClient } from "@/query-client"; - -/** - * What is waiting on a person: boundary refusals and stalled runs, drawn from the trail and gone - * once somebody marks them handled. The trail itself keeps everything; this page is only what is - * open now. - */ - -export const Route = createFileRoute("/_authed/_app/attention")({ - component: AttentionPage, -}); - -const KIND_WORDS: Record = { - refused: "Action refused", - tool_rejected: "Tool call refused", - stalled: "Run stalled", -}; - -function KindIcon({ kind }: { kind: AttentionItem["kind"] }) { - if (kind === "stalled") return ; - if (kind === "tool_rejected") return ; - return ; -} - -function AttentionPage() { - const items = useQuery(attentionListQueryOptions()); - const resolve = useMutation(resolveAttentionMutationOptions(queryClient)); - - return ( - - Refusals and stalled runs that nobody has handled yet. Everything here - is already recorded in the trail; marking an item handled clears it - for everyone and says who did. - - } - title="Attention" - > - - {items.isPending ? null : items.error ? ( -

- The attention list could not be loaded. -

- ) : items.data?.length === 0 ? ( -

- Nothing is waiting. A refusal or a stalled run will appear here. -

- ) : ( - - {items.data?.map((item, index) => ( -
- {index > 0 ? : null} - - - - - - - {KIND_WORDS[item.kind]} · {item.botId} - - - {item.sentence}{" "} - - {new Date(item.at).toLocaleString()} - - - - - - - - -
- ))} - - )} - {resolve.error ? ( -

- {resolve.error.message} -

- ) : null} - - - ); -} diff --git a/server/drizzle.config.ts b/server/drizzle.config.ts index 66cc7440..a0fdf026 100644 --- a/server/drizzle.config.ts +++ b/server/drizzle.config.ts @@ -23,7 +23,6 @@ export default defineConfig({ "./src/db/schema/coworker.ts", "./src/db/schema/components.ts", "./src/db/schema/plugins.ts", - "./src/db/schema/attention.ts", "./src/db/schema/work.ts", ], out: "./drizzle", diff --git a/server/src/app.ts b/server/src/app.ts index e418714d..5a13148c 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -31,8 +31,6 @@ import { createSandboxedRoutes } from "./components/sandboxed-routes"; import type { ComponentStore } from "./components/store"; import type { ComputerGateway } from "./computer/gateway"; import type { PolicyStore } from "./computer/policy-store"; -import { createAttentionRoutes } from "./attention/routes"; -import type { AttentionStore } from "./attention/store"; import { createComputerRoutes } from "./computer/routes"; import type { PageFrameStore } from "./computer/page-frames"; import { configuredAuthProviders, type DeploymentConfig } from "./config"; @@ -160,12 +158,6 @@ export function createApp( * the default coworker, which is exactly the failsafe the router itself falls back to. */ intentRouter?: IntentRouter, - /** - * Resolutions for the attention inbox. Built beside the other stores in index.ts. Absent leaves - * the inbox unmounted rather than degraded: an inbox that cannot subtract what has been handled - * would show everything forever, and one that cannot see the trail would show a false all-quiet. - */ - attentionStore?: AttentionStore, /** * Where the frame a browsing turn ended on is kept. * @@ -757,23 +749,6 @@ export function createApp( ); } - /* - * The attention inbox: what on the trail is waiting for a person. Needs the trail to read and the - * database for resolutions, and without either it is not mounted — an inbox that cannot see - * refusals is not a reduced feature, it is a false "all quiet". - */ - if (auditReader && attentionStore) { - app.route( - "/api/attention", - createAttentionRoutes( - auditReader, - attentionStore, - requireUser, - canUseBot, - ), - ); - } - if (agentProfileStore) { app.route( "/api/agents", diff --git a/server/src/attention/routes.ts b/server/src/attention/routes.ts deleted file mode 100644 index 68652fcb..00000000 --- a/server/src/attention/routes.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * The attention inbox over HTTP. - * - * Deliberately not `/api/admin/...`: the audit page is the administrator's view of everything, and - * is gated accordingly. The inbox is the working person's view of their own Bots' trouble, so it is - * scoped per item by the same `canUseBot` the rest of the surface uses, and an administrator sees - * all of it the way they see all Bots. - */ - -import { Hono } from "hono"; -import type { MiddlewareHandler } from "hono"; -import type { BotAccessCheck } from "../agents/profile-policy"; -import type { AuditReader } from "../audit"; -import type { AppVariables } from "../auth/guards"; -import type { AttentionStore } from "./store"; -import { ATTENTION_EVENT_TYPES, attentionItemsFrom, botOf } from "./view"; - -/** - * How much trail the view reads. Bounded and biased to recency for the same reason the policy - * dry-run is: the inbox answers "what needs me now", and an unresolved refusal from beyond this - * window is answered by the Audit page, which exists for looking back. - */ -const SCAN_LIMIT = 200; - -export function createAttentionRoutes( - auditReader: AuditReader, - store: AttentionStore, - requireUser: MiddlewareHandler<{ Variables: AppVariables }>, - canUseBot: BotAccessCheck, -) { - const routes = new Hono<{ Variables: AppVariables }>(); - - routes.get("/", requireUser, async (context) => { - const { events } = await auditReader.list({ - limit: SCAN_LIMIT, - eventType: ATTENTION_EVENT_TYPES.join(","), - }); - const resolved = await store.resolvedAmong(events.map((one) => one.id)); - const items = attentionItemsFrom(events, resolved); - - if (context.var.actor.role === "admin") { - return context.json({ items }); - } - /* - * Scoped per item rather than per request: one inbox can name several Bots, and which of them - * this person may see is the store's question, asked with the same check the roster and the - * computer use. Sequential on purpose — the distinct Bots in a 200-row window are few, and the - * memo keeps it to one ask per Bot. - */ - const allowed = new Map(); - const visible = []; - for (const item of items) { - let may = allowed.get(item.botId); - if (may === undefined) { - may = await canUseBot(context.var.actor, item.botId); - allowed.set(item.botId, may); - } - if (may) visible.push(item); - } - return context.json({ items: visible }); - }); - - routes.post("/:eventId/resolve", requireUser, async (context) => { - const eventId = context.req.param("eventId"); - const event = await store.event(eventId); - /* - * Only a real attention row can be resolved. Anything else — a made-up id, a trail row of some - * other kind — answers the same way, so this cannot be used to probe what the trail holds. - */ - const kinds: readonly string[] = ATTENTION_EVENT_TYPES; - if (!event || !kinds.includes(event.eventType)) { - return context.json({ error: "There is no such attention item." }, 404); - } - // The same reading the view uses: a tool rejection's target is the tool, not the Bot. - const botId = botOf(event); - if ( - context.var.actor.role !== "admin" && - !(botId && (await canUseBot(context.var.actor, botId))) - ) { - return context.json({ error: "There is no such attention item." }, 404); - } - - const resolution = await store.resolve(eventId, context.var.actor.id); - return context.json({ resolution }); - }); - - return routes; -} diff --git a/server/src/attention/store.ts b/server/src/attention/store.ts deleted file mode 100644 index feebbb5c..00000000 --- a/server/src/attention/store.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * The one piece of state the inbox owns: which trail rows a person has marked handled. - */ - -import { eq, inArray } from "drizzle-orm"; -import type { Database } from "../db/client"; -import { attentionResolutions, auditEvents } from "../db/schema"; - -export type AttentionResolution = { - auditEventId: string; - resolvedBy: string; - resolvedAt: string; -}; - -export type AttentionStore = { - /** The resolved ids among these, for subtracting from the view. */ - resolvedAmong(eventIds: string[]): Promise>; - /** The trail row itself, so a resolve can check what it is resolving. */ - event(eventId: string): Promise<{ - id: string; - eventType: string; - targetType: string; - targetId: string | null; - payload: Record; - } | null>; - /** - * Mark handled. First writer wins by unique index — not by check-then-write — so two replicas - * cannot both believe they resolved it. Returns the resolution that stands, and whether this call - * is the one that wrote it. - */ - resolve( - eventId: string, - userId: string, - ): Promise; -}; - -export function createAttentionStore(database: Database): AttentionStore { - return { - resolvedAmong: async (eventIds) => { - if (eventIds.length === 0) return new Set(); - const rows = await database - .select({ auditEventId: attentionResolutions.auditEventId }) - .from(attentionResolutions) - .where(inArray(attentionResolutions.auditEventId, eventIds)); - return new Set(rows.map((row) => row.auditEventId)); - }, - - event: async (eventId) => { - const rows = await database - .select({ - id: auditEvents.id, - eventType: auditEvents.eventType, - targetType: auditEvents.targetType, - targetId: auditEvents.targetId, - payload: auditEvents.payload, - }) - .from(auditEvents) - .where(eq(auditEvents.id, eventId)) - .limit(1); - const row = rows[0]; - return row - ? { ...row, payload: row.payload as Record } - : null; - }, - - resolve: async (eventId, userId) => { - const inserted = await database - .insert(attentionResolutions) - .values({ auditEventId: eventId, resolvedBy: userId }) - .onConflictDoNothing({ target: attentionResolutions.auditEventId }) - .returning({ - auditEventId: attentionResolutions.auditEventId, - resolvedBy: attentionResolutions.resolvedBy, - resolvedAt: attentionResolutions.resolvedAt, - }); - const row = inserted[0]; - if (row) { - return { - auditEventId: row.auditEventId, - resolvedBy: row.resolvedBy, - resolvedAt: row.resolvedAt.toISOString(), - alreadyResolved: false, - }; - } - // The conflict path: somebody got there first. Read back who, which is the answer the second - // presser actually wants. - const standing = await database - .select() - .from(attentionResolutions) - .where(eq(attentionResolutions.auditEventId, eventId)) - .limit(1); - const existing = standing[0]; - if (!existing) { - // Conflict on insert and absent on read means a concurrent resolve was rolled back between - // the two statements. Vanishingly rare; the honest report is a retryable failure. - throw new Error("The resolution could not be read back. Try again."); - } - return { - auditEventId: existing.auditEventId, - resolvedBy: existing.resolvedBy, - resolvedAt: existing.resolvedAt.toISOString(), - alreadyResolved: true, - }; - }, - }; -} diff --git a/server/src/attention/view.ts b/server/src/attention/view.ts deleted file mode 100644 index 3525e975..00000000 --- a/server/src/attention/view.ts +++ /dev/null @@ -1,110 +0,0 @@ -/** - * The attention inbox: the trail rows that mean a Bot is waiting on a person, minus the ones a - * person has already handled. - * - * A view, not a store. Refusals and stalls are already recorded transactionally by the gateway and - * the stall guard, so deriving the inbox from those rows means it cannot miss one: there is no - * second write to forget, and nothing here runs on the action path. The only state the inbox owns - * is the resolution — who marked a row handled, and when — which lives beside the trail rather than - * in it, because the trail is append-only and must stay that way. - */ - -import type { AuditEvent } from "../audit"; - -/** The trail rows that mean "a Bot is waiting on a person". In one place, for the query. */ -export const ATTENTION_EVENT_TYPES = [ - "computer.action_refused", - "mcp.call_rejected", - "agent.stream_stalled", -] as const; - -export type AttentionKind = "refused" | "tool_rejected" | "stalled"; - -export type AttentionItem = { - /** The trail row's own id: resolving cites the exact row, not a copy of it. */ - id: string; - kind: AttentionKind; - botId: string; - at: string; - /** One sentence a person can act on, built from what the trail recorded. */ - sentence: string; -}; - -const KIND_BY_EVENT_TYPE: Record = { - "computer.action_refused": "refused", - "mcp.call_rejected": "tool_rejected", - "agent.stream_stalled": "stalled", -}; - -const text = (value: unknown): string => - typeof value === "string" ? value : ""; - -/** - * The Bot a row is about. The three event types write it differently, and the difference is not - * cosmetic: the computer and the stall guard put the Bot in `targetId`, but a tool rejection's - * target is the TOOL — `targetType: "mcp_tool"`, `targetId` the ref — and its Bot travels only in - * the payload. Reading `targetId` unconditionally made the inbox call a refusal's Bot - * "google-drive/search_files", which is not a Bot, which `canUseBot` correctly denies, which hid - * every tool rejection from exactly the person it was for. - */ -export function botOf( - event: Pick, -): string { - if (event.targetType === "computer" || event.targetType === "agent") { - return event.targetId ?? text(event.payload.bot); - } - return text(event.payload.bot); -} - -/** What to tell the person. Prefers the sentence the recording code already wrote for one. */ -function sentenceFor(event: AuditEvent, kind: AttentionKind): string { - const payload = event.payload; - if (kind === "stalled") { - return "The Bot's stream went quiet mid-turn and the run was ended."; - } - if (kind === "tool_rejected") { - // The rejection records the decision's own reason, written for a person. Use it whole. - const reason = text(payload.reason); - if (reason) return reason; - const tool = text(payload.tool) || "a tool"; - return `A call to ${tool} was refused by this deployment's boundary.`; - } - // A computer refusal records the gateway's own reason, written for a person. Use it whole. - const decision = - payload.decision && typeof payload.decision === "object" - ? (payload.decision as Record) - : null; - const reason = text(decision?.reason); - if (reason) return reason; - const action = text(payload.action) || "an action"; - return `${action} was refused by this deployment's boundary.`; -} - -/** - * Compose the inbox from trail rows and the set of resolved row ids. - * - * Rows without a Bot are dropped rather than guessed at: an item that cannot say whose trouble it - * is cannot be scoped to a person, and showing it to everybody would leak across the same line - * `canUseBot` exists to hold. - */ -export function attentionItemsFrom( - events: AuditEvent[], - resolvedEventIds: ReadonlySet, -): AttentionItem[] { - const items: AttentionItem[] = []; - for (const event of events) { - const kind = KIND_BY_EVENT_TYPE[event.eventType]; - if (!kind) continue; - if (resolvedEventIds.has(event.id)) continue; - const botId = botOf(event); - if (!botId) continue; - items.push({ - id: event.id, - kind, - botId, - at: event.createdAt, - sentence: sentenceFor(event, kind), - }); - } - return items; -} diff --git a/server/src/db/schema/attention.ts b/server/src/db/schema/attention.ts deleted file mode 100644 index 0a90e61e..00000000 --- a/server/src/db/schema/attention.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { - pgTable, - text, - timestamp, - uniqueIndex, - uuid, -} from "drizzle-orm/pg-core"; - -/** - * A trail row somebody has marked handled. - * - * The attention inbox is a view over the audit trail — refusals and stalls are already recorded - * there, transactionally, by the gateway and the stall guard. What the trail cannot say is that a - * person has dealt with one, because the trail is append-only and must stay that way. So resolution - * is state ABOUT a trail row, held beside it: the row itself is never touched. - * - * Ids by value, no foreign keys, for the trail's own reason (see `actorUserId` on `audit_events`): - * a cascade against an append-only table is an update the trigger refuses, and a person who had - * ever resolved anything could otherwise never be deleted. - */ -export const attentionResolutions = pgTable( - "attention_resolutions", - { - id: uuid("id").primaryKey().defaultRandom(), - auditEventId: uuid("audit_event_id").notNull(), - resolvedBy: text("resolved_by").notNull(), - resolvedAt: timestamp("resolved_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (table) => [ - /** - * One resolution per trail row, enforced where two replicas cannot disagree about it. Two - * people pressing Resolve at once is not a race to be lost: the second insert conflicts, and - * the caller reads back who got there first. - */ - uniqueIndex("attention_resolutions_event_idx").on(table.auditEventId), - ], -); diff --git a/server/src/db/schema/index.ts b/server/src/db/schema/index.ts index 7755b8dc..68ee5d83 100644 --- a/server/src/db/schema/index.ts +++ b/server/src/db/schema/index.ts @@ -1,6 +1,5 @@ /** One import path for every table, with schema files grouped by owner. */ -export * from "./attention"; export * from "./components"; export * from "./computer"; export * from "./core"; diff --git a/server/src/index.ts b/server/src/index.ts index 4ad296c1..15104de5 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -52,7 +52,6 @@ import { createCredentialStore, resolveModelApiKey, } from "./credentials"; -import { createAttentionStore } from "./attention/store"; import { createDatabase } from "./db/client"; import { createPeopleStore } from "./people/store"; import { useRoutineTools } from "./plugins/builtin-routines"; @@ -711,8 +710,6 @@ const app = createApp( identityProviderStore, // Chooses the coworker for an untagged message, on the deployment's own model and key. intentRouter, - // Resolutions for the attention inbox: which trail rows a person has marked handled. - createAttentionStore(database), // What a browsing turn's screen looked like when it finished, so the transcript can show it later. createPageFrameStore(database), // What a due routine actually does: a turn, run as its owner, into the thread they will open. diff --git a/server/tests/attention-store.integration.test.ts b/server/tests/attention-store.integration.test.ts deleted file mode 100644 index 2983a33e..00000000 --- a/server/tests/attention-store.integration.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { afterAll, describe, expect, test } from "bun:test"; -import { randomUUID } from "node:crypto"; -import { inArray } from "drizzle-orm"; -import { createAttentionStore } from "../src/attention/store"; -import { createDatabase } from "../src/db/client"; -import { attentionResolutions } from "../src/db/schema"; -import { TEST_POOL } from "./support/database"; - -const databaseUrl = - process.env.DATABASE_URL ?? - "postgres://openbot:openbot@localhost:5432/openbot"; -const database = createDatabase(databaseUrl, TEST_POOL); -const store = createAttentionStore(database); - -/** Rows this file wrote, removed on the way out. Resolutions are not the trail; they may be. */ -const written: string[] = []; - -afterAll(async () => { - if (written.length > 0) { - await database - .delete(attentionResolutions) - .where(inArray(attentionResolutions.auditEventId, written)); - } - await database.$client.close(); -}); - -describe("attention resolutions", () => { - test("the first resolver wins and the second is told who did", async () => { - const eventId = randomUUID(); - written.push(eventId); - - const first = await store.resolve(eventId, "person-a"); - expect(first.alreadyResolved).toBe(false); - expect(first.resolvedBy).toBe("person-a"); - - // The race, replayed: the unique index answers, not a check-then-write. - const second = await store.resolve(eventId, "person-b"); - expect(second.alreadyResolved).toBe(true); - expect(second.resolvedBy).toBe("person-a"); - }); - - test("resolvedAmong answers exactly the resolved subset", async () => { - const resolved = randomUUID(); - const pending = randomUUID(); - written.push(resolved); - - await store.resolve(resolved, "person-a"); - const answer = await store.resolvedAmong([resolved, pending]); - expect(answer.has(resolved)).toBe(true); - expect(answer.has(pending)).toBe(false); - }); - - test("an empty ask does not touch the database", async () => { - expect((await store.resolvedAmong([])).size).toBe(0); - }); -}); diff --git a/server/tests/attention-view.test.ts b/server/tests/attention-view.test.ts deleted file mode 100644 index 4ec901e9..00000000 --- a/server/tests/attention-view.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { AuditEvent } from "../src/audit"; -import { attentionItemsFrom } from "../src/attention/view"; - -/** - * The inbox must show exactly the trail rows that mean "a Bot is waiting on a person", minus what - * has been handled — and must drop rather than guess when a row cannot say whose trouble it is. - */ - -function event(overrides: Partial): AuditEvent { - return { - id: overrides.id ?? "evt-1", - actorUserId: null, - eventType: overrides.eventType ?? "computer.action_refused", - targetType: overrides.targetType ?? "computer", - targetId: - overrides.targetId === undefined - ? "general-assistant" - : overrides.targetId, - payload: overrides.payload ?? {}, - createdAt: overrides.createdAt ?? "2026-08-25T00:00:00.000Z", - }; -} - -describe("attentionItemsFrom", () => { - test("a computer refusal carries the gateway's own reason, whole", () => { - const items = attentionItemsFrom( - [ - event({ - payload: { - decision: { reason: "“Submit order” on shop.example is blocked." }, - }, - }), - ], - new Set(), - ); - expect(items).toHaveLength(1); - expect(items[0]?.kind).toBe("refused"); - expect(items[0]?.sentence).toBe( - "“Submit order” on shop.example is blocked.", - ); - expect(items[0]?.botId).toBe("general-assistant"); - }); - - test("a tool rejection's Bot is the payload's, never the tool ref in targetId", () => { - // The real row shape: targetType mcp_tool, targetId the tool ref, the Bot in the payload. - const items = attentionItemsFrom( - [ - event({ - eventType: "mcp.call_rejected", - targetType: "mcp_tool", - targetId: "jira/jira_create_issue", - payload: { - bot: "risk-analyst", - tool: "jira_create_issue", - reason: "This Bot holds no grant for that tool.", - }, - }), - ], - new Set(), - ); - expect(items[0]?.kind).toBe("tool_rejected"); - expect(items[0]?.botId).toBe("risk-analyst"); - // The decision's own sentence, whole. - expect(items[0]?.sentence).toBe("This Bot holds no grant for that tool."); - }); - - test("a tool rejection without a payload Bot is dropped, not attributed to the tool", () => { - const items = attentionItemsFrom( - [ - event({ - eventType: "mcp.call_rejected", - targetType: "mcp_tool", - targetId: "jira/jira_create_issue", - payload: { tool: "jira_create_issue" }, - }), - ], - new Set(), - ); - expect(items).toHaveLength(0); - }); - - test("a stall is one plain sentence", () => { - const items = attentionItemsFrom( - [event({ eventType: "agent.stream_stalled", targetType: "agent" })], - new Set(), - ); - expect(items[0]?.kind).toBe("stalled"); - expect(items[0]?.sentence).toContain("quiet"); - }); - - test("a resolved row is subtracted", () => { - const items = attentionItemsFrom( - [event({ id: "handled" }), event({ id: "pending" })], - new Set(["handled"]), - ); - expect(items.map((item) => item.id)).toEqual(["pending"]); - }); - - test("a row that cannot name its Bot is dropped, not shown to everybody", () => { - const items = attentionItemsFrom( - [event({ targetId: null, payload: {} })], - new Set(), - ); - expect(items).toHaveLength(0); - }); - - test("event types outside the three are ignored even if handed in", () => { - const items = attentionItemsFrom( - [event({ eventType: "computer.action_allowed" })], - new Set(), - ); - expect(items).toHaveLength(0); - }); -}); diff --git a/server/tests/routine-endpoint.test.ts b/server/tests/routine-endpoint.test.ts index fbfb96fe..91ccc06e 100644 --- a/server/tests/routine-endpoint.test.ts +++ b/server/tests/routine-endpoint.test.ts @@ -66,7 +66,6 @@ function buildApp( undefined, // peopleStore undefined, // identityProviders undefined, // intentRouter - undefined, // attentionStore undefined, // pageFrames runner, ]; From ec01ee24fb63df58798ebccc90db51369d84ba11 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 10:46:08 -0300 Subject: [PATCH 40/45] Clamp a routine's instruction to three lines on the list --- app/src/components/routines/routines-list.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/components/routines/routines-list.tsx b/app/src/components/routines/routines-list.tsx index 97ff20e2..6ffb8c5b 100644 --- a/app/src/components/routines/routines-list.tsx +++ b/app/src/components/routines/routines-list.tsx @@ -142,7 +142,7 @@ export function RoutinesList() { {routine.timezone} - + {routine.instruction} {/* A set, so it wraps onto its own line rather than crowding the title. */} From 2b61f7da8a15506e8f0d066497e89b73918671ed Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 12:03:18 -0300 Subject: [PATCH 41/45] Judge a schedule by its whole cycle, and close the runs nobody will finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four lifecycle fixes from review. The 15-minute floor now walks the expression's cycle instead of sampling the next two occurrences, so a cron like '45,55 8 * * *' is refused whenever it is created rather than accepted at 08:50 and wedged after its first firing; a refusal that still surfaces at sweep time switches the routine off with its reason on a run row instead of silently burning a due slot every pass forever. The enabled cap counts and writes under a per-owner advisory lock, so two racing creates at nineteen admit exactly one. A stale clock is drained current in one pass by computing the next occurrence from now while the CAS still compares the old stamp. And failOpenRuns — whose only caller was unreachable at the shipped five-minute cadence, and which would have closed a genuinely in-flight run as failed — is replaced by an age-scoped reaper that closes abandoned rows as skipped, which also mops up runs stranded open by a server dying mid-turn. --- server/src/routines/schedule.ts | 103 +++++- server/src/routines/store.ts | 205 +++++++++--- server/src/routines/sweep.ts | 166 ++++++---- server/tests/routine-schedule.test.ts | 51 +++ .../tests/routine-sweep.integration.test.ts | 304 ++++++++++++++++-- .../tests/routines-store.integration.test.ts | 198 ++++++++++-- 6 files changed, 854 insertions(+), 173 deletions(-) diff --git a/server/src/routines/schedule.ts b/server/src/routines/schedule.ts index fd4be6c4..269d3fb7 100644 --- a/server/src/routines/schedule.ts +++ b/server/src/routines/schedule.ts @@ -24,6 +24,16 @@ function hasFiveFields(cron: string): boolean { return cron.trim().split(/\s+/).length === 5; } +/** + * Both memo caches below are keyed by caller-supplied strings, so an unbounded map is a slow leak a + * hostile caller can drive on purpose: one invalid timezone per request grows it forever. Cleared + * wholesale at the cap rather than evicted piecemeal — the population that matters (the handful of + * zones and expressions real routines use) is re-learned in one pass and the code stays one line. + */ +const MAX_MEMOIZED_VERDICTS = 1000; + +const timeZoneVerdicts = new Map(); + /** * The only reliable way to validate an IANA zone name in plain JS/TS: ask Intl to * build a formatter for it and see whether it throws. cron-parser (via luxon) @@ -31,22 +41,104 @@ function hasFiveFields(cron: string): boolean { * a message ("unhandled timestamp: Invalid Date") that says nothing about * timezones — so we check this ourselves, up front, to give a sentence that means * something. + * + * Memoized because this sits on the sweep's hot path: `advanceNextRun` calls in here for every due + * routine on every pass, and building an Intl.DateTimeFormat per call is the expensive way to keep + * re-learning that "UTC" is a timezone. */ function isKnownTimeZone(timezone: string): boolean { + const cached = timeZoneVerdicts.get(timezone); + if (cached !== undefined) return cached; + let known: boolean; try { new Intl.DateTimeFormat(undefined, { timeZone: timezone }); - return true; + known = true; } catch { - return false; + known = false; } + if (timeZoneVerdicts.size >= MAX_MEMOIZED_VERDICTS) timeZoneVerdicts.clear(); + timeZoneVerdicts.set(timezone, known); + return known; +} + +/** + * Where the floor scan starts, fixed so acceptance is a property of the expression rather than of + * the clock at the moment somebody asked. The failure this prevents: `45,55 8 * * *` asked about at + * 08:50 sees 08:55 and then tomorrow's 08:45 as its next two occurrences — a 23h50m gap — so a check + * that samples from "now" accepts at some times of day what it refuses at others, and the accepted + * routine then wedges the moment the sweep tries to advance it past the ten-minute pair. A leap-year + * start, so expressions pinned to 29 February are scanned on days they actually fire. + */ +const FLOOR_SCAN_START = new Date("2024-01-01T00:00:00Z"); + +/** + * How many successive occurrences the floor scan walks. + * + * The intra-day pattern of a five-field cron is the same on every day it fires (minutes × hours), + * so the first firing day already shows every within-day gap, and the densest day the floor allows + * is 96 firings — 200 covers a full day of those plus the wrap into the next firing day, and for + * sparse expressions the iterator simply walks forward until it has seen 200 real firings, however + * far apart they are. Not exhaustive: a sub-floor gap that only exists under a DST compression more + * than 200 occurrences from the scan start can still slip through, which is why the per-call pair + * check below stays as the runtime backstop. + */ +const FLOOR_SCAN_OCCURRENCES = 200; + +/** null means the expression cleared the floor; a string is the refusal to throw. */ +const floorVerdicts = new Map(); + +/** + * Refuse any expression whose own cycle contains an adjacent pair under the floor, deterministically. + * + * Scanned from a fixed start rather than from `after`, because the caller's `after` moves with the + * clock and a floor sampled near it is a floor that depends on when the question was asked. Parse + * and iteration errors are swallowed here on purpose: an unreadable expression is the caller's + * refusal to make (with the unreadable-schedule sentence), and an expression that runs out of + * occurrences mid-scan has shown every gap it has. Memoized because this runs inside + * `nextOccurrence`, which the sweep calls for every due routine on every pass. + */ +function refuseSubFloorCycle(cron: string, timezone: string): void { + const key = `${timezone}\u0000${cron}`; + const cached = floorVerdicts.get(key); + if (cached !== undefined) { + if (cached !== null) throw new ScheduleRefusedError(cached); + return; + } + + let verdict: string | null = null; + try { + const expression = CronExpressionParser.parse(cron, { + tz: timezone, + currentDate: FLOOR_SCAN_START, + }); + let previous = expression.next().toDate().getTime(); + for (let index = 1; index < FLOOR_SCAN_OCCURRENCES; index += 1) { + const current = expression.next().toDate().getTime(); + if (current - previous < MINIMUM_INTERVAL_MS) { + verdict = TOO_FREQUENT_MESSAGE; + break; + } + previous = current; + } + } catch { + // Unreadable, or fewer occurrences than the scan wanted. Either way the gaps seen were fine, + // and unreadability is refused by the parse in `nextOccurrence` with the sentence for it. + } + + if (floorVerdicts.size >= MAX_MEMOIZED_VERDICTS) floorVerdicts.clear(); + floorVerdicts.set(key, verdict); + if (verdict !== null) throw new ScheduleRefusedError(verdict); } /** * Parse, validate against the floor, and return the next occurrence after `after`. * - * One function owns both acceptance and scheduling, so what was accepted is always schedulable: - * the floor is checked on the gap between the next two occurrences, not on a guess about the - * expression's shape. + * One function owns both acceptance and scheduling, so what was accepted is always schedulable. The + * floor is enforced twice, on purpose: `refuseSubFloorCycle` scans the expression's own cycle from a + * fixed start, so acceptance cannot depend on the time of the asking, and the pair check at the + * bottom re-measures the gap actually ahead of `after`, catching the rare shapes the bounded scan + * cannot see (a DST compression far from the scan window). A pair-check throw at sweep time is what + * the sweep's unschedulable-routine switch-off exists for. */ export function nextOccurrence( cron: string, @@ -59,6 +151,7 @@ export function nextOccurrence( if (!isKnownTimeZone(timezone)) { throw new ScheduleRefusedError(UNKNOWN_TIMEZONE_MESSAGE); } + refuseSubFloorCycle(cron, timezone); let first: Date; let second: Date; diff --git a/server/src/routines/store.ts b/server/src/routines/store.ts index 663bc18d..932cc78e 100644 --- a/server/src/routines/store.ts +++ b/server/src/routines/store.ts @@ -175,8 +175,15 @@ export type RoutineStore = { /** Enabled routines whose next run has arrived, oldest due first. */ dueRoutines(limit: number): Promise<{ id: string; nextRunAt: Date }[]>; - /** Compare-and-set the clock forward. False means another sweep got there first. */ - advanceNextRun(id: string, from: Date): Promise; + /** + * Compare-and-set the clock forward. False means another sweep got there first. + * + * The CAS always compares against `from`, the stamp the caller read. `computeFrom` is for the + * stale-backlog case: a routine whose stamp is a month behind used to be stepped one occurrence + * per pass — a fifteen-minute routine idle a month stayed silent another fortnight while its + * clock caught up — so the sweep passes `now` here and one advance makes the clock current. + */ + advanceNextRun(id: string, from: Date, computeFrom?: Date): Promise; /** Open a run row. Its status stays null until something finishes it. */ insertRun(routineId: string): Promise<{ runId: string }>; /** @@ -204,23 +211,39 @@ export type RoutineStore = { error?: string, ): Promise; /** - * Close every open (`status is null`) run for one routine as "failed", with the same error on all - * of them. Returns how many rows it closed. + * Close every run row that has sat open (`status is null`) longer than `olderThanMs` as + * "skipped", with the same reason on all of them. Returns how many rows it closed. + * + * The sweep's reaper, and deliberately NOT scoped to one routine or one firing: it exists for the + * rows nothing else can reach — a server that died mid-turn after the queue item was already + * finished on the 202, or a dispatch-failure close that itself failed. The age bound is what keeps + * it away from live work: a run younger than the cutoff may be a turn some server is still + * running, and closing that row would make the real `finishRun` a silent no-op. "skipped" rather + * than "failed" so an infrastructure death is not counted by the fatigue rule as the routine's own + * failure. + */ + reapAbandonedRuns(olderThanMs: number, error: string): Promise; + /** + * Switch a routine off because its own schedule refuses to advance, and record why. * - * The give-up branch's cleanup, not `finishRun`'s: `insertRun` runs before `dispatch` on every - * attempt, so a dispatch that throws leaves an open run row behind, and the queue's attempt cap - * eventually stops offering that item to anybody — nothing else ever closes those rows. `listFor` - * shows the newest one, so without this a routine that never ran once reads "running now" forever. - * Closing ALL of them, not just the newest, is the more truthful shape: every open row is a real - * dispatch attempt that went nowhere, not just the last one. + * The sweep's off switch, not a person's, so it is not owner-scoped — like everything else in this + * half, the id comes from the sweep's own read, not from a caller. A routine whose stored cron the + * schedule module refuses (a row written before a validation existed, a hand-edited value) throws + * out of `advanceNextRun` on every pass for ever: its clock never moves, it burns one of the + * pass's due slots each time, and the owner sees a routine that silently stopped. Disabling it + * ends that, and the finished "skipped" run row this writes is the announcement — the sweep has no + * channel to speak in, and the run history is what `listFor` surfaces to the owner. */ - failOpenRuns(routineId: string, error: string): Promise; + markUnschedulable(id: string, reason: string): Promise; /** How many failures the routine has at the tail, for the fatigue rule to read. */ consecutiveFailures(routineId: string): Promise; }; type RoutineRow = typeof routines.$inferSelect; +/** What drizzle hands the callback of `database.transaction`. */ +type Transaction = Parameters[0]>[0]; + function toRoutine(row: RoutineRow): Routine { return { id: row.id, @@ -369,9 +392,18 @@ export function createRoutineStore(database: Database): RoutineStore { return only.id; } - /** How many of this person's routines are switched on. The cap counts these, not rows. */ - async function countEnabled(ownerUserId: string): Promise { - const [row] = await database + /** + * How many of this person's routines are switched on. The cap counts these, not rows. + * + * Takes the transaction it must count on: a count made on another pooled connection would not see + * the uncommitted row a racing create is about to add, which is the exact blindness the lock below + * exists to remove. + */ + async function countEnabled( + handle: Transaction, + ownerUserId: string, + ): Promise { + const [row] = await handle .select({ total: sql`count(*)::int` }) .from(routines) .where( @@ -380,6 +412,27 @@ export function createRoutineStore(database: Database): RoutineStore { return row?.total ?? 0; } + /** + * Serialize this owner's cap-guarded writes: count and write under one advisory lock. + * + * The cap used to be a bare count-then-write, and two concurrent creates that both counted 19 both + * inserted — the person held 21. An advisory lock rather than a row lock because the thing being + * guarded is a COUNT: there is no one row whose `for update` covers "how many are enabled". + * Transaction-scoped (`_xact_`), so the commit or the rollback releases it and never us forgetting. + * `hashtext` collisions are harmless — two owners sharing a hash take turns, slower and not wrong. + */ + async function withEnabledCapLock( + ownerUserId: string, + work: (transaction: Transaction) => Promise, + ): Promise { + return await database.transaction(async (transaction) => { + await transaction.execute( + sql`select pg_advisory_xact_lock(hashtext(${`routine-cap-${ownerUserId}`}))`, + ); + return await work(transaction); + }); + } + async function loadOwned( ownerUserId: string, id: string, @@ -417,9 +470,6 @@ export function createRoutineStore(database: Database): RoutineStore { if (patch.enabled !== undefined) values.enabled = patch.enabled; const enabling = patch.enabled === true && !existing.enabled; - if (enabling && (await countEnabled(ownerUserId)) >= MAX_ENABLED_ROUTINES) { - throw new RoutineRefusedError(TOO_MANY_ENABLED); - } const cron = patch.cron ?? existing.cron; const timezone = patch.timezone ?? existing.timezone; @@ -434,11 +484,24 @@ export function createRoutineStore(database: Database): RoutineStore { values.nextRunAt = nextRunFor(cron, timezone, new Date()); } - const [row] = await database - .update(routines) - .set(values) - .where(and(eq(routines.id, id), eq(routines.ownerUserId, ownerUserId))) - .returning(); + /* + * The cap is re-counted inside the lock, in the same transaction as the write. Counting outside + * and writing inside would keep the very race the lock removes: two enables that both counted 19 + * before either committed. + */ + const [row] = await withEnabledCapLock(ownerUserId, async (transaction) => { + if ( + enabling && + (await countEnabled(transaction, ownerUserId)) >= MAX_ENABLED_ROUTINES + ) { + throw new RoutineRefusedError(TOO_MANY_ENABLED); + } + return await transaction + .update(routines) + .set(values) + .where(and(eq(routines.id, id), eq(routines.ownerUserId, ownerUserId))) + .returning(); + }); if (!row) throw new RoutineNotFoundError(); return toRoutine(row); } @@ -456,23 +519,32 @@ export function createRoutineStore(database: Database): RoutineStore { ); const nextRunAt = nextRunFor(input.cron, timezone, new Date()); - if ((await countEnabled(input.ownerUserId)) >= MAX_ENABLED_ROUTINES) { - throw new RoutineRefusedError(TOO_MANY_ENABLED); - } - - const [row] = await database - .insert(routines) - .values({ - id: `routine_${crypto.randomUUID()}`, - ownerUserId: input.ownerUserId, - agentId: input.agentId, - channelId, - instruction, - cron: input.cron, - timezone, - nextRunAt, - }) - .returning(); + // Counted and inserted under the owner's cap lock, so two creates racing at 19 cannot both + // count 19 and hand the person 21: the second waits, counts 20, and gets the refusal. + const [row] = await withEnabledCapLock( + input.ownerUserId, + async (transaction) => { + if ( + (await countEnabled(transaction, input.ownerUserId)) >= + MAX_ENABLED_ROUTINES + ) { + throw new RoutineRefusedError(TOO_MANY_ENABLED); + } + return await transaction + .insert(routines) + .values({ + id: `routine_${crypto.randomUUID()}`, + ownerUserId: input.ownerUserId, + agentId: input.agentId, + channelId, + instruction, + cron: input.cron, + timezone, + nextRunAt, + }) + .returning(); + }, + ); // An insert that returned nothing is not a missing routine, it is a broken database: loud // rather than folded into the not-found sentence a caller is meant to be able to trust. if (!row) throw new Error("inserting a routine returned no row"); @@ -606,7 +678,7 @@ export function createRoutineStore(database: Database): RoutineStore { .limit(limit); }, - async advanceNextRun(id, from) { + async advanceNextRun(id, from, computeFrom) { const [row] = await database .select({ cron: routines.cron, timezone: routines.timezone }) .from(routines) @@ -614,7 +686,10 @@ export function createRoutineStore(database: Database): RoutineStore { .limit(1); if (!row) return false; - const next = nextRunFor(row.cron, row.timezone, from); + // `computeFrom` moves only where the next occurrence is measured from, never what the CAS + // compares against: the sweep uses it to make a month-stale clock current in one pass, and + // the guarantee that exactly one replica moves the row has to survive that. + const next = nextRunFor(row.cron, row.timezone, computeFrom ?? from); /* * THE COMPARE-AND-SET IS THE WHOLE MECHANISM. `where next_run_at = from` means the row only @@ -714,24 +789,55 @@ export function createRoutineStore(database: Database): RoutineStore { .where(and(eq(routineRuns.id, runId), isNull(routineRuns.status))); }, - async failOpenRuns(routineId, error) { - // One UPDATE, not a select-then-loop: every row this WHERE matches is a leaked attempt, and - // there is nothing to decide per row that `status is null` does not already decide. + async reapAbandonedRuns(olderThanMs, error) { + /* + * One UPDATE, not a select-then-loop: every row this WHERE matches is abandoned, and there is + * nothing to decide per row that the age bound does not already decide. Both sides of the age + * comparison are the database's clock — `started_at` was written by its `now()`, so measuring + * it against a replica's `Date.now()` would let ninety seconds of skew reap a run some server + * is still running, which is this file's standing clock discipline. + */ const closed = await database .update(routineRuns) .set({ - status: "failed", + status: "skipped", finishedAt: sql`now()`, - // Same code-point cap as `finishRun`, so a give-up reason cannot be cut mid-surrogate-pair. + // Same code-point cap as `finishRun`, so a reap reason cannot be cut mid-surrogate-pair. error: Array.from(error).slice(0, MAX_RUN_ERROR).join(""), }) .where( - and(eq(routineRuns.routineId, routineId), isNull(routineRuns.status)), + and( + isNull(routineRuns.status), + lte( + routineRuns.startedAt, + sql`now() - (${olderThanMs} * interval '1 millisecond')`, + ), + ), ) .returning({ id: routineRuns.id }); return closed.length; }, + async markUnschedulable(id, reason) { + const disabled = await database + .update(routines) + .set({ enabled: false, updatedAt: sql`now()` }) + .where(eq(routines.id, id)) + .returning({ id: routines.id }); + // Deleted between the sweep's read and this write: gone is gone, and a run row inserted here + // would only violate the foreign key of a routine nobody can see any more. + if (disabled.length === 0) return; + // A finished "skipped" row rather than "failed": no turn ran, so the fatigue rule must not + // count this, and skipped is exactly the vocabulary for a firing that never became a turn. + await database.insert(routineRuns).values({ + id: `routine_run_${crypto.randomUUID()}`, + routineId: id, + status: "skipped", + finishedAt: sql`now()`, + error: Array.from(reason).slice(0, MAX_RUN_ERROR).join(""), + }); + }, + async consecutiveFailures(routineId) { /* * Bounded, then counted here. The bound is the point: this is read on every failed firing, @@ -749,8 +855,9 @@ export function createRoutineStore(database: Database): RoutineStore { eq(routineRuns.routineId, routineId), isNotNull(routineRuns.status), /* - * A SKIP IS NOT A FAILURE, AND DOES NOT BREAK THE STREAK. It means the channel was - * gone, not that the turn failed, so it is not counted; and it does not reset the + * A SKIP IS NOT A FAILURE, AND DOES NOT BREAK THE STREAK. It means no turn ran — the + * channel was gone, the dispatch never reached the server, or a restart abandoned the + * run — not that the turn failed, so it is not counted; and it does not reset the * count either, because a routine whose channel flaps would otherwise never reach the * fatigue rule — it would disable itself over ten missing channels, or never at all. * Excluding it here, rather than reading it and skipping over it below, keeps it from diff --git a/server/src/routines/sweep.ts b/server/src/routines/sweep.ts index d43e2d48..b5e709d2 100644 --- a/server/src/routines/sweep.ts +++ b/server/src/routines/sweep.ts @@ -22,7 +22,7 @@ * prevent. */ import { DEFAULT_MAX_ATTEMPTS, type WorkQueue } from "../work/queue"; -import type { RoutineStore } from "./store"; +import { RoutineRefusedError, type RoutineStore } from "./store"; export const ROUTINE_FIRE_KIND = "routine.fire"; @@ -56,6 +56,19 @@ const DISPATCH_RETRY_DELAY_MS = 60_000; */ export const DEFAULT_GRACE_MS = 10 * 60_000; +/** + * How long a run row may sit open with no outcome before a pass declares it abandoned. + * + * A server that dies mid-turn strands its run row for ever: the queue item was finished on the 202, + * so no retry comes back for it, and nothing else writes that row — the routines page reads + * "running now" for a run no process is running. Twice the server's own turn timeout + * (`DEFAULT_TURN_TIMEOUT_MS` in `./run-turn`, five minutes), so a slow-but-alive turn is never + * closed out from under the server still running it. A local constant rather than an import because + * the sweep runs as a CronJob and must not drag the runtime's import graph — the Intelligence + * client and everything behind it — into that process. + */ +const ABANDONED_RUN_MS = 10 * 60_000; + export type RoutineSweepOptions = { routineStore: RoutineStore; /** The shared `work_items` queue. Not a second queue, and not a timer. */ @@ -139,21 +152,18 @@ export async function offerDueRoutines( */ try { /* - * A STALE STAMP IS NOT A BACKLOG TO REPLAY. `advanceNextRun` moves the clock one occurrence on - * from the stamp it was given, so a routine whose stamp is a month behind — a deployment that - * ran with no worker, a worker that was down — comes back due on the next pass, and the pass - * after that, once per missed occurrence, each with its own offer key and its own real firing. - * Turn the worker on after a quiet month and a person gets thirty summaries of thirty days ago. + * A STALE STAMP IS NOT A BACKLOG TO REPLAY. A routine whose stamp is a month behind — a + * deployment that ran with no worker, a worker that was down — must not fire once per missed + * occurrence when the worker comes back: turn it on after a quiet month and a person would get + * thirty summaries of thirty days ago. * * So this loop offers only firings that are still worth having: a stamp within GRACE of now. - * For anything later than that, advance WITHOUT offering and say nothing — the occurrence is - * past and nobody wants it now — and let successive passes drain the stamp silently until it is - * current. The store deliberately does not decide this: it moves the clock one step and reports - * whether it won, and which steps are worth firing is this file's policy. - * - * Draining costs a slot of `limit` per pass per stale routine, which is the price of a bounded - * pass; the routines still current are read on the same passes, because the ordering is by due - * stamp and a stale one leaves the list as soon as its clock catches up. + * For anything later than that, advance WITHOUT offering and compute the next occurrence from + * NOW rather than from the stale stamp, so one pass makes the clock current. Stepping one + * occurrence per pass instead — the earlier shape — kept a fifteen-minute routine silent a + * further fortnight after a month of downtime, because draining ~2,900 missed occurrences at + * one per five-minute sweep is itself two weeks. The store deliberately does not decide this: + * which stamps are worth firing, and where a stale clock should land, are this file's policy. */ const lateBy = now.getTime() - routine.nextRunAt.getTime(); if (lateBy <= graceMs) { @@ -172,19 +182,52 @@ export async function offerDueRoutines( }, }); offered.push(routine.id); + // False means another sweep advanced it first, which is fine either way: the firing was + // offered under the same key by both, so it still happens once. + await options.routineStore.advanceNextRun( + routine.id, + routine.nextRunAt, + ); + } else { + // The CAS still compares against the stale stamp it read — only the landing point moves. + await options.routineStore.advanceNextRun( + routine.id, + routine.nextRunAt, + now, + ); } - // False means another sweep advanced it first, which is fine either way: the firing was offered - // under the same key by both, so it still happens once. - await options.routineStore.advanceNextRun(routine.id, routine.nextRunAt); } catch (error) { /* - * Said out loud, with the routine in it. A pass that swallowed this would look clean while one - * routine's clock never moved again: it would be read as due and warned about on every pass - * thereafter, but OFFERED only while its stamp is still inside `graceMs` — once the stamp ages - * past the grace window, the guard above skips the offer before this throw is ever reached. So - * the grace policy (the window worth having, above) is what bounds this failure mode to one - * firing: harmless and loud, but invisible to anybody not reading logs. + * A refusal from the store here is the schedule's own: `advanceNextRun` recomputes the next + * occurrence, and a cron the schedule module refuses — a row written before a validation + * existed, a hand-edited value — throws on every pass for ever. Left alone, that routine's + * clock never moves, it burns one of this pass's `limit` slots each time, and its owner sees a + * routine that silently stopped. Deterministic refusals do not heal, so the routine is + * switched off and the reason written where the routines page reads it — the sweep has no + * channel to announce itself in, the way the runner's fatigue switch-off does. Queue and + * database errors are NOT this: they heal, so those routines are left enabled for the next + * pass to try again. */ + if (error instanceof RoutineRefusedError) { + try { + await options.routineStore.markUnschedulable( + routine.id, + error.message, + ); + } catch (markError) { + // Best-effort: a switch-off that failed leaves the loud warning below, and the next pass + // will be back here to try the switch-off again. + console.warn( + JSON.stringify({ + type: "routine-sweep-disable-failed", + routineId: routine.id, + reason: String(markError), + }), + ); + } + } + // Said out loud, with the routine in it: a pass that swallowed this would look clean while + // one routine was switched off, or failed to be. console.warn( JSON.stringify({ type: "routine-sweep-offer-failed", @@ -200,23 +243,6 @@ export async function offerDueRoutines( return { offered }; } -/** - * Which routine a claimed item is about. - * - * The payload is the answer, and the key is the fallback for a row written before the payload was: - * the key is `:` and a routine id carries no colon, so everything up to the first - * one is the routine. A firing whose routine cannot be named at all would be a firing nothing could - * report, which is why this never returns undefined. - */ -function routineIdOf(item: { key: string; payload: Record }) { - const fromPayload = item.payload.routineId; - if (typeof fromPayload === "string" && fromPayload.length > 0) { - return fromPayload; - } - const colon = item.key.indexOf(":"); - return colon === -1 ? item.key : item.key.slice(0, colon); -} - /** * Phase two: claimed items become dispatched firings, with the queue's booleans honoured. * @@ -254,8 +280,38 @@ export async function dispatchClaimedRoutines( skipped: [], }; + /* + * THE REAPER, before any firing is considered. It closes the run rows nothing in the system will + * ever come back for: a server that died mid-turn (the queue item was finished on the 202, so no + * retry returns for that row), and the rows opened by dispatch attempts that threw (the loop below + * deliberately does not close those itself — see the comment at the dispatch). Without it those + * rows read "running now" on the routines page for ever. Age-scoped rather than identity-scoped, + * because age is the one signal that distinguishes an abandoned row from a turn some server is + * still running; the cutoff sits above the turn timeout so a live turn always finishes its own row + * first. Best-effort with its own catch, because a reaper that cannot run must not stop this pass + * from firing what is due. + */ + try { + const reaped = await options.routineStore.reapAbandonedRuns( + ABANDONED_RUN_MS, + "the server never finished this run; it may have restarted mid-turn, or the run may never have been dispatched", + ); + if (reaped > 0) { + console.warn(JSON.stringify({ type: "routine-runs-reaped", reaped })); + } + } catch (error) { + console.warn( + JSON.stringify({ + type: "routine-run-reap-failed", + reason: String(error), + }), + ); + } + for (const item of claimed) { - const routineId = routineIdOf(item); + // The offer above is the only writer of these items and always names the routine in the + // payload; the key is the last-resort stand-in for a row written by hand, the culler's idiom. + const routineId = String(item.payload.routineId ?? item.key); /* * Renewed before acting, because the batch is many and the lease is one. @@ -347,6 +403,18 @@ export async function dispatchClaimedRoutines( * owns that, not this loop. */ const { runId } = await options.routineStore.insertRun(routineId); + /* + * A dispatch that throws leaves the row this attempt opened with no status, AND NOTHING HERE + * CLOSES IT — the reaper above does, once the row is older than any turn could still be + * running. That restraint is deliberate: a dispatch that timed out is not a dispatch that + * failed, because the abort tears down the sweep's side of the call while the server may + * already have accepted it and detached the turn — a turn that will come back minutes later + * and finish this very row. `finishRun` finishes once, so closing the row now would turn that + * turn's real outcome into a silent no-op; the earlier shape of this cleanup ("close every + * open run of the routine at the give-up") mislabelled exactly such in-flight runs as failed. + * Age is the only signal the sweep has that no server is coming back for a row, so age is the + * scope the closing uses. + */ await options.dispatch(runId); if ( !(await options.queue.finish({ @@ -412,18 +480,9 @@ export async function dispatchClaimedRoutines( * and the reason for anybody who queries the table; this is for whoever reads the logs. */ if (item.attempts >= maxAttempts) { - /* - * THE ROW THIS GIVE-UP LEAKED. Every attempt opened a run row before it dispatched - * (`insertRun` above), and a dispatch that throws never reaches `finishRun` — so an item at - * the cap is not just off the queue, it is one or more `routine_runs` rows stuck open with no - * status. `listFor` shows the newest one, so without this the routines page reads "running - * now" for a routine that never ran at all, forever. Closed before the warning so the row is - * never left open even if the log line itself fails. - */ - const closed = await options.routineStore.failOpenRuns( - routineId, - reason, - ); + // The run rows the attempts opened are NOT closed here: one of them may be a turn a wedged + // server accepted after the dispatch timed out, and only age can tell (see the comment at + // the dispatch). The reaper closes them on a later pass; this branch only has to say so. console.warn( JSON.stringify({ type: "routine-fire-gave-up", @@ -431,7 +490,6 @@ export async function dispatchClaimedRoutines( key: item.key, attempts: item.attempts, reason, - closedRuns: closed, }), ); } else if (!released) { diff --git a/server/tests/routine-schedule.test.ts b/server/tests/routine-schedule.test.ts index bf101fc0..d9ef8b13 100644 --- a/server/tests/routine-schedule.test.ts +++ b/server/tests/routine-schedule.test.ts @@ -50,6 +50,57 @@ describe("nextOccurrence", () => { expect(result).toBeInstanceOf(Date); }); + /** + * The floor must be a property of the expression, not of the clock at the moment of asking. + * + * "45,55 8 * * *" has one ten-minute pair per day. A check that samples only the next two + * occurrences after `after` sees that pair when asked before 08:45 and a 23h50m gap when asked at + * 08:50 — so the routine is accepted or refused depending on when the person happened to create + * it, and an accepted one wedges at its first advance: the sweep hands `nextOccurrence` an `after` + * of 08:45, the pair check throws, and `next_run_at` never moves again. + */ + test("refuses a sub-floor adjacent pair no matter when it is asked about", () => { + // Asked from a moment where the next two occurrences are 23h50m apart: the shape the + // next-two-samples check accepted. + expect(() => + nextOccurrence("45,55 8 * * *", "UTC", new Date("2026-01-01T08:50:00Z")), + ).toThrow("Routines may run at most every 15 minutes."); + // And from a moment where the pair is the next thing ahead, for symmetry. + expect(() => + nextOccurrence("45,55 8 * * *", "UTC", new Date("2026-01-01T00:00:00Z")), + ).toThrow(ScheduleRefusedError); + }); + + test("refuses a sub-floor pair that only exists on one day of the year", () => { + // 09:00 and 09:10 every 25 December. Asked five minutes after the 09:05 of one Christmas, the + // next two occurrences are 09:10 and NEXT year's 09:00 — the old sampling accepted that. + expect(() => + nextOccurrence("0,10 9 25 12 *", "UTC", new Date("2026-12-25T09:05:00Z")), + ).toThrow("Routines may run at most every 15 minutes."); + }); + + test("refuses a sub-floor gap that only appears across midnight", () => { + // 00:00, 00:50, 23:00 and 23:50 every day: every same-day gap clears the floor, but 23:50 to + // the next day's 00:00 is ten minutes. A scan that stopped at one day's occurrences would + // accept it. + expect(() => + nextOccurrence( + "0,50 0,23 * * *", + "UTC", + new Date("2026-01-01T00:10:00Z"), + ), + ).toThrow("Routines may run at most every 15 minutes."); + }); + + test("still accepts a listed pair that clears the floor", () => { + const result = nextOccurrence( + "0,30 9 * * *", + "UTC", + new Date("2026-01-01T09:05:00Z"), + ); + expect(result.toISOString()).toBe("2026-01-01T09:30:00.000Z"); + }); + test("refuses an unknown IANA timezone", () => { expect(() => nextOccurrence( diff --git a/server/tests/routine-sweep.integration.test.ts b/server/tests/routine-sweep.integration.test.ts index 2b3a4459..ebae2212 100644 --- a/server/tests/routine-sweep.integration.test.ts +++ b/server/tests/routine-sweep.integration.test.ts @@ -417,14 +417,14 @@ describe("offering the firings that are due", () => { /** * A STALE STAMP IS NOT A BACKLOG TO REPLAY. * - * `advanceNextRun` moves the clock one occurrence on from the stamp it was given, so a routine whose - * stamp is a month behind comes back due pass after pass, once per missed occurrence. Turn the - * worker on after a quiet month and a person gets thirty summaries of thirty days ago; a 15-minute - * routine idle a year would be some thirty-five thousand firings. The stamp still has to drain, so - * the pass advances it — it just says nothing while it does. + * A routine whose stamp is a month behind must not fire once per missed occurrence when the worker + * comes back: a person would get thirty summaries of thirty days ago. And it must not drain one + * occurrence per pass either — that kept a fifteen-minute routine silent a further fortnight after a + * month of downtime, because stepping through ~2,900 missed occurrences at one per five-minute sweep + * is itself two weeks. One pass makes the clock current, silently. */ describe("draining a stale stamp instead of replaying it", () => { - test("a month-old firing is advanced without being offered, and a two-minute-old one fires", async () => { + test("a month-old firing is caught up to current in one pass, and a two-minute-old one fires", async () => { const { routine: stale } = await makeRoutine("A month behind."); const { routine: fresh } = await makeRoutine("Two minutes late."); const staleFrom = await makeDueAt( @@ -441,11 +441,13 @@ describe("draining a stale stamp instead of replaying it", () => { expect(await firingsFor(stale.id)).toHaveLength(0); expect(await firingsFor(fresh.id)).toHaveLength(1); - // Advanced all the same, one occurrence on, so successive passes drain it silently rather than - // reading it as due for ever. + // Advanced past the whole backlog in one move: the next occurrence is computed from the pass's + // own moment, not one day along a thirty-one-day drain, so the routine is silent for the missed + // month and then simply current. (Whether it is still "due" is Postgres's real clock's judgement, + // which is why this asserts the landing point rather than a second pass over a 2001 stamp.) const after = await readRoutine(stale.id); expect(after?.nextRunAt.getTime()).toBeGreaterThan(staleFrom.getTime()); - expect(after?.nextRunAt.toISOString()).toBe("2001-01-02T09:00:00.000Z"); + expect(after?.nextRunAt.toISOString()).toBe("2001-02-02T09:00:00.000Z"); }); test("the grace window is a setting, so a caller can say what counts as worth having", async () => { @@ -467,15 +469,18 @@ describe("draining a stale stamp instead of replaying it", () => { }); /** - * One poisoned routine is one person's problem, not everybody's. + * One poisoned routine is one person's problem, not everybody's — and not silently for ever. * - * A cron the parser cannot read makes `advanceNextRun` throw, and an unguarded loop would take the - * whole pass down with it — for every other person's routine too, on every pass, for as long as the - * bad row exists. The one routine nobody can schedule must not be able to stop the sweep. + * A cron the schedule module refuses makes `advanceNextRun` throw. An unguarded loop would take the + * whole pass down with it, for every other person's routine too; and a loop that only warned would + * read the same row as due on every pass for ever, burning one of the pass's slots while its owner + * saw a routine that quietly stopped. So the pass survives it, says so, and switches the routine off + * with the reason written where the routines page reads it. */ describe("surviving a routine that cannot be scheduled", () => { - test("a poisoned cron is warned about and the next routine is still offered", async () => { - const { routine: poisoned } = await makeRoutine("Unschedulable."); + test("a poisoned cron is warned about, switched off with its reason on the page, and the next routine still offered", async () => { + const { owner: poisonedOwner, routine: poisoned } = + await makeRoutine("Unschedulable."); const { routine: healthy } = await makeRoutine("Perfectly fine."); // Only a direct write can make this row: `create` and `update` both refuse a cron the schedule // module cannot read, which is exactly why the bad row has to be simulated rather than created. @@ -506,24 +511,69 @@ describe("surviving a routine that cannot be scheduled", () => { expect(await firingsFor(healthy.id)).toHaveLength(1); // Said out loud, with the routine in it: a sweep that swallowed this would look clean while one - // routine never advanced again. + // routine was switched off. const complaint = lines.find((line) => line.includes(poisoned.id)); expect(complaint).toBeDefined(); expect(JSON.parse(complaint as string).routineId).toBe(poisoned.id); - // The poisoned routine's clock could not move, so it stays due and stays being warned about on - // every pass thereafter. It is not re-offered forever, though: this pass fired it because its - // stamp was still inside the grace window (two minutes late). Once the stamp ages past graceMs, - // the grace guard skips the offer before this throw is ever reached, so the grace policy is what - // bounds this failure mode to a single firing rather than an unbounded stream of re-offers. - expect((await readRoutine(poisoned.id))?.nextRunAt.getTime()).toBe( + /* + * SWITCHED OFF, NOT LEFT TO WEDGE. The clock could not move, so left enabled this row would be + * read as due and thrown over on every pass for ever — invisible to its owner, who would see a + * routine that simply stopped. Disabled, it stops burning a due slot, and the skipped run row is + * the announcement: the sweep has no channel to speak in, so the page's last-run column is where + * the owner learns why. + */ + const after = await readRoutine(poisoned.id); + expect(after?.enabled).toBe(false); + expect(after?.nextRunAt.getTime()).toBe( new Date("2001-01-01T09:25:00Z").getTime(), ); + const [summary] = await store.listFor(poisonedOwner.id); + expect(summary?.enabled).toBe(false); + expect(summary?.lastRun?.status).toBe("skipped"); // That single firing is the entire blast radius: pin it as exactly one `work_items` row, the // offer that preceded the throw, rather than leaving it inferred from the warning alone. expect(await firingsFor(poisoned.id)).toHaveLength(1); }); + + /** + * The wedge that motivated the deterministic floor: `45,55 8 * * *` used to be ACCEPTED when + * created between the pair (the next two occurrences sampled 23h50m apart), and then the first + * advance handed the schedule an `after` of 08:45, saw the ten-minute pair, and threw — on every + * pass, for ever, while the routine silently never fired again. Creation now refuses it, so the + * row is written directly to stand for the ones already in the wild. + */ + test("a sub-floor cron already in the table is switched off instead of wedging the sweep", async () => { + const { owner, routine } = await makeRoutine("Grandfathered in."); + await database + .update(routines) + .set({ cron: "45,55 8 * * *" }) + .where(eq(routines.id, routine.id)); + await makeDueAt(routine.id, new Date("2001-01-01T08:45:00Z")); + + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + await offerDueRoutines( + sweepOptions({ now: () => new Date("2001-01-01T08:46:00Z") }), + ); + } finally { + warn.mockRestore(); + } + + const [summary] = await store.listFor(owner.id); + expect(summary?.enabled).toBe(false); + expect(summary?.lastRun?.status).toBe("skipped"); + // The reason on the run row is the schedule's own sentence, so the owner (and their Bot) can + // propose a schedule that works rather than guess at what broke. + const runs = await runsFor(routine.id); + expect(runs[0]?.error).toContain("15 minutes"); + + // And a disabled routine is not due: the pass after this one no longer spends a slot on it. + expect((await store.dueRoutines(50)).map((row) => row.id)).not.toContain( + routine.id, + ); + }); }); /** @@ -594,6 +644,16 @@ describe("consuming a claimed firing", () => { expect(report.skipped[0]?.routineId).toBe(routine.id); expect(report.skipped[0]?.reason).toContain("503"); + /* + * The run row this attempt opened stays open FOR NOW, on purpose: a dispatch that timed out may + * be a dispatch a wedged server accepted anyway, with a detached turn coming back for this very + * row, and only age can tell that apart from abandonment. The reaper closes it once it is older + * than any turn could still be running — the age-scoped test further down pins that half. + */ + const runs = await runsFor(routine.id); + expect(runs).toHaveLength(1); + expect(runs[0]?.status).toBeNull(); + /* * THE ROW, NOT THE RETURN VALUE. A consumer that reported a failure and finished the item anyway * would pass any assertion made on the report alone, and the firing would be gone for good. @@ -849,14 +909,16 @@ describe("consuming a claimed firing", () => { }); /** - * THE LEAKED RUN ROW THE ATTEMPT CAP LEFT BEHIND. `insertRun` runs before `dispatch` on every - * attempt, so a dispatch that throws on the very last attempt still leaves an open (`status` null) - * run row nothing else closes: the item stops being claimed at the cap and the queue's own - * machinery has nothing to do with `routine_runs`. Without closing it, `listFor` — the routines - * page's read — keeps showing that open row as the newest run, and a routine that never ran reads - * as "running now" forever. + * NO RUN ROW STAYS OPEN FOR EVER AFTER THE CAP. `insertRun` runs before `dispatch` on every + * attempt, so a dispatch that throws on the very last attempt leaves an open (`status` null) run + * row: the item stops being claimed at the cap and the queue's own machinery has nothing to do + * with `routine_runs`. Nothing closes that row at the give-up itself — a timed-out dispatch may be + * a turn a wedged server accepted, and only age can tell — but the reaper closes it on a later + * pass, so `listFor` (the routines page's read) stops showing "running now" for a routine that + * never ran. At Helm defaults the old give-up branch was unreachable in time anyway (five 1-minute + * retries against a 10-minute grace), so these rows used to leak for ever. */ - test("giving up at the attempt cap also closes the run row it leaked, so the page stops reading 'running'", async () => { + test("a run row leaked at the attempt cap is closed by a later pass, so the page stops reading 'running'", async () => { const { owner, routine } = await makeRoutine(); await offerFiring( routine.id, @@ -875,18 +937,196 @@ describe("consuming a claimed firing", () => { }, }), ); + + // Old enough that no turn could still be running it; the reaper compares on the database's + // clock, so the age is written rather than waited for. + const [leaked] = await runsFor(routine.id); + await database + .update(routineRuns) + .set({ startedAt: new Date(Date.now() - 11 * 60_000) }) + .where(eq(routineRuns.id, leaked?.id as string)); + + await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:40:00Z") }), + ); } finally { warn.mockRestore(); } - // No run row for this routine is left open: every attempt that opened one also got it closed. + // No run row for this routine is left open once the reaper has passed. const runs = await runsFor(routine.id); expect(runs.length).toBeGreaterThan(0); expect(runs.every((run) => run.status !== null)).toBe(true); - // And the page-facing read agrees: the newest run reads "failed", not "running now". + // The page-facing read agrees, and it reads "skipped", not "failed": the turn never ran, so the + // routine's own failure streak must not grow — ten flapping dispatches used to read as ten turn + // failures, enough to trip the fatigue rule and switch a perfectly healthy routine off. + const [summary] = await store.listFor(owner.id); + expect(summary?.lastRun?.status).toBe("skipped"); + expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date); + expect(await store.consecutiveFailures(routine.id)).toBe(0); + }); + + /** + * THE NET MUST NOT CATCH THE NEIGHBOUR'S FISH. An earlier firing of the same routine can be + * genuinely mid-turn while a later firing runs out of dispatch attempts: the dispatch call aborts + * at 30 seconds, but the server's detached turn keeps running for minutes and finishes its row + * itself. The old give-up cleanup closed EVERY open run of the routine, that one included, so the + * real turn's finish then no-oped against an already-"failed" row and an honest success was + * recorded as a failure. Cleanup is age-scoped now, and a fresh row is by definition one a turn + * may still be running. + */ + test("giving up leaves a genuinely in-flight run from an earlier firing untouched", async () => { + const { routine } = await makeRoutine(); + // The earlier firing: already dispatched, its turn still running on the server, its row open. + const inFlight = await store.insertRun(routine.id); + + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + maxAttempts: 1, + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + } finally { + warn.mockRestore(); + } + + // Nothing was mislabelled at the give-up: both rows are still open, because both are young + // enough to be turns some server is running. + const runs = await runsFor(routine.id); + expect(runs.find((run) => run.id === inFlight.runId)?.status).toBeNull(); + + // The in-flight turn now completes, and its outcome lands rather than no-oping against a row + // the give-up already closed — which is exactly what the old cleanup caused. + await store.finishRun(inFlight.runId, "succeeded"); + expect( + (await runsFor(routine.id)).find((run) => run.id === inFlight.runId) + ?.status, + ).toBe("succeeded"); + + // And once the abandoned attempt's row is old enough that no turn could still be running it, + // the reaper closes it — as skipped, never touching the finished run beside it. + const attempt = runs.find((run) => run.id !== inFlight.runId); + await database + .update(routineRuns) + .set({ startedAt: new Date(Date.now() - 11 * 60_000) }) + .where(eq(routineRuns.id, attempt?.id as string)); + const warnAgain = spyOn(console, "warn").mockImplementation(() => {}); + try { + await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:40:00Z") }), + ); + } finally { + warnAgain.mockRestore(); + } + const after = await runsFor(routine.id); + expect(after.find((run) => run.id === attempt?.id)?.status).toBe("skipped"); + expect(after.find((run) => run.id === inFlight.runId)?.status).toBe( + "succeeded", + ); + }); + + /** + * A firing abandoned mid-retry must not strand its run rows. An attempt fails inside the grace + * window and its item is released; by the time the item is claimed again the window has passed and + * the firing is finished as stale. The grace-skip itself opens nothing and closes nothing — the + * open row may be a turn a wedged server accepted, and only age can tell — but the reaper closes + * it once it is old enough, so nothing about the abandoned firing reads "running now" for ever. + */ + test("a firing abandoned by the grace window does not strand its earlier attempt's run row", async () => { + const { routine } = await makeRoutine(); + await offerFiring( + routine.id, + new Date("2001-01-01T09:25:00Z"), + new Date("2001-01-01T09:26:00Z"), + ); + + // Attempt one, inside the window: the dispatch throws and the item is pushed out for retry. + await dispatchClaimedRoutines( + sweepOptions({ + now: at("2001-01-01T09:26:00Z"), + dispatch: async () => { + throw new Error("the server answered 503"); + }, + }), + ); + // The retry delay is written into the past so the next pass can claim the item at all, and the + // leaked row is aged past the reaper's cutoff the same way — the clock is driven, not waited on. + const [row] = await firingsFor(routine.id); + await backdate(row?.key as string, { + runAt: new Date("2001-01-01T09:27:00Z"), + }); + const [leaked] = await runsFor(routine.id); + await database + .update(routineRuns) + .set({ startedAt: new Date(Date.now() - 11 * 60_000) }) + .where(eq(routineRuns.id, leaked?.id as string)); + + // Attempt two, claimed twenty minutes after the occurrence: outside the window, so the firing + // is finished without dispatching — and the same pass's reaper closes the leaked row. + const warn = spyOn(console, "warn").mockImplementation(() => {}); + let report: Awaited>; + try { + report = await dispatchClaimedRoutines( + sweepOptions({ now: at("2001-01-01T09:45:00Z") }), + ); + } finally { + warn.mockRestore(); + } + expect(report.fired).toEqual([]); + expect(report.skipped[0]?.routineId).toBe(routine.id); + + // Nothing about this firing is still open: attempt one's row is closed as skipped — no turn ran, + // so the fatigue rule must not count it — and attempt two never opened one. + const runs = await runsFor(routine.id); + expect(runs).toHaveLength(1); + expect(runs[0]?.status).toBe("skipped"); + expect(await store.consecutiveFailures(routine.id)).toBe(0); + const [finished] = await firingsFor(routine.id); + expect(finished?.finishedAt).not.toBeNull(); + }); + + /** + * THE REAPER, for the row no retry ever comes back for. The server 202s the dispatch, the queue + * item is finished, and then the server dies mid-turn: its run row stays open with nothing in the + * system holding a reference to it — the page reads "running now" for a run no process is running. + * The sweep's consuming pass is on a clock anyway, so it is the thing that mops these up. + */ + test("a run abandoned by a dead server is closed by the next pass, as skipped, with an honest reason", async () => { + const { owner, routine } = await makeRoutine(); + const { runId } = await store.insertRun(routine.id); + // Aged past the reaper's cutoff (twice the server's five-minute turn timeout) by hand: the run + // was opened by a server that died eleven minutes ago. + await database + .update(routineRuns) + .set({ startedAt: new Date(Date.now() - 11 * 60_000) }) + .where(eq(routineRuns.id, runId)); + + // Nothing is claimed in this pass; the reaper alone does the work. + const warn = spyOn(console, "warn").mockImplementation(() => {}); + let report: Awaited>; + try { + report = await dispatchClaimedRoutines(sweepOptions()); + } finally { + warn.mockRestore(); + } + expect(report.considered).toBe(0); + + const runs = await runsFor(routine.id); + expect(runs[0]?.status).toBe("skipped"); + expect(runs[0]?.error).toContain("never finished this run"); const [summary] = await store.listFor(owner.id); - expect(summary?.lastRun?.status).toBe("failed"); + expect(summary?.lastRun?.status).toBe("skipped"); expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date); }); diff --git a/server/tests/routines-store.integration.test.ts b/server/tests/routines-store.integration.test.ts index 33b93d8d..4702aa8f 100644 --- a/server/tests/routines-store.integration.test.ts +++ b/server/tests/routines-store.integration.test.ts @@ -279,6 +279,56 @@ describe("how many a person may have switched on", () => { store.setEnabled(owner.id, created[0] as string, true), ).rejects.toBeInstanceOf(RoutineRefusedError); }); + + /** + * The cap has to hold under concurrency, not just in sequence. A bare count-then-insert lets two + * creates racing at 19 both count 19 and both insert — the person holds 21 — which is why the + * store serializes the count and the write per owner under an advisory lock. This drives the race + * for real: two creates in flight at once on separate pooled connections. + */ + test("two creates racing at the cap admit exactly one", async () => { + const { owner, agentId, channel } = await setUp(); + for (let index = 0; index < MAX_ENABLED_ROUTINES - 1; index += 1) { + await store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: `Routine ${index}`, + cron: DAILY, + }); + } + + const outcomes = await Promise.allSettled([ + store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Racer one.", + cron: DAILY, + }), + store.create({ + ownerUserId: owner.id, + agentId, + channelId: channel.id, + instruction: "Racer two.", + cron: DAILY, + }), + ]); + + const refused = outcomes.filter( + (outcome) => outcome.status === "rejected", + ) as PromiseRejectedResult[]; + expect(outcomes.filter((o) => o.status === "fulfilled")).toHaveLength(1); + expect(refused).toHaveLength(1); + // The loser gets the friendly sentence, not a constraint violation: the same refusal a + // sequential twenty-first create gets. + expect(refused[0]?.reason).toBeInstanceOf(RoutineRefusedError); + + const enabled = (await store.listFor(owner.id)).filter( + (summary) => summary.enabled, + ); + expect(enabled).toHaveLength(MAX_ENABLED_ROUTINES); + }); }); /** @@ -805,6 +855,34 @@ describe("moving a routine's clock", () => { expect(after?.lastRunAt).toBeNull(); }); + test("an explicit computeFrom drains a stale clock current in one move, CAS intact", async () => { + const { routine } = await makeRoutine(); + const from = await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z")); + + // The sweep's stale-backlog call: compare against the month-old stamp it read, but land on the + // occurrence after the moment the sweep is standing in — not one day along a 31-day drain. + const moved = await store.advanceNextRun( + routine.id, + from, + new Date("2001-02-01T10:00:00Z"), + ); + + expect(moved).toBe(true); + const after = await readRoutine(routine.id); + expect(after?.nextRunAt.toISOString()).toBe("2001-02-02T09:00:00.000Z"); + // The bookmark still records the stamp the CAS compared against. + expect(after?.lastRunAt?.getTime()).toBe(from.getTime()); + + // And the CAS still guards: a second call holding the drained stamp moves nothing. + expect( + await store.advanceNextRun( + routine.id, + from, + new Date("2001-03-01T10:00:00Z"), + ), + ).toBe(false); + }); + test("advancing stamps last_run_at with the `from` it was given", async () => { const { routine } = await makeRoutine(); const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z")); @@ -959,29 +1037,37 @@ describe("opening and closing a run", () => { }); /** - * Every dispatch attempt that goes nowhere opens a run row and leaves it open — `insertRun` runs - * before `dispatch`, and a dispatch that throws never reaches `finishRun`. `dueRoutines`/the sweep's - * attempt cap eventually stops retrying, but nothing else closes those rows: `listFor` shows the - * newest one, so the page reads "running now" forever for a routine that never ran at all. - * - * `failOpenRuns` is the sweep's cleanup for exactly that: close every open (`status is null`) run for - * one routine as "failed", not just the newest, because every one of them was a real dispatch attempt - * that went nowhere — closing only the newest would still leave the others open and wrong. + * A server that dies mid-turn strands its run row open forever: the work item was finished on the + * 202, so no retry ever comes back for that row, and nothing else writes it — the routines page + * reads "running now" for a run no process is running. `reapAbandonedRuns` is the sweep's mop for + * exactly that, and its age bound is what keeps it off live work: a young open row may be a turn + * some server is still running, and closing it would turn the real `finishRun` into a silent no-op. */ -describe("closing the runs a dispatch never got to finish", () => { - test("closes every open run for the routine, leaves a finished one untouched, and reports the count", async () => { +describe("reaping the runs the server never finished", () => { + /** Age one run row past the reaper's cutoff by hand; the reaper compares on the database's clock. */ + async function ageRun(runId: string, byMs: number) { + await database + .update(routineRuns) + .set({ startedAt: new Date(Date.now() - byMs) }) + .where(eq(routineRuns.id, runId)); + } + + test("closes only the rows past the cutoff, as skipped, and leaves live work alone", async () => { const { routine } = await makeRoutine(); - const firstOpen = await store.insertRun(routine.id); - const secondOpen = await store.insertRun(routine.id); + const abandoned = await store.insertRun(routine.id); + const inFlight = await store.insertRun(routine.id); const finished = await store.insertRun(routine.id); await store.finishRun(finished.runId, "succeeded"); + await ageRun(abandoned.runId, 11 * 60_000); - const closed = await store.failOpenRuns( - routine.id, - "the server answered 503", + const reaped = await store.reapAbandonedRuns( + 10 * 60_000, + "the server did not finish this run; it may have restarted mid-turn", ); - expect(closed).toBe(2); + // At least the row this test aged: the reaper is deliberately not routine-scoped, so a stray + // abandoned row from elsewhere in the database may be swept up in the same call. + expect(reaped).toBeGreaterThanOrEqual(1); const rows = await database .select() @@ -989,25 +1075,25 @@ describe("closing the runs a dispatch never got to finish", () => { .where(eq(routineRuns.routineId, routine.id)); const byId = new Map(rows.map((row) => [row.id, row])); - for (const { runId } of [firstOpen, secondOpen]) { - const row = byId.get(runId); - expect(row?.status).toBe("failed"); - expect(row?.finishedAt).toBeInstanceOf(Date); - expect(row?.error).toBe("the server answered 503"); - } + const abandonedRow = byId.get(abandoned.runId); + expect(abandonedRow?.status).toBe("skipped"); + expect(abandonedRow?.finishedAt).toBeInstanceOf(Date); + expect(abandonedRow?.error).toContain("did not finish this run"); + + // The fresh open row is a turn some server may still be running: closing it would make the real + // finishRun a silent no-op, which is the exact overreach the age bound exists to prevent. + expect(byId.get(inFlight.runId)?.status).toBeNull(); - // The finished run's outcome is untouched: this cleanup closes leaked attempts, not runs that - // already have an outcome. - const finishedRow = byId.get(finished.runId); - expect(finishedRow?.status).toBe("succeeded"); - expect(finishedRow?.error).toBeNull(); + // And a run with an outcome already has its truth; the reaper closes abandonment, not history. + expect(byId.get(finished.runId)?.status).toBe("succeeded"); }); - test("caps the error the same way finishRun does", async () => { + test("caps the reason the same way finishRun does", async () => { const { routine } = await makeRoutine(); const { runId } = await store.insertRun(routine.id); + await ageRun(runId, 11 * 60_000); - await store.failOpenRuns(routine.id, "x".repeat(600)); + await store.reapAbandonedRuns(10 * 60_000, "x".repeat(600)); const [row] = await database .select() @@ -1015,12 +1101,58 @@ describe("closing the runs a dispatch never got to finish", () => { .where(eq(routineRuns.id, runId)); expect(row?.error).toHaveLength(MAX_RUN_ERROR); }); +}); - test("a routine with nothing open closes nothing", async () => { - const { routine } = await makeRoutine(); - expect(await store.failOpenRuns(routine.id, "no attempts to close")).toBe( - 0, +/** + * The sweep's off switch for a routine whose own schedule refuses to advance — a cron written before + * a validation existed, or hand-edited under it. Left enabled, such a routine throws out of + * `advanceNextRun` on every pass forever: its clock never moves and it burns a due slot each time. + */ +describe("switching off a routine the sweep cannot schedule", () => { + test("disables it and leaves a skipped run carrying the reason where the page reads it", async () => { + const { owner, routine } = await makeRoutine(); + + await store.markUnschedulable( + routine.id, + "Routines may run at most every 15 minutes.", ); + + const [summary] = await store.listFor(owner.id); + expect(summary?.enabled).toBe(false); + // The run row is the announcement: the sweep has no channel to say this in, so the page's + // last-run column is where the owner learns why their routine stopped. + expect(summary?.lastRun?.status).toBe("skipped"); + const [row] = await database + .select() + .from(routineRuns) + .where(eq(routineRuns.routineId, routine.id)); + expect(row?.error).toBe("Routines may run at most every 15 minutes."); + expect(row?.finishedAt).toBeInstanceOf(Date); + }); + + test("a skipped announcement neither counts as a failure nor breaks a streak", async () => { + const { routine } = await makeRoutine(); + const { runId } = await store.insertRun(routine.id); + await store.finishRun(runId, "failed", "it threw"); + + await store.markUnschedulable(routine.id, "unschedulable"); + + // The fatigue rule reads through the announcement to the real failure tail. + expect(await store.consecutiveFailures(routine.id)).toBe(1); + }); + + test("a routine deleted in the meantime is left alone", async () => { + const { owner, routine } = await makeRoutine(); + await store.remove(owner.id, routine.id); + + // Gone is gone: no throw, and no orphaned run row for a routine nobody can see. + await store.markUnschedulable(routine.id, "unschedulable"); + expect( + await database + .select() + .from(routineRuns) + .where(eq(routineRuns.routineId, routine.id)), + ).toEqual([]); }); }); From 4433e95672d517edf46caa37597240298ed9eaa4 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 12:03:18 -0300 Subject: [PATCH 42/45] Drop the attention table on purpose, and index routines by owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The attention revert kept migration 0020 while removing the schema declaration, which left drizzle's latest snapshot claiming a table the schema files no longer knew: the next unrelated db:generate silently emitted DROP TABLE attention_resolutions CASCADE inside whatever migration somebody happened to be writing. Migration 0022 makes that drop explicit and reviewable — the feature never shipped in a release — and restores the invariant that the latest snapshot matches the schema. Migration 0023 gives routines the (owner_user_id, enabled) index that listFor, countEnabled, the owner-scoped writes and the users cascade were all sequential-scanning without. --- .../0022_drop_attention_resolutions.sql | 1 + server/drizzle/0023_routines_owner_index.sql | 1 + server/drizzle/meta/0022_snapshot.json | 2801 ++++++++++++++++ server/drizzle/meta/0023_snapshot.json | 2822 +++++++++++++++++ server/drizzle/meta/_journal.json | 14 + server/src/db/schema/coworker.ts | 6 +- 6 files changed, 5644 insertions(+), 1 deletion(-) create mode 100644 server/drizzle/0022_drop_attention_resolutions.sql create mode 100644 server/drizzle/0023_routines_owner_index.sql create mode 100644 server/drizzle/meta/0022_snapshot.json create mode 100644 server/drizzle/meta/0023_snapshot.json diff --git a/server/drizzle/0022_drop_attention_resolutions.sql b/server/drizzle/0022_drop_attention_resolutions.sql new file mode 100644 index 00000000..23a2e579 --- /dev/null +++ b/server/drizzle/0022_drop_attention_resolutions.sql @@ -0,0 +1 @@ +DROP TABLE "attention_resolutions" CASCADE; \ No newline at end of file diff --git a/server/drizzle/0023_routines_owner_index.sql b/server/drizzle/0023_routines_owner_index.sql new file mode 100644 index 00000000..12c3670b --- /dev/null +++ b/server/drizzle/0023_routines_owner_index.sql @@ -0,0 +1 @@ +CREATE INDEX "routines_by_owner_idx" ON "routines" USING btree ("owner_user_id","enabled"); \ No newline at end of file diff --git a/server/drizzle/meta/0022_snapshot.json b/server/drizzle/meta/0022_snapshot.json new file mode 100644 index 00000000..78eb6249 --- /dev/null +++ b/server/drizzle/meta/0022_snapshot.json @@ -0,0 +1,2801 @@ +{ + "id": "3b578b1a-4ebf-4a96-a729-aefe25c7c1e3", + "prevId": "1ac1d8fc-e594-4a52-a87c-047ee1038713", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": ["channel_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": ["channel_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": ["last_message_agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": ["user_id", "channel_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": ["provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": ["user_id", "role"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": ["computer_id", "tool_call_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": ["user_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": ["routine_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": ["component_name", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": ["component_name", "function_name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": ["server_id", "name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": ["server_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": ["kind", "ref", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": ["skill_id", "ref"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": ["kind", "key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": ["built_in", "remote_ag_ui"] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": ["admin", "user"] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": ["public", "private"] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": ["succeeded", "failed", "skipped"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/0023_snapshot.json b/server/drizzle/meta/0023_snapshot.json new file mode 100644 index 00000000..9ec039ea --- /dev/null +++ b/server/drizzle/meta/0023_snapshot.json @@ -0,0 +1,2822 @@ +{ + "id": "a1053424-96eb-4ace-98e4-63ac0ea99060", + "prevId": "3b578b1a-4ebf-4a96-a729-aefe25c7c1e3", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": ["channel_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": ["channel_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": ["last_message_agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": ["user_id", "channel_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": ["provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": ["user_id", "role"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_page_frame": { + "name": "computer_page_frame", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "computer_page_frame_captured_idx": { + "name": "computer_page_frame_captured_idx", + "columns": [ + { + "expression": "captured_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "computer_page_frame_computer_id_tool_call_id_pk": { + "name": "computer_page_frame_computer_id_tool_call_id_pk", + "columns": ["computer_id", "tool_call_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": ["user_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routine_runs": { + "name": "routine_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "routine_id": { + "name": "routine_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "routine_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "routine_runs_by_routine_idx": { + "name": "routine_runs_by_routine_idx", + "columns": [ + { + "expression": "routine_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routine_runs_routine_id_routines_id_fk": { + "name": "routine_runs_routine_id_routines_id_fk", + "tableFrom": "routine_runs", + "tableTo": "routines", + "columnsFrom": ["routine_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.routines": { + "name": "routines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instruction": { + "name": "instruction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "next_run_at": { + "name": "next_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_run_at": { + "name": "last_run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "routines_due_idx": { + "name": "routines_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "routines_by_owner_idx": { + "name": "routines_by_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "routines_owner_user_id_users_id_fk": { + "name": "routines_owner_user_id_users_id_fk", + "tableFrom": "routines", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "routines_agent_id_agents_id_fk": { + "name": "routines_agent_id_agents_id_fk", + "tableFrom": "routines", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": ["component_name", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": ["component_name", "function_name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": ["server_id", "name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": ["server_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": ["kind", "ref", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": ["skill_id", "ref"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": ["kind", "key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": ["built_in", "remote_ag_ui"] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": ["admin", "user"] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": ["public", "private"] + }, + "public.routine_run_status": { + "name": "routine_run_status", + "schema": "public", + "values": ["succeeded", "failed", "skipped"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index 6e3f5217..b6330597 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -155,6 +155,20 @@ "when": 1787767359748, "tag": "0021_routines", "breakpoints": true + }, + { + "idx": 22, + "version": "7", + "when": 1787841150017, + "tag": "0022_drop_attention_resolutions", + "breakpoints": true + }, + { + "idx": 23, + "version": "7", + "when": 1787841174859, + "tag": "0023_routines_owner_index", + "breakpoints": true } ] } diff --git a/server/src/db/schema/coworker.ts b/server/src/db/schema/coworker.ts index 7040f006..514642a5 100644 --- a/server/src/db/schema/coworker.ts +++ b/server/src/db/schema/coworker.ts @@ -123,7 +123,11 @@ export const routines = pgTable( createdAt: createdAt(), updatedAt: updatedAt(), }, - (table) => [index("routines_due_idx").on(table.enabled, table.nextRunAt)], + (table) => [ + index("routines_due_idx").on(table.enabled, table.nextRunAt), + /** Owner-scoped reads and writes: listFor, countEnabled, and the users cascade all hit this. */ + index("routines_by_owner_idx").on(table.ownerUserId, table.enabled), + ], ); /** One row per firing, which is what the page's "last ran" and the fatigue rule read. */ From 63e851a666cf82c6e8f676e44685daba13d5ab08 Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 12:03:18 -0300 Subject: [PATCH 43/45] Say '2 minutes ago' from one module, not two copies --- .../components/app-sidebar/app-sidebar.tsx | 25 +------------------ app/src/components/routines/routines-list.tsx | 25 +------------------ app/src/lib/relative-time.ts | 23 +++++++++++++++++ 3 files changed, 25 insertions(+), 48 deletions(-) create mode 100644 app/src/lib/relative-time.ts diff --git a/app/src/components/app-sidebar/app-sidebar.tsx b/app/src/components/app-sidebar/app-sidebar.tsx index c512ed97..49052645 100644 --- a/app/src/components/app-sidebar/app-sidebar.tsx +++ b/app/src/components/app-sidebar/app-sidebar.tsx @@ -54,6 +54,7 @@ import { import { useChannelEvents } from "@/lib/channels/use-channel-events"; import { appConfig } from "@/lib/generated/application-config"; import { EASE_OUT, ENTRANCE_SECONDS } from "@/lib/motion"; +import { relativeTime } from "@/lib/relative-time"; import { Button } from "../ui/button"; import { Empty, EmptyDescription, EmptyHeader, EmptyTitle } from "../ui/empty"; import { Channel } from "./channel"; @@ -434,27 +435,3 @@ export function AppSidebar({ ...props }: React.ComponentProps) { ); } - -const RELATIVE_UNITS = [ - { limit: 60_000, divisor: 1_000, unit: "second" }, - { limit: 3_600_000, divisor: 60_000, unit: "minute" }, - { limit: 86_400_000, divisor: 3_600_000, unit: "hour" }, - { limit: 604_800_000, divisor: 86_400_000, unit: "day" }, - { limit: Number.POSITIVE_INFINITY, divisor: 604_800_000, unit: "week" }, -] as const; - -const relativeFormat = new Intl.RelativeTimeFormat(undefined, { - numeric: "auto", -}); - -/** Locale-aware relative timestamp, e.g. "2 minutes ago". */ -function relativeTime(iso: string) { - const elapsed = Date.now() - new Date(iso).getTime(); - const scale = - RELATIVE_UNITS.find(({ limit }) => Math.abs(elapsed) < limit) ?? - RELATIVE_UNITS[RELATIVE_UNITS.length - 1]; - return relativeFormat.format( - -Math.round(elapsed / scale.divisor), - scale.unit, - ); -} diff --git a/app/src/components/routines/routines-list.tsx b/app/src/components/routines/routines-list.tsx index 6ffb8c5b..4697d746 100644 --- a/app/src/components/routines/routines-list.tsx +++ b/app/src/components/routines/routines-list.tsx @@ -25,6 +25,7 @@ import { } from "@/components/ui/item"; import { Separator } from "@/components/ui/separator"; import { Switch } from "@/components/ui/switch"; +import { relativeTime } from "@/lib/relative-time"; import { deleteRoutineMutationOptions, setRoutineEnabledMutationOptions, @@ -35,30 +36,6 @@ import { } from "@/lib/routines/queries"; import { queryClient } from "@/query-client"; -const RELATIVE_UNITS = [ - { limit: 60_000, divisor: 1_000, unit: "second" }, - { limit: 3_600_000, divisor: 60_000, unit: "minute" }, - { limit: 86_400_000, divisor: 3_600_000, unit: "hour" }, - { limit: 604_800_000, divisor: 86_400_000, unit: "day" }, - { limit: Number.POSITIVE_INFINITY, divisor: 604_800_000, unit: "week" }, -] as const; - -const relativeFormat = new Intl.RelativeTimeFormat(undefined, { - numeric: "auto", -}); - -/** Locale-aware relative timestamp, e.g. "2 minutes ago". */ -function relativeTime(iso: string): string { - const elapsed = Date.now() - new Date(iso).getTime(); - const scale = - RELATIVE_UNITS.find(({ limit }) => Math.abs(elapsed) < limit) ?? - RELATIVE_UNITS[RELATIVE_UNITS.length - 1]; - return relativeFormat.format( - -Math.round(elapsed / scale.divisor), - scale.unit, - ); -} - /** * What the last-run cell says, and in what tone. * diff --git a/app/src/lib/relative-time.ts b/app/src/lib/relative-time.ts new file mode 100644 index 00000000..d05c076d --- /dev/null +++ b/app/src/lib/relative-time.ts @@ -0,0 +1,23 @@ +const RELATIVE_UNITS = [ + { limit: 60_000, divisor: 1_000, unit: "second" }, + { limit: 3_600_000, divisor: 60_000, unit: "minute" }, + { limit: 86_400_000, divisor: 3_600_000, unit: "hour" }, + { limit: 604_800_000, divisor: 86_400_000, unit: "day" }, + { limit: Number.POSITIVE_INFINITY, divisor: 604_800_000, unit: "week" }, +] as const; + +const relativeFormat = new Intl.RelativeTimeFormat(undefined, { + numeric: "auto", +}); + +/** Locale-aware relative timestamp, e.g. "2 minutes ago". */ +export function relativeTime(iso: string): string { + const elapsed = Date.now() - new Date(iso).getTime(); + const scale = + RELATIVE_UNITS.find(({ limit }) => Math.abs(elapsed) < limit) ?? + RELATIVE_UNITS[RELATIVE_UNITS.length - 1]; + return relativeFormat.format( + -Math.round(elapsed / scale.divisor), + scale.unit, + ); +} From 7e0f43b6d83be9466d7990b55bcece81838192aa Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 12:03:18 -0300 Subject: [PATCH 44/45] Ask a surviving server whether the worker can reach it before keeping it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit start.sh leaves an answering server alone, and that philosophy kept a server started before WORKER_SHARED_SECRET existed: the worker then got 401 for every handoff and routines never fired, with nothing at start time saying why. Now the script probes /internal/routines/run with this run's secret — 400 means the secret was accepted and only the empty body refused, 401 or 404 means the server cannot take handoffs and is restarted into this run's environment. --- scripts/start.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/scripts/start.sh b/scripts/start.sh index 024bb818..74beebab 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -267,6 +267,39 @@ if [ "$SECRETS_ROTATED" = "true" ]; then pkill -f "bun --env-file=../.env src/index.ts" >/dev/null 2>&1 || true sleep 1 fi +# +# A server that answers as OpenBot can still be one no worker can hand a routine to. The worker +# below is started unconditionally with this run's WORKER_SHARED_SECRET, and every dispatch it makes +# is one POST to this server's /internal/routines/run — a door that a server started from an older +# checkout does not have (404), and that a server started before this secret existed in its +# environment holds shut (401, `workerSharedSecret` undefined or different). Either way routines +# never fire, and nothing at start time says why. +# +# So before keeping a running server, ask it the one thing only a compatible one answers well: POST +# the handoff route with this run's secret and a deliberately empty body. In server/src/app.ts the +# secret is checked before the body is parsed, so a server holding this same secret rejects the +# empty body with 400 — the healthy answer. 401 means it does not hold this secret; 404 means it +# predates the route. Both are cured by a restart into this run's environment, so fall through to +# the launch below. Anything else — including a probe that could not connect at all — keeps the +# philosophy of leaving an answering server alone. +if identifies_as_openbot "$SERVER_PORT" server; then + HANDOFF_STATUS="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 3 \ + -X POST "http://localhost:$SERVER_PORT/internal/routines/run" \ + -H "Authorization: Bearer $WORKER_SHARED_SECRET" \ + -H "Content-Type: application/json" --data '{}' 2>/dev/null || true)" + case "$HANDOFF_STATUS" in + 401) + info " server: up, but refuses the worker's secret (401), so it is restarted to pick it up" + pkill -f "bun --env-file=../.env src/index.ts" >/dev/null 2>&1 || true + sleep 1 + ;; + 404) + info " server: up, but has no /internal/routines/run (404: an older checkout), so it is restarted" + pkill -f "bun --env-file=../.env src/index.ts" >/dev/null 2>&1 || true + sleep 1 + ;; + esac +fi if ! identifies_as_openbot "$SERVER_PORT" server; then if [ "$ONE_COMPUTER_EACH" = "true" ]; then (cd server && PORT="$SERVER_PORT" \ From 4115d9b32293c3613633a7c492e4e5b0647879ba Mon Sep 17 00:00:00 2001 From: Guido Vizoso Date: Thu, 27 Aug 2026 12:03:18 -0300 Subject: [PATCH 45/45] Let the store's own types and caps speak for the routine tools --- server/src/plugins/builtin-routines.ts | 35 ++++++++------------------ 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/server/src/plugins/builtin-routines.ts b/server/src/plugins/builtin-routines.ts index c21861da..e0371420 100644 --- a/server/src/plugins/builtin-routines.ts +++ b/server/src/plugins/builtin-routines.ts @@ -1,8 +1,10 @@ import { + MAX_RUN_ERROR, RoutineNotFoundError, RoutineRefusedError, type Routine, type RoutinePatch, + type RoutineStore, type RoutineSummary, } from "../routines/store"; import { MAX_RESULT_CHARS, type McpCallResult, type McpTool } from "./mcp"; @@ -23,9 +25,6 @@ import { MAX_RESULT_CHARS, type McpCallResult, type McpTool } from "./mcp"; * arrives through {@link useRoutineTools} rather than through a constructor — see the comment there. */ -/** No fetch, no vendor, no third party: the store's own refusal cap, in code points. */ -const MAX_FAILURE_CODE_POINTS = 400; - /** * What the tools act on. * @@ -33,23 +32,10 @@ const MAX_FAILURE_CODE_POINTS = 400; * absent, because nothing a model calls has any business advancing a clock or opening a run row. A * store satisfies this structurally, so wiring it is one call and no adapter. */ -export type RoutineTools = { - create(input: { - ownerUserId: string; - agentId: string; - channelId?: string; - instruction: string; - cron: string; - timezone?: string; - }): Promise; - listFor(ownerUserId: string): Promise; - update( - ownerUserId: string, - id: string, - patch: RoutinePatch, - ): Promise; - remove(ownerUserId: string, id: string): Promise; -}; +export type RoutineTools = Pick< + RoutineStore, + "create" | "listFor" | "update" | "remove" +>; let installed: RoutineTools | null = null; @@ -489,11 +475,10 @@ export async function callTool( return failure("There is no routine of yours with that id."); } // Anything else is a bug or a broken database, and it still has to come back as a sentence - // rather than as a thrown error mid-turn. Capped in code points, like the store caps a run's - // error, so a message carrying an emoji cannot be cut mid-surrogate-pair. + // rather than as a thrown error mid-turn. Capped at the store's own refusal cap, in code + // points like the store caps a run's error, so a message carrying an emoji cannot be cut + // mid-surrogate-pair. const message = error instanceof Error ? error.message : String(error); - return failure( - Array.from(message).slice(0, MAX_FAILURE_CODE_POINTS).join(""), - ); + return failure(Array.from(message).slice(0, MAX_RUN_ERROR).join("")); } }