Run OpenBot on Kubernetes: Bots and all, proven on EKS

.github/workflows/ci.yml

@@ -73,6 +73,83 @@ jobs:
       - run: bun run typecheck
         working-directory: ${{ matrix.package }}
 
+  chart:
+    name: chart (${{ matrix.target }})
+    runs-on: ubuntu-latest
+    strategy:
+      # One red target must not hide whether another is red too.
+      fail-fast: false
+      matrix:
+        target: [self-hosted, eks, eks-sandbox, gke, aks]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
+        with:
+          version: v3.19.0
+      # For the coherence check below, which is a Bun script like everything else here.
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version: 1.3.14
+      # Nothing rendered this chart until now, which is how four values files that produce a server
+      # unable to start were shipped and stayed shipped. Rendering is the cheap half; the refusals in
+      # validation.yaml are the half that catches a missing value before a cluster does.
+      - run: helm dependency build charts/openbot
+      # Structure only. `helm lint` reports a template `fail` as an INFO line and exits 0 even under
+      # `--strict`, which was driven and confirmed, so it cannot gate the refusals below. Rendering
+      # does: `helm template` exits non-zero on one.
+      - run: helm lint charts/openbot --values charts/openbot/ci/${{ matrix.target }}-values.yaml
+      # A key encryption key is a real 32 bytes rather than a placeholder, because the chart checks
+      # its shape. Generated here so no example key is ever a literal in this repository.
+      - name: Render
+        run: |
+          helm template ci charts/openbot \
+            --values charts/openbot/ci/${{ matrix.target }}-values.yaml \
+            --set-string secrets.keyEncryptionKey="$(openssl rand -base64 32)" \
+            --api-versions agents.x-k8s.io/v1beta1/Sandbox \
+            --api-versions extensions.agents.x-k8s.io/v1beta1/SandboxTemplate \
+            > rendered.yaml
+      # Rendering proves the templates run. This proves the result is coherent, which is a different
+      # question: every secret key a container demands has to be one the chart actually writes.
+      # Getting that wrong is invisible until a pod starts, and every shipped target had it wrong.
+      - run: bun scripts/check-rendered-chart.ts rendered.yaml
+      # And that the refusals are load-bearing rather than decorative. A chart full of `fail`
+      # messages nothing ever triggers is a chart that has never been shown to refuse anything, and
+      # every one of these describes a state that shipped in a values file at some point.
+      - name: Refusals fire
+        run: |
+          set -uo pipefail
+          refuses() {
+            local why="$1"; shift
+            if helm template ci charts/openbot \
+                 --values charts/openbot/ci/${{ matrix.target }}-values.yaml \
+                 --set-string secrets.keyEncryptionKey="$(openssl rand -base64 32)" \
+                 --api-versions agents.x-k8s.io/v1beta1/Sandbox \
+                 --api-versions extensions.agents.x-k8s.io/v1beta1/SandboxTemplate \
+                 "$@" >/dev/null 2>&1; then
+              echo "::error::The chart rendered $why, which it is supposed to refuse."
+              return 1
+            fi
+            echo "refused: $why"
+          }
+          # The public example key, which the server will not start with. Only where this chart
+          # holds the secret: with a store, the value is not readable at template time, so the
+          # refusal is deliberately not armed and asserting it here would be asserting a bug.
+          if ! grep -qE '^ *enabled: true' <(sed -n '/^externalSecrets:/,/^[a-z]/p' charts/openbot/ci/${{ matrix.target }}-values.yaml); then
+            refuses "the public example encryption key" \
+              --set-string secrets.keyEncryptionKey="$(head -c 32 /dev/zero | base64)"
+          else
+            echo "skipped: the example-key refusal is not armed when the secret comes from a store"
+          fi
+          # A Bot endpoint with nothing on the request that says who is calling. Armed whether the
+          # secret is this chart's or a store's, because the key list is readable either way.
+          refuses "a managed agent URL with no token" \
+            --set-string config.managedAgent.url=http://agent.default:8000/ag-ui
+          # A browser inside every replica of a replicated API.
+          refuses "an embedded browser across several replicas" \
+            --set server.embeddedComputer=true --set server.replicaCount=2
+
   test:
     name: tests
     runs-on: ubuntu-latest
@@ -261,7 +338,7 @@ jobs:
     name: verify
     runs-on: ubuntu-latest
     if: always()
-    needs: [static, deployables, test, build, migrations, image]
+    needs: [static, deployables, chart, test, build, migrations, image]
     steps:
       - name: Require every check
         env:

.gitignore

@@ -21,3 +21,8 @@ app/src/lib/generated/application-config.ts
 
 # TanStack Router scratch output
 app/.tanstack/
+
+# Helm subchart tarballs, fetched by `helm dependency build`. The lock beside them is NOT ignored:
+# it is what makes that fetch reproducible, and ignoring it meant every build resolved the dependency
+# afresh, so CI and a customer install could take different subchart versions with no diff to show it.
+charts/*/charts/

CHANGELOG.md

@@ -8,6 +8,134 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### A finished turn shows the page it opened, not the one open now
+
+Reopening a conversation made every past turn fetch the screen as it is now, so an answer about
+Hacker News from an hour ago sat under a picture of whatever the Bot had open since.
+
+A page is now photographed where it is opened. The server takes the frame the moment a navigation
+succeeds and keeps it in `computer_page_frame` under the computer and the address, which is the one
+moment the screen is certainly showing the page that was asked for. Reopening the conversation shows
+that frame rather than the live screen, and a turn with nothing kept names the page instead of
+drawing the wrong one.
+
+The surface used to capture it itself once the turn went quiet, and that is a race it cannot win: a
+reopened turn and one that has just finished are indistinguishable from inside the component, the
+same computer is driven by other conversations in between, and a resumed computer starts blank. It
+filed pictures of pages the turn never opened, or none at all. It only reads now.
+
+**Redeploy the computers with the server.** A screenshot only says which page it is of on an
+`agent-computer` built after that field was added, and this is what decides whether a frame is kept.
+Where each Bot has a computer of its own there is nobody to race with, so an old computer's picture is
+accepted and the feature works through a rollout. On ONE SHARED COMPUTER it cannot be: another Bot's
+navigation lands between the navigation and the picture, and a frame that cannot be told apart from
+theirs is refused. So a shared-computer deployment that updates the server and not the computer keeps
+no frames until it does, and says so in the server log each time rather than leaving somebody to
+wonder.
+
+Two things followed from making a past turn a record. Its placeholder is decided by the turn being
+over rather than by whether a live frame happens to be in hand, because a tile that was live a moment
+ago keeps its last screenshot and used to fall through to "Waiting for the assistant's screen…" and
+wait there for ever. And opening one full size shows that same kept frame, with no live stream and no
+wheel: zooming a past turn used to mount the socket and offer Take control, so the one gesture for
+looking closer at what a turn did replaced it with whatever the Bot has open now.
+
+### A conversation keeps the browsing that produced its answers
+
+Every turn in which a Bot used a tool was disappearing from the transcript on reload. The sentence
+the Bot wrote stayed; the browsing that produced it did not, the inline screen went with it, and the
+footer said some messages could not be read.
+
+The history store writes a tool call as `{id, name, args}`. AG-UI describes
+`{id, type: "function", function: {name, arguments}}`. The reader validated against the second,
+treated the first as damage from an interrupted run, and dropped it. It is not damage: it is how
+every tool call is stored, so what looked like a guard against one bad turn was deleting all of the
+real ones. Observed on a live thread where every browsing turn was counted unreadable and every one
+of them was well formed in the store's own dialect.
+
+The two spellings are now read as the same thing. The check stays for turns that really are
+malformed, and a mixed or unrecognised array is still refused rather than half-translated, because a
+reader that rewrites what it does not recognise is worse than one that refuses it.
+
+
+### Run this on Kubernetes
+
+A Helm chart under `charts/openbot`, Bots and all, and the fixes that installing it for real turned
+up. Proven on a real EKS cluster: five workloads, replicas across two nodes, EBS volumes bound, and a
+Bot opening a real page from inside AWS with the decision in the audit trail.
+
+One chart, four targets: EKS, GKE, AKS and somebody's own cluster, with nothing but values between
+them. There is no cloud branching in any template. Every place the clouds genuinely differ is a
+value whose default is what a plain self-hosted cluster does: the cluster's own default StorageClass,
+no RuntimeClass, a plain Kubernetes Secret, an Ingress. Identity is one `serviceAccount.annotations`
+map, which is all IRSA, Workload Identity and AKS workload identity are. Secrets are a plain Secret
+by default and an ExternalSecret against any backend when asked, so Secrets Manager, Secret Manager
+and Key Vault are a values block rather than three code paths. Gateway API is supported beside
+Ingress rather than instead of it. `charts/openbot/ci` holds a values file per target.
+
+Two replicas by default, because horizontal is the point and one replica hides every bug that is
+not. A bad install is refused at `helm install`, naming the value to change, rather than discovered
+in a crash loop: no database or two of them, nobody who could sign in, nobody who would be an
+administrator, a key of the wrong shape, both routers enabled, or a browser asked for inside more
+than one replica.
+
+**A Bot's computer is not in an API pod.** The image runs one beside the API so that a single
+container works on its own, and `EMBEDDED_COMPUTER=off` turns it off. A replica must not carry a
+browser: it is a few hundred megabytes holding one Bot's logins, so scaling the API would scale
+those with it.
+
+**Migrations no longer need a development tool.** `bun x drizzle-kit migrate` cannot run in the
+shipped image at all. The CLI reads a TypeScript config, which needs the esbuild that
+`bun install --production` correctly leaves out, so it printed "Reading config file", exited 1 and
+said nothing else. `EMBEDDED_POSTGRES=on` was therefore starting a container whose database was
+never migrated, and the first symptom was the API reporting that `users` does not exist.
+`server/scripts/migrate.ts` uses the migrator inside `drizzle-orm`, which is a runtime dependency
+already, and keeps the same journal, so a database migrated by either tool is migrated.
+
+**A computer for each Bot, suspended when idle.** `computers.mode: sandbox` gives every Bot its own
+browser as a `Sandbox` from `kubernetes-sigs/agent-sandbox`, which is built for this workload: an
+isolated, stateful, singleton pod with a stable identity and persistent storage. Suspending is one
+field, and it keeps the volumes, so a computer comes back with its logins rather than signed out of
+everything. `shared` stays the default and needs nothing installed in the cluster.
+
+**The NetworkPolicy would have fenced the API off from its own work.** Its egress named DNS and the
+bundled database and nothing else, so on a cluster that enforces policy the API could not have
+reached a Bot's computer or, with a managed database, the database. Both are allowed now, and turning
+the policy on with an external database and no rule for it is refused rather than shipped. Worth
+knowing either way: EKS runs its CNI with `--enable-network-policy=false`, so a policy there installs,
+looks right, and does nothing at all.
+
+**A cluster with no controller is refused at install.** `computers.mode: sandbox` needs the
+agent-sandbox CRD, and without it the install succeeds, every pod is healthy, and the deployment
+looks finished until the first Bot asks for a browser. The chart reads the cluster and refuses,
+naming the one command that fixes it.
+
+**What decides a computer is idle is the audit trail, not the browser.** Asking the browser would
+wake it, so every computer anything asked about would come back up and the bill would never fall.
+
+**Durable work, claimed by whichever replica gets there first.** `work_items` plus
+`select ... for update skip locked` and a lease: no coordinator, no leader election, and a replica
+added is throughput added. The idle-computer culler is its first user; scheduled routines and
+hand-offs between Bots are the other two, which is why it is written once rather than three times
+slightly differently. A CronJob runs the sweep, because a timer in the API fires in every replica and
+suspending a browser somebody just started using is not something to do five times.
+
+**Which run of a computer this is, across a suspend.** A resumed browser counts snapshot
+generations from one again, so a ref the model still holds from before the suspend would match a row
+nothing has overwritten and the boundary would decide about an element on a page that no longer
+exists. The first answer here used the node and the pod address, and resuming a real computer
+disproved it: a suspended sandbox is very often rescheduled onto the same node and handed the same
+address back, so both were identical across a suspend and resume and the check would have said "same
+run" for the exact case it exists to catch. It reads the `Ready` condition's transition time instead,
+which moves every time a computer starts serving again.
+
+**Which run of a computer this is, on more than one replica.** `sessionOf` answered from a map in
+the process that started the computer, which is right until there are two: the replica that took a
+snapshot is usually not the one handling the click, and the second had nothing to answer with. An
+unknown session means "no opinion" and skips the generation check, so on exactly the deployment
+shape it was written for, the check that stops a ref from a replaced computer resolving against a
+live one was silently absent. It now asks the supervisor when it does not know, by listing rather
+than by ensuring, so asking never starts a computer that had stopped.
 ### Notion joins the connector catalogue
 
 Notion is now a governed MCP connector, reached through Notion's own hosted server on the
@@ -35,7 +163,6 @@ read that as a stolen token and revoke the whole connection. Every plugin call t
 token now locks the credential's vault row for the length of the exchange, so a second replica waits
 rather than races, and the rotated token is written back in the same transaction that held the lock.
 Nothing to configure; a connection just stops going stale under concurrent traffic.
-
 ### Knowledge searches instead of guessing
 
 A package can say which of its skills each coworker gets, and the fintech example gives Knowledge the

agent-bot/src/history.ts

@@ -63,16 +63,7 @@ export function toProviderMessages(
       const toolCalls = message.toolCalls?.map((call) => ({
         id: call.id,
         type: "function" as const,
-        function: {
-          /*
-           * A name is required by the provider and is not always present: read back from the thread
-           * store these arrive undefined, and a payload carrying `"name": undefined` is rejected
-           * outright. The call still has to be shown, or the model repeats an action it already
-           * took, so it keeps its id and is named as something the model can read.
-           */
-          name: call.function?.name ?? "tool",
-          arguments: call.function?.arguments ?? "{}",
-        },
+        function: callDetails(call),
       }));
       messages.push({
         role: "assistant",
@@ -109,3 +100,36 @@ export function toProviderMessages(
 
   return messages;
 }
+
+/**
+ * A tool call's name and arguments, in whichever dialect it arrived in.
+ *
+ * TWO SPELLINGS, ONE CALL. AG-UI describes `{id, type: "function", function: {name, arguments}}` and
+ * the history store writes `{id, name, args}`. Read back from a thread, every call arrives in the
+ * second, so code reaching straight for `call.function` finds nothing there.
+ *
+ * That was diagnosed here as "the name is not always present" and papered over with a default, which
+ * turned every restored call into a tool named `tool` with no arguments. The model is then shown a
+ * call it cannot recognise as the one it made, so it makes it again: the exact repetition the
+ * fallback was written to prevent.
+ */
+function callDetails(call: {
+  function?: { name?: unknown; arguments?: unknown };
+  name?: unknown;
+  args?: unknown;
+}): { name: string; arguments: string } {
+  const name = call.function?.name ?? call.name;
+  const args = call.function?.arguments ?? call.args;
+  return {
+    // Still defaulted, because a call with no name at all is rejected outright by the provider and
+    // showing the model something is better than losing the turn. It is now the last resort it was
+    // meant to be rather than the ordinary path.
+    name: typeof name === "string" && name ? name : "tool",
+    arguments:
+      typeof args === "string"
+        ? args
+        : args === undefined || args === null
+          ? "{}"
+          : JSON.stringify(args),
+  };
+}

agent-bot/tests/history.test.ts

@@ -207,3 +207,47 @@ describe("a history that arrives out of order", () => {
     expect(assistant.tool_calls?.[0]?.function.name).toBe("tool");
   });
 });
+
+/**
+ * A call read back from the thread store arrives in the store's dialect, not AG-UI's.
+ *
+ * `{id, name, args}` rather than `{id, type, function: {name, arguments}}`. Reaching straight for
+ * `call.function` finds nothing there, and the default underneath turned every restored call into a
+ * tool named `tool` with no arguments: the model is shown a call it cannot recognise as the one it
+ * made, so it makes it again. That is the repetition the default was written to prevent.
+ */
+describe("a tool call restored from the thread store", () => {
+  test("keeps the name and arguments it was made with", () => {
+    const messages = toProviderMessages({
+      messages: [
+        {
+          id: "m1",
+          role: "assistant",
+          content: null,
+          toolCalls: [
+            {
+              id: "call_1",
+              name: "computer_navigate",
+              args: '{"url":"https://news.ycombinator.com"}',
+            },
+          ],
+        },
+        {
+          id: "m2",
+          role: "tool",
+          toolCallId: "call_1",
+          content: '{"ok":true}',
+        },
+      ],
+    } as never);
+
+    const withCalls = messages.find(
+      (message: Record<string, unknown>) => message.tool_calls,
+    ) as Record<string, unknown>;
+    const call = (withCalls.tool_calls as Array<Record<string, unknown>>)[0];
+    const fn = call.function as Record<string, unknown>;
+
+    expect(fn.name).toBe("computer_navigate");
+    expect(fn.arguments).toBe('{"url":"https://news.ycombinator.com"}');
+  });
+});