diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bdc7671..98fc7192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,25 @@ the membership row like the pin — so one person reading does not clear anybody The deployment gains one nullable column, via migration `0019`. +### The API can reach Intelligence and sign-in when a NetworkPolicy is on + +`networkPolicy.enabled` wrote a rule for the API server that named DNS, the database and the Bots' +computers, and nothing on 443. On a cluster that enforces policy the server could therefore reach +neither CopilotKit Intelligence, nor an identity provider, nor a Bot: nobody could sign in and no +conversation ran. Two of the five shipped `ci/` targets turn the policy on, and on GKE enforcement is +the default and cannot be switched off. + +Nothing said so. The pod passed every probe and stayed Ready, because `/health` answers from a +literal, so the first evidence was a timeout to a hostname that read as the internet being down. + +The API now reaches HTTP and HTTPS everywhere outside the cluster's private ranges, in every +`computers.mode` rather than only `sandbox`, cut by the same exception list the computers' own policy +uses. It still cannot address another pod, a node, or a cloud metadata endpoint. + +`mode: sandbox` had been working only because a rule meant for the Kubernetes API server carried no +destination and so permitted everything. That rule now covers the API server alone, and +`networkPolicy.kubernetesApiCidr` narrows it to your cluster's service range; left empty it stays as +it was, because a chart cannot know that range. ### A finished turn shows the page it opened, not the one open now diff --git a/charts/openbot/templates/networkpolicy.yaml b/charts/openbot/templates/networkpolicy.yaml index 973a093e..d188a428 100644 --- a/charts/openbot/templates/networkpolicy.yaml +++ b/charts/openbot/templates/networkpolicy.yaml @@ -7,8 +7,16 @@ Off by default, because a NetworkPolicy on a cluster with no CNI that enforces o that silently does nothing, and on a cluster that does enforce one a wrong rule is an outage. A deployment that turns this on is saying it knows which of the two it has. -Egress deliberately allows DNS and the database, and nothing else without being asked: a Bot's -computer reaching the open internet is the computers' own policy, not the API's. +Egress allows DNS, the database, the computers, and HTTP and HTTPS to everywhere that is not the +cluster's own private network. THAT LAST ONE IS NOT A CONCESSION, it is what this pod does: sign-in +goes to an identity provider, every conversation goes to Intelligence, and every run goes to a Bot +at an address somebody registered. All three are hostnames rather than CIDRs, and a NetworkPolicy +cannot match a hostname, so there is no narrower rule to write. Leaving it out did not fence the API +off, it stopped the product working, and only in `computers.mode: sandbox` did a rule meant for the +Kubernetes API server quietly cover for it. + +The private ranges stay cut out by exception, the way the computers' policy does it, so this is +still a pod that cannot address another pod, a node, or a cloud metadata endpoint. */}} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -62,9 +70,43 @@ spec: - port: 4100 protocol: TCP {{- end }} + {{- /* + Intelligence, the identity provider, and every Bot: all of them, and all outside the cluster. + + Written as an exception list rather than as a destination list because the destinations are + hostnames the deployment configures and a NetworkPolicy matches addresses. The same shape the + computers' policy below already uses, for the same reason. + */}} + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + # The cluster and everything else on the private network, including the database. + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + # Link-local, which is where every cloud keeps the endpoint that hands out credentials. + - 169.254.0.0/16 + ports: + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP {{- if eq .Values.computers.mode "sandbox" }} - {{- /* The API server, which is where a per-Bot computer is asked for. */}} - - ports: + {{- /* + The Kubernetes API server, which is where a per-Bot computer is asked for. + + Its own rule because it sits on the private network the rule above cuts out, so nothing else + here reaches it. Unscoped unless a deployment says otherwise: the API server answers on a + ClusterIP from the service range, and a chart cannot know that range at template time. Name it + in `networkPolicy.kubernetesApiCidr` and this narrows to it. + */}} + - {{- with .Values.networkPolicy.kubernetesApiCidr }} + to: + - ipBlock: + cidr: {{ . }} + {{- end }} + ports: - port: 443 protocol: TCP - port: 6443 diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml index da8e7596..dbb42f81 100644 --- a/charts/openbot/values.yaml +++ b/charts/openbot/values.yaml @@ -365,8 +365,18 @@ httpRoute: networkPolicy: enabled: false # Where the API may reach out to. A deployment with a managed database adds its CIDR here. + # + # It already reaches HTTP and HTTPS everywhere outside the cluster's private ranges, because that + # is where Intelligence, sign-in and the Bots are. This is for anything on the private side. extraEgress: [] extraIngress: [] + # `computers.mode: sandbox` only. The service range the Kubernetes API server answers on, so the + # rule that lets the API ask for a Bot's computer can name it instead of being left open. + # + # Empty means unscoped, which is the only thing a chart can do by default: the range is the + # cluster's, not the release's. `kubectl get svc kubernetes -o jsonpath='{.spec.clusterIP}'` shows + # which one yours is on; on EKS it is usually 172.20.0.0/16, on GKE and kubeadm 10.96.0.0/12. + kubernetesApiCidr: "" # Where a Bot's computer may reach beyond the public internet. A deployment whose Bots must reach # an internal site adds it here, one address at a time, rather than reopening the private ranges. computerExtraEgress: [] diff --git a/scripts/check-rendered-chart.ts b/scripts/check-rendered-chart.ts index 12e68145..dc5de112 100644 --- a/scripts/check-rendered-chart.ts +++ b/scripts/check-rendered-chart.ts @@ -127,6 +127,33 @@ for (const [name, keys] of written) { } } +/** + * A policy that fences the API off from the services it cannot work without. + * + * The same question as the Secret one above, asked of the other thing a render can be internally + * wrong about: this chart requires CopilotKit Intelligence and an identity provider, reaches both + * over HTTPS at hostnames, and also writes the rule that says where the API may go. Those two had + * never been compared. The server's egress named DNS, the database and the computers, so on any + * cluster that enforces policy nobody could sign in and no conversation ran — and the pod stayed + * Ready throughout, because `/health` answers from a literal. + * + * Asked of the rendered object rather than the template, because the rule that covered for this was + * conditional on `computers.mode` and only one mode ever had it. + */ +const serverPolicy = documents.find( + (document) => + /^kind:\s*NetworkPolicy\s*$/m.test(document) && + /app\.kubernetes\.io\/component:\s*server/.test(document), +); +if (serverPolicy) { + const egress = serverPolicy.split(/^\s{2}egress:\s*$/m)[1] ?? ""; + if (!/port:\s*443\b/.test(egress)) { + problems.push( + "The server's NetworkPolicy has no egress on 443, so the API cannot reach Intelligence or an identity provider. Nothing would report it: /health answers from a literal and every probe reads it.", + ); + } +} + if (problems.length > 0) { for (const problem of problems) console.error(`::error::${problem}`); process.exit(1); @@ -134,6 +161,7 @@ if (problems.length > 0) { console.log( `${documents.length} objects, ${demands.length} secret keys demanded, and every required one is written.` + + (serverPolicy ? " The server's egress reaches 443." : "") + (skippedOptional > 0 ? ` ${skippedOptional} optional key${skippedOptional === 1 ? " was" : "s were"} not checked.` : ""),