feat(supervisor): configurable tolerations for run pods (#4491)

## Summary

Self-hosted Kubernetes deployments can now add tolerations to run pods,
so runs
can schedule onto tainted nodes. Previously the only way to do this was
to patch
the supervisor.

`KUBERNETES_RUNNER_TOLERATIONS` takes a comma separated list of
`key=value:effect`, or `key:effect` to tolerate any value. It applies to
every
run pod, and for runs from a schedule tree it merges with the existing
`KUBERNETES_SCHEDULED_RUN_TOLERATIONS`. Left unset, nothing changes: no
tolerations are added and the pod spec leaves the field off entirely.

The Helm chart takes it as a list:

```yaml
supervisor:
  config:
    kubernetes:
      runnerTolerations:
        - dedicated=runs:NoSchedule
        - spot:NoExecute
```

## Naming

The issue proposed `KUBERNETES_WORKER_TOLERATIONS`. This ships as
`KUBERNETES_RUNNER_TOLERATIONS` instead, because `RUNNER_*` is already
the prefix
for run pod settings (`RUNNER_HEARTBEAT_INTERVAL_SECONDS`,
`RUNNER_ADDITIONAL_ENV_VARS`, and `DOCKER_RUNNER_NETWORKS` for the
Docker
equivalent), whereas "worker" refers to the supervisor itself throughout
this app.

## Validation

Keys and values are checked against the Kubernetes naming rules when the
supervisor starts, so `dedicated=prod runs:NoSchedule` fails immediately
with a
message naming the offending entry. Without that check a bad value is
accepted at
startup and then rejected by the API server on every pod create, which
stops all
runs with the cause buried in an API error.
`KUBERNETES_WORKER_NODETYPE_LABEL` is
trimmed and validated for the same reason: surrounding whitespace is not
valid in
a label value, so a padded value fails every pod create today.

## Node selector off switch

`KUBERNETES_WORKER_NODETYPE_LABEL` accepts an empty string to skip the
node
selector entirely, so runs schedule on any node. This already worked and
the Helm
chart has always shipped it empty, but it was not documented. It is now.

The issue also asked for general node affinity configuration. That is
not
included: the node selector off switch plus tolerations covers the
reported
problem, and a free form affinity setting is a much larger config
surface to
commit to.

Fixes #4458

Author: nicktrn

.server-changes/supervisor-run-pod-tolerations.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: feature
+---
+
+Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes. An invalid toleration now stops the supervisor at startup instead of failing every run pod, so check existing values before upgrading.

apps/supervisor/src/env.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from "crypto";
 import { env as stdEnv } from "std-env";
 import { z } from "zod";
-import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
+import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 export const Env = z
   .object({
@@ -173,7 +173,7 @@ export const Env = z
     // Kubernetes settings
     KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
     KUBERNETES_NAMESPACE: z.string().default("default"),
-    KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
+    KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"),
     KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
@@ -256,65 +256,8 @@ export const Env = z
       .max(100)
       .default(20),
 
-    // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool
-    // Comma-separated list of tolerations in the format: key=value:effect
-    // For Exists operator (no value): key:effect
-    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z
-      .string()
-      .transform((val, ctx) => {
-        const tolerations = val
-          .split(",")
-          .map((entry) => entry.trim())
-          .filter((entry) => entry.length > 0)
-          .map((entry) => {
-            const colonIdx = entry.lastIndexOf(":");
-            if (colonIdx === -1) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (missing effect): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            const effect = entry.slice(colonIdx + 1);
-            const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
-            if (!validEffects.includes(effect)) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
-                  ", "
-                )}`,
-              });
-              return z.NEVER;
-            }
-
-            const keyValue = entry.slice(0, colonIdx);
-            const eqIdx = keyValue.indexOf("=");
-            const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx);
-
-            if (!key) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (empty key): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            if (eqIdx === -1) {
-              return { key, operator: "Exists" as const, effect };
-            }
-
-            return {
-              key,
-              operator: "Equal" as const,
-              value: keyValue.slice(eqIdx + 1),
-              effect,
-            };
-          });
-
-        return tolerations;
-      })
-      .optional(),
+    KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod
+    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only
 
     // Placement tags settings
     PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),

apps/supervisor/src/envUtil.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { BoolEnv, AdditionalEnvVars } from "./envUtil.js";
+import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 describe("BoolEnv", () => {
   it("should parse string 'true' as true", () => {
@@ -78,3 +78,128 @@ describe("AdditionalEnvVars", () => {
     });
   });
 });
+
+describe("NodeLabelValue", () => {
+  it("should keep a clean value untouched", () => {
+    expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker");
+  });
+
+  it("should trim surrounding whitespace, which Kubernetes would reject", () => {
+    expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker");
+    expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker");
+  });
+
+  it("should treat a whitespace-only value as the empty off-switch", () => {
+    expect(NodeLabelValue.parse("")).toBe("");
+    expect(NodeLabelValue.parse("   ")).toBe("");
+  });
+
+  it("should still apply a default only when unset", () => {
+    const withDefault = NodeLabelValue.default("v4-worker");
+    expect(withDefault.parse(undefined)).toBe("v4-worker");
+    expect(withDefault.parse("")).toBe("");
+  });
+
+  it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => {
+    for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) {
+      expect(NodeLabelValue.safeParse(invalid).success).toBe(false);
+    }
+  });
+});
+
+describe("Tolerations", () => {
+  it("should parse key=value entries as Equal", () => {
+    expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse entries without a value as Exists", () => {
+    expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([
+      { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" },
+    ]);
+  });
+
+  it("should keep an empty value as an exact match for a valueless taint", () => {
+    expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" },
+    ]);
+
+    expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Exists", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse an empty string as no tolerations", () => {
+    expect(Tolerations.parse("")).toEqual([]);
+    expect(Tolerations.parse("  ")).toEqual([]);
+  });
+
+  it("should skip blank entries and trim whitespace", () => {
+    expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([
+      { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a missing effect, an unknown effect, and an empty key", () => {
+    for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => {
+    expect(
+      Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule")
+    ).toEqual([
+      { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" },
+      { key: "spot", operator: "Exists", effect: "NoExecute" },
+      { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should accept a DNS-subdomain prefixed key", () => {
+    expect(
+      Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule")
+    ).toEqual([
+      {
+        key: "node.cluster.x-k8s.io/machinepool",
+        operator: "Equal",
+        value: "scheduled-runs",
+        effect: "NoSchedule",
+      },
+    ]);
+  });
+
+  it("should reject a key or value that Kubernetes would reject at pod create", () => {
+    for (const invalid of [
+      "dedicated=prod runs:NoSchedule",
+      "ded icated=runs:NoSchedule",
+      "dedicated=-runs:NoSchedule",
+      `dedicated=${"r".repeat(64)}:NoSchedule`,
+      `${"a".repeat(64)}=runs:NoSchedule`,
+      `example.com/${"a".repeat(64)}=runs:NoSchedule`,
+      "a/b/c=runs:NoSchedule",
+      "Example.com/pool=runs:NoSchedule",
+    ]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should bound the prefix and the name separately, as Kubernetes does", () => {
+    const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`;
+    expect(longestPrefix.length).toBe(253);
+
+    expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1);
+    expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false);
+  });
+
+  it("should tolerate whitespace around the separators", () => {
+    expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a stray extra effect instead of folding it into the value", () => {
+    expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false);
+  });
+});

apps/supervisor/src/envUtil.ts

@@ -16,6 +16,136 @@ export const BoolEnv = baseBoolEnv as Omit<typeof baseBoolEnv, "default"> & {
   default: (value: boolean) => z.ZodDefault<typeof baseBoolEnv>;
 };
 
+const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/;
+const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/;
+const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/;
+const QUALIFIED_NAME_MAX = 63;
+const DNS_SUBDOMAIN_MAX = 253;
+const LABEL_VALUE_MAX = 63;
+
+/**
+ * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream.
+ */
+function isLabelValue(value: string): boolean {
+  return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value);
+}
+
+/**
+ * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and
+ * label keys: an optional DNS subdomain prefix before the slash, then the name.
+ * The two halves have different length limits and different case rules, so a
+ * single pattern with one overall bound gets both ends wrong.
+ */
+function isQualifiedName(key: string): boolean {
+  const slashIdx = key.indexOf("/");
+
+  if (slashIdx === -1) {
+    return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key);
+  }
+
+  const prefix = key.slice(0, slashIdx);
+  const name = key.slice(slashIdx + 1);
+
+  return (
+    prefix.length <= DNS_SUBDOMAIN_MAX &&
+    DNS_SUBDOMAIN.test(prefix) &&
+    name.length <= QUALIFIED_NAME_MAX &&
+    QUALIFIED_NAME.test(name)
+  );
+}
+
+/**
+ * A node label value. Trimmed because Kubernetes rejects surrounding whitespace
+ * outright, so a padded value fails every pod create. Deliberately no `min(1)`:
+ * empty is the off-switch, and the Helm chart ships empty by default.
+ */
+export const NodeLabelValue = z.string().trim().refine(isLabelValue, {
+  message:
+    "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters",
+});
+
+/**
+ * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect`
+ * for the Exists operator. Keys and values are checked against the Kubernetes
+ * naming rules here so a typo fails at startup, rather than 422ing every single
+ * pod create with the cause buried in an API server message.
+ */
+export const Tolerations = z.string().transform((val, ctx) => {
+  return val
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0)
+    .map((entry) => {
+      const colonIdx = entry.lastIndexOf(":");
+      if (colonIdx === -1) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (missing effect): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      const effect = entry.slice(colonIdx + 1).trim();
+      const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
+      if (!validEffects.includes(effect)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
+            ", "
+          )}`,
+        });
+        return z.NEVER;
+      }
+
+      const keyValue = entry.slice(0, colonIdx);
+      const eqIdx = keyValue.indexOf("=");
+      const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim();
+
+      if (!key) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (empty key): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      if (!isQualifiedName(key)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`,
+        });
+        return z.NEVER;
+      }
+
+      if (eqIdx === -1) {
+        return { key, operator: "Exists" as const, effect };
+      }
+
+      const value = keyValue.slice(eqIdx + 1).trim();
+      if (!value) {
+        logger.warn(
+          'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.',
+          { entry, key }
+        );
+      }
+
+      if (!isLabelValue(value)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`,
+        });
+        return z.NEVER;
+      }
+
+      return {
+        key,
+        operator: "Equal" as const,
+        value,
+        effect,
+      };
+    });
+});
+
 export const AdditionalEnvVars = z.preprocess((val) => {
   if (typeof val !== "string") {
     return val;