feat(supervisor): harden org placement override parsing and visibility

Blank env value now means no overrides instead of failing startup.
Tolerations also accept an array of entries to match the Helm list
shape, scalar node selector values are coerced to strings, keys and
values are trimmed, and empty selector values are rejected since they
would pin the org to nothing. Startup fails when an override pins the
large-machine pool while non-large presets are required to stay off
it. The supervisor logs configured override orgs at startup and warns
when an override replaces a default node selector key.

Author: myftija

apps/supervisor/src/env.ts

@@ -266,9 +266,10 @@ export const Env = z
     KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod
     KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only
 
-    // Per-org placement overrides, JSON keyed by org id:
-    // {"<orgId>": {"nodeSelector": {"<key>": "<value>"}, "tolerations": "<csv>"}}
-    KUBERNETES_ORG_PLACEMENT_OVERRIDES: OrgPlacementOverrides.optional(),
+    // Per-org placement overrides, JSON keyed by the internal org id
+    // (the `org` label on run pods):
+    // {"<orgId>": {"nodeSelector": {"<key>": "<value>"}, "tolerations": "<csv or array>"}}
+    KUBERNETES_ORG_PLACEMENT_OVERRIDES: OrgPlacementOverrides,
 
     // Placement tags settings
     PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),
@@ -315,6 +316,22 @@ export const Env = z
         path: ["TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE"],
       });
     }
+    if (data.KUBERNETES_LARGE_MACHINE_AFFINITY_ENABLED && data.KUBERNETES_ORG_PLACEMENT_OVERRIDES) {
+      // Non-large presets carry a hard NotIn on the large-machine pool, so an org
+      // pinned to that pool could never schedule its non-large runs.
+      for (const [orgId, override] of Object.entries(data.KUBERNETES_ORG_PLACEMENT_OVERRIDES)) {
+        const pinnedPool =
+          override.nodeSelector?.[data.KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_KEY];
+
+        if (pinnedPool === data.KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_VALUE) {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `Org "${orgId}" pins run pods to the large-machine pool, but non-large presets are required to stay off it, so those runs would never schedule. Use a different pool or disable KUBERNETES_LARGE_MACHINE_AFFINITY_ENABLED.`,
+            path: ["KUBERNETES_ORG_PLACEMENT_OVERRIDES"],
+          });
+        }
+      }
+    }
     if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_METADATA_URL) {
       ctx.addIssue({
         code: z.ZodIssueCode.custom,

apps/supervisor/src/envUtil.test.ts

@@ -251,6 +251,53 @@ describe("OrgPlacementOverrides", () => {
     }
   });
 
+  it("should treat a blank or missing value as no overrides, like the sibling settings", () => {
+    expect(OrgPlacementOverrides.parse(undefined)).toBeUndefined();
+    expect(OrgPlacementOverrides.parse("")).toBeUndefined();
+    expect(OrgPlacementOverrides.parse("   ")).toBeUndefined();
+  });
+
+  it("should accept tolerations as an array of entries, matching the Helm list shape", () => {
+    expect(
+      OrgPlacementOverrides.parse(
+        JSON.stringify({
+          org_123: { tolerations: ["dedicated=pool:NoSchedule", "spot:NoExecute"] },
+        })
+      )
+    ).toEqual({
+      org_123: {
+        tolerations: [
+          { key: "dedicated", operator: "Equal", value: "pool", effect: "NoSchedule" },
+          { key: "spot", operator: "Exists", effect: "NoExecute" },
+        ],
+      },
+    });
+  });
+
+  it("should coerce scalar node selector values to strings, as Kubernetes labels are", () => {
+    expect(
+      OrgPlacementOverrides.parse(
+        JSON.stringify({ org_123: { nodeSelector: { paid: true, replicas: 3 } } })
+      )
+    ).toEqual({ org_123: { nodeSelector: { paid: "true", replicas: "3" } } });
+  });
+
+  it("should trim whitespace around node selector keys and values", () => {
+    expect(
+      OrgPlacementOverrides.parse(JSON.stringify({ org_123: { nodeSelector: { " pool ": " a " } } }))
+    ).toEqual({ org_123: { nodeSelector: { pool: "a" } } });
+  });
+
+  it("should reject an empty node selector value instead of pinning the org to nothing", () => {
+    for (const value of ["", "   "]) {
+      expect(
+        OrgPlacementOverrides.safeParse(
+          JSON.stringify({ org_123: { nodeSelector: { pool: value } } })
+        ).success
+      ).toBe(false);
+    }
+  });
+
   it("should reject an unknown field, so a typo cannot silently drop an override", () => {
     expect(
       OrgPlacementOverrides.safeParse(

apps/supervisor/src/envUtil.ts

@@ -146,34 +146,68 @@ export const Tolerations = z.string().transform((val, ctx) => {
     });
 });
 
-const NodeSelector = z.record(z.string(), z.string()).superRefine((selector, ctx) => {
-  for (const [key, value] of Object.entries(selector)) {
-    if (!isQualifiedName(key)) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Invalid node selector key "${key}". Must be a Kubernetes label key, optionally prefixed with a DNS subdomain.`,
-      });
-    }
+/**
+ * Scalar values are coerced: YAML/JSON easily produce `true` or `3` where a label
+ * value is meant, and Kubernetes label values are always strings. An empty value
+ * is rejected rather than passed through - as a selector it matches only nodes
+ * carrying a literal empty-valued label, which pins the org to nothing.
+ */
+const NodeSelector = z
+  .record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
+  .transform((selector, ctx) => {
+    const result: Record<string, string> = {};
 
-    if (!isLabelValue(value)) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Invalid node selector value "${value}" for key "${key}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters.`,
-      });
+    for (const [rawKey, rawValue] of Object.entries(selector)) {
+      const key = rawKey.trim();
+      const value = String(rawValue).trim();
+
+      if (!isQualifiedName(key)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid node selector key "${rawKey}". Must be a Kubernetes label key, optionally prefixed with a DNS subdomain.`,
+        });
+        continue;
+      }
+
+      if (!value) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Empty node selector value for key "${key}". Remove the key instead of blanking the value.`,
+        });
+        continue;
+      }
+
+      if (!isLabelValue(value)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid node selector value "${value}" for key "${key}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters.`,
+        });
+        continue;
+      }
+
+      result[key] = value;
     }
-  }
-});
+
+    return result;
+  });
 
 /**
- * Per-organization placement overrides for run pods, as JSON keyed by org id:
+ * Per-organization placement overrides for run pods, as JSON keyed by the
+ * internal org id (the `org` label on run pods):
  * `{"<orgId>": {"nodeSelector": {"<key>": "<value>"}, "tolerations": "<csv>"}}`.
- * Tolerations use the same CSV format as `Tolerations`. Everything is validated
- * at startup for the same reason as tolerations above: a typo would otherwise
- * reject every pod create for that org, with the cause buried in API errors.
+ * Tolerations use the same CSV format as `Tolerations`, or an array of such
+ * entries. Everything is validated at startup for the same reason as
+ * tolerations above: a typo would otherwise reject every pod create for that
+ * org, with the cause buried in API errors. A blank value means no overrides.
  */
 export const OrgPlacementOverrides = z
   .string()
+  .optional()
   .transform((val, ctx) => {
+    if (val === undefined || val.trim() === "") {
+      return undefined;
+    }
+
     try {
       return JSON.parse(val) as unknown;
     } catch {
@@ -185,15 +219,21 @@ export const OrgPlacementOverrides = z
     }
   })
   .pipe(
-    z.record(
-      z.string().min(1),
-      z
-        .object({
-          nodeSelector: NodeSelector.optional(),
-          tolerations: Tolerations.optional(),
-        })
-        .strict()
-    )
+    z
+      .record(
+        z.string().min(1),
+        z
+          .object({
+            nodeSelector: NodeSelector.optional(),
+            tolerations: z
+              .union([z.string(), z.array(z.string())])
+              .transform((val) => (Array.isArray(val) ? val.join(",") : val))
+              .pipe(Tolerations)
+              .optional(),
+          })
+          .strict()
+      )
+      .optional()
   );
 
 export const AdditionalEnvVars = z.preprocess((val) => {

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -70,6 +70,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
         domain: opts.workloadApiDomain,
       });
     }
+
+    if (env.KUBERNETES_ORG_PLACEMENT_OVERRIDES) {
+      this.logger.info("[KubernetesWorkloadManager] Org placement overrides enabled", {
+        orgIds: Object.keys(env.KUBERNETES_ORG_PLACEMENT_OVERRIDES),
+      });
+    }
   }
 
   private addPlacementTags(
@@ -112,10 +118,23 @@ export class KubernetesWorkloadManager implements WorkloadManager {
 
     try {
       const orgOverride = env.KUBERNETES_ORG_PLACEMENT_OVERRIDES?.[opts.orgId];
-      const basePodSpec = withNodeSelector(
-        this.addPlacementTags(this.#defaultPodSpec, opts.placementTags),
-        orgOverride?.nodeSelector
-      );
+      const taggedPodSpec = this.addPlacementTags(this.#defaultPodSpec, opts.placementTags);
+      const basePodSpec = withNodeSelector(taggedPodSpec, orgOverride?.nodeSelector);
+
+      if (orgOverride?.nodeSelector) {
+        const replacedKeys = Object.keys(orgOverride.nodeSelector).filter(
+          (key) =>
+            taggedPodSpec.nodeSelector?.[key] !== undefined &&
+            taggedPodSpec.nodeSelector[key] !== orgOverride.nodeSelector?.[key]
+        );
+
+        if (replacedKeys.length > 0) {
+          this.logger.warn(
+            "[KubernetesWorkloadManager] Org placement override replaces node selector keys",
+            { orgId: opts.orgId, replacedKeys }
+          );
+        }
+      }
       const podSpec = this.opts.checkpointsEnabled
         ? withBlockIoUringSeccompProfile(basePodSpec, opts.runtime)
         : basePodSpec;

docs/self-hosting/env/supervisor.mdx

@@ -48,7 +48,7 @@ mode: "wide"
 | `KUBERNETES_NAMESPACE`                      | No       | default     | The namespace that runs should be in.                       |
 | `KUBERNETES_WORKER_NODETYPE_LABEL`          | No       | v4-worker   | Nodes for runs need `nodetype=<this>`. Empty: any node.     |
 | `KUBERNETES_RUNNER_TOLERATIONS`             | No       | —           | Run pod tolerations. CSV: `key=value:effect`/`key:effect`.  |
-| `KUBERNETES_ORG_PLACEMENT_OVERRIDES`        | No       | —           | Per-org node selector and tolerations. JSON keyed by org ID.|
+| `KUBERNETES_ORG_PLACEMENT_OVERRIDES`        | No       | —           | Per-org node selector and tolerations for run pods, merged over the defaults. JSON keyed by the org's internal ID (the `org` label on run pods). |
 | `KUBERNETES_IMAGE_PULL_SECRETS`             | No       | —           | Image pull secrets (CSV).                                   |
 | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT`   | No       | 10Gi        | Ephemeral storage size limit. Applies to all runs.          |
 | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST` | No       | 2Gi         | Ephemeral storage size request. Applies to all runs.        |

hosting/k8s/helm/values.yaml

@@ -297,7 +297,9 @@ supervisor:
       namespace: "" # Default: uses release namespace
       workerNodetypeLabel: "" # When set, runs will only be scheduled on nodes with "nodetype=<label>"
       runnerTolerations: [] # Run pod tolerations, e.g. ["dedicated=runs:NoSchedule"]
-      orgPlacementOverrides: {} # Per-org run pod placement, e.g. {"<orgId>": {"nodeSelector": {"pool": "dedicated"}, "tolerations": "dedicated=runs:NoSchedule"}}
+      # Per-org run pod placement, keyed by the org's internal ID (the `org` label on run pods).
+      # Merged over the default node selector. e.g. {"<orgId>": {"nodeSelector": {"pool": "dedicated"}, "tolerations": ["dedicated=runs:NoSchedule"]}}
+      orgPlacementOverrides: {}
       ephemeralStorageSizeLimit: "" # Default: 10Gi
       ephemeralStorageSizeRequest: "" # Default: 2Gi´
     podCleaner: