feat(supervisor): configurable tolerations for run pods

.server-changes/supervisor-run-pod-tolerations.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: feature
+---
+
+Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes. An invalid toleration now stops the supervisor at startup instead of failing every run pod, so check existing values before upgrading.

apps/supervisor/src/env.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from "crypto";
 import { env as stdEnv } from "std-env";
 import { z } from "zod";
-import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
+import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 export const Env = z
   .object({
@@ -173,7 +173,7 @@ export const Env = z
     // Kubernetes settings
     KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
     KUBERNETES_NAMESPACE: z.string().default("default"),
-    KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
+    KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"),
     KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
@@ -256,65 +256,8 @@ export const Env = z
       .max(100)
       .default(20),
 
-    // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool
-    // Comma-separated list of tolerations in the format: key=value:effect
-    // For Exists operator (no value): key:effect
-    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z
-      .string()
-      .transform((val, ctx) => {
-        const tolerations = val
-          .split(",")
-          .map((entry) => entry.trim())
-          .filter((entry) => entry.length > 0)
-          .map((entry) => {
-            const colonIdx = entry.lastIndexOf(":");
-            if (colonIdx === -1) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (missing effect): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            const effect = entry.slice(colonIdx + 1);
-            const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
-            if (!validEffects.includes(effect)) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
-                  ", "
-                )}`,
-              });
-              return z.NEVER;
-            }
-
-            const keyValue = entry.slice(0, colonIdx);
-            const eqIdx = keyValue.indexOf("=");
-            const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx);
-
-            if (!key) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (empty key): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            if (eqIdx === -1) {
-              return { key, operator: "Exists" as const, effect };
-            }
-
-            return {
-              key,
-              operator: "Equal" as const,
-              value: keyValue.slice(eqIdx + 1),
-              effect,
-            };
-          });
-
-        return tolerations;
-      })
-      .optional(),
+    KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod
+    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only
 
     // Placement tags settings
     PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),

apps/supervisor/src/envUtil.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { BoolEnv, AdditionalEnvVars } from "./envUtil.js";
+import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 describe("BoolEnv", () => {
   it("should parse string 'true' as true", () => {
@@ -78,3 +78,128 @@ describe("AdditionalEnvVars", () => {
     });
   });
 });
+
+describe("NodeLabelValue", () => {
+  it("should keep a clean value untouched", () => {
+    expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker");
+  });
+
+  it("should trim surrounding whitespace, which Kubernetes would reject", () => {
+    expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker");
+    expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker");
+  });
+
+  it("should treat a whitespace-only value as the empty off-switch", () => {
+    expect(NodeLabelValue.parse("")).toBe("");
+    expect(NodeLabelValue.parse("   ")).toBe("");
+  });
+
+  it("should still apply a default only when unset", () => {
+    const withDefault = NodeLabelValue.default("v4-worker");
+    expect(withDefault.parse(undefined)).toBe("v4-worker");
+    expect(withDefault.parse("")).toBe("");
+  });
+
+  it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => {
+    for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) {
+      expect(NodeLabelValue.safeParse(invalid).success).toBe(false);
+    }
+  });
+});
+
+describe("Tolerations", () => {
+  it("should parse key=value entries as Equal", () => {
+    expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse entries without a value as Exists", () => {
+    expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([
+      { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" },
+    ]);
+  });
+
+  it("should keep an empty value as an exact match for a valueless taint", () => {
+    expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" },
+    ]);
+
+    expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Exists", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse an empty string as no tolerations", () => {
+    expect(Tolerations.parse("")).toEqual([]);
+    expect(Tolerations.parse("  ")).toEqual([]);
+  });
+
+  it("should skip blank entries and trim whitespace", () => {
+    expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([
+      { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a missing effect, an unknown effect, and an empty key", () => {
+    for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => {
+    expect(
+      Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule")
+    ).toEqual([
+      { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" },
+      { key: "spot", operator: "Exists", effect: "NoExecute" },
+      { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should accept a DNS-subdomain prefixed key", () => {
+    expect(
+      Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule")
+    ).toEqual([
+      {
+        key: "node.cluster.x-k8s.io/machinepool",
+        operator: "Equal",
+        value: "scheduled-runs",
+        effect: "NoSchedule",
+      },
+    ]);
+  });
+
+  it("should reject a key or value that Kubernetes would reject at pod create", () => {
+    for (const invalid of [
+      "dedicated=prod runs:NoSchedule",
+      "ded icated=runs:NoSchedule",
+      "dedicated=-runs:NoSchedule",
+      `dedicated=${"r".repeat(64)}:NoSchedule`,
+      `${"a".repeat(64)}=runs:NoSchedule`,
+      `example.com/${"a".repeat(64)}=runs:NoSchedule`,
+      "a/b/c=runs:NoSchedule",
+      "Example.com/pool=runs:NoSchedule",
+    ]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should bound the prefix and the name separately, as Kubernetes does", () => {
+    const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`;
+    expect(longestPrefix.length).toBe(253);
+
+    expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1);
+    expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false);
+  });
+
+  it("should tolerate whitespace around the separators", () => {
+    expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a stray extra effect instead of folding it into the value", () => {
+    expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false);
+  });
+});

apps/supervisor/src/envUtil.ts

@@ -16,6 +16,136 @@ export const BoolEnv = baseBoolEnv as Omit<typeof baseBoolEnv, "default"> & {
   default: (value: boolean) => z.ZodDefault<typeof baseBoolEnv>;
 };
 
+const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/;
+const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/;
+const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/;
+const QUALIFIED_NAME_MAX = 63;
+const DNS_SUBDOMAIN_MAX = 253;
+const LABEL_VALUE_MAX = 63;
+
+/**
+ * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream.
+ */
+function isLabelValue(value: string): boolean {
+  return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value);
+}
+
+/**
+ * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and
+ * label keys: an optional DNS subdomain prefix before the slash, then the name.
+ * The two halves have different length limits and different case rules, so a
+ * single pattern with one overall bound gets both ends wrong.
+ */
+function isQualifiedName(key: string): boolean {
+  const slashIdx = key.indexOf("/");
+
+  if (slashIdx === -1) {
+    return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key);
+  }
+
+  const prefix = key.slice(0, slashIdx);
+  const name = key.slice(slashIdx + 1);
+
+  return (
+    prefix.length <= DNS_SUBDOMAIN_MAX &&
+    DNS_SUBDOMAIN.test(prefix) &&
+    name.length <= QUALIFIED_NAME_MAX &&
+    QUALIFIED_NAME.test(name)
+  );
+}
+
+/**
+ * A node label value. Trimmed because Kubernetes rejects surrounding whitespace
+ * outright, so a padded value fails every pod create. Deliberately no `min(1)`:
+ * empty is the off-switch, and the Helm chart ships empty by default.
+ */
+export const NodeLabelValue = z.string().trim().refine(isLabelValue, {
+  message:
+    "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters",
+});
+
+/**
+ * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect`
+ * for the Exists operator. Keys and values are checked against the Kubernetes
+ * naming rules here so a typo fails at startup, rather than 422ing every single
+ * pod create with the cause buried in an API server message.
+ */
+export const Tolerations = z.string().transform((val, ctx) => {
+  return val
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0)
+    .map((entry) => {
+      const colonIdx = entry.lastIndexOf(":");
+      if (colonIdx === -1) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (missing effect): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      const effect = entry.slice(colonIdx + 1).trim();
+      const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
+      if (!validEffects.includes(effect)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
+            ", "
+          )}`,
+        });
+        return z.NEVER;
+      }
+
+      const keyValue = entry.slice(0, colonIdx);
+      const eqIdx = keyValue.indexOf("=");
+      const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim();
+
+      if (!key) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (empty key): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      if (!isQualifiedName(key)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`,
+        });
+        return z.NEVER;
+      }
+
+      if (eqIdx === -1) {
+        return { key, operator: "Exists" as const, effect };
+      }
+
+      const value = keyValue.slice(eqIdx + 1).trim();
+      if (!value) {
+        logger.warn(
+          'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.',
+          { entry, key }
+        );
+      }
+
+      if (!isLabelValue(value)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`,
+        });
+        return z.NEVER;
+      }
+
+      return {
+        key,
+        operator: "Equal" as const,
+        value,
+        effect,
+      };
+    });
+});
+
 export const AdditionalEnvVars = z.preprocess((val) => {
   if (typeof val !== "string") {
     return val;