From 30a92303fdddfc45ace5d64a3d4643dfd67f1a25 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 20 Oct 2026 22:39:31 +0800 Subject: [PATCH] feat(featureflags): read per-node limit fallbacks from the environment A deployment with no LAUNCH_DARKLY_API_KEY serves every flag's compiled-in fallback, so an int flag resolves to a value only a rebuild can change. That is fine for flags whose fallback is a safe default, but not for the per-node limits: max-sandboxes-per-node falls back to 200, which is sized for cloud node types rather than for the host it runs on. A self-hosted node with 384 CPUs and 1.5 TB of memory refuses the 201st sandbox with ResourceExhausted, and the only way past it is editing a literal in flags.go. Read the fallback of the two per-node limits from the environment: max-sandboxes-per-node (MAX_SANDBOXES_PER_NODE) and max-starting-instances-per-node (MAX_STARTING_INSTANCES_PER_NODE). The override applies to the fallback rather than to the evaluated value, so both modes stay consistent: without a LaunchDarkly key it is the value the offline store serves, and with a key set it is the default for flags that the LaunchDarkly environment does not define. A defined LaunchDarkly flag still wins, so live configuration is not bypassed. Both limits go through envPositiveIntOr, a thin wrapper added around the existing envIntOr. envIntOr deliberately accepts zero and negative values, since pause-admission-disk-headroom-mib uses -1 to turn its check off, but these two limits must stay positive, so the wrapper keeps the fallback for non-positive values. A typo then cannot silently stop a node from accepting sandboxes, and it matches the "Must be > 0" requirement already documented on MaxStartingInstancesPerNode. Signed-off-by: Wei Wang --- packages/shared/pkg/featureflags/flags.go | 17 +++++++++-- .../pkg/featureflags/flags_envint_test.go | 28 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 621a2535dd..3761679304 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -163,6 +163,19 @@ func envIntOr(key string, fallback int) int { return parsed } +// envPositiveIntOr is envIntOr for limits that must stay positive: a zero or +// negative value keeps the fallback, so a typo cannot silently stop a node +// from accepting sandboxes. max-sandboxes-per-node's fallback is sized for +// cloud node types rather than for the host it runs on, which is why the +// per-node limits read their fallback from the environment. +func envPositiveIntOr(key string, fallback int) int { + if v := envIntOr(key, fallback); v > 0 { + return v + } + + return fallback +} + func NewBoolFlag(name string, fallback bool) BoolFlag { flag := BoolFlag{name: name, fallback: fallback} builder := launchDarklyOfflineStore.Flag(flag.name).VariationForAll(fallback) @@ -568,7 +581,7 @@ var ( // future retirements. EgressRetirementTimeoutMsFlag = NewIntFlag("egress-retirement-timeout-ms", 60000) - MaxSandboxesPerNode = NewIntFlag("max-sandboxes-per-node", 200) + MaxSandboxesPerNode = NewIntFlag("max-sandboxes-per-node", envPositiveIntOr("MAX_SANDBOXES_PER_NODE", 200)) APIIdempotencyRoutes = NewJSONFlag("api-idempotency-routes", ldvalue.Null()) APIIdempotencyTTLSeconds = NewIntFlag("api-idempotency-ttl-seconds", 86400) // The LD keys keep the legacy "gcloud-" prefix, but the limits apply to uploads on all storage providers. @@ -761,7 +774,7 @@ var ( // MaxStartingInstancesPerNode limits concurrent sandbox start/resume operations on a single orchestrator node. // Must be > 0. - MaxStartingInstancesPerNode = NewIntFlag("max-starting-instances-per-node", 3) + MaxStartingInstancesPerNode = NewIntFlag("max-starting-instances-per-node", envPositiveIntOr("MAX_STARTING_INSTANCES_PER_NODE", 3)) // MaxConcurrentEvictions caps the number of sandbox evictions that can run // in parallel per API instance. Excess items remain expired in the store diff --git a/packages/shared/pkg/featureflags/flags_envint_test.go b/packages/shared/pkg/featureflags/flags_envint_test.go index 4a7ecf1bf2..6b6fbb07fe 100644 --- a/packages/shared/pkg/featureflags/flags_envint_test.go +++ b/packages/shared/pkg/featureflags/flags_envint_test.go @@ -34,3 +34,31 @@ func TestEnvIntOr(t *testing.T) { }) } } + +//nolint:paralleltest,tparallel // t.Setenv +func TestEnvPositiveIntOr(t *testing.T) { + const key = "MAX_SANDBOXES_PER_NODE_TEST_ONLY" + + for _, tc := range []struct { + name string + set bool + value string + fallback int + want int + }{ + {name: "unset keeps the fallback", fallback: 200, want: 200}, + {name: "empty keeps the fallback", set: true, value: "", fallback: 200, want: 200}, + {name: "a value overrides the fallback", set: true, value: "1200", fallback: 200, want: 1200}, + {name: "garbage keeps the fallback", set: true, value: "many", fallback: 200, want: 200}, + {name: "zero keeps the fallback", set: true, value: "0", fallback: 200, want: 200}, + {name: "negative keeps the fallback", set: true, value: "-1", fallback: 200, want: 200}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.set { + t.Setenv(key, tc.value) + } + + require.Equal(t, tc.want, envPositiveIntOr(key, tc.fallback)) + }) + } +}