From 039653ef51156219babe42aa30f6b2afbc10083d Mon Sep 17 00:00:00 2001 From: root Date: Fri, 4 Sep 2026 10:32:50 +0200 Subject: [PATCH] scheduler: snapshot job volumes in placement request to fix data race The StartJob goroutine iterates job.Volumes to create any pending volumes on the selected host, while the main scheduler loop can replace job.Volumes wholesale in handleActiveJob. This is a concurrent read/write race on the slice field. Snapshot the assigned volumes into the PlacementRequest when it is handled in the main loop, and have StartJob iterate that snapshot instead of the mutable job.Volumes field. --- controller/scheduler/scheduler.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/controller/scheduler/scheduler.go b/controller/scheduler/scheduler.go index 50e16b2a1..a908b1e1b 100755 --- a/controller/scheduler/scheduler.go +++ b/controller/scheduler/scheduler.go @@ -1168,6 +1168,11 @@ func (s *Scheduler) HandlePlacementRequest(req *PlacementRequest) { req.Job.Volumes[i] = vol } + // snapshot the assigned volumes so the StartJob goroutine can + // create pending volumes on the host without racing with + // handleActiveJob replacing job.Volumes + req.Volumes = req.Job.Volumes + if req.Host != nil { log.Info(fmt.Sprintf("placed job on host with existing %s volumes", req.Job.Type), "host.id", req.Host.ID) } @@ -1611,7 +1616,7 @@ outer: continue } - for _, vol := range job.Volumes { + for _, vol := range req.Volumes { if vol.GetState() == ct.VolumeStatePending { log.Info("creating new volume", "host.id", req.Host.ID, "vol.id", vol.ID, "vol.path", vol.Path) if err := req.Host.client.CreateVolume("default", vol.Info()); err != nil { @@ -1642,6 +1647,14 @@ type PlacementRequest struct { Host *Host Err chan error + // Volumes is a snapshot of the job's assigned volumes taken when the + // placement request is handled. It is populated by the main scheduler + // loop and consumed by the StartJob goroutine which creates any + // pending volumes on the selected host. The snapshot avoids racing + // with handleActiveJob, which may replace job.Volumes in the main + // loop while the StartJob goroutine iterates it. + Volumes []*Volume + // ExcludeHosts is a set of host IDs that should be avoided when // placing the job, typically because a previous attempt to add the // job to those hosts failed. If all eligible hosts are excluded,