diff --git a/cmd/gpu-cuda-profile/main.go b/cmd/gpu-cuda-profile/main.go index e7469e6..9cb12b3 100644 --- a/cmd/gpu-cuda-profile/main.go +++ b/cmd/gpu-cuda-profile/main.go @@ -1,25 +1,54 @@ -// Attaches to the NVIDIA CUPTI adapter, runs a real CUDA workload under it, -// and writes gpu-cuda.pb.gz through the existing pprof builder. +// Attaches to the NVIDIA CUPTI adapter and writes gpu-cuda.pb.gz through the +// existing pprof builder. This is cmd/gpu-stub-profile with the synthetic +// producer replaced by an actual GPU. // -// This is cmd/gpu-stub-profile with the synthetic producer replaced by an -// actual GPU. The adapter is not executed: it is a shared object the CUDA -// driver loads into the workload through CUDA_INJECTION64_PATH, so this -// command attaches its uprobes to the .so by path (system-wide, since the -// process it will be mapped into does not exist yet) and then starts the -// workload with that environment variable set. +// The adapter is never executed. It is a shared object the CUDA driver loads +// into a process through CUDA_INJECTION64_PATH, so this command attaches its +// uprobes to the .so by path and the probes arm themselves in whichever +// processes map it. +// +// Two shapes, and the second is the one real deployments need: +// +// - Launch (the default). This command starts the workload itself, with +// CUDA_INJECTION64_PATH and the sampling configuration in its +// environment, and attaches system-wide beforehand because the process +// it will be mapped into does not exist yet. It knows exactly how many +// sampled launches to expect, and it holds the workload's stdin open +// until it has them so the stacks can still be symbolized. +// +// - Attach (-pid). The process is already running and was started by +// somebody else -- a kubelet, a job runner, an operator -- with the +// injection variable already in its environment. Nothing here can +// configure it, so every flag that would have is refused rather than +// ignored; the run is bounded by -duration instead of by an expected +// count; and it ends the moment the target exits, because a stack +// outlives the /proc maps it is symbolized against by nothing at all. +// +// Attach mode is what a sidecar and a node collector both need (issue #124): +// neither can launch what it profiles. It is also strictly weaker today in +// one respect, reported at the end of every attached run -- the adapter +// captures module bytes only while a consumer is present, so modules loaded +// before the attach are missing and their PC samples carry no source line. package main import ( "context" + "errors" "flag" "fmt" "log" "os" "os/exec" + "os/signal" "path/filepath" + "regexp" + "sort" "strings" + "syscall" "time" + "golang.org/x/sys/unix" + "github.com/dpsoft/perf-agent/gpu" "github.com/dpsoft/perf-agent/gpuprobe" "github.com/dpsoft/perf-agent/internal/gpuabi" @@ -36,20 +65,65 @@ import ( "github.com/dpsoft/perf-agent/unwind/procmap" ) -func main() { - var ( - shim = flag.String("shim", "./shim/libperfagent-gpu-nvidia.so", "the CUPTI adapter .so carrying the perfagent USDT probes") - workload = flag.String("workload", "./shim/nvidia/testdata/cuda_workload", "CUDA program to run under the adapter") - iters = flag.Int("iters", 2000, "workload iterations; it launches two kernels per iteration") - sleepUs = flag.Int("sleep-us", 200, "workload sleep between iterations, in microseconds") - period = flag.Int("period", 8, "one-in-N launch sampling period (PERFAGENT_GPU_SAMPLE_PERIOD)") - linger = flag.Int("linger-ms", 30000, "how long the workload may wait to be released after it finishes") - out = flag.String("out", "gpu-cuda.pb.gz", "output pprof profile") - nvSymbols = flag.String("nvidia-symbols", "", +// options is every flag this command has, in one place, so that +// launchOnlyInAttachMode below can be checked for TOTALITY rather than +// trusted. A flag added here and not classified there would be silently +// ignored in attach mode, which is the exact failure the classification +// exists to prevent; a test enumerates the registered set and fails until +// every name is on one side or the other. +type options struct { + shim *string + workload *string + iters *int + sleepUs *int + period *int + linger *int + out *string + pid *int + duration *time.Duration + drainEvery *time.Duration + waitForShim *time.Duration + nvSymbols *string + pcSampling *string + pcAck *bool +} + +func defineFlags(fs *flag.FlagSet) *options { + return &options{ + shim: fs.String("shim", "./shim/libperfagent-gpu-nvidia.so", "the CUPTI adapter .so carrying the perfagent USDT probes"), + workload: fs.String("workload", "./shim/nvidia/testdata/cuda_workload", "CUDA program to run under the adapter"), + iters: fs.Int("iters", 2000, "workload iterations; it launches two kernels per iteration"), + sleepUs: fs.Int("sleep-us", 200, "workload sleep between iterations, in microseconds"), + period: fs.Int("period", 8, "one-in-N launch sampling period (PERFAGENT_GPU_SAMPLE_PERIOD)"), + linger: fs.Int("linger-ms", 30000, "how long the workload may wait to be released after it finishes"), + out: fs.String("out", "gpu-cuda.pb.gz", "output pprof profile"), + + // Attach mode. Non-zero switches this command from "launch a workload + // under the adapter" to "profile a process that is already running", + // which is the shape a sidecar or a node collector has to have: the + // kubelet started the workload, and nothing we do can have been in its + // environment. + pid: fs.Int("pid", 0, + "profile this already-running process instead of launching a workload. It must "+ + "have been started with CUDA_INJECTION64_PATH pointing at -shim; injection "+ + "happens during cuInit and cannot be added to a live process"), + duration: fs.Duration("duration", 30*time.Second, + "how long to profile in -pid mode. The run also ends early on SIGINT or when the "+ + "target exits"), + drainEvery: fs.Duration("drain-every", 2*time.Second, + "in -pid mode, how often to drain the timeline into the profile. The "+ + "timeline's rings hold ONE interval, not the whole run: a long attach to a "+ + "busy process overruns them and loses GPU time outright. Larger intervals "+ + "cost memory; smaller ones cost a little CPU"), + waitForShim: fs.Duration("wait-for-shim", 5*time.Second, + "in -pid mode, how long to wait for the shim to appear in the target's mappings "+ + "before giving up. Non-zero because a process attached to moments after it "+ + "started may not have reached cuInit yet"), + nvSymbols: fs.String("nvidia-symbols", "", "cache directory for NVIDIA's CUDA Toolkit Symbol Server; enables fetching "+ "symbols for libcuda/libcupti/libcuBLAS, which ship stripped. Off unless set: "+ "it reaches the network and discloses the build-ids of the CUDA libraries the "+ - "target has mapped.") + "target has mapped."), // One setting, three values, and no way to ask for two. The default // is the empty string rather than "off" so that an unspecified flag @@ -57,32 +131,151 @@ func main() { // contradicting it — an explicit --gpu-pc-sampling=off against an // exported "serialized" is a disagreement and is refused, but not // setting the flag at all is not. - pcSampling = flag.String("gpu-pc-sampling", "", + pcSampling: fs.String("gpu-pc-sampling", "", "GPU PC-sampling tier: "+strings.Join(gpu.PCSamplingTierNames, " | ")+ " (default off; also read from "+gpu.PCSamplingEnvVar+"). "+ "\"continuous\" does not serialize kernels; \"serialized\" does, and requires "+ - "-gpu-pc-sampling-acknowledge-perturbation") - pcAck = flag.Bool("gpu-pc-sampling-acknowledge-perturbation", false, + "-gpu-pc-sampling-acknowledge-perturbation"), + pcAck: fs.Bool("gpu-pc-sampling-acknowledge-perturbation", false, "acknowledge that the \"serialized\" tier perturbs the workload: it inflates GPU "+ "kernel durations inside a burst, it distorts any CPU and off-CPU profile taken "+ "alongside it with no marking in those profiles at all, and it is unavailable "+ - "where CUDA graphs are in use") - ) + "where CUDA graphs are in use"), + } +} + +// launchOnlyInAttachMode names every flag that configures the process this +// command STARTS, mapped to why it cannot mean anything when it does not +// start one. +// +// These are refused in attach mode rather than ignored, and the difference +// matters more than it looks. The whole of the producer's configuration -- +// its sampling period, its PC-sampling tier -- travels in the target's +// environment and is fixed at cuInit. A -period accepted here would change +// nothing about the target and would then be read off the command line +// afterwards as though it had: the profile would be interpreted at a +// sampling rate it was not taken at, with nothing anywhere saying otherwise. +// A refusal costs one restart; a silently-ignored flag costs a wrong +// conclusion. +// +// Totality is enforced by a test rather than by care: every flag defineFlags +// registers must appear here or in attachSafeFlags, so a flag added later +// cannot default into being silently ignored. +var launchOnlyInAttachMode = map[string]string{ + + "workload": "attach mode profiles a process that is already running", + "iters": "the target's iteration count is its own", + "sleep-us": "the target's pacing is its own", + "linger-ms": "nothing is being released; -duration bounds an attached run", + "period": "the launch sampling period is read by the ADAPTER from " + + "PERFAGENT_GPU_SAMPLE_PERIOD in the target's environment, which was fixed " + + "when the target started", + "gpu-pc-sampling": "the PC-sampling tier is the producer's, selected from " + + gpu.PCSamplingEnvVar + " in the target's environment at cuInit", + "gpu-pc-sampling-acknowledge-perturbation": "there is no tier for this run to " + + "acknowledge; see -gpu-pc-sampling", +} + +// attachSafeFlags are the flags that mean the same thing in both modes: they +// configure THIS process -- where the shim is, where the profile goes, how +// long to run, how symbols are resolved -- rather than the profiled one. +var attachSafeFlags = map[string]bool{ + "shim": true, + "out": true, + "nvidia-symbols": true, + "pid": true, + "duration": true, + "wait-for-shim": true, + "drain-every": true, +} + +// refusedLaunchFlags reports the launch-only flags the operator actually set, +// each with its reason. Only flags that were SET are refused: a default that +// happens to sit in the table is not a request. +func refusedLaunchFlags(fs *flag.FlagSet) []string { + var refused []string + fs.Visit(func(f *flag.Flag) { + if why, ok := launchOnlyInAttachMode[f.Name]; ok { + refused = append(refused, fmt.Sprintf("-%s: %s", f.Name, why)) + } + }) + sort.Strings(refused) + return refused +} + +// anomalyDigits collapses a health line to its KIND by removing the numbers +// in it, so "evicted 10198 launches" and "evicted 20824 launches" are +// recognised as one ongoing condition rather than two events. +var anomalyDigits = regexp.MustCompile(`[0-9]+`) + +func main() { + opt := defineFlags(flag.CommandLine) flag.Parse() + // Which of the two shapes this run has, and the refusal of every flag + // that belongs to the other one. + // + // Attach mode exists because the deployments that matter cannot launch + // what they profile. A sidecar profiles a container the kubelet started; + // a node collector profiles processes that were running before it was + // scheduled. Neither can put anything in the target's environment, which + // is where the whole of the producer's configuration lives. + // + // That is the reason the launch-only flags are REFUSED here rather than + // ignored. -period sets PERFAGENT_GPU_SAMPLE_PERIOD in the child this + // command starts; in attach mode there is no child, the target's period + // is whatever it was started with, and a -period that silently did + // nothing would be read off the command line afterwards as though it had + // applied. The profile would then be interpreted at a sampling rate it + // was not taken at. Every flag below has that shape: it configures a + // process this mode does not create. + attach := *opt.pid != 0 + if attach { + if refused := refusedLaunchFlags(flag.CommandLine); len(refused) > 0 { + log.Fatalf("-pid was given, so these flags cannot take effect and are refused "+ + "rather than ignored:\n %s", strings.Join(refused, "\n ")) + } + if *opt.duration <= 0 { + log.Fatalf("-duration must be positive in -pid mode, got %s", *opt.duration) + } + } + // Tier selection, and it happens BEFORE anything is attached or launched. // Every refusal here is a startup error: an unknown value, a value naming // two tiers, the flag and the environment naming two tiers, or Tier A // without its acknowledgement. None of them is resolved to a tier — a // profile produced under a tier nobody chose is worse than no profile, // because nothing in it says which one ran. - tier, err := gpu.PCSamplingRequest{ - Flag: *pcSampling, - Env: os.Getenv(gpu.PCSamplingEnvVar), - AcknowledgePerturbation: *pcAck, - }.Select() - if err != nil { - log.Fatalf("gpu pc sampling: %v", err) + // + // In attach mode there is no tier to select. The tier is the PRODUCER's: + // the adapter reads it out of the target's environment during cuInit and + // configures CUPTI accordingly, long before this command existed. Reading + // our own environment here would let a stale export in the operator's + // shell decide what this consumer believes the target is doing, and the + // tier gates the ANSWER — whether an execution may be claimed + // gpu_serialized="true" (gpu/timeline.go). A consumer that guessed the + // producer's tier from its own shell would be making that claim on + // nothing. + // + // So attach mode takes the zero value, which is the branch that claims + // nothing: PC samples still arrive and are still attributed through the + // module, and executions are marked "unknown" rather than "false". That + // is correct-but-weaker for a target running Tier A, which is the right + // direction to be wrong in. Recovering the producer's real tier means + // replaying gpu_config_v1 on attach, which is the same missing machinery + // that costs this mode its module bytes; see the report at the end of a + // run. + var tier gpu.PCSamplingTier + if !attach { + var terr error + tier, terr = gpu.PCSamplingRequest{ + Flag: *opt.pcSampling, + Env: os.Getenv(gpu.PCSamplingEnvVar), + AcknowledgePerturbation: *opt.pcAck, + }.Select() + if terr != nil { + log.Fatalf("gpu pc sampling: %v", terr) + } } // Printed at startup as well as standing in every JoinHealth render // below. The startup copy is for the operator who is watching the run @@ -92,7 +285,7 @@ func main() { log.Print(line) } - shimPath, err := filepath.Abs(*shim) + shimPath, err := filepath.Abs(*opt.shim) if err != nil { log.Fatalf("shim path: %v", err) } @@ -104,6 +297,38 @@ func main() { log.Fatalf("adapter %s: %v (build it with: make -C shim nvidia)", shimPath, err) } + // Attach mode's one hard precondition, and it is checked FIRST -- before + // the module store, the symbolizer, the BPF objects, or anything that + // needs a capability. + // + // A wrong -pid is the most likely thing an operator gets wrong, and it + // must not surface as a complaint about CAP_CHECKPOINT_RESTORE from + // setup that a correct -pid would have needed anyway. The cheapest and + // most decisive question goes first. + // + // The shim reaches a process exactly once, when the driver dlopens it + // during cuInit. There is no way to inject it into a process that is past + // that point: not with -pid, not with ptrace, not by any flag this + // command could grow. So a target that does not map the shim now will + // never map it, and every probe this run attaches would sit on a file + // that process does not have. + // + // Refused rather than warned about, because the alternative is a full + // -duration of waiting followed by an empty profile — which is precisely + // the fails-open-and-silent failure the injection check exists to end. + // The launch path can only diagnose this after the fact; here it is + // knowable up front, so it is an error at startup. + // + // The wait is for one case only: a process attached to in the seconds + // after it started, which has not reached cuInit yet. It is not a retry + // loop for a target that will never load the shim. + if attach { + if err := waitForShimIn(*opt.pid, shimPath, *opt.waitForShim); err != nil { + log.Fatalf("attach to pid %d: %v", *opt.pid, err) + } + log.Printf("pid %d maps %s; attaching", *opt.pid, shimPath) + } + // The module store, built HERE because it has three readers and no owner // among them: the cubin listener writes every arriving cubin into it // (gpuprobe.Config.Modules), the Timeline's join resolves a pending PC @@ -144,14 +369,14 @@ func main() { modules := procmap.NewResolver() defer modules.Close() symOpts := []symbolize.LocalOption{symbolize.WithModuleIndex(modules)} - if *nvSymbols != "" { + if *opt.nvSymbols != "" { // Last resort only, after the library's own file has failed. NVIDIA // exports 0.16%-3.2% of .text in these libraries, so almost every // address in them is unnameable locally; the server has a // symbols-only ELF per build-id that named 13 of 13 such frames in a // real capture. - symOpts = append(symOpts, symbolize.WithNVIDIASymbols(&nvsym.Store{Dir: *nvSymbols})) - log.Printf("nvidia symbols: enabled, cache %s", *nvSymbols) + symOpts = append(symOpts, symbolize.WithNVIDIASymbols(&nvsym.Store{Dir: *opt.nvSymbols})) + log.Printf("nvidia symbols: enabled, cache %s", *opt.nvSymbols) } sym, err := symbolize.NewLocalSymbolizer(symOpts...) if err != nil { @@ -161,10 +386,13 @@ func main() { c, err := gpuprobe.Attach(gpuprobe.Config{ ShimPath: shimPath, - // PID 0: the process that will map the adapter has not been started - // yet, so the attachment has to be system-wide. The semaphore the - // uprobe refcount maintains is what arms the probes in it once the - // driver maps the .so. + // Launch mode passes 0 and attach mode passes the target. + // + // Zero is system-wide, and it is what the launch path needs: the + // process that will map the adapter has not been started yet, so + // there is nothing to filter on. The semaphore the uprobe refcount + // maintains is what arms the probes in it once the driver maps the + // .so. // // Attach also binds the startup rendezvous before it creates the // uprobe link, which is what keeps the workload's first sampled @@ -174,7 +402,19 @@ func main() { // this consumer has installed them. See gpuprobe/enroll.go. Nothing // is needed here for that, and nothing here may set // PERFAGENT_GPU_ENROLL_TIMEOUT_MS to 0, which turns it off. - PID: 0, + // + // A non-zero PID does two things, and the second is the one that + // makes late attach viable at all. It narrows the uprobe_multi link + // to that process, and it takes the EAGER registration path in + // gpuprobe: Attach compiles that PID's CFI tables synchronously, + // before the link exists and therefore before any probe can fire. + // The rendezvous cannot help a process that is already running -- it + // passed cuInit before this command was started -- so without the + // eager path a late attach would walk its first stacks with no + // tables, which is exactly the ~38% loss issue #49 measured. Here + // the window is not merely narrowed but absent: the tables are in + // before the first probe exists. + PID: *opt.pid, Backend: gpu.BackendCUPTI, Sink: timeline, Symbolizer: sym, @@ -197,99 +437,187 @@ func main() { } }() - // The sampler jitters each gap around the period so it cannot lock phase - // against the workload's alternating axpy/scale pair (issue #50), but the - // schedule is still a deterministic chain from (seed, period): replaying - // it gives the EXACT sampled count, not an estimate. The workload - // launches exactly two kernels per iteration and the adapter samples on - // every launch, attached or not, so this number is what the consumer must - // see before the workload may be released. - if *iters <= 0 || *period <= 0 { - log.Fatalf("iters and period must both be positive, got iters=%d period=%d", *iters, *period) - } - launches := *iters * 2 - wantSampled := int(gpuabi.SampledCount(uint64(launches), uint32(*period), gpuabi.DefaultSampleSeed)) //nolint:gosec // both bounds-checked positive above - - cmd := exec.Command(*workload, - fmt.Sprint(*iters), fmt.Sprint(*sleepUs), fmt.Sprint(*linger)) - cmd.Env = append(os.Environ(), - "CUDA_INJECTION64_PATH="+shimPath, - fmt.Sprintf("PERFAGENT_GPU_SAMPLE_PERIOD=%d", *period), - "PERFAGENT_GPU_LOG=stderr", - // Set EXPLICITLY on every run including an off one, never left to be - // inherited. os.Environ() may already carry this variable from the - // operator's shell; appending the resolved value last is what keeps a - // stale export from turning a run this agent believes is off into a - // producer that serializes the workload's kernels. - gpu.PCSamplingEnvVar+"="+tier.EnvValue(), - ) - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - // Same release protocol as the stub: the workload's CPU stacks are - // symbolized against /proc//maps, which the kernel destroys the - // instant it exits, so we hold its stdin open until the consumer has - // counted what it needs. - release, err := cmd.StdinPipe() - if err != nil { - log.Fatalf("workload stdin: %v", err) - } - if err := cmd.Start(); err != nil { - log.Fatalf("workload: %v", err) - } - - // Did the injection actually happen? + // The profile accumulates ACROSS snapshots, which is what lets an + // attached run drain the timeline while it is still collecting. + // + // Timeline.Snapshot drains: it swaps in fresh rings and leaves anything + // it could not join eligible for a later call. It was built to be called + // repeatedly. Launch mode calls it once because it can -- it profiles a + // bounded workload and then stops -- and that simplification is exactly + // what breaks under a long attach. Measured on an RTX 3090: a 20 s attach + // to a workload issuing ~5,500 launches/s filled the 65,536-entry + // execution ring at about twelve seconds, and from there on every + // snapshot-less second evicted both executions and the launches waiting + // to join them -- 45,032 executions and 45,315 launches gone, GPU time + // missing from the profile entirely. The bound is not the problem; + // holding a whole run inside it is. Raising it only moves the cliff and + // makes the agent's memory grow with the length of the run, which for a + // collector is unbounded by construction. // - // CUDA_INJECTION64_PATH fails OPEN and SILENT: a driver that cannot load - // the library carries on as though the variable were unset, so a broken - // shim and a workload that launched no kernels produce identical output -- - // an empty profile and no error. The mapping is the one observable that - // separates them. Watched here, while the workload is alive, because - // /proc//maps is gone the moment it is not. - injected := waitForInjection(cmd.Process.Pid, shimPath, 10*time.Second) - - deadline := time.Now().Add(time.Duration(*linger) * time.Millisecond) - for c.Stats().SampledLaunches < uint64(wantSampled) { - if time.Now().After(deadline) { - log.Printf("WARNING: only %d/%d sampled launches observed before the workload was released; "+ - "stacks that arrive after it exits cannot be symbolized", - c.Stats().SampledLaunches, wantSampled) - break + // So the ring holds a drain interval rather than a run, and the profile + // -- not the timeline -- is what accumulates. + builders := pprof.NewProfileBuilders(pprof.BuildersOptions{SampleRate: 1}) + totalSamples := 0 + var lastSnap gpu.Snapshot + var lastProj gpu.ProjectionStats + snapshots := 0 + seenAnomaly := map[string]bool{} + collect := func() { + snap := timeline.Snapshot() + // ProjectExecutionsWith rather than ProjectExecutions so the + // projection's own losses reach the operator: gpu_pc labels dropped + // at the cardinality ceiling are invisible in the profile itself, and + // JoinHealthWith is the only place they are reported. + samples, projStats := gpu.ProjectExecutionsWith(snap, gpu.ProjectionConfig{Modules: store}) + for i := range samples { + builders.AddSample(&samples[i]) + } + totalSamples += len(samples) + snapshots++ + lastSnap, lastProj = snap, projStats + // Health is per-snapshot, and printing all of it would put one clean + // line per interval into a collector's log forever. JoinHealthWith + // returns exactly one summary line when there is nothing wrong, so + // anything past the first is a warning or an anomaly -- which is + // precisely what must not wait for the end of a long run to be seen. + // The final snapshot prints in full below either way. + // + // Reported ONCE PER KIND, because not every counter behind these + // lines is per-snapshot. Snapshot drains the rings but not the launch + // cache, so LaunchCacheStats is cumulative for the life of the run: + // its eviction total is re-reported, larger, in every subsequent + // snapshot. Printed naively that is one ongoing condition wearing the + // costume of a fresh anomaly every interval -- and an operator who + // sees the same alarm ten times learns to skip it, which is the exact + // harm the anomaly exists to prevent. The kind is the line with its + // numbers removed, so a condition that persists is announced when it + // starts and then stays quiet; the final health block below carries + // the totals. + if lines := gpu.JoinHealthWith(snap, projStats); len(lines) > 1 { + for _, line := range lines[1:] { + kind := anomalyDigits.ReplaceAllString(line, "#") + if seenAnomaly[kind] { + continue + } + seenAnomaly[kind] = true + log.Printf("snapshot %d: %s", snapshots, line) + } } - time.Sleep(5 * time.Millisecond) - } - reportInjection(injected, shimPath, cmd.Process.Pid, c.Stats().SampledLaunches) - if err := release.Close(); err != nil { - log.Fatalf("release workload: %v", err) } - if err := cmd.Wait(); err != nil { - log.Fatalf("workload: %v", err) + + // The two shapes diverge here and nowhere else. Everything above -- + // the store, the timeline, the symbolizer, the attach -- is identical, + // and everything below (snapshot, projection, profile, health) is too. + // What differs is only how the run is bounded: launch mode knows exactly + // how many sampled launches to expect and waits for them; attach mode + // cannot know, because it did not start the workload and has no idea what + // it is doing. + var launches, wantSampled int + if attach { + profileAttached(c, *opt.pid, *opt.duration, *opt.drainEvery, collect) + } else { + // The sampler jitters each gap around the period so it cannot lock phase + // against the workload's alternating axpy/scale pair (issue #50), but the + // schedule is still a deterministic chain from (seed, period): replaying + // it gives the EXACT sampled count, not an estimate. The workload + // launches exactly two kernels per iteration and the adapter samples on + // every launch, attached or not, so this number is what the consumer must + // see before the workload may be released. + if *opt.iters <= 0 || *opt.period <= 0 { + log.Fatalf("iters and period must both be positive, got iters=%d period=%d", *opt.iters, *opt.period) + } + launches = *opt.iters * 2 + wantSampled = int(gpuabi.SampledCount(uint64(launches), uint32(*opt.period), gpuabi.DefaultSampleSeed)) //nolint:gosec // both bounds-checked positive above + + cmd := exec.Command(*opt.workload, + fmt.Sprint(*opt.iters), fmt.Sprint(*opt.sleepUs), fmt.Sprint(*opt.linger)) + cmd.Env = append(os.Environ(), + "CUDA_INJECTION64_PATH="+shimPath, + fmt.Sprintf("PERFAGENT_GPU_SAMPLE_PERIOD=%d", *opt.period), + "PERFAGENT_GPU_LOG=stderr", + // Set EXPLICITLY on every run including an off one, never left to be + // inherited. os.Environ() may already carry this variable from the + // operator's shell; appending the resolved value last is what keeps a + // stale export from turning a run this agent believes is off into a + // producer that serializes the workload's kernels. + gpu.PCSamplingEnvVar+"="+tier.EnvValue(), + ) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + // Same release protocol as the stub: the workload's CPU stacks are + // symbolized against /proc//maps, which the kernel destroys the + // instant it exits, so we hold its stdin open until the consumer has + // counted what it needs. + release, err := cmd.StdinPipe() + if err != nil { + log.Fatalf("workload stdin: %v", err) + } + if err := cmd.Start(); err != nil { + log.Fatalf("workload: %v", err) + } + + // Did the injection actually happen? + // + // CUDA_INJECTION64_PATH fails OPEN and SILENT: a driver that cannot load + // the library carries on as though the variable were unset, so a broken + // shim and a workload that launched no kernels produce identical output -- + // an empty profile and no error. The mapping is the one observable that + // separates them. Watched here, while the workload is alive, because + // /proc//maps is gone the moment it is not. + injected := waitForInjection(cmd.Process.Pid, shimPath, 10*time.Second) + + deadline := time.Now().Add(time.Duration(*opt.linger) * time.Millisecond) + for c.Stats().SampledLaunches < uint64(wantSampled) { + if time.Now().After(deadline) { + log.Printf("WARNING: only %d/%d sampled launches observed before the workload was released; "+ + "stacks that arrive after it exits cannot be symbolized", + c.Stats().SampledLaunches, wantSampled) + break + } + time.Sleep(5 * time.Millisecond) + } + reportInjection(injected, shimPath, cmd.Process.Pid, c.Stats().SampledLaunches) + if err := release.Close(); err != nil { + log.Fatalf("release workload: %v", err) + } + if err := cmd.Wait(); err != nil { + log.Fatalf("workload: %v", err) + } + // The adapter's atexit handler runs cuptiActivityFlushAll and flushes + // both batches before the process leaves, so everything is in the ringbuf + // by now. This sleep is for the consumer goroutine to drain the tail of + // batched launches and executions — none of which carries a stack, so + // none of it needs the workload alive. } // The adapter's atexit handler runs cuptiActivityFlushAll and flushes // both batches before the process leaves, so everything is in the ringbuf - // by now. This sleep is for the consumer goroutine to drain the tail of - // batched launches and executions — none of which carries a stack, so - // none of it needs the workload alive. + // by now in launch mode. In attach mode the target is still running and + // its 100 ms drain timer is still firing, so this is the tail of the last + // tick rather than of an exit -- either way it is batched launches and + // executions, none of which carries a stack, so none of it needs a live + // process. time.Sleep(500 * time.Millisecond) cancel() <-done // Release any launch still held for a sampled twin before the snapshot. c.Flush() - snap := timeline.Snapshot() - // ProjectExecutionsWith rather than ProjectExecutions so the projection's - // own losses reach the operator: gpu_pc labels dropped at the cardinality - // ceiling are invisible in the profile itself, and JoinHealthWith below is - // the only place they are reported. - samples, projStats := gpu.ProjectExecutionsWith(snap, gpu.ProjectionConfig{Modules: store}) - if len(samples) == 0 { + // The last drain interval, and in launch mode the only one. + collect() + if totalSamples == 0 { + // In launch mode an empty pipeline is a bug: this command started a + // workload it knows launches kernels. In attach mode it is most + // often a fact about the target -- a process that was idle for the + // whole window launches nothing, and there is no defect to report. + if attach { + log.Fatalf("no samples projected: pid %d launched no kernels in %s. The shim is "+ + "mapped (checked at startup), so the probes were live; the target was "+ + "idle, or its CUDA work happens in a different process. stats=%+v", + *opt.pid, *opt.duration, c.Stats()) + } log.Fatal("no samples projected; the pipeline produced nothing") } - builders := pprof.NewProfileBuilders(pprof.BuildersOptions{SampleRate: 1}) - for i := range samples { - builders.AddSample(&samples[i]) - } - - f, err := os.Create(*out) + f, err := os.Create(*opt.out) if err != nil { log.Fatal(err) } @@ -300,17 +628,28 @@ func main() { break } if err := f.Close(); err != nil { - log.Fatalf("close %s: %v", *out, err) + log.Fatalf("close %s: %v", *opt.out, err) } st := c.Stats() - log.Printf("wrote %s: %d samples, launches=%d expected_sampled=%d stats=%+v", - *out, len(samples), launches, wantSampled, st) + // launches/expected_sampled are LAUNCH-MODE facts and are printed only + // there. They come from replaying the sampler's deterministic schedule + // over a workload this command started with a known iteration count, and + // none of those three things is true in attach mode: printing + // expected_sampled=0 beside a healthy attached run would read as a + // shortfall rather than as an inapplicable number. + if attach { + log.Printf("wrote %s: %d samples from pid %d over %d drain intervals, stats=%+v", + *opt.out, totalSamples, *opt.pid, snapshots, st) + } else { + log.Printf("wrote %s: %d samples, launches=%d expected_sampled=%d stats=%+v", + *opt.out, totalSamples, launches, wantSampled, st) + } // c.Stats() above is ingestion: what arrived off the ringbuf. This is // attribution: what the timeline could join it to, and what it evicted // trying. A run can be perfect on the first and quietly useless on the // second, so both are printed - one line when the join is clean, one // extra line per anomaly when it is not (see gpu.JoinHealthWith). - for _, line := range gpu.JoinHealthWith(snap, projStats) { + for _, line := range gpu.JoinHealthWith(lastSnap, lastProj) { log.Print(line) } // And the store's own account, which is neither of the above: what @@ -321,6 +660,35 @@ func main() { // (the CRCs the PC records join on are not the CRCs the cubins arrived // under, which is hardware assertion 13). log.Printf("module store: %+v", store.Stats()) + reportLateAttachModuleGap(attach, store.Stats().ModulesStored, st.CubinsReceived) +} + +// reportLateAttachModuleGap names the one way an attached profile is +// systematically weaker than a launched one, so it is not mistaken for a +// broken cubin transport. +// +// The adapter captures a module's bytes only while a consumer is already +// present -- capture_enabled() in shim/nvidia/cupti_adapter.cc is +// g_consumer_enrolled || gpu_module_load_v1_enabled(), and in a late attach +// neither holds at the moment the modules load. A CUDA process loads +// essentially all of its modules during startup, so attaching afterwards +// misses essentially all of them, and every PC sample then resolves +// gpu_src_status="no-module". +// +// This is a real gap and not a misconfiguration, which is exactly why it is +// worth a line: the operator's next move is to read the issue, not to go +// looking for a socket that is working perfectly. shim/core/drain.h already +// carries a complete ReplayLog for this transition; nothing calls it yet. +func reportLateAttachModuleGap(attach bool, modulesStored, cubinsReceived uint64) { + if !attach || modulesStored > 0 || cubinsReceived > 0 { + return + } + log.Printf("no modules were captured, which is expected for a late attach and is NOT a "+ + "transport failure: the adapter only captures a module's bytes while a consumer is "+ + "already attached, and this target loaded its modules before we arrived. Every PC "+ + "sample in %s therefore reads gpu_src_status=\"no-module\" and carries no source "+ + "line. Launch-mode runs do not have this gap. Tracking: issue #124.", + "this profile") } // waitForInjection watches for the shim appearing in the workload's mappings. @@ -365,3 +733,157 @@ func reportInjection(mapped bool, shimPath string, pid int, sampled uint64) { "script the CUDA process is a child, and this check does not see it.", shimPath, pid) } + +// waitForShimIn is attach mode's precondition, and it answers a question the +// launch path cannot ask: is this process one the probes can ever fire in? +// +// The launch path's waitForInjection above is a diagnostic — it watches a +// process it started, after the fact, and a miss there is ambiguous because +// the workload may simply not have reached cuInit. Here the answer is +// decisive in one direction: the driver dlopens the shim during cuInit and +// never again, so a process past that point either maps it or never will. +// There is no operation — no flag, no ptrace, no second attach — that adds +// the shim to a running process. +// +// The wait therefore covers exactly one legitimate case: a target attached to +// in the seconds after it started, whose cuInit has not happened yet. It is +// bounded and then it is an error, because the alternative is to attach to a +// process nothing can be observed in and say so only after a full -duration. +func waitForShimIn(pid int, shimPath string, within time.Duration) error { + deadline := time.Now().Add(within) + var last error + for { + mapped, err := gpuprobe.ShimIsMappedIn(pid, shimPath) + switch { + case err == nil && mapped: + return nil + case err != nil: + // Distinguish "cannot look" from "looked and it is not there". + // A pid that does not exist, or one this process may not read, + // is a different fix from a target that never loaded the shim, + // and reporting the second for the first sends the reader to + // the driver instead of to their own command line. + last = err + } + if !time.Now().Before(deadline) { + if last != nil { + return fmt.Errorf("could not read the target's mappings: %w", last) + } + return fmt.Errorf( + "%s is not mapped into pid %d after %s. The CUDA driver loads the shim "+ + "during cuInit, from CUDA_INJECTION64_PATH in the process's own "+ + "environment, and nothing can add it afterwards -- so this process "+ + "cannot be profiled by this shim. Check that the target was started "+ + "with CUDA_INJECTION64_PATH=%s (cat /proc/%d/environ | tr '\\0' '\\n' "+ + "| grep CUDA_INJECTION64_PATH), that the file is the same INODE the "+ + "target was given (a rebuilt or copied shim is a different file to a "+ + "uprobe), and that it loads in the target's environment "+ + "(`make -C shim nvidia-portable` builds one that does). If the target "+ + "is a wrapper script, the CUDA process is a child of it and has a "+ + "different pid", + shimPath, pid, within, shimPath, pid) + } + time.Sleep(50 * time.Millisecond) + } +} + +// profileAttached bounds a run this command did not start. +// +// Three things can end it, and the third is the one that matters for +// correctness rather than convenience. The duration is the operator's budget. +// SIGINT is the operator changing their mind, and it must produce the profile +// collected so far rather than discarding it -- a collector killed at the end +// of a scrape window that wrote nothing would be worse than useless. +// +// The target exiting ends the run IMMEDIATELY, and not as a courtesy: a +// sampled launch's stack is symbolized against /proc//maps, which the +// kernel destroys the instant the process leaves. Every second spent +// collecting after that point produces records whose stacks can no longer be +// resolved. Launch mode solves this by holding the workload's stdin open; +// attach mode has no such handle on a process it did not create, so the best +// it can do is notice and stop. +// +// A pidfd rather than polling /proc/: the pid is not ours, so it can be +// reaped and reused by an unrelated process while we watch. A pidfd is bound +// to the process, not to the number, and it becomes readable exactly once, +// when that process dies. +func profileAttached(c *gpuprobe.Consumer, pid int, d, drainEvery time.Duration, collect func()) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sig) + + gone := make(chan struct{}) + if fd, err := unix.PidfdOpen(pid, 0); err == nil { + go func() { + defer func() { _ = unix.Close(fd) }() + defer close(gone) + fds := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}} //nolint:gosec // a pidfd is well inside int32 + for { + n, err := unix.Poll(fds, -1) + // Poll is interruptible by any signal the runtime delivers, + // and the Go runtime delivers plenty. EINTR here means + // "nothing happened yet", not "the process is gone", and + // treating it as the latter would end every attached run at + // the first GC-related signal. + if errors.Is(err, unix.EINTR) { + continue + } + if n > 0 || err != nil { + return + } + } + }() + } else { + // No pidfd (pre-5.3, or the process left between the mapping check + // and here). The run still works; it just cannot shorten itself when + // the target exits, so stacks arriving after that point fail to + // symbolize and are counted as such. + log.Printf("cannot watch pid %d for exit (%v); the run will not end early if it exits, "+ + "and stacks captured after that point cannot be symbolized", pid, err) + } + + log.Printf("profiling pid %d for %s (ctrl-c to stop early)", pid, d) + // A progress line, because the alternative is an operator watching an + // idle terminal for the length of the budget with no way to tell a + // working attach from a dead one. Coarse on purpose: it is a sign of + // life, not a metric. + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + // The drain is what keeps the timeline's rings holding an interval rather + // than a run. It is deliberately separate from the progress ticker above: + // one is for the operator watching, the other is load-bearing. + drain := time.NewTicker(drainEvery) + defer drain.Stop() + deadline := time.After(d) + for { + select { + case <-deadline: + log.Printf("-duration elapsed") + return + case s := <-sig: + log.Printf("%s: stopping early and writing what was collected", s) + return + case <-gone: + st := c.Stats() + log.Printf("pid %d exited after %d sampled launches; stopping now, because its "+ + "/proc maps are gone and any stack arriving from here on cannot be "+ + "symbolized", pid, st.SampledLaunches) + return + case <-drain.C: + // Deliberately NOT c.Flush() here, though the end-of-run path + // does exactly that. Flush releases every launch being held for a + // sampled twin that has not arrived yet, and mid-run most of + // those twins are simply still in flight -- a batch or two away. + // Flushing on a timer would release them stackless, converting + // launches that were about to be attributed into + // [gpu:launch unsampled]. Held launches are not lost by being + // skipped: they reach the sink when their twin arrives and land + // in the NEXT snapshot. Only the last one, where nothing more is + // coming, has to force the issue. + collect() + case <-ticker.C: + st := c.Stats() + log.Printf("... sampled_launches=%d batches=%d", st.SampledLaunches, st.Batches) + } + } +} diff --git a/cmd/gpu-cuda-profile/main_test.go b/cmd/gpu-cuda-profile/main_test.go new file mode 100644 index 0000000..a367ef9 --- /dev/null +++ b/cmd/gpu-cuda-profile/main_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "flag" + "io" + "os" + "strings" + "testing" + "time" +) + +// The guard this whole classification exists for. +// +// Attach mode's contract is that a flag which cannot take effect is refused +// rather than ignored, and that contract is only as good as the coverage of +// the table behind it. A flag added to defineFlags and classified nowhere +// would fall through refusedLaunchFlags and be accepted in attach mode while +// doing nothing at all -- which is precisely the failure the refusal exists +// to prevent, reintroduced by omission rather than by decision. +// +// So the test is over the REGISTERED set, not over a list written here: it +// enumerates what defineFlags actually installed and fails until every name +// has been put on one side or the other. Adding a flag without thinking about +// attach mode is not possible; adding one and deciding it is attach-safe +// takes one line. +func TestEveryFlagIsClassifiedForAttachMode(t *testing.T) { + fs := flag.NewFlagSet("gpu-cuda-profile", flag.ContinueOnError) + defineFlags(fs) + + var unclassified, both []string + registered := map[string]bool{} + fs.VisitAll(func(f *flag.Flag) { + registered[f.Name] = true + _, launch := launchOnlyInAttachMode[f.Name] + safe := attachSafeFlags[f.Name] + switch { + case launch && safe: + both = append(both, f.Name) + case !launch && !safe: + unclassified = append(unclassified, f.Name) + } + }) + if len(unclassified) > 0 { + t.Errorf("these flags are registered but classified neither launch-only nor "+ + "attach-safe, so -pid would accept them and they would do nothing: %s", + strings.Join(unclassified, ", ")) + } + if len(both) > 0 { + t.Errorf("these flags are in both tables, which cannot be true of any flag: %s", + strings.Join(both, ", ")) + } + // The tables must not outlive the flags they describe either: a name left + // behind after a flag is removed makes the classification look more + // complete than it is. + for name := range launchOnlyInAttachMode { + if !registered[name] { + t.Errorf("launchOnlyInAttachMode names %q, which defineFlags does not register", name) + } + } + for name := range attachSafeFlags { + if !registered[name] { + t.Errorf("attachSafeFlags names %q, which defineFlags does not register", name) + } + } +} + +// A reason that does not say why is a refusal the operator cannot act on. +func TestEveryRefusalExplainsItself(t *testing.T) { + for name, why := range launchOnlyInAttachMode { + if len(why) < 20 { + t.Errorf("-%s is refused with %q, which does not tell the operator what to do "+ + "instead", name, why) + } + } +} + +func TestOnlyTheFlagsActuallySetAreRefused(t *testing.T) { + fs := flag.NewFlagSet("t", flag.ContinueOnError) + fs.SetOutput(io.Discard) + defineFlags(fs) + if err := fs.Parse([]string{"-pid", "1234", "-period", "4", "-out", "x.pb.gz"}); err != nil { + t.Fatal(err) + } + got := refusedLaunchFlags(fs) + if len(got) != 1 || !strings.HasPrefix(got[0], "-period:") { + t.Fatalf("want exactly -period refused, got %v", got) + } + // -iters and -linger-ms are launch-only and have non-zero DEFAULTS. If + // refusal keyed on the value rather than on the flag having been set, + // every attach run would be refused for flags nobody typed. + for _, r := range got { + if strings.HasPrefix(r, "-iters") || strings.HasPrefix(r, "-linger-ms") { + t.Fatalf("a launch-only flag was refused on its default value: %q", r) + } + } +} + +// waitForShimIn must distinguish "I looked and it is not there" from "I could +// not look", because they send the reader to completely different places: the +// first to how the target was started, the second to their own command line +// or to their capabilities. A single generic failure would send everyone to +// the driver. +func TestWaitForShimInSeparatesNotMappedFromCannotLook(t *testing.T) { + t.Run("a process that maps the file", func(t *testing.T) { + // Whatever libc this test binary is linked against is mapped into + // this very process by definition, so this is the positive case with + // no GPU, no shim and no capabilities involved. + libc := findMappedLibrary(t) + if err := waitForShimIn(os.Getpid(), libc, time.Second); err != nil { + t.Fatalf("self maps %s but waitForShimIn says: %v", libc, err) + } + }) + + t.Run("a process that does not map the file", func(t *testing.T) { + err := waitForShimIn(os.Getpid(), "/bin/true", 50*time.Millisecond) + if err == nil { + t.Fatal("want an error: this process does not map /bin/true") + } + // The message must name the mechanism, or the operator has no way to + // know that restarting the target is the fix and re-running this + // command is not. + if !strings.Contains(err.Error(), "cuInit") { + t.Errorf("the refusal does not explain that injection happens at cuInit "+ + "and cannot be added later: %v", err) + } + }) + + t.Run("a pid that cannot be read", func(t *testing.T) { + err := waitForShimIn(1<<21, "/bin/true", 50*time.Millisecond) + if err == nil { + t.Fatal("want an error for a pid that does not exist") + } + if !strings.Contains(err.Error(), "could not read") { + t.Errorf("a pid that cannot be inspected is reported as though it had been "+ + "inspected and found wanting: %v", err) + } + }) +} + +func findMappedLibrary(t *testing.T) string { + t.Helper() + maps, err := os.ReadFile("/proc/self/maps") + if err != nil { + t.Skipf("no /proc/self/maps: %v", err) + } + for _, line := range strings.Split(string(maps), "\n") { + i := strings.Index(line, " /") + if i < 0 { + continue + } + path := strings.TrimSpace(line[i+1:]) + if strings.Contains(path, ".so") && !strings.Contains(path, "(deleted)") { + return path + } + } + t.Skip("this binary maps no shared library to test against") + return "" +} + +// The launch cache's counters are cumulative for the life of the run -- +// Snapshot drains the timeline's rings but not the cache -- so an eviction +// total is re-reported, larger, in every subsequent snapshot. Collapsing a +// line to its kind is what stops one ongoing condition from being announced +// as a fresh anomaly on every drain interval. +func TestAnOngoingConditionIsOneAnomalyAndNotOnePerInterval(t *testing.T) { + kind := func(s string) string { return anomalyDigits.ReplaceAllString(s, "#") } + + // The exact lines a 20s attach produced on the 3090, four intervals apart. + a := "gpu join ANOMALY: launch cache evicted 10198 launches at capacity (65536 live) — too small for the launch rate" + b := "gpu join ANOMALY: launch cache evicted 45034 launches at capacity (65536 live) — too small for the launch rate" + if kind(a) != kind(b) { + t.Errorf("the same growing condition reads as two different anomalies:\n %q\n %q", + kind(a), kind(b)) + } + + // And it must not over-collapse: two genuinely different findings that + // happen to differ only in their numbers are still different findings, + // but two different SENTENCES must stay apart. + c := "gpu join ANOMALY: 353 of 65536 executions unmatched — GPU time arrived with no launch" + if kind(a) == kind(c) { + t.Errorf("two unrelated anomalies collapsed to one kind: %q", kind(a)) + } +} diff --git a/docs/gpu-injection.md b/docs/gpu-injection.md index 4c17b27..8473180 100644 --- a/docs/gpu-injection.md +++ b/docs/gpu-injection.md @@ -146,13 +146,57 @@ because the `perf_uprobe` PMU path requires `CAP_SYS_ADMIN` and that is what gets a per-pod agent rejected by admission policy. It costs a **Linux 6.6** floor. +## Profiling a process that is already running + +A sidecar or a node collector cannot launch what it profiles: the kubelet +started it. `-pid` attaches to a process that is already running. + +``` +gpu-cuda-profile -pid 12345 -duration 60s -out gpu.pb.gz +``` + +The target must have been started with `CUDA_INJECTION64_PATH` already in its +environment — the driver loads the shim during `cuInit` and never again, and +nothing (not `-pid`, not `ptrace`) can add it to a live process afterwards. +That is why the init container sets the variable on the *application* pod +rather than on the profiler. If the shim is not in the target's mappings the +command refuses at startup and says so, rather than collecting nothing for a +full `-duration`. + +Three things behave differently from a launched run, and all three are +consequences of not being the parent: + +- **Flags that configure the target are refused, not ignored.** `-period` and + `-gpu-pc-sampling` are read by the adapter out of the target's environment at + `cuInit`. Accepting them here would change nothing while making the profile + look as though it had been taken at a rate it was not. +- **The run ends when the target exits.** A sampled launch's stack is + symbolized against `/proc//maps`, which the kernel destroys the instant + the process leaves; anything collected past that point has stacks that can + never be resolved. +- **The timeline is drained on an interval** (`-drain-every`, default 2s) + rather than once at the end. Its rings hold one interval, not a whole run — + a 20 s attach to a process issuing ~5,500 launches/s overruns a run-length + ring and loses GPU time outright. + +Capabilities are the same set as a launched run; no `privileged`, no +`CAP_SYS_ADMIN`. A node collector additionally needs `hostPID: true` to see +the pids it is attaching to, which is a genuine privilege increase over the +sidecar and should be stated in a pod spec rather than absorbed quietly. + ## What this does not yet cover - **No published container image** for the init-container pattern. CI uploads the portable shim as a build artifact; there is no registry image to name in a pod spec yet. -- **The agent cannot attach to an already-running process** on the GPU path: - `cmd/gpu-cuda-profile` launches the workload itself. A Kubernetes sidecar has - to attach to a container the kubelet started, so the sidecar shape in - `examples/kubernetes/` cannot run end to end until that exists. Tracked in - issue #124. +- **Retention costs the target memory.** The shim captures every module's + bytes whether or not a consumer has attached, because the driver offers them + exactly once and there is no way to ask again — so a process that is never + profiled can hold up to 512 modules or 64 MiB (the consumer's own module + store bounds) on a profiler's behalf. Past those bounds captures are dropped + and counted; nothing grows without limit. `module_retained_unattached` in the + adapter's exit report is what it is currently holding. +- **The collector (DaemonSet) shape is not built yet** — `-pid` is its + prerequisite, not the whole of it. One profile per pod versus one profile + labelled by pod, attaching to pods that start later, and whether the shim is + one inode per node or one per pod are all open. Tracked in issue #124. diff --git a/shim/core/cubinqueue.cc b/shim/core/cubinqueue.cc index 120e0e6..f47e232 100644 --- a/shim/core/cubinqueue.cc +++ b/shim/core/cubinqueue.cc @@ -138,9 +138,11 @@ size_t CubinQueue::drain(CubinOfferFn offer, unsigned timeout_ms) { } else { // Broader than cubin.cc's cubins_send_failed(), on purpose: that // counter excludes "nobody was listening", because an unprofiled - // process must not accumulate failures. Here the queue only ever - // holds bytes when a consumer was believed present, so a - // no-listener result IS a module the consumer will not have. + // process must not accumulate failures. Here the caller only + // drains when it believes a consumer is present -- the adapter + // retains rather than drains while unattached, precisely so an + // offer is never made into the void -- so a no-listener result IS + // a module the consumer will not have. send_failed_.fetch_add(1, std::memory_order_relaxed); } } diff --git a/shim/core/cubinqueue.h b/shim/core/cubinqueue.h index 15de4c6..b013646 100644 --- a/shim/core/cubinqueue.h +++ b/shim/core/cubinqueue.h @@ -116,7 +116,22 @@ typedef CubinOfferResult (*CubinOfferFn)(const void *bytes, size_t len, uint64_t // Config.CubinMaxBytes default so a cubin this end copies is not one the // other end was always going to refuse. struct CubinQueueLimits { - size_t max_entries = 32; // queued offers + // Sized for RETENTION, not for a 100 ms hop. + // + // These bounds used to describe a queue that was drained every drain + // tick, so 32 entries was generous: nothing could accumulate. Since the + // adapter retains captures while no consumer is attached (a late attach + // would otherwise find every module already gone -- measured 100% loss), + // the queue holds a process's modules for as long as nobody has arrived, + // and 32 is not a process's worth of modules. + // + // 512 and 64 MiB are the CONSUMER's ModuleStore defaults, taken + // deliberately rather than invented: there is no value in retaining more + // modules than the store on the other end can hold, and no reason to + // retain fewer. max_queued_bytes is the real bound -- it is the memory an + // opted-in process can be made to hold -- and max_entries is the count + // guard beside it. + size_t max_entries = 512; // queued offers size_t max_queued_bytes = 64u * 1024 * 1024; // their total size_t max_cubin_bytes = 8u * 1024 * 1024; // one module, the memcpy bound size_t max_interned = 4096; // distinct CRCs remembered diff --git a/shim/core/cubinqueue_test.cc b/shim/core/cubinqueue_test.cc index 7637fce..d3d505a 100644 --- a/shim/core/cubinqueue_test.cc +++ b/shim/core/cubinqueue_test.cc @@ -397,6 +397,52 @@ void test_destruction_releases_undrained_copies() { printf(" destroying a non-empty queue releases its copies\n"); } +// Retention: a queue nobody drains must HOLD, and hold the bytes intact. +// +// This is the property the adapter's late-attach fix rests on. It retains +// captures while no consumer is attached and drains only once one arrives, +// because drain() pops and frees whatever it takes regardless of whether the +// offer landed -- so draining into a socket nobody is listening on is +// indistinguishable from deleting the modules, which is exactly how a late +// attach came to find zero of them. +// +// The failure this guards against is a plausible "optimisation": making the +// queue self-trim, or drop its oldest entry on a timer, on the reasoning that +// a queue is a transient thing. It is not one any more. +void test_an_undrained_queue_retains_everything_intact() { + reset_offers(); + reset_captured(); + CubinQueue q; + const char *payloads[] = {"module-alpha", "module-bravo", "module-charlie"}; + for (int i = 0; i < 3; i++) { + const CubinView v(payloads[i], strlen(payloads[i])); + assert(q.capture(v, fnv1a, nullptr, nullptr)); + } + // Many ticks' worth of doing nothing. Nothing may age out on its own: + // there is no deadline in this policy, only the bounds. + assert(q.depth() == 3); + assert(q.modules_captured() == 3); + assert(q.cubin_queue_full() == 0); + assert(g_offers.empty()); + + // And when a consumer finally arrives, every one of them is offered -- + // with its BYTES, not just its CRC. A retention scheme that kept the + // entries but lost the contents would satisfy every counter above and + // still deliver nothing a consumer could parse. + assert(q.drain(record_offer, 0) == 3); + assert(g_offers.size() == 3); + for (int i = 0; i < 3; i++) assert(g_offers[i].bytes == payloads[i]); + assert(q.depth() == 0); + printf(" an undrained queue retains every capture, bytes intact\n"); +} + +void test_the_retention_bounds_are_the_consumers_store_bounds() { + const CubinQueueLimits d; + assert(d.max_entries == 512); // gpu.ModuleStoreConfig.Capacity + assert(d.max_queued_bytes == 64u * 1024 * 1024); // gpu.ModuleStoreConfig.MaxBytes + printf(" the retention bounds are the consumer's module-store bounds\n"); +} + } // namespace int main() { @@ -413,6 +459,8 @@ int main() { test_a_slow_offer_does_not_block_a_capture(); test_a_healthy_run_reads_zero_on_every_drop_counter(); test_destruction_releases_undrained_copies(); + test_an_undrained_queue_retains_everything_intact(); + test_the_retention_bounds_are_the_consumers_store_bounds(); printf("cubinqueue_test: OK\n"); return 0; } diff --git a/shim/nvidia/cupti_adapter.cc b/shim/nvidia/cupti_adapter.cc index 2322c81..69a8fef 100644 --- a/shim/nvidia/cupti_adapter.cc +++ b/shim/nvidia/cupti_adapter.cc @@ -240,7 +240,7 @@ std::atomic g_launch_ordinal{0}; // The offer budget, read once at init so the drain thread does not re-parse // an environment variable every 100ms. Zero disables offers outright. unsigned g_cubin_timeout_ms = 0; -// True when the startup rendezvous confirmed a consumer. See capture_enabled. +// True when the startup rendezvous confirmed a consumer. See consumer_attached. bool g_consumer_enrolled = false; // Every discard has a counter. Nothing here is allowed to be silent (§6.1). @@ -262,13 +262,22 @@ std::atomic g_buffers{0}; // declined request is not documented, so whatever it does with the records // for that window, the refusal itself is on the record here. std::atomic g_buffer_alloc_failed{0}; -// A MODULE_LOADED callback we declined to copy because no consumer was -// believed present, and one whose descriptor carried no bytes at all. Both -// are modules that will read gpu_src_status "no-module" later, so both are -// counted here rather than being invisible. -std::atomic g_module_unattached{0}; +// A module copied while no consumer was attached -- retained against the +// queue's bounds until one arrives, not declined. It is a gauge of how much +// this process is holding on a profiler's behalf, and the number that says +// whether a late attach found anything to work with. +std::atomic g_module_retained{0}; +// A MODULE_LOADED callback whose descriptor carried no bytes at all. These +// WILL read gpu_src_status "no-module" later -- nothing can be captured from +// a descriptor with nothing in it -- so they are counted rather than being +// invisible. std::atomic g_module_no_bytes{0}; +// Declared here rather than with the rest of the process setup because +// on_cubin_captured below is what fills it: a module load that arrives before +// any consumer is logged for replay instead of being emitted. +perfagent::ReplayLog *g_replay = nullptr; + bool g_names_was_attached = false; // ------------------------------------------------------------ module path @@ -289,7 +298,7 @@ bool g_names_was_attached = false; // semaphore: the rendezvous CONNECT succeeded, which the consumer performs // before it creates the uprobe link, or the semaphore has since armed. In an // unprofiled process both read false and no module is ever copied. -bool capture_enabled() { +bool consumer_attached() { return g_consumer_enrolled || gpu_module_load_v1_enabled(); } @@ -324,14 +333,21 @@ uint64_t cupti_cubin_crc(const void *bytes, size_t len) { // (core/cubin.h); this record announces THAT a module loaded, with its CRC // and size. void on_cubin_captured(void *ctx, uint64_t crc, const void *bytes, size_t len) { - if (!gpu_module_load_v1_enabled()) return; gpu_module_load_v1 r{}; r.cubin_crc = crc; r.module_id = (uint64_t)(uintptr_t)ctx; r.size_bytes = (uint64_t)len; r.load_ns = mono_ns(); r.bytes_ptr = (uint64_t)(uintptr_t)bytes; - gpu_module_load_v1_emit(&r, 1, g_module_seq.fetch_add(1, std::memory_order_relaxed)); + // Live, or logged for replay -- never both, and that is what keeps a + // launched run from receiving every module twice. The log holds exactly + // the records nobody was there to hear, so the replay on the attach edge + // emits exactly the ones that were missed. + if (gpu_module_load_v1_enabled()) { + gpu_module_load_v1_emit(&r, 1, g_module_seq.fetch_add(1, std::memory_order_relaxed)); + return; + } + g_replay->record_module(r); } // CUPTI_CBID_RESOURCE_MODULE_LOADED. @@ -359,10 +375,28 @@ void on_module_loaded(const CUpti_ResourceData *rd) { // g_cubins is constructed before cuptiSubscribe, so a callback cannot // arrive ahead of it -- but a null here would be a segfault in somebody // else's process, which is not a way to find that out. - if (!capture_enabled() || !g_cubins) { - g_module_unattached.fetch_add(1, std::memory_order_relaxed); - return; - } + if (!g_cubins) return; + // Captured whether or not anyone is listening yet, which is the whole of + // the late-attach fix. + // + // This used to be gated on a consumer already being present, and the + // gate was measured: a consumer attaching to a running process found + // ZERO modules -- not most, all of them -- because a CUDA process loads + // essentially every module it will ever use during startup, and the bytes + // are only offered by the vendor once, inside this callback, with a + // buffer that is reused afterwards. There is no second chance at them and + // no API to re-enumerate what is loaded, so a module not copied here is + // gone for the life of the process, and every PC sample that lands in it + // reads gpu_src_status="no-module" forever after. + // + // The cost is bounded and it is the queue's bounds that bound it: 512 + // entries and 64 MiB, the consumer's own ModuleStore limits. Past them + // capture() drops the offer and counts it, so a process that is never + // profiled holds at most that and no more. The shim's PRESENCE is the + // operator's opt-in -- a process without CUDA_INJECTION64_PATH never maps + // it at all -- so the memory is only ever held by a process somebody + // asked to be able to profile. + if (!consumer_attached()) g_module_retained.fetch_add(1, std::memory_order_relaxed); const perfagent::CubinView view(m->pCubin, m->cubinSize); // moduleId travels as the context, not as a captured pointer: nothing // about this call may outlive the callback except the owned copy. @@ -580,7 +614,6 @@ std::vector g_pc_ctxs; // leaked with everything else; see be perfagent::PCDrainSchedule *g_pc_schedule = nullptr; perfagent::Batch *g_pcb = nullptr; -perfagent::ReplayLog *g_replay = nullptr; std::atomic g_pc_seq{0}; std::atomic g_stall_seq{0}; @@ -1923,7 +1956,7 @@ void report(const char *why) { "activity_kernels=%llu activity_other=%llu buffers=%llu buffer_alloc_failed=%llu " "exec_unattached=%llu exec_batch_dropped=%llu exec_no_clock=%llu " "exec_no_time=%llu cupti_dropped=%llu names=%zu " - "modules_captured=%llu module_reload_skipped=%llu module_unattached=%llu " + "modules_captured=%llu module_reload_skipped=%llu module_retained_unattached=%llu " "module_no_bytes=%llu cubin_too_large=%llu cubin_crc_failed=%llu " "cubin_alloc_failed=%llu cubin_queue_full=%llu cubin_queue_depth=%zu " "cubins_sent=%llu cubin_send_failed=%llu " @@ -1952,7 +1985,7 @@ void report(const char *why) { // this process could not explain. (unsigned long long)(g_cubins ? g_cubins->modules_captured() : 0), (unsigned long long)(g_cubins ? g_cubins->module_reload_skipped() : 0), - (unsigned long long)g_module_unattached.load(), + (unsigned long long)g_module_retained.load(), (unsigned long long)g_module_no_bytes.load(), (unsigned long long)(g_cubins ? g_cubins->cubin_too_large() : 0), (unsigned long long)(g_cubins ? g_cubins->cubin_crc_failed() : 0), @@ -2216,7 +2249,30 @@ void on_tick() { // after `if (!g_pc_enabled) return;` would silence the offer half of every // module capture in the DEFAULT configuration -- with modules_captured // still counting up and cubins_sent stuck at zero. - if (g_cubins) g_cubins->drain(perfagent::cubin_offer_to_consumer, g_cubin_timeout_ms); + // + // The replay runs BEFORE the drain, and the order is load-bearing twice + // over. gpu_module_load_v1 carries bytes_ptr into the adapter's own copy, + // and the drain frees that copy the moment it has offered it -- replaying + // afterwards would announce modules by a pointer that had just become + // dangling. And a consumer that received bytes for a module it had never + // been told about would have to hold them speculatively. + // + // It also sits here, above the Tier B gate, for the reason the drain + // does: module capture is not part of PC sampling. Left at the end of + // this function it would never run at all in the default configuration, + // because `if (!g_pc_enabled) return` is between the two. Stall maps and + // config replay on the same edge and are harmless to move -- their + // callbacks are only registered when PC sampling is on, so with it off + // this call is three no-ops and an edge flip. + g_replay->replay_if_newly_attached(consumer_attached()); + + // Drained only when somebody is there to receive it. While unattached the + // entries STAY QUEUED: drain() pops and frees whatever it takes, + // regardless of whether the offer landed, so draining into a socket + // nobody is listening on is indistinguishable from deleting the modules. + // That is precisely how a late attach came to find nothing. + if (g_cubins && consumer_attached()) + g_cubins->drain(perfagent::cubin_offer_to_consumer, g_cubin_timeout_ms); // Graph-launched executions: counted continuously, put on the wire as a // delta whenever it moves. Deliberately BEFORE the Tier B gate --- the @@ -2250,10 +2306,6 @@ void on_tick() { pc_drain_all(perfagent::PCDrainReason::kPeriodic); g_pcb->flush(); - // The stall map and the config record are one-shot and are queried at - // context creation, long before a consumer can attach. ReplayLog replays - // both on the unattached -> attached edge. - g_replay->replay_if_newly_attached(gpu_stall_reason_map_v1_enabled()); } // THE tick. One timer thread, and therefore one thread of ours that can ever @@ -2431,7 +2483,7 @@ extern "C" __attribute__((visibility("default"))) int InitializeInjection(void) snprintf(cubin_name, sizeof(cubin_name), ""); perfagent::EnrollResult enrolled = perfagent::enroll_with_consumer(perfagent::enroll_timeout_ms(2000)); - // The gate for cubin capture -- see capture_enabled(). A confirmed + // The gate for OFFERING captures -- see consumer_attached(). A confirmed // rendezvous is a positive statement that a consumer is attached, made // at a moment when the probe semaphore may still read zero. g_consumer_enrolled = (enrolled == perfagent::kEnrollConfirmed); @@ -2530,6 +2582,15 @@ extern "C" __attribute__((visibility("default"))) int InitializeInjection(void) gpu_config_v1_emit(&r, 1, g_config_seq.fetch_add(1, std::memory_order_relaxed)); }); } + // Registered UNCONDITIONALLY, unlike the two above. Those describe PC + // sampling and are meaningless with it off; module loads are not part of + // PC sampling and happen in every configuration, so putting this inside + // the PC block would leave the default build retaining every module and + // then never announcing one. + g_replay->on_replay_module([](const gpu_module_load_v1 &r) { + if (gpu_module_load_v1_enabled()) + gpu_module_load_v1_emit(&r, 1, g_module_seq.fetch_add(1, std::memory_order_relaxed)); + }); if (g_pc_tier_a) { perfagent::BurstConfig bc; bc.burst_ns = (uint64_t)env_uint("PERFAGENT_GPU_PC_BURST_MS", 50) * 1000000ull;