diff --git a/src/bpf/exectop.bpf.c b/src/bpf/exectop.bpf.c index 6601638..4ef805b 100644 --- a/src/bpf/exectop.bpf.c +++ b/src/bpf/exectop.bpf.c @@ -112,7 +112,16 @@ static __always_inline int in_scope(__u32 tgid, __u32 *depth_out) SEC("tracepoint/sched/sched_process_fork") int on_fork(struct trace_event_raw_sched_process_fork *ctx) { - __u32 parent = (__u32)ctx->parent_pid; + // The parent key is its TGID, not ctx->parent_pid. The tracepoint reports + // *tids*, but `traced` is keyed by tgid on the read path (in_scope uses + // pid_tgid >> 32), so looking the parent up by tid only matched when the + // forking thread happened to be created after we seeded — a thread that + // already existed at attach was never in the map, and every process it + // spawned, and that subtree's entire exec stream, was invisible. The fork + // tracepoint runs in the parent's context, so pid_tgid gives us the tgid. + // (Thread creations still insert a tid-keyed entry here. It is never read, + // since lookups are by tgid, and it is dropped at that thread's exit.) + __u32 parent = (__u32)(bpf_get_current_pid_tgid() >> 32); __u32 child = (__u32)ctx->child_pid; struct traced_key pk = { .tgid = parent }; diff --git a/src/lib/scope.js b/src/lib/scope.js index 9cfa33b..12ccf03 100644 --- a/src/lib/scope.js +++ b/src/lib/scope.js @@ -76,3 +76,56 @@ export async function listContainers() { return []; } } + +// ── seeding a scope that already exists ────────────────────────────────────── +// +// `traced` only ever spread FORWARD, through fork. That is correct for launch +// mode, where the target is parked before it execs and there is no history to +// miss — but for every other front door the interesting processes already +// exist. Attaching to a running app never saw the children it forked before we +// got there, and unscoped mode seeded pid 1 alone, so it caught only processes +// whose entire fork chain postdated attach: a shell started at login was +// invisible, and so was everything typed into it, forever. Duration had nothing +// to do with it; ancestry did. +// +// So: enumerate what is running and seed it, then let fork propagation carry it +// on from there. One graph query, bounded by the process count (a few hundred). + +// Every live process as {pid, ppid, comm}. Kernel threads are included — they +// never execve, so they cost a map entry and nothing else. +export async function procTable() { + const { data } = await yeet.graph.query(`{ procs { stat { pid ppid comm } } }`); + return (data?.procs ?? []) + .map((p) => p.stat) + .filter((s) => s && Number(s.pid)) + .map((s) => ({ pid: Number(s.pid), ppid: Number(s.ppid ?? 0), comm: s.comm ?? null })); +} + +// The already-running descendants of `root`, each with its generation distance +// from the root — the same depth the kernel would have assigned had we been +// attached when it forked, so a pre-existing subtree renders at the right +// indent instead of flattening onto the root. +export function descendantsOf(table, root) { + const kids = new Map(); + for (const s of table) { + const bucket = kids.get(s.ppid); + if (bucket) bucket.push(s); else kids.set(s.ppid, [s]); + } + const out = []; + const seen = new Set([root]); // guards against a ppid cycle after pid reuse + let frontier = [{ pid: root, depth: 0 }]; + while (frontier.length) { + const next = []; + for (const f of frontier) { + for (const c of kids.get(f.pid) ?? []) { + if (seen.has(c.pid)) continue; + seen.add(c.pid); + const row = { pid: c.pid, comm: c.comm, depth: f.depth + 1 }; + out.push(row); + next.push(row); + } + } + frontier = next; + } + return out; +} diff --git a/src/main.jsx b/src/main.jsx index ecb3e25..aa7d49d 100644 --- a/src/main.jsx +++ b/src/main.jsx @@ -19,9 +19,9 @@ */ import { Box, Text, computed, mount, signal } from "yeet:tui"; import { - buckets, flashes, folds, idle, outliers, paused, seedRoot, setCgroup, setPaused, stats, status, tick, + buckets, flashes, folds, idle, outliers, paused, seedExisting, seedRoot, setCgroup, setPaused, stats, status, tick, } from "@/probes/exec.js"; -import { commOf, containerRoot, listContainers } from "@/lib/scope.js"; +import { commOf, containerRoot, descendantsOf, listContainers, procTable } from "@/lib/scope.js"; import { C_FAINT, C_DIM } from "@/lib/format.js"; import TitleBar from "@/components/titlebar.jsx"; import Verdict from "@/components/verdict.jsx"; @@ -51,8 +51,11 @@ async function resolveScope() { ); } await seedRoot(c.pid, c.comm); + // The container is already running, so its tree already exists — seed it + // rather than waiting for each process to fork again. + const existing = await seedExisting(descendantsOf(await procTable(), c.pid)); scope.set(`container ${c.label}`); - status.set(c.cgroup ? "cgroup-scoped" : "pid-subtree"); + status.set(`pid-subtree — ${existing} existing process${existing === 1 ? "" : "es"} seeded`); return; } if (pid) { @@ -64,17 +67,24 @@ async function resolveScope() { status.set("complete tree — target was parked until the probe attached"); } else { scope.set(`pid ${pid}`); - // Anything this pid forked before we attached is invisible until it - // forks again — say so rather than implying the tree is complete. - status.set("pid-subtree (pre-existing children not tracked)"); + // Children that already existed are seeded from the process table, so + // the tree is complete as of attach — what we cannot see is only what + // exited before we got here. + const existing = await seedExisting(descendantsOf(await procTable(), pid)); + status.set(`pid-subtree — ${existing} existing child${existing === 1 ? "" : "ren"} seeded`); } return; } - // No scope given: seed pid 1 so fork propagation covers the host. Honest - // about being unscoped rather than pretending to be targeted. + // No scope given: seed EVERY live process. Seeding pid 1 alone was not + // "whole host" — membership only spreads forward through fork, so it caught + // just the processes whose whole chain postdated attach. A login shell + // predates that, which meant nothing typed into a terminal ever appeared. await seedRoot(1, "systemd"); + const n = await seedExisting( + (await procTable()).filter((s) => s.pid !== 1).map((s) => ({ ...s, depth: 0 })), + ); scope.set("whole host"); - status.set("unscoped — pass --container or --pid to narrow"); + status.set(`unscoped — ${n} live processes seeded; pass --container or --pid to narrow`); } await resolveScope(); diff --git a/src/probes/capture.js b/src/probes/capture.js index ef40df4..95cc246 100644 --- a/src/probes/capture.js +++ b/src/probes/capture.js @@ -13,6 +13,7 @@ import { HashMap, RingBuf } from "yeet:bpf"; import { control } from "./probe.js"; import { normalize } from "../lib/argv.js"; import { createModel } from "../lib/model.js"; +import { descendantsOf, procTable } from "../lib/scope.js"; const traced = new HashMap(control, "traced"); const events = new RingBuf(control, "events"); @@ -28,7 +29,12 @@ const model = createModel(); const raw = []; await traced.update({ tgid: root }, { depth: 0 }); -console.log(`[capture] root=${root} for ${secs}s`); +// Seed the descendants that already exist, exactly as main.jsx does. Without +// this the capture only sees processes forked after we attach, so a root whose +// children (or whose threads) predate us reports a fraction of its tree. +const existing = descendantsOf(await procTable(), root); +for (const r of existing) await traced.update({ tgid: r.pid }, { depth: r.depth }); +console.log(`[capture] root=${root} for ${secs}s (+${existing.length} existing)`); const sub = await events.subscribe((w) => { const e = normalize(w); diff --git a/src/probes/exec.js b/src/probes/exec.js index 7e0b17a..c020d69 100644 --- a/src/probes/exec.js +++ b/src/probes/exec.js @@ -38,6 +38,24 @@ export async function seedRoot(pid, comm) { model.seed(pid, comm ?? String(pid)); } +// Seed processes that ALREADY exist, so a scope isn't limited to what forks +// after we attach. Rows are {pid, comm, depth} from lib/scope.js. Membership +// spreads forward from each of them exactly as it does from the root. +export async function seedExisting(rows) { + let n = 0; + for (const r of rows) { + try { + await traced.update({ tgid: r.pid }, { depth: r.depth }); + if (r.comm) model.seed(r.pid, r.comm); + n++; + } catch { + // The map is capacity-bounded; a full map should cost us this one process + // rather than the whole seed, so keep going and report what landed. + } + } + return n; +} + // Narrow to a cgroup (0 = the pid subtree alone, no cgroup filter). export async function setCgroup(cgid) { await bss.patch({ target_cgid: cgid }); diff --git a/test/capture.sh b/test/capture.sh new file mode 100755 index 0000000..4f8eb05 --- /dev/null +++ b/test/capture.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# End-to-end capture test: does the probe see everything, and only the truth? +# +# The unit tests in test/heuristics.test.mjs cover lib/model.js against recorded +# fixtures — pure aggregation, no kernel. That leaves the seam where the two +# real bugs lived: kernel → traced set → ring buffer → normalized record. This +# test covers that seam by running a workload whose exec count is known exactly +# and asserting the capture reproduces it. +# +# test/capture.sh # run it +# test/capture.sh -v # also print the full capture report +# +# Needs: make (already run), sudo for the probe, python3, curl, base64. +# Everything stays in a scratch dir under /tmp; the curl fetch uses file://, +# so nothing leaves the machine. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$HERE" +VERBOSE=0 +[[ "${1:-}" == "-v" ]] && VERBOSE=1 + +W=$(mktemp -d /tmp/exectop-capture-test.XXXXXX) +SECS=14 +red() { printf '\033[31m%s\033[0m\n' "$*"; } +grn() { printf '\033[32m%s\033[0m\n' "$*"; } +dim() { printf '\033[2m%s\033[0m\n' "$*"; } + +cleanup() { kill "${ROOT:-0}" 2>/dev/null; rm -rf "$W"; } +trap cleanup EXIT + +for tool in python3 curl base64 chmod; do + command -v "$tool" >/dev/null || { red "missing prerequisite: $tool"; exit 2; } +done +[[ -f bin/probe.bpf.o ]] || { red "bin/probe.bpf.o missing — run: make"; exit 2; } + +# Distinctly-named copies of /bin/true, so each phase folds into a row we can +# assert on by name. A real binary per phase makes the expected count exact: +# `comm` is the fold identity, so there is no ambiguity about which exec came +# from where. +for m in mark-loop mark-child mark-thread; do + cp /bin/true "$W/$m" +done +mkdir -p "$W/home/.ssh" && echo "not-a-key" > "$W/home/.ssh/id_rsa" +echo "payload" > "$W/blob.txt" && base64 "$W/blob.txt" > "$W/blob.b64" + +# ── the workload ──────────────────────────────────────────────────────────── +# Phases, in order: +# setup a child process and a python thread, both created BEFORE the probe +# attaches — these are the two regressions. They idle on $W/go. +# ready touch $W/ready, then wait for $W/go +# loop 50 × mark-loop → folding, exact count +# findings four commands that should each be flagged with a reason +# pre the child and the thread each run 10 × their marker +cat > "$W/workload.sh" <<'EOF' +set -u +W="$1" + +# A child forked NOW, execing only after the probe is up. Before the scope +# fix this subtree was never seeded and contributed nothing. +bash -c ' + W="$1"; touch "$W/child-up" + while [[ ! -e "$W/go" ]]; do sleep 0.05; done + for i in $(seq 1 10); do "$W/mark-child"; done +' _ "$W" & + +# A thread created NOW, spawning only after the probe is up. Before the fork +# key fix, membership was looked up by tid, so a thread that predated the seed +# was invisible and so was everything it spawned. +python3 -c ' +import os, sys, time, threading, subprocess +W = sys.argv[1] +def worker(): + # Announce from INSIDE the thread: the test blocks on this file, so the OS + # thread provably exists before the probe attaches. Without it, thread + # creation raced capture startup — and on runs where the thread won that + # race (created after seeding) a tid-keyed parent lookup still worked, so + # the test passed against code that was broken. A flaky test that hides the + # bug it exists to catch is worse than no test. + open(W + "/thread-up", "w").write(str(threading.get_native_id())) + while not os.path.exists(W + "/go"): time.sleep(0.05) + for _ in range(10): subprocess.run([W + "/mark-thread"]) +t = threading.Thread(target=worker); t.start(); t.join() +' "$W" & + +touch "$W/ready" +while [[ ! -e "$W/go" ]]; do sleep 0.05; done + +# Folding: one row, ×50. +for i in $(seq 1 50); do "$W/mark-loop"; done + +# Findings: one exec each, every one a real program doing the real thing. +curl -s -o /dev/null "file://$W/blob.txt" # fetches from the network +base64 -d "$W/blob.b64" > /dev/null # evaluates constructed input +ls "$W/home/.ssh" > /dev/null # touches credential paths +chmod 777 "$W/blob.txt" # widens permissions + +wait +EOF + +# ── run it ────────────────────────────────────────────────────────────────── +bash "$W/workload.sh" "$W" >"$W/workload.log" 2>&1 & +ROOT=$! +# Wait for BOTH pre-existing subtrees to confirm they are up, not merely that +# the workload launched them — that is the precondition the whole test rests on. +for _ in $(seq 1 200); do + [[ -e "$W/ready" && -e "$W/child-up" && -e "$W/thread-up" ]] && break + sleep 0.05 +done +for f in ready child-up thread-up; do + [[ -e "$W/$f" ]] || { red "workload never signalled $f"; exit 2; } +done +dim "root pid $ROOT — child up, thread up (tid $(cat "$W/thread-up")), probe not yet attached" + +# Release the workload once the probe has attached and seeded. +( sleep 3; touch "$W/go" ) & + +dim "capturing for ${SECS}s …" +sudo yeet run src/probes/capture.js -- "$ROOT" "$SECS" >"$W/report.txt" 2>&1 +[[ $VERBOSE == 1 ]] && cat "$W/report.txt" + +# ── assertions ────────────────────────────────────────────────────────────── +fails=0 +report="$W/report.txt" + +# An exact folded count: `×50 ... mark-loop`. Asserting the COUNT, not just +# presence, is the point — a partially-seeded scope shows the row but undercounts. +want_fold() { # name count + if grep -qE "×$2 .*$1" "$report"; then + grn " ok $1 folded ×$2" + else + got=$(grep -oE "×[0-9]+ [^ ]* *[0-9.]+% *$1" "$report" | grep -oE '×[0-9]+' | head -1) + red " FAIL $1 expected ×$2, got ${got:-nothing}" + fails=$((fails + 1)) + fi +} +want_finding() { # reason-substring label + if grep -q "$1" "$report"; then + grn " ok flagged: $2" + else + red " FAIL not flagged: $2 ($1)" + fails=$((fails + 1)) + fi +} + +echo +echo "capture completeness (exact counts):" +want_fold mark-loop 50 +echo +echo "scope regressions — subtrees that existed before attach:" +want_fold mark-child 10 +want_fold mark-thread 10 +echo +echo "outlier tier — each of these should carry a reason:" +want_finding "fetches from the network" "curl" +want_finding "evaluates constructed input" "base64 -d" +want_finding "touches credential paths" "ls ~/.ssh" +want_finding "widens permissions" "chmod 777" + +total=$(grep -oE '^=== [0-9]+ execs' "$report" | grep -oE '[0-9]+' | head -1) +echo +if [[ -n "$total" && "$total" -ge 74 ]]; then + grn " ok $total execs captured (>= 74 known: 50 + 10 + 10 + 4)" +else + red " FAIL total execs = ${total:-none}, expected at least 74" + fails=$((fails + 1)) +fi + +echo +if [[ $fails == 0 ]]; then + grn "PASS — the probe sees the whole tree, folds it, and flags the four findings" +else + red "FAIL — $fails check(s) failed" + dim "full report: cat $report (or rerun with -v)" + # Keep the report for inspection on failure. + trap 'kill "${ROOT:-0}" 2>/dev/null' EXIT + echo "report kept at $report" >&2 +fi +exit $fails