diff --git a/docs/runbook.md b/docs/runbook.md index fbc5381..998a88f 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -4,6 +4,96 @@ This is the canonical procedure for paid no-tools benchmark runs. Treat `results/local/` as scratch space; git history and release snapshots are the archive, not superseded local files. +## 0. Launch long runs under launchd + +A paid `policybench run` takes hours. Start it with `scripts/launch_run.sh`, +which installs a per-run launchd user agent, rather than with `nohup … &` from +a terminal or an agent session. The job then belongs to launchd, not to the +shell, terminal, or Claude Code session that started it: it keeps running when +that process group is torn down, it comes back after a reboot (`RunAtLoad`), +and if it is killed before finishing, launchd relaunches it and the supervisor +resumes from `run_state.json` and the per-scenario CSVs (`KeepAlive` on an +unfinished exit). A run that stops on purpose (budget stop, rounds exhausted, +every scenario complete) unloads itself. + +```bash +export OPENROUTER_API_KEY=... # provider credentials are copied into the job +scripts/launch_run.sh start --name glm53 --model glm-5.3 \ + --scenario-manifest paper/snapshot/20260501/us_scenarios.csv \ + --run-dir results/local/newmodels/glm53/run \ + --budget-usd 40 --max-workers 5 + +scripts/launch_run.sh status glm53 # launchd state, heartbeat summary, last log lines +scripts/launch_run.sh logs glm53 # supervisor.log and launchd.log from the run dir +scripts/launch_run.sh stop glm53 # SIGTERM, then SIGKILL, of the job's process group +scripts/launch_run.sh list +``` + +Run it from the checkout whose code you want to benchmark; that checkout +becomes `PYTHONPATH` and the supervisor executable defaults to its `.venv` +(or the main clone's `.venv` when run from a worktree). Add `--dry-run` to +print the plist instead of installing it. Variables named `*_API_KEY` or +`*_API_TOKEN`, or prefixed `ANTHROPIC_`, `OPENAI_`, `OPENROUTER_`, `GEMINI_`, +`GOOGLE_`, `XAI_`, `DEEPSEEK_`, `LITELLM_`, or `POLICYENGINE_`, are forwarded +into the job because launchd does not inherit a shell's environment; endpoint +overrides (`*_BASE_URL`, `*_API_BASE`) are not, since a Claude Code session +exports `ANTHROPIC_BASE_URL` for its own proxy. Pass anything else with +`--env NAME` or `--env NAME=VALUE`, or keep secrets in a mode-600 file and pass +`--env-file`. The plist itself is written mode 600 under +`~/Library/LaunchAgents/org.policyengine.policybench..plist`. + +The supervisor's stdout and stderr append to `/supervisor.log` +across relaunches; launchd's own output goes to `/launchd.log`. +`.launchd_restarts` counts consecutive unfinished exits (the wrapper gives up +after `--max-restarts`, default 5, and writes `.launchd_gave_up`); +`.launchd_done` marks a finished run. + +### What actually kills a run + +Investigated on 2026-09-04, when two board runs launched with +`nohup caffeinate -i policybench run … & disown` from a Claude Code session +died with `stopped_reason: null` and an empty log: + +- A **reboot** (`sysctl kern.boottime`; the Mac restarted at 10:14) killed the + GLM-5.3 run. `pmset -g log` showed no sleep, which is what made the deaths + look like session restarts. A launchd job with `RunAtLoad` resumes after a + reboot; a `nohup` job does not. +- A **broad pattern kill** from another session, + `for p in $(pgrep -f "codex-api|gpt-6-astra"); do kill -9 $p; done`, took + out the GPT-6 Astra supervisor and its workers three minutes after launch: + the model id is in every worker's command line and is shared with unrelated + tooling that runs the same model. Never `pkill -f` a model id. Match the + run directory instead (`pkill -f -- "--run-dir $RUN_DIR"`), or use + `scripts/launch_run.sh stop`. launchd relaunches a job killed this way. +- The run's own `pkill -f "policybench run --model glm-5.3"` during a + deliberate relaunch. + +What does **not** kill a `nohup … & disown` child: the Bash tool call +returning, the session process exiting, or the session process receiving +SIGTERM or SIGKILL. All three were tested against the Claude Code binary; the +harness only signals the process group of a command that is still running, +timed out, or was aborted, and background tasks (`run_in_background`) of an +exiting agent. Children of a finished foreground command are not tracked. +Note that `caffeinate -i cmd` execs `cmd` in the original pid and forks a +helper, so `pkill -f` on the command line matches the helper too. + +### Regression check + +`scripts/check_run_survival.sh` (macOS only, no model calls) launches +`/bin/sleep` through the launcher next to a plain `nohup` control from a +throwaway process group, kills that whole group with SIGTERM and SIGKILL, +checks that the control died and the launchd job survived, SIGKILLs the job +and checks that launchd relaunched it, then stops it through the launcher and +checks that nothing is left. Run it after touching the launcher or the +wrapper. `tests/test_launch_run.py` covers the plist rendering, credential +forwarding, and the wrapper's exit-code policy without launchd, so it runs in +CI. + +To confirm survival across a Claude Code session restart specifically, start +a run with the launcher from a session, restart or pause that session, and run +`scripts/launch_run.sh status ` from a new one: the launchd pid and the +`supervisor.log` mtime keep advancing. + ## 1. Pick a Run Directory Use a dated, descriptive run directory and keep US and UK artifacts under it. diff --git a/scripts/check_run_survival.sh b/scripts/check_run_survival.sh new file mode 100755 index 0000000..5b98a09 --- /dev/null +++ b/scripts/check_run_survival.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Live regression check (macOS only): a run started by scripts/launch_run.sh +# outlives the process group that launched it, is relaunched by launchd when +# killed before finishing, and stops cleanly. This is the reproducible stand-in +# for "restart the Claude Code session and see whether the supervisor is still +# there": a session teardown, a terminal closing, and a stray `kill -9` are all +# signals aimed at a process group or a pid, and that is exactly what this +# script sends. +# +# scripts/check_run_survival.sh # prints PASS or FAIL, exit 0/1 +# +# It launches `/bin/sleep 3600` through the launcher (no model calls, no spend), +# next to a plain `nohup sleep & disown` control launched from the same +# throwaway shell, then: +# 1. SIGTERMs and SIGKILLs that shell's whole process group. The control dies; +# the launchd job must survive (it lives in launchd's session, not ours). +# 2. SIGKILLs the job's sleep. launchd must relaunch it (KeepAlive on an +# unfinished exit; --throttle-seconds 5 keeps the wait short). +# 3. Stops the job through the launcher and checks that nothing is left. +set -u + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +LAUNCHER="$SCRIPT_DIR/launch_run.sh" +NAME="survival-check-$$" +LABEL="org.policyengine.policybench.$NAME" +DOMAIN="gui/$(id -u)" +WORK=$(mktemp -d "${TMPDIR:-/tmp}/pb-survival.XXXXXX") +RUN_DIR="$WORK/run" +fail=0 + +command -v launchctl >/dev/null 2>&1 || { echo "SKIP: launchctl not found (macOS only)"; exit 0; } + +say() { printf '%s\n' "$*"; } +check() { # $1 = description, $2 = 0/1 (1 = ok) + if [ "$2" -eq 1 ]; then say " ok $1"; else say " FAIL $1"; fail=1; fi +} +job_pid() { + launchctl print "$DOMAIN/$LABEL" 2>/dev/null | sed -n 's/^[[:space:]]*pid = \([0-9]*\).*/\1/p' | head -1 +} +sleep_pid_under() { # $1 = wrapper pid -> pid of the /bin/sleep it runs + pgrep -P "$1" -x sleep 2>/dev/null | head -1 +} +cleanup() { + "$LAUNCHER" stop "$NAME" >/dev/null 2>&1 + pkill -f "^pb-survival-control-$$" 2>/dev/null + rm -rf "$WORK" +} +trap cleanup EXIT + +say "1. launching a dummy run and a nohup control from a throwaway process group" +# A python-spawned bash in its own session stands in for the shell a Claude Code +# Bash tool call (or a terminal tab) would use. +launcher_shell_pgid=$(python3 - "$LAUNCHER" "$NAME" "$RUN_DIR" "$$" <<'PY' +import subprocess, sys +launcher, name, run_dir, tag = sys.argv[1:] +script = f""" +"{launcher}" start --name "{name}" --run-dir "{run_dir}" --throttle-seconds 5 --no-caffeinate -- /bin/sleep 3600 >"{run_dir}.start.log" 2>&1 +nohup bash -c 'exec -a pb-survival-control-{tag} /bin/sleep 3601' >/dev/null 2>&1 & disown +sleep 3 +""" +p = subprocess.Popen(["bash", "-c", script], start_new_session=True) +print(p.pid) # session leader: pid == pgid +p.wait() +PY +) +sleep 1 +control_pid=$(pgrep -f "^pb-survival-control-$$" | head -1) +wrapper_pid=$(job_pid) +check "launchd job $LABEL is running (pid ${wrapper_pid:-none})" "$([ -n "$wrapper_pid" ] && echo 1 || echo 0)" +check "nohup control sleep is running (pid ${control_pid:-none})" "$([ -n "$control_pid" ] && echo 1 || echo 0)" +if [ -n "$wrapper_pid" ]; then + wrapper_pgid=$(ps -o pgid= -p "$wrapper_pid" | tr -d ' ') + wrapper_ppid=$(ps -o ppid= -p "$wrapper_pid" | tr -d ' ') + check "job runs under launchd (ppid $wrapper_ppid) in its own process group ($wrapper_pgid, launcher shell was $launcher_shell_pgid)" \ + "$([ "$wrapper_ppid" = "1" ] && [ "$wrapper_pgid" != "$launcher_shell_pgid" ] && echo 1 || echo 0)" +fi + +say "2. killing the launcher shell's whole process group (SIGTERM, then SIGKILL)" +kill -TERM -- "-$launcher_shell_pgid" 2>/dev/null +sleep 1 +kill -KILL -- "-$launcher_shell_pgid" 2>/dev/null +sleep 2 +check "nohup control died with its process group" "$([ -n "$control_pid" ] && ! kill -0 "$control_pid" 2>/dev/null && echo 1 || echo 0)" +check "launchd job survived the group kill" "$([ -n "$wrapper_pid" ] && kill -0 "$wrapper_pid" 2>/dev/null && echo 1 || echo 0)" + +say "3. SIGKILLing the job's sleep; launchd should relaunch it within ~5s" +sleep_pid=$(sleep_pid_under "$wrapper_pid") +[ -n "$sleep_pid" ] && kill -KILL "$sleep_pid" +new_pid="" +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do + sleep 1 + new_pid=$(job_pid) + [ -n "$new_pid" ] && [ "$new_pid" != "$wrapper_pid" ] && break +done +check "launchd relaunched the job (new wrapper pid ${new_pid:-none}, old $wrapper_pid)" \ + "$([ -n "$new_pid" ] && [ "$new_pid" != "$wrapper_pid" ] && echo 1 || echo 0)" +check "wrapper recorded the unfinished exit in $RUN_DIR/.launchd_restarts" \ + "$([ "$(cat "$RUN_DIR/.launchd_restarts" 2>/dev/null)" = "1" ] && echo 1 || echo 0)" + +say "4. stopping through the launcher" +"$LAUNCHER" stop "$NAME" >/dev/null 2>&1 +sleep 2 +check "job unloaded" "$(launchctl print "$DOMAIN/$LABEL" >/dev/null 2>&1 && echo 0 || echo 1)" +check "no sleep left from the job" "$([ -z "$(pgrep -f "^/bin/sleep 3600$")" ] && echo 1 || echo 0)" +check "plist removed" "$([ ! -f "$HOME/Library/LaunchAgents/$LABEL.plist" ] && echo 1 || echo 0)" + +if [ "$fail" -eq 0 ]; then say "PASS: launchd-launched runs survive the launching process group and are relaunched when killed"; else say "FAIL: see the lines above; logs in $RUN_DIR (kept)"; trap - EXIT; "$LAUNCHER" stop "$NAME" >/dev/null 2>&1; fi +exit "$fail" diff --git a/scripts/launch_run.sh b/scripts/launch_run.sh new file mode 100755 index 0000000..2251b86 --- /dev/null +++ b/scripts/launch_run.sh @@ -0,0 +1,347 @@ +#!/bin/bash +# Launch a supervised PolicyBench run (`policybench run`) as a launchd user +# agent so it outlives the shell, terminal, or Claude Code session that started +# it, survives a reboot, and is relaunched if it is killed before finishing. +# +# scripts/launch_run.sh start --name NAME --model MODEL [options] +# scripts/launch_run.sh status NAME +# scripts/launch_run.sh logs NAME [LINES] +# scripts/launch_run.sh stop NAME +# scripts/launch_run.sh list +# +# `start` options (defaults in brackets): +# --scenario-manifest PATH [paper/snapshot/20260501/us_scenarios.csv] +# --run-dir DIR [results/local/launchd/NAME/run] +# --budget-usd N passed through; omit for no cap +# --max-workers N [5] +# --max-rounds N passed through when given +# --repo DIR checkout whose code runs (PYTHONPATH) [this one] +# --policybench PATH supervisor executable [REPO/.venv/bin/policybench, +# else the main clone's .venv, else `policybench`] +# --env NAME[=VALUE] extra environment for the job (repeatable) +# --env-file FILE sourced by the job at start (keeps secrets out of +# the plist); must not be group/world readable +# --max-restarts N relaunch cap after unfinished exits [5] +# --throttle-seconds N launchd delay before a relaunch [60] +# --no-caffeinate do not wrap the supervisor in `caffeinate -i` +# --dry-run print the plist and the commands; change nothing +# -- COMMAND [ARGS...] run this instead of `policybench run` (used by +# scripts/check_run_survival.sh) +# +# Environment variables named *_API_KEY, *_API_TOKEN, or starting with +# ANTHROPIC_, OPENAI_, OPENROUTER_, GEMINI_, GOOGLE_, XAI_, DEEPSEEK_, LITELLM_, +# or POLICYENGINE_ are copied into the job (launchd does not inherit a shell's +# environment). The plist is written mode 600 because of that. +# +# Portable to bash 3.2 (macOS). Only `start` without --dry-run, `status`, +# `logs`, `stop`, and `list` touch launchd. +set -u + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +DEFAULT_REPO=$(cd "$SCRIPT_DIR/.." && pwd) +LABEL_PREFIX="org.policyengine.policybench" +AGENTS_DIR="${POLICYBENCH_LAUNCH_AGENTS_DIR:-$HOME/Library/LaunchAgents}" +DOMAIN="gui/$(id -u)" + +usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-64}"; } +die() { echo "launch_run: $*" >&2; exit 1; } + +sanitize_name() { + printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '-' +} + +xml_escape() { + printf '%s' "$1" | sed -e 's/&/\&/g' -e 's//\>/g' +} + +resolve_policybench() { # $1 = repo + local repo=$1 common + if [ -x "$repo/.venv/bin/policybench" ]; then + printf '%s' "$repo/.venv/bin/policybench"; return + fi + common=$(git -C "$repo" rev-parse --git-common-dir 2>/dev/null) + if [ -n "$common" ]; then + common=$(cd "$repo" && cd "$common/.." 2>/dev/null && pwd) + if [ -n "$common" ] && [ -x "$common/.venv/bin/policybench" ]; then + printf '%s' "$common/.venv/bin/policybench"; return + fi + fi + command -v policybench 2>/dev/null || true +} + +env_is_forwarded() { # $1 = variable name + case "$1" in + # Endpoint overrides are never forwarded implicitly: a Claude Code session + # exports ANTHROPIC_BASE_URL for its own proxy, which must not redirect a + # benchmark's calls. Pass them with --env when a proxy is intended. + *_BASE_URL|*_API_BASE|*_ENDPOINT) return 1 ;; + *_API_KEY|*_API_TOKEN) return 0 ;; + ANTHROPIC_*|OPENAI_*|OPENROUTER_*|GEMINI_*|GOOGLE_*|XAI_*|DEEPSEEK_*|LITELLM_*|POLICYENGINE_*) return 0 ;; + esac + return 1 +} + +cmd_start() { + local name="" model="" manifest="" run_dir="" budget="" workers=5 rounds="" repo="$DEFAULT_REPO" + local pb="" env_file="" max_restarts=5 throttle=60 caffeinate=1 dry_run=0 + local extra_env="" # newline-separated NAME=VALUE + local exec_override=0 + while [ $# -gt 0 ]; do + case "$1" in + --name) name="$2"; shift 2 ;; + --model) model="$2"; shift 2 ;; + --scenario-manifest) manifest="$2"; shift 2 ;; + --run-dir) run_dir="$2"; shift 2 ;; + --budget-usd) budget="$2"; shift 2 ;; + --max-workers) workers="$2"; shift 2 ;; + --max-rounds) rounds="$2"; shift 2 ;; + --repo) repo=$(cd "$2" && pwd) || die "no such repo: $2"; shift 2 ;; + --policybench) pb="$2"; shift 2 ;; + --env) + if printf '%s' "$2" | grep -q '='; then + extra_env="$extra_env$2 +" + else + [ -n "${!2:-}" ] || die "--env $2: not set in the current environment" + extra_env="$extra_env$2=${!2} +" + fi + shift 2 ;; + --env-file) env_file="$2"; shift 2 ;; + --max-restarts) max_restarts="$2"; shift 2 ;; + --throttle-seconds) throttle="$2"; shift 2 ;; + --no-caffeinate) caffeinate=0; shift ;; + --dry-run) dry_run=1; shift ;; + --) shift; exec_override=1; break ;; + -h|--help) usage 0 ;; + *) die "unknown option $1 (see --help)" ;; + esac + done + [ -n "$name" ] || die "--name is required" + name=$(sanitize_name "$name") + local label="$LABEL_PREFIX.$name" + local plist="$AGENTS_DIR/$label.plist" + [ -n "$run_dir" ] || run_dir="$repo/results/local/launchd/$name/run" + case "$run_dir" in /*) ;; *) run_dir="$(pwd)/$run_dir" ;; esac + if [ -n "$env_file" ]; then + [ -r "$env_file" ] || die "--env-file $env_file is not readable" + case "$env_file" in /*) ;; *) env_file="$(pwd)/$env_file" ;; esac + if [ "$(stat -f '%Lp' "$env_file" 2>/dev/null || stat -c '%a' "$env_file")" != "600" ]; then + die "--env-file must be mode 600 (chmod 600 $env_file)" + fi + fi + + # The command launchd runs. + local args + if [ "$exec_override" -eq 1 ]; then + [ $# -gt 0 ] || die "nothing after --" + case "$1" in + */*) [ -x "$1" ] || die "command is not executable: $1" ;; + *) command -v "$1" >/dev/null 2>&1 || die "command not found on PATH: $1" ;; + esac + args=("$@") + else + [ -n "$model" ] || die "--model is required (or pass a command after --)" + [ -n "$pb" ] || pb=$(resolve_policybench "$repo") + [ -n "$pb" ] || die "cannot find a policybench executable; pass --policybench" + [ -n "$manifest" ] || manifest="$repo/paper/snapshot/20260501/us_scenarios.csv" + case "$manifest" in /*) ;; *) manifest="$(pwd)/$manifest" ;; esac + [ -f "$manifest" ] || die "scenario manifest not found: $manifest" + args=("$pb" run --model "$model" --scenario-manifest "$manifest" --run-dir "$run_dir" --max-workers "$workers") + [ -n "$budget" ] && args+=(--budget-usd "$budget") + [ -n "$rounds" ] && args+=(--max-rounds "$rounds") + fi + local wrapper="$SCRIPT_DIR/policybench_launchd_wrapper.sh" + local program=("$wrapper" --run-dir "$run_dir" --label "$label" --max-restarts "$max_restarts") + [ -n "$env_file" ] && program+=(--env-file "$env_file") + [ "$caffeinate" -eq 0 ] && program+=(--no-caffeinate) + program+=(-- "${args[@]}") + + # Environment forwarded into the job. + local env_lines="PATH=$PATH +HOME=$HOME +PYTHONPATH=$repo +PYTHONUNBUFFERED=1 +" + local var forwarded="" + for var in $(env | sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' | sort -u); do + case "$var" in PATH|HOME|PYTHONPATH|PYTHONUNBUFFERED) continue ;; esac + if env_is_forwarded "$var"; then + env_lines="$env_lines$var=${!var} +" + forwarded="$forwarded $var" + fi + done + env_lines="$env_lines$extra_env" + + # Render the plist. + local xml a line k v + xml=" + + + + Label$(xml_escape "$label") + ProgramArguments + +" + for a in "${program[@]}"; do + xml="$xml $(xml_escape "$a") +" + done + xml="$xml + WorkingDirectory$(xml_escape "$repo") + EnvironmentVariables + +" + while IFS= read -r line; do + [ -n "$line" ] || continue + k=${line%%=*}; v=${line#*=} + xml="$xml $(xml_escape "$k")$(xml_escape "$v") +" + done < + StandardErrorPath$(xml_escape "$run_dir/launchd.log") + + +" + + local how=" +Env: PATH HOME PYTHONPATH PYTHONUNBUFFERED${forwarded}$(printf '%s' "$extra_env" | sed -n 's/^\([^=]*\)=.*/ \1/p' | tr -d '\n') +Check: $0 status $name +Logs: $0 logs $name (supervisor.log and launchd.log live in $run_dir) +Stop: $0 stop $name +launchd: launchctl print $DOMAIN/$label | grep -E 'state|pid'" + + if [ "$dry_run" -eq 1 ]; then + printf '%s' "$xml" + echo "# dry run: would write $plist and run: launchctl bootstrap $DOMAIN $plist" >&2 + printf '%s\n' "$how" | sed 's/^/# /' >&2 + return 0 + fi + + command -v launchctl >/dev/null 2>&1 || die "launchctl not found; this launcher needs macOS launchd" + if launchctl print "$DOMAIN/$label" >/dev/null 2>&1; then + if launchctl print "$DOMAIN/$label" 2>/dev/null | grep -qE '^[[:space:]]*pid = [0-9]+'; then + die "$label is already running. Run '$0 status $name' or '$0 stop $name' first." + fi + # Loaded but idle: a finished job whose self-unload did not complete, or a + # job that gave up. Reap it so the new run can start. + launchctl bootout "$DOMAIN/$label" >/dev/null 2>&1 || true + sleep 1 + fi + mkdir -p "$AGENTS_DIR" "$run_dir" || die "cannot create $AGENTS_DIR or $run_dir" + umask 077 + printf '%s' "$xml" > "$plist" || die "cannot write $plist" + chmod 600 "$plist" + if command -v plutil >/dev/null 2>&1; then + plutil -lint -s "$plist" >/dev/null || die "generated plist does not lint: $plist" + fi + rm -f "$run_dir/.launchd_restarts" "$run_dir/.launchd_done" "$run_dir/.launchd_gave_up" + launchctl bootstrap "$DOMAIN" "$plist" || die "launchctl bootstrap failed (see launchctl print $DOMAIN/$label)" + sleep 1 + local pid + pid=$(launchctl print "$DOMAIN/$label" 2>/dev/null | sed -n 's/^[[:space:]]*pid = \([0-9]*\).*/\1/p' | head -1) + echo "Started $label (launchd pid ${pid:-pending}); the run is independent of this shell and session." + echo "$how" +} + +label_for() { printf '%s.%s' "$LABEL_PREFIX" "$(sanitize_name "$1")"; } + +run_dir_for() { # $1 = label; read the run dir back from the plist + local plist="$AGENTS_DIR/$1.plist" + [ -f "$plist" ] || return 1 + python3 - "$plist" <<'PY' +import plistlib, sys +args = plistlib.load(open(sys.argv[1], "rb"))["ProgramArguments"] +print(args[args.index("--run-dir") + 1]) +PY +} + +cmd_status() { + [ $# -ge 1 ] || die "usage: $0 status NAME" + local label run_dir + label=$(label_for "$1") + if launchctl print "$DOMAIN/$label" >/dev/null 2>&1; then + echo "$label: loaded" + launchctl print "$DOMAIN/$label" | grep -E '^[[:space:]]*(state|pid|last exit code|runs) ' | sed 's/^[[:space:]]*/ /' + else + echo "$label: not loaded (finished, stopped, or never started)" + fi + run_dir=$(run_dir_for "$label") || { echo " no plist at $AGENTS_DIR/$label.plist"; return 0; } + echo " run dir: $run_dir" + for marker in .launchd_done .launchd_gave_up .launchd_restarts; do + [ -f "$run_dir/$marker" ] && echo " $marker: $(cat "$run_dir/$marker")" + done + if [ -f "$run_dir/run_state.json" ]; then + python3 - "$run_dir/run_state.json" <<'PY' +import datetime as dt, json, sys +s = json.load(open(sys.argv[1])) +completed = s.get("completed") +completed = len(completed) if isinstance(completed, list) else completed +upd = dt.datetime.fromtimestamp(s.get("updated_at", 0)).strftime("%Y-%m-%d %H:%M:%S") +print(f" {s.get('model')}: {completed}/{s.get('total')} complete, ${s.get('spent_usd', 0):.2f} spent, " + f"workers={s.get('workers')}, stopped_reason={s.get('stopped_reason')!r}, heartbeat {upd}") +PY + fi + [ -f "$run_dir/supervisor.log" ] && { echo " last supervisor.log lines:"; tail -n 3 "$run_dir/supervisor.log" | sed 's/^/ /'; } +} + +cmd_logs() { + [ $# -ge 1 ] || die "usage: $0 logs NAME [LINES]" + local label run_dir n=${2:-40} + label=$(label_for "$1") + run_dir=$(run_dir_for "$label") || die "no plist for $label" + for f in supervisor.log launchd.log; do + [ -f "$run_dir/$f" ] || continue + echo "==> $run_dir/$f <==" + tail -n "$n" "$run_dir/$f" + done +} + +cmd_stop() { + [ $# -ge 1 ] || die "usage: $0 stop NAME" + local label + label=$(label_for "$1") + if launchctl print "$DOMAIN/$label" >/dev/null 2>&1; then + launchctl bootout "$DOMAIN/$label" && echo "stopped $label (launchd sent SIGTERM, then SIGKILL to the job's process group)" + else + echo "$label was not loaded" + fi + rm -f "$AGENTS_DIR/$label.plist" +} + +cmd_list() { + local p label + for p in "$AGENTS_DIR"/$LABEL_PREFIX.*.plist; do + [ -f "$p" ] || { echo "no PolicyBench launchd jobs in $AGENTS_DIR"; return 0; } + label=$(basename "$p" .plist) + if launchctl print "$DOMAIN/$label" >/dev/null 2>&1; then + echo "$label loaded pid=$(launchctl print "$DOMAIN/$label" | sed -n 's/^[[:space:]]*pid = \([0-9]*\).*/\1/p' | head -1)" + else + echo "$label not loaded" + fi + done +} + +[ $# -ge 1 ] || usage 64 +cmd=$1; shift +case "$cmd" in + start) cmd_start "$@" ;; + status) cmd_status "$@" ;; + logs) cmd_logs "$@" ;; + stop) cmd_stop "$@" ;; + list) cmd_list "$@" ;; + -h|--help|help) usage 0 ;; + *) die "unknown command $cmd (start|status|logs|stop|list)" ;; +esac diff --git a/scripts/policybench_launchd_wrapper.sh b/scripts/policybench_launchd_wrapper.sh new file mode 100755 index 0000000..be5b22d --- /dev/null +++ b/scripts/policybench_launchd_wrapper.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# launchd entry point for a supervised PolicyBench run (see scripts/launch_run.sh). +# +# Runs the supervisor in the foreground (under `caffeinate -i` when available), +# appends its output to the run directory, and translates the outcome into the +# exit code launchd's KeepAlive policy keys on: +# +# exit 0 the run is finished: every scenario complete, or the supervisor +# recorded a stopped_reason (budget stop, rounds exhausted). launchd +# leaves the job alone; the job unloads itself. +# exit 75 the supervisor died without finishing (SIGKILL, reboot, crash, +# a stray `pkill`). launchd relaunches after ThrottleInterval and the +# supervisor resumes from run_state.json and the scenario CSVs. +# +# After --max-restarts consecutive unfinished exits the wrapper gives up (exit 0 +# and a .launchd_gave_up marker) so a persistent crash cannot loop forever. +# +# Portable to bash 3.2 (macOS). Usage: +# policybench_launchd_wrapper.sh --run-dir DIR --label LABEL +# [--max-restarts N] [--env-file FILE] [--no-caffeinate] -- COMMAND [ARGS...] +set -u + +RUN_DIR="" +LABEL="" +MAX_RESTARTS=5 +ENV_FILE="" +USE_CAFFEINATE=1 +while [ $# -gt 0 ]; do + case "$1" in + --run-dir) RUN_DIR="$2"; shift 2 ;; + --label) LABEL="$2"; shift 2 ;; + --max-restarts) MAX_RESTARTS="$2"; shift 2 ;; + --env-file) ENV_FILE="$2"; shift 2 ;; + --no-caffeinate) USE_CAFFEINATE=0; shift ;; + --) shift; break ;; + *) echo "policybench_launchd_wrapper: unknown option $1" >&2; exit 64 ;; + esac +done +[ -n "$RUN_DIR" ] || { echo "policybench_launchd_wrapper: --run-dir is required" >&2; exit 64; } +[ $# -gt 0 ] || { echo "policybench_launchd_wrapper: no command after --" >&2; exit 64; } + +mkdir -p "$RUN_DIR" || exit 73 +LOG="$RUN_DIR/supervisor.log" +STATE="$RUN_DIR/run_state.json" +RESTARTS_FILE="$RUN_DIR/.launchd_restarts" +DONE_MARKER="$RUN_DIR/.launchd_done" +GAVE_UP_MARKER="$RUN_DIR/.launchd_gave_up" + +if [ -n "$ENV_FILE" ]; then + if [ -r "$ENV_FILE" ]; then + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a + else + echo "policybench_launchd_wrapper: cannot read --env-file $ENV_FILE" >&2 + exit 66 + fi +fi + +stamp() { date "+%Y-%m-%d %H:%M:%S"; } +attempt=$(( $(cat "$RESTARTS_FILE" 2>/dev/null || echo 0) + 1 )) +{ + echo "=== $(stamp) launchd attempt $attempt pid $$ label ${LABEL:-?}" + echo "=== command: $*" +} >> "$LOG" + +if [ "$USE_CAFFEINATE" -eq 1 ] && command -v caffeinate >/dev/null 2>&1; then + caffeinate -i "$@" >> "$LOG" 2>&1 +else + "$@" >> "$LOG" 2>&1 +fi +rc=$? +echo "=== $(stamp) exited rc=$rc" >> "$LOG" + +# Outcome from the supervisor's heartbeat file: "done" when every scenario is +# complete or a stopped_reason was recorded, "unfinished" otherwise, "nostate" +# when the file is missing or unreadable (a non-supervisor command, or a crash +# before the first heartbeat). +outcome=$(python3 - "$STATE" <<'PY' 2>/dev/null +import json, sys +try: + state = json.load(open(sys.argv[1])) +except Exception: + print("nostate"); sys.exit() +completed = state.get("completed") +total = state.get("total") +if isinstance(completed, list): + completed = len(completed) +if state.get("stopped_reason") or ( + isinstance(completed, int) and isinstance(total, int) and total > 0 and completed >= total +): + print("done") +else: + print("unfinished") +PY +) +[ -n "$outcome" ] || outcome=nostate + +finish() { # $1 = marker file, $2 = message + echo "=== $(stamp) $2" >> "$LOG" + rm -f "$RESTARTS_FILE" + date "+%Y-%m-%dT%H:%M:%S" > "$1" + if [ -n "$LABEL" ] && command -v launchctl >/dev/null 2>&1; then + # Unload the finished job so RunAtLoad does not replay it at the next login. + # Synchronous on purpose: launchd kills the job's process group as soon as + # this (the main) process exits, so a backgrounded bootout never runs. + rm -f "$HOME/Library/LaunchAgents/$LABEL.plist" + launchctl bootout "gui/$(id -u)/$LABEL" >/dev/null 2>&1 || true + fi + exit 0 +} + +case "$outcome" in + done) finish "$DONE_MARKER" "run finished (heartbeat says complete or stopped); unloading job" ;; + nostate) [ "$rc" -eq 0 ] && finish "$DONE_MARKER" "command exited 0 with no heartbeat file; unloading job" ;; +esac +if [ "$outcome" = "unfinished" ] && [ "$rc" -eq 0 ]; then + finish "$DONE_MARKER" "supervisor exited 0; unloading job" +fi + +echo "$attempt" > "$RESTARTS_FILE" +if [ "$rc" -eq 126 ] || [ "$rc" -eq 127 ]; then + # The command could not even start (not found / not executable); a relaunch + # cannot help and would only spin until the cap. + finish "$GAVE_UP_MARKER" "command could not be executed (rc=$rc); giving up. Fix the command, then relaunch." +fi +if [ "$attempt" -ge "$MAX_RESTARTS" ]; then + finish "$GAVE_UP_MARKER" "unfinished after $attempt attempts (rc=$rc); giving up. Inspect $LOG, then relaunch." +fi +echo "=== $(stamp) unfinished (rc=$rc); asking launchd to relaunch (attempt $attempt of $MAX_RESTARTS)" >> "$LOG" +exit 75 diff --git a/tests/test_launch_run.py b/tests/test_launch_run.py new file mode 100644 index 0000000..5ce8ab7 --- /dev/null +++ b/tests/test_launch_run.py @@ -0,0 +1,332 @@ +"""Tests for scripts/launch_run.sh and scripts/policybench_launchd_wrapper.sh. + +Nothing here touches launchd: the launcher is exercised with --dry-run (it +prints the plist it would install) and the wrapper with a stub supervisor +that writes run_state.json. See scripts/check_run_survival.sh for the live +macOS check that a launched job outlives the process group that started it. +""" + +from __future__ import annotations + +import json +import os +import plistlib +import shutil +import stat +import subprocess +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] +LAUNCHER = REPO / "scripts" / "launch_run.sh" +WRAPPER = REPO / "scripts" / "policybench_launchd_wrapper.sh" + +pytestmark = pytest.mark.skipif( + shutil.which("bash") is None, reason="launcher scripts need bash" +) + + +def _run(args: list[str], env: dict[str, str] | None = None, cwd: Path | None = None): + return subprocess.run( + ["bash", *args], + capture_output=True, + text=True, + env=env, + cwd=cwd, + check=False, + ) + + +def _clean_env(tmp_path: Path, **extra: str) -> dict[str, str]: + env = { + "PATH": os.environ["PATH"], + "HOME": str(tmp_path / "home"), + "POLICYBENCH_LAUNCH_AGENTS_DIR": str(tmp_path / "agents"), + } + env.update(extra) + return env + + +@pytest.fixture +def manifest(tmp_path: Path) -> Path: + path = tmp_path / "scenarios.csv" + path.write_text("scenario_id,scenario_json\nscenario_000,{}\n") + return path + + +def _dry_run(tmp_path: Path, manifest: Path, *extra: str, env: dict | None = None): + result = _run( + [ + str(LAUNCHER), + "start", + "--name", + "Unit Test/run", + "--model", + "glm-5.3", + "--scenario-manifest", + str(manifest), + "--run-dir", + str(tmp_path / "run"), + "--budget-usd", + "12.5", + "--max-workers", + "3", + "--policybench", + "/bin/echo", + "--dry-run", + *extra, + ], + env=env or _clean_env(tmp_path), + ) + assert result.returncode == 0, result.stderr + return plistlib.loads(result.stdout.encode()), result.stderr + + +def test_dry_run_renders_a_valid_launchd_plist(tmp_path: Path, manifest: Path): + plist, stderr = _dry_run(tmp_path, manifest) + assert plist["Label"] == "org.policyengine.policybench.Unit-Test-run" + args = plist["ProgramArguments"] + assert args[0] == str(WRAPPER) + assert args[args.index("--run-dir") + 1] == str(tmp_path / "run") + assert args[args.index("--label") + 1] == plist["Label"] + command = args[args.index("--") + 1 :] + assert command[:3] == ["/bin/echo", "run", "--model"] + assert command[3] == "glm-5.3" + assert command[command.index("--scenario-manifest") + 1] == str(manifest) + assert command[command.index("--budget-usd") + 1] == "12.5" + assert command[command.index("--max-workers") + 1] == "3" + assert plist["RunAtLoad"] is True + assert plist["KeepAlive"] == {"SuccessfulExit": False} + assert plist["ThrottleInterval"] == 60 + assert plist["WorkingDirectory"] == str(REPO) + assert plist["EnvironmentVariables"]["PYTHONPATH"] == str(REPO) + assert plist["EnvironmentVariables"]["PYTHONUNBUFFERED"] == "1" + assert plist["StandardOutPath"] == str(tmp_path / "run" / "launchd.log") + assert "launchctl bootstrap" in stderr + assert "status Unit-Test-run" in stderr + assert not (tmp_path / "agents").exists(), "dry run must not write a plist" + + +def test_dry_run_forwards_only_provider_credentials(tmp_path: Path, manifest: Path): + env = _clean_env( + tmp_path, + OPENAI_API_KEY="sk-test", + OPENROUTER_API_KEY="or-test", + LITELLM_LOG="DEBUG", + UNRELATED_SECRET="nope", + AWS_SECRET_ACCESS_KEY="also-nope", + ANTHROPIC_BASE_URL="http://127.0.0.1:1/session-proxy", + OPENAI_API_BASE="http://127.0.0.1:2", + ) + plist, _ = _dry_run( + tmp_path, manifest, "--env", "EXTRA_FLAG=1", "--env", "LITELLM_LOG", env=env + ) + forwarded = plist["EnvironmentVariables"] + assert forwarded["OPENAI_API_KEY"] == "sk-test" + assert forwarded["OPENROUTER_API_KEY"] == "or-test" + assert forwarded["LITELLM_LOG"] == "DEBUG" + assert forwarded["EXTRA_FLAG"] == "1" + assert "UNRELATED_SECRET" not in forwarded + assert "AWS_SECRET_ACCESS_KEY" not in forwarded + # Endpoint overrides ride along only when asked for with --env. + assert "ANTHROPIC_BASE_URL" not in forwarded + assert "OPENAI_API_BASE" not in forwarded + + +def test_dry_run_accepts_a_command_override(tmp_path: Path, manifest: Path): + plist, _ = _dry_run( + tmp_path, + manifest, + "--throttle-seconds", + "5", + "--no-caffeinate", + "--", + "/bin/sleep", + "30", + ) + args = plist["ProgramArguments"] + assert args[args.index("--") + 1 :] == ["/bin/sleep", "30"] + assert "--no-caffeinate" in args[: args.index("--")] + assert plist["ThrottleInterval"] == 5 + + +def test_start_requires_a_model_or_a_command(tmp_path: Path): + result = _run( + [str(LAUNCHER), "start", "--name", "x", "--dry-run"], env=_clean_env(tmp_path) + ) + assert result.returncode == 1 + assert "--model is required" in result.stderr + + +def test_env_file_must_be_private(tmp_path: Path, manifest: Path): + env_file = tmp_path / "secrets.env" + env_file.write_text("OPENAI_API_KEY=sk\n") + env_file.chmod(0o644) + result = _run( + [ + str(LAUNCHER), + "start", + "--name", + "x", + "--dry-run", + "--env-file", + str(env_file), + "--", + "/bin/true", + ], + env=_clean_env(tmp_path), + ) + assert result.returncode == 1 + assert "mode 600" in result.stderr + + +# -- wrapper ----------------------------------------------------------------- + + +def _stub_supervisor(tmp_path: Path) -> Path: + """A fake `policybench run` driven by environment variables. + + STUB_STATE: JSON written to /run_state.json ("" writes nothing). + STUB_RC: exit code. + """ + stub = tmp_path / "policybench" + stub.write_text( + "#!/bin/bash\n" + 'run_dir="$1"\n' + 'mkdir -p "$run_dir"\n' + 'echo "stub supervisor ran with: $*"\n' + '[ -n "${STUB_STATE:-}" ] && printf "%s" "$STUB_STATE" ' + '> "$run_dir/run_state.json"\n' + 'exit "${STUB_RC:-0}"\n' + ) + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + return stub + + +def _wrapper(tmp_path: Path, run_dir: Path, state: dict | None, rc: int, *extra: str): + stub = _stub_supervisor(tmp_path) + env = _clean_env(tmp_path, STUB_RC=str(rc)) + env["STUB_STATE"] = json.dumps(state) if state is not None else "" + return _run( + [ + str(WRAPPER), + "--run-dir", + str(run_dir), + "--no-caffeinate", + "--max-restarts", + "3", + *extra, + "--", + str(stub), + str(run_dir), + ], + env=env, + ) + + +def test_wrapper_exits_zero_when_every_scenario_is_complete(tmp_path: Path): + run_dir = tmp_path / "run" + result = _wrapper( + tmp_path, run_dir, {"completed": 100, "total": 100, "stopped_reason": None}, 0 + ) + assert result.returncode == 0, result.stderr + assert (run_dir / ".launchd_done").exists() + assert not (run_dir / ".launchd_restarts").exists() + log = (run_dir / "supervisor.log").read_text() + assert "stub supervisor ran with" in log + assert "run finished" in log + + +def test_wrapper_treats_a_recorded_stop_as_finished_despite_exit_1(tmp_path: Path): + run_dir = tmp_path / "run" + state = { + "completed": ["scenario_000"], + "total": 100, + "stopped_reason": "budget: spent $36.00 of $40.00 (stop at 90%)", + } + result = _wrapper(tmp_path, run_dir, state, 1) + assert result.returncode == 0, result.stderr + assert (run_dir / ".launchd_done").exists() + + +def test_wrapper_asks_for_a_relaunch_after_an_unfinished_death(tmp_path: Path): + run_dir = tmp_path / "run" + state = {"completed": ["scenario_000"], "total": 100, "stopped_reason": None} + result = _wrapper(tmp_path, run_dir, state, 137) # SIGKILL + assert result.returncode == 75, result.stderr + assert (run_dir / ".launchd_restarts").read_text().strip() == "1" + assert not (run_dir / ".launchd_done").exists() + assert ( + "asking launchd to relaunch (attempt 1 of 3)" + in (run_dir / "supervisor.log").read_text() + ) + + +def test_wrapper_gives_up_after_max_restarts(tmp_path: Path): + run_dir = tmp_path / "run" + state = {"completed": [], "total": 100, "stopped_reason": None} + codes = [_wrapper(tmp_path, run_dir, state, 1).returncode for _ in range(3)] + assert codes == [75, 75, 0] + assert (run_dir / ".launchd_gave_up").exists() + assert not (run_dir / ".launchd_restarts").exists() + assert "giving up" in (run_dir / "supervisor.log").read_text() + + +def test_wrapper_gives_up_at_once_when_the_command_cannot_start(tmp_path: Path): + run_dir = tmp_path / "run" + env = _clean_env(tmp_path) + result = _run( + [ + str(WRAPPER), + "--run-dir", + str(run_dir), + "--no-caffeinate", + "--", + str(tmp_path / "missing-supervisor"), + ], + env=env, + ) + assert result.returncode == 0 + assert (run_dir / ".launchd_gave_up").exists() + assert "could not be executed (rc=127)" in (run_dir / "supervisor.log").read_text() + + +def test_dry_run_rejects_a_missing_command(tmp_path: Path): + result = _run( + [str(LAUNCHER), "start", "--name", "x", "--dry-run", "--", "/bin/no-such-tool"], + env=_clean_env(tmp_path), + ) + assert result.returncode == 1 + assert "not executable" in result.stderr + + +def test_wrapper_without_a_heartbeat_follows_the_exit_code(tmp_path: Path): + run_dir = tmp_path / "run" + assert _wrapper(tmp_path, run_dir, None, 0).returncode == 0 + assert (run_dir / ".launchd_done").exists() + run_dir2 = tmp_path / "run2" + assert _wrapper(tmp_path, run_dir2, None, 9).returncode == 75 + + +def test_wrapper_sources_the_env_file(tmp_path: Path): + run_dir = tmp_path / "run" + env_file = tmp_path / "secrets.env" + env_file.write_text("STUB_RC=0\nSTUB_STATE=\n") + stub = _stub_supervisor(tmp_path) + env = _clean_env(tmp_path, STUB_RC="7") # the env file must override this + result = _run( + [ + str(WRAPPER), + "--run-dir", + str(run_dir), + "--no-caffeinate", + "--env-file", + str(env_file), + "--", + str(stub), + str(run_dir), + ], + env=env, + ) + assert result.returncode == 0, result.stderr