From 5fd5b4de668632240b144d5a331599aa7b322a3f Mon Sep 17 00:00:00 2001 From: bigboateng Date: Tue, 21 Jul 2026 06:56:27 +0100 Subject: [PATCH 1/2] fix(settle): restore the 'decide' subcommand (missing from #75 merge) decide.go (decision-only stdin/stdout mode) didn't land in the #75 squash, but the Harbor integration depends on it. Restores the file + wires the 'decide' case in main.go. Build/vet/test green. --- labs/18-settle/settle/cmd/settle/decide.go | 81 ++++++++++++++++++++++ labs/18-settle/settle/cmd/settle/main.go | 3 + 2 files changed, 84 insertions(+) create mode 100644 labs/18-settle/settle/cmd/settle/decide.go diff --git a/labs/18-settle/settle/cmd/settle/decide.go b/labs/18-settle/settle/cmd/settle/decide.go new file mode 100644 index 000000000..f1f5f45c6 --- /dev/null +++ b/labs/18-settle/settle/cmd/settle/decide.go @@ -0,0 +1,81 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/operatorstack/settle/detector" +) + +// decide is settle's decision-only mode: a host (e.g. the Harbor orchestrator) +// that runs commands itself calls this to get a verdict without settle touching +// the process. It reads one JSON request on stdin and writes one JSON response +// on stdout. Session history persists via SETTLE_SESSION, exactly as `exec`. +// +// observe: {"mode":"observe","command":"..."} +// -> {"outcome":"allow|deny","action":"...","class":"...","belief":0.0,"signals":[...],"message":"..."} +// record: {"mode":"record","command":"...","exit":1,"err":"...","state":""} +// -> {"ok":true} +// +// Fail open: any malformed input yields an allow (observe) or ok (record). settle +// is an optimizer — a broken decision channel must never block a healthy agent. + +type decideRequest struct { + Mode string `json:"mode"` + Command string `json:"command"` + Exit int `json:"exit"` + Err string `json:"err"` + State string `json:"state"` +} + +type decideResponse struct { + Outcome string `json:"outcome,omitempty"` + Action string `json:"action,omitempty"` + Class string `json:"class,omitempty"` + Belief float64 `json:"belief,omitempty"` + Signals []string `json:"signals,omitempty"` + Message string `json:"message,omitempty"` + OK bool `json:"ok,omitempty"` +} + +func runDecide(_ []string) int { + raw, _ := io.ReadAll(os.Stdin) + var req decideRequest + if err := json.Unmarshal(raw, &req); err != nil { + // Fail open. + writeJSON(decideResponse{Outcome: detector.OutcomeAllow, Action: string(detector.ActionAllow), OK: true}) + return 0 + } + + d, sess := loadDetector() + + switch req.Mode { + case "record": + d.Record(req.Command, req.Exit, req.Err, req.State) + saveDetector(d, sess) + writeJSON(decideResponse{OK: true}) + default: // "observe" (and anything else, defensively) + dec := d.Observe(req.Command) + writeJSON(decideResponse{ + Outcome: dec.Outcome, + Action: string(dec.Action), + Class: string(dec.Class), + Belief: dec.Belief, + Signals: dec.Signals, + Message: dec.Message, + }) + } + return 0 +} + +func writeJSON(v decideResponse) { + b, err := json.Marshal(v) + if err != nil { + fmt.Fprintln(os.Stdout, `{"outcome":"allow","ok":true}`) + return + } + os.Stdout.Write(b) + fmt.Fprintln(os.Stdout) +} diff --git a/labs/18-settle/settle/cmd/settle/main.go b/labs/18-settle/settle/cmd/settle/main.go index 6abe38b0d..084c49714 100644 --- a/labs/18-settle/settle/cmd/settle/main.go +++ b/labs/18-settle/settle/cmd/settle/main.go @@ -24,6 +24,8 @@ func main() { os.Exit(runRun(os.Args[2:])) case "replay": os.Exit(runReplay(os.Args[2:])) + case "decide": + os.Exit(runDecide(os.Args[2:])) case "doctor": os.Exit(runDoctor(os.Args[2:])) case "-h", "--help", "help": @@ -43,6 +45,7 @@ usage: settle exec -- [args...] supervise one command settle run -- [args...] wrap an agent executable settle replay -- ... offline: report what settle would do on a recorded run + settle decide decision-only: read one JSON request on stdin, write verdict on stdout settle doctor self-check env: From a756257f0e906992a2ef3ce03a9b8682720f90ba Mon Sep 17 00:00:00 2001 From: bigboateng Date: Tue, 21 Jul 2026 06:56:27 +0100 Subject: [PATCH 2/2] docs(settle): detection-quality report + honest claim-tier reframe Adds docs/05-detection-quality.md: fixture correctness (0 FP/FN), ~93% precision at the redirect threshold via the recovery curve, ~10% firing rate on real traces, deny rung dormant. Reframes Status/docs-04 from Being-evaluated to Evaluated: net-neutral on Qwen/terminal-bench (workload not loop-bound); causal uplift on loop-prone workloads (small models, online evals) = next experiment. --- labs/18-settle/README.md | 32 +++++-- .../docs/04-calibration-and-experiment.md | 14 +++- labs/18-settle/docs/05-detection-quality.md | 83 +++++++++++++++++++ 3 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 labs/18-settle/docs/05-detection-quality.md diff --git a/labs/18-settle/README.md b/labs/18-settle/README.md index 51e62eed4..d46f31cc4 100644 --- a/labs/18-settle/README.md +++ b/labs/18-settle/README.md @@ -24,11 +24,17 @@ A standalone, compiled Go binary. Isolated — no runtime dependency on Pitot or Harbor (it is designed to emit Pitot events later, but stands alone today). ``` -settle exec -- [args...] # supervise one command (the atomic unit) -settle run -- [args...] # wrap an agent; its shell routes through settle exec -settle doctor # self-check +settle exec -- [args...] # supervise one command (the atomic unit) +settle run -- [args...] # wrap an agent; its shell routes through settle exec +settle replay -- ... # offline: what settle would do on a recorded run +settle decide # decision-only: JSON request on stdin -> verdict on stdout +settle doctor # self-check ``` +`settle decide` is the host-integration surface: a runner that executes commands itself +(e.g. the Harbor orchestrator) calls it to gate at its own chokepoint — see +`labs/11-harbor-submit/SETTLE_INTEGRATION.md`. + The reliable, universal integration is to wire `settle exec` as the harness's command prefix (or PreToolUse hook). `settle run` is a zero-config convenience wrapper that installs a shell shim so an agent honouring `$SHELL` is supervised @@ -48,6 +54,10 @@ without any changes to its internals. - [`docs/04-calibration-and-experiment.md`](docs/04-calibration-and-experiment.md) — deriving the point-of-no-return from real traces; the pre-registered paired experiment; claim-tier discipline. +- [`docs/05-detection-quality.md`](docs/05-detection-quality.md) — offline + evidence: fixture correctness (0 FP/FN), ~93% precision at the redirect + threshold on real traces, ~10% firing rate, and why causal uplift is not yet + claimed. ## Module layout @@ -75,6 +85,16 @@ command-normalization logic is ported into a single Go source of truth ## Status -Detection firing correctly on the golden fixtures is **Observed**. Net effect on -task success is **Being-evaluated** until the pre-registered paired run -(docs/04). No invented numbers; no cross-model uplift claims. +- **Verified** — the classifier is correct on all labeled golden fixtures (0 false + positives / 0 false negatives); fail-open and nonblocking are structural. +- **Observed** — on real Qwen traces it fires on ~10% of trajectories at ~93% precision at + the redirect threshold; the deny rung is conservative and effectively dormant here + (see [`docs/05-detection-quality.md`](docs/05-detection-quality.md)). +- **Evaluated — net-neutral on Qwen/terminal-bench.** The measured effect on task success + is neutral: that workload's failures are step-budget/approach-bound, not loops (only ~13% + loop-shaped, 0% deep enough to deny — see `labs/11-harbor-submit/qwen-budget200` and + `EXPERIMENTS.md`). This is a result, not a shortcoming: settle targets non-progress loops, + which this workload rarely produces. +- **Not yet established** — *causal uplift* on a loop-prone population. The next experiment + is smaller/cheaper models (settle's original thesis) and online evals, where agents + actually thrash. No invented numbers; no cross-model uplift claims. diff --git a/labs/18-settle/docs/04-calibration-and-experiment.md b/labs/18-settle/docs/04-calibration-and-experiment.md index d73d1376a..0ef4ee1b0 100644 --- a/labs/18-settle/docs/04-calibration-and-experiment.md +++ b/labs/18-settle/docs/04-calibration-and-experiment.md @@ -55,10 +55,15 @@ actually is. **Claim tier: Observed** (calibration from Harbor traces). Not a universal constant; re-derive per corpus. -## 2. The pre-registered paired experiment (not yet run) +## 2. The paired experiment -Whether `settle` improves task outcomes is **Being-evaluated** — it is not -claimed by this lab. The experiment, registered here before running: +**Result on Qwen/terminal-bench: net-neutral** (see +`labs/11-harbor-submit/qwen-budget200` and `EXPERIMENTS.md`). That workload's failures are +step-budget/approach-bound — only ~13% of failures are loop-shaped and 0% reach the deny +threshold — so a loop-breaker has little to act on. Detection quality itself is solid +([05-detection-quality.md](05-detection-quality.md)); the *causal uplift* question just +needs a **loop-prone population** (smaller/cheaper models, online evals), which is the next +experiment. The pre-registered design, applied there: - **Design.** Paired, same model, same tasks and seeds: governed (`settle run`) vs. ungoverned. Same-model comparison only — no cross-model "reaches frontier" @@ -78,6 +83,7 @@ claimed by this lab. The experiment, registered here before running: - **Verified / Observed / Being-evaluated** tiers, always labeled. - Detection firing correctly on the golden fixtures = **Observed**. -- Net effect on task success = **Being-evaluated** until the paired run above. +- Net effect on task success = **Evaluated: net-neutral on Qwen/terminal-bench**; causal + uplift on loop-prone workloads remains **Being-evaluated** (next experiment). - No invented numbers. No "proven" for empirical results. No cross-model uplift claims. The one correct headline is in the [README](../README.md). diff --git a/labs/18-settle/docs/05-detection-quality.md b/labs/18-settle/docs/05-detection-quality.md new file mode 100644 index 000000000..d72add6d2 --- /dev/null +++ b/labs/18-settle/docs/05-detection-quality.md @@ -0,0 +1,83 @@ +# 05 — Detection quality (offline) + +What can be said about `settle`'s detector *without* a causal uplift experiment. +Three questions, kept separate because they need different evidence: + +- **Is the classifier correct on cases with known ground truth?** → golden fixtures. +- **When it fires on real traces, is it usually right?** → recovery-probability curve. +- **How often does it fire at all?** → firing rate on the real corpus. + +The honest frame from [01-theory.md](01-theory.md) holds throughout: the true "stuck" +state is *unobservable*, so every number here is a proxy, and precision is bounded, not +proven. + +## 1. Correctness on labeled ground truth (golden fixtures) + +The fixtures in `detector/testdata/` are the labeled set — each has a designed-in correct +answer. On all of them the classifier is exactly right (`go test ./detector/...`, green): + +| fixture | ground truth | settle | ✓ | +|---|---|---|---| +| always-fail | stuck | deny | ✓ | +| syntax-mutating (cmd changes, effect same) | stuck | deny | ✓ | +| oscillation A/B | stuck | deny | ✓ | +| genuine-progress | not stuck | allow throughout | ✓ | +| near-miss (2 fails then resolves) | not stuck | no deny | ✓ | +| hidden-progress (error repeats, state changes) | not stuck | no deny | ✓ | + +**0 false positives, 0 false negatives** on the labeled set. The `hidden-progress` case is +the load-bearing one: an error-signature-only detector would false-positive; keying on the +joint `(error, state)` avoids it. Canonicalization fidelity is separately asserted +(`signature_test.go`): volatile-token variants collapse to one signature; genuinely +different errors do not. + +Caveat: fixtures validate the *logic*, not the real-world base rate. For that, §2–3. + +## 2. Empirical precision when it fires (recovery-probability curve) + +`settle` acts at *dwell d* = d consecutive identical `(error, state)` observations. A +firing is "right" if continuing would **not** have recovered on its own. So + +> precision at threshold d ≈ 1 − P(recovery | d) + +using the recovery curve mined from 1,890 Harbor trajectories (`calibration.json`): + +| threshold (dwell d) | rung | P(recovery \| d) | ≈ precision | support | +|---|---|---|---|---| +| d = 2 | redirect | 0.068 | **~93%** | 44 | +| d = 3 | redirect | 0.167 | ~83% | 12 | +| d ≥ 5 | **deny** | under-observed | — | 2 | + +At its lightest fire threshold (redirect, dwell 2) settle is right ~93% of the time on real +traces. The **deny** threshold (dwell ≥ 5) is essentially never reached in this corpus +(support 2), so its precision isn't estimable here — deny is conservative by construction, +not validated. + +Caveat: the curve conditions on *consecutive same-normalized-command* runs. Agents usually +mutate the command, so dwell rarely grows — which is exactly why deep loops (and denies) +are rare (§3). + +## 3. Firing rate on the real corpus + +Offline `settle replay` over real Qwen trajectories: + +| corpus | trajectories | intervened (redirect) | denied | +|---|---|---|---| +| 277 (mixed qwen-* runs) | 277 | ~29 (~10%) | **0** | +| qwen-budget200 (deep, 200-step) | 40 | ~4 (~10%) | **0** | + +settle nudges ~10% of trajectories, always at the gentle rung; the deny rung never fires +because 5-deep identical-error loops don't occur in these runs. + +## 4. Conclusion (claim tiers) + +- **Verified:** the classifier is correct on all labeled fixtures (0 FP / 0 FN); fail-open + and nonblocking are structural (`ladder.go`, `exec/state.go`). +- **Observed:** on real traces it fires on ~10% of trajectories at ~93% precision (redirect + threshold); the deny rung is conservative and effectively dormant on this workload. +- **Not established (needs the right population):** *causal* effect on task success. On + Qwen/terminal-bench it is **net-neutral** — the target failure mode (deep loops) is rare; + failures are budget/approach-bound (see `labs/11-harbor-submit/qwen-budget200`). Proving + uplift requires a loop-prone population: **smaller/cheaper models** (settle's original + thesis — the loop-awareness cheap models lack) and **online evals**. That is the next + experiment, not a claim made here.