diff --git a/go.work b/go.work index a4368f6c2..250d3d10a 100644 --- a/go.work +++ b/go.work @@ -8,4 +8,5 @@ go 1.26 use ( ./labs/12-product-engineering-loop/product-engineering-loop ./labs/15-pitot/pitot + ./labs/18-settle/settle ) diff --git a/labs/18-settle/README.md b/labs/18-settle/README.md new file mode 100644 index 000000000..51e62eed4 --- /dev/null +++ b/labs/18-settle/README.md @@ -0,0 +1,80 @@ +# Lab 18 — `settle` + +> A zero-inference, partial-observation supervisory controller that wraps any +> agent's command execution, detects non-progress with cheap deterministic +> sensors, and intervenes with confidence-proportional, nonblocking force. Net +> effect on task success is measured per model — not claimed. + +`settle` watches the stream of `(command → outcome → workspace-state)` an agent +produces and, when its sensors agree the loop has stopped making *observable* +progress, injects a control signal whose force scales with its confidence. It +runs **no model inference**: all detection is hashing, counting, diffing, and +regex on the CPU. + +It is the same Ramadge–Wonham supervisory-control theory the repo-root +[`supervisory-control.md`](../../supervisory-control.md) states for Boatstack — +but operating in the regime Boatstack avoids by construction, where the +specification is **unobservable**. That single fact explains everything that +makes `settle` different: why it has an irreducible false-positive rate, why it +uses minimum force, and why it fails *open* where Boatstack fails closed. + +## Surface + +A standalone, compiled Go binary. Isolated — no runtime dependency on Pitot or +Harbor (it is designed to emit Pitot events later, but stands alone today). + +``` +settle exec -- [args...] # supervise one command (the atomic unit) +settle run -- [args...] # wrap an agent; its shell routes through settle exec +settle doctor # self-check +``` + +The reliable, universal integration is to wire `settle exec` as the harness's +command prefix (or PreToolUse hook). `settle run` is a zero-config convenience +wrapper that installs a shell shim so an agent honouring `$SHELL` is supervised +without any changes to its internals. + +## Documentation index + +- [`docs/01-theory.md`](docs/01-theory.md) — the control-theory frame: an + approximate belief-state supervisor for an unobservable spec; why the + false-positive rate is a theorem, not a defect; fail-open vs. fail-closed. +- [`docs/02-detection.md`](docs/02-detection.md) — the recurrent-class detector + and how it subsumes all five sensors; the canonicalization contract; the + belief function. +- [`docs/03-intervention.md`](docs/03-intervention.md) — the minimum-force + ladder; the nonblocking and fail-open invariants; the deny-with-recovery + contract. +- [`docs/04-calibration-and-experiment.md`](docs/04-calibration-and-experiment.md) + — deriving the point-of-no-return from real traces; the pre-registered paired + experiment; claim-tier discipline. + +## Module layout + +``` +settle/ Go module: github.com/operatorstack/settle + detector/ pure, deterministic classifier core (no I/O, no os/exec) + exec/ thin host wrapper: runs a command, hashes workspace state + cmd/settle/ the binary +scripts/ + calibrate.py mine trajectories -> Recovery Probability Curve -> k + check_import_boundary.sh detector/ must never import exec/ +calibration.json the derived point-of-no-return + full curve (transparency) +docs/ theory, detection, intervention, calibration +``` + +**Relationship to lab 17 (`f-prime-governor`).** `settle` is a new architecture, +not lab 17 shipped. It keeps lab 17's defensible ideas (the EV inequality, +minimum-force under partial observability, weak-sensor agreement) and drops the +`F' = p(1−p)` framing, which was decorative — it is only Bernoulli variance, was +computed offline only, and the live governor actually fired on a hardcoded +"last 3 failed". `settle` also fixes lab 17's `swe_breaker.py exit(1)` hard-block, +which violated the house **nonblocking** invariant. The semantic-signature and +command-normalization logic is ported into a single Go source of truth +(`detector/signature.go`). + +## Status + +Detection firing correctly on the golden fixtures is **Observed**. Net effect on +task success is **Being-evaluated** until the pre-registered paired run +(docs/04). No invented numbers; no cross-model uplift claims. diff --git a/labs/18-settle/calibration.json b/labs/18-settle/calibration.json new file mode 100644 index 000000000..8188076e2 --- /dev/null +++ b/labs/18-settle/calibration.json @@ -0,0 +1,129 @@ +{ + "point_of_no_return": 5, + "adopted": "fallback", + "derived_k": 2, + "derived_support": 44, + "fallback_k": 5, + "recovery_floor": 0.15, + "min_support": 20, + "strong_support": 200, + "trajectory_files": 1890, + "curve": [ + { + "k": 1, + "reached": 7170, + "recovered": 8, + "p_recovery": 0.0011 + }, + { + "k": 2, + "reached": 44, + "recovered": 3, + "p_recovery": 0.0682 + }, + { + "k": 3, + "reached": 12, + "recovered": 2, + "p_recovery": 0.1667 + }, + { + "k": 4, + "reached": 5, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 5, + "reached": 2, + "recovered": 1, + "p_recovery": 0.5 + }, + { + "k": 6, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 7, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 8, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 9, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 10, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 11, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 12, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 13, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 14, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 15, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 16, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 17, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 18, + "reached": 1, + "recovered": 0, + "p_recovery": 0.0 + }, + { + "k": 19, + "reached": 1, + "recovered": 1, + "p_recovery": 1.0 + } + ], + "reason": "Data hints k=2, but on thin support (44 sequences, need >=200). Real traces rarely contain byte-identical repeated failing commands, so genuine saturated-failure loops are scarce here. Shipping the conservative fallback (5); the redirect rung already covers k=2..4 non-destructively. Being-evaluated pending richer trace data.", + "claim_tier": "Observed (calibration from Harbor traces); not a universal constant." +} diff --git a/labs/18-settle/docs/01-theory.md b/labs/18-settle/docs/01-theory.md new file mode 100644 index 000000000..55fd98c9f --- /dev/null +++ b/labs/18-settle/docs/01-theory.md @@ -0,0 +1,113 @@ +# 01 — Theory: an approximate supervisor for an unobservable spec + +`settle` is a supervisory controller in the sense of Ramadge & Wonham (1987). +The canonical statement of that framework for this system is the repo-root +[`supervisory-control.md`](../../../supervisory-control.md); this document places +`settle` inside it. Read that doc first — the objects (`Σ`, `Σ_c/Σ_u`, +`Σ_o/Σ_uo`, `P`, `L_m(G)`, controllability, observability, nonblocking) are used +here without redefinition. + +## 1. The one idea + +> Boatstack's specification is co-designed to be **observable**, so an exact +> supervisor exists and it has **no** false-positive rate. `settle`'s +> specification is **unobservable**, so by the Existence theorem no exact +> supervisor exists; `settle` runs an **approximate belief-state supervisor** and +> pays for the unobservability with an irreducible false-positive rate. + +Everything else in `settle` is a consequence of that sentence. + +## 2. The objects, instantiated + +- **Plant `G`.** The agent plus its host, generating a stream of events. The + slice `settle` cares about is the command-execution loop. +- **Controllable event.** *Execute the next command.* This is the one event + `settle` can disable — exactly the `Σ_c` events the host presents synchronously + for permission. `settle` may withhold permission for it; it can never force the + agent to think differently (that is `Σ_u`, uncontrollable). +- **Observation projection `P`.** `settle` never sees the agent's intent or its + true progress. It sees a lossy projection: a normalized command, a semantic + error signature, and a workspace state-hash (`detector.Observation`). This is + `P(s)`, and it is deliberately coarse. +- **Marked language `L_m(G)`.** Command outcomes that represent success + (exit 0 / tests passing). A marked symbol is the only positive evidence + `settle` can observe. +- **Specification `K`.** "Do not remain in a non-progressing cycle" — i.e., do + not dwell in a set of observed states that contains no marked symbol and that + the trajectory does not leave. + +## 3. Why the spec is unobservable (and why that is the whole point) + +The house doc's observability condition (its §2) requires that any two histories +with the same projection demand the same control decision. `settle`'s spec fails +this: + +> A genuinely stuck loop and a slow-but-progressing effort can produce the **same +> projection** `P(s)` — identical repeated error text, while the real, unobserved +> state differs (the agent is narrowing a search, warming a cache, making +> progress the sensors cannot see). + +Two histories, same projection, opposite correct decisions (deny vs. allow). By +the Existence theorem, **no supervisor achieving exactly `K` exists.** This is not +a limitation to apologize for — it is the theorem that *predicts* the +false-positive rate. `settle` therefore does not compute membership in `K`; it +maintains a **belief** `b ≈ P(stuck | P(s))` and acts on it. See +[02-detection.md](02-detection.md) for how `b` is estimated and +[03-intervention.md](03-intervention.md) for how force is chosen given `b`. + +## 4. What survives from the theory, exactly + +Three results transfer with full rigor; nothing is borrowed decoratively. + +1. **Minimum force = the minimally-restrictive supervisor.** The house doc's + `supC(K)` (disable the minimum necessary) is, verbatim, "minimum-force + intervention." Two independent arguments — the economic `EV > 0` inequality + and the control-theoretic minimally-restrictive supervisor — land on the same + control law. That convergence is the justification. + +2. **Nonblocking is a hard invariant.** The house doc requires a blocked state be + blocked *toward* a recovery path, never absorbing. Therefore `settle` may never + disable *every* controllable event: even its highest-confidence deny withholds + one futile command and leaves read-only inspection open. (Lab 17's + `swe_breaker.py exit(1)` violated this.) + +3. **Fail-open, inverting Boatstack's fail-closed.** The house doc's fail-closed + corollary — under observation loss, disable all controllable events + (`γ = Σ_u`) — is the unique safe pattern *for a safety spec*, where the cost of + allowing a bad event is unbounded. `settle` is an **optimizer**, not a safety + gate: the cost of *wrongly blocking* a healthy agent is the expensive error, + and the benefit it protects (saved steps) is bounded. So under uncertainty or + sensor loss, `settle` **allows**. This is why `exec.WorkspaceState` returning + "unmeasurable" degrades to allow, never to deny. + +## 5. The EV frame (the economic half of §4.1) + +An intervention's expected value: + +``` +EV = P(stuck | signals) · (steps saved by breaking the loop) + − P(not stuck | signals) · (cost of interrupting a good trajectory) +``` + +Inference cost is ~0, so attempting an intervention is nearly free. There are +exactly two levers, and they are the entire design space: + +1. **Raise `P(stuck | signals)`** — more accurate, agreeing sensors. +2. **Lower the cost-when-wrong** — a gentler, reversible intervention. + +The governing corollary: **intervention design matters more than detection +accuracy.** A reversible nudge at a 30% false-positive rate is net-positive; a +destructive revert at the same rate is net-negative. That is why the intervention +ladder is designed before the detector is tuned. + +## 6. What `settle` explicitly is not + +- It does **not** judge correctness. "Is this the right fix" is not in the + observable alphabet; deciding it needs the model or a real verifier. Claiming a + $0 controller judges correctness is the trap `settle` refuses. +- It does **not** act while the trajectory is still emitting *new* observations — + a moving loop is exploring, not stuck. (This replaces lab 17's "F′ is high / + sensitive region" with a directly measurable condition: not yet absorbed into a + recurrent class.) +- It does **not** eliminate false positives. The Existence theorem forbids it. It + makes them cheap. diff --git a/labs/18-settle/docs/02-detection.md b/labs/18-settle/docs/02-detection.md new file mode 100644 index 000000000..0e364d381 --- /dev/null +++ b/labs/18-settle/docs/02-detection.md @@ -0,0 +1,102 @@ +# 02 — Detection: one recurrent-class detector, five sensors + +`settle` estimates the belief `b ≈ P(stuck | P(s))` with zero model inference. +The estimate has one primary object and several corroborating sensors, combined +by agreement. All of it is hashing, counting, diffing, and regex. + +## 1. The observation symbol + +Each executed command yields a `detector.Observation` — the projection `P(s)`: + +```go +Observation{ NormCommand, ErrSignature, StateHash, ExitCode } +``` + +The identity used for non-progress detection is the **joint** effect-side symbol: + +``` +symbol = ErrSignature ⊕ StateHash +``` + +Not the error alone. Two attempts with the same symbol produced the same +observable effect — provably no forward motion between them. Keying on the joint +`(error, state)` is what lets `settle` leave a loop alone whose error text +repeats while the workspace keeps changing (hidden progress). This is verified by +the `hidden-progress` golden fixture, which an error-only detector would +false-positive on. + +## 2. The primary object: a non-marked recurrent class + +The real question is not "what is the variance of the pass rate" (lab 17's +`F' = p(1−p)`, which is just Bernoulli variance and needs a whole window to even +estimate `p`). It is: + +> Has the observed trajectory entered a **recurrent class** — a cycle in the +> symbol stream — that contains **no marked (success) symbol**? + +`detector.detectRecurrence` finds the smallest period `p ≤ MaxCyclePeriod` whose +block repeats at the tail of history, and checks whether any marked symbol falls +inside the cycle window. This single detector subsumes the brief's separate +sensors: + +| Brief's sensor | In `settle` | +|---|---| +| state-unchanged / semantic-error repeat | period-1 cycle (fixed point) — strongest, observes *effect* | +| oscillation A→B→A→B | period-`k` cycle | + +`Reps ≥ 2` is required before anything is called a cycle; `Dwell = Period × Reps` +is the number of observations the cycle has consumed (comparable to the +calibrated point-of-no-return `k`). + +## 3. Leading, input-side sensors + +Two cheap sensors read the *input* side and fire *before* the effect-side cycle +confirms — so they are treated as low-confidence leading indicators +(`detector/sensors.go`): + +- **Edit-distance floor.** Normalized Levenshtein between consecutive normalized + commands `≤ EditDistanceFloor` ⇒ trivial mutation (`x=1 → x=2`), i.e. fiddling + not rethinking. +- **Entropy collapse.** Shannon entropy of the command distribution over the + window below `EntropyFloor` ⇒ the agent's idea alphabet is collapsing. (Counts + are summed in sorted order so the result is bit-for-bit deterministic despite + Go's randomized map iteration.) + +Lagging observation-cycles → high confidence. Leading input-signals → low +confidence. **The force ladder falls out of this split for free.** + +## 4. Canonicalization (the sensor-fidelity contract) + +`detector/signature.go` is the single source of truth (ported from lab 17's +`test_semantic.py` / `operatorstack_harbor.py`, which had drifted across three +files): + +- `NormalizeCommand` strips volatile temp paths (`/tmp/…`, `/var/folders/…`) and + collapses whitespace, so retries of the same command group together. +- `SemanticSignature` strips memory addresses (`0x…`), line numbers (`line N`), + and paths, then hashes the **last three non-empty lines** (the core exception). + +The signature returns the canonical *core* string alongside the hash so callers +can log both raw and canonical forms. **Over-stripping** — two genuinely +different errors collapsing to one hash — is a false-positive source that is only +diagnosable if the core is auditable. The `signature_test.go` cases assert both +directions: volatile-token variants must collapse; distinct errors must not. + +## 5. The belief function + +`detector.assess` combines the readings into `b ∈ [0,1]` and the list of signals +that fired. It is monotone (more agreement, longer dwell → never lower belief) +and deliberately simple — the correctness argument is the EV / minimum-force +economics and the recurrent-class structure, not a tuned curve: + +- A confirmed, **unmarked** cycle is the main term; a period-1 fixed point counts + slightly stronger than an oscillation. Belief scales with `Dwell / k`. +- A marked symbol anywhere in the cycle window zeroes the main term — the loop is + still reaching success, so it is not stuck. +- Each leading sensor adds a small increment (weak sensors, strong agreement). +- Leading-only agreement (fiddling detected, no confirmed cycle yet) is capped at + the "inform" band — never enough to deny. + +`b` is an estimate of `P(stuck | observations)`, never a certainty. The spec is +unobservable ([01-theory.md](01-theory.md) §3), so certainty is unavailable in +principle. diff --git a/labs/18-settle/docs/03-intervention.md b/labs/18-settle/docs/03-intervention.md new file mode 100644 index 000000000..1c4fc31ab --- /dev/null +++ b/labs/18-settle/docs/03-intervention.md @@ -0,0 +1,57 @@ +# 03 — Intervention: minimum force, nonblocking, fail-open + +The intervention is designed *before* the detector is tuned, because under +partial observation the cost-when-wrong dominates the expected value +([01-theory.md](01-theory.md) §5). Force is proportional to belief; every rung is +nonblocking. + +## The ladder (`detector/ladder.go`) + +| Belief band | Action | Outcome | What the agent sees | +|---|---|---|---| +| `< Low` | allow | `allow` | nothing (silent) | +| `[Low, Med)` | **inform** | `allow` | "this attempt is repeating with little observable change." | +| `[Med, High)` | **redirect** | `allow` | "stop mutating the command; inspect the mechanism (config, environment, upstream definition)." | +| `≥ High` **and** past the point of no return | **deny** | `deny` | the futile command is withheld; a read-only recovery path is named. | + +`Outcome` is exactly Pitot's `allow | deny`, so a future bridge adapter is a +straight translation with no remapping. + +## The three invariants (each from [01-theory.md](01-theory.md) §4) + +1. **Deny is gated hard.** A deny requires *all* of: belief `≥ High`, a confirmed + cycle (`Reps ≥ 2`), no marked symbol in the cycle, **and** `Dwell ≥ + PointOfNoReturn`. The dwell gate is why a near-miss (two identical failures + that then resolve) is never denied even if leading sensors briefly push belief + up — the low-cost `inform`/`redirect` rungs may fire early, but the + high-cost rung may not. + +2. **Nonblocking.** A deny withholds *this one command only*. The agent can still + run any other command — read-only inspection, a different approach, a + diagnostic. `settle` never disables every controllable event, so delivery is + never wedged. This is the invariant lab 17's `swe_breaker.py exit(1)` + hard-block violated. + +3. **Fail-open.** `settle` is an optimizer, not a safety gate. When state cannot + be measured (`exec.WorkspaceState` returns not-ok — not a git repo, git + absent) or the process cannot be started, `settle` **allows**. A blind sensor + must never block a healthy agent. This inverts Boatstack's fail-closed + corollary by design, because the asymmetry of costs is inverted. + +## The deny contract in the wrapper (`cmd/settle/exec.go`) + +- On **deny**: the command is **not executed and not recorded** — dwell does not + advance on an intervention, so `settle` will not "run up the score" against a + command it is already blocking. The recovery message goes to stderr; exit is + nonzero to signal the agent the command did not run. +- On **inform / redirect**: the command **is** executed and recorded; the + guidance is surfaced afterward. These rungs are reversible and near-zero cost + if the agent was in fact fine — exactly the profile the EV frame rewards. + +## Prevention (a cheaper win than interruption) + +A future extension (noted, not yet built): a pre-execution catalog of known-stuck +patterns — interactive-prompt hangs needing a non-interactive flag (`opam` → +`OPAMYES`, `apt` without `-y`) — flagged before the command runs. That is a +pre-condition gate on a controllable event, the same architecture as Boatstack's +gates, and it stops the wasted steps before they are ever spent. diff --git a/labs/18-settle/docs/04-calibration-and-experiment.md b/labs/18-settle/docs/04-calibration-and-experiment.md new file mode 100644 index 000000000..d73d1376a --- /dev/null +++ b/labs/18-settle/docs/04-calibration-and-experiment.md @@ -0,0 +1,83 @@ +# 04 — Calibration and the experiment + +Two empirical questions, kept honest and separated by claim tier. + +## 1. Calibrating the point of no return (`k`) + +`k` (`Config.PointOfNoReturn`) is the dwell past which a confirmed, unmarked +cycle may be denied. Lab 17 hardcoded `3` as if principled. `settle` derives it +from data with `scripts/calibrate.py`. + +The script mines ATIF-v1.7 trajectory JSON for runs of the *same* normalized +command and builds the **Recovery Probability Curve** `P(recovery | k +consecutive identical failures)`. The point of no return is the smallest `k` +where continuing no longer pays off under the EV frame — `P(recovery | k)` below +a conservative `recovery_floor`. + +Two guardrails encode the theory and the honesty: + +- **`k = 1` is not a candidate.** A loop does not exist until there is a repeat; + `settle` itself only treats the trajectory as recurrent at `Dwell ≥ 2`. The + `k = 1` bucket just measures how often agents change the command, not + non-recovery. +- **Thin support stays advisory.** A derived `k` is only *shipped* if its support + clears `strong_support`; otherwise the conservative fallback is shipped and the + derived value is recorded as a hint. + +### What the current data actually says + +Run against 1,890 Harbor trajectories, the curve is: + +| k | reached | recovered | P(recovery\|k) | +|---|---|---|---| +| 1 | 7170 | 8 | 0.001 | +| 2 | 44 | 3 | 0.068 | +| 3 | 12 | 2 | 0.167 | +| ≥4 | ≤5 | — | sparse | + +The honest reading, recorded in `calibration.json`: + +- The `k=1` bucket is enormous and near-zero-recovery — but that is a **grouping + artifact**, not a deny threshold. Agents almost never re-run the *byte-identical* + failing command; they mutate it. So genuine saturated-failure loops (identical + command *and* identical effect, repeated) are **scarce** in real traces. +- Data hints `k = 2`, but on 44 sequences — below `strong_support` (200). So + `settle` ships the **conservative fallback `k = 5`**, and records `derived_k: 2` + with its support and the reason. The `redirect` rung already covers the `k=2..4` + range non-destructively. + +This finding validates `settle`'s design over lab 17's: because literal repeats +are rare, a controller that leans on the *literal* command hash (lab 17's live +governor) has little to fire on. The semantic-signature and joint `(error,state)` +sensors — which catch "different command, same effect" — are where the signal +actually is. + +**Claim tier: Observed** (calibration from Harbor traces). Not a universal +constant; re-derive per corpus. + +## 2. The pre-registered paired experiment (not yet run) + +Whether `settle` improves task outcomes is **Being-evaluated** — it is not +claimed by this lab. The experiment, registered here before running: + +- **Design.** Paired, same model, same tasks and seeds: governed (`settle run`) + vs. ungoverned. Same-model comparison only — no cross-model "reaches frontier" + claims. +- **Primary metric.** Task success rate and steps-to-success on tasks both arms + solve. +- **Secondary.** Token cost. +- **Safety metric (the false-positive harm).** Log every firing and its + subsequent task outcome (recovered vs. failed) to estimate how often an + intervention hurt a trajectory that was actually fine. +- **Predicted direction, stated before the run:** on tasks with genuine + saturated-failure loops, governed reaches success in fewer steps or converts a + timeout into a success; on clean tasks, governed is within noise of ungoverned + (the fail-open, minimum-force design should make wrongful interference rare). + +## Claim discipline (applies to all `settle` docs and any writeup) + +- **Verified / Observed / Being-evaluated** tiers, always labeled. +- Detection firing correctly on the golden fixtures = **Observed**. +- Net effect on task success = **Being-evaluated** until the paired run above. +- No invented numbers. No "proven" for empirical results. No cross-model uplift + claims. The one correct headline is in the [README](../README.md). diff --git a/labs/18-settle/scripts/calibrate.py b/labs/18-settle/scripts/calibrate.py new file mode 100644 index 000000000..3b3e28b40 --- /dev/null +++ b/labs/18-settle/scripts/calibrate.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Calibrate settle's point-of-no-return (k) from real agent trajectories. + +This is the honest, useful half of lab 17's calibrate_f_prime.py, with the F' = +p(1-p) framing dropped. It mines ATIF-v1.7 trajectory JSON for consecutive +command failures and builds the Recovery Probability Curve P(recovery | k +consecutive failures). The point of no return is the smallest k at which +continuing is no longer expected to pay off: + + EV(retry at k) = P(recovery | k) * value(recovery) + - (1 - P(recovery | k)) * cost(wasted step) + +With value ~ cost (a recovered step is worth roughly what a wasted one costs), +EV crosses zero at P(recovery | k) ~ 0.5; we use a conservative RECOVERY_FLOOR +so settle only denies where recovery is clearly unlikely. k is emitted to +calibration.json, which the Go binary reads as its default PointOfNoReturn. If +no data is available the documented fallback (5) is kept. + +Usage: + python calibrate.py [--artifacts DIR] [--floor 0.15] [--min-support 20] [--out calibration.json] +""" +import argparse +import glob +import json +import os +import re +from collections import defaultdict + +FALLBACK_K = 5 + + +def normalize_command(command: str) -> str: + """Group retries of the same command (ported from settle's NormalizeCommand).""" + if not command: + return "" + cmd = re.sub(r"/(tmp|var/folders)/[a-zA-Z0-9_/-]+", "/tmp/...", command) + cmd = re.sub(r"\s+", " ", cmd) + return cmd.strip() + + +def process_traces(artifact_dir: str): + """Extract per-command exit-code sequences from ATIF-v1.7 trajectory files.""" + command_sequences = defaultdict(list) + files = glob.glob(os.path.join(artifact_dir, "**/*trajectory*.json"), recursive=True) + if not files: + return command_sequences, 0 + + for file_path in files: + try: + with open(file_path, "r") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError): + continue + + current_command = None + current_sequence = [] + for step in data.get("steps", []): + for tc in step.get("tool_calls", []) or []: + if tc.get("function_name") not in ("bash", "run_shell_command"): + continue + cmd = (tc.get("arguments", {}) or {}).get("command") + if not cmd: + continue + call_id = tc.get("tool_call_id") + result_content = "" + obs = step.get("observation", {}) or {} + for r in obs.get("results", []) or []: + if r.get("source_call_id") == call_id: + result_content = r.get("content", "") or "" + lower = result_content.lower() + exit_code = 1 if any( + x in lower for x in ("error", "exception", "failed", "command not found", "traceback", "exit status 1") + ) else 0 + normalized = normalize_command(cmd) + if normalized == current_command: + current_sequence.append(exit_code) + else: + if current_command is not None: + command_sequences[current_command].append(current_sequence) + current_command = normalized + current_sequence = [exit_code] + if current_command is not None: + command_sequences[current_command].append(current_sequence) + + return command_sequences, len(files) + + +def recovery_curve(command_sequences): + """P(recovery | k) = fraction of runs at k consecutive failures that then succeed.""" + reached = defaultdict(int) # runs that reached k consecutive failures + recovered = defaultdict(int) # of those, ones whose next attempt succeeded + for sequences in command_sequences.values(): + for seq in sequences: + k = 0 + for exit_code in seq: + if exit_code != 0: + k += 1 + reached[k] += 1 + else: + if k > 0: + recovered[k] += 1 + k = 0 + return reached, recovered + + +# A repeat must exist before there is a loop at all: settle only treats the +# trajectory as recurrent at dwell >= 2, so k=1 (every first failure ever) is not +# a candidate deny threshold — including it just measures how often agents change +# the command, not non-recovery. Candidates start at 2. +MIN_CANDIDATE_K = 2 +# Support required before a derived k is trusted enough to SHIP as the default +# (as opposed to merely reported). Thin buckets stay advisory. +STRONG_SUPPORT = 200 + + +def choose_k(reached, recovered, floor, min_support): + max_k = max(reached) if reached else 0 + curve = [] + derived_k, derived_support = None, 0 + for k in range(1, max_k + 1): + n = reached[k] + p = (recovered[k] / n) if n else 0.0 + curve.append({"k": k, "reached": n, "recovered": recovered[k], "p_recovery": round(p, 4)}) + if derived_k is None and k >= MIN_CANDIDATE_K and n >= min_support and p < floor: + derived_k, derived_support = k, n + return derived_k, derived_support, curve + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--artifacts", default="labs/11-harbor-submit", help="dir of trajectory JSON") + ap.add_argument("--floor", type=float, default=0.15, help="recovery-probability floor for the point of no return") + ap.add_argument("--min-support", type=int, default=20, help="min runs reaching k before trusting P(recovery|k)") + ap.add_argument("--out", default="labs/18-settle/calibration.json") + args = ap.parse_args() + + sequences, n_files = process_traces(args.artifacts) + reached, recovered = recovery_curve(sequences) + derived_k, derived_support, curve = choose_k(reached, recovered, args.floor, args.min_support) + + strong = derived_k is not None and derived_support >= STRONG_SUPPORT + adopted_k = derived_k if strong else FALLBACK_K + adopted = "derived" if strong else "fallback" + + if derived_k is None: + reason = ( + f"No k>={MIN_CANDIDATE_K} had support>={args.min_support} with " + f"P(recovery|k)<{args.floor}. Shipping fallback ({FALLBACK_K})." + ) + elif strong: + reason = ( + f"Derived k={derived_k} from {derived_support} sequences " + f"(P(recovery|{derived_k})<{args.floor}); support is strong, so it is shipped." + ) + else: + reason = ( + f"Data hints k={derived_k}, but on thin support ({derived_support} sequences, " + f"need >={STRONG_SUPPORT}). Real traces rarely contain byte-identical repeated " + f"failing commands, so genuine saturated-failure loops are scarce here. Shipping the " + f"conservative fallback ({FALLBACK_K}); the redirect rung already covers k={derived_k}..4 " + "non-destructively. Being-evaluated pending richer trace data." + ) + + result = { + "point_of_no_return": adopted_k, + "adopted": adopted, + "derived_k": derived_k, + "derived_support": derived_support, + "fallback_k": FALLBACK_K, + "recovery_floor": args.floor, + "min_support": args.min_support, + "strong_support": STRONG_SUPPORT, + "trajectory_files": n_files, + "curve": curve, + "reason": reason, + "claim_tier": "Observed (calibration from Harbor traces); not a universal constant.", + } + with open(args.out, "w") as f: + json.dump(result, f, indent=2) + f.write("\n") + + print(f"scanned {n_files} trajectory files") + print(f"{'k':>3} | {'reached':>8} | {'recovered':>9} | P(recovery|k)") + for row in curve[:15]: + print(f"{row['k']:>3} | {row['reached']:>8} | {row['recovered']:>9} | {row['p_recovery']:.3f}") + print(f"\npoint_of_no_return = {adopted_k} ({adopted})") + print(reason) + print(f"written to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/labs/18-settle/scripts/check_import_boundary.sh b/labs/18-settle/scripts/check_import_boundary.sh new file mode 100755 index 000000000..a6c829b27 --- /dev/null +++ b/labs/18-settle/scripts/check_import_boundary.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Enforce settle's detector-innocence invariant at build time: the detector +# (classifier) package must never import the exec (host-execution) package. The +# classifier decides only from the observations handed to it; it does not reach +# the OS. This is a structural guarantee, not a convention, so a refactor cannot +# silently blur the boundary. Mirrors Pitot's sensor-innocence check. +set -euo pipefail + +MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../settle" && pwd)" +MODULE_PATH="github.com/operatorstack/settle" + +cd "$MODULE_DIR" + +FORBIDDEN_PKG="${MODULE_PATH}/detector" +FORBIDDEN_DEP="${MODULE_PATH}/exec" + +echo "check_import_boundary: ${FORBIDDEN_PKG} must not depend on ${FORBIDDEN_DEP}" + +DEPS="$(GOWORK=off go list -deps "$FORBIDDEN_PKG")" +if echo "$DEPS" | grep -qx "$FORBIDDEN_DEP"; then + echo "FAIL: ${FORBIDDEN_PKG} imports ${FORBIDDEN_DEP} (directly or transitively)." >&2 + echo " The classifier decides; it must not reach the execution host." >&2 + exit 1 +fi + +echo "OK: detector-innocence boundary holds." diff --git a/labs/18-settle/settle/cmd/settle/doctor.go b/labs/18-settle/settle/cmd/settle/doctor.go new file mode 100644 index 000000000..a16c6bdd6 --- /dev/null +++ b/labs/18-settle/settle/cmd/settle/doctor.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "os" + + "github.com/operatorstack/settle/detector" +) + +// runDoctor is a self-check: it feeds the detector a synthetic saturated-failure +// loop and confirms the ladder escalates to a deny, then feeds a genuine-progress +// sequence and confirms it never intervenes. This exercises the core without any +// external dependency. +func runDoctor(_ []string) int { + cfg := detector.DefaultConfig() + + // 1. A stuck loop must eventually be denied. + d := detector.New(cfg) + denied := false + for i := 0; i < cfg.PointOfNoReturn+3; i++ { + if d.Observe("pytest tests/").Outcome == detector.OutcomeDeny { + denied = true + break + } + d.Record("pytest tests/", 1, "AssertionError: expected 200, got 401", "state-unchanged") + } + if !denied { + fmt.Fprintln(os.Stderr, "settle doctor: FAIL — a saturated-failure loop was never denied") + return 1 + } + + // 2. Genuine progress (new state every step) must never be denied. + p := detector.New(cfg) + for i := 0; i < 10; i++ { + if p.Observe(fmt.Sprintf("step-%d", i)).Outcome == detector.OutcomeDeny { + fmt.Fprintln(os.Stderr, "settle doctor: FAIL — genuine progress was denied") + return 1 + } + p.Record(fmt.Sprintf("step-%d", i), 1, fmt.Sprintf("error variant %d", i), fmt.Sprintf("state-%d", i)) + } + + fmt.Println("settle doctor: OK — denies a saturated-failure loop, leaves genuine progress alone") + return 0 +} diff --git a/labs/18-settle/settle/cmd/settle/exec.go b/labs/18-settle/settle/cmd/settle/exec.go new file mode 100644 index 000000000..81b409ac7 --- /dev/null +++ b/labs/18-settle/settle/cmd/settle/exec.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + "github.com/operatorstack/settle/detector" + sexec "github.com/operatorstack/settle/exec" +) + +// runExec supervises a single command. It is the atomic unit an agent harness +// wires in as its command prefix. Session history is persisted to SETTLE_SESSION +// so successive invocations within one wrapped agent share detector state. +func runExec(args []string) int { + argv := stripDashDash(args) + if len(argv) == 0 { + fmt.Fprintln(os.Stderr, "settle exec: no command given") + return 2 + } + + // Support the `$SHELL -c "cmd"` interposition used by `settle run`: an agent + // that spawns its shell as `$SHELL -c ""` arrives here as `-c `. + var cmdText string + var runArgv []string + if argv[0] == "-c" && len(argv) >= 2 { + cmdText = argv[1] + runArgv = []string{"/bin/sh", "-c", argv[1]} + } else { + cmdText = strings.Join(argv, " ") + runArgv = argv + } + + d, sess := loadDetector() + dec := d.Observe(cmdText) + + // Deny: withhold THIS command only (nonblocking — the agent can still run + // other, read-only commands). The command is not run and not recorded. + if dec.Outcome == detector.OutcomeDeny { + fmt.Fprintln(os.Stderr, dec.Message) + saveDetector(d, sess) + return 1 + } + + dir, _ := os.Getwd() + res, err := sexec.Run(dir, runArgv) + if err != nil { + // Could not start the process. Fail open: surface the error, do not + // pretend the loop is stuck. + fmt.Fprintln(os.Stderr, "settle: could not execute command:", err) + return 1 + } + + state, _ := sexec.WorkspaceState(dir) // unmeasurable state -> "" (fail open) + d.Record(cmdText, res.ExitCode, res.Stderr, state) + saveDetector(d, sess) + + // Inform / redirect: allow the command, then surface guidance. + if dec.Message != "" { + fmt.Fprintln(os.Stderr, dec.Message) + } + return res.ExitCode +} + +// loadDetector builds a detector from defaults, the SETTLE_K override, and any +// persisted session state. It returns the session path (possibly ""). +func loadDetector() (*detector.Detector, string) { + cfg := detector.DefaultConfig() + if k := os.Getenv("SETTLE_K"); k != "" { + if v, err := strconv.Atoi(k); err == nil && v > 0 { + cfg.PointOfNoReturn = v + } + } + d := detector.New(cfg) + + path := os.Getenv("SETTLE_SESSION") + if path != "" { + if b, err := os.ReadFile(path); err == nil { + var st detector.State + if json.Unmarshal(b, &st) == nil { + d.Import(st) + } + } + } + return d, path +} + +func saveDetector(d *detector.Detector, path string) { + if path == "" { + return + } + if b, err := json.Marshal(d.Export()); err == nil { + _ = os.WriteFile(path, b, 0o644) + } +} diff --git a/labs/18-settle/settle/cmd/settle/main.go b/labs/18-settle/settle/cmd/settle/main.go new file mode 100644 index 000000000..6abe38b0d --- /dev/null +++ b/labs/18-settle/settle/cmd/settle/main.go @@ -0,0 +1,60 @@ +// Command settle is a zero-inference, partial-observation controller that wraps +// any agent's command execution and intervenes with minimum force when the loop +// stops making observable progress. +// +// settle exec -- [args...] supervise one command (the atomic unit) +// settle run -- [args...] wrap an agent; its shell routes through settle exec +// settle doctor self-check +package main + +import ( + "fmt" + "os" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + switch os.Args[1] { + case "exec": + os.Exit(runExec(os.Args[2:])) + case "run": + os.Exit(runRun(os.Args[2:])) + case "replay": + os.Exit(runReplay(os.Args[2:])) + case "doctor": + os.Exit(runDoctor(os.Args[2:])) + case "-h", "--help", "help": + usage() + os.Exit(0) + default: + fmt.Fprintf(os.Stderr, "settle: unknown subcommand %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `settle — zero-inference non-progress controller + +usage: + settle exec -- [args...] supervise one command + settle run -- [args...] wrap an agent executable + settle replay -- ... offline: report what settle would do on a recorded run + settle doctor self-check + +env: + SETTLE_SESSION path to the session history file (set automatically by 'run') + SETTLE_K override the point-of-no-return dwell (default: calibrated) +`) +} + +// stripDashDash drops a single leading "--" separator if present. +func stripDashDash(args []string) []string { + if len(args) > 0 && args[0] == "--" { + return args[1:] + } + return args +} diff --git a/labs/18-settle/settle/cmd/settle/replay.go b/labs/18-settle/settle/cmd/settle/replay.go new file mode 100644 index 000000000..588989d9b --- /dev/null +++ b/labs/18-settle/settle/cmd/settle/replay.go @@ -0,0 +1,155 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/operatorstack/settle/detector" +) + +// replay drives the detector over a recorded ATIF-v1.7 agent trajectory and +// reports what settle would have done, without running anything. It is the +// offline "observe" mode: safe, free, and useful for seeing how settle behaves +// on real agent runs before any live wiring. +// +// Honesty caveat, printed in the header: a recorded trajectory carries no +// workspace state-hash, so replay runs settle's error-signature sensor ONLY +// (state is unknown/""). Live settle keys recurrence on the JOINT (error,state), +// which is strictly more conservative. Replay therefore OVER-estimates +// stuck-ness relative to production — treat its denies as an upper bound. + +type atifTrajectory struct { + Steps []atifStep `json:"steps"` +} +type atifStep struct { + ToolCalls []atifToolCall `json:"tool_calls"` + Observation atifObs `json:"observation"` +} +type atifToolCall struct { + ToolCallID string `json:"tool_call_id"` + FunctionName string `json:"function_name"` + Arguments atifArgs `json:"arguments"` +} +type atifArgs struct { + Command string `json:"command"` +} +type atifObs struct { + Results []atifResult `json:"results"` +} +type atifResult struct { + SourceCallID string `json:"source_call_id"` + Content string `json:"content"` +} + +var errorMarkers = []string{"error", "exception", "failed", "command not found", "traceback", "exit status 1"} + +// inferExit heuristically maps terminal output to an exit code, mirroring +// scripts/calibrate.py. It is a heuristic, not a real exit code. +func inferExit(content string) int { + l := strings.ToLower(content) + for _, m := range errorMarkers { + if strings.Contains(l, m) { + return 1 + } + } + return 0 +} + +func runReplay(args []string) int { + argv := stripDashDash(args) + if len(argv) == 0 { + fmt.Fprintln(os.Stderr, "settle replay: give one or more trajectory.json paths") + return 2 + } + fmt.Fprintf(os.Stderr, "note: offline replay has no workspace state; settle runs error-signature-only "+ + "(no joint err+state), so denies are an UPPER bound vs live settle.\n\n") + rc := 0 + for _, path := range argv { + if replayOne(path) != 0 { + rc = 1 + } + } + return rc +} + +func replayOne(path string) int { + b, err := os.ReadFile(path) + if err != nil { + fmt.Fprintln(os.Stderr, "settle replay:", err) + return 1 + } + var traj atifTrajectory + if err := json.Unmarshal(b, &traj); err != nil { + fmt.Fprintln(os.Stderr, "settle replay:", path, err) + return 1 + } + + d := detector.New(detector.DefaultConfig()) + counts := map[detector.Action]int{} + firstDeny := -1 + total := 0 + + fmt.Printf("== %s ==\n", trimName(path)) + for si, step := range traj.Steps { + for _, tc := range step.ToolCalls { + if tc.FunctionName != "bash" && tc.FunctionName != "run_shell_command" { + continue + } + cmd := tc.Arguments.Command + if strings.TrimSpace(cmd) == "" { + continue + } + content := "" + for _, r := range step.Observation.Results { + if r.SourceCallID == tc.ToolCallID { + content = r.Content + break + } + } + total++ + + dec := d.Observe(cmd) + counts[dec.Action]++ + if dec.Action != detector.ActionAllow { + if dec.Action == detector.ActionDeny && firstDeny < 0 { + firstDeny = total + } + fmt.Printf(" step %-3d %-8s belief=%.2f [%s] %s\n", + si, dec.Action, dec.Belief, strings.Join(dec.Signals, ","), + truncate(detector.NormalizeCommand(cmd), 68)) + } + if dec.Outcome == detector.OutcomeDeny { + continue // would not have run, so not recorded + } + d.Record(cmd, inferExit(content), content, "") // no workspace state offline + } + } + + deny := "-" + if firstDeny >= 0 { + deny = fmt.Sprintf("#%d", firstDeny) + } + fmt.Printf(" -> %d commands | inform=%d redirect=%d deny=%d | first deny %s\n\n", + total, counts[detector.ActionInform], counts[detector.ActionRedirect], counts[detector.ActionDeny], deny) + return 0 +} + +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + "…" + } + return s +} + +// trimName shortens a long trajectory path to the task-identifying tail. +func trimName(path string) string { + parts := strings.Split(path, "/") + for i, p := range parts { + if p == "job" && i+1 < len(parts) { + return strings.Join(parts[i+1:], "/") + } + } + return path +} diff --git a/labs/18-settle/settle/cmd/settle/run.go b/labs/18-settle/settle/cmd/settle/run.go new file mode 100644 index 000000000..350a6f818 --- /dev/null +++ b/labs/18-settle/settle/cmd/settle/run.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// runRun wraps an agent executable. It allocates a fresh session file and +// installs a shell shim so the agent's `$SHELL -c ""` invocations route +// through `settle exec`, giving settle a view of every command the agent runs +// without any cooperation from the agent's internals. +// +// This is best-effort interposition: it works for agents that honour $SHELL. +// The reliable, universal surface is wiring `settle exec` directly as the +// harness command prefix; `run` is the zero-config convenience wrapper. +func runRun(args []string) int { + argv := stripDashDash(args) + if len(argv) == 0 { + fmt.Fprintln(os.Stderr, "settle run: no agent command given") + return 2 + } + + self, err := os.Executable() + if err != nil { + fmt.Fprintln(os.Stderr, "settle run:", err) + return 1 + } + + tmp, err := os.MkdirTemp("", "settle-session-") + if err != nil { + fmt.Fprintln(os.Stderr, "settle run:", err) + return 1 + } + defer os.RemoveAll(tmp) + + sess := filepath.Join(tmp, "session.json") + shim := filepath.Join(tmp, "settle-sh") + script := "#!/bin/sh\nexec " + shellQuote(self) + " exec -- \"$@\"\n" + if err := os.WriteFile(shim, []byte(script), 0o755); err != nil { + fmt.Fprintln(os.Stderr, "settle run:", err) + return 1 + } + + child := exec.Command(argv[0], argv[1:]...) + child.Stdin, child.Stdout, child.Stderr = os.Stdin, os.Stdout, os.Stderr + child.Env = append(os.Environ(), + "SETTLE_SESSION="+sess, + "SHELL="+shim, + ) + + if err := child.Run(); err != nil { + var ee *exec.ExitError + if ok := asExit(err, &ee); ok { + return ee.ExitCode() + } + fmt.Fprintln(os.Stderr, "settle run:", err) + return 1 + } + return 0 +} + +func asExit(err error, target **exec.ExitError) bool { + if ee, ok := err.(*exec.ExitError); ok { + *target = ee + return true + } + return false +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} diff --git a/labs/18-settle/settle/detector/belief.go b/labs/18-settle/settle/detector/belief.go new file mode 100644 index 000000000..3f69f1da8 --- /dev/null +++ b/labs/18-settle/settle/detector/belief.go @@ -0,0 +1,54 @@ +package detector + +// assess turns the raw sensor readings into a belief in [0,1] and the list of +// signals that fired. It is monotone: more agreement and longer dwell never +// lower the belief. The math is deliberately simple — the correctness argument +// is the EV / minimum-force economics and the recurrent-class structure, not a +// tuned curve. belief is an estimate of P(stuck | observations), never a +// certainty; the spec is unobservable, so certainty is unavailable in principle. +func assess(cfg Config, rec Recurrence, editFloor, entropyCollapse bool) (belief float64, signals []string) { + // Primary, effect-side evidence: the trajectory is in a confirmed, unmarked + // recurrent class. A success symbol in the window (rec.Marked) means the loop + // is still reaching marked states — not stuck. + if rec.Period >= 1 && rec.Reps >= 2 && !rec.Marked { + periodFactor := 1.0 + if rec.Period > 1 { + periodFactor = 0.9 // oscillation is slightly weaker evidence than a fixed point + } + dwellFrac := float64(rec.Dwell) / float64(cfg.PointOfNoReturn) + if dwellFrac > 1 { + dwellFrac = 1 + } + belief = (0.5 + 0.4*dwellFrac) * periodFactor + if rec.Period == 1 { + signals = append(signals, "state-unchanged") + } else { + signals = append(signals, "oscillation") + } + } + + // Leading, input-side corroboration. Each adds a small increment — weak + // sensors, strong agreement, at zero extra compute. + if editFloor { + belief += 0.08 + signals = append(signals, "edit-distance-floor") + } + if entropyCollapse { + belief += 0.08 + signals = append(signals, "entropy-collapse") + } + + // Leading-only: the input side agrees the agent is fiddling, but the effect + // side has not yet confirmed a cycle. This is enough to inform, never to deny. + if rec.Reps < 2 && editFloor && entropyCollapse { + if belief < cfg.LowThreshold+0.03 { + belief = cfg.LowThreshold + 0.03 + } + signals = append(signals, "leading-only") + } + + if belief > 1 { + belief = 1 + } + return belief, signals +} diff --git a/labs/18-settle/settle/detector/config.go b/labs/18-settle/settle/detector/config.go new file mode 100644 index 000000000..177be991c --- /dev/null +++ b/labs/18-settle/settle/detector/config.go @@ -0,0 +1,46 @@ +package detector + +// Config holds the detector thresholds. Every field is a plain number so the +// detector stays deterministic and free of I/O; the binary layer loads +// calibration.json and overrides PointOfNoReturn from mined data. +type Config struct { + // Window is the ring-buffer length of retained observations. + Window int + // MaxCyclePeriod bounds the oscillation period the recurrence detector looks for. + MaxCyclePeriod int + // PointOfNoReturn (k) is the dwell length past which a confirmed, unmarked + // cycle may be denied. Calibrated from the recovery-probability curve, not + // guessed. See scripts/calibrate.py and calibration.json. + PointOfNoReturn int + + // Belief thresholds selecting the intervention rung. + LowThreshold float64 // >= this: inform + MedThreshold float64 // >= this: redirect + HighThreshold float64 // >= this (and past PointOfNoReturn): deny + + // EditDistanceFloor: normalized Levenshtein <= this between consecutive + // commands counts as a trivial mutation (fiddling, not rethinking). + EditDistanceFloor float64 + // EntropyFloor: command-distribution Shannon entropy (bits) below this over a + // full window counts as idea-collapse. + EntropyFloor float64 + // MinWindowForEntropy is the minimum command count before entropy is trusted. + MinWindowForEntropy int +} + +// DefaultConfig returns the built-in defaults. PointOfNoReturn defaults to 5 as +// a documented fallback; the binary overrides it with the calibrated value. +// (Lab 17 hardcoded 3 as if principled — settle derives it instead.) +func DefaultConfig() Config { + return Config{ + Window: 32, + MaxCyclePeriod: 4, + PointOfNoReturn: 5, + LowThreshold: 0.25, + MedThreshold: 0.50, + HighThreshold: 0.80, + EditDistanceFloor: 0.15, + EntropyFloor: 1.0, + MinWindowForEntropy: 4, + } +} diff --git a/labs/18-settle/settle/detector/detector.go b/labs/18-settle/settle/detector/detector.go new file mode 100644 index 000000000..9f7857e0b --- /dev/null +++ b/labs/18-settle/settle/detector/detector.go @@ -0,0 +1,73 @@ +package detector + +// Detector is the pure, deterministic core: a stream of (command, outcome, +// state) in, a Decision out. Same inputs always produce the same output. It +// performs no I/O and never touches the OS — execution and state measurement +// live in the exec package, which the detector must not import. +type Detector struct { + cfg Config + obs []Observation + cmds []string +} + +// New returns a Detector with the given config. +func New(cfg Config) *Detector { return &Detector{cfg: cfg} } + +// Observe decides whether the command about to run should be allowed, from the +// history of prior observations plus the leading sensors (which include this +// pending command). It does not mutate history. +func (d *Detector) Observe(rawCommand string) Decision { + norm := NormalizeCommand(rawCommand) + rec := detectRecurrence(d.obs, d.cfg.MaxCyclePeriod) + + cmds := make([]string, 0, len(d.cmds)+1) + cmds = append(cmds, d.cmds...) + cmds = append(cmds, norm) + + editFloor := editFloorVote(cmds, d.cfg.EditDistanceFloor) + entropy := entropyCollapseVote(cmds, d.cfg.EntropyFloor, d.cfg.MinWindowForEntropy) + + belief, signals := assess(d.cfg, rec, editFloor, entropy) + return decide(d.cfg, rec, belief, signals) +} + +// Record appends the observed outcome of a command that was actually run, +// trimming history to the configured window. A denied command is never run and +// so is never recorded — dwell does not advance on interventions. +func (d *Detector) Record(rawCommand string, exitCode int, errText, stateHash string) Observation { + _, sig := SemanticSignature(errText) + if exitCode == 0 { + sig = "" // success carries no error signature; it is a marked symbol + } + o := Observation{ + NormCommand: NormalizeCommand(rawCommand), + ErrSignature: sig, + StateHash: stateHash, + ExitCode: exitCode, + } + d.obs = append(d.obs, o) + d.cmds = append(d.cmds, o.NormCommand) + if len(d.obs) > d.cfg.Window { + d.obs = d.obs[len(d.obs)-d.cfg.Window:] + d.cmds = d.cmds[len(d.cmds)-d.cfg.Window:] + } + return o +} + +// State is the serializable detector history, used to persist a session across +// separate `settle exec` invocations. +type State struct { + Obs []Observation `json:"obs"` + Cmds []string `json:"cmds"` +} + +// Export snapshots the history for persistence. +func (d *Detector) Export() State { + return State{Obs: append([]Observation(nil), d.obs...), Cmds: append([]string(nil), d.cmds...)} +} + +// Import restores a previously exported history. +func (d *Detector) Import(s State) { + d.obs = append([]Observation(nil), s.Obs...) + d.cmds = append([]string(nil), s.Cmds...) +} diff --git a/labs/18-settle/settle/detector/fixtures_test.go b/labs/18-settle/settle/detector/fixtures_test.go new file mode 100644 index 000000000..ab886181f --- /dev/null +++ b/labs/18-settle/settle/detector/fixtures_test.go @@ -0,0 +1,126 @@ +package detector + +import ( + "bufio" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// fixtureStep is one line of a golden fixture: a single executed command and its +// observed outcome. The state field is an opaque token standing in for the +// workspace state-hash the exec package would compute in production. +type fixtureStep struct { + Cmd string `json:"cmd"` + Exit int `json:"exit"` + Err string `json:"err"` + State string `json:"state"` +} + +func loadSteps(t *testing.T, name string) []fixtureStep { + t.Helper() + f, err := os.Open(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var steps []fixtureStep + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + var s fixtureStep + if err := json.Unmarshal([]byte(line), &s); err != nil { + t.Fatalf("%s: %v", name, err) + } + steps = append(steps, s) + } + if err := sc.Err(); err != nil { + t.Fatal(err) + } + return steps +} + +// replay drives a fresh detector through a fixture the way the exec wrapper +// does: Observe before each command, and Record its outcome only if it was +// allowed (a denied command is never run, so never recorded). +func replay(steps []fixtureStep) []Decision { + d := New(DefaultConfig()) + decs := make([]Decision, 0, len(steps)) + for _, s := range steps { + dec := d.Observe(s.Cmd) + decs = append(decs, dec) + if dec.Outcome == OutcomeDeny { + continue + } + d.Record(s.Cmd, s.Exit, s.Err, s.State) + } + return decs +} + +func anyAction(decs []Decision, a Action) bool { + for _, d := range decs { + if d.Action == a { + return true + } + } + return false +} + +func anyClass(decs []Decision, c Class) bool { + for _, d := range decs { + if d.Class == c { + return true + } + } + return false +} + +// TestStuckLoopsAreDenied: the three non-progressing shapes must each escalate +// to a deny — a repeated fixed point (identical error+state), a syntax-mutating +// loop (command changes, effect identical), and an A/B oscillation. +func TestStuckLoopsAreDenied(t *testing.T) { + for _, name := range []string{"always-fail.jsonl", "syntax-mutating.jsonl", "oscillation-ab.jsonl"} { + decs := replay(loadSteps(t, name)) + if !anyAction(decs, ActionDeny) { + t.Errorf("%s: expected the ladder to reach a deny, but it never did", name) + } + } +} + +// TestProgressIsNeverDenied: genuine progress, a near-miss that resolves, and +// hidden progress (error repeats but state keeps changing) must never be denied. +func TestProgressIsNeverDenied(t *testing.T) { + for _, name := range []string{"genuine-progress.jsonl", "near-miss.jsonl", "hidden-progress.jsonl"} { + decs := replay(loadSteps(t, name)) + if anyAction(decs, ActionDeny) { + t.Errorf("%s: expected no deny, but the ladder reached one (false positive)", name) + } + } +} + +// TestHiddenProgressIsNotStuck is the false-positive-mechanism test: an +// error-signature-only detector would deny this loop (the error text never +// changes). Because settle keys recurrence on the JOINT (error, state), the +// changing state keeps it out of the stuck class entirely. +func TestHiddenProgressIsNotStuck(t *testing.T) { + decs := replay(loadSteps(t, "hidden-progress.jsonl")) + if anyClass(decs, ClassStuck) { + t.Error("hidden-progress: joint (err,state) observation should prevent a stuck classification") + } +} + +// TestDeterminism: identical inputs must produce byte-identical decisions. +func TestDeterminism(t *testing.T) { + steps := loadSteps(t, "oscillation-ab.jsonl") + a := replay(steps) + b := replay(steps) + if !reflect.DeepEqual(a, b) { + t.Error("detector output is not deterministic across identical runs") + } +} diff --git a/labs/18-settle/settle/detector/ladder.go b/labs/18-settle/settle/detector/ladder.go new file mode 100644 index 000000000..f2b2ae703 --- /dev/null +++ b/labs/18-settle/settle/detector/ladder.go @@ -0,0 +1,84 @@ +package detector + +// Control outcomes. These match Pitot's schema.Outcome* values so a bridge +// adapter is a straight translation with no remapping. +const ( + OutcomeAllow = "allow" + OutcomeDeny = "deny" +) + +// Class is settle's estimate of the loop's macro-state. +type Class string + +const ( + ClassProgressing Class = "progressing" + ClassUnsettled Class = "unsettled" + ClassStuck Class = "stuck" +) + +// Action is the rung of the minimum-force intervention ladder. Force rises with +// belief; every rung is nonblocking — even Deny withholds only the one futile +// command and leaves read-only moves open. +type Action string + +const ( + ActionAllow Action = "allow" + ActionInform Action = "inform" + ActionRedirect Action = "redirect" + ActionDeny Action = "deny" +) + +// Decision is the controller's answer for one pending command. Outcome maps 1:1 +// to Pitot's control.response. +type Decision struct { + Outcome string // OutcomeAllow | OutcomeDeny + Action Action // the finer-grained rung + Class Class // estimated macro-state + Belief float64 // P(stuck | observations), in [0,1] + Signals []string // which sensors fired + Message string // guidance surfaced to the agent (empty when silent) +} + +// decide maps a belief to a ladder rung. Deny is reserved for the top of the +// ladder AND a confirmed, unmarked cycle whose dwell has passed the calibrated +// point of no return — the one place the high-cost intervention is paid. This +// hard dwell gate is why a near-miss (a couple of repeats that then resolve) is +// never denied even if leading sensors briefly push the belief up. +func decide(cfg Config, rec Recurrence, belief float64, signals []string) Decision { + d := Decision{ + Outcome: OutcomeAllow, + Action: ActionAllow, + Class: ClassProgressing, + Belief: belief, + Signals: signals, + } + + canDeny := rec.Period >= 1 && rec.Reps >= 2 && !rec.Marked && rec.Dwell >= cfg.PointOfNoReturn + + switch { + case belief >= cfg.HighThreshold && canDeny: + d.Outcome = OutcomeDeny + d.Action = ActionDeny + d.Class = ClassStuck + d.Message = "settle: blocked a provably non-progressing command (" + classifyCycle(rec) + + "). Execution is paused for THIS command only; read-only inspection remains available. " + + "Diagnose the mechanism producing this result, then issue a different command." + case belief >= cfg.MedThreshold: + d.Action = ActionRedirect + d.Class = ClassUnsettled + d.Message = "settle: no forward motion across repeated attempts. Stop mutating the command; " + + "inspect the mechanism producing this result (config, environment, upstream definition) before the next attempt." + case belief >= cfg.LowThreshold: + d.Action = ActionInform + d.Class = ClassUnsettled + d.Message = "settle: this attempt is repeating with little observable change. Retrying variants is unlikely to clear it." + } + return d +} + +func classifyCycle(rec Recurrence) string { + if rec.Period == 1 { + return "identical error and unchanged workspace state repeated" + } + return "oscillating between a small set of repeated states" +} diff --git a/labs/18-settle/settle/detector/observation.go b/labs/18-settle/settle/detector/observation.go new file mode 100644 index 000000000..0128b3f9c --- /dev/null +++ b/labs/18-settle/settle/detector/observation.go @@ -0,0 +1,25 @@ +package detector + +// Observation is settle's view of one executed command: the projection P(s). +// It is a letter of the observed alphabet. settle never sees the agent's true +// intent or any hidden progress — only these fields. This is exactly why the +// spec is unobservable and the false-positive rate is irreducible. +type Observation struct { + NormCommand string `json:"norm_command"` + ErrSignature string `json:"err_signature"` // "" when the command succeeded + StateHash string `json:"state_hash"` + ExitCode int `json:"exit_code"` +} + +// Marked reports whether this observation reached a marked (success) symbol of +// the marked language L_m(G). A marked symbol inside a cycle proves the loop is +// not stuck. +func (o Observation) Marked() bool { return o.ExitCode == 0 } + +// symbol is the effect-side identity used for recurrent-class detection: the +// joint (error signature, post-state). Two attempts with the same symbol +// produced the same observable effect — provably no forward motion between +// them. Using the JOINT (err, state), not the error alone, is what lets settle +// avoid denying a loop whose error text repeats while the workspace keeps +// changing (hidden progress the error-only view would miss). +func (o Observation) symbol() string { return o.ErrSignature + "\x1f" + o.StateHash } diff --git a/labs/18-settle/settle/detector/recurrence.go b/labs/18-settle/settle/detector/recurrence.go new file mode 100644 index 000000000..abf9bcf27 --- /dev/null +++ b/labs/18-settle/settle/detector/recurrence.go @@ -0,0 +1,63 @@ +package detector + +// Recurrence describes whether the tail of the observation history has entered a +// repeating class and whether that class contains a marked (success) symbol. +// +// This one detector subsumes the brief's separate sensors: Period 1 is the +// state-unchanged / semantic-error-repeat case (strongest, observes effect), +// Period > 1 is the oscillation case (A->B->A->B). The distinction between +// "stuck" and "still exploring" is exactly whether the observation trajectory +// has been absorbed into such a class or is still emitting new symbols. +type Recurrence struct { + Period int // 0 = no cycle; 1 = fixed point; >1 = oscillation + Reps int // consecutive repetitions of the period-block (>=2 to be a cycle) + Dwell int // observations consumed by the cycle = Period*Reps + Marked bool // a success symbol appears within the cycle window +} + +// detectRecurrence finds the smallest period whose block repeats at least twice +// at the tail of obs. Smallest period wins so AAAA is reported as a 1-cycle, not +// a 2-cycle. +func detectRecurrence(obs []Observation, maxPeriod int) Recurrence { + n := len(obs) + syms := make([]string, n) + for i, o := range obs { + syms[i] = o.symbol() + } + for p := 1; p <= maxPeriod; p++ { + if n < 2*p { + break + } + reps := 1 + for { + need := (reps + 1) * p + if n < need { + break + } + // The trailing window of length need is periodic with period p iff + // syms[i] == syms[i+p] for every i in [n-need, n-p). + ok := true + for i := n - need; i < n-p; i++ { + if syms[i] != syms[i+p] { + ok = false + break + } + } + if !ok { + break + } + reps++ + } + if reps >= 2 { + rec := Recurrence{Period: p, Reps: reps, Dwell: p * reps} + for i := n - rec.Dwell; i < n; i++ { + if obs[i].Marked() { + rec.Marked = true + break + } + } + return rec + } + } + return Recurrence{} +} diff --git a/labs/18-settle/settle/detector/sensors.go b/labs/18-settle/settle/detector/sensors.go new file mode 100644 index 000000000..2b78f8000 --- /dev/null +++ b/labs/18-settle/settle/detector/sensors.go @@ -0,0 +1,88 @@ +package detector + +import ( + "math" + "sort" +) + +// normLevenshtein returns the byte-level edit distance divided by the longer +// length, in [0,1]. Commands are effectively ASCII so byte distance is fine. +func normLevenshtein(a, b string) float64 { + if a == b { + return 0 + } + la, lb := len(a), len(b) + if la == 0 || lb == 0 { + return 1 + } + prev := make([]int, lb+1) + for j := 0; j <= lb; j++ { + prev[j] = j + } + for i := 1; i <= la; i++ { + cur := make([]int, lb+1) + cur[0] = i + for j := 1; j <= lb; j++ { + cost := 1 + if a[i-1] == b[j-1] { + cost = 0 + } + cur[j] = min3(prev[j]+1, cur[j-1]+1, prev[j-1]+cost) + } + prev = cur + } + m := la + if lb > m { + m = lb + } + return float64(prev[lb]) / float64(m) +} + +func min3(a, b, c int) int { + if b < a { + a = b + } + if c < a { + a = c + } + return a +} + +// editFloorVote fires when the newest command is a near-trivial mutation of the +// previous one — the agent is fiddling (x=1 -> x=2), not rethinking. A leading +// indicator: it predicts non-progress before the effect-side cycle confirms it. +func editFloorVote(cmds []string, floor float64) bool { + if len(cmds) < 2 { + return false + } + return normLevenshtein(cmds[len(cmds)-2], cmds[len(cmds)-1]) <= floor +} + +// entropyCollapseVote fires when the command distribution over the window has +// collapsed toward a single template — the agent has stopped generating new +// ideas. Also a leading indicator. Counts are summed in sorted order so the +// result is bit-for-bit deterministic regardless of Go's random map iteration. +func entropyCollapseVote(cmds []string, floor float64, minWindow int) bool { + if len(cmds) < minWindow { + return false + } + counts := map[string]int{} + for _, c := range cmds { + counts[c]++ + } + if len(counts) <= 1 { + return true + } + vals := make([]int, 0, len(counts)) + for _, c := range counts { + vals = append(vals, c) + } + sort.Ints(vals) + n := float64(len(cmds)) + h := 0.0 + for _, c := range vals { + p := float64(c) / n + h -= p * math.Log2(p) + } + return h < floor +} diff --git a/labs/18-settle/settle/detector/signature.go b/labs/18-settle/settle/detector/signature.go new file mode 100644 index 000000000..e685a6622 --- /dev/null +++ b/labs/18-settle/settle/detector/signature.go @@ -0,0 +1,61 @@ +package detector + +import ( + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" +) + +var ( + reVolatilePath = regexp.MustCompile(`/(tmp|var/folders)/[a-zA-Z0-9_/-]+`) + reWhitespace = regexp.MustCompile(`\s+`) + reHexAddr = regexp.MustCompile(`0x[0-9a-fA-F]+`) + reLineNo = regexp.MustCompile(`line \d+`) + reAnyPath = regexp.MustCompile(`/[a-zA-Z0-9_./-]+`) +) + +// NormalizeCommand groups retries of the same command by stripping volatile +// temp paths and collapsing whitespace. Ported unchanged from the Harbor +// governor's normalize_command so signatures stay comparable across the two +// implementations (this is now the single source of truth; the Harbor Python +// path is deprecated toward it). +func NormalizeCommand(cmd string) string { + if cmd == "" { + return "" + } + cmd = reVolatilePath.ReplaceAllString(cmd, "/tmp/...") + cmd = reWhitespace.ReplaceAllString(cmd, " ") + return strings.TrimSpace(cmd) +} + +// SemanticSignature canonicalizes error text — stripping memory addresses, line +// numbers, and paths — then hashes the last three non-empty lines (the core +// exception). Two failures with the same signature are the same semantic error +// even if their volatile tokens differ. Ported from lab 17's test_semantic.py / +// operatorstack_harbor.py. +// +// The returned core is retained so callers can log both the raw and canonical +// strings: over-stripping (distinct errors collapsing to one hash) is a +// false-positive source that is only diagnosable if the core is auditable. +func SemanticSignature(text string) (core, hash string) { + if strings.TrimSpace(text) == "" { + return "", "" + } + text = reHexAddr.ReplaceAllString(text, "0x...") + text = reLineNo.ReplaceAllString(text, "line ...") + text = reAnyPath.ReplaceAllString(text, "/...") + + var lines []string + for _, l := range strings.Split(text, "\n") { + if strings.TrimSpace(l) != "" { + lines = append(lines, l) + } + } + if len(lines) > 3 { + lines = lines[len(lines)-3:] + } + core = strings.Join(lines, "\n") + sum := sha256.Sum256([]byte(core)) + return core, hex.EncodeToString(sum[:]) +} diff --git a/labs/18-settle/settle/detector/signature_test.go b/labs/18-settle/settle/detector/signature_test.go new file mode 100644 index 000000000..26860a3d7 --- /dev/null +++ b/labs/18-settle/settle/detector/signature_test.go @@ -0,0 +1,58 @@ +package detector + +import "testing" + +// The semantic signature must collapse the same underlying error across volatile +// tokens (temp paths, line numbers, memory addresses) and must NOT collapse +// genuinely different errors. These pairs are ported from lab 17's +// test_semantic.py, which is the canonicalization contract the Harbor governor +// used. Over-stripping (distinct errors sharing a hash) is a false-positive +// source, so the negative case is as important as the positive ones. + +const ( + errTempPathsA = `Traceback (most recent call last): + File "/tmp/pytest-of-root/pytest-12/test_auth.py", line 45, in test_login + assert response.status_code == 200 +AssertionError: Expected 200, got 401` + + errTempPathsB = `Traceback (most recent call last): + File "/var/folders/8j/sfr9qqcj73j4p6nhwcfpr0th0000gn/T/tmp.azvk3Qr3Ky/test_auth.py", line 48, in test_login + assert response.status_code == 200 +AssertionError: Expected 200, got 401` + + errMemAddrA = `TypeError: Object of type is not JSON serializable + File "/app/serializers.py", line 112, in serialize_user` + + errMemAddrB = `TypeError: Object of type is not JSON serializable + File "/app/serializers.py", line 112, in serialize_user` +) + +func TestSemanticSignatureCollapsesVolatileTokens(t *testing.T) { + cases := []struct{ name, a, b string }{ + {"temp-paths-and-line-numbers", errTempPathsA, errTempPathsB}, + {"memory-addresses", errMemAddrA, errMemAddrB}, + } + for _, c := range cases { + _, ha := SemanticSignature(c.a) + _, hb := SemanticSignature(c.b) + if ha != hb { + t.Errorf("%s: expected identical signatures despite volatile tokens\n a=%s\n b=%s", c.name, ha, hb) + } + } +} + +func TestSemanticSignatureSeparatesDifferentErrors(t *testing.T) { + _, ha := SemanticSignature("AssertionError: Expected 200, got 401") + _, hb := SemanticSignature("KeyError: 'user_id' not found in payload") + if ha == hb { + t.Fatal("distinct errors collapsed to a single signature (over-stripping)") + } +} + +func TestNormalizeCommandStripsVolatilePaths(t *testing.T) { + a := NormalizeCommand("cat /tmp/abc123/out.log | grep x") + b := NormalizeCommand("cat /var/folders/8j/xyz/out.log | grep x") + if a != b { + t.Errorf("expected volatile temp paths to normalize equal:\n a=%q\n b=%q", a, b) + } +} diff --git a/labs/18-settle/settle/detector/testdata/always-fail.jsonl b/labs/18-settle/settle/detector/testdata/always-fail.jsonl new file mode 100644 index 000000000..0fc115146 --- /dev/null +++ b/labs/18-settle/settle/detector/testdata/always-fail.jsonl @@ -0,0 +1,7 @@ +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} +{"cmd":"pytest tests/test_auth.py","exit":1,"err":"AssertionError: Expected 200, got 401","state":"s0"} diff --git a/labs/18-settle/settle/detector/testdata/genuine-progress.jsonl b/labs/18-settle/settle/detector/testdata/genuine-progress.jsonl new file mode 100644 index 000000000..895873e63 --- /dev/null +++ b/labs/18-settle/settle/detector/testdata/genuine-progress.jsonl @@ -0,0 +1,6 @@ +{"cmd":"go build ./cmd/server","exit":1,"err":"undefined: handleLogin","state":"p1"} +{"cmd":"go build ./cmd/server","exit":0,"err":"","state":"p2"} +{"cmd":"go test ./auth","exit":1,"err":"FAIL auth TestSession session expired","state":"p3"} +{"cmd":"golangci-lint run","exit":1,"err":"ineffassign: ineffectual assignment to err","state":"p4"} +{"cmd":"go test ./auth","exit":0,"err":"","state":"p5"} +{"cmd":"git commit -m wire-session","exit":0,"err":"","state":"p6"} diff --git a/labs/18-settle/settle/detector/testdata/hidden-progress.jsonl b/labs/18-settle/settle/detector/testdata/hidden-progress.jsonl new file mode 100644 index 000000000..904b17c71 --- /dev/null +++ b/labs/18-settle/settle/detector/testdata/hidden-progress.jsonl @@ -0,0 +1,7 @@ +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h1"} +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h2"} +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h3"} +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h4"} +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h5"} +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h6"} +{"cmd":"pytest tests/","exit":1,"err":"AssertionError: 3 tests failed","state":"h7"} diff --git a/labs/18-settle/settle/detector/testdata/near-miss.jsonl b/labs/18-settle/settle/detector/testdata/near-miss.jsonl new file mode 100644 index 000000000..0e30c2f2d --- /dev/null +++ b/labs/18-settle/settle/detector/testdata/near-miss.jsonl @@ -0,0 +1,3 @@ +{"cmd":"pytest tests/test_db.py","exit":1,"err":"sqlite3.OperationalError: database is locked","state":"s0"} +{"cmd":"pytest tests/test_db.py","exit":1,"err":"sqlite3.OperationalError: database is locked","state":"s0"} +{"cmd":"pytest tests/test_db.py","exit":0,"err":"","state":"s1"} diff --git a/labs/18-settle/settle/detector/testdata/oscillation-ab.jsonl b/labs/18-settle/settle/detector/testdata/oscillation-ab.jsonl new file mode 100644 index 000000000..55a892806 --- /dev/null +++ b/labs/18-settle/settle/detector/testdata/oscillation-ab.jsonl @@ -0,0 +1,10 @@ +{"cmd":"make test","exit":1,"err":"error: undefined symbol foo","state":"sA"} +{"cmd":"make test","exit":1,"err":"error: type mismatch bar","state":"sB"} +{"cmd":"make test","exit":1,"err":"error: undefined symbol foo","state":"sA"} +{"cmd":"make test","exit":1,"err":"error: type mismatch bar","state":"sB"} +{"cmd":"make test","exit":1,"err":"error: undefined symbol foo","state":"sA"} +{"cmd":"make test","exit":1,"err":"error: type mismatch bar","state":"sB"} +{"cmd":"make test","exit":1,"err":"error: undefined symbol foo","state":"sA"} +{"cmd":"make test","exit":1,"err":"error: type mismatch bar","state":"sB"} +{"cmd":"make test","exit":1,"err":"error: undefined symbol foo","state":"sA"} +{"cmd":"make test","exit":1,"err":"error: type mismatch bar","state":"sB"} diff --git a/labs/18-settle/settle/detector/testdata/syntax-mutating.jsonl b/labs/18-settle/settle/detector/testdata/syntax-mutating.jsonl new file mode 100644 index 000000000..ae767f1c3 --- /dev/null +++ b/labs/18-settle/settle/detector/testdata/syntax-mutating.jsonl @@ -0,0 +1,7 @@ +{"cmd":"python run.py --retries=1","exit":1,"err":"KeyError: 'user_id'","state":"s0"} +{"cmd":"python run.py --retries=2","exit":1,"err":"KeyError: 'user_id'","state":"s0"} +{"cmd":"python run.py --retries=3","exit":1,"err":"KeyError: 'user_id'","state":"s0"} +{"cmd":"python run.py --retries=4","exit":1,"err":"KeyError: 'user_id'","state":"s0"} +{"cmd":"python run.py --retries=5","exit":1,"err":"KeyError: 'user_id'","state":"s0"} +{"cmd":"python run.py --retries=6","exit":1,"err":"KeyError: 'user_id'","state":"s0"} +{"cmd":"python run.py --retries=7","exit":1,"err":"KeyError: 'user_id'","state":"s0"} diff --git a/labs/18-settle/settle/doc.go b/labs/18-settle/settle/doc.go new file mode 100644 index 000000000..476b3711a --- /dev/null +++ b/labs/18-settle/settle/doc.go @@ -0,0 +1,31 @@ +// Package settle is the module root for the settle non-progress controller. +// +// settle is a zero-inference, partial-observation supervisory controller that +// wraps any agent's command execution. It watches the stream of +// (command, outcome, workspace-state) an agent produces and, when cheap +// deterministic sensors agree the loop has stopped making observable progress, +// injects a control signal whose force is proportional to its confidence. It +// runs no model inference: all detection is hashing, counting, diffing, and +// regex. +// +// Theory. settle is a Ramadge-Wonham supervisor operating in the regime the +// house control-theory doc (repo-root supervisory-control.md) describes for +// Boatstack, but where the specification is *unobservable*. Boatstack's spec is +// co-designed to be observable, so it has no false-positive rate. settle's spec +// ("do not remain in a non-progressing cycle") is provably not observable: a +// futile loop and slow-but-genuine progress can share the same observation +// projection P(s). By the Existence theorem no exact supervisor exists, so +// settle runs an approximate belief-state supervisor and pays for the +// unobservability with an irreducible false-positive rate. See docs/01-theory.md. +// +// Module layout: +// +// detector/ pure, deterministic classifier core (no I/O, no os/exec) +// exec/ thin host wrapper: runs a command, hashes workspace state +// cmd/settle/ the compiled binary (exec, run, doctor subcommands) +// +// Architectural invariant: detector/ must never import exec/. The classifier is +// innocent of execution; it decides only from the observations handed to it. +// The invariant is enforced at build time by scripts/check_import_boundary.sh +// (mirroring Pitot's sensor-innocence boundary). +package settle diff --git a/labs/18-settle/settle/exec/run.go b/labs/18-settle/settle/exec/run.go new file mode 100644 index 000000000..9e8dded07 --- /dev/null +++ b/labs/18-settle/settle/exec/run.go @@ -0,0 +1,46 @@ +// Package exec is settle's thin host-execution wrapper: the only package that +// touches the OS. It runs a command, captures the outcome, and hashes workspace +// state. The detector package must never import this one. +package exec + +import ( + "bytes" + "errors" + "io" + "os" + "os/exec" +) + +// Result is the observed outcome of running one command. +type Result struct { + ExitCode int + Stderr string +} + +// Run executes argv in dir, streaming stdout/stderr to the terminal while also +// capturing stderr for the semantic-signature sensor. A nonzero exit is data, +// not an error: it is returned in Result with a nil error. A nil error with a +// nonzero ExitCode is the normal failing-command case. +func Run(dir string, argv []string) (Result, error) { + if len(argv) == 0 { + return Result{}, errors.New("settle exec: empty command") + } + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + var errBuf bytes.Buffer + cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf) + + err := cmd.Run() + res := Result{Stderr: errBuf.String()} + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + res.ExitCode = ee.ExitCode() + return res, nil + } + return res, err // could not start the process at all + } + return res, nil +} diff --git a/labs/18-settle/settle/exec/state.go b/labs/18-settle/settle/exec/state.go new file mode 100644 index 000000000..13e255934 --- /dev/null +++ b/labs/18-settle/settle/exec/state.go @@ -0,0 +1,29 @@ +package exec + +import ( + "crypto/sha256" + "encoding/hex" + "os/exec" +) + +// WorkspaceState returns a cheap, deterministic hash of the git working-tree +// state (porcelain status + unstaged diff). This is settle's post-state sensor: +// it is what distinguishes "same error, workspace changed" (progress) from +// "same error, workspace identical" (a fixed point). +// +// The bool reports whether the state could be measured. On any failure (not a +// git repo, git absent) the caller treats the state as unknown and fails OPEN — +// settle is an optimizer, not a safety gate, so a blind sensor must never block +// a healthy agent. This inverts Boatstack's fail-closed corollary by design. +func WorkspaceState(dir string) (string, bool) { + status, err := exec.Command("git", "-C", dir, "status", "--porcelain").Output() + if err != nil { + return "", false + } + diff, err := exec.Command("git", "-C", dir, "diff").Output() + if err != nil { + return "", false + } + sum := sha256.Sum256(append(status, diff...)) + return hex.EncodeToString(sum[:]), true +} diff --git a/labs/18-settle/settle/go.mod b/labs/18-settle/settle/go.mod new file mode 100644 index 000000000..102359e84 --- /dev/null +++ b/labs/18-settle/settle/go.mod @@ -0,0 +1,3 @@ +module github.com/operatorstack/settle + +go 1.26