diff --git a/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retro-derive-proposals.md b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retro-derive-proposals.md new file mode 100644 index 000000000..72195cb51 --- /dev/null +++ b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retro-derive-proposals.md @@ -0,0 +1,5 @@ +### `retro derive` turns your repeated instructions into reviewable proposals + +The new `retro derive` command reads the transcript files you name — Claude Code sessions, plain-text logs, or a neutral event format — finds the instructions you keep giving across sessions, and classifies each one as a missing observation, verb, setpoint, or guard, with a suggested typed promotion. An instruction the classifier cannot place is still shown, marked unclassified, and generates no proposal. + +The command only proposes: it writes no file, changes no state, and runs nothing, and it reads only the transcripts you explicitly pass — Boatstack never scans for transcripts on its own. Promote a proposal by hand through the normal reviewed delivery flow. The idea behind it: an instruction you keep repeating is evidence your system is missing a typed control, and the fix is to add that control — not to save the prompt. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/SKILL.md b/labs/12-product-engineering-loop/product-engineering-loop/SKILL.md index 9c4ef693a..a6306c085 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/SKILL.md +++ b/labs/12-product-engineering-loop/product-engineering-loop/SKILL.md @@ -254,6 +254,8 @@ This is a two-slice ZCA projection: the reviewer brief minimizes review effort, Read [failure-moves.md](references/failure-moves.md) before proposing a loop change. +For a retro over past sessions, run the read-only `.product-loop/bin/boatstack-helper retro derive --input [--input ...]`. It detects operator instructions that recur across sessions and classifies each as a missing observation, verb, setpoint, or guard, with a suggested typed promotion. It reads only the transcript files the user names, works fully offline, and writes nothing. A recurring instruction is evidence of a missing typed control — promote it by hand through the normal reviewed delivery flow; never turn it into a saved prompt, and never apply a proposal automatically. + 1. Classify the observed failure below the surface symptom. 2. State a mechanism and the exact failure population the move targets. 3. Estimate cost, risk, and possible regressions. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/coverage_conformance_test.go b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/coverage_conformance_test.go index 325776591..cfce070a0 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/coverage_conformance_test.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/coverage_conformance_test.go @@ -82,6 +82,10 @@ var nonDeliveryVerbs = map[string]bool{ "workspace-sync": true, // Flow layer itself is read-only navigation over the machine, not a transition. "flow": true, + // Retro derivation reads operator-supplied transcripts and proposes typed + // promotions; it mutates nothing, so it registers no delivery transition. + // control-law: retro-proposes-never-enforces + "retro": true, } // dispatchVerbs parses main.go and returns the set of command verbs the run() diff --git a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go index cd1e01419..076e1f32c 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go @@ -1439,7 +1439,7 @@ func workspaceSyncCommand(arguments []string) int { func run() int { if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: boatstack-helper ") + fmt.Fprintln(os.Stderr, "usage: boatstack-helper ") return 2 } switch os.Args[1] { @@ -1553,6 +1553,8 @@ func run() int { return migrateConfigCommand(os.Args[2:]) case "flow": return flowCommand(os.Args[2:]) + case "retro": + return retroCommand(os.Args[2:]) case "version": fmt.Printf("Boatstack %s (%s)\n", boatstack.Version, boatstack.SourceCommit) return 0 diff --git a/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/retro.go b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/retro.go new file mode 100644 index 000000000..836367464 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/retro.go @@ -0,0 +1,66 @@ +package main + +import ( + "flag" + "fmt" + "os" + + boatstack "github.com/operatorstack/boatstack/boatstack" +) + +// retroCommand is the derive-only entry point for transcript mining. The CLI +// boundary owns the ONLY I/O in the pipeline: it reads the operator-supplied +// paths and prints the report to stdout. Below this boundary the derivation +// is capability-free (no filesystem, network, subprocess, or clock), and +// nothing anywhere in the pipeline writes, mutates state, or runs a command. +// control-law: retro-proposes-never-enforces +func retroCommand(arguments []string) int { + if len(arguments) == 0 || arguments[0] != "derive" { + fmt.Fprintln(os.Stderr, "usage: boatstack-helper retro derive --input [--input ...] [--format events|claudecode|plaintext] [--json]") + return 2 + } + flags := flag.NewFlagSet("retro derive", flag.ContinueOnError) + var inputs stringList + flags.Var(&inputs, "input", "transcript file to mine (repeatable)") + format := flags.String("format", "", "transcript format: events, claudecode, or plaintext (default: sniff per file)") + jsonOutput := flags.Bool("json", false, "print the structured derivation report") + if err := flags.Parse(arguments[1:]); err != nil { + return 2 + } + inputs = append(inputs, flags.Args()...) + if len(inputs) == 0 { + fmt.Fprintln(os.Stderr, "retro derive requires at least one --input transcript; Boatstack never scans for transcripts on its own") + return 2 + } + loaded := make([]boatstack.RetroInput, 0, len(inputs)) + for _, path := range inputs { + content, err := os.ReadFile(path) + if err != nil { + return fail(err) + } + loaded = append(loaded, boatstack.RetroInput{Name: path, Content: content}) + } + report, err := boatstack.RetroDerive(*format, loaded) + if err != nil { + return fail(err) + } + if *jsonOutput { + value, marshalErr := boatstack.MarshalJSON(report) + if marshalErr != nil { + return fail(marshalErr) + } + fmt.Print(string(value)) + } else { + fmt.Print(boatstack.FormatRetroReport(report)) + } + return 0 +} + +// stringList is a repeatable string flag. +type stringList []string + +func (s *stringList) String() string { return fmt.Sprint([]string(*s)) } +func (s *stringList) Set(value string) error { + *s = append(*s, value) + return nil +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/classify.go b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/classify.go new file mode 100644 index 000000000..37a5984ad --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/classify.go @@ -0,0 +1,94 @@ +package retromine + +import "strings" + +// Gap classification names WHICH typed construct a recurring instruction is +// compensating for. The four gap types are the four ways a controller can be +// missing a term: +// +// missing_observation — the operator keeps asking what the system could show +// missing_verb — the operator keeps describing an action to take +// missing_setpoint — the operator keeps restating a goal or condition to +// pursue ("until", "every time", "at least") +// missing_guard — the operator keeps warning what must not happen +// +// The classifier is a deterministic keyword lexicon over the normalized +// instruction, with fixed precedence guard > setpoint > observation > verb: +// a guard misclassified as a verb could become an action proposal, so the +// constraining readings win. Anything the lexicon cannot place lands in +// unclassified, which is REPORTED but never generates a proposal. +// control-law: retro-proposes-never-enforces +const ( + GapObservation = "missing_observation" + GapVerb = "missing_verb" + GapSetpoint = "missing_setpoint" + GapGuard = "missing_guard" + GapUnclassified = "unclassified" +) + +// The lexicons match either whole tokens or normalized phrases. Normalization +// has already lowered the text and stripped punctuation ("don't" → "don t"). +var ( + guardPhrases = []string{"don t", "do not", "make sure not", "must not", "never", "only if", "unless", "be careful", "avoid", "without asking", "instead of"} + guardTokens = []string{"dont", "stop"} + + setpointPhrases = []string{"until", "at least", "at most", "within", "every time", "each time", "whenever", "keep doing", "always", "from now on", "before you finish", "when green", "when it passes"} + + observationPhrases = []string{"check the", "check whether", "check if", "what is the", "what s the", "show me", "look at", "status of", "is it", "did it", "how is", "where is", "monitor"} + + verbTokens = []string{"run", "merge", "publish", "push", "rerun", "retry", "open", "record", "fix", "update", "deploy", "rebase", "commit", "create", "install", "sync", "clean", "make"} +) + +// ClassifyGap places one normalized instruction into a gap type. +func ClassifyGap(normalized string) string { + padded := " " + normalized + " " + containsPhrase := func(phrases []string) bool { + for _, phrase := range phrases { + if strings.Contains(padded, " "+phrase+" ") { + return true + } + } + return false + } + tokens := map[string]bool{} + for _, token := range strings.Fields(normalized) { + tokens[token] = true + } + containsToken := func(list []string) bool { + for _, token := range list { + if tokens[token] { + return true + } + } + return false + } + switch { + case containsPhrase(guardPhrases) || containsToken(guardTokens): + return GapGuard + case containsPhrase(setpointPhrases): + return GapSetpoint + case containsPhrase(observationPhrases): + return GapObservation + case containsToken(verbTokens): + return GapVerb + default: + return GapUnclassified + } +} + +// SuggestedShape names the typed construct to add for a gap type — prose +// pointing a human at the right kind of promotion, never a diff. +func SuggestedShape(gapType string) string { + switch gapType { + case GapObservation: + return "Add a typed observation: a read-only status or frontier field that answers this without being asked." + case GapVerb: + return "Add or prescribe a typed verb: a deterministic command the flow names at the right state." + case GapSetpoint: + return "Add a typed setpoint: a persisted goal or condition (like delivery.terminal) the flow pursues so this stops being restated." + case GapGuard: + return "Add a typed guard: an enforced precondition or denial (a gate or policy) instead of a remembered warning." + default: + return "" + } +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/report.go b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/report.go new file mode 100644 index 000000000..63053404d --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/report.go @@ -0,0 +1,57 @@ +package retromine + +// The report is the miner's entire output surface: typed proposals for the +// classified recurrences, and the unclassified recurrences named so nothing +// is silently dropped. It is data for a human to review — the derivation +// proposes, and promotion into a real state, verb, setpoint, or guard is +// always a reviewed change made by hand. +// control-law: retro-proposes-never-enforces +const ReportSchemaVersion = 1 + +// Proposal is one recurring instruction promoted to a typed suggestion. +type Proposal struct { + GapType string `json:"gap_type"` + Occurrences int `json:"occurrences"` + Sessions []string `json:"sessions"` + Exemplar string `json:"exemplar"` + SuggestedShape string `json:"suggested_shape"` + Evidence []EventRef `json:"evidence"` +} + +// Report is the full derivation result over one set of transcripts. +type Report struct { + SchemaVersion int `json:"schema_version"` + EventsScanned int `json:"events_scanned"` + OperatorEvents int `json:"operator_events"` + Proposals []Proposal `json:"proposals"` + // Unclassified recurrences are surfaced — a recurrence the lexicon cannot + // place is still steady-state error worth a human look — but they never + // become proposals (fail-closed). + Unclassified []Cluster `json:"unclassified,omitempty"` +} + +// BuildReport mines the events and classifies every recurrence. +func BuildReport(events []Event) Report { + report := Report{SchemaVersion: ReportSchemaVersion, EventsScanned: len(events), Proposals: []Proposal{}} + for _, event := range events { + if event.Role == RoleOperator { + report.OperatorEvents++ + } + } + for _, cluster := range DetectRecurrence(events) { + gapType := ClassifyGap(cluster.Normalized) + if gapType == GapUnclassified { + report.Unclassified = append(report.Unclassified, cluster) + continue + } + report.Proposals = append(report.Proposals, Proposal{ + GapType: gapType, + Occurrences: cluster.Occurrences, + Sessions: cluster.Sessions, + Exemplar: cluster.Exemplar, + SuggestedShape: SuggestedShape(gapType), + Evidence: cluster.Evidence, + }) + } + return report +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/retro.go b/labs/12-product-engineering-loop/product-engineering-loop/retro.go new file mode 100644 index 000000000..8309766ea --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/retro.go @@ -0,0 +1,61 @@ +package boatstack + +import ( + "fmt" + "strings" + + "github.com/operatorstack/boatstack/boatstack/internal/retromine" +) + +// RetroInput is one transcript handed to the retro derivation: a name (for +// evidence references and per-file session identity) and its raw content. +// The derivation layer takes bytes, never paths — every capability the miner +// lacks (filesystem, network, subprocess, clock) stays lacking here; only +// the CLI boundary reads files, from operator-supplied paths only. +// control-law: retro-derivation-is-offline-and-deterministic +type RetroInput struct { + Name string + Content []byte +} + +// RetroDerive parses every input with the named adapter format ("" sniffs +// per file: events | claudecode | plaintext) and mines the combined events +// for recurring operator instructions, classified into typed-gap proposals. +// It proposes only: no file is written, no state is touched, no command is +// run, and nothing is enforced — promotion is always a reviewed change made +// by hand. control-law: retro-proposes-never-enforces +func RetroDerive(format string, inputs []RetroInput) (retromine.Report, error) { + events := []retromine.Event{} + for _, input := range inputs { + parsed, err := retromine.ParseTranscript(format, input.Name, input.Content) + if err != nil { + return retromine.Report{}, err + } + events = append(events, parsed...) + } + return retromine.BuildReport(events), nil +} + +// FormatRetroReport renders the derivation for a human reviewer. +func FormatRetroReport(report retromine.Report) string { + var b strings.Builder + fmt.Fprintf(&b, "Retro derivation: %d event(s) scanned, %d from the operator.\n", + report.EventsScanned, report.OperatorEvents) + if len(report.Proposals) == 0 && len(report.Unclassified) == 0 { + b.WriteString("No recurring operator instruction found across sessions. Nothing to promote.\n") + return b.String() + } + for i, proposal := range report.Proposals { + fmt.Fprintf(&b, "\n%d. [%s] seen %d time(s) across %d session(s)\n", i+1, + proposal.GapType, proposal.Occurrences, len(proposal.Sessions)) + fmt.Fprintf(&b, " Instruction: %q\n", proposal.Exemplar) + fmt.Fprintf(&b, " Promote it: %s\n", proposal.SuggestedShape) + } + for _, cluster := range report.Unclassified { + fmt.Fprintf(&b, "\n?. [unclassified] seen %d time(s) across %d session(s): %q\n", + cluster.Occurrences, len(cluster.Sessions), cluster.Exemplar) + b.WriteString(" Recurs, but no gap type matched; review it by hand. No proposal is generated.\n") + } + b.WriteString("\nDerivation proposes; it never enforces. Promote a proposal by hand through the normal reviewed delivery flow.\n") + return b.String() +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/retro_conformance_test.go b/labs/12-product-engineering-loop/product-engineering-loop/retro_conformance_test.go new file mode 100644 index 000000000..8979e60ad --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/retro_conformance_test.go @@ -0,0 +1,140 @@ +package boatstack + +// control-law: retro-proposes-never-enforces +// +// `retro derive` closes the loop the whole program serves: a recurring +// operator instruction is steady-state error, and the remedy is a TYPED +// promotion — an observation, verb, setpoint, or guard — never a saved +// prompt and never an automatic change. The derivation therefore only ever +// produces a report: it writes no file, mutates no state, runs no command, +// and an unclassifiable recurrence is surfaced without a proposal +// (fail-closed). Below the CLI's read-only file loading, the pipeline is +// capability-free (pinned structurally in the retromine conformance suite). +// +// Test classes: positive (each gap type classifies from planted recurring +// phrasing, with a suggested typed shape), negative (an unmatched recurrence +// lands in unclassified with zero proposals), bypass (derivation leaves the +// filesystem byte-identical), failure-state (empty input → empty report; +// a malformed transcript is a typed error, not a partial report). + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func neutralTranscript(instruction string) []byte { + var b strings.Builder + for _, session := range []string{"s1", "s2", "s3"} { + fmt.Fprintf(&b, `{"session_id":%q,"role":"operator","text":%q}`+"\n", session, instruction) + fmt.Fprintf(&b, `{"session_id":%q,"role":"agent","text":"done"}`+"\n", session) + } + return []byte(b.String()) +} + +// Positive: each gap type classifies from its phrasing and carries a +// suggested typed shape. +func TestRetroDeriveClassifiesEachGapType(t *testing.T) { + for _, test := range []struct { + instruction string + wantGap string + }{ + {"never force push to the main branch", "missing_guard"}, + {"watch the checks until every one passes then merge", "missing_setpoint"}, + {"check the status of the deployment pipeline", "missing_observation"}, + {"run the full test suite again please", "missing_verb"}, + } { + t.Run(test.wantGap, func(t *testing.T) { + report, err := RetroDerive("events", []RetroInput{{Name: "t.jsonl", Content: neutralTranscript(test.instruction)}}) + if err != nil { + t.Fatal(err) + } + if len(report.Proposals) != 1 { + t.Fatalf("proposals = %#v, want exactly one", report.Proposals) + } + proposal := report.Proposals[0] + if proposal.GapType != test.wantGap { + t.Fatalf("gap = %q, want %q", proposal.GapType, test.wantGap) + } + if proposal.Occurrences != 3 || len(proposal.Sessions) != 3 { + t.Fatalf("unexpected recurrence evidence: %#v", proposal) + } + if proposal.SuggestedShape == "" { + t.Fatal("proposal carries no suggested typed shape") + } + rendered := FormatRetroReport(report) + if !strings.Contains(rendered, test.wantGap) || !strings.Contains(rendered, "never enforces") { + t.Fatalf("rendering incomplete:\n%s", rendered) + } + }) + } +} + +// Negative: a recurrence the lexicon cannot place is surfaced as +// unclassified and generates zero proposals. +func TestUnclassifiedRecurrenceGeneratesNoProposal(t *testing.T) { + report, err := RetroDerive("events", []RetroInput{{Name: "t.jsonl", Content: neutralTranscript("the quarterly numbers look pretty good overall")}}) + if err != nil { + t.Fatal(err) + } + if len(report.Proposals) != 0 { + t.Fatalf("unclassified recurrence produced proposals: %#v", report.Proposals) + } + if len(report.Unclassified) != 1 { + t.Fatalf("unclassified recurrence not surfaced: %#v", report) + } + if rendered := FormatRetroReport(report); !strings.Contains(rendered, "No proposal is generated") { + t.Fatalf("unclassified recurrence not explained:\n%s", rendered) + } +} + +// Bypass: derivation leaves the filesystem byte-identical — it consumes +// bytes and produces a report, nothing else. +func TestRetroDeriveWritesNothing(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "transcript.jsonl") + if err := os.WriteFile(path, neutralTranscript("never force push to the main branch"), 0o644); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if _, err := RetroDerive("", []RetroInput{{Name: path, Content: content}}); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("derivation changed the directory: %v", entries) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(after) != string(content) { + t.Fatal("derivation modified its input") + } +} + +// Failure-state: empty input yields an empty report; a malformed transcript +// is a typed error, never a partial report. +func TestRetroDeriveFailureStates(t *testing.T) { + report, err := RetroDerive("events", nil) + if err != nil { + t.Fatal(err) + } + if report.EventsScanned != 0 || len(report.Proposals) != 0 { + t.Fatalf("empty input produced content: %#v", report) + } + if rendered := FormatRetroReport(report); !strings.Contains(rendered, "Nothing to promote") { + t.Fatalf("empty report not explained:\n%s", rendered) + } + if _, err := RetroDerive("events", []RetroInput{{Name: "bad.jsonl", Content: []byte("not json\n")}}); err == nil { + t.Fatal("malformed transcript accepted") + } +}