feat(boatstack): retro derive — recurring instructions become typed proposals

labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retro-derive-proposals.md

@@ -0,0 +1,5 @@
+### `retro derive` turns your repeated instructions into reviewable proposals
+
+The new `retro derive` command reads the transcript files you name — Claude Code sessions, plain-text logs, or a neutral event format — finds the instructions you keep giving across sessions, and classifies each one as a missing observation, verb, setpoint, or guard, with a suggested typed promotion. An instruction the classifier cannot place is still shown, marked unclassified, and generates no proposal.
+
+The command only proposes: it writes no file, changes no state, and runs nothing, and it reads only the transcripts you explicitly pass — Boatstack never scans for transcripts on its own. Promote a proposal by hand through the normal reviewed delivery flow. The idea behind it: an instruction you keep repeating is evidence your system is missing a typed control, and the fix is to add that control — not to save the prompt.

labs/12-product-engineering-loop/product-engineering-loop/SKILL.md

@@ -254,6 +254,8 @@ This is a two-slice ZCA projection: the reviewer brief minimizes review effort,
 
 Read [failure-moves.md](references/failure-moves.md) before proposing a loop change.
 
+For a retro over past sessions, run the read-only `.product-loop/bin/boatstack-helper retro derive --input <transcript> [--input <transcript> ...]`. It detects operator instructions that recur across sessions and classifies each as a missing observation, verb, setpoint, or guard, with a suggested typed promotion. It reads only the transcript files the user names, works fully offline, and writes nothing. A recurring instruction is evidence of a missing typed control — promote it by hand through the normal reviewed delivery flow; never turn it into a saved prompt, and never apply a proposal automatically.
+
 1. Classify the observed failure below the surface symptom.
 2. State a mechanism and the exact failure population the move targets.
 3. Estimate cost, risk, and possible regressions.

labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/coverage_conformance_test.go

@@ -82,6 +82,10 @@ var nonDeliveryVerbs = map[string]bool{
 	"workspace-sync":        true,
 	// Flow layer itself is read-only navigation over the machine, not a transition.
 	"flow": true,
+	// Retro derivation reads operator-supplied transcripts and proposes typed
+	// promotions; it mutates nothing, so it registers no delivery transition.
+	// control-law: retro-proposes-never-enforces
+	"retro": true,
 }
 
 // dispatchVerbs parses main.go and returns the set of command verbs the run()

labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/main.go

@@ -1439,7 +1439,7 @@ func workspaceSyncCommand(arguments []string) int {
 
 func run() int {
 	if len(os.Args) < 2 {
-		fmt.Fprintln(os.Stderr, "usage: boatstack-helper <attach|detach|detached-status|context|activate|deactivate|init|update|check-update|repair-status|operation-status|prepare-update-pr|publish-update-pr|release-classify|next-patch|export|check-source-plan|planning-write|check-plan|record-approval|activate-plan|delivery-status|next-status|recovery-status|repair-state|mutation-status|undo|run-preflight|record-change|ignore-delivery|record-delivery-gate|record-pr-visual-evidence|capture-evidence|provision-capability|capability-register|record-pr-visual-publication|check-safety|migrate-config|safety-hook|ambient-safety-hook|diagnose-hook|render-denial|pr-context|check-pr|publish-pr|workspace-cut|workspace-cleanup|workspace-reap|workspace-status|workspace-sync|flow|doctor|version>")
+		fmt.Fprintln(os.Stderr, "usage: boatstack-helper <attach|detach|detached-status|context|activate|deactivate|init|update|check-update|repair-status|operation-status|prepare-update-pr|publish-update-pr|release-classify|next-patch|export|check-source-plan|planning-write|check-plan|record-approval|activate-plan|delivery-status|next-status|recovery-status|repair-state|mutation-status|undo|run-preflight|record-change|ignore-delivery|record-delivery-gate|record-pr-visual-evidence|capture-evidence|provision-capability|capability-register|record-pr-visual-publication|check-safety|migrate-config|safety-hook|ambient-safety-hook|diagnose-hook|render-denial|pr-context|check-pr|publish-pr|workspace-cut|workspace-cleanup|workspace-reap|workspace-status|workspace-sync|flow|retro|doctor|version>")
 		return 2
 	}
 	switch os.Args[1] {
@@ -1553,6 +1553,8 @@ func run() int {
 		return migrateConfigCommand(os.Args[2:])
 	case "flow":
 		return flowCommand(os.Args[2:])
+	case "retro":
+		return retroCommand(os.Args[2:])
 	case "version":
 		fmt.Printf("Boatstack %s (%s)\n", boatstack.Version, boatstack.SourceCommit)
 		return 0

labs/12-product-engineering-loop/product-engineering-loop/cmd/boatstack-helper/retro.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	boatstack "github.com/operatorstack/boatstack/boatstack"
+)
+
+// retroCommand is the derive-only entry point for transcript mining. The CLI
+// boundary owns the ONLY I/O in the pipeline: it reads the operator-supplied
+// paths and prints the report to stdout. Below this boundary the derivation
+// is capability-free (no filesystem, network, subprocess, or clock), and
+// nothing anywhere in the pipeline writes, mutates state, or runs a command.
+// control-law: retro-proposes-never-enforces
+func retroCommand(arguments []string) int {
+	if len(arguments) == 0 || arguments[0] != "derive" {
+		fmt.Fprintln(os.Stderr, "usage: boatstack-helper retro derive --input <transcript> [--input <transcript> ...] [--format events|claudecode|plaintext] [--json]")
+		return 2
+	}
+	flags := flag.NewFlagSet("retro derive", flag.ContinueOnError)
+	var inputs stringList
+	flags.Var(&inputs, "input", "transcript file to mine (repeatable)")
+	format := flags.String("format", "", "transcript format: events, claudecode, or plaintext (default: sniff per file)")
+	jsonOutput := flags.Bool("json", false, "print the structured derivation report")
+	if err := flags.Parse(arguments[1:]); err != nil {
+		return 2
+	}
+	inputs = append(inputs, flags.Args()...)
+	if len(inputs) == 0 {
+		fmt.Fprintln(os.Stderr, "retro derive requires at least one --input transcript; Boatstack never scans for transcripts on its own")
+		return 2
+	}
+	loaded := make([]boatstack.RetroInput, 0, len(inputs))
+	for _, path := range inputs {
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return fail(err)
+		}
+		loaded = append(loaded, boatstack.RetroInput{Name: path, Content: content})
+	}
+	report, err := boatstack.RetroDerive(*format, loaded)
+	if err != nil {
+		return fail(err)
+	}
+	if *jsonOutput {
+		value, marshalErr := boatstack.MarshalJSON(report)
+		if marshalErr != nil {
+			return fail(marshalErr)
+		}
+		fmt.Print(string(value))
+	} else {
+		fmt.Print(boatstack.FormatRetroReport(report))
+	}
+	return 0
+}
+
+// stringList is a repeatable string flag.
+type stringList []string
+
+func (s *stringList) String() string { return fmt.Sprint([]string(*s)) }
+func (s *stringList) Set(value string) error {
+	*s = append(*s, value)
+	return nil
+}

labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/classify.go

@@ -0,0 +1,94 @@
+package retromine
+
+import "strings"
+
+// Gap classification names WHICH typed construct a recurring instruction is
+// compensating for. The four gap types are the four ways a controller can be
+// missing a term:
+//
+//	missing_observation — the operator keeps asking what the system could show
+//	missing_verb        — the operator keeps describing an action to take
+//	missing_setpoint    — the operator keeps restating a goal or condition to
+//	                      pursue ("until", "every time", "at least")
+//	missing_guard       — the operator keeps warning what must not happen
+//
+// The classifier is a deterministic keyword lexicon over the normalized
+// instruction, with fixed precedence guard > setpoint > observation > verb:
+// a guard misclassified as a verb could become an action proposal, so the
+// constraining readings win. Anything the lexicon cannot place lands in
+// unclassified, which is REPORTED but never generates a proposal.
+// control-law: retro-proposes-never-enforces
+const (
+	GapObservation  = "missing_observation"
+	GapVerb         = "missing_verb"
+	GapSetpoint     = "missing_setpoint"
+	GapGuard        = "missing_guard"
+	GapUnclassified = "unclassified"
+)
+
+// The lexicons match either whole tokens or normalized phrases. Normalization
+// has already lowered the text and stripped punctuation ("don't" → "don t").
+var (
+	guardPhrases = []string{"don t", "do not", "make sure not", "must not", "never", "only if", "unless", "be careful", "avoid", "without asking", "instead of"}
+	guardTokens  = []string{"dont", "stop"}
+
+	setpointPhrases = []string{"until", "at least", "at most", "within", "every time", "each time", "whenever", "keep doing", "always", "from now on", "before you finish", "when green", "when it passes"}
+
+	observationPhrases = []string{"check the", "check whether", "check if", "what is the", "what s the", "show me", "look at", "status of", "is it", "did it", "how is", "where is", "monitor"}
+
+	verbTokens = []string{"run", "merge", "publish", "push", "rerun", "retry", "open", "record", "fix", "update", "deploy", "rebase", "commit", "create", "install", "sync", "clean", "make"}
+)
+
+// ClassifyGap places one normalized instruction into a gap type.
+func ClassifyGap(normalized string) string {
+	padded := " " + normalized + " "
+	containsPhrase := func(phrases []string) bool {
+		for _, phrase := range phrases {
+			if strings.Contains(padded, " "+phrase+" ") {
+				return true
+			}
+		}
+		return false
+	}
+	tokens := map[string]bool{}
+	for _, token := range strings.Fields(normalized) {
+		tokens[token] = true
+	}
+	containsToken := func(list []string) bool {
+		for _, token := range list {
+			if tokens[token] {
+				return true
+			}
+		}
+		return false
+	}
+	switch {
+	case containsPhrase(guardPhrases) || containsToken(guardTokens):
+		return GapGuard
+	case containsPhrase(setpointPhrases):
+		return GapSetpoint
+	case containsPhrase(observationPhrases):
+		return GapObservation
+	case containsToken(verbTokens):
+		return GapVerb
+	default:
+		return GapUnclassified
+	}
+}
+
+// SuggestedShape names the typed construct to add for a gap type — prose
+// pointing a human at the right kind of promotion, never a diff.
+func SuggestedShape(gapType string) string {
+	switch gapType {
+	case GapObservation:
+		return "Add a typed observation: a read-only status or frontier field that answers this without being asked."
+	case GapVerb:
+		return "Add or prescribe a typed verb: a deterministic command the flow names at the right state."
+	case GapSetpoint:
+		return "Add a typed setpoint: a persisted goal or condition (like delivery.terminal) the flow pursues so this stops being restated."
+	case GapGuard:
+		return "Add a typed guard: an enforced precondition or denial (a gate or policy) instead of a remembered warning."
+	default:
+		return ""
+	}
+}

labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/report.go

@@ -0,0 +1,57 @@
+package retromine
+
+// The report is the miner's entire output surface: typed proposals for the
+// classified recurrences, and the unclassified recurrences named so nothing
+// is silently dropped. It is data for a human to review — the derivation
+// proposes, and promotion into a real state, verb, setpoint, or guard is
+// always a reviewed change made by hand.
+// control-law: retro-proposes-never-enforces
+const ReportSchemaVersion = 1
+
+// Proposal is one recurring instruction promoted to a typed suggestion.
+type Proposal struct {
+	GapType        string     `json:"gap_type"`
+	Occurrences    int        `json:"occurrences"`
+	Sessions       []string   `json:"sessions"`
+	Exemplar       string     `json:"exemplar"`
+	SuggestedShape string     `json:"suggested_shape"`
+	Evidence       []EventRef `json:"evidence"`
+}
+
+// Report is the full derivation result over one set of transcripts.
+type Report struct {
+	SchemaVersion  int        `json:"schema_version"`
+	EventsScanned  int        `json:"events_scanned"`
+	OperatorEvents int        `json:"operator_events"`
+	Proposals      []Proposal `json:"proposals"`
+	// Unclassified recurrences are surfaced — a recurrence the lexicon cannot
+	// place is still steady-state error worth a human look — but they never
+	// become proposals (fail-closed).
+	Unclassified []Cluster `json:"unclassified,omitempty"`
+}
+
+// BuildReport mines the events and classifies every recurrence.
+func BuildReport(events []Event) Report {
+	report := Report{SchemaVersion: ReportSchemaVersion, EventsScanned: len(events), Proposals: []Proposal{}}
+	for _, event := range events {
+		if event.Role == RoleOperator {
+			report.OperatorEvents++
+		}
+	}
+	for _, cluster := range DetectRecurrence(events) {
+		gapType := ClassifyGap(cluster.Normalized)
+		if gapType == GapUnclassified {
+			report.Unclassified = append(report.Unclassified, cluster)
+			continue
+		}
+		report.Proposals = append(report.Proposals, Proposal{
+			GapType:        gapType,
+			Occurrences:    cluster.Occurrences,
+			Sessions:       cluster.Sessions,
+			Exemplar:       cluster.Exemplar,
+			SuggestedShape: SuggestedShape(gapType),
+			Evidence:       cluster.Evidence,
+		})
+	}
+	return report
+}

labs/12-product-engineering-loop/product-engineering-loop/retro.go

@@ -0,0 +1,61 @@
+package boatstack
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/operatorstack/boatstack/boatstack/internal/retromine"
+)
+
+// RetroInput is one transcript handed to the retro derivation: a name (for
+// evidence references and per-file session identity) and its raw content.
+// The derivation layer takes bytes, never paths — every capability the miner
+// lacks (filesystem, network, subprocess, clock) stays lacking here; only
+// the CLI boundary reads files, from operator-supplied paths only.
+// control-law: retro-derivation-is-offline-and-deterministic
+type RetroInput struct {
+	Name    string
+	Content []byte
+}
+
+// RetroDerive parses every input with the named adapter format ("" sniffs
+// per file: events | claudecode | plaintext) and mines the combined events
+// for recurring operator instructions, classified into typed-gap proposals.
+// It proposes only: no file is written, no state is touched, no command is
+// run, and nothing is enforced — promotion is always a reviewed change made
+// by hand. control-law: retro-proposes-never-enforces
+func RetroDerive(format string, inputs []RetroInput) (retromine.Report, error) {
+	events := []retromine.Event{}
+	for _, input := range inputs {
+		parsed, err := retromine.ParseTranscript(format, input.Name, input.Content)
+		if err != nil {
+			return retromine.Report{}, err
+		}
+		events = append(events, parsed...)
+	}
+	return retromine.BuildReport(events), nil
+}
+
+// FormatRetroReport renders the derivation for a human reviewer.
+func FormatRetroReport(report retromine.Report) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Retro derivation: %d event(s) scanned, %d from the operator.\n",
+		report.EventsScanned, report.OperatorEvents)
+	if len(report.Proposals) == 0 && len(report.Unclassified) == 0 {
+		b.WriteString("No recurring operator instruction found across sessions. Nothing to promote.\n")
+		return b.String()
+	}
+	for i, proposal := range report.Proposals {
+		fmt.Fprintf(&b, "\n%d. [%s] seen %d time(s) across %d session(s)\n", i+1,
+			proposal.GapType, proposal.Occurrences, len(proposal.Sessions))
+		fmt.Fprintf(&b, "   Instruction: %q\n", proposal.Exemplar)
+		fmt.Fprintf(&b, "   Promote it: %s\n", proposal.SuggestedShape)
+	}
+	for _, cluster := range report.Unclassified {
+		fmt.Fprintf(&b, "\n?. [unclassified] seen %d time(s) across %d session(s): %q\n",
+			cluster.Occurrences, len(cluster.Sessions), cluster.Exemplar)
+		b.WriteString("   Recurs, but no gap type matched; review it by hand. No proposal is generated.\n")
+	}
+	b.WriteString("\nDerivation proposes; it never enforces. Promote a proposal by hand through the normal reviewed delivery flow.\n")
+	return b.String()
+}