diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6250f19..9398942 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ # Contributing -Boatstack is a generated content distribution. Propose changes to workflow semantics, templates, evidence rules, or generated presentation in [Intelligence Flow](https://github.com/operatorstack/intelligence-flow/tree/c550ef95f440e8b463ae0d436953e10ad91e1ab6/labs/12-product-engineering-loop). +Boatstack is a generated content distribution. Propose changes to workflow semantics, templates, evidence rules, or generated presentation in [Intelligence Flow](https://github.com/operatorstack/intelligence-flow/tree/b37bfc311634ab20082ce1d768f0c953530d72cc/labs/12-product-engineering-loop). The Boatstack repository receives product/runtime changes through a generated pull request. Review the PR's `UPSTREAM.json`, tests, adapter diff, and context-size change; do not hand-edit generated output on `main`. `.github/workflows` is the exception: it is Boatstack's executable control plane, excluded from scheduled projection and changed only through a separate manually reviewed Boatstack PR. diff --git a/UPSTREAM.json b/UPSTREAM.json index 55ca006..8d00aa3 100644 --- a/UPSTREAM.json +++ b/UPSTREAM.json @@ -12,7 +12,7 @@ }, "files": { ".gitignore": "a7079e923a776f14f1bb3a6aa0a11a133a8e1dfb35af020f327623357b7e3957", - "CONTRIBUTING.md": "370fe1191864a3a3b4acd5ca4bc22807f70da794743f376f992a140c534e0619", + "CONTRIBUTING.md": "dde868caa9c38db8ceaa264ee034fdd33eb413e57b570845dfeb5d44c1e9d8d7", "README.md": "3ce3e95e511089b44e946a44b8d5f4f81d019ece5336db65b2cab1f9dc4d4dad", "assets/boatstack-journey.svg": "e465befc50c8ce30f3e07e8fd97012931beeb053392c8fbf38ad645023b3cc63", "assets/boatstack-mark.svg": "be1f984da1bfa69fa5d1f986d8343d21f7e20921b71db888c928b4d2e54b09b5", @@ -129,6 +129,12 @@ "boatstack/internal/deliverycontrol/trajectorylog.go": "a1da7e7252b33f63f232c101de683b4a515e80243801caf69fda53d24233e42f", "boatstack/internal/deliverycontrol/trajectorylog_test.go": "227dd6ed9ce181d517a37b67ef4d64dd93779a533eae804798ab54de35c7f13e", "boatstack/internal/deliverycontrol/transition.go": "b43abb0e99d29697b27b0bb8ee2e2f5f31f3471a2983f25d18ae3564ee246775", + "boatstack/internal/retromine/adapters.go": "7736ebe7fe200dc6aca2799dab964a49031e4f840a049dc872f0bb62053a1ed8", + "boatstack/internal/retromine/cluster.go": "ddb6b3506ac192ab37f0341e57437ddf365207c42163da423a06bbfd856758f5", + "boatstack/internal/retromine/event.go": "4d41cbf37209d01bc5cc083d144f835950700aa7f6e39d0661b525b5e045227a", + "boatstack/internal/retromine/retromine_conformance_test.go": "33883893db1e455901d9c0e93767c39b475fae728aa9ee9dd925a2c21e5b0ec4", + "boatstack/internal/retromine/testdata/session-alpha.jsonl": "45ed8fa691af9db3f957e87175dbbeffd8fdd2f001324e86a5fd68e29ffbe244", + "boatstack/internal/retromine/testdata/session-beta.txt": "8e85e6a4442e3c892166df97ec879d998b3a47bdad5ef2bf785674b3149c36c2", "boatstack/migrate.go": "eaf589e2b266238068e42c6d78e01dc040266d28e342cb24f09e33e8541749b3", "boatstack/migrate_effect_grade.go": "bccb58e770001aa9554d8e7f151663d907f152508a61118f56fd90465ba6f32e", "boatstack/migrate_effect_grade_test.go": "fea1d1057bc6d8eaf015e377864a3adab29ef5731f597fe0a38b96fa80355d14", @@ -215,10 +221,10 @@ "docs/benchmark-corpus-audit.md": "f2d206fe8579a514f9da82b2c96c19b343ac004be67617e1bd34f0f8e0e5e6c6", "docs/benchmark-submission-audit.md": "9518abdd17690729c6423f87cab20418ed47b0915b5faa44b9ef975e9e9c3b79", "docs/configuration.md": "221f979506a3a9de357e5277f1329c345bf175346ec8dfc8fdd1212fb100dea1", - "docs/evidence-engineered-coding.md": "42c0efc8f5947b4b21f7f99ea8b86a303025bf3760b8fe17caf07ed47acfe609", + "docs/evidence-engineered-coding.md": "2a2b905e044d89cad2855f10c45261e980b0507c60e275165c8de8ed2a663f42", "docs/generated-files.md": "437791765b0a4015032ae21d1a6618563cad92b7402819e4f963bf5ae16284a3", "docs/getting-started.md": "51c2823f21e35140d31e6d5083dc4b89fddd24721ac6acc474154a4da53ee9f8", - "docs/public-claims.json": "b47e678c1a830a0bd6ceea4fdc3fae7b55794099770df61f6819b9802212f561", + "docs/public-claims.json": "6f78d9f8b0ce76079f23a43d84008045dc7e51e5004a2f3e394784e44a3b752e", "docs/public-surface.md": "713f7a050b5f339cf948299103ef3800417dccfecf2cc1a4166397ea6f978907", "docs/research-and-design.md": "8d78678108f0a6c924e1ff9b32c0f81aae9d1f779e0082843b6f99ad993ae2b6", "docs/safety.md": "7b9b5c515d36e683767ec8d3d9d6d119ac93650b2f629d351deadd4c600ed6a6", @@ -232,7 +238,7 @@ "labs/diagram-json/compiled/evidence.md": "1ba1c989ade070a8ef9a508fbd788d100d7292f2dbacbb2bce895468019f619d", "labs/diagram-json/compiled/tasks.json": "88f60851abf79d851e9fccc754ff3040034ae595306bc87d64784c19eb403e71", "labs/diagram-json/compiled/test-matrix.json": "424657ff505768e50fa113801fd8363364a18269d5297480907a993d44063a39", - "labs/diagram-json/plan.lock.json": "77ff19d1baf7a9872a2e0e052a1b02fdc804607eff4106bc677cf70c995e970e", + "labs/diagram-json/plan.lock.json": "2c2182f842a9ab47814c685ad0762bb63da4859fc27f583a25a32597e97d7b79", "labs/diagram-json/plan.md": "3cc4f533b8d69386deff16b3a594a3ba09d4c0c3db636cccd8c4380084ce6a51", "labs/diagram-json/questions.md": "74733b015002c8a6777c558e7e997fa48c94850b9bd39054fe9366c97ecf728d", "labs/diagram-json/request.md": "0808fc41c36779c404f4a3a121167da6e76cac56df526e70f9ed6d3e0d4c02ed", @@ -370,12 +376,13 @@ "release-notes/2026-07-28-operator-frontier-next-actor.md": "7e769625a2beb8a204d2d79158c18bdac350656de5c55998988a8a5aba2c319a", "release-notes/2026-07-28-post-publish-prescriptions.md": "a0b728df569bdba40e3b6107179b35873bad00e1ed1c2574e7eecf438b6962f4", "release-notes/2026-07-28-pr-phase-observation.md": "8c5615013eb88ce9561d30897e47f6fa967e0157c4c245f0c35a1f31e4e132e2", - "release-notes/2026-07-28-protected-native-auto-merge.md": "67dc76a6e7ce51034a0eadc541ba7a8946cfcabe5433cedc25db55321dfb8b62" + "release-notes/2026-07-28-protected-native-auto-merge.md": "67dc76a6e7ce51034a0eadc541ba7a8946cfcabe5433cedc25db55321dfb8b62", + "release-notes/2026-07-28-retromine-recurrence-detector.md": "27790993a02e73f3a2700dce7340d044aeb0e52f785ded68271ff6673add9e25" }, "generator": "operatorstack/intelligence-flow:boatstack-distribution", "schema_version": 1, "source": { - "commit": "c550ef95f440e8b463ae0d436953e10ad91e1ab6", + "commit": "b37bfc311634ab20082ce1d768f0c953530d72cc", "path": "labs/12-product-engineering-loop", "repository": "operatorstack/intelligence-flow" } diff --git a/boatstack/internal/retromine/adapters.go b/boatstack/internal/retromine/adapters.go new file mode 100644 index 0000000..93eba64 --- /dev/null +++ b/boatstack/internal/retromine/adapters.go @@ -0,0 +1,223 @@ +package retromine + +import ( + "encoding/json" + "fmt" + "io" + "strings" +) + +// Adapters perform the lossy projection from one host's transcript format +// into neutral events. Lossiness is one-directional by design: an adapter may +// SKIP entries it does not understand (host formats grow shapes constantly), +// but a line that fails to parse as the format at all is a typed error — +// mis-parsing must never silently become "no recurrence found". +// control-law: retro-derivation-is-offline-and-deterministic + +// Format names for ParseTranscript. +const ( + FormatNeutral = "events" + FormatClaudeCode = "claudecode" + FormatPlaintext = "plaintext" +) + +// ParseTranscript dispatches to the named adapter, or sniffs the format from +// content when format is empty: a JSON object line with a "role" field is the +// neutral format, one with "type"/"message" is a Claude Code session line, +// anything else is plain text. +func ParseTranscript(format, source string, content []byte) ([]Event, error) { + if format == "" { + format = sniffFormat(content) + } + switch format { + case FormatNeutral: + return ParseNeutralEvents(source, strings.NewReader(string(content))) + case FormatClaudeCode: + return ParseClaudeCodeSession(source, strings.NewReader(string(content))) + case FormatPlaintext: + return ParsePlaintextTranscript(source, strings.NewReader(string(content))) + default: + return nil, fmt.Errorf("unknown transcript format %q (supported: events, claudecode, plaintext)", format) + } +} + +func sniffFormat(content []byte) string { + for _, line := range strings.Split(string(content), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if !strings.HasPrefix(line, "{") { + return FormatPlaintext + } + var probe map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &probe); err != nil { + return FormatPlaintext + } + if _, ok := probe["role"]; ok { + return FormatNeutral + } + return FormatClaudeCode + } + return FormatPlaintext +} + +// claudeCodeLine is the subset of a Claude Code session JSONL entry the +// projection needs. Message content is either a plain string or an array of +// typed blocks; only text blocks carry conversational text, and tool_result +// blocks mark tool output. +type claudeCodeLine struct { + Type string `json:"type"` + SessionID string `json:"sessionId"` + Timestamp string `json:"timestamp"` + Message struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + } `json:"message"` +} + +// ParseClaudeCodeSession projects a Claude Code session JSONL stream into +// neutral events. Entries whose type is not user/assistant (summaries, +// hooks, system reminders) are skipped — projection is lossy — but a line +// that is not valid JSON is a typed error. +func ParseClaudeCodeSession(source string, r io.Reader) ([]Event, error) { + scanner := newLineScanner(r) + events := []Event{} + line := 0 + for scanner.Scan() { + line++ + raw := strings.TrimSpace(scanner.Text()) + if raw == "" { + continue + } + var entry claudeCodeLine + if err := json.Unmarshal([]byte(raw), &entry); err != nil { + return nil, fmt.Errorf("parse claudecode session %s line %d: %w", source, line, err) + } + role := "" + switch entry.Type { + case "user": + role = RoleOperator + case "assistant": + role = RoleAgent + default: + continue + } + text, isToolPayload := claudeCodeText(entry.Message.Content) + if isToolPayload { + role = RoleTool + } + if strings.TrimSpace(text) == "" { + continue + } + sessionID := entry.SessionID + if sessionID == "" { + sessionID = source + } + events = append(events, Event{ + Source: source, SessionID: sessionID, Timestamp: entry.Timestamp, + Role: role, Text: text, + }) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read claudecode session %s: %w", source, err) + } + return assignSessionIndexes(events), nil +} + +// claudeCodeText extracts conversational text from a message content value. +// The bool reports that the content was ONLY tool payload (tool results), +// which projects as RoleTool so it never counts as an operator instruction. +func claudeCodeText(content json.RawMessage) (string, bool) { + if len(content) == 0 { + return "", false + } + var plain string + if err := json.Unmarshal(content, &plain); err == nil { + return plain, false + } + var blocks []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal(content, &blocks); err != nil { + return "", false + } + texts := []string{} + sawTool := false + for _, block := range blocks { + switch block.Type { + case "text": + if strings.TrimSpace(block.Text) != "" { + texts = append(texts, block.Text) + } + case "tool_result", "tool_use": + sawTool = true + } + } + if len(texts) == 0 { + return "", sawTool + } + return strings.Join(texts, "\n"), false +} + +// plaintextPrefixes maps a line prefix to a role for the plain-text adapter. +// Order matters: first match wins. Unprefixed text continues the current +// speaker's turn; before any prefix appears, text defaults to the operator — +// fail-open into the INPUT only (the worst a misclassified line can do is +// create one more proposal for a human to reject; it can never act). +var plaintextPrefixes = []struct { + prefix string + role string +}{ + {"user:", RoleOperator}, + {"operator:", RoleOperator}, + {"h:", RoleOperator}, + {">", RoleOperator}, + {"assistant:", RoleAgent}, + {"agent:", RoleAgent}, + {"a:", RoleAgent}, + {"tool:", RoleTool}, +} + +// ParsePlaintextTranscript projects a prefix-annotated plain-text transcript +// (`User: …` / `Agent: …`) into neutral events. Consecutive lines of one +// speaker merge into one event; the whole file is one session identified by +// its source name. +func ParsePlaintextTranscript(source string, r io.Reader) ([]Event, error) { + scanner := newLineScanner(r) + events := []Event{} + currentRole := RoleOperator + var current []string + flush := func() { + text := strings.TrimSpace(strings.Join(current, "\n")) + current = nil + if text == "" { + return + } + events = append(events, Event{Source: source, SessionID: source, Role: currentRole, Text: text}) + } + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + matched := false + lower := strings.ToLower(trimmed) + for _, candidate := range plaintextPrefixes { + if strings.HasPrefix(lower, candidate.prefix) { + flush() + currentRole = candidate.role + current = append(current, strings.TrimSpace(trimmed[len(candidate.prefix):])) + matched = true + break + } + } + if !matched { + current = append(current, line) + } + } + flush() + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read plaintext transcript %s: %w", source, err) + } + return assignSessionIndexes(events), nil +} diff --git a/boatstack/internal/retromine/cluster.go b/boatstack/internal/retromine/cluster.go new file mode 100644 index 0000000..74d8649 --- /dev/null +++ b/boatstack/internal/retromine/cluster.go @@ -0,0 +1,182 @@ +package retromine + +import ( + "regexp" + "sort" + "strings" +) + +// Recurrence detection: normalize each operator instruction, reduce it to +// token 3-shingles, and greedily cluster by Jaccard similarity in a fixed +// order. A cluster is a recurrence candidate only when the same instruction +// shape appears at least minOccurrences times across at least minSessions +// distinct sessions — repetition inside one conversation is conversation, +// not steady-state error. Everything here is deterministic: inputs are +// sorted before clustering, so identical inputs in any order produce +// identical clusters. +// control-law: retro-derivation-is-offline-and-deterministic +const ( + // jaccardThreshold is the shingle-set similarity at which two + // instructions count as the same instruction shape. + jaccardThreshold = 0.6 + // minInstructionTokens filters acknowledgements and one-word replies + // ("g", "ok", "yes please") out of the instruction pool. + minInstructionTokens = 4 + // minOccurrences and minSessions define recurrence. + minOccurrences = 3 + minSessions = 2 + // exemplarCap bounds the quoted exemplar so a report never embeds a wall + // of transcript text. + exemplarCap = 240 +) + +// Cluster is one recurring instruction shape with its evidence. +type Cluster struct { + Exemplar string `json:"exemplar"` + Normalized string `json:"normalized"` + Occurrences int `json:"occurrences"` + Sessions []string `json:"sessions"` + Evidence []EventRef `json:"evidence"` +} + +var ( + fencedCodePattern = regexp.MustCompile("(?s)```.*?```") + inlineCodeSpaces = regexp.MustCompile("\\s+") + nonWordPattern = regexp.MustCompile(`[^a-z0-9 ]+`) +) + +// normalizeInstruction reduces an operator message to its comparable shape: +// fenced code stripped (pasted logs are payload, not instruction), lowered, +// punctuation removed, whitespace collapsed. +func normalizeInstruction(text string) string { + text = fencedCodePattern.ReplaceAllString(text, " ") + text = strings.ToLower(text) + text = nonWordPattern.ReplaceAllString(text, " ") + return strings.TrimSpace(inlineCodeSpaces.ReplaceAllString(text, " ")) +} + +// shingles returns the token 3-shingle set; short instructions fall back to +// one whole-text shingle so they remain comparable. +func shingles(normalized string) map[string]bool { + tokens := strings.Fields(normalized) + set := map[string]bool{} + if len(tokens) < 3 { + if len(tokens) > 0 { + set[strings.Join(tokens, " ")] = true + } + return set + } + for i := 0; i+3 <= len(tokens); i++ { + set[strings.Join(tokens[i:i+3], " ")] = true + } + return set +} + +func jaccard(a, b map[string]bool) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + intersection := 0 + for key := range a { + if b[key] { + intersection++ + } + } + union := len(a) + len(b) - intersection + if union == 0 { + return 0 + } + return float64(intersection) / float64(union) +} + +type candidate struct { + ref EventRef + text string + normalized string + shingleSet map[string]bool +} + +// DetectRecurrence finds the recurring operator instruction shapes across the +// given events. Only operator events participate; ordering of the input does +// not matter. +func DetectRecurrence(events []Event) []Cluster { + candidates := []candidate{} + // The per-session Index is intrinsic to each event (parser-assigned from + // transcript order), so the caller's concatenation order is irrelevant. + for _, event := range events { + if event.Role != RoleOperator { + continue + } + normalized := normalizeInstruction(event.Text) + if len(strings.Fields(normalized)) < minInstructionTokens { + continue + } + candidates = append(candidates, candidate{ + ref: EventRef{SessionID: event.SessionID, Index: event.Index}, + text: strings.TrimSpace(event.Text), + normalized: normalized, + shingleSet: shingles(normalized), + }) + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].ref.SessionID != candidates[j].ref.SessionID { + return candidates[i].ref.SessionID < candidates[j].ref.SessionID + } + return candidates[i].ref.Index < candidates[j].ref.Index + }) + + type bucket struct { + representative candidate + members []candidate + } + buckets := []*bucket{} + for _, c := range candidates { + placed := false + for _, b := range buckets { + if jaccard(c.shingleSet, b.representative.shingleSet) >= jaccardThreshold { + b.members = append(b.members, c) + placed = true + break + } + } + if !placed { + buckets = append(buckets, &bucket{representative: c, members: []candidate{c}}) + } + } + + clusters := []Cluster{} + for _, b := range buckets { + sessions := map[string]bool{} + refs := make([]EventRef, 0, len(b.members)) + for _, member := range b.members { + sessions[member.ref.SessionID] = true + refs = append(refs, member.ref) + } + if len(b.members) < minOccurrences || len(sessions) < minSessions { + continue + } + names := make([]string, 0, len(sessions)) + for session := range sessions { + names = append(names, session) + } + sort.Strings(names) + exemplar := b.representative.text + if len(exemplar) > exemplarCap { + exemplar = exemplar[:exemplarCap] + "…" + } + clusters = append(clusters, Cluster{ + Exemplar: exemplar, + Normalized: b.representative.normalized, + Occurrences: len(b.members), + Sessions: names, + Evidence: refs, + }) + } + sort.Slice(clusters, func(i, j int) bool { + if clusters[i].Occurrences != clusters[j].Occurrences { + return clusters[i].Occurrences > clusters[j].Occurrences + } + return clusters[i].Normalized < clusters[j].Normalized + }) + return clusters +} diff --git a/boatstack/internal/retromine/event.go b/boatstack/internal/retromine/event.go new file mode 100644 index 0000000..5ac2f0c --- /dev/null +++ b/boatstack/internal/retromine/event.go @@ -0,0 +1,118 @@ +// Package retromine detects recurring operator instructions in coding-agent +// transcripts, offline and deterministically. It exists to serve one control +// law: a recurring operator instruction is steady-state error — evidence that +// the controller is missing a typed observation, verb, setpoint, or guard — +// and the remedy is promotion into a typed construct, never a saved prompt. +// This package is the SENSOR of that loop: it parses transcripts into neutral +// events, finds recurrence, and (in the classify layer) names the gap. It +// proposes; it never enforces, writes, or calls anything. +// +// Purity contract: no network, no subprocesses, no filesystem — inputs arrive +// as io.Readers, randomness and wall clocks are not used, and identical +// inputs in any order produce identical output. +// control-law: retro-derivation-is-offline-and-deterministic +package retromine + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "strings" +) + +// Role classifies who produced an event. Only operator events feed the +// recurrence detector: the law is about what the OPERATOR keeps having to +// say, never about what an agent generates. +const ( + RoleOperator = "operator" + RoleAgent = "agent" + RoleTool = "tool" +) + +// Event is the neutral transcript unit every adapter projects into. +// The schema is deliberately minimal: source (which adapter/file), a session +// identity (recurrence across sessions is the signal; within one session it +// is just conversation), an optional RFC3339 timestamp, a role, and the text. +type Event struct { + Source string `json:"source"` + SessionID string `json:"session_id"` + Timestamp string `json:"ts,omitempty"` + Role string `json:"role"` + Text string `json:"text"` + // Index is the event's position within its session, assigned by the + // parser from transcript order. It is intrinsic to the event — never to + // the order a caller happens to concatenate inputs in — which is what + // keeps detection order-independent. + Index int `json:"index"` +} + +// EventRef points back into the parsed input so every proposal is traceable +// to its evidence without embedding whole transcripts anywhere. +type EventRef struct { + SessionID string `json:"session_id"` + Index int `json:"index"` +} + +// ParseNeutralEvents reads the neutral JSONL format (one Event per line). +// A syntactically invalid line is a typed error naming its position — never +// a silent partial parse. Blank lines are permitted. +func ParseNeutralEvents(source string, r io.Reader) ([]Event, error) { + scanner := newLineScanner(r) + events := []Event{} + line := 0 + for scanner.Scan() { + line++ + raw := strings.TrimSpace(scanner.Text()) + if raw == "" { + continue + } + var event Event + if err := json.Unmarshal([]byte(raw), &event); err != nil { + return nil, fmt.Errorf("parse neutral events %s line %d: %w", source, line, err) + } + if event.Source == "" { + event.Source = source + } + if err := validateEvent(source, line, event); err != nil { + return nil, err + } + events = append(events, event) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read neutral events %s: %w", source, err) + } + return assignSessionIndexes(events), nil +} + +// assignSessionIndexes stamps each event's intrinsic per-session position in +// transcript order, overwriting whatever the input carried so the value is +// always the parser's, never a caller's claim. +func assignSessionIndexes(events []Event) []Event { + counters := map[string]int{} + for i := range events { + events[i].Index = counters[events[i].SessionID] + counters[events[i].SessionID]++ + } + return events +} + +func validateEvent(source string, line int, event Event) error { + switch event.Role { + case RoleOperator, RoleAgent, RoleTool: + default: + return fmt.Errorf("parse neutral events %s line %d: unknown role %q", source, line, event.Role) + } + if strings.TrimSpace(event.SessionID) == "" { + return fmt.Errorf("parse neutral events %s line %d: session_id is required", source, line) + } + return nil +} + +// newLineScanner returns a scanner sized for long transcript lines (tool +// results and pasted logs routinely exceed bufio's default token size). +func newLineScanner(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + return scanner +} diff --git a/boatstack/internal/retromine/retromine_conformance_test.go b/boatstack/internal/retromine/retromine_conformance_test.go new file mode 100644 index 0000000..448c1ca --- /dev/null +++ b/boatstack/internal/retromine/retromine_conformance_test.go @@ -0,0 +1,218 @@ +package retromine + +// control-law: retro-derivation-is-offline-and-deterministic +// +// The recurrence miner is a pure sensor: no network, no subprocesses, no +// filesystem, no clocks, no randomness — inputs arrive as bytes and readers, +// and identical inputs in ANY order produce byte-identical output. The +// import-purity test enforces the capability boundary structurally, the +// determinism test enforces the output contract, and the adapter tests pin +// the lossy-projection rule: skip what you do not understand, error on what +// fails to parse. +// +// Test classes: positive (a planted instruction repeated 3× across 2 +// sessions clusters; both host adapters project equivalent content to +// equivalent events), negative (2 occurrences or 1 session never clusters; +// acknowledgements below the token floor never enter the pool), relation +// (input order does not change the report), failure-state (malformed JSONL +// is a typed error naming the line, never a silent partial parse), bypass +// (the package imports grant no I/O capability at all). + +import ( + "fmt" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func operatorEvent(session string, text string) Event { + return Event{Source: "test", SessionID: session, Role: RoleOperator, Text: text} +} + +func plantedEvents() []Event { + return []Event{ + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-a", "add a logout button to the settings page"), + {Source: "test", SessionID: "session-a", Role: RoleAgent, Text: "make the pr, monitor the checks, and merge when green"}, + operatorEvent("session-b", "please make the PR, monitor the checks and merge when green"), + operatorEvent("session-b", "ok"), + operatorEvent("session-c", "make the pr monitor the checks and merge when it is green"), + } +} + +// Positive: three occurrences across three sessions cluster; the agent's +// identical text and the sub-floor acknowledgement never enter the pool. +func TestRecurringInstructionClusters(t *testing.T) { + clusters := DetectRecurrence(plantedEvents()) + if len(clusters) != 1 { + t.Fatalf("clusters = %#v, want exactly one", clusters) + } + cluster := clusters[0] + if cluster.Occurrences != 3 { + t.Fatalf("occurrences = %d, want 3", cluster.Occurrences) + } + if !reflect.DeepEqual(cluster.Sessions, []string{"session-a", "session-b", "session-c"}) { + t.Fatalf("sessions = %v", cluster.Sessions) + } + if !strings.Contains(cluster.Exemplar, "monitor the checks") { + t.Fatalf("exemplar lost the instruction: %q", cluster.Exemplar) + } +} + +// Negative: two occurrences, or three inside one session, are not recurrence. +func TestBelowThresholdNeverClusters(t *testing.T) { + twoOccurrences := []Event{ + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-b", "make the pr, monitor the checks, and merge when green"), + } + if clusters := DetectRecurrence(twoOccurrences); len(clusters) != 0 { + t.Fatalf("two occurrences clustered: %#v", clusters) + } + oneSession := []Event{ + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + } + if clusters := DetectRecurrence(oneSession); len(clusters) != 0 { + t.Fatalf("single-session repetition clustered: %#v", clusters) + } +} + +// Relation: input order does not change the report. +func TestDetectionIsOrderIndependent(t *testing.T) { + events := plantedEvents() + reversed := make([]Event, 0, len(events)) + for i := len(events) - 1; i >= 0; i-- { + reversed = append(reversed, events[i]) + } + forward := DetectRecurrence(events) + backward := DetectRecurrence(reversed) + if !reflect.DeepEqual(forward, backward) { + t.Fatalf("order changed the report:\n%#v\n---\n%#v", forward, backward) + } +} + +// Positive: normalization strips pasted logs, so the same instruction with a +// different fenced payload is the same instruction shape. +func TestFencedPayloadDoesNotSplitClusters(t *testing.T) { + events := []Event{ + operatorEvent("s1", "fix the failing windows shard\n```\nlog A\n```"), + operatorEvent("s2", "fix the failing windows shard\n```\ncompletely different log B\n```"), + operatorEvent("s3", "fix the failing windows shard please"), + } + if clusters := DetectRecurrence(events); len(clusters) != 1 { + t.Fatalf("payload variance split the cluster: %#v", clusters) + } +} + +// Positive + relation: the two host adapters project equivalent content into +// equivalent instruction streams — the miner is agent-agnostic by contract. +func TestAdaptersProjectEquivalentContent(t *testing.T) { + claudeLines := strings.Join([]string{ + `{"type":"user","sessionId":"cc-1","message":{"role":"user","content":"make the pr, monitor the checks, and merge when green"}}`, + `{"type":"assistant","sessionId":"cc-1","message":{"role":"assistant","content":[{"type":"text","text":"On it."}]}}`, + `{"type":"user","sessionId":"cc-1","message":{"role":"user","content":[{"type":"tool_result","content":"exit 0"}]}}`, + `{"type":"summary","summary":"irrelevant"}`, + }, "\n") + fromClaude, err := ParseTranscript("", "cc-1.jsonl", []byte(claudeLines)) + if err != nil { + t.Fatal(err) + } + plain := "User: make the pr, monitor the checks, and merge when green\nAgent: On it.\n" + fromPlain, err := ParseTranscript("", "plain.txt", []byte(plain)) + if err != nil { + t.Fatal(err) + } + pick := func(events []Event) []string { + out := []string{} + for _, e := range events { + if e.Role == RoleOperator { + out = append(out, normalizeInstruction(e.Text)) + } + } + return out + } + if !reflect.DeepEqual(pick(fromClaude), pick(fromPlain)) { + t.Fatalf("adapters disagree:\n%v\n---\n%v", pick(fromClaude), pick(fromPlain)) + } + // The tool result projected as tool, never operator. + for _, e := range fromClaude { + if e.Role == RoleOperator && strings.Contains(e.Text, "exit 0") { + t.Fatalf("tool payload classified as operator: %#v", e) + } + } +} + +// Failure-state: malformed JSONL is a typed error naming the line — never a +// silent partial parse. +func TestMalformedLinesAreTypedErrors(t *testing.T) { + if _, err := ParseTranscript(FormatNeutral, "bad.jsonl", []byte(`{"role":"operator","session_id":"s","text":"x"}`+"\nnot json\n")); err == nil || !strings.Contains(err.Error(), "line 2") { + t.Fatalf("malformed neutral line not surfaced: %v", err) + } + if _, err := ParseTranscript(FormatClaudeCode, "bad.jsonl", []byte("{broken\n")); err == nil || !strings.Contains(err.Error(), "line 1") { + t.Fatalf("malformed claudecode line not surfaced: %v", err) + } + if _, err := ParseTranscript(FormatNeutral, "bad.jsonl", []byte(`{"role":"wizard","session_id":"s","text":"x"}`+"\n")); err == nil || !strings.Contains(err.Error(), "unknown role") { + t.Fatalf("unknown role accepted: %v", err) + } +} + +// Bypass: the package's non-test imports grant no I/O capability — no +// network, no subprocesses, no filesystem, no clocks, no randomness. The +// boundary is structural, not behavioral. +func TestPackageImportsGrantNoCapabilities(t *testing.T) { + disallowed := []string{"net", "os", "syscall", "time", "math/rand", "crypto/rand", "path/filepath", "io/ioutil"} + fset := token.NewFileSet() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + name := entry.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, filepath.Join(".", name), nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range file.Imports { + path := strings.Trim(spec.Path.Value, `"`) + for _, banned := range disallowed { + if path == banned || strings.HasPrefix(path, banned+"/") { + t.Fatalf("%s imports %s — the miner must stay capability-free", name, path) + } + } + } + } +} + +// Golden determinism over the synthetic fixtures: parse both fixture +// transcripts, mine them together, and pin the whole report. +func TestFixtureGoldenReport(t *testing.T) { + events := []Event{} + for _, fixture := range []string{"session-alpha.jsonl", "session-beta.txt"} { + content, err := os.ReadFile(filepath.Join("testdata", fixture)) + if err != nil { + t.Fatal(err) + } + parsed, err := ParseTranscript("", fixture, content) + if err != nil { + t.Fatal(err) + } + events = append(events, parsed...) + } + clusters := DetectRecurrence(events) + if len(clusters) != 1 { + t.Fatalf("fixture clusters = %#v", clusters) + } + got := fmt.Sprintf("%dx across %v: %s", clusters[0].Occurrences, clusters[0].Sessions, clusters[0].Normalized) + want := "3x across [cc-alpha session-beta.txt]: open the pr watch ci until every check passes then merge it" + if got != want { + t.Fatalf("golden drift:\n got %q\nwant %q", got, want) + } +} diff --git a/boatstack/internal/retromine/testdata/session-alpha.jsonl b/boatstack/internal/retromine/testdata/session-alpha.jsonl new file mode 100644 index 0000000..8c754c5 --- /dev/null +++ b/boatstack/internal/retromine/testdata/session-alpha.jsonl @@ -0,0 +1,5 @@ +{"type":"user","sessionId":"cc-alpha","message":{"role":"user","content":"open the PR, watch CI until every check passes, then merge it"}} +{"type":"assistant","sessionId":"cc-alpha","message":{"role":"assistant","content":[{"type":"text","text":"Opening the PR now."}]}} +{"type":"user","sessionId":"cc-alpha","message":{"role":"user","content":"open the pr, watch ci until every check passes, then merge it."}} +{"type":"user","sessionId":"cc-alpha","message":{"role":"user","content":"also rename the config key while you are at it"}} +{"type":"summary","summary":"synthetic fixture"} diff --git a/boatstack/internal/retromine/testdata/session-beta.txt b/boatstack/internal/retromine/testdata/session-beta.txt new file mode 100644 index 0000000..da22b82 --- /dev/null +++ b/boatstack/internal/retromine/testdata/session-beta.txt @@ -0,0 +1,3 @@ +User: open the PR — watch CI until every check passes, then merge it +Agent: Working on it. +User: thanks diff --git a/docs/evidence-engineered-coding.md b/docs/evidence-engineered-coding.md index 80c9c38..50bf412 100644 --- a/docs/evidence-engineered-coding.md +++ b/docs/evidence-engineered-coding.md @@ -146,6 +146,6 @@ Delivery and system improvement also remain separate. A failed task may suggest ## What is evidence-backed -The current moves were derived from the Intelligence Flow benchmark corpus and product-repository studies. The generated source commit is [`c550ef95f440e8b463ae0d436953e10ad91e1ab6`](https://github.com/operatorstack/intelligence-flow/tree/c550ef95f440e8b463ae0d436953e10ad91e1ab6/labs/12-product-engineering-loop). +The current moves were derived from the Intelligence Flow benchmark corpus and product-repository studies. The generated source commit is [`b37bfc311634ab20082ce1d768f0c953530d72cc`](https://github.com/operatorstack/intelligence-flow/tree/b37bfc311634ab20082ce1d768f0c953530d72cc/labs/12-product-engineering-loop). The evidence supports specific failure mechanisms and guardrails. It does not establish that Boatstack is optimal, that control-theory notation proves software quality, or that one workflow dominates every team. Those are evaluation questions, so the distribution preserves measurements, provenance, gaps, and negative results. diff --git a/docs/public-claims.json b/docs/public-claims.json index a52036e..f7e6236 100644 --- a/docs/public-claims.json +++ b/docs/public-claims.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "source_commit": "c550ef95f440e8b463ae0d436953e10ad91e1ab6", + "source_commit": "b37bfc311634ab20082ce1d768f0c953530d72cc", "statuses": ["verified", "observed", "still_being_evaluated"], "claims": [ { @@ -12,7 +12,7 @@ "readable_evidence": "why-these-steps.md#portable-workflow-and-state", "implementation": ["../boatstack/export.go", "../boatstack/references/artifacts.md", "../boatstack/references/workflow.md"], "verification": ["../boatstack/export_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "human-decisions", @@ -23,7 +23,7 @@ "readable_evidence": "why-these-steps.md#human-decisions", "implementation": ["../boatstack/references/workflow.md", "../boatstack/plan.go"], "verification": ["../boatstack/plan_test.go", "../boatstack/planning_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "validation-provenance", @@ -34,7 +34,7 @@ "readable_evidence": "why-these-steps.md#validation-provenance", "implementation": ["validation-and-evidence.md", "../boatstack/plan.go"], "verification": ["../boatstack/plan_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "irreversible-operations", @@ -46,7 +46,7 @@ "readable_evidence": "why-these-steps.md#irreversible-operations", "implementation": ["safety.md", "../boatstack/safety.go", "../boatstack/hooks.go"], "verification": ["../boatstack/safety_test.go", "../boatstack/hooks_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "reviewer-ready-pr", @@ -57,7 +57,7 @@ "readable_evidence": "why-these-steps.md#reviewer-ready-pr", "implementation": ["../boatstack/pr.go", "getting-started.md"], "verification": ["../boatstack/pr_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "phase-scoped-delivery", @@ -68,7 +68,7 @@ "readable_evidence": "why-these-steps.md#phase-scoped-delivery", "implementation": ["../boatstack/delivery.go", "../boatstack/safety.go", "../boatstack/hooks.go", "../boatstack/references/workflow.md"], "verification": ["../boatstack/delivery_test.go", "../boatstack/pr_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "model-neutral-contract", @@ -79,7 +79,7 @@ "readable_evidence": "why-these-steps.md#model-choice-and-budget", "implementation": ["research-and-design.md", "../boatstack/references/workflow.md"], "verification": ["../boatstack/export_test.go", "../boatstack/planning_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "cross-model-failures", @@ -90,7 +90,7 @@ "readable_evidence": "why-these-steps.md#model-choice-and-budget", "implementation": ["research-and-design.md"], "verification": ["benchmark-corpus-audit.md", "benchmark-submission-audit.md"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "lower-cost-outcomes", @@ -101,7 +101,7 @@ "readable_evidence": "why-these-steps.md#model-choice-and-budget", "implementation": ["research-and-design.md"], "verification": ["benchmark-corpus-audit.md", "benchmark-submission-audit.md"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "git-worktree-activation", @@ -112,7 +112,7 @@ "readable_evidence": "why-these-steps.md#git-worktree-activation", "implementation": ["../boatstack/runtime_cache.go", "../boatstack/hooks.go"], "verification": ["../boatstack/runtime_cache_test.go", "../boatstack/hooks_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" }, { "id": "visible-updates", @@ -123,7 +123,7 @@ "readable_evidence": "why-these-steps.md#visible-updates", "implementation": ["../boatstack/update.go", "../boatstack/init.go"], "verification": ["../boatstack/update_test.go", "../boatstack/init_test.go", "../boatstack/export_test.go"], - "last_verified_version": "source:c550ef95f440e8b463ae0d436953e10ad91e1ab6" + "last_verified_version": "source:b37bfc311634ab20082ce1d768f0c953530d72cc" } ] } diff --git a/labs/diagram-json/plan.lock.json b/labs/diagram-json/plan.lock.json index 6111b81..878af88 100644 --- a/labs/diagram-json/plan.lock.json +++ b/labs/diagram-json/plan.lock.json @@ -6,7 +6,7 @@ "plan_path": "labs/diagram-json/plan.md", "plan_sha256": "3cc4f533b8d69386deff16b3a594a3ba09d4c0c3db636cccd8c4380084ce6a51", "schema_version": 1, - "source_commit": "c550ef95f440e8b463ae0d436953e10ad91e1ab6", + "source_commit": "b37bfc311634ab20082ce1d768f0c953530d72cc", "source_plan_path": "labs/diagram-json/source-plan.md", "source_plan_sha256": "e10593ddaa7522ab80cc991d0a09399257139799e37f737794cd49d68a39985b", "spec_path": "labs/diagram-json/spec.md", diff --git a/release-notes/2026-07-28-retromine-recurrence-detector.md b/release-notes/2026-07-28-retromine-recurrence-detector.md new file mode 100644 index 0000000..8b4f9d8 --- /dev/null +++ b/release-notes/2026-07-28-retromine-recurrence-detector.md @@ -0,0 +1,5 @@ +### Boatstack can now detect the instructions you keep repeating to your agent + +A new offline analysis engine reads coding-agent transcripts — Claude Code session files, plain-text logs, or a neutral event format any tool can emit — and finds the operator instructions that recur across sessions. Repetition inside one conversation does not count; the signal is the same instruction shape appearing in session after session, because an instruction you keep restating is evidence the system is missing a typed control, not a prompt to be saved. + +The engine is deterministic and capability-free by construction: no network, no subprocesses, no filesystem access, no clocks — its imports are conformance-tested to grant no I/O at all, and identical transcripts in any order produce identical results. Nothing is exposed to you yet; the user-facing `retro derive` command that turns detected recurrence into reviewable proposals arrives in the next update.