From b568c05d7b95ae4c53931de45e6f64975f1c1a93 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Tue, 28 Jul 2026 18:01:19 +0100 Subject: [PATCH] =?UTF-8?q?feat(boatstack):=20retromine=20=E2=80=94=20offl?= =?UTF-8?q?ine=20recurrence=20detection=20over=20transcripts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New pure package internal/retromine: the sensor for the control law that a recurring operator instruction is steady-state error. Neutral JSONL event schema; adapters for Claude Code session JSONL and prefix-annotated plain text (lossy projection: skip unknown shapes, typed error on unparseable lines); deterministic recurrence detection (normalize, strip fenced payloads, token 3-shingles, Jaccard >= 0.6, greedy clustering in sorted order) with recurrence defined as >= 3 occurrences across >= 2 sessions. Per-session event indexes are parser-assigned so caller concatenation order cannot change the report. Capability-free by construction: no network, subprocesses, filesystem, clocks, or randomness — the import list is conformance-tested. Synthetic fixtures only. No user-facing verb yet; retro derive follows. control-law: retro-derivation-is-offline-and-deterministic Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- ...026-07-28-retromine-recurrence-detector.md | 5 + .../internal/retromine/adapters.go | 223 ++++++++++++++++++ .../internal/retromine/cluster.go | 182 ++++++++++++++ .../internal/retromine/event.go | 118 +++++++++ .../retromine/retromine_conformance_test.go | 218 +++++++++++++++++ .../retromine/testdata/session-alpha.jsonl | 5 + .../retromine/testdata/session-beta.txt | 3 + 7 files changed, 754 insertions(+) create mode 100644 labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retromine-recurrence-detector.md create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/adapters.go create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/cluster.go create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/event.go create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/retromine_conformance_test.go create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-alpha.jsonl create mode 100644 labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-beta.txt diff --git a/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retromine-recurrence-detector.md b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retromine-recurrence-detector.md new file mode 100644 index 000000000..8b4f9d812 --- /dev/null +++ b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-28-retromine-recurrence-detector.md @@ -0,0 +1,5 @@ +### Boatstack can now detect the instructions you keep repeating to your agent + +A new offline analysis engine reads coding-agent transcripts — Claude Code session files, plain-text logs, or a neutral event format any tool can emit — and finds the operator instructions that recur across sessions. Repetition inside one conversation does not count; the signal is the same instruction shape appearing in session after session, because an instruction you keep restating is evidence the system is missing a typed control, not a prompt to be saved. + +The engine is deterministic and capability-free by construction: no network, no subprocesses, no filesystem access, no clocks — its imports are conformance-tested to grant no I/O at all, and identical transcripts in any order produce identical results. Nothing is exposed to you yet; the user-facing `retro derive` command that turns detected recurrence into reviewable proposals arrives in the next update. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/adapters.go b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/adapters.go new file mode 100644 index 000000000..93eba64d3 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/adapters.go @@ -0,0 +1,223 @@ +package retromine + +import ( + "encoding/json" + "fmt" + "io" + "strings" +) + +// Adapters perform the lossy projection from one host's transcript format +// into neutral events. Lossiness is one-directional by design: an adapter may +// SKIP entries it does not understand (host formats grow shapes constantly), +// but a line that fails to parse as the format at all is a typed error — +// mis-parsing must never silently become "no recurrence found". +// control-law: retro-derivation-is-offline-and-deterministic + +// Format names for ParseTranscript. +const ( + FormatNeutral = "events" + FormatClaudeCode = "claudecode" + FormatPlaintext = "plaintext" +) + +// ParseTranscript dispatches to the named adapter, or sniffs the format from +// content when format is empty: a JSON object line with a "role" field is the +// neutral format, one with "type"/"message" is a Claude Code session line, +// anything else is plain text. +func ParseTranscript(format, source string, content []byte) ([]Event, error) { + if format == "" { + format = sniffFormat(content) + } + switch format { + case FormatNeutral: + return ParseNeutralEvents(source, strings.NewReader(string(content))) + case FormatClaudeCode: + return ParseClaudeCodeSession(source, strings.NewReader(string(content))) + case FormatPlaintext: + return ParsePlaintextTranscript(source, strings.NewReader(string(content))) + default: + return nil, fmt.Errorf("unknown transcript format %q (supported: events, claudecode, plaintext)", format) + } +} + +func sniffFormat(content []byte) string { + for _, line := range strings.Split(string(content), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if !strings.HasPrefix(line, "{") { + return FormatPlaintext + } + var probe map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &probe); err != nil { + return FormatPlaintext + } + if _, ok := probe["role"]; ok { + return FormatNeutral + } + return FormatClaudeCode + } + return FormatPlaintext +} + +// claudeCodeLine is the subset of a Claude Code session JSONL entry the +// projection needs. Message content is either a plain string or an array of +// typed blocks; only text blocks carry conversational text, and tool_result +// blocks mark tool output. +type claudeCodeLine struct { + Type string `json:"type"` + SessionID string `json:"sessionId"` + Timestamp string `json:"timestamp"` + Message struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + } `json:"message"` +} + +// ParseClaudeCodeSession projects a Claude Code session JSONL stream into +// neutral events. Entries whose type is not user/assistant (summaries, +// hooks, system reminders) are skipped — projection is lossy — but a line +// that is not valid JSON is a typed error. +func ParseClaudeCodeSession(source string, r io.Reader) ([]Event, error) { + scanner := newLineScanner(r) + events := []Event{} + line := 0 + for scanner.Scan() { + line++ + raw := strings.TrimSpace(scanner.Text()) + if raw == "" { + continue + } + var entry claudeCodeLine + if err := json.Unmarshal([]byte(raw), &entry); err != nil { + return nil, fmt.Errorf("parse claudecode session %s line %d: %w", source, line, err) + } + role := "" + switch entry.Type { + case "user": + role = RoleOperator + case "assistant": + role = RoleAgent + default: + continue + } + text, isToolPayload := claudeCodeText(entry.Message.Content) + if isToolPayload { + role = RoleTool + } + if strings.TrimSpace(text) == "" { + continue + } + sessionID := entry.SessionID + if sessionID == "" { + sessionID = source + } + events = append(events, Event{ + Source: source, SessionID: sessionID, Timestamp: entry.Timestamp, + Role: role, Text: text, + }) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read claudecode session %s: %w", source, err) + } + return assignSessionIndexes(events), nil +} + +// claudeCodeText extracts conversational text from a message content value. +// The bool reports that the content was ONLY tool payload (tool results), +// which projects as RoleTool so it never counts as an operator instruction. +func claudeCodeText(content json.RawMessage) (string, bool) { + if len(content) == 0 { + return "", false + } + var plain string + if err := json.Unmarshal(content, &plain); err == nil { + return plain, false + } + var blocks []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal(content, &blocks); err != nil { + return "", false + } + texts := []string{} + sawTool := false + for _, block := range blocks { + switch block.Type { + case "text": + if strings.TrimSpace(block.Text) != "" { + texts = append(texts, block.Text) + } + case "tool_result", "tool_use": + sawTool = true + } + } + if len(texts) == 0 { + return "", sawTool + } + return strings.Join(texts, "\n"), false +} + +// plaintextPrefixes maps a line prefix to a role for the plain-text adapter. +// Order matters: first match wins. Unprefixed text continues the current +// speaker's turn; before any prefix appears, text defaults to the operator — +// fail-open into the INPUT only (the worst a misclassified line can do is +// create one more proposal for a human to reject; it can never act). +var plaintextPrefixes = []struct { + prefix string + role string +}{ + {"user:", RoleOperator}, + {"operator:", RoleOperator}, + {"h:", RoleOperator}, + {">", RoleOperator}, + {"assistant:", RoleAgent}, + {"agent:", RoleAgent}, + {"a:", RoleAgent}, + {"tool:", RoleTool}, +} + +// ParsePlaintextTranscript projects a prefix-annotated plain-text transcript +// (`User: …` / `Agent: …`) into neutral events. Consecutive lines of one +// speaker merge into one event; the whole file is one session identified by +// its source name. +func ParsePlaintextTranscript(source string, r io.Reader) ([]Event, error) { + scanner := newLineScanner(r) + events := []Event{} + currentRole := RoleOperator + var current []string + flush := func() { + text := strings.TrimSpace(strings.Join(current, "\n")) + current = nil + if text == "" { + return + } + events = append(events, Event{Source: source, SessionID: source, Role: currentRole, Text: text}) + } + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + matched := false + lower := strings.ToLower(trimmed) + for _, candidate := range plaintextPrefixes { + if strings.HasPrefix(lower, candidate.prefix) { + flush() + currentRole = candidate.role + current = append(current, strings.TrimSpace(trimmed[len(candidate.prefix):])) + matched = true + break + } + } + if !matched { + current = append(current, line) + } + } + flush() + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read plaintext transcript %s: %w", source, err) + } + return assignSessionIndexes(events), nil +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/cluster.go b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/cluster.go new file mode 100644 index 000000000..74d864957 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/cluster.go @@ -0,0 +1,182 @@ +package retromine + +import ( + "regexp" + "sort" + "strings" +) + +// Recurrence detection: normalize each operator instruction, reduce it to +// token 3-shingles, and greedily cluster by Jaccard similarity in a fixed +// order. A cluster is a recurrence candidate only when the same instruction +// shape appears at least minOccurrences times across at least minSessions +// distinct sessions — repetition inside one conversation is conversation, +// not steady-state error. Everything here is deterministic: inputs are +// sorted before clustering, so identical inputs in any order produce +// identical clusters. +// control-law: retro-derivation-is-offline-and-deterministic +const ( + // jaccardThreshold is the shingle-set similarity at which two + // instructions count as the same instruction shape. + jaccardThreshold = 0.6 + // minInstructionTokens filters acknowledgements and one-word replies + // ("g", "ok", "yes please") out of the instruction pool. + minInstructionTokens = 4 + // minOccurrences and minSessions define recurrence. + minOccurrences = 3 + minSessions = 2 + // exemplarCap bounds the quoted exemplar so a report never embeds a wall + // of transcript text. + exemplarCap = 240 +) + +// Cluster is one recurring instruction shape with its evidence. +type Cluster struct { + Exemplar string `json:"exemplar"` + Normalized string `json:"normalized"` + Occurrences int `json:"occurrences"` + Sessions []string `json:"sessions"` + Evidence []EventRef `json:"evidence"` +} + +var ( + fencedCodePattern = regexp.MustCompile("(?s)```.*?```") + inlineCodeSpaces = regexp.MustCompile("\\s+") + nonWordPattern = regexp.MustCompile(`[^a-z0-9 ]+`) +) + +// normalizeInstruction reduces an operator message to its comparable shape: +// fenced code stripped (pasted logs are payload, not instruction), lowered, +// punctuation removed, whitespace collapsed. +func normalizeInstruction(text string) string { + text = fencedCodePattern.ReplaceAllString(text, " ") + text = strings.ToLower(text) + text = nonWordPattern.ReplaceAllString(text, " ") + return strings.TrimSpace(inlineCodeSpaces.ReplaceAllString(text, " ")) +} + +// shingles returns the token 3-shingle set; short instructions fall back to +// one whole-text shingle so they remain comparable. +func shingles(normalized string) map[string]bool { + tokens := strings.Fields(normalized) + set := map[string]bool{} + if len(tokens) < 3 { + if len(tokens) > 0 { + set[strings.Join(tokens, " ")] = true + } + return set + } + for i := 0; i+3 <= len(tokens); i++ { + set[strings.Join(tokens[i:i+3], " ")] = true + } + return set +} + +func jaccard(a, b map[string]bool) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + intersection := 0 + for key := range a { + if b[key] { + intersection++ + } + } + union := len(a) + len(b) - intersection + if union == 0 { + return 0 + } + return float64(intersection) / float64(union) +} + +type candidate struct { + ref EventRef + text string + normalized string + shingleSet map[string]bool +} + +// DetectRecurrence finds the recurring operator instruction shapes across the +// given events. Only operator events participate; ordering of the input does +// not matter. +func DetectRecurrence(events []Event) []Cluster { + candidates := []candidate{} + // The per-session Index is intrinsic to each event (parser-assigned from + // transcript order), so the caller's concatenation order is irrelevant. + for _, event := range events { + if event.Role != RoleOperator { + continue + } + normalized := normalizeInstruction(event.Text) + if len(strings.Fields(normalized)) < minInstructionTokens { + continue + } + candidates = append(candidates, candidate{ + ref: EventRef{SessionID: event.SessionID, Index: event.Index}, + text: strings.TrimSpace(event.Text), + normalized: normalized, + shingleSet: shingles(normalized), + }) + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].ref.SessionID != candidates[j].ref.SessionID { + return candidates[i].ref.SessionID < candidates[j].ref.SessionID + } + return candidates[i].ref.Index < candidates[j].ref.Index + }) + + type bucket struct { + representative candidate + members []candidate + } + buckets := []*bucket{} + for _, c := range candidates { + placed := false + for _, b := range buckets { + if jaccard(c.shingleSet, b.representative.shingleSet) >= jaccardThreshold { + b.members = append(b.members, c) + placed = true + break + } + } + if !placed { + buckets = append(buckets, &bucket{representative: c, members: []candidate{c}}) + } + } + + clusters := []Cluster{} + for _, b := range buckets { + sessions := map[string]bool{} + refs := make([]EventRef, 0, len(b.members)) + for _, member := range b.members { + sessions[member.ref.SessionID] = true + refs = append(refs, member.ref) + } + if len(b.members) < minOccurrences || len(sessions) < minSessions { + continue + } + names := make([]string, 0, len(sessions)) + for session := range sessions { + names = append(names, session) + } + sort.Strings(names) + exemplar := b.representative.text + if len(exemplar) > exemplarCap { + exemplar = exemplar[:exemplarCap] + "…" + } + clusters = append(clusters, Cluster{ + Exemplar: exemplar, + Normalized: b.representative.normalized, + Occurrences: len(b.members), + Sessions: names, + Evidence: refs, + }) + } + sort.Slice(clusters, func(i, j int) bool { + if clusters[i].Occurrences != clusters[j].Occurrences { + return clusters[i].Occurrences > clusters[j].Occurrences + } + return clusters[i].Normalized < clusters[j].Normalized + }) + return clusters +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/event.go b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/event.go new file mode 100644 index 000000000..5ac2f0c78 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/event.go @@ -0,0 +1,118 @@ +// Package retromine detects recurring operator instructions in coding-agent +// transcripts, offline and deterministically. It exists to serve one control +// law: a recurring operator instruction is steady-state error — evidence that +// the controller is missing a typed observation, verb, setpoint, or guard — +// and the remedy is promotion into a typed construct, never a saved prompt. +// This package is the SENSOR of that loop: it parses transcripts into neutral +// events, finds recurrence, and (in the classify layer) names the gap. It +// proposes; it never enforces, writes, or calls anything. +// +// Purity contract: no network, no subprocesses, no filesystem — inputs arrive +// as io.Readers, randomness and wall clocks are not used, and identical +// inputs in any order produce identical output. +// control-law: retro-derivation-is-offline-and-deterministic +package retromine + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "strings" +) + +// Role classifies who produced an event. Only operator events feed the +// recurrence detector: the law is about what the OPERATOR keeps having to +// say, never about what an agent generates. +const ( + RoleOperator = "operator" + RoleAgent = "agent" + RoleTool = "tool" +) + +// Event is the neutral transcript unit every adapter projects into. +// The schema is deliberately minimal: source (which adapter/file), a session +// identity (recurrence across sessions is the signal; within one session it +// is just conversation), an optional RFC3339 timestamp, a role, and the text. +type Event struct { + Source string `json:"source"` + SessionID string `json:"session_id"` + Timestamp string `json:"ts,omitempty"` + Role string `json:"role"` + Text string `json:"text"` + // Index is the event's position within its session, assigned by the + // parser from transcript order. It is intrinsic to the event — never to + // the order a caller happens to concatenate inputs in — which is what + // keeps detection order-independent. + Index int `json:"index"` +} + +// EventRef points back into the parsed input so every proposal is traceable +// to its evidence without embedding whole transcripts anywhere. +type EventRef struct { + SessionID string `json:"session_id"` + Index int `json:"index"` +} + +// ParseNeutralEvents reads the neutral JSONL format (one Event per line). +// A syntactically invalid line is a typed error naming its position — never +// a silent partial parse. Blank lines are permitted. +func ParseNeutralEvents(source string, r io.Reader) ([]Event, error) { + scanner := newLineScanner(r) + events := []Event{} + line := 0 + for scanner.Scan() { + line++ + raw := strings.TrimSpace(scanner.Text()) + if raw == "" { + continue + } + var event Event + if err := json.Unmarshal([]byte(raw), &event); err != nil { + return nil, fmt.Errorf("parse neutral events %s line %d: %w", source, line, err) + } + if event.Source == "" { + event.Source = source + } + if err := validateEvent(source, line, event); err != nil { + return nil, err + } + events = append(events, event) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read neutral events %s: %w", source, err) + } + return assignSessionIndexes(events), nil +} + +// assignSessionIndexes stamps each event's intrinsic per-session position in +// transcript order, overwriting whatever the input carried so the value is +// always the parser's, never a caller's claim. +func assignSessionIndexes(events []Event) []Event { + counters := map[string]int{} + for i := range events { + events[i].Index = counters[events[i].SessionID] + counters[events[i].SessionID]++ + } + return events +} + +func validateEvent(source string, line int, event Event) error { + switch event.Role { + case RoleOperator, RoleAgent, RoleTool: + default: + return fmt.Errorf("parse neutral events %s line %d: unknown role %q", source, line, event.Role) + } + if strings.TrimSpace(event.SessionID) == "" { + return fmt.Errorf("parse neutral events %s line %d: session_id is required", source, line) + } + return nil +} + +// newLineScanner returns a scanner sized for long transcript lines (tool +// results and pasted logs routinely exceed bufio's default token size). +func newLineScanner(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + return scanner +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/retromine_conformance_test.go b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/retromine_conformance_test.go new file mode 100644 index 000000000..448c1ca9c --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/retromine_conformance_test.go @@ -0,0 +1,218 @@ +package retromine + +// control-law: retro-derivation-is-offline-and-deterministic +// +// The recurrence miner is a pure sensor: no network, no subprocesses, no +// filesystem, no clocks, no randomness — inputs arrive as bytes and readers, +// and identical inputs in ANY order produce byte-identical output. The +// import-purity test enforces the capability boundary structurally, the +// determinism test enforces the output contract, and the adapter tests pin +// the lossy-projection rule: skip what you do not understand, error on what +// fails to parse. +// +// Test classes: positive (a planted instruction repeated 3× across 2 +// sessions clusters; both host adapters project equivalent content to +// equivalent events), negative (2 occurrences or 1 session never clusters; +// acknowledgements below the token floor never enter the pool), relation +// (input order does not change the report), failure-state (malformed JSONL +// is a typed error naming the line, never a silent partial parse), bypass +// (the package imports grant no I/O capability at all). + +import ( + "fmt" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func operatorEvent(session string, text string) Event { + return Event{Source: "test", SessionID: session, Role: RoleOperator, Text: text} +} + +func plantedEvents() []Event { + return []Event{ + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-a", "add a logout button to the settings page"), + {Source: "test", SessionID: "session-a", Role: RoleAgent, Text: "make the pr, monitor the checks, and merge when green"}, + operatorEvent("session-b", "please make the PR, monitor the checks and merge when green"), + operatorEvent("session-b", "ok"), + operatorEvent("session-c", "make the pr monitor the checks and merge when it is green"), + } +} + +// Positive: three occurrences across three sessions cluster; the agent's +// identical text and the sub-floor acknowledgement never enter the pool. +func TestRecurringInstructionClusters(t *testing.T) { + clusters := DetectRecurrence(plantedEvents()) + if len(clusters) != 1 { + t.Fatalf("clusters = %#v, want exactly one", clusters) + } + cluster := clusters[0] + if cluster.Occurrences != 3 { + t.Fatalf("occurrences = %d, want 3", cluster.Occurrences) + } + if !reflect.DeepEqual(cluster.Sessions, []string{"session-a", "session-b", "session-c"}) { + t.Fatalf("sessions = %v", cluster.Sessions) + } + if !strings.Contains(cluster.Exemplar, "monitor the checks") { + t.Fatalf("exemplar lost the instruction: %q", cluster.Exemplar) + } +} + +// Negative: two occurrences, or three inside one session, are not recurrence. +func TestBelowThresholdNeverClusters(t *testing.T) { + twoOccurrences := []Event{ + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-b", "make the pr, monitor the checks, and merge when green"), + } + if clusters := DetectRecurrence(twoOccurrences); len(clusters) != 0 { + t.Fatalf("two occurrences clustered: %#v", clusters) + } + oneSession := []Event{ + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + operatorEvent("session-a", "make the pr, monitor the checks, and merge when green"), + } + if clusters := DetectRecurrence(oneSession); len(clusters) != 0 { + t.Fatalf("single-session repetition clustered: %#v", clusters) + } +} + +// Relation: input order does not change the report. +func TestDetectionIsOrderIndependent(t *testing.T) { + events := plantedEvents() + reversed := make([]Event, 0, len(events)) + for i := len(events) - 1; i >= 0; i-- { + reversed = append(reversed, events[i]) + } + forward := DetectRecurrence(events) + backward := DetectRecurrence(reversed) + if !reflect.DeepEqual(forward, backward) { + t.Fatalf("order changed the report:\n%#v\n---\n%#v", forward, backward) + } +} + +// Positive: normalization strips pasted logs, so the same instruction with a +// different fenced payload is the same instruction shape. +func TestFencedPayloadDoesNotSplitClusters(t *testing.T) { + events := []Event{ + operatorEvent("s1", "fix the failing windows shard\n```\nlog A\n```"), + operatorEvent("s2", "fix the failing windows shard\n```\ncompletely different log B\n```"), + operatorEvent("s3", "fix the failing windows shard please"), + } + if clusters := DetectRecurrence(events); len(clusters) != 1 { + t.Fatalf("payload variance split the cluster: %#v", clusters) + } +} + +// Positive + relation: the two host adapters project equivalent content into +// equivalent instruction streams — the miner is agent-agnostic by contract. +func TestAdaptersProjectEquivalentContent(t *testing.T) { + claudeLines := strings.Join([]string{ + `{"type":"user","sessionId":"cc-1","message":{"role":"user","content":"make the pr, monitor the checks, and merge when green"}}`, + `{"type":"assistant","sessionId":"cc-1","message":{"role":"assistant","content":[{"type":"text","text":"On it."}]}}`, + `{"type":"user","sessionId":"cc-1","message":{"role":"user","content":[{"type":"tool_result","content":"exit 0"}]}}`, + `{"type":"summary","summary":"irrelevant"}`, + }, "\n") + fromClaude, err := ParseTranscript("", "cc-1.jsonl", []byte(claudeLines)) + if err != nil { + t.Fatal(err) + } + plain := "User: make the pr, monitor the checks, and merge when green\nAgent: On it.\n" + fromPlain, err := ParseTranscript("", "plain.txt", []byte(plain)) + if err != nil { + t.Fatal(err) + } + pick := func(events []Event) []string { + out := []string{} + for _, e := range events { + if e.Role == RoleOperator { + out = append(out, normalizeInstruction(e.Text)) + } + } + return out + } + if !reflect.DeepEqual(pick(fromClaude), pick(fromPlain)) { + t.Fatalf("adapters disagree:\n%v\n---\n%v", pick(fromClaude), pick(fromPlain)) + } + // The tool result projected as tool, never operator. + for _, e := range fromClaude { + if e.Role == RoleOperator && strings.Contains(e.Text, "exit 0") { + t.Fatalf("tool payload classified as operator: %#v", e) + } + } +} + +// Failure-state: malformed JSONL is a typed error naming the line — never a +// silent partial parse. +func TestMalformedLinesAreTypedErrors(t *testing.T) { + if _, err := ParseTranscript(FormatNeutral, "bad.jsonl", []byte(`{"role":"operator","session_id":"s","text":"x"}`+"\nnot json\n")); err == nil || !strings.Contains(err.Error(), "line 2") { + t.Fatalf("malformed neutral line not surfaced: %v", err) + } + if _, err := ParseTranscript(FormatClaudeCode, "bad.jsonl", []byte("{broken\n")); err == nil || !strings.Contains(err.Error(), "line 1") { + t.Fatalf("malformed claudecode line not surfaced: %v", err) + } + if _, err := ParseTranscript(FormatNeutral, "bad.jsonl", []byte(`{"role":"wizard","session_id":"s","text":"x"}`+"\n")); err == nil || !strings.Contains(err.Error(), "unknown role") { + t.Fatalf("unknown role accepted: %v", err) + } +} + +// Bypass: the package's non-test imports grant no I/O capability — no +// network, no subprocesses, no filesystem, no clocks, no randomness. The +// boundary is structural, not behavioral. +func TestPackageImportsGrantNoCapabilities(t *testing.T) { + disallowed := []string{"net", "os", "syscall", "time", "math/rand", "crypto/rand", "path/filepath", "io/ioutil"} + fset := token.NewFileSet() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + name := entry.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, filepath.Join(".", name), nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, spec := range file.Imports { + path := strings.Trim(spec.Path.Value, `"`) + for _, banned := range disallowed { + if path == banned || strings.HasPrefix(path, banned+"/") { + t.Fatalf("%s imports %s — the miner must stay capability-free", name, path) + } + } + } + } +} + +// Golden determinism over the synthetic fixtures: parse both fixture +// transcripts, mine them together, and pin the whole report. +func TestFixtureGoldenReport(t *testing.T) { + events := []Event{} + for _, fixture := range []string{"session-alpha.jsonl", "session-beta.txt"} { + content, err := os.ReadFile(filepath.Join("testdata", fixture)) + if err != nil { + t.Fatal(err) + } + parsed, err := ParseTranscript("", fixture, content) + if err != nil { + t.Fatal(err) + } + events = append(events, parsed...) + } + clusters := DetectRecurrence(events) + if len(clusters) != 1 { + t.Fatalf("fixture clusters = %#v", clusters) + } + got := fmt.Sprintf("%dx across %v: %s", clusters[0].Occurrences, clusters[0].Sessions, clusters[0].Normalized) + want := "3x across [cc-alpha session-beta.txt]: open the pr watch ci until every check passes then merge it" + if got != want { + t.Fatalf("golden drift:\n got %q\nwant %q", got, want) + } +} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-alpha.jsonl b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-alpha.jsonl new file mode 100644 index 000000000..8c754c599 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-alpha.jsonl @@ -0,0 +1,5 @@ +{"type":"user","sessionId":"cc-alpha","message":{"role":"user","content":"open the PR, watch CI until every check passes, then merge it"}} +{"type":"assistant","sessionId":"cc-alpha","message":{"role":"assistant","content":[{"type":"text","text":"Opening the PR now."}]}} +{"type":"user","sessionId":"cc-alpha","message":{"role":"user","content":"open the pr, watch ci until every check passes, then merge it."}} +{"type":"user","sessionId":"cc-alpha","message":{"role":"user","content":"also rename the config key while you are at it"}} +{"type":"summary","summary":"synthetic fixture"} diff --git a/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-beta.txt b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-beta.txt new file mode 100644 index 000000000..da22b82d4 --- /dev/null +++ b/labs/12-product-engineering-loop/product-engineering-loop/internal/retromine/testdata/session-beta.txt @@ -0,0 +1,3 @@ +User: open the PR — watch CI until every check passes, then merge it +Agent: Working on it. +User: thanks