fix(crush): JSON keeps structural errors, not the bare word "error"

crush/json.go

@@ -31,6 +31,14 @@ type JSONCrusher struct {
 	MinItems int
 }
 
+// jsonMinKeep / jsonMaxKeep bound the adaptive keep budget for the distinct
+// (non-floor) middle items: never fewer than jsonMinKeep representatives, never
+// an unbounded tail. adaptiveKeepCount scales between them by diversity.
+const (
+	jsonMinKeep = 3
+	jsonMaxKeep = 40
+)
+
 // NewJSONCrusher returns a JSONCrusher with sensible defaults.
 func NewJSONCrusher() *JSONCrusher {
 	return &JSONCrusher{HeadKeep: 5, TailKeep: 3, MinItems: 8}
@@ -52,23 +60,78 @@ func (c *JSONCrusher) Compress(req Request) (Result, error) {
 
 	terms := queryTerms(req.Query)
 	keep := make([]bool, len(items))
-	seen := make(map[string]bool, len(items))
+
+	// Signature-dedup + adaptive scored budget (headroom-parity): a large array
+	// of near-identical rows (search hits, log lines differing only by ids/line
+	// numbers) collapses to a representative sample, instead of the old rule that
+	// kept every row merely MENTIONING "error" (which pinned entire arrays of Go
+	// signatures like `func H(ctx context.Context) error`).
+	// Collapse near-duplicate rows to ONE representative per signature, keeping
+	// the highest-scoring member of each group so a query-relevant / soft-error
+	// row is never lost to a plainer sibling that shares its shape.
+	type softItem struct {
+		idx   int
+		text  string
+		score int
+	}
+	sigOf := map[string]int{} // signature -> index into reps
+	var reps []softItem
+
+	score := func(s string) int {
+		sc := 0
+		if matchesAny(s, terms) {
+			sc += 2 // query relevance
+		}
+		if looksError(s) { // broad mention → soft boost only, not an absolute keep
+			sc++
+		}
+		if looksFragile(s) {
+			sc++
+		}
+		return sc
+	}
 
 	for i, raw := range items {
-		// Always keep head and tail.
+		// Always keep head and tail (callers care about boundaries).
 		if i < c.HeadKeep || i >= len(items)-c.TailKeep {
 			keep[i] = true
 			continue
 		}
 		s := strings.ToLower(string(raw))
-		// Drop exact duplicates of content already kept or seen.
-		if seen[s] {
+		// Absolute floor — never dropped: caller must-keeps and STRUCTURAL errors
+		// (log levels, stack traces, structured level fields). NOT the bare word
+		// "error". looksHardError takes the ORIGINAL text (uppercase is a signal).
+		if mustKeep(s, req.MustKeep) || looksHardError(string(raw)) {
+			keep[i] = true
 			continue
 		}
-		seen[s] = true
-		// Must-keep: error-like, builder-flagged, or query-relevant items.
-		if looksError(s) || mustKeep(s, req.MustKeep) || matchesAny(s, terms) {
-			keep[i] = true
+		sc := score(s)
+		sig := lineSig(s)
+		if j, ok := sigOf[sig]; ok {
+			if sc > reps[j].score { // better representative of this shape
+				reps[j] = softItem{idx: i, text: string(raw), score: sc}
+			}
+			continue // sibling of an already-tracked group → offload
+		}
+		sigOf[sig] = len(reps)
+		reps = append(reps, softItem{idx: i, text: string(raw), score: sc})
+	}
+
+	// Keep the top-K representatives: K adapts to their diversity (redundant →
+	// few, diverse → more) and is capped so a huge array can't retain an
+	// unbounded tail. Stable sort keeps first-seen order within equal scores.
+	if len(reps) > 0 {
+		strs := make([]string, len(reps))
+		for j, it := range reps {
+			strs[j] = it.text
+		}
+		k := adaptiveKeepCount(strs, jsonMinKeep, jsonMaxKeep)
+		if k > len(reps) {
+			k = len(reps)
+		}
+		sort.SliceStable(reps, func(a, b int) bool { return reps[a].score > reps[b].score })
+		for j := 0; j < k; j++ {
+			keep[reps[j].idx] = true
 		}
 	}
 

crush/json_regression_test.go

@@ -0,0 +1,56 @@
+package crush
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/initializ/ctxzip/ccr"
+)
+
+// The bug: every row contains the word "error" (a Go return type), so the old
+// looksError substring keep pinned all 400 rows → 0% compression.
+func TestJSONCrusher_CodeSignaturesCompress(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("[")
+	for i := 0; i < 400; i++ {
+		if i > 0 {
+			b.WriteByte(',')
+		}
+		fmt.Fprintf(&b, `{"file":"src/pkg/module_%d.go","line":%d,"match":"func Handle%d(ctx context.Context) error"}`, i%20, i, i)
+	}
+	b.WriteString("]")
+	in := b.String()
+	res, _ := NewJSONCrusher().Compress(Request{Content: in, Store: ccr.NewMemoryStore(ccr.MemoryConfig{})})
+	if len(res.Markers) == 0 {
+		t.Fatalf("expected compression, got passthrough (%d bytes)", len(res.Compressed))
+	}
+	if len(res.Compressed) >= len(in) {
+		t.Fatalf("did not shrink: %d -> %d", len(in), len(res.Compressed))
+	}
+	t.Logf("compressed %d -> %d bytes", len(in), len(res.Compressed))
+}
+
+// A genuine structural error (FATAL level) must still survive verbatim.
+func TestJSONCrusher_RealErrorRowKept(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("[")
+	for i := 0; i < 50; i++ {
+		if i > 0 {
+			b.WriteByte(',')
+		}
+		if i == 30 {
+			b.WriteString(`{"level":"FATAL","msg":"db pool exhausted"}`)
+			continue
+		}
+		fmt.Fprintf(&b, `{"level":"INFO","msg":"ok %d"}`, i)
+	}
+	b.WriteString("]")
+	res, _ := NewJSONCrusher().Compress(Request{Content: b.String(), Store: ccr.NewMemoryStore(ccr.MemoryConfig{})})
+	if !strings.Contains(res.Compressed, "FATAL") {
+		t.Fatalf("FATAL row was dropped: %s", res.Compressed)
+	}
+	if len(res.Markers) == 0 {
+		t.Fatalf("expected the 49 INFO rows to collapse")
+	}
+}

crush/relevance.go

@@ -52,6 +52,34 @@ func looksError(s string) bool {
 	return false
 }
 
+// hardErrorRe matches STRUCTURAL error signals — log-level tokens, stack
+// traces, structured level fields, compiler error codes, k8s failure states —
+// as opposed to the mere mention of the word "error" (a Go `error` return type,
+// a line saying "0 errors"). looksError stays deliberately broad and is used as
+// a soft ranking boost; looksHardError is the tighter test used where an item
+// must be kept ABSOLUTELY, so a uniform array that all merely mentions "error"
+// can still compress instead of pinning every row. Case-sensitive by design:
+// loggers emit uppercase levels, and callers pass the ORIGINAL (not lowercased)
+// text.
+var hardErrorRe = regexp.MustCompile(strings.Join([]string{
+	// Uppercase log levels + Kubernetes failure states (word-bounded).
+	`\b(?:ERROR|FATAL|FAILED|FAILURE|PANIC|CRITICAL|EXCEPTION|SIGSEGV|SIGABRT|OOMKilled|CrashLoopBackOff|ImagePullBackOff|Evicted)\b`,
+	// Anchored lowercase: "error:", "panic:", "error[E0433]", "traceback:".
+	`(?i:\b(?:error|fatal|panic|exception|traceback|failed|failure|denied|rejected)\b\s*[:\[])`,
+	// Line-start error/level keyword.
+	`(?im:^\s*(?:error|fatal|panic|exception|traceback)\b)`,
+	// Structured level fields: {"level":"error"} / level=fatal.
+	`(?i:"level"\s*:\s*"(?:error|fatal|critical|panic)")`,
+	`(?i:\blevel=(?:error|fatal|critical|panic)\b)`,
+}, "|"))
+
+// looksHardError reports whether s carries a structural error signal (see
+// hardErrorRe). Pass the ORIGINAL text, not a lowercased copy — uppercase log
+// levels are a deliberate signal.
+func looksHardError(s string) bool {
+	return hardErrorRe.MatchString(s)
+}
+
 // IsErrorLike reports whether s matches the built-in error floor — the terms
 // compression never drops. Exported for feedback loops in host runtimes:
 // a token already on the floor was KEPT, so it cannot be the reason a model