diff --git a/crush/json.go b/crush/json.go index 72bb80f..90c510d 100644 --- a/crush/json.go +++ b/crush/json.go @@ -31,6 +31,14 @@ type JSONCrusher struct { MinItems int } +// jsonMinKeep / jsonMaxKeep bound the adaptive keep budget for the distinct +// (non-floor) middle items: never fewer than jsonMinKeep representatives, never +// an unbounded tail. adaptiveKeepCount scales between them by diversity. +const ( + jsonMinKeep = 3 + jsonMaxKeep = 40 +) + // NewJSONCrusher returns a JSONCrusher with sensible defaults. func NewJSONCrusher() *JSONCrusher { return &JSONCrusher{HeadKeep: 5, TailKeep: 3, MinItems: 8} @@ -52,23 +60,78 @@ func (c *JSONCrusher) Compress(req Request) (Result, error) { terms := queryTerms(req.Query) keep := make([]bool, len(items)) - seen := make(map[string]bool, len(items)) + + // Signature-dedup + adaptive scored budget (headroom-parity): a large array + // of near-identical rows (search hits, log lines differing only by ids/line + // numbers) collapses to a representative sample, instead of the old rule that + // kept every row merely MENTIONING "error" (which pinned entire arrays of Go + // signatures like `func H(ctx context.Context) error`). + // Collapse near-duplicate rows to ONE representative per signature, keeping + // the highest-scoring member of each group so a query-relevant / soft-error + // row is never lost to a plainer sibling that shares its shape. + type softItem struct { + idx int + text string + score int + } + sigOf := map[string]int{} // signature -> index into reps + var reps []softItem + + score := func(s string) int { + sc := 0 + if matchesAny(s, terms) { + sc += 2 // query relevance + } + if looksError(s) { // broad mention → soft boost only, not an absolute keep + sc++ + } + if looksFragile(s) { + sc++ + } + return sc + } for i, raw := range items { - // Always keep head and tail. + // Always keep head and tail (callers care about boundaries). if i < c.HeadKeep || i >= len(items)-c.TailKeep { keep[i] = true continue } s := strings.ToLower(string(raw)) - // Drop exact duplicates of content already kept or seen. - if seen[s] { + // Absolute floor — never dropped: caller must-keeps and STRUCTURAL errors + // (log levels, stack traces, structured level fields). NOT the bare word + // "error". looksHardError takes the ORIGINAL text (uppercase is a signal). + if mustKeep(s, req.MustKeep) || looksHardError(string(raw)) { + keep[i] = true continue } - seen[s] = true - // Must-keep: error-like, builder-flagged, or query-relevant items. - if looksError(s) || mustKeep(s, req.MustKeep) || matchesAny(s, terms) { - keep[i] = true + sc := score(s) + sig := lineSig(s) + if j, ok := sigOf[sig]; ok { + if sc > reps[j].score { // better representative of this shape + reps[j] = softItem{idx: i, text: string(raw), score: sc} + } + continue // sibling of an already-tracked group → offload + } + sigOf[sig] = len(reps) + reps = append(reps, softItem{idx: i, text: string(raw), score: sc}) + } + + // Keep the top-K representatives: K adapts to their diversity (redundant → + // few, diverse → more) and is capped so a huge array can't retain an + // unbounded tail. Stable sort keeps first-seen order within equal scores. + if len(reps) > 0 { + strs := make([]string, len(reps)) + for j, it := range reps { + strs[j] = it.text + } + k := adaptiveKeepCount(strs, jsonMinKeep, jsonMaxKeep) + if k > len(reps) { + k = len(reps) + } + sort.SliceStable(reps, func(a, b int) bool { return reps[a].score > reps[b].score }) + for j := 0; j < k; j++ { + keep[reps[j].idx] = true } } diff --git a/crush/json_regression_test.go b/crush/json_regression_test.go new file mode 100644 index 0000000..b2b23f7 --- /dev/null +++ b/crush/json_regression_test.go @@ -0,0 +1,56 @@ +package crush + +import ( + "fmt" + "strings" + "testing" + + "github.com/initializ/ctxzip/ccr" +) + +// The bug: every row contains the word "error" (a Go return type), so the old +// looksError substring keep pinned all 400 rows → 0% compression. +func TestJSONCrusher_CodeSignaturesCompress(t *testing.T) { + var b strings.Builder + b.WriteString("[") + for i := 0; i < 400; i++ { + if i > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `{"file":"src/pkg/module_%d.go","line":%d,"match":"func Handle%d(ctx context.Context) error"}`, i%20, i, i) + } + b.WriteString("]") + in := b.String() + res, _ := NewJSONCrusher().Compress(Request{Content: in, Store: ccr.NewMemoryStore(ccr.MemoryConfig{})}) + if len(res.Markers) == 0 { + t.Fatalf("expected compression, got passthrough (%d bytes)", len(res.Compressed)) + } + if len(res.Compressed) >= len(in) { + t.Fatalf("did not shrink: %d -> %d", len(in), len(res.Compressed)) + } + t.Logf("compressed %d -> %d bytes", len(in), len(res.Compressed)) +} + +// A genuine structural error (FATAL level) must still survive verbatim. +func TestJSONCrusher_RealErrorRowKept(t *testing.T) { + var b strings.Builder + b.WriteString("[") + for i := 0; i < 50; i++ { + if i > 0 { + b.WriteByte(',') + } + if i == 30 { + b.WriteString(`{"level":"FATAL","msg":"db pool exhausted"}`) + continue + } + fmt.Fprintf(&b, `{"level":"INFO","msg":"ok %d"}`, i) + } + b.WriteString("]") + res, _ := NewJSONCrusher().Compress(Request{Content: b.String(), Store: ccr.NewMemoryStore(ccr.MemoryConfig{})}) + if !strings.Contains(res.Compressed, "FATAL") { + t.Fatalf("FATAL row was dropped: %s", res.Compressed) + } + if len(res.Markers) == 0 { + t.Fatalf("expected the 49 INFO rows to collapse") + } +} diff --git a/crush/relevance.go b/crush/relevance.go index a8ed8bd..3851171 100644 --- a/crush/relevance.go +++ b/crush/relevance.go @@ -52,6 +52,34 @@ func looksError(s string) bool { return false } +// hardErrorRe matches STRUCTURAL error signals — log-level tokens, stack +// traces, structured level fields, compiler error codes, k8s failure states — +// as opposed to the mere mention of the word "error" (a Go `error` return type, +// a line saying "0 errors"). looksError stays deliberately broad and is used as +// a soft ranking boost; looksHardError is the tighter test used where an item +// must be kept ABSOLUTELY, so a uniform array that all merely mentions "error" +// can still compress instead of pinning every row. Case-sensitive by design: +// loggers emit uppercase levels, and callers pass the ORIGINAL (not lowercased) +// text. +var hardErrorRe = regexp.MustCompile(strings.Join([]string{ + // Uppercase log levels + Kubernetes failure states (word-bounded). + `\b(?:ERROR|FATAL|FAILED|FAILURE|PANIC|CRITICAL|EXCEPTION|SIGSEGV|SIGABRT|OOMKilled|CrashLoopBackOff|ImagePullBackOff|Evicted)\b`, + // Anchored lowercase: "error:", "panic:", "error[E0433]", "traceback:". + `(?i:\b(?:error|fatal|panic|exception|traceback|failed|failure|denied|rejected)\b\s*[:\[])`, + // Line-start error/level keyword. + `(?im:^\s*(?:error|fatal|panic|exception|traceback)\b)`, + // Structured level fields: {"level":"error"} / level=fatal. + `(?i:"level"\s*:\s*"(?:error|fatal|critical|panic)")`, + `(?i:\blevel=(?:error|fatal|critical|panic)\b)`, +}, "|")) + +// looksHardError reports whether s carries a structural error signal (see +// hardErrorRe). Pass the ORIGINAL text, not a lowercased copy — uppercase log +// levels are a deliberate signal. +func looksHardError(s string) bool { + return hardErrorRe.MatchString(s) +} + // IsErrorLike reports whether s matches the built-in error floor — the terms // compression never drops. Exported for feedback loops in host runtimes: // a token already on the floor was KEPT, so it cannot be the reason a model