From 46c5d441baa898c4875d4b3f3b0af71102869fc9 Mon Sep 17 00:00:00 2001 From: yusing Date: Wed, 9 Sep 2026 04:21:28 +0000 Subject: [PATCH 01/13] feat(router): add provider-free context compaction Handle Responses compaction locally without provider summaries, using conservative history reducers and encrypted router-owned envelopes. Restore native history across compaction, truncation, restart, and fresh context while preserving provider-owned state. Add HTTP, envelope, reducer, restoration, and Codex compatibility tests, and document the compaction contract and ownership boundary. --- AGENTS.md | 2 + README.md | 20 +- doc/architecture/compaction.md | 32 ++ doc/architecture/index.md | 2 + doc/spec/compaction.md | 69 ++++ doc/spec/ctp.md | 5 +- doc/spec/index.md | 2 + internal/router/context_compaction.go | 246 ++++++++++++ .../router/context_compaction_codex_test.go | 191 +++++++++ .../router/context_compaction_envelope.go | 156 ++++++++ .../context_compaction_envelope_test.go | 83 ++++ internal/router/context_compaction_http.go | 377 ++++++++++++++++++ .../router/context_compaction_http_test.go | 283 +++++++++++++ internal/router/context_compaction_test.go | 150 +++++++ internal/router/server.go | 11 +- 15 files changed, 1624 insertions(+), 5 deletions(-) create mode 100644 doc/architecture/compaction.md create mode 100644 doc/spec/compaction.md create mode 100644 internal/router/context_compaction.go create mode 100644 internal/router/context_compaction_codex_test.go create mode 100644 internal/router/context_compaction_envelope.go create mode 100644 internal/router/context_compaction_envelope_test.go create mode 100644 internal/router/context_compaction_http.go create mode 100644 internal/router/context_compaction_http_test.go create mode 100644 internal/router/context_compaction_test.go diff --git a/AGENTS.md b/AGENTS.md index 1dd8c12f..ef0946a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ When parts of this file is stale after your work, update this file. | Portable verified-row, source-capability, Go-lexical, and shell-header semantics | `internal/verifiedrow`, `internal/sourcekind`, `internal/golex`, `internal/shellsyntax` | | Versioned plugin shared-core adapter and private WASM bridge | `internal/router/toolplugin/core-v1.mjs`, `internal/router/toolplugin/core-v1.d.ts`, `internal/sharedwasm` | | Router lifecycle, launch flags, modes, and HTTP endpoints | `internal/router/server.go`, `internal/router/flags.go` | +| Provider-free context pruning, local envelopes, and native-history restoration | `internal/router/context_compaction*.go` | | Third-party native subagent projection, Grok authentication/translation, and model-catalog metadata | `internal/router/subagent_bridge.go`, `internal/router/grok_*.go` | | Codex-facing WebSocket sessions, incremental history, and steering | `internal/router/server_websocket.go` | | Codex authentication and upstream Responses transport | `internal/router/client.go`, `internal/router/client_websocket.go` | @@ -47,6 +48,7 @@ When parts of this file is stale after your work, update this file. | --- | --- | | Root engine | `go test .` | | Router request, response, recovery, workspace, plugin, or transport | `go test ./internal/router` | +| Context compaction client compatibility | `HPATCH_COMPACTION_CODEX_BIN="$(command -v codex)" go test ./internal/router -run '^TestCompactionInstalledCodex$'` (isolated loopback fixtures, no provider inference) | | Portable core or `mekugi:core/v1` adapter | `go generate ./internal/router/toolplugin`, then `go test ./...` and `bun test ./internal/router/toolplugin/tests/core.test.ts` | | TypeScript plugin source | `go generate ./internal/router/toolplugin`, then `bun test ./internal/router/toolplugin/tests` | | Router or shell-helper process entry point | `go test ./cmd/mekugi ./cmd/shell` | diff --git a/README.md b/README.md index 86519c5b..7f0cd64e 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,19 @@ go install github.com/yusing/mekugi/cmd/mekugi@latest \ Add `$GOBIN`, or `$(go env GOPATH)/bin` when unset, to the `PATH` used by both Mekugi and Codex. The fixed `shell` helper must be available to Codex's executor. +Context compaction is handled locally, without a provider-generated summary. +Codex still decides when to compact using your settings. Hpatch preserves the +retained native history in an encrypted item and restores it before the next +provider request. The first compaction creates an owner-only key at +`$XDG_CONFIG_HOME/hpatch/compaction.key` (normally +`~/.config/hpatch/compaction.key` on Linux). Keep that key to resume compacted +sessions, including when moving them to another installation. + +The initial reducers handle only recognized search listings and verbose Go test +results. Uncertain evidence is retained; when nothing can safely be reduced, +compaction reports an error instead of dropping context or asking a provider +for a summary. The retained history is not guaranteed to fit a 30k-token window. + Then launch: ```sh @@ -179,6 +192,9 @@ HTTP/SSE clients and can fall back to HTTP for those requests when ChatGPT explicitly rejects the WebSocket upgrade. It never silently replays a dropped request or accepted steering. Grok provider requests remain on HTTP. +See the [context-compaction contract](doc/spec/compaction.md) for supported forms +and preservation behavior. No compaction threshold or scope is overridden. + ### Options | Flag | Default | Purpose | @@ -194,14 +210,14 @@ request or accepted steering. Grok provider requests remain on HTTP. | `--metrics-output PATH` | Disabled | Write the final metrics snapshot on shutdown, overwriting the destination | | `--debug` | Disabled | Record diagnostics, capture, metrics, patched instructions, runtime reads, and an AX report; print all artifact paths on exit | -For a transport-only session: +To disable Hpatch tool and model-string transformations: ```sh mekugi --mode passthrough codex ``` Passthrough does not load the plugin registry, so it does not require Node.js or -plugin grammar validation. Capture remains available. +plugin grammar validation. Local context compaction and capture remain available. ### Grok subagents diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md new file mode 100644 index 00000000..ae13458d --- /dev/null +++ b/doc/architecture/compaction.md @@ -0,0 +1,32 @@ +# Context-compaction boundary + +## CTR-COMPACTION-001 — Router-owned pruning and local envelopes + +The router's context-compaction HTTP boundary precedes ordinary Responses +projection and upstream transport. It owns local standalone and V2 completion, +restoration of its own input envelopes, and bounded request decoding. Codex owns +configuration resolution, trigger timing, retained client-side context, and +installation of the returned compaction result. + +Command-aware reducers own only evidence selection. They do not execute commands, +interpret opaque provider state, infer task completion, or choose a fixed context +budget. Unknown forms retain their evidence. + +The envelope owner authenticates and encrypts the retained native item array. +Its persistent key belongs to the Hpatch configuration directory, not a thread, +temporary plugin runtime, provider credential, or capture stream. Cross-process +locking serializes first creation. Compression is an internal envelope-storage +detail, not the semantic compaction mechanism or a token-usage measurement. + +Restoration aligns the client's carried prefix against the retained native +timeline backwards by stable item/call identity or canonical JSON. Supported client truncation +must match a unique authenticated original. Fresh canonical context is never +content-deduplicated into an older instruction, and unmatched current context +is retained in relative order; the post-envelope suffix is appended unchanged. +Restored items enter the existing Hpatch and CTP boundaries as native history. +Neither those boundaries nor the provider interpret router-owned ciphertext. + +No provider transport is available to the local compaction handler. Errors leave +Codex without a replacement window rather than silently losing context or issuing +a model summary. Capture observes local HTTP traffic but local compaction does not +fabricate provider token usage. diff --git a/doc/architecture/index.md b/doc/architecture/index.md index 95e427d7..6138fa83 100644 --- a/doc/architecture/index.md +++ b/doc/architecture/index.md @@ -9,6 +9,7 @@ pjdoc: - subagents.md - commentary.md - mentor.md + - compaction.md - ctp.md - syntax.md - core.md @@ -30,6 +31,7 @@ Each listed file owns one ownership contract. Related facts are cited by stable - [`CTR-SUBAGENTS-001`](subagents.md): router-owned third-party provider bridge - [`CTR-COMMENTARY-001`](commentary.md): router-owned subagent commentary projection - [`CTR-MENTOR-001`](mentor.md): router-owned subagent model schedule +- [`CTR-COMPACTION-001`](compaction.md): router-owned pruning and local envelopes - [`CTR-CTP-001`](ctp.md): router-owned compact provider representation - [`CTR-SYNTAX-001`](syntax.md): shared compact-script framing - [`CTR-CORE-001`](core.md): virtual workspace and immutable-baseline edit planning diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md new file mode 100644 index 00000000..8199c82f --- /dev/null +++ b/doc/spec/compaction.md @@ -0,0 +1,69 @@ +# Provider-free context compaction + +## REQ-COMPACTION-001 — Router-owned context pruning + +Codex owns when to compact: its effective model context window, automatic +compaction threshold and counting scope, model changes, and manual compaction +requests remain unchanged. Hpatch does not rewrite Codex settings or schedule +an earlier trigger. + +The router handles `POST /v1/responses/compact` locally. It also handles streaming +`POST /v1/responses` requests identified by Codex metadata as +`responses_compaction_v2`. Neither path calls a provider. Other metadata-tagged +compaction implementations fail explicitly instead of requesting a provider +summary. Local handling applies in both router modes. + +Compaction preserves all history items and their order. Its initial reducers +only change completed historical tool output: + +- Successful Go test results with a structured result or native Codex exec header can omit routine run/pause/continue and + pass lines. The exact call, exit status, package summaries, and other output + remain. +- Successful search listings from recognized `rg`, `hgrep`, or `find` calls can + be replaced with a reference to a later retained read result containing the + exact same complete listing. The original call and completion metadata stay. +- The last tool result, failed or unfinished operations, live handles, unknown + output formats, unsupported wrappers, compound commands, pipelines, dynamic + shell expressions, and unrecognized commands remain unchanged. +- User corrections, commentary, reasoning, existing provider-encrypted items, + decisions, identifiers, unknown fields, and current work are not blanket-pruned. + +If no safe reduction is available, compaction fails with HTTP 422. It does not +discard protected context just to fit a budget, report a fabricated summary, +or fall back to provider compaction. The proposed 30k-token output target and +5k allowance remain an evaluation hypothesis, not an enforced limit or a proven +preservation guarantee. Encoded size is not the restored model context size. + +The retained native timeline travels inline in an authenticated, encrypted +router-owned compaction item. Legacy output also carries original user messages +for Codex's own user-input handling. V2 emits exactly one completed compaction +item. On subsequent requests, the router restores the timeline before any +projection or forwarding. It reconciles carried native items without duplicating +matched messages, preserves newly injected context and post-compaction input, +and restores full items when Codex retained truncated versions. No-ID truncations +must uniquely match the authenticated original; ambiguous matches fail explicitly. +Unreconciled carried user content, including unsupported multimodal truncation, +also fails rather than becoming a duplicated request. +Fresh canonical instructions retain their current position even when their text +matches an older instruction. + +The versioned local payload is never sent upstream as provider-encrypted state. +Existing provider-owned compaction and reasoning payloads remain untouched. +Malformed, unknown-version, nested, unauthenticated, or unreadable local envelopes +fail closed. Both encoded requests and restored history obey router memory bounds. + +The installation-owned key is `compaction.key` in Hpatch's configuration directory. +It is created lazily with owner-only permissions, shared safely across simultaneous +routers, and retained across process exits. Resuming on another installation needs +the same key. The original key is never silently replaced. There is no transcript +archive, session cache requirement, or model-operated retrieval step. + +Acceptance checks cover local HTTP completion without provider calls, native +restoration with fresh context and suffixes, repeated compaction, restart and +concurrent key creation, damaged or missing keys, and conservative output pruning. +Installed Codex 0.153.4 has passed loopback legacy and V2 automatic-compaction +round trips with both counting scopes and a large user request that is truncated +by the client's V2 retention step, then restored in full without duplication. +Other versions and manual/resumed client flows need corresponding +runtime coverage. Paired outcome evaluation is still required before claiming +an optimal output budget or task-critical semantic preservation on real histories. diff --git a/doc/spec/ctp.md b/doc/spec/ctp.md index f8c7baeb..90c187c1 100644 --- a/doc/spec/ctp.md +++ b/doc/spec/ctp.md @@ -10,8 +10,9 @@ model protocol fail before the router listens. Passthrough mode uses native and CTP/2 is a reversible representation between the ordinary Mekugi request projection and the model provider. It is not another Responses protocol or an edit-engine feature. Responses objects, roles, instruction priority, identifiers, statuses, reasoning, schemas, grammar definitions, streaming, -usage, conversation selection, and compaction remain provider-owned and native. CTP/2 changes only -eligible model-visible request strings and assistant text. Newly emitted tool names, tool inputs, +and usage remain native and outside CTP/2 ownership. Context pruning and router-owned compaction +items belong to `REQ-COMPACTION-001`. CTP/2 changes only eligible model-visible request strings and +assistant text. Newly emitted tool names, tool inputs, and function arguments remain native. `contrib/codex/file-editing-instructions.md` owns the model-visible interpretation and emission diff --git a/doc/spec/index.md b/doc/spec/index.md index e4288fa5..0af5f820 100644 --- a/doc/spec/index.md +++ b/doc/spec/index.md @@ -29,6 +29,7 @@ pjdoc: - guide.md - comparison.md - benchmark.md + - compaction.md - ctp.md --- # mekugi specification @@ -60,6 +61,7 @@ Each listed file owns one requirement. Related facts are cited by stable ID or l - [`REQ-GUIDE-001`](guide.md): concise agent guidance - [`REQ-COMPARE-001`](comparison.md): token comparison scenarios - [`REQ-BENCH-001`](benchmark.md): historical-commit correctness and paired model evaluation +- [`REQ-COMPACTION-001`](compaction.md): provider-free context pruning and native-history restoration - [`REQ-CTP-001`](ctp.md): lossless token-positive model-visible data-plane encoding All listed requirements are must-haves for this increment. diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go new file mode 100644 index 00000000..1e5dfed0 --- /dev/null +++ b/internal/router/context_compaction.go @@ -0,0 +1,246 @@ +package router + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strings" + + "mvdan.cc/sh/v3/expand" + "mvdan.cc/sh/v3/syntax" +) + +// reduceContextCompaction changes only recognized, completed historical tool +// output. Requests, decisions, opaque state, calls, order, and the last result +// remain intact. It makes no claim that the surviving history fits a token cap. +func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { + output := slices.Clone(input) + type item struct { + Type string `json:"type"` + Name string `json:"name"` + CallID string `json:"call_id"` + Arguments string `json:"arguments"` + Input string `json:"input"` + Output json.RawMessage `json:"output"` + } + items := make([]item, len(input)) + calls := make(map[string]int) + duplicates := make(map[string]bool) + lastResult := -1 + for index, raw := range input { + if json.Unmarshal(raw, &items[index]) != nil { + continue + } + current := items[index] + switch current.Type { + case "function_call", "custom_tool_call": + if _, exists := calls[current.CallID]; exists { + duplicates[current.CallID] = true + } + calls[current.CallID] = index + case "function_call_output", "custom_tool_call_output": + lastResult = index + } + } + for index, current := range items { + if index == lastResult || (current.Type != "function_call_output" && current.Type != "custom_tool_call_output") { + continue + } + callIndex, exists := calls[current.CallID] + if !exists || current.CallID == "" || duplicates[current.CallID] || callIndex >= index { + continue + } + call := items[callIndex] + command := call.Input + switch { + case call.Type == "function_call" && (call.Name == "exec_command" || call.Name == "functions.exec_command"): + var args struct { + Command string `json:"cmd"` + Shell string `json:"shell"` + } + if json.Unmarshal([]byte(call.Arguments), &args) != nil || (args.Shell != "" && args.Shell != "bash" && args.Shell != "/bin/bash") { + continue + } + command = args.Command + case call.Type == "custom_tool_call" && (call.Name == "shell" || call.Name == "functions.shell"): + // Interpreter/header semantics belong to the shell runtime. Unknown + // headers must not be interpreted as plain Bash by this reducer. + if strings.HasPrefix(command, "#!") { + continue + } + default: + continue + } + kind := contextCompactionCommand(command) + if kind == "" { + continue + } + encode, text, ok := contextCompactionOutput(current.Output) + if !ok { + continue + } + reduced := text + switch kind { + case "go-test": + var kept strings.Builder + removed := 0 + for line := range strings.SplitAfterSeq(text, "\n") { + if contextCompactionGoRoutine.MatchString(strings.TrimSuffix(line, "\n")) { + removed++ + } else { + kept.WriteString(line) + } + } + if removed > 0 { + reduced = fmt.Sprintf("[hpatch: omitted %d Go test progress/pass lines]\n%s", removed, kept.String()) + } + case "search": + // A later byte-identical output is explicit replacement evidence, + // not an assumption that rerunning a search gives its old answer. + // Only read outputs qualify, so this source cannot itself be reduced. + if len(text) < 256 { + continue + } + for later := index + 1; later < len(items); later++ { + candidate := items[later] + if candidate.Type != "function_call_output" || candidate.CallID == "" || duplicates[candidate.CallID] { + continue + } + sourceIndex, exists := calls[candidate.CallID] + if !exists || sourceIndex <= index || sourceIndex >= later { + continue + } + source := items[sourceIndex] + if source.Type != "function_call" || (source.Name != "exec_command" && source.Name != "functions.exec_command") { + continue + } + var args struct { + Command string `json:"cmd"` + Shell string `json:"shell"` + } + if json.Unmarshal([]byte(source.Arguments), &args) != nil || args.Shell != "" { + continue + } + if contextCompactionCommand(args.Command) != "read" { + continue + } + _, evidence, ok := contextCompactionOutput(candidate.Output) + if ok && evidence == text { + reduced = fmt.Sprintf("[hpatch compaction: matching search listing retained verbatim in tool result %q; original command and successful exit status retained]\n", candidate.CallID) + break + } + } + } + if len(reduced) >= len(text) { + continue + } + var fields map[string]json.RawMessage + if json.Unmarshal(input[index], &fields) != nil { + continue + } + fields["output"] = encode(reduced) + output[index] = mustMarshalJSON(fields) + } + return output +} + +// Accept structured results or Codex's native completed-exec header. Unknown +// formats, failures, and live handles stay untouched rather than guessing state. +// Native header source: Codex core/src/tools/context.rs response_header. +func contextCompactionOutput(raw json.RawMessage) (func(string) json.RawMessage, string, bool) { + var serialized string + if json.Unmarshal(raw, &serialized) != nil { + return nil, "", false + } + var envelope map[string]json.RawMessage + if json.Unmarshal([]byte(serialized), &envelope) != nil || envelope == nil { + matches := contextCompactionNativeResult.FindStringSubmatch(serialized) + if matches == nil { + return nil, "", false + } + return func(text string) json.RawMessage { return mustMarshalJSON(matches[1] + text) }, matches[2], true + } + + var exitCode *int + var output string + if json.Unmarshal(envelope["exit_code"], &exitCode) != nil || exitCode == nil || *exitCode != 0 || + json.Unmarshal(envelope["output"], &output) != nil { + return nil, "", false + } + for _, key := range []string{"session_id", "cell_id"} { + if value, exists := envelope[key]; exists && string(value) != "null" { + return nil, "", false + } + } + return func(text string) json.RawMessage { + envelope["output"] = mustMarshalJSON(text) + return mustMarshalJSON(string(mustMarshalJSON(envelope))) + }, output, true + +} + +var contextCompactionNativeResult = regexp.MustCompile(`(?s)\A((?:Chunk ID: [^\r\n]+\n)?Wall time: [0-9]+(?:\.[0-9]+)? seconds\nProcess exited with code 0\n(?:Original token count: [0-9]+\n)?(?:Output|Final output):\n)(.*)\z`) + +var contextCompactionGoRoutine = regexp.MustCompile(`^(=== (RUN|PAUSE|CONT) +\S+|[ \t]*--- PASS: \S+ \([0-9]+(\.[0-9]+)?s\))$`) + +// Parse, never execute or expand dynamic shell syntax. Compound commands, +// redirections, wrappers, assignments, and substitutions are outside this +// initial reducer's evidence contract, even if they contain a familiar name. +func contextCompactionCommand(command string) string { + program, err := syntax.NewParser(syntax.Variant(syntax.LangBash)).Parse(strings.NewReader(command), "") + if err != nil || len(program.Stmts) != 1 { + return "" + } + statement := program.Stmts[0] + call, ok := statement.Cmd.(*syntax.CallExpr) + if !ok || len(call.Args) == 0 || len(call.Assigns) != 0 || len(statement.Redirs) != 0 || + statement.Negated || statement.Background || statement.Coprocess || statement.Disown { + return "" + } + args := make([]string, len(call.Args)) + for index, word := range call.Args { + static := true + syntax.Walk(word, func(node syntax.Node) bool { + switch node.(type) { + case nil, *syntax.Word, *syntax.Lit, *syntax.SglQuoted, *syntax.DblQuoted: + return true + default: + static = false + return false + } + }) + if !static { + return "" + } + args[index], err = expand.Literal(nil, word) + if err != nil { + return "" + } + } + switch args[0] { + case "go": + if len(args) > 1 && args[1] == "test" { + return "go-test" + } + case "rg", "hgrep": + for _, arg := range args[1:] { + if arg == "--pre" || strings.HasPrefix(arg, "--pre=") { + return "" + } + } + return "search" + case "find": + for _, arg := range args[1:] { + switch arg { + case "-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf": + return "" + } + } + return "search" + + case "cat", "hread": + return "read" + } + return "" +} diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go new file mode 100644 index 00000000..5f637385 --- /dev/null +++ b/internal/router/context_compaction_codex_test.go @@ -0,0 +1,191 @@ +package router + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +// Opt-in because Codex is not a Go test dependency. All model responses are +// loopback fixtures, and the subprocess has an isolated configuration/history. +func TestCompactionInstalledCodex(t *testing.T) { + binary := os.Getenv("HPATCH_COMPACTION_CODEX_BIN") + if binary == "" { + t.Skip("set HPATCH_COMPACTION_CODEX_BIN to exercise an installed Codex client") + } + for _, probe := range []struct { + legacy bool + scope string + }{{false, "total"}, {false, "body_after_prefix"}, {true, "total"}, {true, "body_after_prefix"}} { + t.Run(fmt.Sprintf("legacy=%v/scope=%s", probe.legacy, probe.scope), func(t *testing.T) { + prompt := "Run the Go tests, then print the working directory, then report completion. Preserve the test result.\n" + + strings.Repeat("Keep the original user constraint. ", 10000) + + "\nThis final instruction must also survive intact." + + directory, home := t.TempDir(), t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module compactionprobe\n\ngo 1.26\n", + "probe_test.go": `package compactionprobe +import ("fmt"; "testing") +func TestProbe(t *testing.T) { + for i := range 100 { t.Run(fmt.Sprintf("Case%d", i), func(t *testing.T) {}) } +} +`, + } { + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + compactor := &contextCompactor{keyPath: filepath.Join(home, "compaction.key")} + var normal, compacted atomic.Int32 + var restored atomic.Bool + model := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request map[string]json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + metadata, _ := decodeCodexTurnMetadata(r.Header) + if metadata.RequestKind == "compaction" || r.URL.Path == "/v1/responses/compact" { + t.Error("client compaction reached the model fixture") + http.Error(w, "unexpected provider compaction", 500) + return + } + step := normal.Add(1) + var item map[string]any + inputTokens := 1000 + switch step { + case 1, 2: + command, callID := "go test -v ./...", "probe_go" + if step == 2 { + command, callID, inputTokens = "pwd", "probe_pwd", 300000 + } + item = map[string]any{ + "type": "function_call", "id": fmt.Sprintf("fc_probe_%d", step), "call_id": callID, + "name": "exec_command", "status": "completed", + "arguments": string(mustMarshalJSON(map[string]any{"cmd": command, "workdir": directory, "yield_time_ms": 10000, "max_output_tokens": 15000})), + } + default: + var input []map[string]json.RawMessage + _ = json.Unmarshal(request["input"], &input) + userCopies := 0 + for _, record := range input { + if jsonString(record, "type") == "message" && jsonString(record, "role") == "user" { + var content []map[string]json.RawMessage + _ = json.Unmarshal(record["content"], &content) + for _, part := range content { + if jsonString(part, "text") == prompt { + userCopies++ + } + } + } + if jsonString(record, "type") == "function_call_output" && jsonString(record, "call_id") == "probe_go" { + output := jsonString(record, "output") + restored.Store(strings.Contains(output, "[hpatch: omitted") && strings.Contains(output, "compactionprobe")) + } + if strings.HasPrefix(jsonString(record, "encrypted_content"), "hpatch.compaction.") { + t.Error("local ciphertext reached the model fixture") + } + } + if userCopies != 1 { + t.Errorf("restored full user request copies = %d, want exactly one", userCopies) + } + item = map[string]any{ + "type": "message", "id": "msg_probe_done", "role": "assistant", "status": "completed", + "content": []any{map[string]any{"type": "output_text", "text": "COMPACTION_OK", "annotations": []any{}}}, + } + } + w.Header().Set("Content-Type", "text/event-stream") + responseID := fmt.Sprintf("resp_probe_%d", step) + events := []map[string]any{ + {"type": "response.created", "response": map[string]any{"id": responseID, "status": "in_progress"}}, + {"type": "response.output_item.added", "output_index": 0, "item": item}, + {"type": "response.output_item.done", "output_index": 0, "item": item}, + {"type": "response.completed", "response": map[string]any{ + "id": responseID, "status": "completed", "output": []any{item}, + "usage": map[string]any{"input_tokens": inputTokens, "output_tokens": 10, "total_tokens": inputTokens + 10}, + }}, + } + for sequence, event := range events { + event["sequence_number"] = sequence + _, _ = fmt.Fprintf(w, "event: %s\ndata: %s\n\n", event["type"], mustMarshalJSON(event)) + } + }) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + metadata, _ := decodeCodexTurnMetadata(r.Header) + if metadata.RequestKind == "compaction" || r.URL.Path == "/v1/responses/compact" { + compacted.Add(1) + } + body, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(strings.NewReader(string(body))) + tracked := &trackedResponseWriter{ResponseWriter: w} + compactor.handler(model)(tracked, r) + if tracked.statusCode >= 400 { + var fields map[string]json.RawMessage + _ = json.Unmarshal(body, &fields) + var items []map[string]json.RawMessage + _ = json.Unmarshal(fields["input"], &items) + for _, item := range items { + if jsonString(item, "type") == "message" { + var content []map[string]json.RawMessage + _ = json.Unmarshal(item["content"], &content) + for _, part := range content { + text := jsonString(part, "text") + t.Logf("fixture message role=%s id=%s metadata=%s bytes=%d prefix=%q", jsonString(item, "role"), jsonString(item, "id"), item["internal_chat_message_metadata_passthrough"], len(text), text[:min(len(text), 140)]) + } + } + } + } + + })) + defer server.Close() + config := fmt.Sprintf(`model = "gpt-5.3-codex" +model_provider = "loopback" +model_context_window = 1000000 +model_auto_compact_token_limit = 200000 +model_auto_compact_token_limit_scope = %q +tool_output_token_limit = 20000 +[model_providers.loopback] +name = "Azure" +base_url = %q +wire_api = "responses" +requires_openai_auth = false +supports_websockets = false +request_max_retries = 0 +stream_max_retries = 0 +[otel] +exporter = "none" +trace_exporter = "none" +metrics_exporter = "none" +`, probe.scope, server.URL+"/v1") + if err := os.WriteFile(filepath.Join(home, "config.toml"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + args := []string{"exec", "--skip-git-repo-check", "--ephemeral", "--json", "--dangerously-bypass-approvals-and-sandbox"} + if probe.legacy { + args = append(args, "--disable", "remote_compaction_v2") + } + args = append(args, "-") + ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second) + defer cancel() + command := exec.CommandContext(ctx, binary, args...) + command.Stdin = strings.NewReader(prompt) + command.Dir = directory + command.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"), "CODEX_HOME=" + home} + output, err := command.CombinedOutput() + if err != nil || compacted.Load() != 1 || normal.Load() != 3 || !restored.Load() { + t.Fatalf("client round trip: err=%v normal=%d compactions=%d restored=%v; output=%s", err, normal.Load(), compacted.Load(), restored.Load(), output) + } + }) + } +} diff --git a/internal/router/context_compaction_envelope.go b/internal/router/context_compaction_envelope.go new file mode 100644 index 00000000..1fd68f9d --- /dev/null +++ b/internal/router/context_compaction_envelope.go @@ -0,0 +1,156 @@ +package router + +import ( + "bytes" + "compress/zlib" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/gofrs/flock" +) + +const ( + contextCompactionPrefix = "hpatch.compaction.v1:" + contextCompactionIDPrefix = "cmp_hpatch_" +) + +// The key is installation-owned, not session-owned: resumed and forked Codex +// histories must remain readable after a router restart. Only encrypted retained +// history travels in the envelope; no transcript archive or retrieval is used. +type contextCompactor struct { + keyPath string +} + +func (c *contextCompactor) cipher(ctx context.Context, create bool) (cipher.AEAD, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if c.keyPath == "" { + return nil, errors.New("hpatch compaction key path is not configured") + } + if create { + if err := os.MkdirAll(filepath.Dir(c.keyPath), 0o700); err != nil { + return nil, fmt.Errorf("create compaction key directory: %w", err) + } + } + lock := flock.New(c.keyPath + ".lock") + defer lock.Close() + locked, err := lock.TryLockContext(ctx, 50*time.Millisecond) + if err != nil || !locked { + return nil, errors.Join(ctx.Err(), err, errors.New("could not acquire compaction key lock")) + } + key, err := os.ReadFile(c.keyPath) + if errors.Is(err, os.ErrNotExist) && create { + key = make([]byte, 32) + rand.Read(key) + file, createErr := os.OpenFile(c.keyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if createErr != nil { + return nil, fmt.Errorf("create compaction key: %w", createErr) + } + _, writeErr := file.Write(key) + err = errors.Join(writeErr, file.Sync(), file.Close()) + } + if err != nil { + return nil, fmt.Errorf("read compaction key (required to resume compacted histories): %w", err) + } + if len(key) != 32 { + return nil, errors.New("invalid compaction key; restore the original key before resuming compacted histories") + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCMWithRandomNonce(block) +} + +func (c *contextCompactor) seal(ctx context.Context, items []json.RawMessage) (json.RawMessage, error) { + plaintext, err := marshalProtocolJSON(items) + if err != nil { + return nil, err + } + if len(plaintext) > responsesRequestBufferBytes { + return nil, errors.New("retained compaction history exceeds the router buffer budget") + } + var compressed bytes.Buffer + compressor := zlib.NewWriter(&compressed) + if _, err := compressor.Write(plaintext); err != nil { + return nil, err + } + if err := compressor.Close(); err != nil { + return nil, err + } + aead, err := c.cipher(ctx, true) + if err != nil { + return nil, err + } + encoded := contextCompactionPrefix + base64.RawStdEncoding.EncodeToString(aead.Seal(nil, nil, compressed.Bytes(), []byte(contextCompactionPrefix))) + digest := sha256.Sum256([]byte(encoded)) + return marshalProtocolJSON(map[string]any{ + "type": "compaction", + "id": fmt.Sprintf("%s%x", contextCompactionIDPrefix, digest[:16]), + "encrypted_content": encoded, + }) +} + +func (c *contextCompactor) open(ctx context.Context, raw json.RawMessage) ([]json.RawMessage, bool, error) { + var item struct { + Type string `json:"type"` + ID string `json:"id"` + Content string `json:"encrypted_content"` + } + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + return nil, false, nil + } + item.Type = jsonString(fields, "type") + item.ID = jsonString(fields, "id") + item.Content = jsonString(fields, "encrypted_content") + local := strings.HasPrefix(item.Content, "hpatch.compaction.") || strings.HasPrefix(item.ID, contextCompactionIDPrefix) + if !local { + return nil, false, nil + } + if item.Type != "compaction" || !strings.HasPrefix(item.Content, contextCompactionPrefix) { + return nil, true, errors.New("unsupported or damaged hpatch compaction envelope") + } + encrypted, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(item.Content, contextCompactionPrefix)) + if err != nil { + return nil, true, errors.New("invalid hpatch compaction envelope encoding") + } + aead, err := c.cipher(ctx, false) + if err != nil { + return nil, true, err + } + compressed, err := aead.Open(nil, nil, encrypted, []byte(contextCompactionPrefix)) + if err != nil { + return nil, true, errors.New("hpatch compaction envelope authentication failed") + } + decompressor, err := zlib.NewReader(bytes.NewReader(compressed)) + if err != nil { + return nil, true, errors.New("invalid hpatch compaction envelope payload") + } + defer decompressor.Close() + plaintext, err := io.ReadAll(io.LimitReader(decompressor, responsesRequestBufferBytes+1)) + if err != nil || len(plaintext) > responsesRequestBufferBytes { + return nil, true, errors.New("hpatch compaction envelope exceeds the router buffer budget or is damaged") + } + if err := ctx.Err(); err != nil { + return nil, true, err + } + var items []json.RawMessage + if json.Unmarshal(plaintext, &items) != nil || len(items) == 0 { + return nil, true, errors.New("invalid retained hpatch compaction history") + } + return items, true, nil +} diff --git a/internal/router/context_compaction_envelope_test.go b/internal/router/context_compaction_envelope_test.go new file mode 100644 index 00000000..647557cb --- /dev/null +++ b/internal/router/context_compaction_envelope_test.go @@ -0,0 +1,83 @@ +package router + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestCompactionEnvelopeSurvivesRestartAndRejectsDamage(t *testing.T) { + path := filepath.Join(t.TempDir(), "compaction.key") + first := &contextCompactor{keyPath: path} + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Preserve this requirement."}), + compactTestCall("last", "go test ./..."), + compactTestOutput("last", "unfinished diagnostics", nil), + mustMarshalJSON(map[string]any{"type": "reasoning", "encrypted_content": "provider-owned-opaque-state"}), + } + sealed, err := first.seal(t.Context(), items) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(sealed), "Preserve this requirement") || strings.Contains(string(sealed), "unfinished diagnostics") { + t.Fatal("envelope contains plaintext history") + } + restarted := &contextCompactor{keyPath: path} + restored, local, err := restarted.open(t.Context(), sealed) + if err != nil || !local || string(mustMarshalJSON(restored)) != string(mustMarshalJSON(items)) { + t.Fatalf("restart round trip failed: local=%v, err=%v", local, err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("key permissions: %v, %v", info, err) + } + var damaged map[string]json.RawMessage + if err := json.Unmarshal(sealed, &damaged); err != nil { + t.Fatal(err) + } + encoded := jsonString(damaged, "encrypted_content") + position := len(encoded) / 2 + replacement := byte('A') + if encoded[position] == replacement { + replacement = 'B' + } + damaged["encrypted_content"] = mustMarshalJSON(encoded[:position] + string(replacement) + encoded[position+1:]) + if _, local, err := restarted.open(t.Context(), mustMarshalJSON(damaged)); !local || err == nil { + t.Fatal("damaged local envelope was accepted or treated as provider state") + } + damaged["encrypted_content"] = mustMarshalJSON("provider-looking-value") + if _, local, err := restarted.open(t.Context(), mustMarshalJSON(damaged)); !local || err == nil { + t.Fatal("local item with a damaged prefix could escape to the provider") + } + if _, local, err := (&contextCompactor{keyPath: filepath.Join(t.TempDir(), "absent.key")}).open(t.Context(), sealed); !local || err == nil { + t.Fatal("missing key did not fail closed") + } +} + +func TestCompactionEnvelopeConcurrentKeyCreationAndProviderIsolation(t *testing.T) { + path := filepath.Join(t.TempDir(), "compaction.key") + items := []json.RawMessage{mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Keep me."})} + var workers sync.WaitGroup + for range 8 { + workers.Go(func() { + compactor := &contextCompactor{keyPath: path} + sealed, err := compactor.seal(t.Context(), items) + if err != nil { + t.Error(err) + return + } + if _, _, err := (&contextCompactor{keyPath: path}).open(t.Context(), sealed); err != nil { + t.Error(err) + } + }) + } + workers.Wait() + provider := mustMarshalJSON(map[string]any{"type": "compaction", "encrypted_content": "provider-owned"}) + if _, local, err := (&contextCompactor{}).open(t.Context(), provider); local || err != nil { + t.Fatal("provider-owned compaction was interpreted locally") + } +} + diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go new file mode 100644 index 00000000..374f0468 --- /dev/null +++ b/internal/router/context_compaction_http.go @@ -0,0 +1,377 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "slices" + "strings" + "time" +) + +// Both compact interfaces terminate here, before tool projection, CTP, or any +// provider call. Codex remains the sole owner of scheduling and configuration. +func (c *contextCompactor) handler(next http.Handler) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + body, err := readResponsesRequest(io.LimitReader(request.Body, responsesRequestBufferBytes+1)) + if err != nil { + http.Error(writer, err.Error(), http.StatusBadRequest) + return + } + if len(body) > responsesRequestBufferBytes { + http.Error(writer, "Responses request exceeds the router buffer budget", http.StatusRequestEntityTooLarge) + return + } + metadata, metadataValid := decodeCodexTurnMetadata(request.Header) + standalone := request.URL.Path == "/v1/responses/compact" + compacting := standalone || (metadataValid && metadata.RequestKind == "compaction") + parsed, err := parseResponsesRequest(body) + if err != nil { + http.Error(writer, err.Error(), http.StatusBadRequest) + return + } + var input []json.RawMessage + if json.Unmarshal(parsed.fields["input"], &input) != nil || len(input) == 0 { + if !compacting { + request.Body = io.NopCloser(bytes.NewReader(body)) + next.ServeHTTP(writer, request) + return + } + http.Error(writer, "local compaction requires a nonempty input item array", http.StatusBadRequest) + return + } + local := false + for _, raw := range input { + var item map[string]json.RawMessage + _ = json.Unmarshal(raw, &item) + local = local || strings.HasPrefix(jsonString(item, "encrypted_content"), "hpatch.compaction.") || strings.HasPrefix(jsonString(item, "id"), contextCompactionIDPrefix) + } + if !compacting && !local { + request.Body = io.NopCloser(bytes.NewReader(body)) + next.ServeHTTP(writer, request) + return + } + + for _, item := range input { + var fields map[string]json.RawMessage + if json.Unmarshal(item, &fields) != nil || fields == nil { + http.Error(writer, "compaction input items must be objects", http.StatusBadRequest) + return + } + } + input, err = c.restore(request.Context(), input) + if err != nil { + http.Error(writer, err.Error(), http.StatusUnprocessableEntity) + return + } + if !compacting { + parsed.setInput(mustMarshalJSON(input)) + body, err = parsed.wireBody(parsed.fields) + if err != nil || len(body) > responsesRequestBufferBytes { + http.Error(writer, "restored history exceeds the router buffer budget", http.StatusRequestEntityTooLarge) + return + } + request.Body = io.NopCloser(bytes.NewReader(body)) + request.ContentLength = int64(len(body)) + next.ServeHTTP(writer, request) + return + } + if parsed.model() == "" { + http.Error(writer, "compaction requires a model", http.StatusBadRequest) + return + } + if !standalone { + var details struct { + Implementation string `json:"implementation"` + } + _ = json.Unmarshal(metadata.Compaction, &details) + if !parsed.streamResponse || details.Implementation != "responses_compaction_v2" { + http.Error(writer, "local compaction requires the standalone compact endpoint or streaming compaction V2; provider summaries are disabled", http.StatusUnprocessableEntity) + return + } + // This is a request control, not part of the durable history. + var last map[string]json.RawMessage + _ = json.Unmarshal(input[len(input)-1], &last) + if jsonString(last, "type") == "compaction_trigger" { + input = input[:len(input)-1] + } + } + reduced := reduceContextCompaction(input) + if slices.EqualFunc(input, reduced, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { + http.Error(writer, "no safe context reduction is available for this history; protected context was not discarded and no provider compaction was requested", http.StatusUnprocessableEntity) + return + } + capsule, err := c.seal(request.Context(), reduced) + if err != nil { + http.Error(writer, err.Error(), http.StatusUnprocessableEntity) + return + } + var sealedItem struct { + ID string `json:"id"` + } + _ = json.Unmarshal(capsule, &sealedItem) + responseID := "resp_" + strings.TrimPrefix(sealedItem.ID, "cmp_") + if standalone { + // Legacy Codex replaces its history wholesale. Keep user messages + // visible to its own user-input handling as well as in the capsule. + var output []json.RawMessage + for _, item := range reduced { + var fields map[string]json.RawMessage + _ = json.Unmarshal(item, &fields) + if jsonString(fields, "type") == "message" && jsonString(fields, "role") == "user" { + output = append(output, item) + } + } + output = append(output, capsule) + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "object": "response.compaction", "id": responseID, + "created_at": time.Now().Unix(), "output": output, + }) + return + } + // V2 retains its own selected user/context items, then appends exactly + // one compaction item. No synthetic assistant prose or provider usage. + writer.Header().Set("Content-Type", "text/event-stream") + for sequence, event := range []map[string]any{ + {"type": "response.created", "response": map[string]any{"id": responseID, "status": "in_progress", "output": []any{}}}, + {"type": "response.output_item.added", "output_index": 0, "item": capsule}, + {"type": "response.output_item.done", "output_index": 0, "item": capsule}, + {"type": "response.completed", "response": map[string]any{"id": responseID, "status": "completed", "output": []json.RawMessage{capsule}}}, + } { + event["sequence_number"] = sequence + if _, err := fmt.Fprintf(writer, "event: %s\ndata: %s\n\n", event["type"], mustMarshalJSON(event)); err != nil { + return + } + } + } +} + +// Restore the native timeline once. Codex may carry a subset of the original +// messages alongside the capsule and inject fresh canonical context between +// them. Align those carried items with the snapshot rather than blindly dropping +// the prefix or duplicating every user message. Unmatched current context stays. +func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) ([]json.RawMessage, error) { + type restoredItem struct { + raw json.RawMessage + fromEnvelope bool + } + var output []restoredItem + for _, item := range input { + + if err := ctx.Err(); err != nil { + return nil, err + } + retained, local, err := c.open(ctx, item) + if err != nil { + return nil, err + } + if !local { + output = append(output, restoredItem{raw: item}) + continue + } + retainedItems := make([]restoredItem, len(retained)) + positions := make(map[string][]int, len(retained)) + for index, raw := range retained { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil || fields == nil { + return nil, errors.New("invalid item in retained compaction history") + } + if strings.HasPrefix(jsonString(fields, "encrypted_content"), "hpatch.compaction.") || strings.HasPrefix(jsonString(fields, "id"), contextCompactionIDPrefix) { + return nil, errors.New("nested local compaction envelope is not supported") + } + retainedItems[index] = restoredItem{raw: raw, fromEnvelope: true} + identity := contextCompactionItemIdentity(raw) + positions[identity] = append(positions[identity], index) + } + // Codex retains the newest end of history. Match backwards so a + // repeated no-ID user message anchors fresh context at its latest + // occurrence, rather than before an older conflicting instruction. + matched := make([]int, len(output)) + limit := len(retained) + for index := len(output) - 1; index >= 0; index-- { + matched[index] = -1 + carried := output[index].raw + if !output[index].fromEnvelope && contextCompactionFreshContext(carried) { + continue + } + matches := positions[contextCompactionItemIdentity(carried)] + match, _ := slices.BinarySearch(matches, limit) + if match > 0 { + matched[index] = matches[match-1] + } else { + for candidate := range limit { + if contextCompactionTruncatedMatch(carried, retained[candidate]) { + if matched[index] >= 0 { + return nil, errors.New("ambiguous truncated message in compacted history") + } + matched[index] = candidate + } + } + var carriedFields map[string]json.RawMessage + _ = json.Unmarshal(carried, &carriedFields) + if matched[index] < 0 && (contextCompactionTruncation.Match(carried) || (jsonString(carriedFields, "type") == "message" && jsonString(carriedFields, "role") == "user")) { + return nil, errors.New("cannot reconcile truncated compacted history without losing context") + } + } + if matched[index] >= 0 { + limit = matched[index] + } + } + var merged, pending []restoredItem + cursor := 0 + for position, carried := range output { + index := matched[position] + if index < 0 { + pending = append(pending, carried) + continue + } + + merged = append(merged, retainedItems[cursor:index]...) + merged = append(merged, pending...) + pending = nil + merged = append(merged, retainedItems[index]) + cursor = index + 1 + } + merged = append(merged, retainedItems[cursor:]...) + output = append(merged, pending...) + } + result := make([]json.RawMessage, len(output)) + for index, item := range output { + result[index] = item.raw + } + if len(mustMarshalJSON(result)) > responsesRequestBufferBytes { + return nil, errors.New("restored compaction history exceeds the router buffer budget") + } + return result, nil + +} + +// Newly injected canonical context must keep its current position even when +// its text equals an older instruction. Content alone is not an event identity. +func contextCompactionFreshContext(raw json.RawMessage) bool { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + if jsonString(fields, "type") != "message" { + return false + } + if role := jsonString(fields, "role"); role == "developer" || role == "system" { + return true + } + var metadata struct { + Kinds []string `json:"content_item_kinds"` + } + _ = json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) + var content []map[string]json.RawMessage + _ = json.Unmarshal(fields["content"], &content) + if len(metadata.Kinds) > 0 { + // Follow Codex's conservative authorization classification: unknown, + // incomplete, mixed, and media-preparation input remains real user input. + return len(metadata.Kinds) == len(content) && !slices.ContainsFunc(metadata.Kinds, func(kind string) bool { + return kind == "" || kind == "unknown" || strings.HasPrefix(kind, "user.") || + kind == "images.preparation_error" || kind == "images.unsupported" || kind == "audio.unsupported" + }) + } + // Older clients omit classifications on freshly reinjected context. + // Source: Codex context/user_instructions.rs and world_state/environment.rs. + return len(content) > 0 && !slices.ContainsFunc(content, func(part map[string]json.RawMessage) bool { + text := strings.TrimSpace(jsonString(part, "text")) + return !((strings.HasPrefix(text, "") && strings.HasSuffix(text, "")) || + (strings.HasPrefix(text, "# AGENTS.md instructions") && strings.HasSuffix(text, ""))) + }) + +} + +var contextCompactionTruncation = regexp.MustCompile(`…[0-9]+ tokens truncated…`) + +// Verify a client-produced truncation against the authenticated original; do +// not guess the client's budget or reproduce its retention policy. +// Source: Codex compact_remote_v2.rs truncate_message_text_to_token_budget; +// utils/string/src/truncate.rs format_truncation_marker/assemble_truncated_output. +func contextCompactionTruncatedMatch(carried, original json.RawMessage) bool { + var left, right map[string]json.RawMessage + _ = json.Unmarshal(carried, &left) + _ = json.Unmarshal(original, &right) + if jsonString(left, "type") != "message" || jsonString(right, "type") != "message" || + jsonString(left, "role") != jsonString(right, "role") { + return false + } + var leftContent, rightContent []map[string]json.RawMessage + if json.Unmarshal(left["content"], &leftContent) != nil || json.Unmarshal(right["content"], &rightContent) != nil || + len(leftContent) == 0 || len(leftContent) > len(rightContent) { + return false + } + // Truncation can introduce "unknown" content classifications. Other + // metadata still participates in identity and must not be discarded. + for _, fields := range []map[string]json.RawMessage{left, right} { + delete(fields, "content") + var metadata map[string]json.RawMessage + if json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) == nil { + var kinds []string + _ = json.Unmarshal(metadata["content_item_kinds"], &kinds) + if !slices.ContainsFunc(kinds, func(kind string) bool { return kind != "unknown" }) { + delete(metadata, "content_item_kinds") + if len(metadata) == 0 { + delete(fields, "internal_chat_message_metadata_passthrough") + } else { + fields["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(metadata) + } + } + } + } + if contextCompactionCanonicalJSON(mustMarshalJSON(left)) != contextCompactionCanonicalJSON(mustMarshalJSON(right)) { + return false + } + shortened := len(leftContent) < len(rightContent) + for index, part := range leftContent { + full := rightContent[index] + leftText, rightText := jsonString(part, "text"), jsonString(full, "text") + delete(part, "text") + delete(full, "text") + if contextCompactionCanonicalJSON(mustMarshalJSON(part)) != contextCompactionCanonicalJSON(mustMarshalJSON(full)) { + return false + } + if leftText == rightText { + continue + } + markers := contextCompactionTruncation.FindAllStringIndex(leftText, -1) + if len(markers) != 1 { + return false + } + start, end := markers[0][0], markers[0][1] + head, tail := leftText[:start], leftText[end:] + if !strings.HasPrefix(rightText, head) || !strings.HasSuffix(rightText, tail) || len(head)+len(tail) >= len(rightText) { + return false + } + shortened = true + } + return shortened +} + +func contextCompactionItemIdentity(raw json.RawMessage) string { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + kind, role := jsonString(fields, "type"), jsonString(fields, "role") + if id := jsonString(fields, "id"); id != "" { + return kind + "\x00" + role + "\x00id:" + id + } + if id := jsonString(fields, "call_id"); id != "" { + return kind + "\x00call:" + id + } + return contextCompactionCanonicalJSON(raw) +} + +func contextCompactionCanonicalJSON(raw json.RawMessage) string { + // Codex serializes typed items in a different field order. Use numbers + // without float conversion so canonicalization cannot merge large IDs. + var value any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + _ = decoder.Decode(&value) + return string(mustMarshalJSON(value)) +} diff --git a/internal/router/context_compaction_http_test.go b/internal/router/context_compaction_http_test.go new file mode 100644 index 00000000..e7aa8d22 --- /dev/null +++ b/internal/router/context_compaction_http_test.go @@ -0,0 +1,283 @@ +package router + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "strings" + "testing" +) + +func compactHTTPHistory() []json.RawMessage { + return []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": "Fix only the router. Do not commit."}}}), + compactTestCall("tests", "go test -v ./internal/router"), + compactTestOutput("tests", strings.Repeat("=== RUN TestRoute\n--- PASS: TestRoute (0.01s)\n", 50)+"PASS\nok example/router 0.1s\n", 0), + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": "Keep the failed test diagnostics too."}}}), + compactTestCall("last", "go test ./..."), + compactTestOutput("last", "failure: keep this evidence", 1), + } +} + +func TestCompactionHTTPDoesNotInterpretTextAndBlocksEscapedEnvelopes(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + calls := 0 + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ }) + for _, body := range []string{ + `{"model":"gpt-5","input":"Explain hpatch.compaction.v1: please."}`, + `{"model":"gpt-5","input":[{"type":"message","role":"user","content":"cmp_hpatch_example"}]}`, + } { + response := httptest.NewRecorder() + compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))) + if response.Code != http.StatusOK { + t.Fatal("ordinary text was interpreted as a capsule") + } + } + body := `{"model":"gpt-5","input":[{"type":"compaction","encrypted_content":"hpatch\u002ecompaction\u002ev1:broken"}]}` + response := httptest.NewRecorder() + compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))) + if response.Code < 400 || calls != 2 { + t.Fatal("escaped local envelope was forwarded") + } +} +func TestCompactionHTTPProviderFreeRoundTrip(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + calls := 0 + var delivered []json.RawMessage + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(body, &fields); err != nil { + t.Fatal(err) + } + _ = json.Unmarshal(fields["input"], &delivered) + if strings.Contains(string(body), contextCompactionPrefix) { + t.Fatal("local envelope escaped to provider") + } + if jsonString(fields, "instructions") != "current instructions" { + t.Fatal("current instructions changed") + } + }) + input := compactHTTPHistory() + request := httptest.NewRequest(http.MethodPost, "/v1/responses/compact", strings.NewReader(string(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": input})))) + response := httptest.NewRecorder() + compactor.handler(next)(response, request) + if response.Code != http.StatusOK || calls != 0 { + t.Fatalf("compaction = %d, provider calls = %d: %s", response.Code, calls, response.Body.String()) + } + var compacted struct { + Output []json.RawMessage `json:"output"` + } + if err := json.Unmarshal(response.Body.Bytes(), &compacted); err != nil { + t.Fatal(err) + } + // Model the legacy client retaining user items, injecting fresh context, + // and appending a later user request. Restoration must keep each once. + fresh := mustMarshalJSON(map[string]any{"type": "message", "role": "developer", "content": []any{map[string]string{"type": "input_text", "text": "Fresh session context."}}}) + followup := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": "Continue."}}}) + window := slices.Insert(compacted.Output, 1, fresh) + window = append(window, followup) + request = httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(string(mustMarshalJSON(map[string]any{"model": "gpt-5", "instructions": "current instructions", "input": window})))) + response = httptest.NewRecorder() + (&contextCompactor{keyPath: compactor.keyPath}).handler(next)(response, request) + want := slices.Insert(reduceContextCompaction(input), 3, fresh) + want = append(want, followup) + if response.Code != http.StatusOK || calls != 1 || string(mustMarshalJSON(delivered)) != string(mustMarshalJSON(want)) { + t.Fatalf("native restoration failed: status=%d, provider calls=%d", response.Code, calls) + } +} + +func TestCompactionHTTPStreamingV2AndFailures(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("compaction reached provider") }) + input := append(compactHTTPHistory(), mustMarshalJSON(map[string]any{"type": "compaction_trigger"})) + request := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(string(mustMarshalJSON(map[string]any{ + "model": "gpt-5", "input": input, "stream": true, "tools": []any{}, "parallel_tool_calls": true, + })))) + request.Header.Set(codexTurnMetadataHeader, string(mustMarshalJSON(map[string]any{ + "request_kind": "compaction", "compaction": map[string]any{"implementation": "responses_compaction_v2", "trigger": "auto", "reason": "context_limit", "phase": "mid_turn", "strategy": "memento"}, + }))) + response := httptest.NewRecorder() + compactor.handler(next)(response, request) + if response.Code != http.StatusOK || strings.Count(response.Body.String(), "event: response.output_item.done\n") != 1 || !strings.Contains(response.Body.String(), "event: response.completed\n") { + t.Fatalf("V2 result = %d: %s", response.Code, response.Body.String()) + } + for _, body := range []string{ + `{"model":"gpt-5","input":[]}`, + `{"model":"gpt-5","input":[null]}`, + `{"model":"gpt-5","input":[{"type":"message","role":"user","content":"protected"}]}`, + `{"model":"gpt-5","input":[{"type":"compaction","id":"cmp_hpatch_broken","encrypted_content":"broken"}]}`, + } { + response := httptest.NewRecorder() + compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses/compact", strings.NewReader(body))) + if response.Code < 400 { + t.Fatalf("unsafe compaction accepted: %s", body) + } + } +} + +func TestCompactionRestoreRepeatedAndTruncatedCarriedItems(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + first := mustMarshalJSON(map[string]any{"type": "message", "id": "msg_user", "role": "user", "content": "full original request"}) + items := append([]json.RawMessage{first}, compactHTTPHistory()...) + capsule, err := compactor.seal(t.Context(), items) + if err != nil { + t.Fatal(err) + } + truncated := mustMarshalJSON(map[string]any{"type": "message", "id": "msg_user", "role": "user", "content": "full..."}) + got, err := compactor.restore(t.Context(), []json.RawMessage{truncated, capsule}) + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(items)) { + t.Fatalf("truncated carried message not restored: %v", err) + } + second, err := compactor.seal(t.Context(), reduceContextCompaction(got)) + if err != nil { + t.Fatal(err) + } + got, err = compactor.restore(t.Context(), []json.RawMessage{capsule, second}) + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(reduceContextCompaction(items))) { + t.Fatalf("repeated envelope duplicated or lost history: %v", err) + } +} + +func TestCompactionRestoreFreshAuthorityAndRepeatedUserAnchors(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + message := func(role, text string) json.RawMessage { + return mustMarshalJSON(map[string]any{"type": "message", "role": role, "content": []any{map[string]string{"type": "input_text", "text": text}}}) + } + a, b, yes := message("developer", "Current rule A"), message("developer", "Intermediate conflicting rule B"), message("user", "yes") + items := []json.RawMessage{a, yes, b, yes, compactTestCall("last", "pwd")} + capsule, err := compactor.seal(t.Context(), items) + if err != nil { + t.Fatal(err) + } + got, err := compactor.restore(t.Context(), []json.RawMessage{a, yes, capsule}) + want := slices.Insert(slices.Clone(items), 3, a) + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(want)) { + t.Fatalf("fresh canonical authority was lost or placed at an old user anchor: %v", err) + } +} + +func TestCompactionRestoreNoIDTruncation(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + message := func(text string) json.RawMessage { + return mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": text}}}) + } + old, correction := message("middle1234"), message("Actually, do not make the old change.") + items := []json.RawMessage{old, correction, compactTestCall("last", "pwd")} + capsule, err := compactor.seal(t.Context(), items) + if err != nil { + t.Fatal(err) + } + var carried map[string]json.RawMessage + _ = json.Unmarshal(message("midd…1 tokens truncated…1234"), &carried) + carried["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(map[string]any{"content_item_kinds": []string{"unknown"}}) + for _, prefix := range [][]json.RawMessage{{mustMarshalJSON(carried)}, {mustMarshalJSON(carried), correction}} { + got, err := compactor.restore(t.Context(), append(prefix, capsule)) + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(items)) { + t.Fatalf("no-ID truncated request duplicated or moved past its correction: %v", err) + } + } + ambiguous, err := compactor.seal(t.Context(), []json.RawMessage{message("middle1234"), message("middXX1234")}) + if err != nil { + t.Fatal(err) + } + if _, err := compactor.restore(t.Context(), []json.RawMessage{mustMarshalJSON(carried), ambiguous}); err == nil { + t.Fatal("ambiguous truncation silently selected an original request") + } +} + +func TestCompactionRestoreClassifiedUsersAndSnapshotProvenance(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + classified := func(text string) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", + "content": []any{map[string]string{"type": "input_text", "text": text}}, + "internal_chat_message_metadata_passthrough": map[string]any{"content_item_kinds": []string{"user.text"}}, + }) + } + developer := mustMarshalJSON(map[string]any{"type": "message", "role": "developer", "content": "Current policy"}) + user := classified("middle1234") + items := []json.RawMessage{developer, user} + capsule, err := compactor.seal(t.Context(), items) + if err != nil { + t.Fatal(err) + } + got, err := compactor.restore(t.Context(), []json.RawMessage{classified("midd…1 tokens truncated…1234"), capsule}) + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(items)) { + t.Fatalf("classified user was not reconciled: %v", err) + } + got, err = compactor.restore(t.Context(), []json.RawMessage{capsule, capsule, capsule}) + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(items)) { + t.Fatalf("historical canonical context was duplicated across envelopes: %v", err) + } + fresh := mustMarshalJSON(map[string]any{"type": "message", "role": "developer", "content": "New policy"}) + got, err = compactor.restore(t.Context(), []json.RawMessage{fresh, user, capsule}) + want := []json.RawMessage{developer, fresh, user} + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(want)) { + t.Fatalf("classified user lost the fresh-instruction anchor: %v", err) + } +} + +func TestCompactionRestoreRejectsUnreconciledPartialUserContent(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + original := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{ + map[string]string{"type": "input_text", "text": "before"}, + map[string]string{"type": "input_image", "image_url": "data:image/png;base64,fixture"}, + map[string]string{"type": "input_text", "text": "after"}, + }}) + capsule, err := compactor.seal(t.Context(), []json.RawMessage{original}) + if err != nil { + t.Fatal(err) + } + partial := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{ + map[string]string{"type": "input_text", "text": "before"}, + map[string]string{"type": "input_text", "text": "after"}, + }}) + if _, err := compactor.restore(t.Context(), []json.RawMessage{partial, capsule}); err == nil { + t.Fatal("unreconciled multimodal user content was silently duplicated") + } +} + +func TestCompactionRestoreLegacyCanonicalUserContext(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + message := func(text string) json.RawMessage { + return mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": text}}}) + } + old := message("\n/old\n") + current := message("\n/current\n") + user := message("Continue carefully.") + capsule, err := compactor.seal(t.Context(), []json.RawMessage{old, user}) + if err != nil { + t.Fatal(err) + } + got, err := compactor.restore(t.Context(), []json.RawMessage{current, user, capsule}) + want := []json.RawMessage{old, current, user} + if err != nil || string(mustMarshalJSON(got)) != string(mustMarshalJSON(want)) { + t.Fatalf("legacy canonical user context lost or misplaced: %v", err) + } +} + +func TestCompactionResponsesHaveDistinctIdentities(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("provider called") }) + body := string(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": compactHTTPHistory()})) + previous := "" + for range 2 { + response := httptest.NewRecorder() + compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses/compact", strings.NewReader(body))) + var result map[string]json.RawMessage + _ = json.Unmarshal(response.Body.Bytes(), &result) + id := jsonString(result, "id") + if response.Code != http.StatusOK || id == "" || id == previous { + t.Fatal("separate compactions reused a response identity") + } + previous = id + } +} diff --git a/internal/router/context_compaction_test.go b/internal/router/context_compaction_test.go new file mode 100644 index 00000000..a097ea06 --- /dev/null +++ b/internal/router/context_compaction_test.go @@ -0,0 +1,150 @@ +package router + +import ( + "encoding/json" + "strings" + "testing" +) + +func compactTestCall(id, command string) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": "function_call", "name": "exec_command", "call_id": id, + "arguments": string(mustMarshalJSON(map[string]any{"cmd": command, "workdir": "/workspace"})), + }) +} + +func compactTestOutput(id, output string, exitCode any) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": id, + "output": string(mustMarshalJSON(map[string]any{ + "output": output, "exit_code": exitCode, "wall_time_seconds": 1, + })), + }) +} + +func TestContextCompactionPreservesIntentAndContinuation(t *testing.T) { + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Only change the router. Do not commit."}), + compactTestCall("tests", "go test -v ./internal/router"), + compactTestOutput("tests", strings.Repeat("=== RUN TestRoute\n--- PASS: TestRoute (0.01s)\n", 20)+"PASS\nok example/router 0.1s\n", 0), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "Router tests passed, but the endpoint is not finished."}), + mustMarshalJSON(map[string]any{"type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Need to check continuation before deployment."}}, "encrypted_content": "opaque"}), + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Actually keep the existing endpoint too."}), + compactTestCall("last", "go test -v ./internal/router"), + compactTestOutput("last", "=== RUN TestRoute\n--- PASS: TestRoute (0.01s)\nPASS\nok example/router 0.1s\n", 0), + } + before := string(mustMarshalJSON(items)) + got := reduceContextCompaction(items) + if len(got) != len(items) { + t.Fatal("compaction removed conversation items") + } + for index := range items { + if index != 2 && string(got[index]) != string(items[index]) { + t.Fatalf("protected item %d changed", index) + } + } + if string(got[2]) == string(items[2]) || !strings.Contains(string(got[2]), "ok example/router") { + t.Fatalf("routine test detail was not reduced with package evidence retained: %s (command kind %q)", got[2], contextCompactionCommand("go test -v ./internal/router")) + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("compaction modified the input") + } + if string(mustMarshalJSON(reduceContextCompaction(got))) != string(mustMarshalJSON(got)) { + t.Fatal("repeated compaction changed already compacted context") + } +} + +func TestContextCompactionKeepsUncertainExecutionEvidence(t *testing.T) { + for _, test := range []struct { + name, command, output string + exitCode any + }{ + {"failed", "go test -v ./...", "=== RUN TestA\n--- FAIL: TestA (0.1s)\nassertion details\nFAIL\n", 1}, + {"running", "go test -v ./...", "=== RUN TestA\n", nil}, + {"compound", "go test -v ./...; echo done", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, + {"pipeline", "go test -v ./... | tee result", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, + {"substitution", "go test $(echo ./...)", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, + {"wrapper", "rtk go test -v ./...", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, + {"unknown", "make check", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, + {"no_matches", "rg missing .", "", 1}, + {"search_error", "rg match missing", "rg: missing: No such file or directory", 2}, + } { + t.Run(test.name, func(t *testing.T) { + items := []json.RawMessage{compactTestCall("first", test.command), compactTestOutput("first", test.output, test.exitCode), + compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0)} + if string(mustMarshalJSON(reduceContextCompaction(items))) != string(mustMarshalJSON(items)) { + t.Fatal("uncertain, unfinished, failed, or unrecognized evidence changed") + } + }) + } +} + +func TestContextCompactionSearchRequiresRetainedExactEvidence(t *testing.T) { + listing := strings.Repeat("src/example.go\n", 40) + items := []json.RawMessage{ + compactTestCall("search", "rg --files src"), compactTestOutput("search", listing, 0), + compactTestCall("read", "cat files.txt"), compactTestOutput("read", listing, 0), + } + got := reduceContextCompaction(items) + if string(got[1]) == string(items[1]) || !strings.Contains(string(got[1]), "read") { + t.Fatal("duplicate search listing lacks retained-source reference") + } + if string(got[3]) != string(items[3]) { + t.Fatal("replacement evidence was modified") + } + items[3] = compactTestOutput("read", "different evidence\n", 0) + if string(mustMarshalJSON(reduceContextCompaction(items))) != string(mustMarshalJSON(items)) { + t.Fatal("search evidence removed without exact retained replacement") + } +} + +func TestContextCompactionKeepsUnknownTestDiagnostics(t *testing.T) { + items := []json.RawMessage{ + compactTestCall("tests", "go test -v ./..."), + compactTestOutput("tests", strings.Repeat("=== RUN TestA\n--- PASS: TestA (0.1s)\n", 20)+" test.go:10: important diagnostic\nPASS\nok example 0.1s\n", 0), + compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0), + } + got := reduceContextCompaction(items) + if string(got[1]) == string(items[1]) || !strings.Contains(string(got[1]), "important diagnostic") { + t.Fatal("non-routine diagnostic removed") + } +} + +func TestContextCompactionKeepsLiveHandlesAndAmbiguousCalls(t *testing.T) { + log := strings.Repeat("=== RUN TestA\n--- PASS: TestA (0.1s)\n", 20) + live := mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": "tests", + "output": string(mustMarshalJSON(map[string]any{"output": log, "exit_code": 0, "session_id": 42})), + }) + for _, items := range [][]json.RawMessage{ + {compactTestCall("tests", "go test -v ./..."), live, compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0)}, + {compactTestCall("tests", "go test -v ./..."), compactTestCall("tests", "pwd"), compactTestOutput("tests", log, 0), compactTestOutput("last", "/workspace\n", 0)}, + } { + if string(mustMarshalJSON(reduceContextCompaction(items))) != string(mustMarshalJSON(items)) { + t.Fatal("live continuation or ambiguous call identity changed") + } + } +} + +func TestContextCompactionNativeExecOutput(t *testing.T) { + header := "Chunk ID: abc\nWall time: 1.2500 seconds\nProcess exited with code 0\nOriginal token count: 900\nOutput:\n" + log := strings.Repeat("=== RUN TestNative\n--- PASS: TestNative (0.1s)\n", 30) + "PASS\nok example 0.1s\n" + for _, prefix := range []string{header, strings.Replace(header, "code 0", "code 1", 1), strings.Replace(header, "Process exited with code 0", "Process running with session ID 42", 1)} { + result := mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "tests", "output": prefix + log}) + items := []json.RawMessage{compactTestCall("tests", "go test -v ./..."), result, compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0)} + got := reduceContextCompaction(items) + var fields map[string]json.RawMessage + _ = json.Unmarshal(got[1], &fields) + text := jsonString(fields, "output") + if !strings.HasPrefix(text, prefix) { + t.Fatal("native execution header changed") + } + if prefix == header { + if text == prefix+log || !strings.Contains(text, "ok example") { + t.Fatal("native successful test output not reduced") + } + } else if string(got[1]) != string(result) { + t.Fatal("native failure or running output changed") + } + } +} diff --git a/internal/router/server.go b/internal/router/server.go index fd84e3f7..0a737a5e 100644 --- a/internal/router/server.go +++ b/internal/router/server.go @@ -246,7 +246,16 @@ func RunSession(ctx context.Context, args []string, issues *CriticalErrors, read webSocketEndpoint := responsesWebSocketHandler(ctx, *flags.timeout, provider, issues, mekugiCalls, compactTokens, mentor) defer webSocketEndpoint.Close() mux.Handle("GET /v1/responses", webSocketEndpoint) - mux.HandleFunc("POST /v1/responses", responsesHandler(ctx, *flags.timeout, provider, issues, mekugiCalls, compactTokens, mentor)) + // The compact boundary also restores router-owned envelopes before any + // ordinary request reaches projection or upstream transport. + compactionDirectory, err := mekugiDataDirectory() + if err != nil { + return err + } + compaction := &contextCompactor{keyPath: filepath.Join(compactionDirectory, "compaction.key")} + responses := compaction.handler(responsesHandler(ctx, *flags.timeout, provider, issues, mekugiCalls, compactTokens, mentor)) + mux.HandleFunc("POST /v1/responses", responses) + mux.HandleFunc("POST /v1/responses/compact", responses) server := &http.Server{ ErrorLog: log.New(io.Discard, "", 0), // Disable net/http terminal diagnostics while Codex owns it. From a46528542d6806ffd6b4acdb0388c92f249196cd Mon Sep 17 00:00:00 2001 From: yusing Date: Wed, 9 Sep 2026 04:53:00 +0000 Subject: [PATCH 02/13] fix(router): enable Codex local compaction requests Preserve the OpenAI provider identity and disable request compression so Codex routes manual and automatic compaction through the local JSON endpoints. Extend loopback coverage for synthetic ChatGPT authentication, manual compaction, and compressed-request rejection, and document the launcher behavior. --- README.md | 12 +- cmd/mekugi/wrap.go | 13 +- cmd/mekugi/wrap_test.go | 20 ++- doc/spec/compaction.md | 18 +- .../router/context_compaction_codex_test.go | 170 +++++++++++++++++- 5 files changed, 207 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 7f0cd64e..634e7af0 100644 --- a/README.md +++ b/README.md @@ -174,11 +174,13 @@ down when Codex exits. Multiple sessions can run independently. Codex handles terminal Ctrl-C, and its exit status is preserved. The wrapper uses the fixed Codex ChatGPT upstream and overrides provider -selection for that invocation only. Standalone serving, fixed ports, custom -providers, and provider-selection arguments such as `--oss` are not supported. -It also forces `include_collaboration_mode_instructions=false` for the invocation, -so Codex does not inject collaboration-mode instructions, even if enabled in your -config or command-line overrides. No configuration files are changed. +selection for that invocation only. Codex displays this provider as `OpenAI`; +its requests still go through the private Mekugi router. Standalone serving, +fixed ports, custom providers, and provider-selection arguments such as `--oss` +are not supported. It also forces +`include_collaboration_mode_instructions=false` for the invocation, so Codex +does not inject collaboration-mode instructions, even if enabled in your config +or command-line overrides. No configuration files are changed. The wrapper enables WebSockets between Codex and Mekugi for that invocation, without changing Codex configuration. Mekugi keeps the ChatGPT connection open diff --git a/cmd/mekugi/wrap.go b/cmd/mekugi/wrap.go index 377ffe23..8b828659 100644 --- a/cmd/mekugi/wrap.go +++ b/cmd/mekugi/wrap.go @@ -132,15 +132,24 @@ func wrapCodex(ctx context.Context, routerArgs, args []string) (code int, runErr } func codexArgs(baseURL string, args []string) []string { - // Keep overrides in the final command's config layer: Codex subcommands + // Keep router overrides in the final command's config layer: Codex subcommands // can replace pre-subcommand -c settings with their own. Never cross --. index := slices.Index(args, "--") if index < 0 { index = len(args) } + // Codex gates remote compaction on the OpenAI provider identity. Keep that + // identity for our fixed ChatGPT upstream so manual and automatic compact + // requests use the local compact interfaces instead of model summaries. + // Source: Codex model-provider/src/provider.rs ConfiguredModelProvider::capabilities. + // Disable both the feature toggle and final config value: Codex merges them + // in different layers for the TUI and subcommands. + // The local JSON boundary does not accept Codex's ChatGPT Zstd request bodies. return slices.Insert(slices.Clone(args), index, "-c", `model_provider="mekugi_wrap"`, - "-c", fmt.Sprintf(`model_providers.mekugi_wrap={name="mekugi",base_url=%q,wire_api="responses",requires_openai_auth=true,supports_websockets=true}`, baseURL), + "-c", fmt.Sprintf(`model_providers.mekugi_wrap={name="OpenAI",base_url=%q,wire_api="responses",requires_openai_auth=true,supports_websockets=true}`, baseURL), + "--disable", "enable_request_compression", + "-c", `features.enable_request_compression=false`, "-c", `include_collaboration_mode_instructions=false`, ) } diff --git a/cmd/mekugi/wrap_test.go b/cmd/mekugi/wrap_test.go index 0e2724b3..a688c631 100644 --- a/cmd/mekugi/wrap_test.go +++ b/cmd/mekugi/wrap_test.go @@ -24,12 +24,16 @@ func TestCodexArgsPreservesArguments(t *testing.T) { forwarded := []string{"exec", "-c", "model=\"example\"", "--", "a prompt with spaces"} args := codexArgs("http://127.0.0.1:12345/v1", forwarded) index := slices.Index(forwarded, "--") - if !slices.Equal(args[:index], forwarded[:index]) || !slices.Equal(args[index+6:], forwarded[index:]) { + if !slices.Equal(args[:index], forwarded[:index]) || !slices.Equal(args[index+10:], forwarded[index:]) { t.Fatalf("forwarded arguments changed: %q", args) } + if !slices.Equal(args[index+4:index+6], []string{"--disable", "enable_request_compression"}) { + t.Fatal("wrapped requests must disable the compression feature toggle") + } var config struct { - IncludeCollaborationModeInstructions *bool `toml:"include_collaboration_mode_instructions"` - ModelProvider string `toml:"model_provider"` + Features map[string]bool `toml:"features"` + IncludeCollaborationModeInstructions *bool `toml:"include_collaboration_mode_instructions"` + ModelProvider string `toml:"model_provider"` Providers map[string]struct { Name string `toml:"name"` BaseURL string `toml:"base_url"` @@ -39,7 +43,10 @@ func TestCodexArgsPreservesArguments(t *testing.T) { } `toml:"model_providers"` } var settings []string - for i := index; i < index+6; i += 2 { + for i := index; i < index+10; i += 2 { + if i == index+4 { + continue + } if args[i] != "-c" { t.Fatalf("not a config override: %q", args) } @@ -51,8 +58,11 @@ func TestCodexArgsPreservesArguments(t *testing.T) { if config.IncludeCollaborationModeInstructions == nil || *config.IncludeCollaborationModeInstructions { t.Fatalf("collaboration mode instructions not disabled: %q", args) } + if enabled, set := config.Features["enable_request_compression"]; !set || enabled { + t.Fatal("wrapped requests must remain uncompressed JSON") + } provider := config.Providers[config.ModelProvider] - if provider.Name == "" || provider.BaseURL != "http://127.0.0.1:12345/v1" || provider.WireAPI != "responses" || !provider.Auth || !provider.WebSockets { + if provider.Name != "OpenAI" || provider.BaseURL != "http://127.0.0.1:12345/v1" || provider.WireAPI != "responses" || !provider.Auth || !provider.WebSockets { t.Fatalf("provider = %+v", provider) } withoutDelimiter := []string{"exec", "-c", `model="example"`, "prompt"} diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index 8199c82f..f704a707 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -7,6 +7,13 @@ compaction threshold and counting scope, model changes, and manual compaction requests remain unchanged. Hpatch does not rewrite Codex settings or schedule an earlier trigger. +The launcher preserves the `OpenAI` provider identity for its fixed ChatGPT +upstream while routing the base URL to the local listener. Codex uses that +identity to select remote compaction for both manual and automatic triggers; +an unrecognized provider name selects model summarization instead. +The invocation also disables Codex request compression because the local router +accepts uncompressed JSON, not ChatGPT Zstd request bodies. + The router handles `POST /v1/responses/compact` locally. It also handles streaming `POST /v1/responses` requests identified by Codex metadata as `responses_compaction_v2`. Neither path calls a provider. Other metadata-tagged @@ -61,9 +68,10 @@ archive, session cache requirement, or model-operated retrieval step. Acceptance checks cover local HTTP completion without provider calls, native restoration with fresh context and suffixes, repeated compaction, restart and concurrent key creation, damaged or missing keys, and conservative output pruning. -Installed Codex 0.153.4 has passed loopback legacy and V2 automatic-compaction -round trips with both counting scopes and a large user request that is truncated -by the client's V2 retention step, then restored in full without duplication. -Other versions and manual/resumed client flows need corresponding -runtime coverage. Paired outcome evaluation is still required before claiming +Installed Codex 0.153.4 has passed loopback legacy and V2 round trips with +synthetic ChatGPT authentication: automatic compaction with both counting scopes, +and the manual compact operation used by `/compact`. A large user request is +truncated by the client's V2 retention step, then restored in full without +duplication. Other versions and resumed client flows need corresponding runtime +coverage. Paired outcome evaluation is still required before claiming an optimal output budget or task-critical semantic preservation on real histories. diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go index 5f637385..0bcc2abb 100644 --- a/internal/router/context_compaction_codex_test.go +++ b/internal/router/context_compaction_codex_test.go @@ -2,6 +2,7 @@ package router import ( "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -26,8 +27,9 @@ func TestCompactionInstalledCodex(t *testing.T) { for _, probe := range []struct { legacy bool scope string - }{{false, "total"}, {false, "body_after_prefix"}, {true, "total"}, {true, "body_after_prefix"}} { - t.Run(fmt.Sprintf("legacy=%v/scope=%s", probe.legacy, probe.scope), func(t *testing.T) { + manual bool + }{{false, "total", false}, {false, "body_after_prefix", false}, {true, "total", false}, {true, "body_after_prefix", false}, {false, "total", true}, {true, "total", true}} { + t.Run(fmt.Sprintf("legacy=%v/scope=%s/manual=%v", probe.legacy, probe.scope, probe.manual), func(t *testing.T) { prompt := "Run the Go tests, then print the working directory, then report completion. Preserve the test result.\n" + strings.Repeat("Keep the original user constraint. ", 10000) + "\nThis final instruction must also survive intact." @@ -46,6 +48,19 @@ func TestProbe(t *testing.T) { t.Fatal(err) } } + // Synthetic ChatGPT auth exercises the same compression gate as the + // launcher, without reading credentials or contacting an auth service. + // Source: Codex app-server/tests/common/auth_fixtures.rs. + encode := base64.RawURLEncoding.EncodeToString + idToken := encode([]byte(`{"alg":"none","typ":"JWT"}`)) + "." + + encode([]byte(`{"https://api.openai.com/auth":{"chatgpt_plan_type":"pro"}}`)) + "." + encode([]byte("signature")) + auth := map[string]any{ + "auth_mode": "chatgpt", "last_refresh": time.Now().UTC().Format(time.RFC3339), + "tokens": map[string]any{"id_token": idToken, "access_token": "loopback-test-access", "refresh_token": "loopback-test-refresh"}, + } + if err := os.WriteFile(filepath.Join(home, "auth.json"), mustMarshalJSON(auth), 0o600); err != nil { + t.Fatal(err) + } compactor := &contextCompactor{keyPath: filepath.Join(home, "compaction.key")} var normal, compacted atomic.Int32 var restored atomic.Bool @@ -70,6 +85,9 @@ func TestProbe(t *testing.T) { if step == 2 { command, callID, inputTokens = "pwd", "probe_pwd", 300000 } + if probe.manual { + inputTokens = 1000 + } item = map[string]any{ "type": "function_call", "id": fmt.Sprintf("fc_probe_%d", step), "call_id": callID, "name": "exec_command", "status": "completed", @@ -97,7 +115,7 @@ func TestProbe(t *testing.T) { t.Error("local ciphertext reached the model fixture") } } - if userCopies != 1 { + if compacted.Load() > 0 && userCopies != 1 { t.Errorf("restored full user request copies = %d, want exactly one", userCopies) } item = map[string]any{ @@ -122,6 +140,19 @@ func TestProbe(t *testing.T) { } }) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/v1/models" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"models":[]}`) + return + } + if r.Header.Get("Content-Encoding") != "" { + t.Error("Codex sent compressed JSON to the router") + http.Error(w, "request compression unsupported", http.StatusBadRequest) + return + } + if r.Header.Get("Authorization") == "" { + t.Error("synthetic ChatGPT authentication was not applied") + } metadata, _ := decodeCodexTurnMetadata(r.Header) if metadata.RequestKind == "compaction" || r.URL.Path == "/v1/responses/compact" { compacted.Add(1) @@ -154,12 +185,17 @@ model_provider = "loopback" model_context_window = 1000000 model_auto_compact_token_limit = 200000 model_auto_compact_token_limit_scope = %q +approval_policy = "never" +sandbox_mode = "danger-full-access" tool_output_token_limit = 20000 +cli_auth_credentials_store_mode = "file" +[features] +enable_request_compression = false [model_providers.loopback] -name = "Azure" +name = "OpenAI" base_url = %q wire_api = "responses" -requires_openai_auth = false +requires_openai_auth = true supports_websockets = false request_max_retries = 0 stream_max_retries = 0 @@ -172,20 +208,136 @@ metrics_exporter = "none" t.Fatal(err) } args := []string{"exec", "--skip-git-repo-check", "--ephemeral", "--json", "--dangerously-bypass-approvals-and-sandbox"} + if probe.manual { + args = []string{"app-server"} + } + // Mirror the launcher's final overrides even when the caller enables + // compression through both CLI feature flags and subcommand config. + args = append(args, "--enable", "enable_request_compression", "-c", "features.enable_request_compression=true", + "--disable", "enable_request_compression", "-c", "features.enable_request_compression=false") if probe.legacy { args = append(args, "--disable", "remote_compaction_v2") } - args = append(args, "-") + if !probe.manual { + args = append(args, "-") + } ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second) defer cancel() command := exec.CommandContext(ctx, binary, args...) - command.Stdin = strings.NewReader(prompt) command.Dir = directory command.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"), "CODEX_HOME=" + home} - output, err := command.CombinedOutput() - if err != nil || compacted.Load() != 1 || normal.Load() != 3 || !restored.Load() { + var output []byte + var err error + wantNormal := int32(3) + if probe.manual { + wantNormal = 4 + err = runManualCompactionProbe(command, directory, prompt) + } else { + command.Stdin = strings.NewReader(prompt) + output, err = command.CombinedOutput() + } + if err != nil || compacted.Load() != 1 || normal.Load() != wantNormal || !restored.Load() { + t.Fatalf("client round trip: err=%v normal=%d compactions=%d restored=%v; output=%s", err, normal.Load(), compacted.Load(), restored.Load(), output) } }) } } + +// The app-server operation uses the same Op::Compact as the TUI's /compact. +// Source: Codex app-server/tests/suite/v2/compaction.rs. +func runManualCompactionProbe(command *exec.Cmd, directory, prompt string) error { + stdin, err := command.StdinPipe() + if err != nil { + return err + } + stdout, err := command.StdoutPipe() + if err != nil { + return err + } + if err := command.Start(); err != nil { + return err + } + defer func() { + _ = stdin.Close() + _ = command.Process.Kill() + _ = command.Wait() + }() + encoder, decoder := json.NewEncoder(stdin), json.NewDecoder(stdout) + send := func(id int, method string, params any) error { + return encoder.Encode(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}) + } + type rpcMessage struct { + ID int `json:"id"` + Method string `json:"method"` + Result json.RawMessage `json:"result"` + Params json.RawMessage `json:"params"` + Error json.RawMessage `json:"error"` + } + receive := func(id int, method string) (rpcMessage, error) { + for { + var message rpcMessage + if err := decoder.Decode(&message); err != nil { + return message, fmt.Errorf("app-server read: %w", err) + } + if len(message.Error) > 0 || message.Method == "error" { + return message, fmt.Errorf("app-server error: %+v", message) + } + if (id != 0 && message.ID == id) || (method != "" && message.Method == method) { + return message, nil + } + } + } + if err := send(1, "initialize", map[string]any{"clientInfo": map[string]any{"name": "hpatch_compaction_test", "version": "1"}}); err != nil { + return err + } + if _, err := receive(1, ""); err != nil { + return err + } + if err := encoder.Encode(map[string]any{"jsonrpc": "2.0", "method": "initialized"}); err != nil { + return err + } + if err := send(2, "thread/start", map[string]any{"cwd": directory}); err != nil { + return err + } + message, err := receive(2, "") + if err != nil { + return err + } + var started struct { + Thread struct { + ID string `json:"id"` + } `json:"thread"` + } + if err := json.Unmarshal(message.Result, &started); err != nil { + return err + } + for index, text := range []string{prompt, "", "Report the preserved test result."} { + method := "turn/start" + params := map[string]any{"threadId": started.Thread.ID} + if index == 1 { + method = "thread/compact/start" + } else { + params["input"] = []any{map[string]any{"type": "text", "text": text, "textElements": []any{}}} + } + if err := send(index+3, method, params); err != nil { + return err + } + if _, err := receive(index+3, ""); err != nil { + return err + } + message, err := receive(0, "turn/completed") + if err != nil { + return err + } + var completed struct { + Turn struct { + Status string `json:"status"` + } `json:"turn"` + } + if err := json.Unmarshal(message.Params, &completed); err != nil || completed.Turn.Status != "completed" { + return fmt.Errorf("app-server turn failed: %s: %v", message.Params, err) + } + } + return nil +} From 68eb79df1eefe297beb22c4a9e753584aed8982d Mon Sep 17 00:00:00 2001 From: yusing Date: Wed, 9 Sep 2026 15:27:27 +0000 Subject: [PATCH 03/13] feat(router): retire completed history during local compaction Extend provider-free compaction with conservative retirement of older finished operations, repeated narration, source rows, recognized documentation bodies, and unreferenced transport metadata. Preserve authority, active work, references, diagnostics, failures, unknown states, and native restoration while replacing eligible call/result groups with versioned factual records. Add reference-closure, profitability, carrier, ledger, metadata, narration, and retirement coverage across legacy and V2 Codex flows, and document the expanded lossy-retention contract. --- README.md | 15 +- doc/architecture/compaction.md | 55 +- doc/spec/compaction.md | 116 +- internal/router/context_compaction.go | 21 +- .../router/context_compaction_closure_test.go | 170 +++ .../router/context_compaction_codex_test.go | 99 +- internal/router/context_compaction_http.go | 9 +- .../router/context_compaction_http_test.go | 34 + .../router/context_compaction_ledger_test.go | 529 +++++++++ .../router/context_compaction_metadata.go | 242 ++++ .../context_compaction_metadata_test.go | 325 ++++++ .../router/context_compaction_narration.go | 217 ++++ .../context_compaction_narration_test.go | 137 +++ .../router/context_compaction_operation.go | 382 ++++++ .../router/context_compaction_read_tool.go | 602 ++++++++++ .../context_compaction_read_tool_test.go | 559 +++++++++ internal/router/context_compaction_records.go | 157 +++ .../router/context_compaction_records_test.go | 77 ++ ...ontext_compaction_reference_decode_test.go | 312 +++++ .../router/context_compaction_repeated.go | 195 ++++ .../context_compaction_repeated_test.go | 439 +++++++ .../router/context_compaction_retirement.go | 1032 +++++++++++++++++ .../context_compaction_retirement_test.go | 574 +++++++++ internal/router/context_compaction_source.go | 569 +++++++++ .../router/context_compaction_source_test.go | 425 +++++++ 25 files changed, 7236 insertions(+), 56 deletions(-) create mode 100644 internal/router/context_compaction_closure_test.go create mode 100644 internal/router/context_compaction_ledger_test.go create mode 100644 internal/router/context_compaction_metadata.go create mode 100644 internal/router/context_compaction_metadata_test.go create mode 100644 internal/router/context_compaction_narration.go create mode 100644 internal/router/context_compaction_narration_test.go create mode 100644 internal/router/context_compaction_operation.go create mode 100644 internal/router/context_compaction_read_tool.go create mode 100644 internal/router/context_compaction_read_tool_test.go create mode 100644 internal/router/context_compaction_records.go create mode 100644 internal/router/context_compaction_records_test.go create mode 100644 internal/router/context_compaction_reference_decode_test.go create mode 100644 internal/router/context_compaction_repeated.go create mode 100644 internal/router/context_compaction_repeated_test.go create mode 100644 internal/router/context_compaction_retirement.go create mode 100644 internal/router/context_compaction_retirement_test.go create mode 100644 internal/router/context_compaction_source.go create mode 100644 internal/router/context_compaction_source_test.go diff --git a/README.md b/README.md index 634e7af0..d5a79902 100644 --- a/README.md +++ b/README.md @@ -125,10 +125,17 @@ provider request. The first compaction creates an owner-only key at `~/.config/hpatch/compaction.key` on Linux). Keep that key to resume compacted sessions, including when moving them to another installation. -The initial reducers handle only recognized search listings and verbose Go test -results. Uncertain evidence is retained; when nothing can safely be reduced, -compaction reports an error instead of dropping context or asking a provider -for a summary. The retained history is not guaranteed to fit a 30k-token window. +Compaction can discard unmarked historical details from older finished operations, +even while the task is still open. It keeps factual execution records, requests, +visible decisions, diagnostic excerpts, referenced evidence, and recent/live work. +Recognized applied patch bodies and associated older opaque reasoning can also be +retired. Older failed-command output and truncated documentation can lose +unreferenced bulk while retaining errors, warnings, and provenance. +Discarded details are not currently retrievable through Hpatch. + +Unknown or ambiguous execution states remain intact. If nothing qualifies, +compaction reports an error rather than asking a provider for a summary. +Compaction does not guarantee a fixed retained-history size. Then launch: diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index ae13458d..05ce52fa 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -8,9 +8,54 @@ restoration of its own input envelopes, and bounded request decoding. Codex owns configuration resolution, trigger timing, retained client-side context, and installation of the returned compaction result. -Command-aware reducers own only evidence selection. They do not execute commands, -interpret opaque provider state, infer task completion, or choose a fixed context -budget. Unknown forms retain their evidence. +The selector owns evidence reduction and the approved lossy retirement of finished +operations. Static carrier parsing establishes invocation facts; terminal output +establishes completion, not task closure. It does not execute scripts or interpret +opaque reasoning. Unknown lifecycle/dependency states stay protected. + +Retirement records replace complete native call/result pairs in their original +relative order. Consecutive newly generated records may share framing after +reference closure; authenticated native restoration remains the envelope owner's +responsibility. Reasoning groups are retired atomically with eligible calls, +retaining visible summaries. The recent frontier, explicit evidence references, +live operations, and unknown completion states constrain selection. Completed +failures may lose identified historical bulk, but keep their exact invocation, +status, diagnostic blocks, and unresolved details. Reference closure and complete-group +profitability reach a stable result before replacement; restoring a consumer also +restores every dependency exposed by its original content. Supported text and +JavaScript escapes are decoded for reference matching, while suspicious encodings +retain evidence conservatively. These records are factual history, not new +instructions or an external archive. The envelope contains selected history only; +deferred retrieval must not be implied by a digest or retirement marker. + +Completed native output reduction is independent of whole-group retirement: it +may shorten historical output while leaving calls and opaque reasoning native. +Both paths share reference-preservation rules. Static documentation reducers own +only recognized bodies and unambiguous body fragments in truncated documentation; +provenance, warnings, references, unknown metadata, and uncertain fragments remain +exact. They do not reconstruct missing structure or infer a successful outcome. + +Historical ordinary-assistant narration reduction consolidates exact repeated text rather +than guessing whether an arbitrary sentence is routine progress. It preserves +decision-bearing text together with its qualifications and keeps other prose unchanged. +Agent-message content remains byte-exact for client reconciliation, including +items without a unique ID whose identity depends on their canonical JSON. +User/developer authority and the active frontier do not become historical narration +merely because they are old. No reduction step calls a provider or claims that +omitted details can be retrieved. + +The selector may remove unreferenced transport turn/time fields from older native +items only when unique stable IDs preserve reconciliation, and from factual records +only when retained content does not reference their exact values. Content +classifications, roles, phases, opaque payloads, and unknown metadata remain with +their original owners. Whole-output references stay native; exact verified-row +evidence may travel in a factual record without retaining the entire successful +operation group. + +Ordinary historical assistant narration is not carried beside the capsule by the +supported local Codex compaction flows, so its unreferenced transport item IDs may +be omitted. User and agent-message identities remain stable because those items +can be carried by the client. Referenced IDs and recent items remain protected. The envelope owner authenticates and encrypts the retained native item array. Its persistent key belongs to the Hpatch configuration directory, not a thread, @@ -18,6 +63,10 @@ temporary plugin runtime, provider credential, or capture stream. Cross-process locking serializes first creation. Compression is an internal envelope-storage detail, not the semantic compaction mechanism or a token-usage measurement. +Legacy responses expose real-user messages beside the envelope for Codex's input +handling. Historical canonical context stays only inside the envelope, so this +router-authored carry list cannot be misclassified as a fresh instruction event. + Restoration aligns the client's carried prefix against the retained native timeline backwards by stable item/call identity or canonical JSON. Supported client truncation must match a unique authenticated original. Fresh canonical context is never diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index f704a707..8e03b107 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -20,8 +20,16 @@ The router handles `POST /v1/responses/compact` locally. It also handles streami compaction implementations fail explicitly instead of requesting a provider summary. Local handling applies in both router modes. -Compaction preserves all history items and their order. Its initial reducers -only change completed historical tool output: +Compaction preserves user/developer instructions, corrections, authorization, +and the active execution frontier. Older ordinary-assistant narration can omit an +earlier byte-identical body when a later occurrence remains. Other prose, including +routine progress, stays unchanged; agent-message content remains byte-exact for +client reconciliation. Decisions, qualifications, outcomes, unresolved issues, +and continuation state remain intact. The approved policy permits loss of unmarked historical +details from finished operations inside an ongoing task. It does not claim that +completion proves irrelevance. + +Before lossy retirement, evidence reducers can shorten redundant output: - Successful Go test results with a structured result or native Codex exec header can omit routine run/pause/continue and pass lines. The exact call, exit status, package summaries, and other output @@ -29,21 +37,101 @@ only change completed historical tool output: - Successful search listings from recognized `rg`, `hgrep`, or `find` calls can be replaced with a reference to a later retained read result containing the exact same complete listing. The original call and completion metadata stay. -- The last tool result, failed or unfinished operations, live handles, unknown - output formats, unsupported wrappers, compound commands, pipelines, dynamic - shell expressions, and unrecognized commands remain unchanged. -- User corrections, commentary, reasoning, existing provider-encrypted items, - decisions, identifiers, unknown fields, and current work are not blanket-pruned. +- Repeated file excerpts in completed successful tool results can be replaced + with references to identical source rows retained in later results. Matching + requires at least four consecutive complete `LINE:HASH` rows and 256 bytes. + Unique text, call identity, completion metadata, and the later evidence remain. + This includes Code Mode's completed `input_text` result arrays carrying + structured shell output; it does not require interpreting the script. + +Finished-operation retirement: +- Keep the newest eight tool invocations and their results, live/unmatched or + duplicate identities, unknown completion states, and explicitly + referenced calls or retained-script producers. Referenced verified rows and + numeric ranges may survive exactly in factual records instead of pinning their + whole successful group. Reference matching decodes nested argument/result + envelopes and supported escaped text; suspicious encodings retain evidence. +- Recognize native shell/exec calls, terminal stdin polls, static result-preserving + Code Mode exec/poll carriers, static OpenAI documentation search/fetch/OpenAPI + calls, and router-generated awaited apply-patch carriers. Static JSON-compatible + JavaScript literals are parsed without execution; dynamic expressions remain native. + A literal shell-carrier progress notice is retained verbatim and checked against + the corresponding result part; its evidence references remain protected. + Successful documentation results retain provenance, identities, hierarchy, + pagination, annotations, and unknown metadata while recognized unmarked bodies + may shrink. In truncated documentation, only historical body fragments with + unambiguous boundaries may shrink; provenance, warnings, references, and + uncertain material remain. Reduction must not fabricate missing structure or + represent a truncated result as complete. A known successful + result need not shrink individually for its complete group to be profitable. + Dynamic scripts and status-overwriting projections are not retirement evidence. +- Replace eligible calls/results with factual, versioned assistant-role records, + not executable-looking truncated calls. Preserve relative timeline order; + consecutive newly generated historical records may share explanatory framing. + Keep exact shell invocation arguments, call identity, observed completion + metadata, test outcome lines, and diagnostic excerpts. Standard Python + tracebacks preserve the entire remaining output because multiline exceptions + and notes have no reliable generic end marker. +- A completed failed command is eligible for removal of positively identified, + unreferenced historical bulk, not removal of its failure. Keep its exact + invocation, exit status, error and diagnostic blocks, traceback chains, + unresolved details, and referenced evidence. Terminal completion is not + success or proof that a failure was resolved. Live and unknown completion + states remain protected. +- For successfully applied translated patches, keep affected paths, operation + kinds, a patch digest, and exact application facts, diagnostics, and referenced + evidence. Unreferenced verified source rows in a successful report may shrink + under the same source-evidence rules as other completed output. The digest + identifies the omitted patch body; it is not a retrieval mechanism. +- Preserve visible reasoning summaries. Retire opaque reasoning only with its + complete eligible reasoning/tool group, before the recent frontier. A group + containing unknown, crossing, or protected calls stays native. + Evaluate savings for the complete reasoning/tool group, not each call in + isolation: a small completion record may grow when the eligible group shrinks. + Reference closure and profitability settle to a stable result before any + replacements are applied. If profitability restores original content, every + reference exposed by that content is followed before selection completes. + Older completed shell outputs may also shrink without changing native calls or + reasoning in a blocked group. Whole-output references stay intact; referenced + full or partial verified rows and numeric ranges remain exact. Live, + unknown, and recent output is not made eligible by this output-only pass; + failed output obeys the diagnostic-preservation rule above. + Existing provider-owned compaction items remain untouched. + Factual records use a compact versioned representation with exact invocation + values, ordered output parts, and unknown metadata. Unreferenced transport item + IDs and transport turn/time bookkeeping may be omitted from retired records; + explicitly referenced turn/time values remain exact. This does not change + native event identity or discard substantive content. +- Preserve recorded reference targets across repeated compaction. Previously + retired details cannot be reconstructed if work is reopened. No archival or + model-operated retrieval facility is implemented yet. + +Older native items with unique stable IDs may omit unreferenced nested transport +turn IDs and timestamps. Older ordinary assistant narration may also omit an +unreferenced transport item ID: the supported local Codex legacy and V2 workflows +do not carry those messages alongside the compaction item. User and agent-message +identities remain stable for carried-history reconciliation. Roles, phases, +content classifications, opaque payloads, substantive content, and unknown metadata +remain with their existing owners. Turn/time cleanup leaves no-ID, duplicate-ID, +malformed, unknown-kind, and recent items unchanged. Legacy and V2 carried messages +still reconcile by their original stable identity. + +Operation completion is not task completion. Execution records report observed +facts without inventing scope closure, successful validation, or a workspace +version. User corrections and visible reasoning/decision text are not blanket-pruned. -If no safe reduction is available, compaction fails with HTTP 422. It does not +If no supported reduction is available, compaction fails with HTTP 422. It does not discard protected context just to fit a budget, report a fabricated summary, -or fall back to provider compaction. The proposed 30k-token output target and -5k allowance remain an evaluation hypothesis, not an enforced limit or a proven -preservation guarantee. Encoded size is not the restored model context size. +or fall back to provider compaction. The target replay must retain 50,000 or fewer +visible-string tokens using the first native replay boundary and `o200k_base`, +excluding opaque reasoning and request/tool framing. This acceptance target is +not a universal cap on arbitrary histories or a complete provider-context count. +Encoded size and downstream projection savings do not establish this target. The retained native timeline travels inline in an authenticated, encrypted -router-owned compaction item. Legacy output also carries original user messages -for Codex's own user-input handling. V2 emits exactly one completed compaction +router-owned compaction item. Legacy output also carries original real-user messages +for Codex's own user-input handling. Historical canonical instructions and +environment context stay only in the envelope, avoiding false fresh injections. V2 emits exactly one completed compaction item. On subsequent requests, the router restores the timeline before any projection or forwarding. It reconciles carried native items without duplicating matched messages, preserves newly injected context and post-compaction input, @@ -55,7 +143,7 @@ Fresh canonical instructions retain their current position even when their text matches an older instruction. The versioned local payload is never sent upstream as provider-encrypted state. -Existing provider-owned compaction and reasoning payloads remain untouched. +Existing provider-owned compaction payloads remain untouched; reasoning retirement follows the complete-group rule above. Malformed, unknown-version, nested, unauthenticated, or unreadable local envelopes fail closed. Both encoded requests and restored history obey router memory bounds. diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 1e5dfed0..298adeb7 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -11,10 +11,16 @@ import ( "mvdan.cc/sh/v3/syntax" ) -// reduceContextCompaction changes only recognized, completed historical tool -// output. Requests, decisions, opaque state, calls, order, and the last result -// remain intact. It makes no claim that the surviving history fits a token cap. +// reduceContextCompaction retains authority and the active frontier while +// reducing redundant evidence and retiring eligible finished operations under +// the explicit lossy retention policy. It does not guarantee a fixed token cap. func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { + // Clean transport metadata while stable native IDs are still present. + // Narration reduction may remove an unreferenced ordinary-assistant ID, + // after which the metadata pass must conservatively leave that item alone. + input = reduceContextCompactionMetadata(input) + input = reduceContextCompactionNarration(input) + protected := contextCompactionReferencedResults(input) output := slices.Clone(input) type item struct { Type string `json:"type"` @@ -44,7 +50,7 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { } } for index, current := range items { - if index == lastResult || (current.Type != "function_call_output" && current.Type != "custom_tool_call_output") { + if index == lastResult || protected[current.CallID] || (current.Type != "function_call_output" && current.Type != "custom_tool_call_output") { continue } callIndex, exists := calls[current.CallID] @@ -98,7 +104,7 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { case "search": // A later byte-identical output is explicit replacement evidence, // not an assumption that rerunning a search gives its old answer. - // Only read outputs qualify, so this source cannot itself be reduced. + // Keep the referenced read intact in this and subsequent compactions. if len(text) < 256 { continue } @@ -127,6 +133,7 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { } _, evidence, ok := contextCompactionOutput(candidate.Output) if ok && evidence == text { + protected[candidate.CallID] = true reduced = fmt.Sprintf("[hpatch compaction: matching search listing retained verbatim in tool result %q; original command and successful exit status retained]\n", candidate.CallID) break } @@ -142,7 +149,9 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { fields["output"] = encode(reduced) output[index] = mustMarshalJSON(fields) } - return output + retained := reduceContextCompactionSource(input, + retireCompactionOperations(reduceRepeatedCompactionRows(output, protected))) + return consolidateContextCompactionRecords(input, retained) } // Accept structured results or Codex's native completed-exec header. Unknown diff --git a/internal/router/context_compaction_closure_test.go b/internal/router/context_compaction_closure_test.go new file mode 100644 index 00000000..aa1b980f --- /dev/null +++ b/internal/router/context_compaction_closure_test.go @@ -0,0 +1,170 @@ +package router + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestCompactionRetirementReclosesReferencesAfterProfitabilityRestore(t *testing.T) { + items := retirementHistory() + note := "[hpatch compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result \"operation_01\"]\n" + var sourceOutput string + for rowWords := 8; rowWords <= 512 && sourceOutput == ""; rowWords *= 2 { + for fillerLines := range 80 { + candidate := "17:abcd " + strings.Repeat("exact evidence ", rowWords) + "\n" + note + + strings.Repeat("unmarked historical detail\n", fillerLines) + if compactionClosureProfitabilityFixture(t, items[1:4], candidate) { + sourceOutput = candidate + break + } + } + } + if sourceOutput == "" { + t.Fatal("could not construct a row-preserving profitability boundary") + } + + items[3] = compactTestOutput("operation_00", sourceOutput, 0) + items[6] = compactTestOutput("operation_01", strings.Repeat("later exact evidence\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Continue from exact row 17:abcd.", + })) + + got := reduceContextCompaction(items) + for index := 1; index <= 6; index++ { + if string(got[index]) != string(items[index]) { + t.Fatalf("profitability restore left referenced operation item %d retired", index) + } + } +} + +// compactionClosureProfitabilityFixture finds an output that is profitable to +// retire before its exact row is retained, but not after. The complete pipeline +// test above then verifies that restoring this consumer exposes and follows its +// original operation reference. +func compactionClosureProfitabilityFixture(t *testing.T, group []json.RawMessage, output string) bool { + t.Helper() + var reasoning, call, result map[string]json.RawMessage + if json.Unmarshal(group[0], &reasoning) != nil || json.Unmarshal(group[1], &call) != nil || json.Unmarshal(group[2], &result) != nil { + t.Fatal("invalid profitability fixture") + } + operation, ok := compactionOperationCall(call) + if !ok { + t.Fatal("profitability fixture call was not recognized") + } + result["output"] = compactTestOutputValue(output, 0) + base, ok := compactionRetiredOutput(result["output"], operation) + if !ok { + t.Fatal("profitability fixture output was not recognized") + } + withRow, ok := compactionRetiredOutputKeepingRows(result["output"], operation, map[string]bool{"17:abcd": true}, nil) + if !ok { + t.Fatal("row-preserving profitability fixture output was not recognized") + } + profitable := func(retiredOutput json.RawMessage) bool { + original := []json.RawMessage{group[0], group[1], mustMarshalJSON(result)} + replacement := []json.RawMessage{ + compactionRetiredReasoning(reasoning), + compactionRetiredCall(call, operation), + compactionRetiredResult(result, retiredOutput), + } + beforeTokens, beforeOK := compactionVisibleStringTokens(original...) + afterTokens, afterOK := compactionVisibleStringTokens(replacement...) + return beforeOK && afterOK && + len(mustMarshalJSON(original))-len(mustMarshalJSON(replacement)) > 0 && + beforeTokens-afterTokens >= 0 + } + return profitable(base) && !profitable(withRow) +} + +func compactTestOutputValue(output string, exitCode any) json.RawMessage { + return mustMarshalJSON(string(mustMarshalJSON(map[string]any{"exit_code": exitCode, "output": output}))) +} + +func TestCompactionRetirementDecodesRetainedCarrierReferences(t *testing.T) { + t.Run("supported JavaScript escape keeps exact row", func(t *testing.T) { + items := retirementHistory() + referenced := "17:abcd exact source evidence\n" + items[3] = compactTestOutput("operation_00", referenced+strings.Repeat("old detail\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "exec", "call_id": "pending_dynamic", + "input": `const row="17\u003aabcd"; text(row);`, + })) + + got := reduceContextCompaction(items) + wire := string(mustMarshalJSON(got)) + if !strings.Contains(wire, "historical facts v4") || + !strings.Contains(wire, strings.TrimSpace(referenced)) || strings.Contains(wire, "old detail") { + t.Fatal("escaped carrier reference did not preserve only its exact source row") + } + }) + + t.Run("percent escape keeps exact row without pinning unrelated evidence", func(t *testing.T) { + items := retirementHistory() + referenced := "17:abcd exact percent-decoded source evidence\n" + items[3] = compactTestOutput("operation_00", referenced+strings.Repeat("old detail\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "exec", "call_id": "pending_dynamic", + "input": `const row="17%3aabcd"; text(row);`, + })) + + got := reduceContextCompaction(items) + wire := string(mustMarshalJSON(got)) + if !strings.Contains(wire, "historical facts v4") || + !strings.Contains(wire, strings.TrimSpace(referenced)) || strings.Contains(wire, "old detail") { + t.Fatal("percent-decoded carrier reference did not preserve only its exact source row") + } + }) +} + +func TestCompactionRetirementPreservesReferencedTransportMetadata(t *testing.T) { + items := retirementHistory() + setFields := func(index int, values map[string]any) { + t.Helper() + var fields map[string]json.RawMessage + if json.Unmarshal(items[index], &fields) != nil { + t.Fatal("invalid metadata fixture item") + } + for key, value := range values { + fields[key] = mustMarshalJSON(value) + } + items[index] = mustMarshalJSON(fields) + } + setFields(2, map[string]any{ + "turn_id": "turn_call_keep", + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_nested_keep", "create_time": 2002, + "content_item_kinds": []any{"function_call"}, + }, + }) + setFields(3, map[string]any{ + "create_time": 1003, + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_nested_drop", "create_time": 2003, + "content_item_kinds": []any{"function_call_output"}, + }, + }) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": map[string]any{ + "turns": "Use turn_call_keep and escaped turn\u005fnested\u005fkeep.", + "observed_create_time": 1003, + }, + })) + + got := reduceContextCompaction(items) + wire := string(mustMarshalJSON(got)) + if !strings.Contains(wire, "historical facts v4") { + t.Fatal("metadata fixture operation was not retired") + } + for _, retained := range []string{"turn_call_keep", "turn_nested_keep", "1003"} { + if !strings.Contains(wire, retained) { + t.Fatalf("referenced transport metadata %q was omitted", retained) + } + } + for _, omitted := range []string{"turn_nested_drop", "2002", "2003"} { + if strings.Contains(wire, omitted) { + t.Fatalf("unreferenced transport metadata %q was retained", omitted) + } + } +} diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go index 0bcc2abb..583997b9 100644 --- a/internal/router/context_compaction_codex_test.go +++ b/internal/router/context_compaction_codex_test.go @@ -25,24 +25,37 @@ func TestCompactionInstalledCodex(t *testing.T) { t.Skip("set HPATCH_COMPACTION_CODEX_BIN to exercise an installed Codex client") } for _, probe := range []struct { - legacy bool - scope string - manual bool - }{{false, "total", false}, {false, "body_after_prefix", false}, {true, "total", false}, {true, "body_after_prefix", false}, {false, "total", true}, {true, "total", true}} { - t.Run(fmt.Sprintf("legacy=%v/scope=%s/manual=%v", probe.legacy, probe.scope, probe.manual), func(t *testing.T) { + legacy bool + scope string + manual bool + retirement bool + }{{false, "total", false, false}, {false, "body_after_prefix", false, false}, {true, "total", false, false}, {true, "body_after_prefix", false, false}, {false, "total", true, false}, {true, "total", true, false}, {false, "total", false, true}, {true, "total", false, true}, {false, "total", true, true}, {true, "total", true, true}} { + t.Run(fmt.Sprintf("legacy=%v/scope=%s/manual=%v/retirement=%v", probe.legacy, probe.scope, probe.manual, probe.retirement), func(t *testing.T) { prompt := "Run the Go tests, then print the working directory, then report completion. Preserve the test result.\n" + strings.Repeat("Keep the original user constraint. ", 10000) + "\nThis final instruction must also survive intact." + agentMarker := "HPATCH_INSTALLED_COMPACTION_AGENT_MARKER_4D147B" directory, home := t.TempDir(), t.TempDir() - for name, content := range map[string]string{ - "go.mod": "module compactionprobe\n\ngo 1.26\n", - "probe_test.go": `package compactionprobe + probeSource := `package compactionprobe import ("fmt"; "testing") func TestProbe(t *testing.T) { for i := range 100 { t.Run(fmt.Sprintf("Case%d", i), func(t *testing.T) {}) } } -`, +` + if probe.retirement { + probeSource = `package compactionprobe +import ("fmt"; "strings"; "testing") +func TestProbe(t *testing.T) { + fmt.Print(strings.Repeat("unmarked finished-operation detail\n", 4000)) + for i := range 100 { t.Run(fmt.Sprintf("Case%d", i), func(t *testing.T) {}) } +} +` + } + for name, content := range map[string]string{ + "go.mod": "module compactionprobe\n\ngo 1.26\n", + "AGENTS.md": "Installed compaction fixture marker: " + agentMarker + "\n", + "probe_test.go": probeSource, } { if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil { t.Fatal(err) @@ -61,6 +74,10 @@ func TestProbe(t *testing.T) { if err := os.WriteFile(filepath.Join(home, "auth.json"), mustMarshalJSON(auth), 0o600); err != nil { t.Fatal(err) } + operationCount := int32(2) + if probe.retirement { + operationCount = 10 + } compactor := &contextCompactor{keyPath: filepath.Join(home, "compaction.key")} var normal, compacted atomic.Int32 var restored atomic.Bool @@ -79,45 +96,77 @@ func TestProbe(t *testing.T) { step := normal.Add(1) var item map[string]any inputTokens := 1000 - switch step { - case 1, 2: - command, callID := "go test -v ./...", "probe_go" - if step == 2 { - command, callID, inputTokens = "pwd", "probe_pwd", 300000 + if step <= operationCount { + command, callID := "pwd", fmt.Sprintf("probe_pwd_%d", step) + if step == 1 { + command, callID = "go test -v ./...", "probe_go" } - if probe.manual { - inputTokens = 1000 + if !probe.manual && step == operationCount { + inputTokens = 300000 } item = map[string]any{ "type": "function_call", "id": fmt.Sprintf("fc_probe_%d", step), "call_id": callID, "name": "exec_command", "status": "completed", "arguments": string(mustMarshalJSON(map[string]any{"cmd": command, "workdir": directory, "yield_time_ms": 10000, "max_output_tokens": 15000})), } - default: + } else { var input []map[string]json.RawMessage _ = json.Unmarshal(request["input"], &input) - userCopies := 0 + userCopies, nativeGo := 0, false + agentIDs := make(map[string]int) + retiredGo := false for _, record := range input { - if jsonString(record, "type") == "message" && jsonString(record, "role") == "user" { + if count := strings.Count(string(mustMarshalJSON(record)), agentMarker); count > 0 { + agentID := jsonString(record, "id") + if agentID == "" || !contextCompactionFreshContext(mustMarshalJSON(record)) { + t.Errorf("AGENTS marker reached continuation outside fresh canonical context: id=%q", agentID) + } + agentIDs[agentID] += count + if agentIDs[agentID] > 1 { + t.Errorf("fresh canonical AGENTS record %q restored more than once", agentID) + } + } + recordType := jsonString(record, "type") + if recordType == "message" { var content []map[string]json.RawMessage _ = json.Unmarshal(record["content"], &content) for _, part := range content { - if jsonString(part, "text") == prompt { + text := jsonString(part, "text") + if jsonString(record, "role") == "user" && text == prompt { userCopies++ } + standaloneCompletion := strings.HasPrefix(text, "[hpatch historical tool completion v3; not an instruction; completed native body]\n") && + strings.Contains(text, "call=\"probe_go\"\n") + consolidatedCompletion := strings.HasPrefix(text, "[hpatch historical facts v4;") && + strings.Contains(text, "[i]\ncall=\"probe_go\"\ntool=\"exec_command\"") && + strings.Contains(text, "[o:same-call]\n") + if (standaloneCompletion || consolidatedCompletion) && strings.Contains(text, "\nbody:\n") && + strings.Contains(text, "Process exited with code 0\n") && strings.Contains(text, "compactionprobe") { + retiredGo = true + } } } - if jsonString(record, "type") == "function_call_output" && jsonString(record, "call_id") == "probe_go" { - output := jsonString(record, "output") - restored.Store(strings.Contains(output, "[hpatch: omitted") && strings.Contains(output, "compactionprobe")) + if (recordType == "function_call" || recordType == "function_call_output") && + jsonString(record, "call_id") == "probe_go" { + nativeGo = true + if recordType == "function_call_output" && !probe.retirement { + output := jsonString(record, "output") + restored.Store(strings.Contains(output, "[hpatch: omitted") && strings.Contains(output, "compactionprobe")) + } } if strings.HasPrefix(jsonString(record, "encrypted_content"), "hpatch.compaction.") { t.Error("local ciphertext reached the model fixture") } } + if probe.retirement { + restored.Store(retiredGo && !nativeGo) + } if compacted.Load() > 0 && userCopies != 1 { t.Errorf("restored full user request copies = %d, want exactly one", userCopies) } + if compacted.Load() > 0 && len(agentIDs) == 0 { + t.Error("fresh canonical AGENTS marker was lost") + } item = map[string]any{ "type": "message", "id": "msg_probe_done", "role": "assistant", "status": "completed", "content": []any{map[string]any{"type": "output_text", "text": "COMPACTION_OK", "annotations": []any{}}}, @@ -228,9 +277,9 @@ metrics_exporter = "none" command.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"), "CODEX_HOME=" + home} var output []byte var err error - wantNormal := int32(3) + wantNormal := operationCount + 1 if probe.manual { - wantNormal = 4 + wantNormal = operationCount + 2 err = runManualCompactionProbe(command, directory, prompt) } else { command.Stdin = strings.NewReader(prompt) diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go index 374f0468..c9abfb6a 100644 --- a/internal/router/context_compaction_http.go +++ b/internal/router/context_compaction_http.go @@ -103,7 +103,7 @@ func (c *contextCompactor) handler(next http.Handler) http.HandlerFunc { } reduced := reduceContextCompaction(input) if slices.EqualFunc(input, reduced, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { - http.Error(writer, "no safe context reduction is available for this history; protected context was not discarded and no provider compaction was requested", http.StatusUnprocessableEntity) + http.Error(writer, "no supported context reduction is available for this history; protected context was not discarded and no provider compaction was requested", http.StatusUnprocessableEntity) return } capsule, err := c.seal(request.Context(), reduced) @@ -117,13 +117,14 @@ func (c *contextCompactor) handler(next http.Handler) http.HandlerFunc { _ = json.Unmarshal(capsule, &sealedItem) responseID := "resp_" + strings.TrimPrefix(sealedItem.ID, "cmp_") if standalone { - // Legacy Codex replaces its history wholesale. Keep user messages - // visible to its own user-input handling as well as in the capsule. + // Legacy Codex replaces its history wholesale. Keep real user messages + // visible to its user-input handling. Historical canonical context stays + // only in the capsule so it cannot be mistaken for a fresh injection. var output []json.RawMessage for _, item := range reduced { var fields map[string]json.RawMessage _ = json.Unmarshal(item, &fields) - if jsonString(fields, "type") == "message" && jsonString(fields, "role") == "user" { + if jsonString(fields, "type") == "message" && jsonString(fields, "role") == "user" && !contextCompactionFreshContext(item) { output = append(output, item) } } diff --git a/internal/router/context_compaction_http_test.go b/internal/router/context_compaction_http_test.go index e7aa8d22..1d06b7e3 100644 --- a/internal/router/context_compaction_http_test.go +++ b/internal/router/context_compaction_http_test.go @@ -94,6 +94,40 @@ func TestCompactionHTTPProviderFreeRoundTrip(t *testing.T) { } } +func TestCompactionLegacyDoesNotReexportCanonicalContext(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + message := func(text string) json.RawMessage { + return mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": text}}}) + } + contextItems := []json.RawMessage{ + message("# AGENTS.md instructions\nPreserve the workspace."), + message("\n/old\n"), + } + input := append(slices.Clone(contextItems), compactHTTPHistory()...) + response := httptest.NewRecorder() + compactor.handler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("compaction reached provider") + }))(response, httptest.NewRequest(http.MethodPost, "/v1/responses/compact", + strings.NewReader(string(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": input}))))) + var compacted struct { + Output []json.RawMessage `json:"output"` + } + if response.Code != http.StatusOK || json.Unmarshal(response.Body.Bytes(), &compacted) != nil { + t.Fatalf("local compaction failed: %d", response.Code) + } + // Canonical context belongs in the authenticated history, not in the legacy + // real-user carry list where restoration would mistake it for a fresh event. + for _, item := range compacted.Output { + if contextCompactionFreshContext(item) { + t.Fatal("historical canonical context reexported as fresh context") + } + } + got, err := compactor.restore(t.Context(), compacted.Output) + if err != nil || contextCompactionCanonicalJSON(mustMarshalJSON(got)) != contextCompactionCanonicalJSON(mustMarshalJSON(reduceContextCompaction(input))) { + t.Fatalf("historical context duplicated or lost: %v", err) + } +} + func TestCompactionHTTPStreamingV2AndFailures(t *testing.T) { compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("compaction reached provider") }) diff --git a/internal/router/context_compaction_ledger_test.go b/internal/router/context_compaction_ledger_test.go new file mode 100644 index 00000000..5f7cb159 --- /dev/null +++ b/internal/router/context_compaction_ledger_test.go @@ -0,0 +1,529 @@ +package router + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +func compactionLedgerTestPayload(t *testing.T, raw json.RawMessage) (string, string) { + t.Helper() + var item struct { + Type string `json:"type"` + Role string `json:"role"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &item) != nil || item.Type != "message" || item.Role != "assistant" || + len(item.Content) != 1 || item.Content[0].Type != "output_text" { + t.Fatal("factual record is not one assistant output-text message") + } + header, payload, ok := strings.Cut(item.Content[0].Text, "\n") + if !ok { + t.Fatal("factual record has no versioned header") + } + return header, payload +} + +func compactionLedgerTestManifest(t *testing.T, payload string) ([]json.RawMessage, string) { + t.Helper() + header, body, hasBody := strings.Cut(payload, "\nbody:\n") + values := make(map[string]json.RawMessage) + for _, line := range strings.Split(header, "\n") { + key, value, ok := strings.Cut(line, "=") + if !ok || value == "" { + t.Fatalf("invalid labeled factual record field: %s", line) + } + values[key] = json.RawMessage(value) + } + fields := []json.RawMessage{values["call"], values["metadata"]} + switch { + case values["arguments"] != nil: + fields = []json.RawMessage{values["call"], values["tool"], values["metadata"], values["arguments"]} + case values["data"] != nil: + fields = append(fields, values["data"]) + if values["body-bytes"] != nil { + fields = append(fields, values["body-bytes"]) + } + case values["result"] != nil: + fields = append(fields, values["result"]) + if values["body-bytes"] != nil { + fields = append(fields, values["body-bytes"]) + } + case values["body-bytes"] != nil: + fields = append(fields, values["body-bytes"]) + } + for _, raw := range fields { + if len(raw) == 0 || !json.Valid(raw) { + t.Fatalf("invalid factual record labeled JSON: %s", header) + } + } + if !hasBody { + body = "" + } + return fields, body +} + +func compactionLedgerTestJSONEqual(t *testing.T, got json.RawMessage, want any) { + t.Helper() + var gotValue any + if json.Unmarshal(got, &gotValue) != nil { + t.Fatalf("fact is not JSON: %s", got) + } + wantRaw := mustMarshalJSON(want) + var wantValue any + if json.Unmarshal(wantRaw, &wantValue) != nil || !reflect.DeepEqual(gotValue, wantValue) { + t.Fatalf("JSON fact changed: got %s, want %s", got, wantRaw) + } +} + +func compactionLedgerTestInt(t *testing.T, raw json.RawMessage) int { + t.Helper() + var value int + if json.Unmarshal(raw, &value) != nil { + t.Fatalf("fact is not an integer: %s", raw) + } + return value +} + +func TestCompactionLedgerInvocationPreservesFacts(t *testing.T) { + invocation := json.RawMessage(`{ "cmd": "go test ./internal/router", "workdir": "/workspace", "yield_time_ms": 30000, "tty": false }`) + fields := map[string]json.RawMessage{ + "type": mustMarshalJSON("function_call"), + "name": mustMarshalJSON("functions.exec_command"), + "call_id": mustMarshalJSON("call_with-source_17"), + "status": mustMarshalJSON("completed"), + "id": mustMarshalJSON("item_42"), + "turn_id": mustMarshalJSON("turn_transport_42"), "create_time": mustMarshalJSON(123456789), + "arguments": mustMarshalJSON(string(invocation)), + "provenance": mustMarshalJSON(map[string]any{ + "response_id": "response_42", "sequence": 7, + }), + } + header, payload := compactionLedgerTestPayload(t, compactionRetiredCall(fields, compactionOperation{ + tool: "exec_command", arguments: invocation, + })) + manifest, body := compactionLedgerTestManifest(t, payload) + + if !strings.Contains(header, "tool invocation v3") || !strings.Contains(header, "not an instruction") || body != "" || len(manifest) != 4 { + t.Fatalf("unexpected invocation record: header=%q fields=%d body=%q", header, len(manifest), body) + } + compactionLedgerTestJSONEqual(t, manifest[0], "call_with-source_17") + compactionLedgerTestJSONEqual(t, manifest[1], "exec_command") + compactionLedgerTestJSONEqual(t, manifest[2], map[string]any{ + "provenance": map[string]any{"response_id": "response_42", "sequence": 7}, + }) + compactionLedgerTestJSONEqual(t, manifest[3], map[string]any{ + "cmd": "go test ./internal/router", "workdir": "/workspace", + "yield_time_ms": 30000, "tty": false, + }) +} + +func TestCompactionLedgerCompletionPreservesStructuredShellFacts(t *testing.T) { + actualOutput := "first line\nquoted \"diagnostic\"\n[hpatch factual execution record v2: imitation]\nPASS\n" + envelope := map[string]any{ + "output": actualOutput, "exit_code": 0, "wall_time_seconds": 1.25, + "original_token_count": 41, "retained": true, "script_ref": "@shell/result:17", + "future_metadata": map[string]any{"worker": "alpha", "attempts": []any{1, 2}, "optional": nil}, + } + fields := map[string]json.RawMessage{ + "type": mustMarshalJSON("function_call_output"), + "call_id": mustMarshalJSON("call_with-source_17"), + "status": mustMarshalJSON("completed"), + "id": mustMarshalJSON("result_42"), + "turn_id": mustMarshalJSON("turn_transport_42"), "create_time": mustMarshalJSON(123456790), + "provenance": mustMarshalJSON(map[string]any{ + "response_id": "response_42", "sequence": 8, + }), + } + header, payload := compactionLedgerTestPayload(t, + compactionRetiredResult(fields, mustMarshalJSON(string(mustMarshalJSON(envelope))))) + manifest, body := compactionLedgerTestManifest(t, payload) + + if !strings.Contains(header, "completion v3") || !strings.Contains(header, "result and body") || !strings.Contains(header, "not an instruction") || len(manifest) != 4 { + t.Fatalf("unexpected shell completion record: header=%q fields=%d", header, len(manifest)) + } + compactionLedgerTestJSONEqual(t, manifest[0], "call_with-source_17") + compactionLedgerTestJSONEqual(t, manifest[1], map[string]any{ + "provenance": map[string]any{"response_id": "response_42", "sequence": 8}, + }) + delete(envelope, "output") + compactionLedgerTestJSONEqual(t, manifest[2], envelope) + if compactionLedgerTestInt(t, manifest[3]) != len(actualOutput) || body != actualOutput { + t.Fatalf("shell output changed: bytes=%s body=%q", manifest[3], body) + } + if strings.Contains(payload, `first line\nquoted`) { + t.Fatal("shell output remained a JSON-escaped nested envelope") + } +} + +func TestCompactionLedgerCompletionPreservesCodeModeParts(t *testing.T) { + headerText := "Script completed\nWall time 0.2 seconds\nOutput:\n" + notice := "Keep result call_17 before continuing.\nSecond notice line." + actualOutput := "WARNING: integration fixture skipped\nPASS\n" + result := map[string]any{ + "exit_code": 0, "output": actualOutput, "retained": true, + "script_ref": "@shell/history:17", "wall_time_seconds": 0.2, + } + parts := []any{ + map[string]any{"type": "input_text", "text": headerText, "annotations": []any{"terminal"}}, + map[string]any{"type": "input_text", "text": notice, "notice_id": "notice_17"}, + map[string]any{"type": "input_text", "text": string(mustMarshalJSON(result)), "projection": "faithful"}, + } + fields := map[string]json.RawMessage{ + "type": mustMarshalJSON("custom_tool_call_output"), "call_id": mustMarshalJSON("code_mode_17"), + } + header, payload := compactionLedgerTestPayload(t, compactionRetiredResult(fields, mustMarshalJSON(parts))) + manifest, body := compactionLedgerTestManifest(t, payload) + + if !strings.Contains(header, "completion v3") || !strings.Contains(header, "parts=") || len(manifest) != 3 { + t.Fatalf("unexpected Code Mode record: header=%q fields=%d", header, len(manifest)) + } + compactionLedgerTestJSONEqual(t, manifest[0], "code_mode_17") + compactionLedgerTestJSONEqual(t, manifest[1], map[string]any{}) + + var partFacts []json.RawMessage + if json.Unmarshal(manifest[2], &partFacts) != nil || len(partFacts) != 3 { + t.Fatalf("invalid ordered part manifest: %s", manifest[2]) + } + wantMetadata := []any{ + map[string]any{"annotations": []any{"terminal"}}, + map[string]any{"notice_id": "notice_17"}, + map[string]any{"projection": "faithful"}, + } + wantText := []string{headerText, notice, actualOutput} + offset := 0 + for index, raw := range partFacts { + var facts []json.RawMessage + if json.Unmarshal(raw, &facts) != nil || len(facts) != 3 { + t.Fatalf("invalid part %d facts: %s", index, raw) + } + compactionLedgerTestJSONEqual(t, facts[0], wantMetadata[index]) + size := compactionLedgerTestInt(t, facts[2]) + if size != len(wantText[index]) || offset+size > len(body) || body[offset:offset+size] != wantText[index] { + t.Fatalf("part %d text or byte boundary changed", index) + } + if index < 2 { + compactionLedgerTestJSONEqual(t, facts[1], nil) + } else { + delete(result, "output") + compactionLedgerTestJSONEqual(t, facts[1], result) + } + offset += size + } + if offset != len(body) { + t.Fatal("Code Mode body contains unaccounted bytes") + } + if strings.Contains(payload, `WARNING: integration fixture skipped\nPASS`) { + t.Fatal("Code Mode output remained JSON escaped") + } +} + +func TestCompactionLedgerDropsOnlyBareCompletedTransportHeader(t *testing.T) { + notice := "Keep this exact progress notice." + actualOutput := "WARNING: exact diagnostic\nPASS\n" + result := map[string]any{ + "exit_code": 0, "output": actualOutput, "chunk_id": "chunk_terminal", + "wall_time_seconds": 0.2, "original_token_count": 9, + } + parts := []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.2 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": notice, "notice_id": "notice_kept"}, + map[string]any{"type": "input_text", "text": string(mustMarshalJSON(result)), "projection": "faithful"}, + } + fields := map[string]json.RawMessage{ + "type": mustMarshalJSON("custom_tool_call_output"), "call_id": mustMarshalJSON("bare_header"), + } + header, payload := compactionLedgerTestPayload(t, compactionRetiredResult(fields, mustMarshalJSON(parts))) + manifest, body := compactionLedgerTestManifest(t, payload) + if !strings.Contains(header, "completion v3") || !strings.Contains(header, "parts=") || len(manifest) != 3 { + t.Fatal("bare-header completion framing changed") + } + var facts []json.RawMessage + if json.Unmarshal(manifest[2], &facts) != nil || len(facts) != 2 { + t.Fatalf("bare completed transport header was retained: %s", manifest[2]) + } + wantText := []string{notice, actualOutput} + offset := 0 + for index, raw := range facts { + var part []json.RawMessage + if json.Unmarshal(raw, &part) != nil || len(part) != 3 { + t.Fatal("invalid retained part") + } + var metadata map[string]json.RawMessage + _ = json.Unmarshal(part[0], &metadata) + if _, exists := metadata["type"]; exists { + t.Fatal("duplicate input_text type was retained") + } + size := compactionLedgerTestInt(t, part[2]) + if body[offset:offset+size] != wantText[index] { + t.Fatal("notice or diagnostic changed") + } + offset += size + } + compactionLedgerTestJSONEqual(t, facts[0], []any{ + map[string]any{"notice_id": "notice_kept"}, nil, len(notice), + }) + delete(result, "output") + compactionLedgerTestJSONEqual(t, facts[1], []any{ + map[string]any{"projection": "faithful"}, result, len(actualOutput), + }) + if strings.Contains(body, "Script completed") { + t.Fatal("bare successful transport header remained in the body") + } +} + +func TestCompactionLedgerCompletionPreservesPatchAndUnknownRepresentations(t *testing.T) { + fields := map[string]json.RawMessage{ + "type": mustMarshalJSON("custom_tool_call_output"), "call_id": mustMarshalJSON("patch_17"), + } + report := "in example.go\nfiles add=0 update=1 move=0 delete=0\n" + parts := []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": report}, + } + header, payload := compactionLedgerTestPayload(t, compactionRetiredResult(fields, mustMarshalJSON(parts))) + manifest, body := compactionLedgerTestManifest(t, payload) + if !strings.Contains(header, "completion v3") || !strings.Contains(header, "parts=") || len(manifest) != 3 || !strings.HasSuffix(body, report) { + t.Fatal("patch application report or Code Mode ordering changed") + } + + unknown := json.RawMessage(`{"opaque":["not","a","known","result"],"status":"mystery"}`) + header, payload = compactionLedgerTestPayload(t, compactionRetiredResult(fields, unknown)) + manifest, body = compactionLedgerTestManifest(t, payload) + if !strings.Contains(header, "completion v3") || !strings.Contains(header, "JSON result") || body != "" || len(manifest) != 3 { + t.Fatal("unknown output record framing changed") + } + compactionLedgerTestJSONEqual(t, manifest[2], map[string]any{ + "opaque": []any{"not", "a", "known", "result"}, "status": "mystery", + }) + + native := "Chunk ID: abc123\nWall time: 0.3 seconds\nProcess exited with code 0\nOriginal token count: 9\nFinal output:\nok example/router\n" + header, payload = compactionLedgerTestPayload(t, compactionRetiredResult(fields, mustMarshalJSON(native))) + manifest, body = compactionLedgerTestManifest(t, payload) + if !strings.Contains(header, "completion v3") || !strings.Contains(header, "completed native body") || len(manifest) != 3 || + compactionLedgerTestInt(t, manifest[2]) != len(native) || body != native { + t.Fatal("native completed output was escaped or changed") + } +} + +func TestCompactionRetirementPinsReferencedNativeItemAliases(t *testing.T) { + tests := []struct { + name string + index int + id string + }{ + {name: "reasoning item", index: 1, id: "rs_00"}, + {name: "call item", index: 2, id: "fc_native_00"}, + {name: "result item", index: 3, id: "out_native_00"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + items := retirementHistory() + var fields map[string]json.RawMessage + if json.Unmarshal(items[test.index], &fields) != nil { + t.Fatal("invalid test item") + } + fields["id"] = mustMarshalJSON(test.id) + items[test.index] = mustMarshalJSON(fields) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": "Continue using the evidence from " + test.id + ".", + })) + + got := retireCompactionOperations(items) + for index := 1; index <= 3; index++ { + if string(got[index]) != string(items[index]) { + t.Fatalf("referenced native item alias did not pin group item %d", index) + } + } + }) + } +} + +func TestCompactionStripTransportBookkeepingPreservesUnrecognizedMetadata(t *testing.T) { + for name, raw := range map[string]json.RawMessage{ + "malformed": json.RawMessage(`{`), + "null": json.RawMessage(`null`), + "array": json.RawMessage(`["opaque"]`), + "string": json.RawMessage(`"opaque"`), + "empty": json.RawMessage(`{}`), + "unknown only": json.RawMessage(`{"future":{"sequence":4}}`), + } { + t.Run(name, func(t *testing.T) { + fields := map[string]json.RawMessage{ + "internal_chat_message_metadata_passthrough": raw, + } + before := string(fields["internal_chat_message_metadata_passthrough"]) + compactionStripTransportBookkeeping(fields) + if got, exists := fields["internal_chat_message_metadata_passthrough"]; !exists || string(got) != before { + t.Fatalf("unrecognized metadata changed: got %s, want %s", got, before) + } + }) + } + + fields := map[string]json.RawMessage{ + "internal_chat_message_metadata_passthrough": json.RawMessage(`{"turn_id":"turn_17","create_time":123}`), + } + compactionStripTransportBookkeeping(fields) + if _, exists := fields["internal_chat_message_metadata_passthrough"]; exists { + t.Fatal("empty transport metadata was retained") + } +} + +func TestCompactionLedgerOmitsOnlyUnreferencedTransportBookkeeping(t *testing.T) { + items := retirementHistory() + setFields := func(index int, values map[string]any) { + var fields map[string]json.RawMessage + if json.Unmarshal(items[index], &fields) != nil { + t.Fatal("invalid test item") + } + for key, value := range values { + fields[key] = mustMarshalJSON(value) + } + items[index] = mustMarshalJSON(fields) + } + setFields(1, map[string]any{ + "id": "rs_native_00", "turn_id": "turn_reasoning_00", "create_time": 1001, + }) + setFields(2, map[string]any{ + "id": "fc_native_00", "turn_id": "turn_call_00", "create_time": 1002, + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_call_nested_00", "create_time": 2002, + "content_item_kinds": []any{"function_call"}, "future": map[string]any{"attempt": 3}, + }, + "workspace_version": "workspace-v17", + "unknown_call_metadata": map[string]any{"attempt": 3, "scope": "router"}, + }) + setFields(3, map[string]any{ + "id": "out_native_00", "turn_id": "turn_result_00", "create_time": 1003, + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_result_nested_00", "create_time": 2003, + "content_item_kinds": []any{"function_call_output"}, "future": map[string]any{"sequence": 4}, + }, + "result_version": "result-v9", + "unknown_result_metadata": map[string]any{"source": "local", "sequence": 4}, + }) + + before := string(mustMarshalJSON(items)) + got := retireCompactionOperations(items) + if string(mustMarshalJSON(items)) != before { + t.Fatal("retirement mutated its input") + } + for index := 1; index <= 3; index++ { + if string(got[index]) == string(items[index]) { + t.Fatalf("unreferenced unit item %d was not retired", index) + } + } + wire := string(mustMarshalJSON(got[:4])) + for _, bookkeeping := range []string{ + "rs_native_00", "fc_native_00", "out_native_00", + "turn_reasoning_00", "turn_call_00", "turn_result_00", + "1001", "1002", "1003", + } { + if strings.Contains(wire, bookkeeping) { + t.Fatalf("unreferenced transport bookkeeping %q was retained", bookkeeping) + } + } + + callHeader, callPayload := compactionLedgerTestPayload(t, got[2]) + callManifest, callBody := compactionLedgerTestManifest(t, callPayload) + if !strings.Contains(callHeader, "tool invocation v3") || callBody != "" || len(callManifest) != 4 { + t.Fatal("retired invocation moved or changed kind") + } + compactionLedgerTestJSONEqual(t, callManifest[0], "operation_00") + compactionLedgerTestJSONEqual(t, callManifest[2], map[string]any{ + "internal_chat_message_metadata_passthrough": map[string]any{ + "content_item_kinds": []any{"function_call"}, "future": map[string]any{"attempt": 3}, + }, + "workspace_version": "workspace-v17", + "unknown_call_metadata": map[string]any{"attempt": 3, "scope": "router"}, + }) + compactionLedgerTestJSONEqual(t, callManifest[3], map[string]any{ + "cmd": "hread internal/router/example.go", "workdir": "/workspace", + }) + + resultHeader, resultPayload := compactionLedgerTestPayload(t, got[3]) + resultManifest, _ := compactionLedgerTestManifest(t, resultPayload) + if !strings.Contains(resultHeader, "completion v3") || !strings.Contains(resultHeader, "result and body") || len(resultManifest) != 4 { + t.Fatal("retired completion moved or changed kind") + } + compactionLedgerTestJSONEqual(t, resultManifest[0], "operation_00") + compactionLedgerTestJSONEqual(t, resultManifest[1], map[string]any{ + "internal_chat_message_metadata_passthrough": map[string]any{ + "content_item_kinds": []any{"function_call_output"}, "future": map[string]any{"sequence": 4}, + }, + "result_version": "result-v9", + "unknown_result_metadata": map[string]any{"source": "local", "sequence": 4}, + }) + + if repeated := retireCompactionOperations(got); string(mustMarshalJSON(repeated)) != string(mustMarshalJSON(got)) { + t.Fatal("bookkeeping retirement was not stable on repeat compaction") + } +} + +func TestCompactionRetirementIgnoresOwnReasoningAlias(t *testing.T) { + items := retirementHistory() + var reasoning map[string]json.RawMessage + if json.Unmarshal(items[1], &reasoning) != nil { + t.Fatal("invalid reasoning item") + } + reasoning["id"] = mustMarshalJSON("rs_native_00") + items[1] = mustMarshalJSON(reasoning) + + first := compactTestCall("operation_00", "printf rs_native_00") + items[2] = first + items = append(items[:4], append([]json.RawMessage{ + compactTestCall("same_reasoning_group", "pwd"), + compactTestOutput("same_reasoning_group", "/workspace\n", 0), + }, items[4:]...)...) + + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3, 4, 5} { + if string(got[index]) == string(items[index]) { + t.Fatalf("own reasoning alias created a self-dependency at %d", index) + } + } +} + +func TestCompactionRetirementRejectsVisibleTokenGrowth(t *testing.T) { + items := retirementHistory() + items[2] = compactTestCall("operation_00", "printf done") + items[3] = compactTestOutput("operation_00", strings.Repeat(" ", 4096), 0) + + var reasoning, call, result map[string]json.RawMessage + _ = json.Unmarshal(items[1], &reasoning) + _ = json.Unmarshal(items[2], &call) + _ = json.Unmarshal(items[3], &result) + operation, ok := compactionOperationCall(call) + if !ok { + t.Fatal("profitability fixture call was not recognized") + } + retiredOutput, ok := compactionRetiredOutput(result["output"], operation) + if !ok { + t.Fatal("profitability fixture output was not recognized") + } + replacements := []json.RawMessage{ + compactionRetiredReasoning(reasoning), + compactionRetiredCall(call, operation), + compactionRetiredResult(result, retiredOutput), + } + beforeBytes := len(items[1]) + len(items[2]) + len(items[3]) + afterBytes := len(replacements[0]) + len(replacements[1]) + len(replacements[2]) + beforeTokens, beforeOK := compactionVisibleStringTokens(items[1], items[2], items[3]) + afterTokens, afterOK := compactionVisibleStringTokens(replacements...) + if !beforeOK || !afterOK || afterBytes >= beforeBytes || afterTokens <= beforeTokens { + t.Fatalf("fixture must shrink bytes but grow visible tokens: bytes %d -> %d, tokens %d -> %d", + beforeBytes, afterBytes, beforeTokens, afterTokens) + } + + got := retireCompactionOperations(items) + for index := 1; index <= 3; index++ { + if string(got[index]) != string(items[index]) { + t.Fatalf("token-growing complete group item %d was retired", index) + } + } +} diff --git a/internal/router/context_compaction_metadata.go b/internal/router/context_compaction_metadata.go new file mode 100644 index 00000000..a12a2f4a --- /dev/null +++ b/internal/router/context_compaction_metadata.go @@ -0,0 +1,242 @@ +package router + +import ( + "bytes" + "encoding/json" + "slices" + "strings" +) + +var contextCompactionMetadataKinds = map[string]bool{ + "message": true, + "reasoning": true, + "function_call": true, + "custom_tool_call": true, + "function_call_output": true, + "custom_tool_call_output": true, + "agent_message": true, +} + +// reduceContextCompactionMetadata removes redundant transport correlation from +// older native items only when stable item identity and retained content make +// restoration independent of that correlation. +func reduceContextCompactionMetadata(input []json.RawMessage) []json.RawMessage { + cutoff := contextCompactionMetadataRecentCutoff(input) + if cutoff == 0 { + return input + } + + ids := make(map[string]int) + for _, raw := range input { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil || fields == nil { + continue + } + if id, ok := contextCompactionMetadataID(fields["id"]); ok { + ids[id]++ + } + } + + references := contextCompactionMetadataReferences(input) + + var output []json.RawMessage + for index, raw := range input { + if index >= cutoff { + break + } + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil || fields == nil || !contextCompactionMetadataKinds[jsonString(fields, "type")] { + continue + } + id, ok := contextCompactionMetadataID(fields["id"]) + if !ok || ids[id] != 1 { + continue + } + var metadata map[string]json.RawMessage + if json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) != nil || metadata == nil { + continue + } + + values := make(map[string]string, 2) + valid := true + for _, key := range []string{"turn_id", "create_time"} { + value, exists := metadata[key] + if !exists { + continue + } + reference, ok := contextCompactionMetadataReference(key, value) + if !ok { + valid = false + break + } + values[key] = reference + } + if !valid || len(values) == 0 { + continue + } + + nextMetadata := make(map[string]json.RawMessage, len(metadata)) + for key, value := range metadata { + nextMetadata[key] = value + } + for key, reference := range values { + referenced := references.unsafe || slices.ContainsFunc(references.text, func(text string) bool { + return strings.Contains(text, reference) + }) + if key == "create_time" && slices.Contains(references.numbers, reference) { + referenced = true + } + if !referenced { + delete(nextMetadata, key) + } + } + if len(nextMetadata) == len(metadata) { + continue + } + + nextFields := make(map[string]json.RawMessage, len(fields)) + for key, value := range fields { + nextFields[key] = value + } + if len(nextMetadata) == 0 { + delete(nextFields, "internal_chat_message_metadata_passthrough") + } else { + nextFields["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(nextMetadata) + } + if output == nil { + output = slices.Clone(input) + } + output[index] = mustMarshalJSON(nextFields) + } + if output == nil { + return input + } + return output +} + +type contextCompactionMetadataReferenceSet struct { + text []string + numbers []string + unsafe bool +} + +func contextCompactionMetadataReferences(input []json.RawMessage) contextCompactionMetadataReferenceSet { + var references contextCompactionMetadataReferenceSet + decoded := make(map[string]bool) + for _, raw := range input { + raw = contextCompactionMetadataWithoutBookkeeping(raw) + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var root any + if decoder.Decode(&root) != nil { + continue + } + queue := []any{root} + for len(queue) > 0 { + value := queue[0] + queue = queue[1:] + switch value := value.(type) { + case string: + compactionSourceVisitDecodedReferences(value, func(text string) { + references.text = append(references.text, text) + }, &references.unsafe) + if decoded[value] { + continue + } + decoded[value] = true + nestedDecoder := json.NewDecoder(strings.NewReader(value)) + nestedDecoder.UseNumber() + var nested any + if nestedDecoder.Decode(&nested) == nil { + queue = append(queue, nested) + } + case json.Number: + references.numbers = append(references.numbers, value.String()) + case []any: + queue = append(queue, value...) + case map[string]any: + for _, nested := range value { + queue = append(queue, nested) + } + } + } + } + return references +} + +func contextCompactionMetadataRecentCutoff(input []json.RawMessage) int { + var calls []int + for index, raw := range input { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + continue + } + switch jsonString(fields, "type") { + case "function_call", "custom_tool_call": + calls = append(calls, index) + } + } + if len(calls) <= compactionRecentOperations { + return 0 + } + return calls[len(calls)-compactionRecentOperations] +} + +func contextCompactionMetadataID(raw json.RawMessage) (string, bool) { + var id string + return id, json.Unmarshal(raw, &id) == nil && id != "" +} + +func contextCompactionMetadataReference(key string, raw json.RawMessage) (string, bool) { + if key == "turn_id" { + var value string + return value, json.Unmarshal(raw, &value) == nil && value != "" + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var value any + if decoder.Decode(&value) != nil { + return "", false + } + number, ok := value.(json.Number) + return number.String(), ok && number.String() != "" +} + +func contextCompactionMetadataWithoutBookkeeping(raw json.RawMessage) json.RawMessage { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil || fields == nil { + return raw + } + _, directTurn := fields["turn_id"] + _, directCreated := fields["create_time"] + var metadata map[string]json.RawMessage + metadataOK := json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) == nil && metadata != nil + nestedTurn, nestedCreated := false, false + if metadataOK { + _, nestedTurn = metadata["turn_id"] + _, nestedCreated = metadata["create_time"] + } + if !directTurn && !directCreated && !nestedTurn && !nestedCreated { + return raw + } + nextFields := make(map[string]json.RawMessage, len(fields)) + for key, value := range fields { + nextFields[key] = value + } + delete(nextFields, "turn_id") + delete(nextFields, "create_time") + if nestedTurn || nestedCreated { + nextMetadata := make(map[string]json.RawMessage, len(metadata)) + for key, value := range metadata { + nextMetadata[key] = value + } + delete(nextMetadata, "turn_id") + delete(nextMetadata, "create_time") + if len(nextMetadata) == 0 { + delete(nextFields, "internal_chat_message_metadata_passthrough") + } else { + nextFields["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(nextMetadata) + } + } + return mustMarshalJSON(nextFields) +} diff --git a/internal/router/context_compaction_metadata_test.go b/internal/router/context_compaction_metadata_test.go new file mode 100644 index 00000000..ecce3ecc --- /dev/null +++ b/internal/router/context_compaction_metadata_test.go @@ -0,0 +1,325 @@ +package router + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" +) + +func compactionMetadataTestItem(kind, id, turnID string) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": kind, "id": id, "role": "user", "phase": "analysis", + "content": []any{map[string]any{"type": "input_text", "text": "full original text"}}, + "opaque_payload": map[string]any{"digest": "kept", "sequence": 7}, + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": turnID, "create_time": 123456789, + "content_item_kinds": []string{"user.text"}, + "unknown": map[string]any{"worker": "alpha", "attempt": 3}, + }, + }) +} + +func compactionMetadataTestHistory(prefix ...json.RawMessage) []json.RawMessage { + items := append([]json.RawMessage(nil), prefix...) + for index := range compactionRecentOperations + 1 { + items = append(items, mustMarshalJSON(map[string]any{ + "type": "function_call", "call_id": fmt.Sprintf("recent_%02d", index), + "name": "unknown", "arguments": "{}", + })) + } + return items +} + +func TestCompactionMetadataCleansRecognizedOlderNativeItems(t *testing.T) { + for _, kind := range []string{ + "message", "reasoning", "function_call", "custom_tool_call", + "function_call_output", "custom_tool_call_output", "agent_message", + } { + t.Run(kind, func(t *testing.T) { + item := compactionMetadataTestItem(kind, "stable_"+kind, "turn_old_"+kind) + input := compactionMetadataTestHistory(item) + before := string(mustMarshalJSON(input)) + got := reduceContextCompactionMetadata(input) + if string(mustMarshalJSON(input)) != before { + t.Fatal("metadata cleanup mutated its input") + } + if string(got[0]) == string(item) { + t.Fatal("eligible transport metadata was not removed") + } + + var fields map[string]json.RawMessage + if json.Unmarshal(got[0], &fields) != nil { + t.Fatal("cleaned item is invalid") + } + for _, key := range []string{"id", "role", "phase", "content", "opaque_payload"} { + var original map[string]json.RawMessage + _ = json.Unmarshal(item, &original) + if string(fields[key]) != string(original[key]) { + t.Fatalf("protected field %q changed", key) + } + } + var metadata map[string]json.RawMessage + if json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) != nil { + t.Fatal("retained metadata is invalid") + } + if _, exists := metadata["turn_id"]; exists { + t.Fatal("turn_id was retained") + } + if _, exists := metadata["create_time"]; exists { + t.Fatal("create_time was retained") + } + compactionLedgerTestJSONEqual(t, metadata["content_item_kinds"], []string{"user.text"}) + compactionLedgerTestJSONEqual(t, metadata["unknown"], map[string]any{"worker": "alpha", "attempt": 3}) + if repeated := reduceContextCompactionMetadata(got); string(mustMarshalJSON(repeated)) != string(mustMarshalJSON(got)) { + t.Fatal("metadata cleanup is not idempotent") + } + }) + } +} + +func TestCompactionMetadataIntegratedAtFinalReduction(t *testing.T) { + item := compactionMetadataTestItem("message", "integrated_message", "turn_integrated") + got := reduceContextCompaction(compactionMetadataTestHistory(item)) + var fields map[string]json.RawMessage + _ = json.Unmarshal(got[0], &fields) + var metadata map[string]json.RawMessage + _ = json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) + if _, exists := metadata["turn_id"]; exists { + t.Fatal("final compaction pipeline retained redundant transport metadata") + } +} + +func TestCompactionMetadataPrecedesAssistantIDRemoval(t *testing.T) { + oldAssistant := mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "old_assistant_transport", + "content": "An older factual observation.", + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_assistant_drop", "create_time": 123456789, + }, + }) + user := mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", "id": "user_identity_keep", + "content": "Keep referenced turn_user_keep and all authority unchanged.", + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_user_keep", "create_time": 123456790, + }, + }) + agent := mustMarshalJSON(map[string]any{ + "type": "agent_message", "id": "agent_identity_keep", "author": "/root/worker", "recipient": "/root", + "content": "Completed evidence.", + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_agent_drop", "create_time": 123456791, + }, + }) + got := reduceContextCompaction(compactionMetadataTestHistory(oldAssistant, user, agent)) + if len(got) == 0 { + t.Fatal("compaction removed the fixture") + } + var assistantFields map[string]json.RawMessage + if json.Unmarshal(got[0], &assistantFields) != nil { + t.Fatal("old assistant item is invalid") + } + if _, exists := assistantFields["id"]; exists { + t.Fatal("unreferenced old ordinary-assistant transport ID survived") + } + if _, exists := assistantFields["internal_chat_message_metadata_passthrough"]; exists { + t.Fatal("assistant ID removal prevented prior transport metadata cleanup") + } + + var userFields, agentFields map[string]json.RawMessage + _ = json.Unmarshal(got[1], &userFields) + _ = json.Unmarshal(got[2], &agentFields) + if jsonString(userFields, "id") != "user_identity_keep" || jsonString(agentFields, "id") != "agent_identity_keep" { + t.Fatal("authority or V2-eligible agent identity changed") + } + var userMetadata map[string]json.RawMessage + _ = json.Unmarshal(userFields["internal_chat_message_metadata_passthrough"], &userMetadata) + if jsonString(userMetadata, "turn_id") != "turn_user_keep" { + t.Fatal("referenced authority metadata was not retained") + } +} + +func TestCompactionMetadataPreservesIneligibleItems(t *testing.T) { + t.Run("no id", func(t *testing.T) { + var fields map[string]json.RawMessage + _ = json.Unmarshal(compactionMetadataTestItem("message", "remove", "turn_no_id"), &fields) + delete(fields, "id") + item := mustMarshalJSON(fields) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item)) + if string(got[0]) != string(item) { + t.Fatal("no-ID item changed") + } + }) + t.Run("invalid id shapes", func(t *testing.T) { + for name, rawID := range map[string]json.RawMessage{ + "empty": mustMarshalJSON(""), + "number": mustMarshalJSON(17), + "null": mustMarshalJSON(nil), + } { + t.Run(name, func(t *testing.T) { + var fields map[string]json.RawMessage + _ = json.Unmarshal(compactionMetadataTestItem("message", "replace", "turn_invalid_id"), &fields) + fields["id"] = rawID + item := mustMarshalJSON(fields) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item)) + if string(got[0]) != string(item) { + t.Fatal("invalid top-level ID was accepted") + } + }) + } + }) + t.Run("duplicate id", func(t *testing.T) { + first := compactionMetadataTestItem("message", "duplicate", "turn_first") + second := compactionMetadataTestItem("reasoning", "duplicate", "turn_second") + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(first, second)) + if string(got[0]) != string(first) || string(got[1]) != string(second) { + t.Fatal("duplicate top-level ID was accepted") + } + }) + t.Run("unknown kind", func(t *testing.T) { + item := compactionMetadataTestItem("future_item", "future_1", "turn_future") + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item)) + if string(got[0]) != string(item) { + t.Fatal("unknown item kind changed") + } + }) + t.Run("malformed metadata", func(t *testing.T) { + for name, metadata := range map[string]any{ + "null": nil, + "array": []any{"opaque"}, + "wrong turn": map[string]any{"turn_id": 17, "content_item_kinds": []string{"user.text"}}, + "wrong time": map[string]any{"create_time": "yesterday", "content_item_kinds": []string{"user.text"}}, + } { + t.Run(name, func(t *testing.T) { + var fields map[string]json.RawMessage + _ = json.Unmarshal(compactionMetadataTestItem("message", "malformed_"+name, "turn"), &fields) + fields["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(metadata) + item := mustMarshalJSON(fields) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item)) + if string(got[0]) != string(item) { + t.Fatal("malformed metadata changed") + } + }) + } + }) + t.Run("recent frontier", func(t *testing.T) { + item := compactionMetadataTestItem("message", "recent_message", "turn_recent") + input := append(compactionMetadataTestHistory(), item) + got := reduceContextCompactionMetadata(input) + if string(got[len(got)-1]) != string(item) { + t.Fatal("recent item changed") + } + }) + t.Run("empty metadata removed", func(t *testing.T) { + item := mustMarshalJSON(map[string]any{ + "type": "message", "id": "empty_metadata", "role": "assistant", "content": "kept", + "internal_chat_message_metadata_passthrough": map[string]any{ + "turn_id": "turn_empty", "create_time": 123, + }, + }) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item)) + var fields map[string]json.RawMessage + _ = json.Unmarshal(got[0], &fields) + if _, exists := fields["internal_chat_message_metadata_passthrough"]; exists { + t.Fatal("empty transport metadata object was retained") + } + }) +} + +func TestCompactionMetadataPreservesReferencedValues(t *testing.T) { + item := compactionMetadataTestItem("message", "referenced_message", "turn_keep") + reference := mustMarshalJSON(map[string]any{ + "type": "message", "id": "reference_message", "role": "assistant", + "content": "Continue with transport evidence turn_keep.", + }) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item, reference)) + var fields map[string]json.RawMessage + _ = json.Unmarshal(got[0], &fields) + var metadata map[string]json.RawMessage + _ = json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) + compactionLedgerTestJSONEqual(t, metadata["turn_id"], "turn_keep") + if _, exists := metadata["create_time"]; exists { + t.Fatal("unreferenced create_time was retained with referenced turn_id") + } +} +func TestCompactionMetadataPreservesEncodedAndNumericReferences(t *testing.T) { + tests := []struct { + name string + content any + wantTurnID bool + wantCreateTime bool + }{ + {name: "escaped turn ID", content: `text("turn\u005fkeep")`, wantTurnID: true}, + {name: "numeric create time", content: map[string]any{"observed": 123456789}, wantCreateTime: true}, + {name: "unrelated decoded code point", content: `text("unrelated\u{3a}escape")`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + item := compactionMetadataTestItem("message", "encoded_reference_item", "turn_keep") + reference := mustMarshalJSON(map[string]any{ + "type": "message", "id": "encoded_reference", "role": "assistant", "content": test.content, + }) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item, reference)) + var fields map[string]json.RawMessage + _ = json.Unmarshal(got[0], &fields) + var metadata map[string]json.RawMessage + _ = json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) + if _, exists := metadata["turn_id"]; exists != test.wantTurnID { + t.Fatalf("turn_id presence = %t, want %t", exists, test.wantTurnID) + } + if _, exists := metadata["create_time"]; exists != test.wantCreateTime { + t.Fatalf("create_time presence = %t, want %t", exists, test.wantCreateTime) + } + }) + } +} + +func TestCompactionMetadataRestoresSameIDCarriedMessages(t *testing.T) { + historical := mustMarshalJSON(map[string]any{ + "type": "message", "role": "developer", + "content": []any{map[string]any{"type": "input_text", "text": "Historical policy"}}, + }) + original := compactionMetadataTestItem("message", "message_same_id", "turn_transport") + history := compactionMetadataTestHistory(historical, original) + cleaned := reduceContextCompactionMetadata(history) + if string(cleaned[1]) == string(original) { + t.Fatal("restoration fixture did not remove transport metadata") + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.seal(t.Context(), cleaned) + if err != nil { + t.Fatal(err) + } + fresh := mustMarshalJSON(map[string]any{ + "type": "message", "role": "developer", + "content": []any{map[string]any{"type": "input_text", "text": "Fresh policy"}}, + }) + + var truncatedFields map[string]json.RawMessage + _ = json.Unmarshal(original, &truncatedFields) + truncatedFields["content"] = mustMarshalJSON([]any{ + map[string]any{"type": "input_text", "text": "full…1 tokens truncated…text"}, + }) + truncated := mustMarshalJSON(truncatedFields) + + for _, test := range []struct { + name string + carried json.RawMessage + }{ + {name: "legacy carried full", carried: original}, + {name: "V2 carried truncated", carried: truncated}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := compactor.restore(t.Context(), []json.RawMessage{fresh, test.carried, capsule}) + want := append([]json.RawMessage{cleaned[0], fresh}, cleaned[1:]...) + if err != nil || contextCompactionCanonicalJSON(mustMarshalJSON(got)) != contextCompactionCanonicalJSON(mustMarshalJSON(want)) { + t.Fatalf("same-ID carried message was not restored in canonical order: %v", err) + } + if strings.Contains(string(mustMarshalJSON(got)), "turn_transport") { + t.Fatal("removed transport metadata reappeared") + } + }) + } +} diff --git a/internal/router/context_compaction_narration.go b/internal/router/context_compaction_narration.go new file mode 100644 index 00000000..e525f421 --- /dev/null +++ b/internal/router/context_compaction_narration.go @@ -0,0 +1,217 @@ +package router + +import ( + "encoding/json" + "maps" + "regexp" + "slices" + "strings" +) + +var ( + compactionNarrationPath = regexp.MustCompile(`(?:^|[ \t(\["'])(?:[.~]/|[A-Za-z0-9_.-]+/)[^ \t\r\n)\]"']+`) +) + +// reduceContextCompactionNarration shortens only older assistant-authored +// prose. Authority-bearing roles and the recent operation frontier stay exact. +func reduceContextCompactionNarration(input []json.RawMessage) []json.RawMessage { + cutoff := contextCompactionMetadataRecentCutoff(input) + if cutoff == 0 { + return input + } + + callIDs := make(map[string]bool) + assistantIDs := make(map[string]bool) + fields := make([]map[string]json.RawMessage, len(input)) + for index, raw := range input { + if json.Unmarshal(raw, &fields[index]) != nil { + continue + } + kind, id := jsonString(fields[index], "type"), jsonString(fields[index], "call_id") + switch kind { + case "function_call", "custom_tool_call": + if id != "" { + callIDs[id] = true + } + } + if index < cutoff && kind == "message" && jsonString(fields[index], "role") == "assistant" { + if itemID := jsonString(fields[index], "id"); itemID != "" && compactionCallID.MatchString(itemID) { + assistantIDs[itemID] = true + } + } + } + referencedAssistantIDs, unsafeAssistantIDReference := contextCompactionReferencedNarrationIDs(fields, assistantIDs) + + type occurrence struct { + index int + text string + } + var occurrences []occurrence + for index := 0; index < cutoff; index++ { + if contextCompactionNarrationKind(fields[index]) { + for _, text := range contextCompactionNarrationTexts(fields[index]["content"]) { + occurrences = append(occurrences, occurrence{index: index, text: text}) + } + } + } + later := make(map[string]int) + for _, occurrence := range occurrences { + later[contextCompactionNarrationKey(fields[occurrence.index], occurrence.text)] = occurrence.index + } + + output := input + cloned := false + for index := 0; index < cutoff; index++ { + if !contextCompactionNarrationKind(fields[index]) { + continue + } + mapped, contentChanged := contextCompactionMapNarration(fields[index]["content"], func(text string) string { + if text == "" || contextCompactionNarrationReferences(text, callIDs) { + return text + } + key := contextCompactionNarrationKey(fields[index], text) + if later[key] > index { + return contextCompactionNarrationReplacement(text, + "[hpatch: exact repeated historical narration omitted; later identical occurrence retained]") + } + return text + }) + removeID := false + if jsonString(fields[index], "type") == "message" && jsonString(fields[index], "role") == "assistant" { + id := jsonString(fields[index], "id") + removeID = id != "" && compactionCallID.MatchString(id) && !unsafeAssistantIDReference && !referencedAssistantIDs[id] + } + if !contentChanged && !removeID { + continue + } + next := maps.Clone(fields[index]) + if contentChanged { + next["content"] = mapped + } + if removeID { + delete(next, "id") + } + if !cloned { + output = slices.Clone(input) + cloned = true + } + output[index] = mustMarshalJSON(next) + } + return output +} + +func contextCompactionReferencedNarrationIDs(fields []map[string]json.RawMessage, assistantIDs map[string]bool) (map[string]bool, bool) { + referenced := make(map[string]bool) + unsafeEncoding := false + for _, item := range fields { + for key, raw := range item { + if key == "id" || key == "call_id" || key == "turn_id" || key == "create_time" || key == "encrypted_content" { + continue + } + compactionVisitReferenceStrings(raw, func(text string) { + compactionSourceVisitDecodedReferences(text, func(decoded string) { + for _, word := range compactionReferenceWord.FindAllString(decoded, -1) { + if assistantIDs[word] { + referenced[word] = true + } + } + }, &unsafeEncoding) + }) + } + } + return referenced, unsafeEncoding +} + +func contextCompactionNarrationKind(fields map[string]json.RawMessage) bool { + kind, role := jsonString(fields, "type"), jsonString(fields, "role") + // Codex V2 may carry agent_message items. A no-ID item is reconciled by + // canonical content, so rewriting even exactly repeated agent prose can + // destroy its restoration identity. Ordinary assistant messages are not + // carried by the supported local continuation path. + return kind == "message" && role == "assistant" +} + +func contextCompactionNarrationTexts(raw json.RawMessage) []string { + var direct string + if json.Unmarshal(raw, &direct) == nil { + return []string{direct} + } + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) != nil { + return nil + } + var result []string + for _, part := range parts { + var text string + if json.Unmarshal(part["text"], &text) == nil { + result = append(result, text) + } + } + return result +} + +func contextCompactionMapNarration(raw json.RawMessage, mapText func(string) string) (json.RawMessage, bool) { + var direct string + if json.Unmarshal(raw, &direct) == nil { + mapped := mapText(direct) + return mustMarshalJSON(mapped), mapped != direct + } + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) != nil { + return raw, false + } + changed := false + for index, part := range parts { + var text string + if json.Unmarshal(part["text"], &text) != nil { + continue + } + mapped := mapText(text) + if mapped == text { + continue + } + parts[index] = maps.Clone(part) + parts[index]["text"] = mustMarshalJSON(mapped) + changed = true + } + if !changed { + return raw, false + } + return mustMarshalJSON(parts), true +} + +func contextCompactionNarrationKey(fields map[string]json.RawMessage, text string) string { + return jsonString(fields, "type") + "\x00" + jsonString(fields, "role") + "\x00" + text +} + +func contextCompactionNarrationReplacement(original, replacement string) string { + if len(replacement) >= len(original) { + return original + } + before, beforeOK := compactionVisibleStringTokens(mustMarshalJSON(original)) + after, afterOK := compactionVisibleStringTokens(mustMarshalJSON(replacement)) + if !beforeOK || !afterOK || after >= before { + return original + } + return replacement +} + +func contextCompactionNarrationReferences(text string, callIDs map[string]bool) bool { + if strings.Contains(text, "`") || strings.Contains(text, "http://") || strings.Contains(text, "https://") || + compactionNarrationPath.MatchString(text) { + return true + } + unsafeEncoding, referenced := false, false + compactionSourceVisitDecodedReferences(text, func(decoded string) { + if compactionRowReference.MatchString(decoded) || compactionSourceRangeReference.MatchString(decoded) || + compactionScriptReference.MatchString(decoded) { + referenced = true + } + for _, word := range compactionReferenceWord.FindAllString(decoded, -1) { + if callIDs[word] { + referenced = true + } + } + }, &unsafeEncoding) + return referenced || unsafeEncoding +} diff --git a/internal/router/context_compaction_narration_test.go b/internal/router/context_compaction_narration_test.go new file mode 100644 index 00000000..fbe89f9d --- /dev/null +++ b/internal/router/context_compaction_narration_test.go @@ -0,0 +1,137 @@ +package router + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" +) + +func TestCompactionNarrationReducesOnlyExactRepeatedProse(t *testing.T) { + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Only change the router."}), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "id": "routine_id", "content": "I'll inspect " + strings.Repeat("the implementation carefully and methodically ", 12) + "now."}), + compactTestCall("old_step", "pwd"), compactTestOutput("old_step", "/workspace\n", 0), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "id": "repeat_first", "content": strings.Repeat("The accepted decision remains qualified. ", 8)}), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "id": "repeat_last", "content": strings.Repeat("The accepted decision remains qualified. ", 8)}), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "id": "qualified", "content": "I'll inspect the implementation, but the unresolved failure must remain visible."}), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "id": "assistant_keep", "content": strings.Repeat("I'll inspect operation_00 before continuing. ", 8)}), + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Preserve assistant_keep exactly."}), + } + for index := range 12 { + id := fmt.Sprintf("operation_%02d", index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + + got := reduceContextCompactionNarration(items) + wire := string(mustMarshalJSON(got)) + for _, id := range []string{"assistant_keep"} { + if !strings.Contains(wire, id) { + t.Fatalf("narration reduction lost stable item ID %q", id) + } + } + if !strings.Contains(string(got[1]), "implementation carefully and methodically") { + t.Fatal("non-repeated narration was guessed to be routine") + } + if strings.Contains(wire, "routine_id") { + t.Fatal("unreferenced old assistant transport ID was retained") + } + if !strings.Contains(string(got[4]), "exact repeated historical narration omitted") || + !strings.Contains(string(got[5]), "The accepted decision remains qualified.") { + t.Fatal("exact repetition was not consolidated into its later complete occurrence") + } + if !strings.Contains(string(got[6]), "unresolved failure must remain visible") || + string(got[7]) != string(items[7]) || string(got[8]) != string(items[8]) || string(got[0]) != string(items[0]) { + t.Fatal("qualified, referenced, or authority-bearing narration changed") + } +} + +func TestCompactionNarrationPreservesRecentAndAmbiguousProse(t *testing.T) { + var items []json.RawMessage + for index := range 12 { + id := fmt.Sprintf("step_%02d", index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + recent := mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "recent_narration", + "content": "I'll inspect " + strings.Repeat("the implementation carefully and methodically ", 12) + "now.", + }) + ambiguous := mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "ambiguous_narration", + "content": "The implementation may require another inspection depending on the retained state.", + }) + items = append(items, recent, ambiguous) + got := reduceContextCompactionNarration(items) + if string(got[len(got)-2]) != string(recent) || string(got[len(got)-1]) != string(ambiguous) { + t.Fatal("recent or ambiguous narration changed") + } +} + +func TestCompactionNarrationPreservesConditionalAuthorization(t *testing.T) { + conditional := mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "conditional_authorization", + "content": "I'll run " + strings.Repeat("the verification carefully and methodically ", 8) + "after you approve the external operation.", + }) + items := []json.RawMessage{conditional, compactTestCall("authorized_step", "pwd"), compactTestOutput("authorized_step", "/workspace\n", 0)} + for index := range 9 { + id := fmt.Sprintf("later_%02d", index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + got := reduceContextCompactionNarration(items) + var fields map[string]json.RawMessage + if json.Unmarshal(got[0], &fields) != nil || jsonString(fields, "content") != jsonString(mustUnmarshalObject(t, conditional), "content") { + t.Fatal("conditional authorization narration changed") + } +} + +func mustUnmarshalObject(t *testing.T, raw json.RawMessage) map[string]json.RawMessage { + t.Helper() + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + t.Fatal("invalid test object") + } + return fields +} + +func TestCompactionNarrationKeepsV2AgentIdentityWhenAssistantIDIsOmitted(t *testing.T) { + assistant := mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "assistant_transport_old", + "content": "An older assistant observation.", + }) + agent := mustMarshalJSON(map[string]any{ + "type": "agent_message", "author": "/root/explorer", "recipient": "/root", + "content": []any{map[string]any{"type": "input_text", "text": "Message Type: NEW_TASK\nTask name: /root/explorer\nSender: /root\nPayload:\nInspect the source."}}, + }) + items := []json.RawMessage{assistant, agent, agent} + for index := range 12 { + id := fmt.Sprintf("identity_%02d", index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + + reduced := reduceContextCompaction(items) + var retainedAgents []json.RawMessage + for _, raw := range reduced { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + if jsonString(fields, "id") == "assistant_transport_old" { + t.Fatal("unreferenced old ordinary-assistant transport ID survived") + } + if jsonString(fields, "type") == "agent_message" { + retainedAgents = append(retainedAgents, raw) + } + } + if len(retainedAgents) != 2 || string(retainedAgents[0]) != string(agent) || string(retainedAgents[1]) != string(agent) { + t.Fatal("repeated no-ID V2-eligible agent message changed") + } + + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.seal(t.Context(), reduced) + if err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{retainedAgents[1], capsule}) + if err != nil || contextCompactionCanonicalJSON(mustMarshalJSON(restored)) != contextCompactionCanonicalJSON(mustMarshalJSON(reduced)) { + t.Fatalf("V2-carried agent message did not reconcile after assistant ID omission: %v", err) + } +} diff --git a/internal/router/context_compaction_operation.go b/internal/router/context_compaction_operation.go new file mode 100644 index 00000000..cbad1ad3 --- /dev/null +++ b/internal/router/context_compaction_operation.go @@ -0,0 +1,382 @@ +package router + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "math" + "math/big" + "strconv" + "strings" + + sitter "github.com/tree-sitter/go-tree-sitter" +) + +type compactionOperation struct { + tool string + arguments json.RawMessage + notice *string + patchReport string +} + +// Decode only static, result-preserving carriers. Never run a script or infer +// success from an arbitrary script's own description of its effects. +func compactionOperationCall(fields map[string]json.RawMessage) (compactionOperation, bool) { + name := strings.TrimPrefix(jsonString(fields, "name"), "functions.") + switch jsonString(fields, "type") { + case "function_call": + if name == "exec_command" || name == "write_stdin" { + arguments := json.RawMessage(jsonString(fields, "arguments")) + var object map[string]json.RawMessage + if json.Unmarshal(arguments, &object) == nil && object != nil { + return compactionOperation{tool: name, arguments: arguments}, true + } + } + case "custom_tool_call": + if name == "shell" { + return compactionOperation{tool: "shell", arguments: mustMarshalJSON(map[string]any{"input": jsonString(fields, "input")})}, true + } + if name == "exec" { + return compactionCodeModeOperation(jsonString(fields, "input")) + } + } + return compactionOperation{}, false +} + +func compactionCodeModeOperation(source string) (compactionOperation, bool) { + parser := sitter.NewParser() + defer parser.Close() + if parser.SetLanguage(codeModeJavaScriptLanguage) != nil { + return compactionOperation{}, false + } + bytes := []byte(source) + tree := parser.Parse(bytes, nil) + if tree == nil { + return compactionOperation{}, false + } + defer tree.Close() + root := tree.RootNode() + if root == nil || root.HasError() { + return compactionOperation{}, false + } + var statements []*sitter.Node + for index := range root.NamedChildCount() { + child := root.NamedChild(uint(index)) + if child.Kind() != "comment" { + statements = append(statements, child) + } + } + argument := func(node *sitter.Node, callee string) *sitter.Node { + if node == nil || node.Kind() != "call_expression" { + return nil + } + function, args := node.ChildByFieldName("function"), node.ChildByFieldName("arguments") + if function == nil || function.Utf8Text(bytes) != callee || args == nil || args.NamedChildCount() != 1 { + return nil + } + return args.NamedChild(0) + } + unquote := func(node *sitter.Node) (string, bool) { + if node == nil || node.Kind() != "string" { + return "", false + } + var value string + literal := node.Utf8Text(bytes) + if json.Unmarshal([]byte(literal), &value) == nil { + return value, true + } + value, err := strconv.Unquote(literal) + return value, err == nil + } + // This is the router's own translated-patch carrier. Its awaited host + // error propagates before the report; see hpatchHistory.carrierInput. + if strings.HasPrefix(source, hpatchApplyExecMarker) && len(statements) == 2 { + first, second := statements[0], statements[1] + if first.Kind() != "expression_statement" || second.Kind() != "expression_statement" { + return compactionOperation{}, false + } + await := first.NamedChild(0) + if await == nil || await.Kind() != "await_expression" { + return compactionOperation{}, false + } + patch, ok := unquote(argument(await.NamedChild(0), "tools.apply_patch")) + report, reportOK := unquote(argument(second.NamedChild(0), "text")) + if !ok || !reportOK || report == "" || !strings.HasPrefix(patch, "*** Begin Patch\n") || !strings.HasSuffix(strings.TrimSpace(patch), "*** End Patch") { + return compactionOperation{}, false + } + var targets []map[string]string + for line := range strings.SplitSeq(patch, "\n") { + for _, operation := range []string{"Add File", "Update File", "Delete File", "Move to"} { + if path, found := strings.CutPrefix(line, "*** "+operation+": "); found && path != "" { + targets = append(targets, map[string]string{"operation": operation, "path": path}) + } + } + } + if len(targets) == 0 { + return compactionOperation{}, false + } + return compactionOperation{tool: "apply_patch", patchReport: report, arguments: mustMarshalJSON(map[string]any{ + "targets": targets, "patch_sha256": fmt.Sprintf("%x", sha256.Sum256([]byte(patch))), + "patch_body": "retired; historical details are not currently retrievable", + })}, true + } + // Shell carriers may emit a literal progress notice between the awaited + // execution and its faithful result projection. Preserve and verify it. + var notice *string + if len(statements) == 3 && statements[1].Kind() == "expression_statement" { + value, ok := unquote(argument(statements[1].NamedChild(0), "text")) + if !ok { + return compactionOperation{}, false + } + notice = &value + statements = []*sitter.Node{statements[0], statements[2]} + } + // Support a direct projection or the generated const-result projection. + var awaited *sitter.Node + projectionAddsMetadata := false + switch len(statements) { + case 1: + if statements[0].Kind() != "expression_statement" { + return compactionOperation{}, false + } + projected := argument(statements[0].NamedChild(0), "text") + if inner := argument(projected, "JSON.stringify"); inner != nil { + projected = inner + } + awaited = projected + case 2: + if statements[0].Kind() != "lexical_declaration" || statements[0].NamedChildCount() != 1 || statements[1].Kind() != "expression_statement" { + return compactionOperation{}, false + } + declaration := statements[0].NamedChild(0) + name, value := declaration.ChildByFieldName("name"), declaration.ChildByFieldName("value") + if name == nil || name.Kind() != "identifier" || value == nil { + return compactionOperation{}, false + } + projected := argument(statements[1].NamedChild(0), "text") + if inner := argument(projected, "JSON.stringify"); inner != nil { + projected = inner + } + if projected != nil && projected.Kind() == "call_expression" { + projectionAddsMetadata = true + + function, args := projected.ChildByFieldName("function"), projected.ChildByFieldName("arguments") + if function == nil || function.Utf8Text(bytes) != "Object.assign" || args == nil || args.NamedChildCount() != 3 || + args.NamedChild(0).Utf8Text(bytes) != "{}" { + return compactionOperation{}, false + } + rawMetadata, ok := compactionStaticJSONObject(args.NamedChild(2), bytes) + if !ok { + return compactionOperation{}, false + } + var metadata map[string]json.RawMessage + if json.Unmarshal(rawMetadata, &metadata) != nil || metadata == nil { + return compactionOperation{}, false + } + for key := range metadata { + if key != "retained" && key != "script_ref" { + return compactionOperation{}, false + } + } + projected = args.NamedChild(1) + } + if projected == nil || projected.Kind() != "identifier" || projected.Utf8Text(bytes) != name.Utf8Text(bytes) { + return compactionOperation{}, false + } + awaited = value + default: + return compactionOperation{}, false + } + if awaited == nil || awaited.Kind() != "await_expression" || awaited.NamedChildCount() != 1 { + return compactionOperation{}, false + } + for _, tool := range []string{"exec_command", "write_stdin"} { + args := argument(awaited.NamedChild(0), "tools."+tool) + if args == nil { + continue + } + raw, ok := compactionStaticJSONObject(args, bytes) + if ok { + return compactionOperation{tool: tool, arguments: raw, notice: notice}, true + } + } + if notice == nil && !projectionAddsMetadata { + for _, tool := range []string{ + compactionDocsSearchTool, + compactionDocsFetchTool, + compactionDocsOpenAPITool, + } { + args := argument(awaited.NamedChild(0), "tools."+tool) + if args == nil { + continue + } + + raw, ok := compactionStaticJSONObject(args, bytes) + if ok { + return compactionOperation{tool: tool, arguments: raw}, true + } + } + } + return compactionOperation{}, false +} + +func compactionStaticJSONObject(node *sitter.Node, source []byte) (json.RawMessage, bool) { + if node == nil || node.Kind() != "object" { + return nil, false + } + + object := make(map[string]json.RawMessage) + pairs := 0 + for index := range node.NamedChildCount() { + pair := node.NamedChild(uint(index)) + if pair.Kind() != "pair" { + return nil, false + } + keyNode, valueNode := pair.ChildByFieldName("key"), pair.ChildByFieldName("value") + key, ok := compactionStaticJSONKey(keyNode, source) + if !ok || key == "__proto__" { + return nil, false + } + if _, duplicate := object[key]; duplicate { + return nil, false + } + value, ok := compactionStaticJSONValue(valueNode, source) + if !ok { + return nil, false + } + object[key] = value + pairs++ + } + if !compactionStaticJSONCommaCount(node, pairs) { + return nil, false + } + return mustMarshalJSON(object), true +} + +func compactionStaticJSONKey(node *sitter.Node, source []byte) (string, bool) { + if node == nil { + return "", false + } + switch node.Kind() { + case "string": + literal := node.Utf8Text(source) + if !compactionStaticJSONSurrogatesPaired(literal) { + return "", false + } + var key string + if json.Unmarshal([]byte(literal), &key) != nil { + return "", false + } + return key, true + case "property_identifier": + key := node.Utf8Text(source) + for index, character := range key { + if character == '_' || character == '$' || + character >= 'a' && character <= 'z' || + character >= 'A' && character <= 'Z' || + index > 0 && character >= '0' && character <= '9' { + continue + } + return "", false + } + return key, key != "" + default: + return "", false + } +} + +func compactionStaticJSONValue(node *sitter.Node, source []byte) (json.RawMessage, bool) { + if node == nil { + return nil, false + } + switch node.Kind() { + case "object": + return compactionStaticJSONObject(node, source) + case "array": + values := make([]json.RawMessage, 0, node.NamedChildCount()) + for index := range node.NamedChildCount() { + value, ok := compactionStaticJSONValue(node.NamedChild(uint(index)), source) + if !ok { + return nil, false + } + values = append(values, value) + } + if !compactionStaticJSONCommaCount(node, len(values)) { + return nil, false + } + return mustMarshalJSON(values), true + case "string", "true", "false", "null": + raw := json.RawMessage(node.Utf8Text(source)) + if !json.Valid(raw) { + return nil, false + } + return append(json.RawMessage(nil), raw...), true + case "number", "unary_expression": + raw := json.RawMessage(node.Utf8Text(source)) + if !json.Valid(raw) || !compactionStaticJSONNumberIsFaithful(string(raw)) { + return nil, false + } + return append(json.RawMessage(nil), raw...), true + default: + return nil, false + } +} + +func compactionStaticJSONNumberIsFaithful(raw string) bool { + value, err := strconv.ParseFloat(raw, 64) + if err != nil || math.IsInf(value, 0) || math.IsNaN(value) { + return false + } + canonical := strconv.FormatFloat(value, 'g', -1, 64) + originalValue, originalOK := new(big.Rat).SetString(raw) + canonicalValue, canonicalOK := new(big.Rat).SetString(canonical) + return originalOK && canonicalOK && originalValue.Cmp(canonicalValue) == 0 +} + +func compactionStaticJSONSurrogatesPaired(literal string) bool { + for index := 1; index < len(literal)-1; { + if literal[index] != '\\' { + index++ + continue + } + if index+1 >= len(literal)-1 || literal[index+1] != 'u' { + index += 2 + continue + } + if index+6 > len(literal)-1 { + return false + } + value, err := strconv.ParseUint(literal[index+2:index+6], 16, 16) + if err != nil { + return false + } + if value >= 0xdc00 && value <= 0xdfff { + return false + } + if value < 0xd800 || value > 0xdbff { + index += 6 + continue + } + next := index + 6 + if next+6 > len(literal)-1 || literal[next] != '\\' || literal[next+1] != 'u' { + return false + } + low, err := strconv.ParseUint(literal[next+2:next+6], 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return false + } + index = next + 6 + } + return true +} + +func compactionStaticJSONCommaCount(node *sitter.Node, values int) bool { + commas := 0 + for index := range node.ChildCount() { + child := node.Child(uint(index)) + if child != nil && child.Kind() == "," { + commas++ + } + } + return commas == max(0, values-1) +} diff --git a/internal/router/context_compaction_read_tool.go b/internal/router/context_compaction_read_tool.go new file mode 100644 index 00000000..860b9110 --- /dev/null +++ b/internal/router/context_compaction_read_tool.go @@ -0,0 +1,602 @@ +package router + +import ( + "encoding/json" + "fmt" + "maps" + "regexp" + "slices" + "strings" +) + +const ( + compactionDocsSearchTool = "mcp__openaiDeveloperDocs__search_openai_docs" + compactionDocsFetchTool = "mcp__openaiDeveloperDocs__fetch_openai_doc" + compactionDocsOpenAPITool = "mcp__openaiDeveloperDocs__get_openapi_spec" +) + +func compactionReadTool(tool string) bool { + switch tool { + case compactionDocsSearchTool, compactionDocsFetchTool, compactionDocsOpenAPITool: + return true + default: + return false + } +} + +// compactionRetiredReadToolOutput accepts only the faithful two-part projection +// of a successful MCP CallToolResult. Tool and content metadata stay intact; +// only a recognized documentation body is replaced. +func compactionRetiredReadToolOutput(raw json.RawMessage, tool string) (json.RawMessage, bool) { + if !compactionReadTool(tool) { + return nil, false + } + + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) != nil || len(parts) != 2 || + jsonString(parts[0], "type") != "input_text" || + !strings.HasPrefix(jsonString(parts[0], "text"), "Script completed\n") || + jsonString(parts[1], "type") != "input_text" { + return nil, false + } + + var serialized string + if json.Unmarshal(parts[1]["text"], &serialized) != nil { + return nil, false + } + + var result map[string]json.RawMessage + if json.Unmarshal([]byte(serialized), &result) != nil || result == nil { + return compactionRetiredTruncatedReadToolOutput(parts, serialized, tool) + } + if rawError, exists := result["isError"]; exists { + var isError bool + if json.Unmarshal(rawError, &isError) != nil || isError { + return nil, false + } + } + + var content []map[string]json.RawMessage + if json.Unmarshal(result["content"], &content) != nil || len(content) == 0 { + return nil, false + } + + changed := false + for index, block := range content { + if jsonString(block, "type") != "text" { + return nil, false + } + + var body string + if json.Unmarshal(block["text"], &body) != nil { + return nil, false + } + + reduced, ok := compactionRetiredReadToolText(tool, body) + if !ok { + continue + } + content[index] = maps.Clone(block) + content[index]["text"] = mustMarshalJSON(reduced) + changed = true + } + if !changed { + return raw, true + } + + result = maps.Clone(result) + result["content"] = mustMarshalJSON(content) + parts[1] = maps.Clone(parts[1]) + parts[1]["text"] = mustMarshalJSON(string(mustMarshalJSON(result))) + return mustMarshalJSON(parts), true +} + +var compactionTruncatedReadBody = regexp.MustCompile(`\\"(?:snippet|snippets|body|content|text|markdown|highlights)\\"[ \t]*:[ \t]*\\"`) + +// compactionRetiredTruncatedReadToolOutput repairs no unknown structure. It +// accepts one client truncation only when that marker is wholly inside a +// recognized escaped documentation-body string and replacing that complete +// value makes the original CallToolResult parse again. +func compactionRetiredTruncatedReadToolOutput(parts []map[string]json.RawMessage, serialized, tool string) (json.RawMessage, bool) { + markers := contextCompactionTruncation.FindAllStringIndex(serialized, -1) + if len(markers) != 1 { + return nil, false + } + marker := markers[0] + for _, match := range compactionTruncatedReadBody.FindAllStringIndex(serialized, -1) { + start := match[1] + end := compactionEscapedJSONStringEnd(serialized, start) + if end < 0 || marker[0] <= start || marker[1] >= end { + continue + } + evidence, ok := compactionTruncatedEscapedBodyEvidence(serialized[start:end], marker[0]-start, marker[1]-start) + if !ok { + continue + } + replacement := fmt.Sprintf( + "[hpatch compaction: client-truncated historical documentation body had %d retained encoded bytes around an unavailable middle; unambiguous body span omitted]\n%s", + end-start, evidence) + encoded, ok := compactionEncodeEscapedJSONStringBody(replacement) + if !ok || len(encoded) >= end-start { + continue + } + repaired := serialized[:start] + encoded + serialized[end:] + objectStart := strings.IndexByte(repaired, '{') + if objectStart < 0 { + continue + } + var result map[string]json.RawMessage + if json.Unmarshal([]byte(repaired[objectStart:]), &result) != nil || result == nil { + continue + } + if rawError, exists := result["isError"]; exists { + var isError bool + if json.Unmarshal(rawError, &isError) != nil || isError { + continue + } + } + var content []map[string]json.RawMessage + if json.Unmarshal(result["content"], &content) != nil || len(content) == 0 { + continue + } + for _, block := range content { + if jsonString(block, "type") != "text" { + return nil, false + } + var body string + if json.Unmarshal(block["text"], &body) != nil { + return nil, false + } + } + repaired = repaired[:objectStart] + string(mustMarshalJSON(result)) + copyParts := slices.Clone(parts) + copyParts[len(copyParts)-1] = maps.Clone(copyParts[len(copyParts)-1]) + copyParts[len(copyParts)-1]["text"] = mustMarshalJSON(repaired) + return mustMarshalJSON(copyParts), true + } + return nil, false +} + +func compactionEscapedJSONStringEnd(text string, start int) int { + for index := start; index < len(text); index++ { + if text[index] != '"' { + continue + } + backslashes := 0 + for previous := index - 1; previous >= start && text[previous] == '\\'; previous-- { + backslashes++ + } + if backslashes == 1 { + return index - 1 + } + } + return -1 +} + +func compactionDecodeEscapedJSONStringBody(raw string) (string, bool) { + var nested string + if json.Unmarshal([]byte(`"`+raw+`"`), &nested) != nil { + return "", false + } + var body string + return body, json.Unmarshal([]byte(`"`+nested+`"`), &body) == nil +} + +func compactionTruncatedEscapedBodyEvidence(raw string, markerStart, markerEnd int) (string, bool) { + if markerStart <= 0 || markerEnd <= markerStart || markerEnd >= len(raw) { + return "", false + } + prefixEnd := strings.LastIndex(raw[:markerStart], `\\n`) + suffixOffset := strings.Index(raw[markerEnd:], `\\n`) + if prefixEnd < 0 || suffixOffset < 0 { + return "", false + } + prefixEnd += len(`\\n`) + suffixStart := markerEnd + suffixOffset + len(`\\n`) + prefix, prefixOK := compactionDecodeEscapedJSONStringBody(raw[:prefixEnd]) + suffix, suffixOK := compactionDecodeEscapedJSONStringBody(raw[suffixStart:]) + if !prefixOK || !suffixOK { + return "", false + } + + var result strings.Builder + result.WriteString(compactionReadBodyEvidence(prefix)) + result.WriteString("[hpatch: exact encoded truncation-boundary line follows]\n") + result.WriteString(raw[prefixEnd:suffixStart]) + result.WriteByte('\n') + result.WriteString(compactionReadBodyEvidence(suffix)) + return result.String(), true +} + +func compactionEncodeEscapedJSONStringBody(body string) (string, bool) { + nested := string(mustMarshalJSON(body)) + nested = nested[1 : len(nested)-1] + outer := string(mustMarshalJSON(nested)) + if len(outer) < 2 { + return "", false + } + return outer[1 : len(outer)-1], true +} + +func compactionRetiredReadToolText(tool, body string) (string, bool) { + switch tool { + case compactionDocsSearchTool: + if result, ok := compactionRetiredSearchResults(body); ok { + return compactionSuccessfulReadHeader(len(body), "search result") + result, true + } + case compactionDocsFetchTool: + if result, ok := compactionRetiredDocumentText(body); ok { + return compactionSuccessfulReadHeader(len(body), "fetched document") + result, true + } + case compactionDocsOpenAPITool: + if result, ok := compactionRetiredOpenAPI(body); ok { + return compactionSuccessfulReadHeader(len(body), "OpenAPI document") + result, true + } + } + return "", false +} + +func compactionSuccessfulReadHeader(originalBytes int, kind string) string { + return fmt.Sprintf("[hpatch compaction: successful read-tool return; %s was %d original bytes; unmarked historical body omitted and not currently retrievable]\n", kind, originalBytes) +} + +func compactionRetiredSearchResults(text string) (string, bool) { + var object map[string]json.RawMessage + if json.Unmarshal([]byte(text), &object) == nil && object != nil { + for _, key := range []string{"results", "hits", "data", "items", "documents"} { + raw, exists := object[key] + if !exists { + continue + } + reduced, ok := compactionRetiredSearchItems(raw) + if !ok { + return "", false + } + object = maps.Clone(object) + object[key] = reduced + return string(mustMarshalJSON(object)), true + } + return "", false + } + + var items []map[string]json.RawMessage + if json.Unmarshal([]byte(text), &items) != nil { + return "", false + } + reduced, ok := compactionRetiredSearchItemMaps(items) + if !ok { + return "", false + } + return string(reduced), true +} + +func compactionRetiredSearchItems(raw json.RawMessage) (json.RawMessage, bool) { + var items []map[string]json.RawMessage + if json.Unmarshal(raw, &items) != nil { + return nil, false + } + return compactionRetiredSearchItemMaps(items) +} + +func compactionRetiredSearchItemMaps(items []map[string]json.RawMessage) (json.RawMessage, bool) { + if len(items) == 0 { + return nil, false + } + + changed := false + for index, item := range items { + identified := false + for _, key := range []string{"url", "source_url", "title", "id", "objectID", "citation", "citation_id", "source"} { + if raw, exists := item[key]; exists && len(raw) > 0 && string(raw) != "null" { + identified = true + break + } + } + if !identified { + return nil, false + } + + reduced := maps.Clone(item) + itemChanged := false + for _, key := range []string{"snippet", "snippets", "body", "content", "text", "markdown", "highlights"} { + raw, exists := item[key] + if !exists || len(raw) < 256 { + continue + } + body, ok := compactionRetiredReadJSONBody(raw) + if !ok { + return nil, false + } + reduced[key] = body + itemChanged = true + } + for _, key := range []string{"_snippetResult", "_highlightResult"} { + raw, exists := item[key] + if !exists { + continue + } + annotations, ok := compactionRetiredSearchAnnotations(raw) + if !ok { + continue + } + reduced[key] = annotations + itemChanged = true + } + if itemChanged { + items[index] = reduced + changed = true + } + } + if !changed { + return nil, false + } + return mustMarshalJSON(items), true +} + +func compactionRetiredSearchAnnotations(raw json.RawMessage) (json.RawMessage, bool) { + var annotations map[string]json.RawMessage + if json.Unmarshal(raw, &annotations) != nil || annotations == nil { + return nil, false + } + + reduced := maps.Clone(annotations) + changed := false + for _, key := range []string{"snippet", "body", "content", "text", "markdown"} { + value, exists := annotations[key] + if !exists { + continue + } + retired, ok := compactionRetiredSearchAnnotationValue(value) + if !ok { + continue + } + reduced[key] = retired + changed = true + } + if !changed { + return nil, false + } + return mustMarshalJSON(reduced), true +} + +func compactionRetiredSearchAnnotationValue(raw json.RawMessage) (json.RawMessage, bool) { + var text string + if json.Unmarshal(raw, &text) == nil { + if len(text) < 256 { + return nil, false + } + return mustMarshalJSON(compactionRetiredReadBodyString(text)), true + } + + var annotation map[string]json.RawMessage + if json.Unmarshal(raw, &annotation) == nil && annotation != nil { + value, exists := annotation["value"] + if !exists { + return nil, false + } + var text string + if json.Unmarshal(value, &text) != nil || len(text) < 256 { + return nil, false + } + reduced := maps.Clone(annotation) + reduced["value"] = mustMarshalJSON(compactionRetiredReadBodyString(text)) + return mustMarshalJSON(reduced), true + } + + var entries []json.RawMessage + if json.Unmarshal(raw, &entries) != nil || len(entries) == 0 { + return nil, false + } + changed := false + for index, entry := range entries { + reduced, ok := compactionRetiredSearchAnnotationValue(entry) + if !ok { + continue + } + entries[index] = reduced + changed = true + } + if !changed { + return nil, false + } + return mustMarshalJSON(entries), true +} + +func compactionRetiredReadJSONBody(raw json.RawMessage) (json.RawMessage, bool) { + var text string + if json.Unmarshal(raw, &text) == nil { + return mustMarshalJSON(compactionRetiredReadBodyString(text)), true + } + + var stringsOnly []string + if json.Unmarshal(raw, &stringsOnly) == nil && len(stringsOnly) > 0 { + for index, text := range stringsOnly { + stringsOnly[index] = compactionRetiredReadBodyString(text) + } + return mustMarshalJSON(stringsOnly), true + } + + var entries []map[string]json.RawMessage + if json.Unmarshal(raw, &entries) == nil && len(entries) > 0 { + changed := false + for index, entry := range entries { + reduced, ok := compactionRetiredReadBodyObject(entry) + if !ok { + continue + } + entries[index] = reduced + changed = true + } + if !changed { + return nil, false + } + return mustMarshalJSON(entries), true + } + + var object map[string]json.RawMessage + if json.Unmarshal(raw, &object) == nil && object != nil { + reduced, ok := compactionRetiredReadBodyObject(object) + if ok { + return mustMarshalJSON(reduced), true + } + } + return nil, false +} + +func compactionRetiredReadBodyObject(object map[string]json.RawMessage) (map[string]json.RawMessage, bool) { + reduced := maps.Clone(object) + changed := false + for _, key := range []string{"snippet", "body", "content", "text", "markdown", "highlight"} { + raw, exists := object[key] + if !exists || len(raw) < 64 { + continue + } + + var text string + if json.Unmarshal(raw, &text) != nil { + return nil, false + } + reduced[key] = mustMarshalJSON(compactionRetiredReadBodyString(text)) + changed = true + } + return reduced, changed +} + +func compactionRetiredReadBodyString(text string) string { + return fmt.Sprintf("[hpatch compaction: historical document body retired (%d original bytes); selected provenance and diagnostics follow]\n%s", + len(text), compactionReadBodyEvidence(text)) +} + +func compactionRetiredDocumentText(text string) (string, bool) { + if len(text) < 512 { + return "", false + } + evidence := compactionReadBodyEvidence(text) + if evidence == "" { + return "", false + } + return evidence, true +} + +func compactionReadBodyEvidence(text string) string { + lines := strings.SplitAfter(text, "\n") + keep := make([]bool, len(lines)) + inFrontmatter := false + for index, line := range lines { + trimmed := strings.TrimSpace(line) + lower := strings.ToLower(trimmed) + + if index == 0 && trimmed == "---" { + inFrontmatter = true + } + if inFrontmatter { + keep[index] = true + if index > 0 && trimmed == "---" { + inFrontmatter = false + } + continue + } + + if strings.HasPrefix(trimmed, "#") || + strings.Contains(line, "https://") || strings.Contains(line, "http://") || + strings.Contains(line, "【") && strings.Contains(line, "】") || + strings.HasPrefix(lower, "source:") || strings.HasPrefix(lower, "url:") || + strings.HasPrefix(lower, "citation:") || strings.HasPrefix(lower, "title:") || + strings.HasPrefix(lower, "retrieved:") || strings.HasPrefix(lower, "updated:") || + strings.HasPrefix(lower, "published:") { + keep[index] = true + } + if compactionDiagnostic.MatchString(line) { + for nearby := max(0, index-2); nearby < min(len(lines), index+3); nearby++ { + keep[nearby] = true + } + } + } + + var result strings.Builder + for index, line := range lines { + if keep[index] { + result.WriteString(line) + } + } + return result.String() +} + +func compactionRetiredOpenAPI(text string) (string, bool) { + var document map[string]json.RawMessage + if json.Unmarshal([]byte(text), &document) != nil || document == nil { + return "", false + } + if jsonString(document, "openapi") == "" && jsonString(document, "swagger") == "" { + return "", false + } + + var paths map[string]json.RawMessage + if json.Unmarshal(document["paths"], &paths) != nil || paths == nil { + return "", false + } + + changed := false + reducedPaths := maps.Clone(paths) + for path, raw := range paths { + var item map[string]json.RawMessage + if json.Unmarshal(raw, &item) != nil || item == nil { + return "", false + } + + reducedItem := maps.Clone(item) + for key, value := range item { + if compactionOpenAPIMethod(key) { + var operation map[string]json.RawMessage + if json.Unmarshal(value, &operation) != nil || operation == nil { + return "", false + } + reducedOperation := maps.Clone(operation) + for _, bodyKey := range []string{"description", "parameters", "requestBody", "responses", "callbacks"} { + body, exists := operation[bodyKey] + if !exists || len(body) < 64 { + continue + } + reducedOperation[bodyKey] = mustMarshalJSON(fmt.Sprintf( + "[hpatch compaction: historical OpenAPI %s retired (%d serialized bytes)]", bodyKey, len(body))) + changed = true + } + reducedItem[key] = mustMarshalJSON(reducedOperation) + continue + } + if key == "description" || key == "parameters" { + if len(value) >= 64 { + reducedItem[key] = mustMarshalJSON(fmt.Sprintf( + "[hpatch compaction: historical OpenAPI path %s retired (%d serialized bytes)]", key, len(value))) + changed = true + } + } + } + reducedPaths[path] = mustMarshalJSON(reducedItem) + } + + reduced := maps.Clone(document) + reduced["paths"] = mustMarshalJSON(reducedPaths) + for _, key := range []string{"components", "webhooks"} { + raw, exists := document[key] + if !exists || len(raw) < 64 { + continue + } + reduced[key] = mustMarshalJSON(fmt.Sprintf( + "[hpatch compaction: historical OpenAPI %s retired (%d serialized bytes)]", key, len(raw))) + changed = true + } + if !changed { + return "", false + } + return string(mustMarshalJSON(reduced)), true +} + +func compactionOpenAPIMethod(value string) bool { + switch value { + case "get", "put", "post", "delete", "options", "head", "patch", "trace": + return true + default: + return false + } +} diff --git a/internal/router/context_compaction_read_tool_test.go b/internal/router/context_compaction_read_tool_test.go new file mode 100644 index 00000000..19c07646 --- /dev/null +++ b/internal/router/context_compaction_read_tool_test.go @@ -0,0 +1,559 @@ +package router + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +const testDocsSearchTool = "mcp__openaiDeveloperDocs__search_openai_docs" + +func readToolSource(tool, arguments string) string { + return "const result = await tools." + tool + "(" + arguments + "); text(result);" +} + +func readToolResultOutput(id, header string, result map[string]any) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": id, + "output": []any{ + map[string]any{"type": "input_text", "text": header}, + map[string]any{"type": "input_text", "text": string(mustMarshalJSON(result))}, + }, + }) +} + +func readToolHistory(t *testing.T, operationIndex int, tool, source string, output json.RawMessage) []json.RawMessage { + t.Helper() + id := fmt.Sprintf("operation_%02d", operationIndex) + items := retirementHistory() + foundCall, foundOutput := false, false + for index, raw := range items { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil || jsonString(fields, "call_id") != id { + continue + } + switch jsonString(fields, "type") { + case "function_call", "custom_tool_call": + items[index] = mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "exec", "call_id": id, "input": source, + }) + foundCall = true + case "function_call_output", "custom_tool_call_output": + items[index] = output + foundOutput = true + } + } + if !foundCall || !foundOutput { + t.Fatalf("history did not contain call/result pair %q", id) + } + return items +} + +func TestCompactionCodeModeRecognizesOnlyFaithfulDocumentationReads(t *testing.T) { + tools := []string{ + testDocsSearchTool, + "mcp__openaiDeveloperDocs__fetch_openai_doc", + "mcp__openaiDeveloperDocs__get_openapi_spec", + } + for _, tool := range tools { + t.Run(tool, func(t *testing.T) { + arguments := `{"query":"Responses API"}` + for _, source := range []string{ + readToolSource(tool, arguments), + "text(JSON.stringify(await tools." + tool + "(" + arguments + ")));", + } { + operation, ok := compactionCodeModeOperation(source) + if !ok || operation.tool != tool || string(operation.arguments) != arguments { + t.Fatalf("faithful static documentation read was not recognized: %#v", operation) + } + } + }) + } + + for _, source := range []string{ + readToolSource("mcp__openaiDeveloperDocs__other", `{"query":"Responses API"}`), + `const result = await tools.` + testDocsSearchTool + `({"query":"Responses API"}); text(result.content.map(value => value.text));`, + `const result = await tools.` + testDocsSearchTool + `({"query":"Responses API"}); text(JSON.stringify(Object.assign({}, result, {"retained":true})));`, + `const result = await tools.` + testDocsSearchTool + `({"query":"Responses API"}); await tools.other(); text(result);`, + } { + if _, ok := compactionCodeModeOperation(source); ok { + t.Fatalf("unknown, transformed, or procedural documentation read was accepted: %s", source) + } + } +} + +func TestCompactionCodeModeDecodesStaticObjectLiteralArguments(t *testing.T) { + tests := []struct { + name string + quoted string + unquoted string + wantTool string + wantResult string + }{ + { + name: "write stdin", + quoted: `text(await tools.write_stdin({"session_id":42,"chars":""}));`, + unquoted: `text(await tools.write_stdin({session_id:42,chars:""}));`, + wantTool: "write_stdin", + wantResult: `{"chars":"","session_id":42}`, + }, + { + name: "documentation query", + quoted: `text(await tools.` + testDocsSearchTool + `({"query":"Responses API","limit":10}));`, + unquoted: `text(await tools.` + testDocsSearchTool + `({query:"Responses API",limit:10}));`, + wantTool: testDocsSearchTool, + wantResult: `{"limit":10,"query":"Responses API"}`, + }, + { + name: "nested JSON literals", + quoted: `text(await tools.exec_command({"cmd":"inspect","meta":{"enabled":true,"values":[1,null,false]}}));`, + unquoted: `text(await tools.exec_command({cmd:"inspect",meta:{enabled:true,values:[1,null,false]}}));`, + wantTool: "exec_command", + wantResult: `{"cmd":"inspect","meta":{"enabled":true,"values":[1,null,false]}}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, source := range []string{test.quoted, test.unquoted} { + operation, ok := compactionCodeModeOperation(source) + if !ok || operation.tool != test.wantTool || string(operation.arguments) != test.wantResult { + t.Fatalf("static object literal was not decoded faithfully: %#v", operation) + } + } + }) + } + + metadata := `const result = await tools.exec_command({cmd:"pwd"}); ` + + `text(JSON.stringify(Object.assign({}, result, {retained:true,script_ref:"@shell/static"})));` + if operation, ok := compactionCodeModeOperation(metadata); !ok || + operation.tool != "exec_command" || string(operation.arguments) != `{"cmd":"pwd"}` { + t.Fatalf("static generated metadata was not decoded: %#v", operation) + } + + for _, arguments := range []string{ + `{query:"x",...other}`, + `{["query"]:"x"}`, + `{get query(){return "x"}}`, + `{query(){return "x"}}`, + `{query}`, + `{query:buildQuery()}`, + "{query:`value-${suffix}`}", + `{query:"first",query:"second"}`, + `{__proto__:{polluted:true},query:"x"}`, + `{query:{value:one}}`, + `{query:'single quoted'}`, + } { + source := `text(await tools.` + testDocsSearchTool + `(` + arguments + `));` + if _, ok := compactionCodeModeOperation(source); ok { + t.Fatalf("dynamic or special-semantics object literal was accepted: %s", arguments) + } + } +} +func TestCompactionCodeModeRejectsNonfaithfulStaticJSONLiterals(t *testing.T) { + for _, arguments := range []string{ + `{value:0.1}`, + `{value:1.5}`, + `{value:1e3}`, + `{"\ud83d\ude00":"paired surrogate"}`, + } { + source := `text(await tools.` + testDocsSearchTool + `(` + arguments + `));` + if _, ok := compactionCodeModeOperation(source); !ok { + t.Fatalf("ordinary static literal was rejected: %s", arguments) + } + } + + for _, arguments := range []string{ + `{value:1.0000000000000001}`, + `{value:9007199254740993}`, + `{value:1e400}`, + `{"\ud800":"lone high surrogate"}`, + `{"\udc00":"lone low surrogate"}`, + } { + source := `text(await tools.` + testDocsSearchTool + `(` + arguments + `));` + if _, ok := compactionCodeModeOperation(source); ok { + t.Fatalf("nonfaithful static literal was accepted: %s", arguments) + } + } +} + +func TestCompactionRetiresSuccessfulDocumentationSearch(t *testing.T) { + arguments := `{"query":"Responses API","limit":10}` + source := readToolSource(testDocsSearchTool, arguments) + body := strings.Repeat("large unmarked search snippet\n", 1200) + result := map[string]any{ + "isError": false, + "_meta": map[string]any{"request_id": "docs-request-17", "source": "OpenAI developer documentation"}, + "structuredContent": map[string]any{"count": 1, "next_cursor": "cursor-2"}, + "content": []any{map[string]any{ + "type": "text", + "annotations": map[string]any{"audience": []string{"assistant"}, "priority": 0.8}, + "text": string(mustMarshalJSON(map[string]any{ + "count": 1, "outcome": "success", + "results": []any{map[string]any{ + "id": "responses-guide", "title": "Responses API guide", + "url": "https://developers.openai.com/api/docs/guides/responses", + "citation": "【responses-guide】", "snippet": body, + "snippets": []any{map[string]any{ + "source_url": "https://developers.openai.com/api/docs/reference/responses", + "citation": "【responses-reference】", + "text": body + "\nWARNING: preview search index", + }}, + }}, + })), + }}, + } + output := readToolResultOutput("operation_00", "Script completed\nWall time 0.1 seconds\nOutput:\n", result) + items := readToolHistory(t, 0, testDocsSearchTool, source, output) + got := retireCompactionOperations(items) + wire := string(mustMarshalJSON(got)) + + if string(got[2]) == string(items[2]) || string(got[3]) == string(items[3]) { + t.Fatal("eligible documentation read was not retired") + } + for _, evidence := range []string{ + "successful read-tool return", "Responses API", "docs-request-17", "isError", + "OpenAI developer documentation", "cursor-2", "responses-guide", + "https://developers.openai.com/api/docs/guides/responses", "【responses-guide】", + "https://developers.openai.com/api/docs/reference/responses", "【responses-reference】", + "WARNING: preview search index", "priority", "0.8", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("documentation read evidence %q was lost", evidence) + } + } + if strings.Count(wire, "large unmarked search snippet") > 4 { + t.Fatal("bulky search snippet was not retired") + } + + operation, ok := compactionCodeModeOperation(source) + if !ok || contextCompactionCanonicalJSON(operation.arguments) != contextCompactionCanonicalJSON(json.RawMessage(arguments)) { + t.Fatal("exact documentation read invocation was not retained") + } +} +func TestCompactionRetiresActualDocumentationSearchSchema(t *testing.T) { + body := strings.Repeat("unmarked Algolia search body\n", 900) + search := map[string]any{ + "hits": []any{map[string]any{ + "url": "https://developers.openai.com/api/docs/guides/responses#streaming", + "url_without_anchor": "https://developers.openai.com/api/docs/guides/responses", + "anchor": "streaming", + "content": body, + "type": "lvl2", + "hierarchy": map[string]any{"lvl0": "Guides", "lvl1": "Responses", "lvl2": "Streaming"}, + "objectID": "docs-responses-streaming", + "_snippetResult": map[string]any{ + "content": map[string]any{ + "value": body + "WARNING: streaming preview behavior\n", + "matchLevel": "full", "matchedWords": []string{"streaming"}, + "annotation_extension": "snippet-metadata", + }, + }, + "_highlightResult": map[string]any{ + "content": map[string]any{ + "value": body, "matchLevel": "partial", "fullyHighlighted": false, + }, + "hierarchy": map[string]any{ + "lvl0": map[string]any{"value": "Guides", "matchLevel": "none"}, + "lvl1": map[string]any{"value": "Responses", "matchLevel": "full"}, + }, + }, + "unknown_hit_metadata": map[string]any{"rank": 7, "source": "developer-docs-index"}, + }}, + "nbHits": 1, "page": 2, "nextCursor": "algolia-cursor-3", + "unknown_root_metadata": map[string]any{"processingTimeMS": 4}, + } + result := map[string]any{ + "_meta": map[string]any{"request_id": "algolia-request-4"}, + "content": []any{map[string]any{ + "type": "text", + "annotations": map[string]any{"audience": []string{"assistant"}, "priority": 0.9}, + "text": string(mustMarshalJSON(search)), + }}, + } + source := readToolSource(testDocsSearchTool, `{"query":"streaming","cursor":"algolia-cursor-2"}`) + items := readToolHistory(t, 0, testDocsSearchTool, source, + readToolResultOutput("operation_00", "Script completed\nWall time 0.1 seconds\nOutput:\n", result)) + wire := string(mustMarshalJSON(retireCompactionOperations(items))) + + for _, evidence := range []string{ + "successful read-tool return", + "https://developers.openai.com/api/docs/guides/responses#streaming", + "https://developers.openai.com/api/docs/guides/responses", + "streaming", "lvl2", "Guides", "Responses", "docs-responses-streaming", + "_snippetResult", "_highlightResult", "matchLevel", "matchedWords", + "fullyHighlighted", "snippet-metadata", "unknown_hit_metadata", + "developer-docs-index", "nbHits", "page", "algolia-cursor-3", + "unknown_root_metadata", "processingTimeMS", "algolia-request-4", + "annotations", "audience", "priority", "WARNING: streaming preview behavior", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("actual search-schema evidence %q was lost", evidence) + } + } + if strings.Count(wire, "unmarked Algolia search body") > 6 { + t.Fatal("actual search-schema body and annotation copies were not retired") + } +} + +func TestCompactionSuccessfulReadWithoutBodyReductionStaysEligible(t *testing.T) { + tool := compactionDocsFetchTool + source := readToolSource(tool, `{"url":"https://developers.openai.com/api/docs/index"}`) + result := map[string]any{ + "isError": false, + "_meta": map[string]any{"request_id": "short-read-2"}, + "content": []any{map[string]any{ + "type": "text", "text": "Short successful documentation result.", "annotations": map[string]any{"priority": 1}, + }}, + } + output := readToolResultOutput("operation_00", "Script completed\nWall time 0.1 seconds\nOutput:\n", result) + + var fields map[string]json.RawMessage + if json.Unmarshal(output, &fields) != nil { + t.Fatal("invalid test output") + } + reduced, ok := compactionRetiredReadToolOutput(fields["output"], tool) + if !ok || string(reduced) != string(fields["output"]) { + t.Fatal("known successful short read did not remain eligible with its exact output") + } + + items := readToolHistory(t, 0, tool, source, output) + items = append(items[:4], append([]json.RawMessage{ + compactTestCall("large_companion", "hread large.go"), + compactTestOutput("large_companion", strings.Repeat("unmarked companion body\n", 600), 0), + }, items[4:]...)...) + got := retireCompactionOperations(items) + for _, index := range []int{2, 3, 4, 5} { + if string(got[index]) == string(items[index]) { + t.Fatalf("profitable complete group was only partially retired at %d", index) + } + } + wire := string(mustMarshalJSON(got)) + for _, evidence := range []string{ + "Short successful documentation result.", "short-read-2", + "https://developers.openai.com/api/docs/index", "large_companion", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("profitable group lost successful read evidence %q", evidence) + } + } +} + +func TestCompactionRetiresDocumentationBodiesConservatively(t *testing.T) { + t.Run("fetched markdown", func(t *testing.T) { + tool := "mcp__openaiDeveloperDocs__fetch_openai_doc" + source := readToolSource(tool, `{"url":"https://developers.openai.com/api/docs/guides/responses"}`) + markdown := "# Responses API\nSource: OpenAI developer documentation\n" + + "https://developers.openai.com/api/docs/guides/responses\n" + + "Citation: 【responses-fetch】\n\n## Create a response\n" + + strings.Repeat("technical example body with request fields\n", 1200) + + "WARNING: preview behavior may change\n" + result := map[string]any{ + "_meta": map[string]any{"request_id": "fetch-request-9"}, + "content": []any{map[string]any{"type": "text", "text": markdown}}, + } + items := readToolHistory(t, 0, tool, source, + readToolResultOutput("operation_00", "Script completed\nWall time 0.1 seconds\nOutput:\n", result)) + wire := string(mustMarshalJSON(retireCompactionOperations(items))) + for _, evidence := range []string{ + "successful read-tool return", "# Responses API", "Source: OpenAI developer documentation", + "https://developers.openai.com/api/docs/guides/responses", "【responses-fetch】", + "## Create a response", "WARNING: preview behavior may change", "fetch-request-9", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("fetched-document evidence %q was lost", evidence) + } + } + if strings.Count(wire, "technical example body") > 2 { + t.Fatal("fetched technical body was not retired") + } + }) + + t.Run("OpenAPI document", func(t *testing.T) { + tool := "mcp__openaiDeveloperDocs__get_openapi_spec" + source := readToolSource(tool, `{"url":"https://api.openai.com/v1/responses","languages":["javascript"]}`) + spec := map[string]any{ + "openapi": "3.1.0", + "info": map[string]any{"title": "OpenAI API", "version": "2026-09-01"}, + "servers": []any{map[string]any{"url": "https://api.openai.com/v1"}}, + "paths": map[string]any{ + "/responses": map[string]any{"post": map[string]any{ + "operationId": "createResponse", "summary": "Create a response", + "description": strings.Repeat("large endpoint description ", 1200), + "responses": map[string]any{"200": map[string]any{"description": strings.Repeat("large schema ", 1200)}}, + }}, + }, + "components": map[string]any{"schemas": map[string]any{"Response": map[string]any{ + "description": strings.Repeat("large component schema ", 1200), + }}}, + } + result := map[string]any{ + "_meta": map[string]any{"source": "OpenAI OpenAPI"}, + "content": []any{map[string]any{"type": "text", "text": string(mustMarshalJSON(spec))}}, + } + items := readToolHistory(t, 0, tool, source, + readToolResultOutput("operation_00", "Script completed\nWall time 0.1 seconds\nOutput:\n", result)) + wire := string(mustMarshalJSON(retireCompactionOperations(items))) + for _, evidence := range []string{ + "successful read-tool return", "3.1.0", "OpenAI API", "2026-09-01", + "https://api.openai.com/v1", "/responses", "createResponse", "Create a response", + "OpenAI OpenAPI", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("OpenAPI evidence %q was lost", evidence) + } + } + for _, body := range []string{"large endpoint description", "large schema", "large component schema"} { + if strings.Contains(wire, body) { + t.Fatalf("OpenAPI body %q was not retired", body) + } + } + }) +} +func TestCompactionDocumentationReadFailuresStayNative(t *testing.T) { + goodSource := readToolSource(testDocsSearchTool, `{"query":"Responses API"}`) + goodResult := map[string]any{ + "content": []any{map[string]any{"type": "text", "text": `{"results":[{"url":"https://developers.openai.com","snippet":"body"}]}`}}, + } + cases := []struct { + name string + source string + output json.RawMessage + }{ + {"reported error", goodSource, readToolResultOutput("operation_00", "Script completed\n", map[string]any{ + "isError": true, "content": []any{map[string]any{"type": "text", "text": "permission error: citation unavailable"}}, + })}, + {"unknown error flag", goodSource, readToolResultOutput("operation_00", "Script completed\n", map[string]any{ + "isError": "false", "content": []any{map[string]any{"type": "text", "text": "ambiguous status"}}, + })}, + {"media", goodSource, readToolResultOutput("operation_00", "Script completed\n", map[string]any{ + "content": []any{map[string]any{"type": "image", "data": "opaque-media"}}, + })}, + {"failed header", goodSource, readToolResultOutput("operation_00", "Script failed\n", goodResult)}, + {"live header", goodSource, readToolResultOutput("operation_00", "Script running\n", goodResult)}, + {"malformed result", goodSource, mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\n"}, + map[string]any{"type": "input_text", "text": "not a CallToolResult"}, + }, + })}, + {"unknown tool", readToolSource("mcp__openaiDeveloperDocs__other", `{"query":"Responses API"}`), + readToolResultOutput("operation_00", "Script completed\n", goodResult)}, + {"mapped projection", `const result = await tools.` + testDocsSearchTool + `({"query":"Responses API"}); text(result.content.map(value => value.text));`, + readToolResultOutput("operation_00", "Script completed\n", goodResult)}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + items := readToolHistory(t, 0, testDocsSearchTool, test.source, test.output) + got := retireCompactionOperations(items) + if string(got[2]) != string(items[2]) || string(got[3]) != string(items[3]) { + t.Fatal("unsupported or unsuccessful documentation result was retired") + } + }) + } +} +func TestCompactionTruncatedDocumentationSearchProjectionStaysNative(t *testing.T) { + source := readToolSource(testDocsSearchTool, `{"query":"responses streaming"}`) + truncated := "Warning: truncated output (original token count exceeded)\nTotal output lines: 1\n\n" + + `{"content":[{"type":"text","text":"{\"hits\":[{\"url\":\"https://developers.openai.com/api/docs/guides/responses#streaming\",` + + `\"url_without_anchor\":\"https://developers.openai.com/api/docs/guides/responses\",\"anchor\":\"streaming\",` + + `\"content\":\"` + strings.Repeat("truncated historical search body ", 1200) + + `\",\"type\":\"lvl2\",\"hierarchy\":{\"lvl0\":\"Guides\"},\"objectID\":\"responses-streaming\",` + + `\"_snippetResult\":{\"content\":{\"value\":\"incomplete` + output := mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": truncated}, + }, + }) + items := readToolHistory(t, 0, testDocsSearchTool, source, output) + got := retireCompactionOperations(items) + if string(got[2]) != string(items[2]) || string(got[3]) != string(items[3]) { + t.Fatal("truncated non-JSON documentation projection was retired") + } + for _, evidence := range []string{ + "Warning: truncated output", "Total output lines: 1", + "url_without_anchor", "_snippetResult", "truncated historical search body", + } { + if !strings.Contains(string(got[3]), evidence) { + t.Fatalf("truncated projection evidence %q was lost", evidence) + } + } +} + +func TestCompactionTruncatedDocumentationSearchRetiresCompleteBodySpan(t *testing.T) { + source := readToolSource(testDocsSearchTool, `{"query":"responses streaming"}`) + truncated := "Warning: truncated output (original token count exceeded)\nTotal output lines: 1\n\n" + + `{"content":[{"type":"text","text":"{\"hits\":[{\"url\":\"https://developers.openai.com/api/docs/guides/responses\",` + + `\"content\":\"# Responses guide\\n` + strings.Repeat(`unmarked historical body\\n`, 500) + + `uncertain boundary\` + `…500 tokens truncated…` + `ncontinues here\\n` + + strings.Repeat(`more unmarked historical body\\n`, 500) + + `WARNING: retained limitation\\n\",\"future\":\"exact unknown metadata\"}]}"}],"future_result":"keep exact"}` + output := mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": truncated}, + }, + }) + items := readToolHistory(t, 0, testDocsSearchTool, source, output) + got := retireCompactionOperations(items) + if string(got[2]) == string(items[2]) || string(got[3]) == string(items[3]) { + t.Fatal("complete recognized body span around client truncation stayed native") + } + wire := string(got[3]) + for _, evidence := range []string{ + "Warning: truncated output", "Total output lines: 1", "developers.openai.com", + "uncertain boundary", "continues here", "WARNING: retained limitation", + "exact unknown metadata", "future_result", "keep exact", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("truncated projection evidence %q was lost", evidence) + } + } + if strings.Count(wire, "unmarked historical body") > 2 { + t.Fatal("unambiguous truncated historical body span was not retired") + } +} + +func TestCompactionDocumentationReadProtectionAndRecentFrontier(t *testing.T) { + result := map[string]any{ + "content": []any{map[string]any{ + "type": "text", + "text": `{"results":[{"id":"citation-17","url":"https://developers.openai.com","snippet":"` + + strings.Repeat("historical referenced body ", 500) + `"}]}`, + }}, + } + source := readToolSource(testDocsSearchTool, `{"query":"Responses API"}`) + + t.Run("referenced call", func(t *testing.T) { + items := readToolHistory(t, 0, testDocsSearchTool, source, + readToolResultOutput("operation_00", "Script completed\n", result)) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Continue from operation_00.", + })) + got := retireCompactionOperations(items) + if string(got[2]) != string(items[2]) || string(got[3]) != string(items[3]) { + t.Fatal("explicitly referenced documentation call was retired") + } + }) + + t.Run("newest eight", func(t *testing.T) { + items := readToolHistory(t, 16, testDocsSearchTool, source, + readToolResultOutput("operation_16", "Script completed\n", result)) + before := string(mustMarshalJSON(items)) + got := retireCompactionOperations(items) + for index, raw := range items { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) == nil && jsonString(fields, "call_id") == "operation_16" && + string(got[index]) != string(raw) { + t.Fatal("documentation read inside newest-eight frontier changed") + } + } + if !strings.Contains(before, "historical referenced body") { + t.Fatal("test setup lost the native body") + } + }) +} diff --git a/internal/router/context_compaction_records.go b/internal/router/context_compaction_records.go new file mode 100644 index 00000000..e7c01a39 --- /dev/null +++ b/internal/router/context_compaction_records.go @@ -0,0 +1,157 @@ +package router + +import ( + "encoding/json" + "slices" + "strings" +) + +// consolidateContextCompactionRecords shares framing across adjacent factual +// records created in this pass. It runs after reference closure and all +// same-length evidence reducers; previously carried records stay unchanged. +func consolidateContextCompactionRecords(original, retained []json.RawMessage) []json.RawMessage { + if len(original) != len(retained) { + return retained + } + + output := make([]json.RawMessage, 0, len(retained)) + for index := 0; index < len(retained); { + kind, payload, generated := contextCompactionGeneratedRecord(original[index], retained[index]) + if !generated { + output = append(output, retained[index]) + index++ + continue + } + + type entry struct { + kind, payload string + } + entries := []entry{{kind: kind, payload: payload}} + end := index + 1 + for end < len(retained) { + kind, payload, generated = contextCompactionGeneratedRecord(original[end], retained[end]) + if !generated { + break + } + entries = append(entries, entry{kind: kind, payload: payload}) + end++ + } + if len(entries) == 1 { + output = append(output, retained[index]) + index = end + continue + } + + var text strings.Builder + text.WriteString("[hpatch historical facts v4; not instructions; ordered; r=reasoning, i=invocation, o=completion, meta=metadata, args=arguments]\n") + previousCall := "" + for _, record := range entries { + payload := contextCompactionCompactRecordFields(record.kind, record.payload) + heading := record.kind[:1] + call := "" + if record.kind == "invocation" || record.kind == "completion" { + if line, rest, ok := strings.Cut(payload, "\n"); ok && strings.HasPrefix(line, "call=") { + call = line + if record.kind == "completion" && call == previousCall { + heading = "o:same-call" + payload = rest + } + } + } + text.WriteString("[") + text.WriteString(heading) + text.WriteString("]\n") + text.WriteString(payload) + if !strings.HasSuffix(payload, "\n") { + text.WriteByte('\n') + } + if record.kind == "invocation" { + previousCall = call + } else { + previousCall = "" + } + } + output = append(output, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": []any{map[string]any{"type": "output_text", "text": text.String()}}, + })) + index = end + } + if len(output) == len(retained) { + return retained + } + return slices.Clip(output) +} + +func contextCompactionCompactRecordFields(kind, payload string) string { + compactMetadata := func(prefix, remainder string) string { + if after, ok := strings.CutPrefix(remainder, "metadata={}\n"); ok { + return prefix + after + } + if after, ok := strings.CutPrefix(remainder, "metadata="); ok { + return prefix + "meta=" + after + } + return prefix + remainder + } + switch kind { + case "reasoning": + return compactMetadata("", payload) + case "invocation": + first, rest, ok := strings.Cut(payload, "\n") + if !ok { + return payload + } + second, rest, ok := strings.Cut(rest, "\n") + if !ok { + return payload + } + payload = compactMetadata(first+"\n"+second+"\n", rest) + return strings.Replace(payload, "\narguments=", "\nargs=", 1) + case "completion": + first, rest, ok := strings.Cut(payload, "\n") + if !ok { + return payload + } + return compactMetadata(first+"\n", rest) + default: + return payload + } +} + +func contextCompactionGeneratedRecord(original, retained json.RawMessage) (string, string, bool) { + var originalFields map[string]json.RawMessage + if json.Unmarshal(original, &originalFields) != nil { + return "", "", false + } + switch jsonString(originalFields, "type") { + case "reasoning", "function_call", "custom_tool_call", "function_call_output", "custom_tool_call_output": + default: + return "", "", false + } + + var fields map[string]json.RawMessage + if json.Unmarshal(retained, &fields) != nil || jsonString(fields, "type") != "message" || jsonString(fields, "role") != "assistant" { + return "", "", false + } + var content []map[string]json.RawMessage + if json.Unmarshal(fields["content"], &content) != nil || len(content) != 1 || jsonString(content[0], "type") != "output_text" { + return "", "", false + } + text := jsonString(content[0], "text") + header, payload, ok := strings.Cut(text, "\n") + if !ok { + return "", "", false + } + var kind string + switch { + case strings.HasPrefix(header, "[hpatch historical reasoning fact v3;"): + kind = "reasoning" + case strings.HasPrefix(header, "[hpatch historical tool invocation v3;"): + kind = "invocation" + case strings.HasPrefix(header, "[hpatch historical tool completion v3;"): + kind = "completion" + default: + return "", "", false + } + return kind, payload, true +} diff --git a/internal/router/context_compaction_records_test.go b/internal/router/context_compaction_records_test.go new file mode 100644 index 00000000..5dbcaa6a --- /dev/null +++ b/internal/router/context_compaction_records_test.go @@ -0,0 +1,77 @@ +package router + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestCompactionConsolidatesAdjacentNewRecords(t *testing.T) { + items := retirementHistory() + got := reduceContextCompaction(items) + if len(got) >= len(items)-20 { + t.Fatalf("adjacent factual records were not consolidated: %d -> %d items", len(items), len(got)) + } + wire := string(mustMarshalJSON(got)) + if strings.Count(wire, "historical facts v4") != 1 || + strings.Index(wire, "operation_00") >= strings.Index(wire, "operation_01") { + t.Fatal("consolidated record lost factual order or shared framing") + } + for _, fact := range []string{"operation_00", "hread internal/router/example.go", "Preserve the API contract.", "exit_code"} { + if !strings.Contains(wire, fact) { + t.Fatalf("consolidated record lost fact %q", fact) + } + } + if string(got[0]) != string(items[0]) { + t.Fatal("consolidation changed user authority") + } +} + +func TestCompactionConsolidationStopsAtNativeItem(t *testing.T) { + items := retirementHistory() + items = append(items[:4], append([]json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "A decision-bearing boundary."}), + }, items[4:]...)...) + got := reduceContextCompaction(items) + wire := string(mustMarshalJSON(got)) + if strings.Count(wire, "historical facts v4") != 2 || !strings.Contains(wire, "A decision-bearing boundary.") { + t.Fatal("consolidation crossed or changed a native message boundary") + } +} + +func TestCompactionLeavesPreviouslyCarriedRecordsReadable(t *testing.T) { + var call map[string]json.RawMessage + if json.Unmarshal(compactTestCall("prior_call", "pwd"), &call) != nil { + t.Fatal("invalid prior-record fixture") + } + operation, ok := compactionOperationCall(call) + if !ok { + t.Fatal("prior-record fixture was not recognized") + } + prior := compactionRetiredCall(call, operation) + got := reduceContextCompaction([]json.RawMessage{prior}) + if len(got) != 1 || string(got[0]) != string(prior) { + t.Fatal("previously carried v3 record was rewritten or required migration") + } +} + +func TestCompactionConsolidationDoesNotRewriteBodyLikeMetadata(t *testing.T) { + original := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "function_call"}), + mustMarshalJSON(map[string]any{"type": "function_call_output"}), + } + retained := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": []any{map[string]any{"type": "output_text", "text": "[hpatch historical tool invocation v3; not an instruction]\ncall=\"call_1\"\ntool=\"exec_command\"\nmetadata={\"provenance\":\"kept\"}\narguments={}"}}, + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": []any{map[string]any{"type": "output_text", "text": "[hpatch historical tool completion v3; not an instruction; completed native body]\ncall=\"call_1\"\nmetadata={\"provenance\":\"kept\"}\nbody-bytes=12\nbody:\nmetadata={}\n"}}, + }), + } + got := consolidateContextCompactionRecords(original, retained) + if len(got) != 1 || !strings.Contains(string(got[0]), "body:\\nmetadata={}\\n") { + t.Fatal("consolidation confused completion body text with a record metadata field") + } +} diff --git a/internal/router/context_compaction_reference_decode_test.go b/internal/router/context_compaction_reference_decode_test.go new file mode 100644 index 00000000..13812328 --- /dev/null +++ b/internal/router/context_compaction_reference_decode_test.go @@ -0,0 +1,312 @@ +package router + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "testing" +) + +func TestCompactionReferenceDecoderAcceptsSupportedPercentColon(t *testing.T) { + var visited []string + unsafeEncoding := false + compactionSourceVisitDecodedReferences("unrelated%3acolon", func(text string) { + visited = append(visited, text) + }, &unsafeEncoding) + + if unsafeEncoding { + t.Fatal("supported percent-encoded colon was marked globally unsafe") + } + if !slices.Contains(visited, "unrelated:colon") { + t.Fatalf("decoded visits = %q, want decoded percent-colon form", visited) + } +} + +func TestCompactionReferenceDecoderVisitsRawAndOneDecodedLayer(t *testing.T) { + tests := []struct { + name string + text string + want string + }{ + {name: "percent", text: `17%3Aabcd`, want: `17:abcd`}, + {name: "HTML decimal", text: `17:abcd`, want: `17:abcd`}, + {name: "HTML hexadecimal", text: `17:abcd`, want: `17:abcd`}, + {name: "HTML named", text: `17:abcd`, want: `17:abcd`}, + {name: "JavaScript fixed unicode", text: `"turn\u005fkeep"`, want: `"turn_keep"`}, + {name: "JavaScript code point", text: "`17\\u{3a}abcd`", want: "`17:abcd`"}, + {name: "JavaScript code point leading zeros", text: "`17\\u{00000003a}abcd`", want: "`17:abcd`"}, + {name: "JavaScript hexadecimal", text: `'@shell\x2fresult'`, want: `'@shell/result'`}, + {name: "mixed direct encodings", text: `"turn\u{3a}keep%5fexact"`, want: `"turn:keep_exact"`}, + {name: "surrogate pair", text: `"face\uD83D\uDE00"`, want: `"face😀"`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var visited []string + unsafeEncoding := false + compactionSourceVisitDecodedReferences(test.text, func(text string) { + visited = append(visited, text) + }, &unsafeEncoding) + if unsafeEncoding { + t.Fatal("supported encoding was marked globally unsafe") + } + if len(visited) < 2 || visited[0] != test.text || !slices.Contains(visited, test.want) { + t.Fatalf("decoded visits = %q, want raw %q then decoded %q", visited, test.text, test.want) + } + }) + } + + var visited []string + unsafeEncoding := false + compactionSourceVisitDecodedReferences(`17%2558abcd`, func(text string) { + visited = append(visited, text) + }, &unsafeEncoding) + if unsafeEncoding || slices.Contains(visited, `17:abcd`) || !slices.Contains(visited, `17%58abcd`) { + t.Fatalf("nested encoding was interpreted more than once: unsafe=%t visits=%q", unsafeEncoding, visited) + } +} + +func TestCompactionReferenceDecoderMarksMalformedEncodingsUnsafe(t *testing.T) { + for _, text := range []string{ + `"17\u03Zabcd"`, + `"17\u{110000}abcd"`, + `"17\x3Zabcd"`, + } { + t.Run(text, func(t *testing.T) { + unsafeEncoding := false + compactionSourceVisitDecodedReferences(text, func(string) {}, &unsafeEncoding) + if !unsafeEncoding { + t.Fatal("malformed or ambiguous encoding was not marked unsafe") + } + }) + } +} + +func TestCompactionReferenceDecoderLeavesIncompleteUntypedEncodingsRaw(t *testing.T) { + for _, input := range []string{ + `17%3`, + `17%3Zabcd`, + `17%u003Aabcd`, + `17:abcd`, + `17�`, + `17&colon`, + } { + t.Run(input, func(t *testing.T) { + unsafeEncoding := false + var visited []string + compactionSourceVisitDecodedReferences(input, func(text string) { + visited = append(visited, text) + }, &unsafeEncoding) + if unsafeEncoding || len(visited) != 1 || visited[0] != input { + t.Fatalf("untyped incomplete encoding was interpreted: unsafe=%t visits=%q", unsafeEncoding, visited) + } + }) + } +} + +func TestCompactionReferenceDecoderDoesNotGuessOrdinaryBackslashText(t *testing.T) { + input := `The files remain under C:\users\xray, fmt uses %d, and work is 100% complete.` + unsafeEncoding := false + compactionSourceVisitDecodedReferences(input, func(string) {}, &unsafeEncoding) + if unsafeEncoding { + t.Fatal("ordinary backslash or percent text was treated as a malformed encoding") + } +} + +func TestCompactionReferenceDecoderAcceptsPercentEncodedBinaryBytes(t *testing.T) { + input := "unrelated%FFbinary operation%5F00" + unsafeEncoding := false + var visited []string + compactionSourceVisitDecodedReferences(input, func(text string) { + visited = append(visited, text) + }, &unsafeEncoding) + if unsafeEncoding { + t.Fatal("syntactically valid percent-encoded binary byte was marked globally unsafe") + } + want := "unrelated\xffbinary operation_00" + if !slices.Contains(visited, want) { + t.Fatalf("decoded visits = %q, want byte-preserving form %q", visited, want) + } +} + +func TestCompactionReferenceDecoderPreservesEncodedRowWithoutGlobalPin(t *testing.T) { + firstRows := compactionSourceTestRows("", 18) + secondRows := compactionSourceTestRows("other.go", 18) + items := []json.RawMessage{ + compactTestCall("source-first", "hread first.go"), + compactTestOutput("source-first", strings.Join(firstRows, ""), 0), + compactTestCall("source-second", "hread second.go"), + compactTestOutput("source-second", strings.Join(secondRows, ""), 0), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": `Keep exact row 3%3A0003 beside unrelated binary %FF and state%3Aready.`, + }), + } + items = append(items, compactionSourceTestRecent()...) + + got := reduceContextCompactionSource(items, items) + for index, rows := range map[int][]string{1: firstRows, 3: secondRows} { + text := compactionSourceTestOutputText(t, got[index]) + if !strings.Contains(text, rows[2]) { + t.Fatalf("encoded row reference was not retained in result %d", index) + } + if string(got[index]) == string(items[index]) || strings.Contains(text, rows[10]) { + t.Fatalf("supported encoded text globally pinned result %d", index) + } + } +} + +func TestCompactionReferenceDecoderPreservesEncodedScriptReference(t *testing.T) { + items := retirementHistory() + items[3] = mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": "operation_00", + "output": string(mustMarshalJSON(map[string]any{ + "exit_code": 0, "script_ref": "@shell/result:17", + "output": strings.Repeat("old source\n", 500), + })), + }) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": "Use the result named `@shell/result%3A17`.", + })) + + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3} { + if string(got[index]) != string(items[index]) { + t.Fatalf("encoded script reference did not preserve producer item %d", index) + } + } +} + +func TestCompactionReferenceDecoderPreservesEncodedMetadataReference(t *testing.T) { + item := compactionMetadataTestItem("message", "metadata_encoded", "turn:keep") + reference := mustMarshalJSON(map[string]any{ + "type": "message", "id": "metadata_reference", "role": "assistant", + "content": `const turn = "turn:keep"; const state = "ready%3Atrue";`, + }) + got := reduceContextCompactionMetadata(compactionMetadataTestHistory(item, reference)) + var fields, metadata map[string]json.RawMessage + if json.Unmarshal(got[0], &fields) != nil || json.Unmarshal(fields["internal_chat_message_metadata_passthrough"], &metadata) != nil { + t.Fatal("metadata fixture was not retained as JSON") + } + if _, exists := metadata["turn_id"]; !exists { + t.Fatal("HTML-encoded turn reference was removed") + } + if _, exists := metadata["create_time"]; exists { + t.Fatal("unrelated supported encoded text globally pinned metadata") + } +} + +func TestCompactionReferenceDecoderPreservesEncodedAssistantID(t *testing.T) { + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "assistant_keep", + "content": "The completed result is recorded.", + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": "assistant_drop", + "content": "Another completed result is recorded.", + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", + "content": "Continue from `assistant\\u{5f}keep`; status%3Aready.", + }), + } + for index := range 12 { + id := fmt.Sprintf("reference_%02d", index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + + got := reduceContextCompactionNarration(items) + if !strings.Contains(string(got[0]), `"id":"assistant_keep"`) { + t.Fatal("escaped assistant ID reference was removed") + } + if strings.Contains(string(got[1]), `"id":"assistant_drop"`) { + t.Fatal("unrelated supported encoded text globally pinned assistant IDs") + } + + items[2] = mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", "content": `malformed "assistant\u{110000}"`, + }) + got = reduceContextCompactionNarration(items) + if !strings.Contains(string(got[1]), `"id":"assistant_drop"`) { + t.Fatal("malformed encoding did not conservatively preserve assistant IDs") + } +} + +func TestCompactionReferenceDecoderBinaryPercentDoesNotPinAssistantIDs(t *testing.T) { + const keepID = "123e4567-e89b-12d3-a456-426614174000" + const dropID = "123e4567-e89b-12d3-a456-426614174001" + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": keepID, + "content": "The referenced result is recorded.", + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": dropID, + "content": "The unrelated result is recorded.", + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", + "content": "Use 123e4567%2De89b%2D12d3%2Da456%2D426614174000; binary=%FF.", + }), + } + for index := range 12 { + id := fmt.Sprintf("binary_reference_%02d", index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + + got := reduceContextCompactionNarration(items) + if !strings.Contains(string(got[0]), `"id":"`+keepID+`"`) { + t.Fatal("encoded UUID reference beside a binary percent byte was removed") + } + if strings.Contains(string(got[1]), `"id":"`+dropID+`"`) { + t.Fatal("unrelated binary percent byte globally pinned assistant UUIDs") + } +} + +func TestCompactionReferenceDecoderPrintfTextDoesNotPinAssistantIDs(t *testing.T) { + const keepID = "assistant_literal_keep" + const dropID = "assistant_printf_drop" + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": keepID, + "content": "The referenced result is recorded.", + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "id": dropID, + "content": "The unrelated result is recorded.", + }), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", + "content": `Keep assistant_literal_keep; fmt.Sprintf("%d %2d %3.2f %% %3Z", n).`, + }), + } + for index := range 12 { + id := fmt.Sprintf("printf_reference_%02d", index) + items = append(items, compactTestCall(id, `printf "%d %2d %3.2f %%\n" 1 2 3`), compactTestOutput(id, "%d %2d %3.2f %%\n", 0)) + } + + got := reduceContextCompactionNarration(items) + if !strings.Contains(string(got[0]), `"id":"`+keepID+`"`) { + t.Fatal("raw literal assistant reference beside printf formats was removed") + } + if strings.Contains(string(got[1]), `"id":"`+dropID+`"`) { + t.Fatal("ordinary printf formats globally pinned assistant IDs") + } +} + +func TestCompactionReferenceDecoderDoesNotEvaluateTemplateInterpolation(t *testing.T) { + input := "const row = `17\\u{3a}abcd${tools.exec_command({cmd: 'false'})}`;" + unsafeEncoding := false + var decoded string + compactionSourceVisitDecodedReferences(input, func(text string) { + if text != input { + decoded = text + } + }, &unsafeEncoding) + if unsafeEncoding || !strings.Contains(decoded, "17:abcd") || + !strings.Contains(decoded, "${tools.exec_command({cmd: 'false'})}") { + t.Fatalf("static decoding changed template interpolation: unsafe=%t decoded=%q", unsafeEncoding, decoded) + } +} diff --git a/internal/router/context_compaction_repeated.go b/internal/router/context_compaction_repeated.go new file mode 100644 index 00000000..ace78127 --- /dev/null +++ b/internal/router/context_compaction_repeated.go @@ -0,0 +1,195 @@ +package router + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" +) + +// Repeated verified source excerpts are replacement evidence even when Code +// Mode carries the read through an opaque script. We do not interpret that +// script or infer which command ran: only completed, successful output qualifies. +func reduceRepeatedCompactionRows(input []json.RawMessage, protected map[string]bool) []json.RawMessage { + type source struct { + lines []string + start int + callID string + } + // A retained-source reference must identify one call and one result. + calls, results := make(map[string]int), make(map[string]int) + positions := make(map[string]int) + for index, raw := range input { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + id := jsonString(fields, "call_id") + switch jsonString(fields, "type") { + case "function_call", "custom_tool_call": + calls[id]++ + positions[id] = index + case "function_call_output", "custom_tool_call_output": + results[id]++ + } + } + sources := make(map[string]source) + lastResult := true + for index := len(input) - 1; index >= 0; index-- { + var fields map[string]json.RawMessage + if json.Unmarshal(input[index], &fields) != nil { + continue + } + kind, callID := jsonString(fields, "type"), jsonString(fields, "call_id") + if (kind != "function_call_output" && kind != "custom_tool_call_output") || callID == "" { + continue + } + if calls[callID] != 1 || results[callID] != 1 || positions[callID] >= index { + lastResult = false + continue + } + var retained []string + output := mapCompactionCompletedOutput(fields["output"], func(text string) string { + lines := strings.SplitAfter(text, "\n") + var result strings.Builder + for row := 0; row < len(lines); { + key := compactionRowWindow(lines, row) + prior, found := sources[key] + if lastResult || protected[callID] || key == "" || !found { + result.WriteString(lines[row]) + row++ + continue + } + count := 4 + for row+count < len(lines) && prior.start+count < len(prior.lines) && + compactionSourceRow.MatchString(lines[row+count]) && lines[row+count] == prior.lines[prior.start+count] { + count++ + } + size := 0 + for _, line := range lines[row : row+count] { + size += len(line) + } + note := "" + // Bound formatting by the span it might replace, even when a + // very long call ID is matched by many separate short excerpts. + if size >= 256 && size > len(prior.callID) { + note = fmt.Sprintf("[hpatch compaction: %d source rows (%s through %s) retained verbatim in later tool result %q]\n", + count, strings.Fields(lines[row])[0], strings.Fields(lines[row+count-1])[0], prior.callID) + } + if note == "" || len(note) >= size { + + // Retain the whole rejected match. Rescanning every suffix + // would be quadratic for large excerpts or long call IDs. + for _, line := range lines[row : row+count] { + result.WriteString(line) + } + row += count + continue + } + result.WriteString(note) + row += count + } + reduced := result.String() + retained = append(retained, reduced) + return reduced + }) + if string(output) != string(fields["output"]) { + fields["output"] = output + input[index] = mustMarshalJSON(fields) + } + // Index only the final retained text, after the complete result is + // processed. References cannot target this same result or deleted rows. + for _, text := range retained { + lines := strings.SplitAfter(text, "\n") + for row := range lines { + if key := compactionRowWindow(lines, row); key != "" { + if _, exists := sources[key]; !exists { + sources[key] = source{lines: lines, start: row, callID: callID} + } + } + } + } + lastResult = false + } + return input +} + +// Require complete verified-row lines, not partial substrings of diagnostics. +var compactionSourceRow = regexp.MustCompile(`^[1-9][0-9]*:[0-9a-f]{4} [^\r\n]*\r?\n$`) + +func compactionRowWindow(lines []string, start int) string { + if start+4 > len(lines) { + return "" + } + for _, line := range lines[start : start+4] { + if !compactionSourceRow.MatchString(line) { + return "" + } + } + return strings.Join(lines[start:start+4], "") +} + +func mapCompactionCompletedOutput(raw json.RawMessage, transform func(string) string) json.RawMessage { + if encode, text, ok := contextCompactionOutput(raw); ok { + if reduced := transform(text); reduced != text { + return encode(reduced) + } + return raw + } + // Codex stores Code Mode output as input_text content blocks. The first + // block is the executor's status; subsequent blocks hold its emitted values. + // A yielded or failed execution is not a completed source of evidence. + var parts []json.RawMessage + if json.Unmarshal(raw, &parts) != nil || len(parts) < 2 { + return raw + } + var status map[string]json.RawMessage + if json.Unmarshal(parts[0], &status) != nil || jsonString(status, "type") != "input_text" || + !strings.HasPrefix(jsonString(status, "text"), "Script completed\n") { + return raw + } + changed := false + for index := 1; index < len(parts); index++ { + var part map[string]json.RawMessage + if json.Unmarshal(parts[index], &part) != nil || jsonString(part, "type") != "input_text" { + continue + } + encode, text, ok := contextCompactionOutput(part["text"]) + if !ok { + continue + } + if reduced := transform(text); reduced != text { + part["text"] = encode(reduced) + parts[index] = mustMarshalJSON(part) + changed = true + } + } + if changed { + return mustMarshalJSON(parts) + } + return raw +} + +// Replacement notes are durable references, not just progress prose. Protect +// their targets on later compactions as well as within the current reduction. +var compactionRetainedReference = regexp.MustCompile(`(?m)^\[hpatch compaction: (?:matching search listing retained verbatim in tool result |[0-9]+ source rows \([^\r\n]*\) retained verbatim in later tool result )("(?:\\.|[^"\\])*")`) + +func contextCompactionReferencedResults(input []json.RawMessage) map[string]bool { + protected := make(map[string]bool) + for _, raw := range input { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + continue + } + if kind := jsonString(fields, "type"); kind != "function_call_output" && kind != "custom_tool_call_output" { + continue + } + compactionVisitReferenceStrings(fields["output"], func(text string) { + for _, match := range compactionRetainedReference.FindAllStringSubmatch(text, -1) { + if id, err := strconv.Unquote(match[1]); err == nil { + protected[id] = true + } + } + }) + } + return protected +} diff --git a/internal/router/context_compaction_repeated_test.go b/internal/router/context_compaction_repeated_test.go new file mode 100644 index 00000000..2b3e1f7c --- /dev/null +++ b/internal/router/context_compaction_repeated_test.go @@ -0,0 +1,439 @@ +package router + +import ( + "bufio" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/tiktoken-go/tokenizer" +) + +func compactCodeModeOutput(id, text string) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": id, + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": string(mustMarshalJSON(map[string]any{ + "exit_code": 0, "output": text, "script_ref": "@shell/retained", + }))}, + }, + }) +} + +func TestCompactionRepeatedCodeModeSource(t *testing.T) { + var excerpt strings.Builder + for row := 1; row <= 12; row++ { + fmt.Fprintf(&excerpt, "%d:abcd source declaration with enough exact text to distinguish this retained source row\n", row) + } + call := func(id string) json.RawMessage { + return mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": id, "input": "opaque script"}) + } + items := []json.RawMessage{ + call("old"), compactCodeModeOutput("old", "old read header\n"+excerpt.String()+"unique old diagnostic\n"), + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Keep the earlier constraint."}), + call("middle"), compactCodeModeOutput("middle", excerpt.String()), + call("new"), compactCodeModeOutput("new", "new read header\n"+excerpt.String()), + } + ambiguous := append([]json.RawMessage{call("old")}, items...) + if got := reduceContextCompaction(ambiguous); string(got[2]) != string(ambiguous[2]) { + t.Fatal("ambiguous call identity was used for source replacement") + } + before := string(mustMarshalJSON(items)) + got := reduceContextCompaction(items) + for index := range items { + if index != 1 && index != 4 && string(got[index]) != string(items[index]) { + t.Fatalf("protected item %d changed", index) + } + } + if !strings.Contains(string(got[1]), "unique old diagnostic") || !strings.Contains(string(got[1]), "retained verbatim") || + !strings.Contains(string(got[4]), "retained verbatim") { + t.Fatal("duplicate excerpt was not reduced with unique evidence retained") + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("input was mutated") + } + if string(mustMarshalJSON(reduceContextCompaction(got))) != string(mustMarshalJSON(got)) { + t.Fatal("repeated reduction changed the retained references") + } + extended := append(append([]json.RawMessage(nil), got...), call("newest"), compactCodeModeOutput("newest", excerpt.String())) + if again := reduceContextCompaction(extended); string(again[6]) != string(got[6]) { + t.Fatal("a subsequent compaction pruned already-referenced evidence") + } + // The search reducer and source-row reducer must not invalidate one + // another's references, including across repeated compactions. + search := []json.RawMessage{ + compactTestCall("search", "rg rows file"), compactTestOutput("search", excerpt.String(), 0), + compactTestCall("read", "cat file"), compactTestOutput("read", excerpt.String(), 0), + call("last"), compactCodeModeOutput("last", excerpt.String()), + } + searched := reduceContextCompaction(search) + if !strings.Contains(string(searched[1]), "matching search listing") || string(searched[3]) != string(search[3]) { + t.Fatal("search replacement no longer points to verbatim retained evidence") + } + if again := reduceContextCompaction(searched); string(again[3]) != string(search[3]) { + t.Fatal("repeated compaction invalidated a search reference") + } + longID := strings.Repeat("long-id", 10000) + separated := strings.Repeat(excerpt.String()+"non-row separator\n", 100) + oversized := []json.RawMessage{call("first"), compactCodeModeOutput("first", separated), + call(longID), compactCodeModeOutput(longID, separated)} + if reduced := reduceContextCompaction(oversized); string(reduced[1]) != string(oversized[1]) { + t.Fatal("oversized replacement reference expanded the output") + } + for _, replacement := range []struct{ old, new string }{ + {"Script completed", "Script running"}, + {`\"exit_code\":0`, `\"exit_code\":1`}, + {"1:abcd source", "1:abce source"}, + } { + altered := append([]json.RawMessage(nil), items...) + altered[1] = json.RawMessage(strings.ReplaceAll(string(items[1]), replacement.old, replacement.new)) + // For changed text, require that changed evidence survives; unchanged + // trailing rows may still have an exact retained replacement. + reduced := reduceContextCompaction(altered) + if replacement.old == "1:abcd source" { + if !strings.Contains(string(reduced[1]), "1:abce source") { + t.Fatal("changed source evidence was removed") + } + } else if string(reduced[1]) != string(altered[1]) { + t.Fatal("unfinished or failed output changed") + } + } +} + +func TestCompactionReferencedResultsScansOnlySpecialResultNotes(t *testing.T) { + note := fmt.Sprintf( + "[hpatch compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", + "later-source", + ) + failed := compactTestOutput("failed-consumer", note, 1) + if !contextCompactionReferencedResults([]json.RawMessage{failed})["later-source"] { + t.Fatal("surviving failed result note did not protect its referenced evidence") + } + + decoys := []json.RawMessage{ + compactTestOutput("plain-id", "later-source", 1), + compactTestOutput("prefixed-note", "ordinary output: "+note, 1), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": note, + }), + } + if protected := contextCompactionReferencedResults(decoys); len(protected) != 0 { + t.Fatal("non-special or non-result text was treated as a replacement note") + } +} + +func compactionReplayAllowsOnlyMetadataCleanup(before, after json.RawMessage) bool { + var left, right map[string]json.RawMessage + if json.Unmarshal(before, &left) != nil || json.Unmarshal(after, &right) != nil || left == nil || right == nil { + return false + } + var leftMetadata, rightMetadata map[string]json.RawMessage + if json.Unmarshal(left["internal_chat_message_metadata_passthrough"], &leftMetadata) != nil || leftMetadata == nil { + return false + } + if raw, exists := right["internal_chat_message_metadata_passthrough"]; exists { + if json.Unmarshal(raw, &rightMetadata) != nil || rightMetadata == nil { + return false + } + } else { + rightMetadata = map[string]json.RawMessage{} + } + + removed := false + for key, leftValue := range leftMetadata { + rightValue, exists := rightMetadata[key] + if key == "turn_id" || key == "create_time" { + if !exists { + removed = true + continue + } + } else if !exists { + return false + } + if contextCompactionCanonicalJSON(leftValue) != contextCompactionCanonicalJSON(rightValue) { + return false + } + } + for key := range rightMetadata { + if _, exists := leftMetadata[key]; !exists { + return false + } + } + delete(left, "internal_chat_message_metadata_passthrough") + delete(right, "internal_chat_message_metadata_passthrough") + return removed && contextCompactionCanonicalJSON(mustMarshalJSON(left)) == contextCompactionCanonicalJSON(mustMarshalJSON(right)) +} + +// Opt-in private-history replay. Only aggregate sizes are reported; no +// conversation text is copied into fixtures or printed on failure. +func TestCompactionRolloutReplay(t *testing.T) { + path := os.Getenv("HPATCH_COMPACTION_ROLLOUT") + if path == "" { + t.Skip("set HPATCH_COMPACTION_ROLLOUT to check a local rollout") + } + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + var input []json.RawMessage + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 64*1024), responsesRequestBufferBytes) + for scanner.Scan() { + var record struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + } + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + t.Fatal("invalid rollout record") + } + // Reproduce the first compaction boundary, not a concatenation of + // pre-compaction history and later continuation records. + if record.Type == "compacted" { + break + } + if record.Type == "response_item" { + input = append(input, record.Payload) + } + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + reduced := reduceContextCompaction(input) + changed := len(input) - len(reduced) + if changed == 0 { + for index := range input { + if string(input[index]) != string(reduced[index]) { + changed++ + } + } + } + retainedCursor := 0 + for index := range input { + var fields map[string]json.RawMessage + _ = json.Unmarshal(input[index], &fields) + kind := jsonString(fields, "type") + if kind == "function_call_output" || kind == "custom_tool_call_output" || kind == "function_call" || kind == "custom_tool_call" || kind == "reasoning" { + continue + } + if kind == "agent_message" || kind == "message" && jsonString(fields, "role") == "assistant" { + continue + } + found := false + for retainedCursor < len(reduced) { + candidate := reduced[retainedCursor] + retainedCursor++ + if string(input[index]) == string(candidate) || compactionReplayAllowsOnlyMetadataCleanup(input[index], candidate) { + found = true + break + } + } + if !found { + t.Fatalf("protected item %d changed or moved out of order", index) + } + } + if changed == 0 { + t.Fatal("rollout has no supported reduction") + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(string(mustMarshalJSON(map[string]any{ + "model": "loopback", "input": input, "stream": true, + })))) + request.Header.Set(codexTurnMetadataHeader, string(mustMarshalJSON(map[string]any{ + "request_kind": "compaction", "compaction": map[string]any{"implementation": "responses_compaction_v2", "trigger": "manual", "reason": "user_request", "phase": "mid_turn", "strategy": "memento"}, + }))) + compactor.handler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("compaction reached provider") }))(response, request) + if response.Code != http.StatusOK { + t.Fatalf("local compaction status %d", response.Code) + } + var capsule json.RawMessage + for _, line := range strings.Split(response.Body.String(), "\n") { + if !strings.HasPrefix(line, "data: ") { + continue + } + var event struct { + Type string `json:"type"` + Item json.RawMessage `json:"item"` + } + if json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &event) == nil && event.Type == "response.output_item.done" { + capsule = event.Item + } + } + if len(capsule) == 0 { + t.Fatal("stream omitted completed compaction item") + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{capsule}) + + if err != nil || contextCompactionCanonicalJSON(mustMarshalJSON(restored)) != contextCompactionCanonicalJSON(mustMarshalJSON(reduced)) { + t.Fatalf("native history round trip failed: error=%v; item counts=%d -> %d", err, len(reduced), len(restored)) + } + // Check the ordinary continuation boundary too, not only envelope opening. + suffix := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Continue the current task."}) + nextInput := []json.RawMessage{capsule, suffix} + continued := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(string(mustMarshalJSON(map[string]any{ + "model": "loopback", "input": nextInput, "stream": true, + })))) + seen := false + compactor.handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Input []json.RawMessage `json:"input"` + } + if json.NewDecoder(r.Body).Decode(&body) != nil { + t.Fatal("invalid restored continuation") + } + want := append(append([]json.RawMessage(nil), reduced...), suffix) + if contextCompactionCanonicalJSON(mustMarshalJSON(body.Input)) != contextCompactionCanonicalJSON(mustMarshalJSON(want)) { + t.Fatal("retired material reappeared or continuation was lost") + } + seen = true + }))(httptest.NewRecorder(), continued) + if !seen { + t.Fatal("continuation did not reach the model boundary") + } + legacy := httptest.NewRecorder() + compactor.handler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("legacy compaction reached provider") + }))(legacy, httptest.NewRequest(http.MethodPost, "/v1/responses/compact", + strings.NewReader(string(mustMarshalJSON(map[string]any{"model": "loopback", "input": input}))))) + var legacyResponse struct { + Output []json.RawMessage `json:"output"` + } + if legacy.Code != http.StatusOK || json.Unmarshal(legacy.Body.Bytes(), &legacyResponse) != nil { + t.Fatalf("legacy compaction failed: status=%d", legacy.Code) + } + legacyRestored, err := compactor.restore(t.Context(), legacyResponse.Output) + if err != nil || contextCompactionCanonicalJSON(mustMarshalJSON(legacyRestored)) != contextCompactionCanonicalJSON(mustMarshalJSON(reduced)) { + t.Fatalf("legacy replay duplicated or lost selected context: %v", err) + } + // Report only aggregate structural reasons, never private transcript text. + calls := make(map[string]map[string]json.RawMessage) + originalResults := make(map[string]json.RawMessage) + for _, raw := range input { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + kind := jsonString(fields, "type") + if kind == "function_call" || kind == "custom_tool_call" { + calls[jsonString(fields, "call_id")] = fields + } else if kind == "function_call_output" || kind == "custom_tool_call_output" { + originalResults[jsonString(fields, "call_id")] = raw + } + } + buckets := make(map[string][]json.RawMessage) + for _, raw := range reduced { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + kind := jsonString(fields, "type") + if kind != "function_call_output" && kind != "custom_tool_call_output" { + continue + } + call := calls[jsonString(fields, "call_id")] + operation, known := compactionOperationCall(call) + reason := "protected or recent" + if !known { + reason = "unsupported carrier" + } else if _, ok := compactionRetiredOutput(fields["output"], operation); !ok { + reason = "unknown or unsuccessful completion" + } + if string(raw) != string(originalResults[jsonString(fields, "call_id")]) { + reason = "evidence reduced, native" + } + buckets[reason] = append(buckets[reason], raw) + } + for _, reason := range []string{"protected or recent", "unsupported carrier", "unknown or unsuccessful completion", "evidence reduced, native"} { + t.Logf("retained result classification: %s (%d items)", reason, len(buckets[reason])) + logCompactionTokenProfile(t, nil, buckets[reason]) + } + retainedTokens := logCompactionTokenProfile(t, input, reduced) + if requested := os.Getenv("HPATCH_COMPACTION_MAX_TOKENS"); requested != "" { + limit, err := strconv.Atoi(requested) + if err != nil || limit <= 0 { + t.Fatal("HPATCH_COMPACTION_MAX_TOKENS must be a positive integer") + } + if retainedTokens > limit { + t.Errorf("retained visible context exceeds target: %d > %d tokens", retainedTokens, limit) + } + } + t.Logf("changed history items=%d; native history bytes=%d -> %d", changed, len(mustMarshalJSON(input)), len(mustMarshalJSON(reduced))) +} + +// Local diagnostics only: string-token estimates exclude opaque ciphertext and +// do not pretend to measure provider-hidden reasoning or request/tool framing. +func logCompactionTokenProfile(t *testing.T, before, after []json.RawMessage) int { + t.Helper() + codec, err := tokenizer.ForModel(tokenizer.GPT5) + if err != nil { + t.Fatal("compaction profile tokenizer unavailable") + } + type measurement struct { + tokens int + opaqueBytes int + } + profile := func(items []json.RawMessage) map[string]measurement { + result := make(map[string]measurement) + for _, raw := range items { + var fields map[string]any + if json.Unmarshal(raw, &fields) != nil { + t.Fatal("invalid profile item") + } + kind, _ := fields["type"].(string) + label := "other" + switch kind { + case "message": + label = "messages" + case "reasoning": + label = "reasoning" + case "custom_tool_call", "function_call": + label = "calls" + case "custom_tool_call_output", "function_call_output": + label = "results" + } + m := result[label] + var visit func(any) + visit = func(value any) { + switch value := value.(type) { + case string: + count, err := codec.Count(value) + if err != nil { + t.Fatal("unable to tokenize profile text") + } + m.tokens += count + case []any: + for _, part := range value { + visit(part) + } + case map[string]any: + for key, part := range value { + if key == "encrypted_content" { + if text, ok := part.(string); ok { + m.opaqueBytes += len(text) + } + continue + } + visit(part) + } + } + } + visit(fields) + result[label] = m + } + return result + } + beforeTotal, afterTotal := 0, 0 + left, right := profile(before), profile(after) + for _, category := range []string{"messages", "calls", "results", "reasoning", "other"} { + beforeTotal += left[category].tokens + afterTotal += right[category].tokens + t.Logf("visible string token estimate (%s), %s: %d -> %d; opaque bytes excluded: %d -> %d", + codec.GetName(), category, left[category].tokens, right[category].tokens, left[category].opaqueBytes, right[category].opaqueBytes) + } + t.Logf("total visible string token estimate (%s): %d -> %d", codec.GetName(), beforeTotal, afterTotal) + return afterTotal +} diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go new file mode 100644 index 00000000..ed220a51 --- /dev/null +++ b/internal/router/context_compaction_retirement.go @@ -0,0 +1,1032 @@ +package router + +import ( + "encoding/json" + "fmt" + "maps" + "regexp" + "slices" + "strings" + "sync" + + "github.com/tiktoken-go/tokenizer" +) + +// A recent suffix is a continuity buffer, not evidence that older work is +// irrelevant. The user-approved loss contract applies to finished operations +// within an open task; it does not require semantic task-closure inference. +const compactionRecentOperations = 8 + +type compactionRetirement struct { + call, result int + operation compactionOperation + output json.RawMessage + rows map[string]bool + ranges [][2]string + callRecord, resultRecord json.RawMessage + eligible bool +} + +func retireCompactionOperations(input []json.RawMessage) []json.RawMessage { + fields := make([]map[string]json.RawMessage, len(input)) + calls, results := make(map[string][]int), make(map[string][]int) + var callOrder []int + for index, raw := range input { + _ = json.Unmarshal(raw, &fields[index]) + id := jsonString(fields[index], "call_id") + switch jsonString(fields[index], "type") { + case "function_call", "custom_tool_call": + calls[id] = append(calls[id], index) + callOrder = append(callOrder, index) + case "function_call_output", "custom_tool_call_output": + results[id] = append(results[id], index) + } + } + if len(callOrder) <= compactionRecentOperations { + return input + } + cutoff := callOrder[len(callOrder)-compactionRecentOperations] + plans := make(map[string]*compactionRetirement) + for id, positions := range calls { + if id == "" || len(positions) != 1 || len(results[id]) != 1 || !compactionCallID.MatchString(id) { + continue + } + call, result := positions[0], results[id][0] + if call >= cutoff || result >= cutoff || result <= call { + continue + } + if jsonString(fields[result], "type") != jsonString(fields[call], "type")+"_output" { + continue + } + operation, ok := compactionOperationCall(fields[call]) + if !ok { + continue + } + output, ok := compactionRetiredOutput(fields[result]["output"], operation) + if !ok { + continue + } + plan := &compactionRetirement{call: call, result: result, operation: operation, output: output, eligible: true} + plans[id] = plan + } + metadataReferences := contextCompactionMetadataReferences(input) + // Reasoning and the calls it precedes form an atomic provider-history + // group. Never leave opaque reasoning attached to partially retired calls. + type group struct { + start, end int + ids []string + reasoningRecord json.RawMessage + blocked bool + } + var groups []group + groupAt := make([]int, len(input)) + for index := range groupAt { + groupAt[index] = -1 + } + current := -1 + for index, item := range fields { + kind, role := jsonString(item, "type"), jsonString(item, "role") + boundary := kind == "reasoning" || (kind == "message" && (role == "user" || role == "developer" || role == "system")) + if boundary { + if current >= 0 { + groups[current].end = index + } + current = -1 + } + if kind == "reasoning" { + groups = append(groups, group{start: index, end: len(input)}) + current = len(groups) - 1 + } + groupAt[index] = current + if current < 0 { + continue + } + switch kind { + case "function_call", "custom_tool_call": + groups[current].ids = append(groups[current].ids, jsonString(item, "call_id")) + case "reasoning", "message", "agent_message", "function_call_output", "custom_tool_call_output": + default: + groups[current].blocked = true + } + } + // Retirement must not trade a smaller envelope for more model-visible + // context. Evaluate the same strings the replay metric counts, including + // the reasoning fact that replaces opaque reasoning for a complete group. + measurePlan := func(plan *compactionRetirement) (json.RawMessage, json.RawMessage, int, int, bool) { + callRecord := compactionRetiredCallWithReferences(fields[plan.call], plan.operation, metadataReferences) + resultRecord := compactionRetiredResultWithReferences(fields[plan.result], plan.output, metadataReferences) + beforeTokens, beforeOK := compactionVisibleStringTokens(input[plan.call], input[plan.result]) + afterTokens, afterOK := compactionVisibleStringTokens(callRecord, resultRecord) + return callRecord, resultRecord, + len(input[plan.call]) + len(input[plan.result]) - len(callRecord) - len(resultRecord), + beforeTokens - afterTokens, beforeOK && afterOK + } + measureGroup := func(g *group, cache bool) (int, int, bool) { + bytesSaved, tokensSaved := 0, 0 + for _, id := range g.ids { + plan := plans[id] + if plan == nil || !plan.eligible { + return 0, 0, false + } + callRecord, resultRecord, planBytes, planTokens, ok := measurePlan(plan) + if !ok { + return 0, 0, false + } + if cache { + plan.callRecord, plan.resultRecord = callRecord, resultRecord + } + bytesSaved += planBytes + + tokensSaved += planTokens + } + reasoningRecord := compactionRetiredReasoningWithReferences(fields[g.start], metadataReferences) + if cache { + g.reasoningRecord = reasoningRecord + } + beforeTokens, beforeOK := compactionVisibleStringTokens(input[g.start]) + afterTokens, afterOK := compactionVisibleStringTokens(reasoningRecord) + return bytesSaved + len(input[g.start]) - len(reasoningRecord), + tokensSaved + beforeTokens - afterTokens, beforeOK && afterOK + } + profitable := func(bytesSaved, tokensSaved int, ok bool) bool { + return ok && bytesSaved > 0 && tokensSaved >= 0 + } + for id, plan := range plans { + if groupAt[plan.call] >= 0 { + continue + } + _, _, bytesSaved, tokensSaved, ok := measurePlan(plan) + if !profitable(bytesSaved, tokensSaved, ok) { + delete(plans, id) + } + } + for index := range groups { + g := &groups[index] + if bytesSaved, tokensSaved, ok := measureGroup(g, false); !profitable(bytesSaved, tokensSaved, ok) { + g.blocked = true + } + } + type referenceText struct { + raw json.RawMessage + owner string + rows bool + } + var queue []referenceText + enqueueOriginal := func(id string, plan *compactionRetirement) { + queue = append(queue, referenceText{fields[plan.call]["input"], id, true}, + referenceText{fields[plan.call]["arguments"], id, true}, + referenceText{fields[plan.result]["output"], id, false}) + } + revision := 0 + var pin func(string) + pin = func(id string) { + plan := plans[id] + if plan == nil || !plan.eligible { + return + } + plan.eligible = false + revision++ + enqueueOriginal(id, plan) + if g := groupAt[plan.call]; g >= 0 && !groups[g].blocked { + groups[g].blocked = true + for _, related := range groups[g].ids { + pin(related) + } + } + } + for _, g := range groups { + blocked := g.blocked || g.end > cutoff + for index := g.start; index < g.end; index++ { + kind := jsonString(fields[index], "type") + if kind == "function_call" || kind == "custom_tool_call" || kind == "function_call_output" || kind == "custom_tool_call_output" { + p := plans[jsonString(fields[index], "call_id")] + if p == nil || !p.eligible || p.call < g.start || p.result >= g.end { + blocked = true + } + } + } + if blocked { + for _, id := range g.ids { + pin(id) + } + } + } + // Native response item IDs are aliases for their operation unit. They are + // transport bookkeeping in retired records, but an explicit retained + // reference to one must keep the original operation and reasoning group. + aliasOwners := make(map[string][]string) + addAlias := func(alias string, owners ...string) { + if !compactionCallID.MatchString(alias) { + return + } + for _, owner := range owners { + if owner != "" { + aliasOwners[alias] = append(aliasOwners[alias], owner) + } + } + } + for id, plan := range plans { + addAlias(jsonString(fields[plan.call], "id"), id) + addAlias(jsonString(fields[plan.result], "id"), id) + } + for _, g := range groups { + addAlias(jsonString(fields[g.start], "id"), g.ids...) + } + unitOwners := make(map[string]map[string]bool) + for id := range plans { + unitOwners[id] = map[string]bool{id: true} + } + for _, g := range groups { + for _, owner := range g.ids { + if unitOwners[owner] == nil { + continue + } + for _, related := range g.ids { + unitOwners[owner][related] = true + } + } + } + + // Explicit call references pin their original operation. Verified-row + // references instead travel inside eligible factual completion records. + // Do not infer that a filename mention needs every historical body. + rowOwners := make(map[string][]string) + unsafeRowOwners := make(map[string]bool) + scriptOwners := make(map[string][]string) + for id, plan := range plans { + evidence := fields[plan.result]["output"] + if plan.operation.notice != nil { + // The verified notice is a reference consumer, not a source-row + // producer. Only the execution result can supply its evidence. + var parts []map[string]json.RawMessage + _ = json.Unmarshal(evidence, &parts) + evidence = parts[2]["text"] + } + unsafeEncoding := false + compactionVisitReferenceStrings(evidence, func(text string) { + compactionSourceVisitDecodedReferences(text, func(decoded string) { + for _, row := range compactionRowReference.FindAllString(decoded, -1) { + rowOwners[row] = append(rowOwners[row], id) + } + for _, reference := range compactionScriptReference.FindAllString(decoded, -1) { + scriptOwners[reference] = append(scriptOwners[reference], id) + } + }, &unsafeEncoding) + if unsafeEncoding { + unsafeRowOwners[id] = true + } + }) + if plan.operation.notice != nil { + queue = append(queue, referenceText{mustMarshalJSON(*plan.operation.notice), id, true}) + } + if plan.eligible { + queue = append(queue, referenceText{plan.operation.arguments, id, true}, referenceText{plan.output, id, false}) + } + } + for _, item := range fields { + kind, id := jsonString(item, "type"), jsonString(item, "call_id") + switch kind { + case "function_call", "custom_tool_call": + if p := plans[id]; p == nil || !p.eligible { + // A retained failed/live carrier still contains the decoded + // notice's references, even if execution never emitted it. + if operation, ok := compactionOperationCall(item); ok && operation.notice != nil { + queue = append(queue, referenceText{mustMarshalJSON(*operation.notice), id, true}) + } + queue = append(queue, referenceText{item["input"], id, true}, referenceText{item["arguments"], id, true}) + } + case "function_call_output", "custom_tool_call_output": + if p := plans[id]; p == nil || !p.eligible { + queue = append(queue, referenceText{item["output"], id, false}) + } + default: + queue = append(queue, referenceText{item["content"], "", true}, referenceText{item["summary"], "", true}) + } + } + retainRow := func(id, row string) { + plan := plans[id] + if plan == nil || !plan.eligible { + return + } + if plan.rows == nil { + plan.rows = make(map[string]bool) + } + plan.rows[row] = true + } + retainRange := func(id string, rowRange [2]string) { + plan := plans[id] + if plan == nil || !plan.eligible { + return + } + plan.ranges = append(plan.ranges, rowRange) + } + + seenIDs, seenRows, seenScripts := make(map[string]bool), make(map[string]bool), make(map[string]bool) + nextReference := 0 + visitReference := func(reference referenceText, text string) { + idText := compactionRowReference.ReplaceAllString(text, "") + for _, word := range compactionReferenceWord.FindAllString(idText, -1) { + targets := append([]string{word}, aliasOwners[word]...) + for _, id := range targets { + if !unitOwners[reference.owner][id] && !seenIDs[id] { + seenIDs[id] = true + pin(id) + } + } + } + if !reference.rows { + return + } + for _, script := range compactionScriptReference.FindAllString(text, -1) { + if !seenScripts[script] { + seenScripts[script] = true + for _, id := range scriptOwners[script] { + pin(id) + } + } + } + for _, match := range compactionSourceRangeReference.FindAllStringSubmatch(text, -1) { + rowRange := [2]string{match[1], match[2]} + for row, owners := range rowOwners { + if !compactionSourceRowReferenced(row, nil, [][2]string{rowRange}) { + continue + } + for _, id := range owners { + retainRange(id, rowRange) + } + } + for id := range unsafeRowOwners { + retainRange(id, rowRange) + } + } + for _, row := range compactionRowReference.FindAllString(text, -1) { + if !seenRows[row] { + seenRows[row] = true + for _, id := range rowOwners[row] { + retainRow(id, row) + } + } + for id := range unsafeRowOwners { + retainRow(id, row) + } + } + } + drainReferences := func() { + for nextReference < len(queue) { + reference := queue[nextReference] + nextReference++ + unsafeEncoding := false + compactionVisitReferenceStrings(reference.raw, func(text string) { + compactionSourceVisitDecodedReferences(text, func(decoded string) { + visitReference(reference, decoded) + }, &unsafeEncoding) + }) + if unsafeEncoding { + // An unsupported colon or slash encoding can conceal any call, + // row, range, or script identity. Keep every other candidate + // rather than guess which evidence the retained text names. + for id := range plans { + if !unitOwners[reference.owner][id] { + pin(id) + } + } + } + } + } + + // Pinning is monotonic. Repeat reference closure only when profitability + // restores original content, which can expose references that were absent + // from the proposed factual record. At most one pass per retired candidate + // can restore content, so this reaches a bounded stable result. + for { + drainReferences() + for _, plan := range plans { + if !plan.eligible || (len(plan.rows) == 0 && len(plan.ranges) == 0) { + continue + } + retained, ok := compactionRetiredOutputKeepingRows( + fields[plan.result]["output"], plan.operation, plan.rows, plan.ranges) + if !ok { + return input + } + plan.output = retained + } + + beforeProfitability := revision + for id, plan := range plans { + if !plan.eligible || groupAt[plan.call] >= 0 { + continue + } + callRecord, resultRecord, bytesSaved, tokensSaved, ok := measurePlan(plan) + if !profitable(bytesSaved, tokensSaved, ok) { + pin(id) + continue + } + plan.callRecord, plan.resultRecord = callRecord, resultRecord + } + for index := range groups { + g := &groups[index] + if g.blocked || g.end > cutoff { + continue + } + bytesSaved, tokensSaved, ok := measureGroup(g, true) + if !profitable(bytesSaved, tokensSaved, ok) { + g.blocked = true + for _, id := range g.ids { + pin(id) + } + } + } + if revision == beforeProfitability { + break + } + } + + output := slices.Clone(input) + for _, plan := range plans { + if plan.eligible { + output[plan.call] = plan.callRecord + output[plan.result] = plan.resultRecord + } + } + for _, g := range groups { + if !g.blocked && g.end <= cutoff && len(g.ids) > 0 { + output[g.start] = g.reasoningRecord + } + } + return output +} + +func compactionVisitReferenceStrings(raw json.RawMessage, visit func(string)) { + var root any + if len(raw) == 0 || json.Unmarshal(raw, &root) != nil { + return + } + + queue := []any{root} + decoded := make(map[string]bool) + for len(queue) > 0 { + value := queue[0] + queue = queue[1:] + switch value := value.(type) { + case string: + visit(value) + if decoded[value] { + continue + } + decoded[value] = true + var nested any + if json.Unmarshal([]byte(value), &nested) == nil { + queue = append(queue, nested) + } + case []any: + queue = append(queue, value...) + case map[string]any: + for _, nested := range value { + queue = append(queue, nested) + } + } + } +} + +var ( + compactionCallID = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + compactionReferenceWord = regexp.MustCompile(`[A-Za-z0-9_-]+`) + compactionScriptReference = regexp.MustCompile(`@shell/[A-Za-z0-9_./:-]+`) + compactionRowReference = regexp.MustCompile(`\b[1-9][0-9]*:[0-9a-f]{4}\b`) + compactionDiagnostic = regexp.MustCompile(`(?i)\b(error|fail|failed|failure|warning|warn|skipped|skip|timeout|panic|fatal|denied|coverage|cancelled|canceled)\b|timed out|no matches|not found`) + compactionTestOutcome = regexp.MustCompile(`^(ok[ \t]|PASS$|FAIL|[?][ \t]|Go test:|Tests:|Test Files:|Test Suites:|Ran [0-9]+ tests|[0-9]+ (passed|failed|skipped))`) + compactionPythonTraceback = regexp.MustCompile(`^Traceback \(most recent call last\):[ \t]*\r?\n?$`) + compactionNativeFailedResult = regexp.MustCompile(`(?s)\A((?:Chunk ID: [^\r\n]+\n)?Wall time: [0-9]+(?:\.[0-9]+)? seconds\nProcess exited with code (?:-[0-9]+|[1-9][0-9]*)\n(?:Original token count: [0-9]+\n)?(?:Output|Final output):\n)(.*)\z`) +) + +func compactionRetiredText(text string) string { + return compactionRetiredTextKeepingRows(text, nil, nil) +} + +// compactionFailedOutput accepts only terminal nonzero shell results. A live +// handle or unknown completion shape is not evidence that an operation ended. +func compactionFailedOutput(raw json.RawMessage) (func(string) json.RawMessage, string, bool) { + rewriteEnvelope := func(rawEnvelope json.RawMessage) (map[string]json.RawMessage, string, bool) { + var serialized string + if json.Unmarshal(rawEnvelope, &serialized) != nil { + return nil, "", false + } + var result map[string]json.RawMessage + if json.Unmarshal([]byte(serialized), &result) != nil || result == nil { + return nil, "", false + } + var exitCode *int + var output string + if json.Unmarshal(result["exit_code"], &exitCode) != nil || exitCode == nil || *exitCode == 0 || + json.Unmarshal(result["output"], &output) != nil { + return nil, "", false + } + for _, key := range []string{"session_id", "cell_id"} { + if value, exists := result[key]; exists && string(value) != "null" { + return nil, "", false + } + } + return result, output, true + } + + if result, output, ok := rewriteEnvelope(raw); ok { + return func(text string) json.RawMessage { + copyResult := maps.Clone(result) + copyResult["output"] = mustMarshalJSON(text) + return mustMarshalJSON(string(mustMarshalJSON(copyResult))) + }, output, true + } + + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) == nil && (len(parts) == 2 || len(parts) == 3) { + header := jsonString(parts[0], "text") + if jsonString(parts[0], "type") != "input_text" || + (!strings.HasPrefix(header, "Script completed\n") && !strings.HasPrefix(header, "Script failed")) { + return nil, "", false + } + for _, part := range parts { + if jsonString(part, "type") != "input_text" { + return nil, "", false + } + } + if result, output, ok := rewriteEnvelope(parts[len(parts)-1]["text"]); ok { + return func(text string) json.RawMessage { + copyResult := maps.Clone(result) + copyResult["output"] = mustMarshalJSON(text) + copyParts := slices.Clone(parts) + copyParts[len(copyParts)-1] = maps.Clone(copyParts[len(copyParts)-1]) + copyParts[len(copyParts)-1]["text"] = mustMarshalJSON(string(mustMarshalJSON(copyResult))) + return mustMarshalJSON(copyParts) + }, output, true + } + } + + var native string + if json.Unmarshal(raw, &native) == nil { + if match := compactionNativeFailedResult.FindStringSubmatch(native); match != nil { + return func(text string) json.RawMessage { return mustMarshalJSON(match[1] + text) }, match[2], true + } + } + return nil, "", false +} + +// Failed operations retain every unclassified line. Only known routine test +// progress/pass lines and unreferenced verified source rows are positively +// identified as historical bulk and eligible for omission. +func compactionRetiredFailedText(text string, referenced map[string]bool, ranges [][2]string) string { + var result strings.Builder + removed := 0 + for line := range strings.SplitAfterSeq(text, "\n") { + trimmed := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") + if contextCompactionGoRoutine.MatchString(trimmed) || + (compactionCompleteSourceRow.MatchString(line) && !compactionTextReferencesRows(line, referenced, ranges)) { + removed++ + continue + } + result.WriteString(line) + } + if removed == 0 { + return text + } + return fmt.Sprintf("[hpatch: omitted %d positively identified routine/source lines from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) +} + +func compactionRetiredTextKeepingRows(text string, referenced map[string]bool, ranges [][2]string) string { + lines := strings.SplitAfter(text, "\n") + keep := make([]bool, len(lines)) + for index, line := range lines { + sourceRow := compactionCompleteSourceRow.MatchString(line) + if compactionTextReferencesRows(line, referenced, ranges) { + keep[index] = true + } + if sourceRow { + continue + } + if compactionDiagnostic.MatchString(line) { + for nearby := max(0, index-2); nearby < min(len(lines), index+3); nearby++ { + keep[nearby] = true + } + } + if compactionTestOutcome.MatchString(line) { + keep[index] = true + } + } + // Python exception messages, notes, and chains have no reliable generic + // end marker. Keep the suffix rather than dropping an actionable detail. + for start, line := range lines { + if !compactionPythonTraceback.MatchString(line) { + continue + } + for index := start; index < len(lines); index++ { + keep[index] = true + } + break + } + + var result strings.Builder + fmt.Fprintf(&result, "[hpatch: output details omitted; unavailable; original bytes=%d]\n", len(text)) + for index, line := range lines { + if keep[index] { + result.WriteString(line) + } + } + return result.String() +} + +func compactionTextReferencesRows(text string, referenced map[string]bool, ranges [][2]string) bool { + if len(referenced) == 0 && len(ranges) == 0 { + return false + } + found, unsafeEncoding := false, false + compactionVisitReferenceStrings(mustMarshalJSON(text), func(value string) { + compactionSourceVisitDecodedReferences(value, func(decoded string) { + for _, row := range compactionRowReference.FindAllString(decoded, -1) { + if compactionSourceRowReferenced(row, referenced, ranges) { + found = true + } + } + }, &unsafeEncoding) + }) + return found || unsafeEncoding +} + +func compactionRetiredOutput(raw json.RawMessage, operation compactionOperation) (json.RawMessage, bool) { + return compactionRetiredOutputKeepingRows(raw, operation, nil, nil) +} + +func compactionRetiredOutputKeepingRows(raw json.RawMessage, operation compactionOperation, referenced map[string]bool, ranges [][2]string) (json.RawMessage, bool) { + if compactionReadTool(operation.tool) { + reduced, ok := compactionRetiredReadToolOutput(raw, operation.tool) + if ok && (len(referenced) > 0 || len(ranges) > 0) { + return raw, true + } + return reduced, ok + } + + if operation.patchReport == "" && operation.notice == nil { + if encode, text, ok := contextCompactionOutput(raw); ok { + return encode(compactionRetiredTextKeepingRows(text, referenced, ranges)), true + } + if encode, text, ok := compactionFailedOutput(raw); ok { + return encode(compactionRetiredFailedText(text, referenced, ranges)), true + } + } + var parts []map[string]json.RawMessage + resultIndex := 1 + if operation.notice != nil { + resultIndex = 2 + } + if json.Unmarshal(raw, &parts) != nil || len(parts) != resultIndex+1 || + jsonString(parts[0], "type") != "input_text" || !strings.HasPrefix(jsonString(parts[0], "text"), "Script completed\n") || + jsonString(parts[resultIndex], "type") != "input_text" { + return nil, false + } + if operation.notice != nil && (jsonString(parts[1], "type") != "input_text" || jsonString(parts[1], "text") != *operation.notice) { + return nil, false + } + if operation.patchReport != "" { + // The generated report is emitted only after awaited application. + if jsonString(parts[1], "text") != operation.patchReport { + return nil, false + } + parts[1] = maps.Clone(parts[1]) + parts[1]["text"] = mustMarshalJSON(compactionRetiredPatchReport(operation.patchReport, referenced, ranges)) + return mustMarshalJSON(parts), true + } + encode, text, ok := contextCompactionOutput(parts[resultIndex]["text"]) + if !ok { + return nil, false + } + parts[resultIndex]["text"] = encode(compactionRetiredTextKeepingRows(text, referenced, ranges)) + return mustMarshalJSON(parts), true +} + +func compactionRetiredPatchReport(text string, referenced map[string]bool, ranges [][2]string) string { + var result strings.Builder + removed := 0 + for line := range strings.SplitAfterSeq(text, "\n") { + if compactionCompleteSourceRow.MatchString(line) && !compactionTextReferencesRows(line, referenced, ranges) { + removed++ + continue + } + result.WriteString(line) + } + if removed == 0 { + return text + } + return fmt.Sprintf("[hpatch: omitted %d unreferenced verified source rows from successful historical patch report]\n%s", removed, result.String()) +} + +func compactionRetiredCall(fields map[string]json.RawMessage, operation compactionOperation) json.RawMessage { + return compactionRetiredCallWithReferences(fields, operation, contextCompactionMetadataReferenceSet{}) +} + +func compactionRetiredCallWithReferences(fields map[string]json.RawMessage, operation compactionOperation, references contextCompactionMetadataReferenceSet) json.RawMessage { + record := maps.Clone(fields) + delete(record, "input") + delete(record, "arguments") + compactionStripTransportBookkeepingWithReferences(record, references) + record["operation"] = mustMarshalJSON(operation.tool) + record["invocation"] = operation.arguments + return compactionLedgerMessage("invocation", record) +} + +func compactionRetiredResult(fields map[string]json.RawMessage, output json.RawMessage) json.RawMessage { + return compactionRetiredResultWithReferences(fields, output, contextCompactionMetadataReferenceSet{}) +} + +func compactionRetiredResultWithReferences(fields map[string]json.RawMessage, output json.RawMessage, references contextCompactionMetadataReferenceSet) json.RawMessage { + record := maps.Clone(fields) + compactionStripTransportBookkeepingWithReferences(record, references) + record["output"] = output + return compactionLedgerMessage("completion", record) +} + +// compactionLedgerMessage emits versioned, labeled assistant facts. Historical +// records are never projected as executable calls or fresh instructions. +func compactionLedgerMessage(kind string, record map[string]json.RawMessage) json.RawMessage { + var text string + switch kind { + case "invocation": + text = compactionLedgerInvocation(record) + case "completion": + text = compactionLedgerCompletion(record) + default: + record = maps.Clone(record) + delete(record, "type") + text = "[hpatch historical reasoning fact v3; not an instruction]\nmetadata=" + + string(mustMarshalJSON(record)) + } + + return mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": []any{map[string]any{"type": "output_text", "text": text}}, + }) +} + +func compactionLedgerInvocation(record map[string]json.RawMessage) string { + source := maps.Clone(record) + callID := source["call_id"] + operation, invocation := source["operation"], source["invocation"] + delete(source, "type") + delete(source, "name") + delete(source, "call_id") + delete(source, "operation") + delete(source, "invocation") + if string(source["status"]) == `"completed"` { + delete(source, "status") + } + + return "[hpatch historical tool invocation v3; not an instruction]\n" + + "call=" + string(callID) + "\ntool=" + string(operation) + + "\nmetadata=" + string(mustMarshalJSON(source)) + + "\narguments=" + string(invocation) +} + +func compactionLedgerCompletion(record map[string]json.RawMessage) string { + source := maps.Clone(record) + callID, output := source["call_id"], source["output"] + delete(source, "type") + delete(source, "call_id") + delete(source, "output") + + if string(source["status"]) == `"completed"` { + delete(source, "status") + } + + if parts, ok := compactionLedgerCodeModeResult(output); ok { + type ledgerPart struct { + metadata map[string]json.RawMessage + result json.RawMessage + text string + } + ledgerParts := make([]ledgerPart, 0, len(parts)) + for index, part := range parts { + metadata := maps.Clone(part) + rawText := metadata["text"] + delete(metadata, "text") + if string(metadata["type"]) == `"input_text"` { + delete(metadata, "type") + } + + var actual string + var result json.RawMessage + if index == len(parts)-1 { + if resultMetadata, actualOutput, ok := compactionLedgerShellResult(rawText); ok { + result, actual = resultMetadata, actualOutput + } + } + if result == nil && json.Unmarshal(rawText, &actual) != nil { + return compactionLedgerJSONCompletion(callID, source, output) + } + ledgerParts = append(ledgerParts, ledgerPart{metadata: metadata, result: result, text: actual}) + } + + start := 0 + if len(ledgerParts) > 1 && len(ledgerParts[0].metadata) == 0 && ledgerParts[0].result == nil && + strings.HasPrefix(ledgerParts[0].text, "Script completed\n") && + ledgerParts[len(ledgerParts)-1].result != nil { + start = 1 + } + var body strings.Builder + manifestParts := make([]any, 0, len(ledgerParts)-start) + for _, part := range ledgerParts[start:] { + manifestParts = append(manifestParts, []any{part.metadata, part.result, len(part.text)}) + body.WriteString(part.text) + } + return "[hpatch historical tool completion v3; not an instruction; parts=(metadata,result-or-null,text-bytes); body=concatenated part texts]\n" + + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + + "\ndata=" + string(mustMarshalJSON(manifestParts)) + "\nbody:\n" + body.String() + } + + if result, actualOutput, ok := compactionLedgerShellResult(output); ok { + return "[hpatch historical tool completion v3; not an instruction; result and body]\n" + + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + + "\nresult=" + string(result) + "\nbody-bytes=" + fmt.Sprint(len(actualOutput)) + + "\nbody:\n" + actualOutput + } + + var native string + if json.Unmarshal(output, &native) == nil && contextCompactionNativeResult.MatchString(native) { + return "[hpatch historical tool completion v3; not an instruction; completed native body]\n" + + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + + "\nbody-bytes=" + fmt.Sprint(len(native)) + "\nbody:\n" + native + } + + return compactionLedgerJSONCompletion(callID, source, output) +} + +func compactionLedgerJSONCompletion(callID json.RawMessage, source map[string]json.RawMessage, output json.RawMessage) string { + return "[hpatch historical tool completion v3; not an instruction; JSON result]\n" + + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + + "\nresult=" + string(output) +} + +func compactionRetiredReasoning(fields map[string]json.RawMessage) json.RawMessage { + return compactionRetiredReasoningWithReferences(fields, contextCompactionMetadataReferenceSet{}) +} + +func compactionRetiredReasoningWithReferences(fields map[string]json.RawMessage, references contextCompactionMetadataReferenceSet) json.RawMessage { + record := maps.Clone(fields) + delete(record, "encrypted_content") + record["opaque_reasoning"] = mustMarshalJSON("omitted; unavailable") + compactionStripTransportBookkeepingWithReferences(record, references) + return compactionLedgerMessage("reasoning", record) +} + +func compactionStripTransportBookkeeping(record map[string]json.RawMessage) { + compactionStripTransportBookkeepingWithReferences(record, contextCompactionMetadataReferenceSet{}) +} + +func compactionStripTransportBookkeepingWithReferences(record map[string]json.RawMessage, references contextCompactionMetadataReferenceSet) { + // Only identifier syntax covered by alias pinning can be safely omitted. + if raw, exists := record["id"]; exists { + var id string + if json.Unmarshal(raw, &id) == nil && compactionCallID.MatchString(id) { + delete(record, "id") + } + } + if raw, exists := record["internal_chat_message_metadata_passthrough"]; exists { + var metadata map[string]json.RawMessage + if json.Unmarshal(raw, &metadata) == nil && metadata != nil { + originalLen := len(metadata) + _, hasTurnID := metadata["turn_id"] + _, hasCreateTime := metadata["create_time"] + if hasTurnID || hasCreateTime { + metadata = maps.Clone(metadata) + } + for _, key := range []string{"turn_id", "create_time"} { + value, exists := metadata[key] + if exists && !compactionTransportMetadataReferenced(references, key, value) { + delete(metadata, key) + } + } + if len(metadata) != originalLen { + if len(metadata) == 0 { + delete(record, "internal_chat_message_metadata_passthrough") + } else { + record["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(metadata) + } + } + } + } + for _, key := range []string{"turn_id", "create_time"} { + value, exists := record[key] + if exists && !compactionTransportMetadataReferenced(references, key, value) { + delete(record, key) + } + } +} + +func compactionTransportMetadataReferenced(references contextCompactionMetadataReferenceSet, key string, raw json.RawMessage) bool { + reference, ok := contextCompactionMetadataReference(key, raw) + if !ok || references.unsafe { + return true + } + if slices.ContainsFunc(references.text, func(text string) bool { + return strings.Contains(text, reference) + }) { + return true + } + return key == "create_time" && slices.Contains(references.numbers, reference) +} + +func compactionLedgerShellResult(raw json.RawMessage) (json.RawMessage, string, bool) { + var serialized string + if json.Unmarshal(raw, &serialized) != nil { + return nil, "", false + } + + var result map[string]json.RawMessage + if json.Unmarshal([]byte(serialized), &result) != nil || result == nil { + return nil, "", false + } + + var exitCode *int + var output string + if json.Unmarshal(result["exit_code"], &exitCode) != nil || exitCode == nil || + json.Unmarshal(result["output"], &output) != nil { + return nil, "", false + } + for _, key := range []string{"session_id", "cell_id"} { + if value, exists := result[key]; exists && string(value) != "null" { + return nil, "", false + } + } + + metadata := maps.Clone(result) + delete(metadata, "output") + return mustMarshalJSON(metadata), output, true +} + +func compactionLedgerCodeModeResult(raw json.RawMessage) ([]map[string]json.RawMessage, bool) { + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) != nil || (len(parts) != 2 && len(parts) != 3) { + return nil, false + } + for index, part := range parts { + if jsonString(part, "type") != "input_text" { + return nil, false + } + var text string + if json.Unmarshal(part["text"], &text) != nil { + return nil, false + } + if index == 0 && !strings.HasPrefix(text, "Script completed\n") { + return nil, false + } + } + return parts, true +} + +var ( + compactionRetirementTokenOnce sync.Once + compactionRetirementTokenCodec tokenizer.Codec + compactionRetirementTokenCodecErr error +) + +func compactionVisibleStringTokens(items ...json.RawMessage) (int, bool) { + compactionRetirementTokenOnce.Do(func() { + compactionRetirementTokenCodec, compactionRetirementTokenCodecErr = tokenizer.ForModel(tokenizer.GPT5) + }) + if compactionRetirementTokenCodecErr != nil { + + return 0, false + } + total := 0 + for _, raw := range items { + var value any + if json.Unmarshal(raw, &value) != nil { + return 0, false + } + var visit func(any) bool + visit = func(current any) bool { + switch current := current.(type) { + case string: + count, err := compactionRetirementTokenCodec.Count(current) + if err != nil { + return false + } + total += count + case []any: + for _, nested := range current { + if !visit(nested) { + return false + } + } + case map[string]any: + for key, nested := range current { + if key != "encrypted_content" && !visit(nested) { + return false + } + } + } + return true + } + if !visit(value) { + return 0, false + } + } + return total, true +} diff --git a/internal/router/context_compaction_retirement_test.go b/internal/router/context_compaction_retirement_test.go new file mode 100644 index 00000000..40ac8573 --- /dev/null +++ b/internal/router/context_compaction_retirement_test.go @@ -0,0 +1,574 @@ +package router + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func retirementHistory() []json.RawMessage { + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Do not commit. Keep the original API. This task is still open."}), + } + for index := range 24 { + id := fmt.Sprintf("operation_%02d", index) + items = append(items, + mustMarshalJSON(map[string]any{"type": "reasoning", "id": fmt.Sprintf("rs_%02d", index), "summary": []any{map[string]any{"type": "summary_text", "text": "Preserve the API contract."}}, "encrypted_content": "opaque-state"}), + compactTestCall(id, "hread internal/router/example.go"), + compactTestOutput(id, strings.Repeat(fmt.Sprintf("historical-body-%02d\n", index), 300), 0)) + } + return items +} + +func TestCompactionRetiresFinishedOperationsInOpenTask(t *testing.T) { + items := retirementHistory() + before := string(mustMarshalJSON(items)) + got := retireCompactionOperations(items) + if string(got[0]) != string(items[0]) { + t.Fatal("user authority changed") + } + wire := string(mustMarshalJSON(got)) + if strings.Contains(wire, "historical-body-00") || !strings.Contains(wire, "historical-body-17") { + t.Fatal("old completed output was not retired, or recent output was lost") + } + if !strings.Contains(wire, "hread internal/router/example.go") || !strings.Contains(wire, "operation_00") || !strings.Contains(wire, "Preserve the API contract.") { + t.Fatal("execution facts or visible reasoning were lost") + } + if len(wire) >= len(before)/2 { + t.Fatal("finished-operation retirement did not provide material relief") + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("input was mutated") + } + if string(mustMarshalJSON(retireCompactionOperations(got))) != wire { + t.Fatal("repeated retirement changed the retained ledger") + } +} + +func TestCompactionRetirementStaticCodeModeCarriers(t *testing.T) { + arguments := `{"cmd":"hread example.go","login":false}` + good := `const result = await tools.exec_command(` + arguments + `); text(JSON.stringify(Object.assign({}, result, {"retained":true,"script_ref":"@shell/history"})));` + for _, source := range []string{good, `text(await tools.exec_command(` + arguments + `));`} { + operation, ok := compactionCodeModeOperation(source) + if !ok || operation.tool != "exec_command" || string(operation.arguments) != arguments { + t.Fatal("static result-preserving carrier was not recognized") + } + items := retirementHistory() + items[2] = mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "operation_00", "input": source}) + items[3] = compactCodeModeOutput("operation_00", strings.Repeat("unmarked old read detail\n", 500)) + got := retireCompactionOperations(items) + if strings.Contains(string(got[3]), "unmarked old read detail") { + t.Fatal("finished Code Mode output was not retired") + } + } + for _, source := range []string{ + strings.Replace(good, `"retained":true`, `"exit_code":0`, 1), + `const result = await tools.exec_command(buildArguments()); text(result);`, + `const result = await tools.exec_command(` + arguments + `); text({"exit_code":0,"output":"fake"});`, + `const result = await tools.exec_command(` + arguments + `); text(result); await tools.other();`, + `if (false) { text(await tools.exec_command(` + arguments + `)); }`, + } { + if _, ok := compactionCodeModeOperation(source); ok { + t.Fatal("dynamic or status-forging carrier was accepted") + } + } +} + +func TestCompactionRetirementPreservesCarrierNotice(t *testing.T) { + notice := "Keep the evidence at 17:abcd for the next edit." + source := `const result = await tools.exec_command({"cmd":"hread example.go"}); text(` + + string(mustMarshalJSON(notice)) + `); text(JSON.stringify(result));` + items := retirementHistory() + items[2] = mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "operation_00", "input": source}) + items[3] = mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": notice}, + map[string]any{"type": "input_text", "text": string(mustMarshalJSON(map[string]any{ + "exit_code": 0, "output": strings.Repeat("old unmarked detail\n", 500), + }))}, + }, + }) + items[6] = compactTestOutput("operation_01", "17:abcd referenced source\n"+strings.Repeat("needed evidence\n", 300), 0) + got := retireCompactionOperations(items) + if string(got[3]) == string(items[3]) || strings.Contains(string(got[3]), "old unmarked detail") { + t.Fatal("static carrier with a literal notice was not retired") + } + wire := string(mustMarshalJSON(got)) + if !strings.Contains(string(got[3]), notice) || !strings.Contains(wire, "17:abcd referenced source") || + strings.Contains(wire, "needed evidence") { + t.Fatal("carrier notice or row-aware factual evidence was lost") + } + mismatch := append([]json.RawMessage(nil), items...) + mismatch[3] = json.RawMessage(strings.Replace(string(items[3]), notice, "different notice", 1)) + got = retireCompactionOperations(mismatch) + if string(got[2]) != string(mismatch[2]) || string(got[3]) != string(mismatch[3]) { + t.Fatal("mismatched notice/result projection was retired") + } + failed := append([]json.RawMessage(nil), items...) + escapedSource := strings.Replace(source, "17:abcd", `17\u003aabcd`, 1) + failed[2] = mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "operation_00", "input": escapedSource}) + failed[3] = mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", "output": "Script failed", + }) + got = retireCompactionOperations(failed) + wire = string(mustMarshalJSON(got)) + if string(got[2]) != string(failed[2]) || string(got[3]) != string(failed[3]) || + !strings.Contains(wire, "17:abcd referenced source") { + t.Fatal("an unfinished carrier or its row-aware factual evidence was lost") + } +} + +func TestCompactionRetiresAppliedPatchBodyNotFailedPatch(t *testing.T) { + patch := "*** Begin Patch\n*** Add File: example.go\n+" + strings.Repeat("// old implementation detail\n+", 500) + "\n*** End Patch\n" + report := "in example.go\nfiles add=1 update=0 move=0 delete=0\n" + source := hpatchApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" + result := mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", + "output": []any{map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": report}}, + }) + items := retirementHistory() + items[2] = mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "operation_00", "input": source}) + items[3] = result + got := retireCompactionOperations(items) + if strings.Contains(string(got[2]), "old implementation detail") || !strings.Contains(string(got[2]), "example.go") || !strings.Contains(string(got[3]), "files add=1") { + t.Fatal("applied edit was not replaced with truthful target/outcome records") + } + items[3] = json.RawMessage(strings.Replace(string(result), "Script completed", "Script failed", 1)) + got = retireCompactionOperations(items) + if string(got[2]) != string(items[2]) || string(got[3]) != string(items[3]) { + t.Fatal("failed patch carrier was retired") + } +} + +func TestCompactionRetiredPatchReportKeepsApplicationFactsAndReferencedRows(t *testing.T) { + var rows strings.Builder + for row := 1; row <= 40; row++ { + fmt.Fprintf(&rows, "%d:%04x source declaration with error word that is not a runtime diagnostic %s\n", + row, row, strings.Repeat("detail ", 20)) + } + report := "in internal/router/example.go\nfiles add=0 update=1 move=0 delete=0\n" + rows.String() + + "Done!\nWARNING: retained application qualification\n" + patch := "*** Begin Patch\n*** Update File: internal/router/example.go\n@@\n-old\n+new\n*** End Patch\n" + source := hpatchApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" + result := mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "operation_00", + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": report}, + }, + }) + items := retirementHistory() + items[2] = mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "operation_00", "input": source}) + items[3] = result + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep exact patch evidence row 17:0011.", + })) + + got := retireCompactionOperations(items) + wire := string(mustMarshalJSON(got[1:4])) + for _, fact := range []string{ + "internal/router/example.go", "Update File", "files add=0 update=1 move=0 delete=0", + "Done!", "WARNING: retained application qualification", "17:0011", + } { + if !strings.Contains(wire, fact) { + t.Fatalf("patch retirement lost application fact %q", fact) + } + } + if strings.Contains(wire, "18:0012") || strings.Count(wire, "error word") != 1 { + t.Fatal("unreferenced verified source rows survived because their source text resembled diagnostics") + } +} + +func TestCompactionRetirementMeasuresCompleteGroup(t *testing.T) { + items := retirementHistory() + // A cheap successful companion must not pin the much larger completed read. + items = append(items[:4], append([]json.RawMessage{ + compactTestCall("small_companion", "pwd"), compactTestOutput("small_companion", "/workspace\n", 0), + }, items[4:]...)...) + got := retireCompactionOperations(items) + if strings.Contains(string(got[3]), "historical-body-00") { + t.Fatal("a small completed companion pinned the entire eligible group") + } + for _, index := range []int{1, 2, 3, 4, 5} { + if string(got[index]) == string(items[index]) { + t.Fatalf("eligible reasoning/tool group was only partially retired at %d", index) + } + } + if !strings.Contains(string(got[4]), "pwd") || !strings.Contains(string(got[5]), "exit_code") { + t.Fatal("small companion lost its invocation or observed completion") + } +} + +func TestCompactionRetirementRetiresCompletedFailedReasoningGroups(t *testing.T) { + items := retirementHistory() + // One reasoning response issued a successful and a terminal failed call. + items = append(items[:4], append([]json.RawMessage{ + compactTestCall("group_failure", "go test ./..."), compactTestOutput("group_failure", "unresolved failure", 1), + }, items[4:]...)...) + got := retireCompactionOperations(items) + for index := 1; index <= 5; index++ { + if string(got[index]) == string(items[index]) { + t.Fatal("complete reasoning/tool group with terminal failure stayed native") + } + } + if wire := string(mustMarshalJSON(got[1:6])); !strings.Contains(wire, "group_failure") || + !strings.Contains(wire, "unresolved failure") || !strings.Contains(wire, `\"exit_code\":1`) { + t.Fatal("failed operation facts were lost from retired group") + } +} +func TestCompactionRetirementPinsFailureLiveAndReferencedEvidence(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", + strings.Repeat("=== RUN TestHistorical\n--- PASS: TestHistorical (0.1s)\n", 200)+"unique unresolved failure\n", 1) + items[6] = mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "operation_01", + "output": string(mustMarshalJSON(map[string]any{"exit_code": 0, "session_id": 42, "output": "still running"}))}) + items = append(items, mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "The evidence in operation_02 is needed for the remaining investigation."})) + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3} { + if string(got[index]) == string(items[index]) { + t.Fatalf("terminal failed operation item %d stayed native", index) + } + } + if wire := string(mustMarshalJSON(got[1:4])); !strings.Contains(wire, "unique unresolved failure") || + !strings.Contains(wire, `\"exit_code\":1`) { + t.Fatal("terminal failure facts disappeared") + } + for _, index := range []int{4, 5, 6, 7, 8, 9} { + if string(got[index]) != string(items[index]) { + t.Fatalf("protected operation/associated reasoning at %d changed", index) + } + } + if !strings.Contains(string(mustMarshalJSON(got)), "historical-body-02") { + t.Fatal("explicitly referenced evidence disappeared") + } +} + +func TestCompactionRetirementPreservesDiagnosticsAndUnknownLifecycle(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", strings.Repeat("routine output\n", 300)+"WARNING: coverage excludes integration tests\nok example/router 0.2s\n", 0) + items[6] = mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "operation_01", "output": "unknown completion format"}) + got := retireCompactionOperations(items) + wire := string(mustMarshalJSON(got)) + if !strings.Contains(wire, "WARNING: coverage excludes integration tests") || !strings.Contains(wire, "ok example/router 0.2s") { + t.Fatal("diagnostic or validation scope was lost") + } + if string(got[5]) != string(items[5]) || string(got[6]) != string(items[6]) { + t.Fatal("unknown completion state was retired") + } +} + +func TestCompactionRetirementPreservesAmbiguousCalls(t *testing.T) { + items := retirementHistory() + items = append([]json.RawMessage{compactTestCall("operation_00", "pwd")}, items...) + got := retireCompactionOperations(items) + for _, index := range []int{0, 2, 3, 4} { + if string(got[index]) != string(items[index]) { + t.Fatalf("ambiguous operation at %d changed", index) + } + } +} + +func TestCompactionRetirementFollowsOnlySurvivingReplacementNotes(t *testing.T) { + note := func(target string) string { + return fmt.Sprintf("[hpatch compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", target) + } + assertNative := func(t *testing.T, got, want []json.RawMessage) { + t.Helper() + for _, index := range []int{1, 2, 3, 4, 5, 6} { + if string(got[index]) != string(want[index]) { + t.Fatalf("surviving replacement-note closure lost native item %d", index) + } + } + } + + t.Run("generated repeated excerpt note retires with its consumer", func(t *testing.T) { + items := retirementHistory() + var excerpt strings.Builder + for row := 1; row <= 12; row++ { + fmt.Fprintf(&excerpt, "%d:abcd source declaration with enough exact text to retire the earlier repeated excerpt\n", row) + } + items[3] = compactTestOutput("operation_00", + "older read header\n"+excerpt.String()+"unique older evidence\n"+strings.Repeat("consumer historical detail\n", 500), 0) + items[6] = compactTestOutput("operation_01", excerpt.String(), 0) + + got := reduceContextCompaction(items) + wire := string(mustMarshalJSON(got)) + if !strings.Contains(wire, "historical facts v4") || strings.Contains(wire, "retained verbatim in later tool result") { + t.Fatal("generated replacement note survived its retired consumer") + } + if again := reduceContextCompaction(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { + t.Fatal("generated replacement-note retirement was not stable") + } + }) + + t.Run("retired note releases its target", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", + note("operation_01")+strings.Repeat("retired consumer detail\n", 500), 0) + items[6] = compactTestOutput("operation_01", strings.Repeat("later source evidence\n", 500), 0) + before := string(mustMarshalJSON(items)) + + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3, 4, 5, 6} { + if string(got[index]) == string(items[index]) { + t.Fatalf("discarded replacement note pinned native item %d", index) + } + } + if strings.Contains(string(mustMarshalJSON(got)), "retained verbatim in later tool result") { + t.Fatal("retired replacement note survived in factual records") + } + if len(mustMarshalJSON(got)) > len(mustMarshalJSON(items)) { + t.Fatal("replacement-note retirement increased retained history") + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("replacement-note retirement mutated its input") + } + if again := retireCompactionOperations(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { + t.Fatal("replacement-note retirement was not stable") + } + }) + + t.Run("failed consumer keeps its target", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", note("operation_01"), 1) + items[6] = compactTestOutput("operation_01", strings.Repeat("later source evidence\n", 500), 0) + assertNative(t, retireCompactionOperations(items), items) + assertNative(t, reduceContextCompaction(items), items) + }) + + t.Run("consumer pinned later restores its original note", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", note("operation_01")+strings.Repeat("pinned consumer detail\n", 500), 0) + items[6] = compactTestOutput("operation_01", strings.Repeat("later source evidence\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep operation_00.", + })) + assertNative(t, retireCompactionOperations(items), items) + assertNative(t, reduceContextCompaction(items), items) + }) + + t.Run("replacement-note cycle follows a surviving root", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", note("operation_01")+strings.Repeat("cycle detail\n", 500), 0) + items[6] = compactTestOutput("operation_01", note("operation_00")+strings.Repeat("cycle detail\n", 500), 0) + + retired := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3, 4, 5, 6} { + if string(retired[index]) == string(items[index]) { + t.Fatalf("unrooted replacement-note cycle pinned native item %d", index) + } + } + + rooted := append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep operation_00.", + })) + assertNative(t, retireCompactionOperations(rooted), rooted) + assertNative(t, reduceContextCompaction(rooted), rooted) + }) +} + +func TestCompactionRetirementPinsRetainedScriptProducer(t *testing.T) { + items := retirementHistory() + items[3] = mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "operation_00", + "output": string(mustMarshalJSON(map[string]any{"exit_code": 0, "script_ref": "@shell/source-reference", + "output": strings.Repeat("old source\n", 500)}))}) + items = append(items, compactTestCall("pending", "hread @shell/source-reference")) + got := retireCompactionOperations(items) + if string(got[2]) != string(items[2]) || string(got[3]) != string(items[3]) { + t.Fatal("retained-script reference lost its producer") + } +} + +func TestCompactionRetirementPinsDecodedNestedReferences(t *testing.T) { + assertPinned := func(t *testing.T, items []json.RawMessage) { + t.Helper() + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3} { + if string(got[index]) != string(items[index]) { + t.Fatalf("referenced operation evidence at %d changed", index) + } + } + } + + t.Run("call ID with newline and escaped unicode", func(t *testing.T) { + items := retirementHistory() + items = append(items, json.RawMessage(`{"type":"message","role":"assistant","content":"continued evidence:\noperation_\u0030\u0030"}`)) + assertPinned(t, items) + }) + + t.Run("verified row in nested JSON", func(t *testing.T) { + items := retirementHistory() + referencedLine := `{\"result\":{\"row\":\"17\u003aabcd\",\"detail\":\"exact\"}}` + items[3] = compactTestOutput("operation_00", + referencedLine+"\n"+strings.Repeat("old detail\n", 500), 0) + arguments := `{"cmd":"{\"target\":\"17\\u003aabcd\"}","workdir":"/workspace"}` + items = append(items, mustMarshalJSON(map[string]any{ + "type": "function_call", "name": "exec_command", "call_id": "pending_row", "arguments": arguments, + })) + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3} { + if string(got[index]) == string(items[index]) { + t.Fatal("decoded verified-row reference pinned the native reasoning group") + } + } + var completion map[string]json.RawMessage + var content []map[string]json.RawMessage + _ = json.Unmarshal(got[3], &completion) + _ = json.Unmarshal(completion["content"], &content) + completionText := jsonString(content[0], "text") + if !strings.Contains(completionText, referencedLine) { + t.Fatal("decoded verified-row reference was lost from factual completion") + } + if strings.Contains(completionText, "old detail") { + t.Fatal("factual completion retained unreferenced output") + } + }) + + t.Run("percent encoded row remains exact", func(t *testing.T) { + items := retirementHistory() + referencedLine := `{"row":"17%3aabcd","detail":"exact"}` + items[3] = compactTestOutput("operation_00", + referencedLine+"\n"+strings.Repeat("old detail\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Retain 17:abcd.", + })) + got := retireCompactionOperations(items) + for _, index := range []int{1, 2, 3} { + if string(got[index]) == string(items[index]) { + t.Fatal("percent-encoded row pinned the native reasoning group") + } + } + var completion map[string]json.RawMessage + var content []map[string]json.RawMessage + _ = json.Unmarshal(got[3], &completion) + _ = json.Unmarshal(completion["content"], &content) + completionText := jsonString(content[0], "text") + if !strings.Contains(completionText, referencedLine) { + t.Fatal("percent-encoded row was lost from factual completion") + } + if strings.Contains(completionText, "old detail") { + t.Fatal("factual completion retained unreferenced output") + } + }) + + t.Run("retained script in nested JSON", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", + `{"result":{"script_ref":"@shell\u002fsource-reference","detail":"`+strings.Repeat("old detail ", 500)+`"}}`, 0) + arguments := `{"cmd":"{\"script_ref\":\"@shell\\u002fsource-reference\"}","workdir":"/workspace"}` + items = append(items, mustMarshalJSON(map[string]any{ + "type": "function_call", "name": "exec_command", "call_id": "pending_script", "arguments": arguments, + })) + assertPinned(t, items) + }) +} + +func TestCompactionRetirementCarriesReferencedRowsInFactualCompletion(t *testing.T) { + items := retirementHistory() + var body strings.Builder + for row := 1; row <= 20; row++ { + fmt.Fprintf(&body, "%d:%04x source declaration %s\n", row, row, strings.Repeat("detail ", 20)) + } + partial := "context line contains partial row 9:0009 with exact surrounding evidence\n" + body.WriteString(partial) + items[3] = compactTestOutput("operation_00", body.String(), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": "Retain 2:0002, range 5:0005..7:0007, and partial 9:0009.", + })) + before := string(mustMarshalJSON(items)) + + got := reduceContextCompaction(items) + completionText := string(mustMarshalJSON(got)) + if !strings.Contains(completionText, "historical facts v4") { + t.Fatal("row-only evidence reference pinned the native reasoning group") + } + for _, retained := range []string{"2:0002", "5:0005", "6:0006", "7:0007", partial} { + if !strings.Contains(completionText, strings.TrimSpace(retained)) { + t.Fatalf("factual completion lost referenced evidence %q", retained) + } + } + if strings.Contains(completionText, "10:000a source declaration") { + t.Fatal("factual completion retained unreferenced source evidence") + } + if len(mustMarshalJSON(got)) > len(mustMarshalJSON(items)) { + t.Fatal("row-aware retirement increased retained history") + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("row-aware retirement mutated its input") + } + if again := reduceContextCompaction(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { + t.Fatal("row-aware factual completion was not stable across repeated compaction") + } + + whole := retirementHistory() + whole = append(whole, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep complete output from operation_00.", + })) + kept := reduceContextCompaction(whole) + for _, index := range []int{1, 2, 3} { + if string(kept[index]) != string(whole[index]) { + t.Fatal("whole-output reference no longer pins native operation") + } + } +} + +func TestCompactionRetirementPreservesFullPythonTraceback(t *testing.T) { + items := retirementHistory() + traceback := strings.Repeat("routine before\n", 300) + + "Traceback (most recent call last):\n" + + " File \"first.py\", line 10, in \n" + + " outer()\n" + + " File \"worker.py\", line 20, in outer\n" + + " inner()\n" + + " File \"worker.py\", line 30, in inner\n" + + " raise ValueError(\"masked failure\")\n" + + "ValueError: masked failure\n" + + "first line of a multiline exception message\n" + + "second line of the exception message\n" + + "important remediation note\n" + + strings.Repeat("routine after\n", 300) + items[3] = compactTestOutput("operation_00", traceback, 0) + + wire := string(mustMarshalJSON(retireCompactionOperations(items))) + for _, evidence := range []string{ + "Traceback (most recent call last):", + "first.py", + "worker.py", + "raise ValueError", + "ValueError: masked failure", + "first line of a multiline exception message", + "second line of the exception message", + "important remediation note", + } { + if !strings.Contains(wire, evidence) { + t.Fatalf("Python traceback evidence %q was lost", evidence) + } + } + if strings.Count(wire, "routine before") > 2 { + t.Fatal("bulk unmarked output before the traceback was not retired") + } + if strings.Count(wire, "routine after") != 300 { + t.Fatal("the suffix after the traceback was not preserved conservatively") + } +} + +func TestCompactionRetirementDoesNotTreatVerifiedSearchSourceAsDiagnostics(t *testing.T) { + var output strings.Builder + for row := 1; row <= 100; row++ { + fmt.Fprintf(&output, "\"path with spaces/example.go\":%d:abcd func example() error { return nil }\n", row) + } + output.WriteString("WARNING: integration coverage is incomplete\nok example/router 0.1s\n") + retired := compactionRetiredText(output.String()) + if strings.Contains(retired, `example.go":1:abcd`) { + t.Fatal("an error type inside verified source was retained as a runtime diagnostic") + } + if !strings.Contains(retired, "WARNING: integration coverage is incomplete") || + !strings.Contains(retired, "ok example/router 0.1s") { + t.Fatal("actual diagnostic or test outcome was lost") + } +} diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go new file mode 100644 index 00000000..f0905991 --- /dev/null +++ b/internal/router/context_compaction_source.go @@ -0,0 +1,569 @@ +package router + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "unicode/utf8" +) + +type compactionSourcePair struct { + call, result int + operation compactionOperation + known bool + scriptRefs []string +} + +func reduceContextCompactionSource(original, retained []json.RawMessage) []json.RawMessage { + if len(original) != len(retained) { + return retained + } + + originalFields := make([]map[string]json.RawMessage, len(original)) + retainedFields := make([]map[string]json.RawMessage, len(retained)) + calls, results := make(map[string][]int), make(map[string][]int) + var callOrder []int + for index := range original { + if json.Unmarshal(original[index], &originalFields[index]) != nil || + json.Unmarshal(retained[index], &retainedFields[index]) != nil { + continue + } + id := jsonString(originalFields[index], "call_id") + switch jsonString(originalFields[index], "type") { + case "function_call", "custom_tool_call": + calls[id] = append(calls[id], index) + callOrder = append(callOrder, index) + case "function_call_output", "custom_tool_call_output": + results[id] = append(results[id], index) + } + } + if len(callOrder) <= compactionRecentOperations { + return retained + } + cutoff := callOrder[len(callOrder)-compactionRecentOperations] + + pairs := make(map[string]compactionSourcePair) + for id, callPositions := range calls { + if id == "" || !compactionCallID.MatchString(id) || len(callPositions) != 1 || len(results[id]) != 1 { + continue + } + call, result := callPositions[0], results[id][0] + callType := jsonString(originalFields[call], "type") + if call >= cutoff || result >= cutoff || result <= call || + jsonString(originalFields[result], "type") != callType+"_output" || + jsonString(retainedFields[call], "type") != callType || + jsonString(retainedFields[result], "type") != callType+"_output" || + jsonString(retainedFields[call], "call_id") != id || + jsonString(retainedFields[result], "call_id") != id { + continue + } + if !compactionSourceShellCall(originalFields[call]) || compactionSourceIdentityCrosses(originalFields, call, result) { + continue + } + operation, known := compactionOperationCall(originalFields[call]) + pairs[id] = compactionSourcePair{ + call: call, result: result, operation: operation, known: known, + scriptRefs: compactionSourceResultScriptRefs(originalFields[result]["output"]), + } + } + if len(pairs) == 0 { + return retained + } + output := retained + cloned := false + + protected := contextCompactionReferencedResults(retained) + for id := range contextCompactionReferencedResults(original) { + protected[id] = true + } + rowReferences := make(map[string]bool) + var ranges [][2]string + unsafeEncoding := false + for index, fields := range originalFields { + if fields == nil || retainedFields[index] == nil { + continue + } + kind, owner := jsonString(fields, "type"), jsonString(fields, "call_id") + var references []json.RawMessage + switch kind { + case "function_call", "custom_tool_call": + operation, known := compactionOperationCall(fields) + normalized := known + if known && operation.patchReport != "" { + normalized = false + if positions := results[owner]; len(positions) == 1 { + _, normalized = compactionRetiredOutput(originalFields[positions[0]]["output"], operation) + } + } + if normalized { + references = append(references, operation.arguments) + if operation.notice != nil { + references = append(references, mustMarshalJSON(*operation.notice)) + } + } else { + references = append(references, fields["input"], fields["arguments"]) + } + case "function_call_output", "custom_tool_call_output": + continue + default: + references = append(references, fields["content"], fields["summary"]) + } + for _, raw := range references { + compactionVisitReferenceStrings(raw, func(text string) { + compactionSourceVisitDecodedReferences(text, func(decoded string) { + for _, word := range compactionReferenceWord.FindAllString(decoded, -1) { + if word != owner { + if _, exists := pairs[word]; exists { + protected[word] = true + } + } + } + for _, reference := range compactionScriptReference.FindAllString(decoded, -1) { + for id, pair := range pairs { + if id != owner && slices.Contains(pair.scriptRefs, reference) { + protected[id] = true + } + } + } + for _, match := range compactionVisibleLineReference.FindAllStringSubmatch(decoded, -1) { + for id := range pairs { + if id != owner && strings.HasSuffix(id, match[1]) { + protected[id] = true + } + } + } + for _, match := range compactionSourceRangeReference.FindAllStringSubmatch(decoded, -1) { + ranges = append(ranges, [2]string{match[1], match[2]}) + } + for _, row := range compactionRowReference.FindAllString(decoded, -1) { + rowReferences[row] = true + } + }, &unsafeEncoding) + }) + } + } + if unsafeEncoding { + for id := range pairs { + protected[id] = true + } + } + + for id, pair := range pairs { + if protected[id] { + continue + } + fields := retainedFields[pair.result] + mapped := fields["output"] + if pair.known && !compactionOutputNeedsSourcePreservation(mapped, rowReferences, ranges) { + if reduced, ok := compactionRetiredOutput(mapped, pair.operation); ok && len(reduced) < len(mapped) { + mapped = reduced + } + } + if string(mapped) == string(fields["output"]) { + mapped = mapCompactionCompletedOutput(mapped, func(text string) string { + return compactionPruneSourceText(text, rowReferences, ranges) + }) + } + if string(mapped) == string(fields["output"]) { + continue + } + if !cloned { + output = slices.Clone(retained) + cloned = true + } + copyFields := make(map[string]json.RawMessage, len(fields)) + for key, value := range fields { + copyFields[key] = value + } + copyFields["output"] = mapped + output[pair.result] = mustMarshalJSON(copyFields) + } + return output +} + +func compactionSourceShellCall(fields map[string]json.RawMessage) bool { + name := strings.TrimPrefix(jsonString(fields, "name"), "functions.") + switch jsonString(fields, "type") { + case "function_call": + return name == "exec_command" || name == "write_stdin" + case "custom_tool_call": + return name == "shell" || name == "exec" + default: + return false + } +} + +func compactionSourceIdentityCrosses(fields []map[string]json.RawMessage, call, result int) bool { + for index := call + 1; index < result; index++ { + switch jsonString(fields[index], "type") { + case "function_call", "custom_tool_call", "function_call_output", "custom_tool_call_output": + return true + } + } + return false +} + +func compactionSourceResultScriptRefs(raw json.RawMessage) []string { + var references []string + visitEnvelope := func(raw json.RawMessage) { + var serialized string + if json.Unmarshal(raw, &serialized) != nil { + return + } + var envelope map[string]json.RawMessage + if json.Unmarshal([]byte(serialized), &envelope) != nil { + return + } + if reference := jsonString(envelope, "script_ref"); reference != "" { + references = append(references, reference) + } + } + + visitEnvelope(raw) + var parts []map[string]json.RawMessage + if json.Unmarshal(raw, &parts) == nil { + for _, part := range parts { + if jsonString(part, "type") == "input_text" { + visitEnvelope(part["text"]) + } + } + } + return references +} + +func compactionSourceVisitDecodedReferences(text string, visit func(string), unsafeEncoding *bool) { + visit(text) + decoded, changed, unsafe := compactionSourceDecodeReferenceEscapes(text) + if changed { + // Always visit the original above as well as this single decoded layer. + // In particular, an escape which produces another escape spelling is + // not interpreted again. + visit(decoded) + } + if unsafe { + *unsafeEncoding = true + } +} + +// compactionSourceDecodeReferenceEscapes decodes one static source layer. It +// understands URL/HTML character encodings and JavaScript string/template +// escapes, but never evaluates interpolation or joins expressions. Generated +// escape spellings remain literal so a second interpretation cannot hide the +// original evidence. +func compactionSourceDecodeReferenceEscapes(text string) (decoded string, changed, unsafe bool) { + var result strings.Builder + result.Grow(len(text)) + for index := 0; index < len(text); { + switch text[index] { + case '\\': + value, width, recognized, valid := compactionSourceDecodeJSEscape(text[index:]) + if !recognized { + result.WriteByte(text[index]) + index++ + continue + } + if !valid { + unsafe = true + result.WriteByte(text[index]) + index++ + continue + } + changed = true + result.WriteString(value) + index += width + case '%': + if index+2 < len(text) && compactionSourceHexValue(text[index+1]) >= 0 && compactionSourceHexValue(text[index+2]) >= 0 { + result.WriteByte(byte(compactionSourceHexValue(text[index+1])<<4 | compactionSourceHexValue(text[index+2]))) + changed = true + index += 3 + continue + } + result.WriteByte(text[index]) + index++ + case '&': + value, width, recognized, valid := compactionSourceDecodeHTMLEscape(text[index:]) + if recognized && valid { + result.WriteString(value) + changed = true + index += width + continue + } + if recognized && !valid { + unsafe = true + } + result.WriteByte(text[index]) + index++ + default: + result.WriteByte(text[index]) + index++ + } + } + decoded = result.String() + return decoded, changed, unsafe +} + +func compactionSourceDecodeJSEscape(text string) (value string, width int, recognized, valid bool) { + if len(text) < 2 || text[0] != '\\' { + return "", 0, false, false + } + switch text[1] { + case '\\', '\'', '"', '`', '/': + return text[1:2], 2, true, true + case 'b': + return "\b", 2, true, true + case 'f': + return "\f", 2, true, true + case 'n': + return "\n", 2, true, true + case 'r': + return "\r", 2, true, true + case 't': + return "\t", 2, true, true + case 'v': + return "\v", 2, true, true + case '\n': + return "", 2, true, true + case '\r': + if len(text) >= 3 && text[2] == '\n' { + return "", 3, true, true + } + return "", 2, true, true + case '0': + if len(text) > 2 && text[2] >= '0' && text[2] <= '9' { + return "", 0, true, false + } + return "\x00", 2, true, true + case 'x': + if len(text) < 3 || compactionSourceHexValue(text[2]) < 0 { + return "", 0, false, false + } + if len(text) < 4 || compactionSourceHexValue(text[2]) < 0 || compactionSourceHexValue(text[3]) < 0 { + return "", 0, true, false + } + return string(rune(compactionSourceHexValue(text[2])<<4 | compactionSourceHexValue(text[3]))), 4, true, true + case 'u': + if len(text) < 3 || text[2] != '{' && compactionSourceHexValue(text[2]) < 0 { + return "", 0, false, false + } + return compactionSourceDecodeJSUnicodeEscape(text) + default: + return "", 0, false, false + } +} + +func compactionSourceDecodeJSUnicodeEscape(text string) (value string, width int, recognized, valid bool) { + if len(text) >= 3 && text[2] == '{' { + end := strings.IndexByte(text[3:], '}') + if end < 0 { + return "", 0, true, false + } + end += 3 + digits := text[3:end] + if len(digits) == 0 { + return "", 0, true, false + } + parsed, err := strconv.ParseUint(digits, 16, 32) + codePoint := rune(parsed) + if err != nil || !utf8.ValidRune(codePoint) { + return "", 0, true, false + } + return string(codePoint), end + 1, true, true + } + first, ok := compactionSourceParseJSCodeUnit(text) + if !ok { + return "", 0, true, false + } + if first >= 0xd800 && first <= 0xdbff { + if len(text) < 12 || text[6] != '\\' || text[7] != 'u' { + return "", 0, true, false + } + second, secondOK := compactionSourceParseJSCodeUnit(text[6:]) + if !secondOK || second < 0xdc00 || second > 0xdfff { + return "", 0, true, false + } + codePoint := rune(0x10000 + (first-0xd800)<<10 + second - 0xdc00) + return string(codePoint), 12, true, true + } + if first >= 0xdc00 && first <= 0xdfff { + return "", 0, true, false + } + return string(first), 6, true, true +} + +func compactionSourceParseJSCodeUnit(text string) (rune, bool) { + if len(text) < 6 || text[0] != '\\' || text[1] != 'u' { + return 0, false + } + return compactionSourceParseHex(text[2:6]) +} + +func compactionSourceParseHex(text string) (rune, bool) { + var value rune + for index := range len(text) { + digit := compactionSourceHexValue(text[index]) + if digit < 0 { + return 0, false + } + value = value<<4 | rune(digit) + } + return value, true +} + +func compactionSourceHexValue(value byte) int { + switch { + case value >= '0' && value <= '9': + return int(value - '0') + case value >= 'a' && value <= 'f': + return int(value-'a') + 10 + case value >= 'A' && value <= 'F': + return int(value-'A') + 10 + default: + return -1 + } +} + +func compactionSourceDecodeHTMLEscape(text string) (value string, width int, recognized, valid bool) { + if strings.HasPrefix(text, ":") { + return ":", len(":"), true, true + } + if strings.HasPrefix(text, "/") { + return "/", len("/"), true, true + } + if strings.HasPrefix(text, "&#") { + base, start := 10, 2 + if len(text) > start && (text[start] == 'x' || text[start] == 'X') { + base, start = 16, start+1 + } + end := start + for end < len(text) && ((base == 10 && text[end] >= '0' && text[end] <= '9') || + (base == 16 && compactionSourceHexValue(text[end]) >= 0)) { + end++ + } + if end == start || end >= len(text) || text[end] != ';' { + return "", 0, false, false + } + codePoint, err := strconv.ParseInt(text[start:end], base, 32) + if err != nil || !utf8.ValidRune(rune(codePoint)) { + return "", 0, false, false + } + return string(rune(codePoint)), end + 1, true, true + } + return "", 0, false, false +} + +func compactionOutputNeedsSourcePreservation(raw json.RawMessage, referenced map[string]bool, ranges [][2]string) bool { + preserve := false + mapCompactionCompletedOutput(raw, func(text string) string { + if strings.HasPrefix(text, "[hpatch compaction: retired finished-operation output (") || + strings.HasPrefix(text, "[hpatch: output details omitted; unavailable; original bytes=") { + preserve = true + return text + } + if compactionTextReferencesRows(text, referenced, ranges) { + preserve = true + } + return text + }) + return preserve +} + +func compactionSourceRowReferenced(token string, referenced map[string]bool, ranges [][2]string) bool { + if referenced[token] { + return true + } + lineText, _, ok := strings.Cut(token, ":") + if !ok { + return false + } + line, err := strconv.Atoi(lineText) + if err != nil { + return false + } + for _, rowRange := range ranges { + startText, _, startOK := strings.Cut(rowRange[0], ":") + endText, _, endOK := strings.Cut(rowRange[1], ":") + start, startErr := strconv.Atoi(startText) + end, endErr := strconv.Atoi(endText) + if startOK && endOK && startErr == nil && endErr == nil && + line >= min(start, end) && line <= max(start, end) { + return true + } + } + return false +} + +func compactionPruneSourceText(text string, referenced map[string]bool, ranges [][2]string) string { + lines := strings.SplitAfter(text, "\n") + tokens := make([]string, len(lines)) + keep := make([]bool, len(lines)) + for index, line := range lines { + if match := compactionCompleteSourceRow.FindStringSubmatch(line); match != nil { + tokens[index] = match[1] + } + if tokens[index] != "" && compactionSourceRowReferenced(tokens[index], referenced, ranges) { + keep[index] = true + } + } + + for _, rowRange := range ranges { + start, end := -1, -1 + for index, token := range tokens { + if start < 0 && token == rowRange[0] { + start = index + } + if start >= 0 && token == rowRange[1] { + end = index + } + } + if end >= start && start >= 0 { + for index := start; index <= end; index++ { + keep[index] = true + } + } + } + + for start, line := range lines { + if !compactionPythonTraceback.MatchString(line) { + continue + } + for index := start; index < len(lines); index++ { + keep[index] = true + } + break + } + + var result strings.Builder + for index := 0; index < len(lines); { + if tokens[index] == "" || keep[index] { + result.WriteString(lines[index]) + index++ + continue + } + end, size := index, 0 + for end < len(lines) && tokens[end] != "" && !keep[end] { + size += len(lines[end]) + end++ + } + count := end - index + note := fmt.Sprintf("[hpatch compaction: omitted %d unreferenced verified source rows (%s through %s); omitted rows are not retained]\n", + count, tokens[index], tokens[end-1]) + if count < 4 || size < 256 || len(note) >= size { + for _, line := range lines[index:end] { + result.WriteString(line) + } + } else { + result.WriteString(note) + } + index = end + } + return result.String() +} + +var ( + compactionCompleteSourceRow = regexp.MustCompile(`^(?:"(?:\\.|[^"\\])*":)?([1-9][0-9]*:[0-9a-f]{4}) [^\r\n]*\r?\n$`) + compactionSourceRangeReference = regexp.MustCompile(`\b([1-9][0-9]*:[0-9a-f]{4})\.\.([1-9][0-9]*:[0-9a-f]{4})\b`) + compactionVisibleLineReference = regexp.MustCompile(`(?m)^(?:!V)?=([A-Za-z0-9_-]+),[1-9][0-9]*,[1-9][0-9]*$`) +) diff --git a/internal/router/context_compaction_source_test.go b/internal/router/context_compaction_source_test.go new file mode 100644 index 00000000..eb5b06ef --- /dev/null +++ b/internal/router/context_compaction_source_test.go @@ -0,0 +1,425 @@ +package router + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "testing" +) + +func compactionSourceTestRows(quotedPath string, count int) []string { + rows := make([]string, count) + for index := range rows { + prefix := fmt.Sprintf("%d:%04x", index+1, index+1) + if quotedPath != "" { + prefix = fmt.Sprintf("%q:%s", quotedPath, prefix) + } + rows[index] = prefix + " source declaration with enough exact text to make source pruning profitable\n" + } + return rows +} + +func compactionSourceTestRecent() []json.RawMessage { + var recent []json.RawMessage + for index := range compactionRecentOperations { + id := fmt.Sprintf("recent-%d", index) + recent = append(recent, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + return recent +} + +func compactionSourceTestOutputText(t *testing.T, raw json.RawMessage) string { + t.Helper() + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + t.Fatal("invalid result record") + } + var text string + mapCompactionCompletedOutput(fields["output"], func(value string) string { + if text != "" { + t.Fatal("test result contains multiple output bodies") + } + text = value + return value + }) + if text == "" { + t.Fatal("result has no completed output body") + } + return text +} + +func TestCompactionSourcePrunesOnlyUnreferencedRows(t *testing.T) { + rows := compactionSourceTestRows("internal/router/source.go", 18) + source := "hgrep output follows\n" + strings.Join(rows, "") + "warning: keep this non-row diagnostic\n" + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Inspect the source before deciding."}}}), + compactTestCall("source-old", "hgrep -n declaration internal/router/source.go"), + compactTestOutput("source-old", source, 0), + mustMarshalJSON(map[string]any{"type": "function_call", "name": "unknown", "call_id": "unknown-old", "arguments": "{}"}), + compactTestOutput("unknown-old", "unknown companion\n", 0), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "Keep exact row 3:0003 and range 6:0006..8:0008 for the next edit."}), + } + items = append(items, compactionSourceTestRecent()...) + + before := string(mustMarshalJSON(items)) + got := reduceContextCompaction(items) + if len(got) != len(items) { + t.Fatal("source pruning removed timeline items") + } + text := compactionSourceTestOutputText(t, got[2]) + for _, want := range []string{ + "hgrep output follows", + rows[2], + rows[5], + rows[6], + rows[7], + "warning: keep this non-row diagnostic", + "[hpatch compaction: omitted", + } { + if !strings.Contains(text, want) { + t.Fatalf("source pruning lost %q", want) + } + } + for _, removed := range []string{rows[10], rows[17]} { + if strings.Contains(text, removed) { + t.Fatalf("unreferenced source row remained: %q", removed) + } + } + if string(got[3]) != string(items[3]) || string(got[4]) != string(items[4]) { + t.Fatal("unknown companion changed") + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("source pruning modified its input") + } + if again := reduceContextCompaction(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { + t.Fatal("source pruning was not idempotent") + } +} +func TestCompactionSourceKeepsProtectedOutputsByteExact(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + tests := []struct { + name string + result json.RawMessage + extra []json.RawMessage + recent bool + }{ + {name: "live", result: mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": "source-old", + "output": string(mustMarshalJSON(map[string]any{"output": rows, "exit_code": 0, "session_id": 42})), + })}, + {name: "whole call reference", result: compactTestOutput("source-old", rows, 0), extra: []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "Use complete output from source-old."}), + }}, + {name: "visible line reference", result: compactTestOutput("source-old", rows, 0), extra: []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "!V=old,2,4\n"}), + }}, + {name: "recent", result: compactTestOutput("source-old", rows, 0), recent: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + items := []json.RawMessage{compactTestCall("source-old", "hread source.go"), test.result} + items = append(items, test.extra...) + if !test.recent { + items = append(items, compactionSourceTestRecent()...) + } + got := reduceContextCompactionSource(items, items) + if string(got[1]) != string(test.result) { + t.Fatal("protected source output changed") + } + }) + } +} + +func TestCompactionSourceKeepsTracebackSuffix(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + traceback := "Traceback (most recent call last):\n File \"check.py\", line 4, in \nValueError: important failure\n" + items := []json.RawMessage{ + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", rows+traceback, 0), + } + items = append(items, compactionSourceTestRecent()...) + + got := reduceContextCompactionSource(items, items) + if !strings.Contains(compactionSourceTestOutputText(t, got[1]), traceback) { + t.Fatal("Python traceback suffix was not preserved byte-exact") + } +} + +func TestCompactionSourceUsesCompletedCodeModeBody(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "exec", "call_id": "source-old", + "input": "const result = await tools.some_runtime_call({}); text(result)", + }), + compactCodeModeOutput("source-old", rows), + } + items = append(items, compactionSourceTestRecent()...) + + got := reduceContextCompaction(items) + text := compactionSourceTestOutputText(t, got[1]) + if !strings.Contains(text, "[hpatch compaction: omitted") || strings.Contains(text, "10:000a source declaration") { + t.Fatal("completed Code Mode source body was not pruned") + } +} + +func TestCompactionSourceDecodesReferenceLiteralsConservatively(t *testing.T) { + rows := compactionSourceTestRows("", 18) + base := []json.RawMessage{ + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", strings.Join(rows, ""), 0), + } + base = append(base, compactionSourceTestRecent()...) + + escaped := slices.Clone(base) + escaped = append(escaped[:2], append([]json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": `const target = "3\u003a0003";`, + }), + }, escaped[2:]...)...) + got := reduceContextCompactionSource(escaped, escaped) + text := compactionSourceTestOutputText(t, got[1]) + if !strings.Contains(text, rows[2]) || strings.Contains(text, rows[10]) { + t.Fatal("escaped JavaScript row reference was not decoded") + } + + codePoint := slices.Clone(base) + codePoint = append(codePoint[:2], append([]json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": `const target = "3\u{3a}0003";`, + }), + }, codePoint[2:]...)...) + got = reduceContextCompactionSource(codePoint, codePoint) + text = compactionSourceTestOutputText(t, got[1]) + if !strings.Contains(text, rows[2]) || strings.Contains(text, rows[10]) { + t.Fatal("JavaScript code-point row reference was not decoded") + } +} +func TestCompactionSourceReducesSuccessfulOutputInBlockedGroup(t *testing.T) { + rows := compactionSourceTestRows("", 18) + live := mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": "live-old", + "output": string(mustMarshalJSON(map[string]any{ + "output": "still running\n", "exit_code": 0, "session_id": 42, + })), + }) + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Keep the blocked group native."}}, + "encrypted_content": "opaque", + }), + compactTestCall("finished-old", "make inspect"), + compactTestOutput("finished-old", strings.Repeat("unmarked historical detail\n", 100), 0), + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", strings.Join(rows, ""), 0), + compactTestCall("failed-old", "make inspect"), + compactTestOutput("failed-old", "important failure\n", 1), + compactTestCall("live-old", "make inspect"), + live, + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", + "content": "The next edit still requires 3:0003.", + }), + } + items = append(items, compactionSourceTestRecent()...) + before := string(mustMarshalJSON(items)) + + got := reduceContextCompaction(items) + finished := compactionSourceTestOutputText(t, got[2]) + if string(got[2]) == string(items[2]) || strings.Contains(finished, "unmarked historical detail\n") || + !strings.Contains(finished, "output details omitted; unavailable") { + t.Fatal("successful unreferenced output in blocked group was not reduced") + } + source := compactionSourceTestOutputText(t, got[4]) + if !strings.Contains(source, rows[2]) || strings.Contains(source, rows[10]) { + t.Fatal("referenced source row was not preserved while unreferenced rows were reduced") + } + for _, index := range []int{0, 1, 3, 5, 6, 7, 8, 9} { + if string(got[index]) != string(items[index]) { + t.Fatalf("blocked-group item %d changed", index) + } + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("output-only reduction mutated its input") + } +} + +func TestCompactionSourcePreservesReferencedPartialRow(t *testing.T) { + body := strings.Repeat("unmarked historical detail\n", 100) + + "diagnostic mentions partial target 3:0003 without a verified source line\n" + items := []json.RawMessage{ + compactTestCall("finished-old", "make inspect"), + compactTestOutput("finished-old", body, 0), + mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep 3:0003 for the next edit.", + }), + } + items = append(items, compactionSourceTestRecent()...) + got := reduceContextCompactionSource(items, items) + if string(got[1]) != string(items[1]) { + t.Fatal("successful output containing a referenced partial row was reduced") + } +} + +func TestCompactionSourceNeverRestoresRetiredHistory(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + original := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Inspect before editing."}}, + "encrypted_content": "opaque", + }), + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", rows, 0), + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "!V=old,2,4\n"}), + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Continue."}), + } + original = append(original, compactionSourceTestRecent()...) + upstream := retireCompactionOperations(reduceRepeatedCompactionRows(slices.Clone(original), contextCompactionReferencedResults(original))) + if string(upstream[0]) == string(original[0]) || string(upstream[1]) == string(original[1]) || string(upstream[2]) == string(original[2]) { + t.Fatal("fixture reasoning group was not fully retired") + } + before := string(mustMarshalJSON(upstream)) + got := reduceContextCompactionSource(original, upstream) + if string(mustMarshalJSON(got)) != before { + t.Fatal("source postpass resurrected already-retired history") + } + if string(mustMarshalJSON(upstream)) != before { + t.Fatal("source postpass mutated the retained input") + } +} + +func TestCompactionSourceDoesNotMutateRetainedInput(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + original := []json.RawMessage{ + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", rows, 0), + } + original = append(original, compactionSourceTestRecent()...) + retained := slices.Clone(original) + before := string(mustMarshalJSON(retained)) + wantResult := string(retained[1]) + got := reduceContextCompactionSource(original, retained) + if string(got[1]) == wantResult { + t.Fatal("fixture source output was not reduced") + } + if len(mustMarshalJSON(got)) > len(mustMarshalJSON(retained)) { + t.Fatal("source postpass increased serialized history") + } + if string(mustMarshalJSON(retained)) != before { + t.Fatal("source postpass mutated the retained input") + } +} + +func TestCompactionSourceGeneratedCarrierReferences(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + input := `const result = await tools.exec_command({"cmd":"hread source.go"}); text(JSON.stringify(Object.assign({}, result, {"retained":true,"script_ref":"@shell/retained"})));` + call := mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "exec", "call_id": "source-old", "input": input, + }) + var callFields map[string]json.RawMessage + _ = json.Unmarshal(call, &callFields) + if operation, ok := compactionOperationCall(callFields); !ok || string(operation.arguments) != `{"cmd":"hread source.go"}` { + t.Fatal("fixture generated carrier was not decoded") + } + base := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Inspect source."}}, + }), + call, + compactCodeModeOutput("source-old", rows), + mustMarshalJSON(map[string]any{"type": "function_call", "name": "unknown", "call_id": "unknown-old", "arguments": "{}"}), + compactTestOutput("unknown-old", "unknown companion\n", 0), + } + base = append(base, compactionSourceTestRecent()...) + + got := reduceContextCompaction(base) + text := compactionSourceTestOutputText(t, got[2]) + if string(got[2]) == string(base[2]) || strings.Contains(text, "10:000a source declaration") || + !strings.Contains(text, "[hpatch:") { + t.Fatal("generated carrier self metadata pinned its own source output") + } + + external := slices.Clone(base) + external = append(external[:5], append([]json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "Use the complete @shell/retained result."}), + }, external[5:]...)...) + got = reduceContextCompaction(external) + if string(got[2]) != string(external[2]) { + t.Fatal("external script reference did not pin complete source output") + } +} + +func TestCompactionSourceFailedPatchRetainsOriginalReferences(t *testing.T) { + sourceRows := compactionSourceTestRows("", 18) + patch := "*** Begin Patch\n*** Update File: source.go\n@@\n-" + strings.TrimSuffix(sourceRows[2], "\n") + "\n+replacement\n*** End Patch\n" + report := "in source.go\nfiles add=0 update=1 move=0 delete=0\n" + patchInput := hpatchApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" + patchCall := mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "exec", "call_id": "patch-consumer", "input": patchInput, + }) + base := []json.RawMessage{ + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", strings.Join(sourceRows, ""), 0), + patchCall, + } + base = append(base, compactionSourceTestRecent()...) + + failed := append(slices.Clone(base[:3]), + mustMarshalJSON(map[string]any{"type": "custom_tool_call_output", "call_id": "patch-consumer", "output": "Script failed"})) + failed = append(failed, base[3:]...) + got := reduceContextCompactionSource(failed, failed) + text := compactionSourceTestOutputText(t, got[1]) + if !strings.Contains(text, sourceRows[2]) || strings.Contains(text, sourceRows[10]) { + t.Fatal("failed translated patch did not retain references from its original body") + } + + success := append(slices.Clone(base[:3]), mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "patch-consumer", + "output": []any{ + map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, + map[string]any{"type": "input_text", "text": report}, + }, + })) + success = append(success, base[3:]...) + got = reduceContextCompactionSource(success, success) + if strings.Contains(compactionSourceTestOutputText(t, got[1]), sourceRows[2]) { + t.Fatal("successful translated patch body was treated as a retained reference consumer") + } +} + +func TestCompactionSourceKeepsAmbiguousIdentities(t *testing.T) { + rows := strings.Join(compactionSourceTestRows("", 18), "") + call := compactTestCall("source-old", "hread source.go") + result := compactTestOutput("source-old", rows, 0) + tests := []struct { + name string + items []json.RawMessage + index int + }{ + {name: "duplicate", items: []json.RawMessage{call, call, result}, index: 2}, + {name: "unmatched", items: []json.RawMessage{result}, index: 0}, + {name: "mismatched", items: []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "shell", "call_id": "source-old", "input": "hread source.go"}), + result, + }, index: 1}, + {name: "crossing", items: []json.RawMessage{ + call, + compactTestCall("other-old", "pwd"), + result, + compactTestOutput("other-old", "/workspace\n", 0), + }, index: 2}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.items = append(test.items, compactionSourceTestRecent()...) + want := test.items[test.index] + got := reduceContextCompactionSource(test.items, test.items) + if string(got[test.index]) != string(want) { + t.Fatal("ambiguous call/result identity was reduced") + } + }) + } +} From f211f3926b0d996670b2f2b75f7feddbaa85c0df Mon Sep 17 00:00:00 2001 From: yusing Date: Thu, 10 Sep 2026 14:06:10 +0000 Subject: [PATCH 04/13] feat(router): support provider-free WebSocket compaction Share compaction preparation, envelope restoration, and local completion framing across HTTP and WebSocket transports. Restore local history before provider projection, reset cached response linkage, preserve pending steering across local completion, and reject envelopes sent through steering. Add WebSocket round-trip and fail-closed coverage, and document local history replacement, continuation, and resumption behavior. --- doc/architecture/compaction.md | 10 +- doc/spec/compaction.md | 12 +- internal/router/context_compaction_http.go | 244 +++++++++------- .../context_compaction_websocket_test.go | 262 ++++++++++++++++++ internal/router/debug_test.go | 2 +- internal/router/server.go | 6 +- internal/router/server_websocket.go | 47 +++- .../router/server_websocket_capture_test.go | 2 +- internal/router/server_websocket_test.go | 6 +- 9 files changed, 470 insertions(+), 121 deletions(-) create mode 100644 internal/router/context_compaction_websocket_test.go diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index 05ce52fa..34128edf 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -2,7 +2,7 @@ ## CTR-COMPACTION-001 — Router-owned pruning and local envelopes -The router's context-compaction HTTP boundary precedes ordinary Responses +The router's shared context-compaction boundary precedes ordinary Responses projection and upstream transport. It owns local standalone and V2 completion, restoration of its own input envelopes, and bounded request decoding. Codex owns configuration resolution, trigger timing, retained client-side context, and @@ -77,5 +77,9 @@ Neither those boundaries nor the provider interpret router-owned ciphertext. No provider transport is available to the local compaction handler. Errors leave Codex without a replacement window rather than silently losing context or issuing -a model summary. Capture observes local HTTP traffic but local compaction does not -fabricate provider token usage. +a model summary. HTTP/SSE and WebSocket adapters share native restoration and +local completion framing. The WebSocket adapter owns per-message metadata, +incremental history, and resetting the provider cache relationship after restoring +an envelope. Local completion replaces its native history with the capsule. +Capture observes local traffic but local compaction does not fabricate provider +token usage. diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index 8e03b107..03429506 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -15,10 +15,16 @@ The invocation also disables Codex request compression because the local router accepts uncompressed JSON, not ChatGPT Zstd request bodies. The router handles `POST /v1/responses/compact` locally. It also handles streaming -`POST /v1/responses` requests identified by Codex metadata as -`responses_compaction_v2`. Neither path calls a provider. Other metadata-tagged +`POST /v1/responses` and WebSocket `response.create` requests identified by +Codex metadata as `responses_compaction_v2`. None of these paths calls a provider. +Other metadata-tagged compaction implementations fail explicitly instead of requesting a provider -summary. Local handling applies in both router modes. +summary. Local handling applies in both router modes. WebSocket continuation and +resumption restore local envelopes before projection, without forwarding local +response IDs or treating the restored timeline as provider-cached history. +A locally completed compaction replaces its WebSocket history with the capsule; +it does not retain the unpruned parent beside it. Steering cannot carry local +envelopes. Compaction preserves user/developer instructions, corrections, authorization, and the active execution frontier. Older ordinary-assistant narration can omit an diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go index c9abfb6a..8663dbf6 100644 --- a/internal/router/context_compaction_http.go +++ b/internal/router/context_compaction_http.go @@ -14,8 +14,8 @@ import ( "time" ) -// Both compact interfaces terminate here, before tool projection, CTP, or any -// provider call. Codex remains the sole owner of scheduling and configuration. +// Both transports use the same boundary before projection or provider calls. +// Codex remains the sole owner of scheduling and configuration. func (c *contextCompactor) handler(next http.Handler) http.HandlerFunc { return func(writer http.ResponseWriter, request *http.Request) { body, err := readResponsesRequest(io.LimitReader(request.Body, responsesRequestBufferBytes+1)) @@ -27,53 +27,25 @@ func (c *contextCompactor) handler(next http.Handler) http.HandlerFunc { http.Error(writer, "Responses request exceeds the router buffer budget", http.StatusRequestEntityTooLarge) return } - metadata, metadataValid := decodeCodexTurnMetadata(request.Header) - standalone := request.URL.Path == "/v1/responses/compact" - compacting := standalone || (metadataValid && metadata.RequestKind == "compaction") parsed, err := parseResponsesRequest(body) if err != nil { http.Error(writer, err.Error(), http.StatusBadRequest) return } - var input []json.RawMessage - if json.Unmarshal(parsed.fields["input"], &input) != nil || len(input) == 0 { - if !compacting { - request.Body = io.NopCloser(bytes.NewReader(body)) - next.ServeHTTP(writer, request) - return - } - http.Error(writer, "local compaction requires a nonempty input item array", http.StatusBadRequest) - return - } - local := false - for _, raw := range input { - var item map[string]json.RawMessage - _ = json.Unmarshal(raw, &item) - local = local || strings.HasPrefix(jsonString(item, "encrypted_content"), "hpatch.compaction.") || strings.HasPrefix(jsonString(item, "id"), contextCompactionIDPrefix) - } - if !compacting && !local { - request.Body = io.NopCloser(bytes.NewReader(body)) - next.ServeHTTP(writer, request) - return - } - - for _, item := range input { - var fields map[string]json.RawMessage - if json.Unmarshal(item, &fields) != nil || fields == nil { - http.Error(writer, "compaction input items must be objects", http.StatusBadRequest) - return - } - } - input, err = c.restore(request.Context(), input) + standalone := request.URL.Path == "/v1/responses/compact" + capsule, err := c.prepare(request.Context(), &parsed, request.Header, standalone) if err != nil { - http.Error(writer, err.Error(), http.StatusUnprocessableEntity) + status := http.StatusUnprocessableEntity + if failure, ok := errors.AsType[*contextCompactionRequestError](err); ok { + status = failure.status + } + http.Error(writer, err.Error(), status) return } - if !compacting { - parsed.setInput(mustMarshalJSON(input)) + if len(capsule) == 0 { body, err = parsed.wireBody(parsed.fields) - if err != nil || len(body) > responsesRequestBufferBytes { - http.Error(writer, "restored history exceeds the router buffer budget", http.StatusRequestEntityTooLarge) + if err != nil { + http.Error(writer, err.Error(), http.StatusBadRequest) return } request.Body = io.NopCloser(bytes.NewReader(body)) @@ -81,76 +53,146 @@ func (c *contextCompactor) handler(next http.Handler) http.HandlerFunc { next.ServeHTTP(writer, request) return } - if parsed.model() == "" { - http.Error(writer, "compaction requires a model", http.StatusBadRequest) - return + if standalone { + writer.Header().Set("Content-Type", "application/json") + } else { + writer.Header().Set("Content-Type", "text/event-stream") } - if !standalone { - var details struct { - Implementation string `json:"implementation"` - } - _ = json.Unmarshal(metadata.Compaction, &details) - if !parsed.streamResponse || details.Implementation != "responses_compaction_v2" { - http.Error(writer, "local compaction requires the standalone compact endpoint or streaming compaction V2; provider summaries are disabled", http.StatusUnprocessableEntity) - return - } - // This is a request control, not part of the durable history. - var last map[string]json.RawMessage - _ = json.Unmarshal(input[len(input)-1], &last) - if jsonString(last, "type") == "compaction_trigger" { - input = input[:len(input)-1] - } + _ = writeContextCompactionResponse(writer, capsule, parsed.fields["input"], standalone) + } +} + +type contextCompactionRequestError struct { + status int + message string +} + +func (e *contextCompactionRequestError) Error() string { return e.message } + +func hasLocalContextCompaction(input []json.RawMessage) bool { + for _, raw := range input { + var item map[string]json.RawMessage + _ = json.Unmarshal(raw, &item) + if strings.HasPrefix(jsonString(item, "encrypted_content"), "hpatch.compaction.") || strings.HasPrefix(jsonString(item, "id"), contextCompactionIDPrefix) { + return true } - reduced := reduceContextCompaction(input) - if slices.EqualFunc(input, reduced, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { - http.Error(writer, "no supported context reduction is available for this history; protected context was not discarded and no provider compaction was requested", http.StatusUnprocessableEntity) - return + } + return false +} + +// prepare restores native input in place and returns a capsule only for a local +// completion. It has no provider transport, including for unsupported requests. +func (c *contextCompactor) prepare(ctx context.Context, parsed *parsedResponsesRequest, headers http.Header, standalone bool) (json.RawMessage, error) { + fail := func(status int, message string) (json.RawMessage, error) { + return nil, &contextCompactionRequestError{status: status, message: message} + } + metadata, valid := decodeCodexTurnMetadata(headers) + compacting := standalone || (valid && metadata.RequestKind == "compaction") + var input []json.RawMessage + if json.Unmarshal(parsed.fields["input"], &input) != nil || len(input) == 0 { + if !compacting { + return nil, nil } - capsule, err := c.seal(request.Context(), reduced) - if err != nil { - http.Error(writer, err.Error(), http.StatusUnprocessableEntity) - return + return fail(http.StatusBadRequest, "local compaction requires a nonempty input item array") + } + local := hasLocalContextCompaction(input) + if !compacting && !local { + return nil, nil + } + for _, item := range input { + var fields map[string]json.RawMessage + if json.Unmarshal(item, &fields) != nil || fields == nil { + return fail(http.StatusBadRequest, "compaction input items must be objects") } - var sealedItem struct { - ID string `json:"id"` + } + input, err := c.restore(ctx, input) + if err != nil { + return fail(http.StatusUnprocessableEntity, err.Error()) + } + parsed.setInput(mustMarshalJSON(input)) + if local { + // Provider-side history cannot represent a router-owned capsule. Send + // the restored timeline in full instead of trimming a cached prefix or + // naming a response that was completed only by the router. + parsed.cachedInput = 0 + if _, exists := parsed.fields["previous_response_id"]; exists { + parsed.fields["previous_response_id"] = json.RawMessage("null") } - _ = json.Unmarshal(capsule, &sealedItem) - responseID := "resp_" + strings.TrimPrefix(sealedItem.ID, "cmp_") - if standalone { - // Legacy Codex replaces its history wholesale. Keep real user messages - // visible to its user-input handling. Historical canonical context stays - // only in the capsule so it cannot be mistaken for a fresh injection. - var output []json.RawMessage - for _, item := range reduced { - var fields map[string]json.RawMessage - _ = json.Unmarshal(item, &fields) - if jsonString(fields, "type") == "message" && jsonString(fields, "role") == "user" && !contextCompactionFreshContext(item) { - output = append(output, item) - } - } - output = append(output, capsule) - writer.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(writer).Encode(map[string]any{ - "object": "response.compaction", "id": responseID, - "created_at": time.Now().Unix(), "output": output, - }) - return + + } + body, err := parsed.wireBody(parsed.fields) + if err != nil || len(body) > responsesRequestBufferBytes { + return fail(http.StatusRequestEntityTooLarge, "restored history exceeds the router buffer budget") + } + if !compacting { + return nil, nil + } + if parsed.model() == "" { + return fail(http.StatusBadRequest, "compaction requires a model") + } + if !standalone { + var details struct { + Implementation string `json:"implementation"` } - // V2 retains its own selected user/context items, then appends exactly - // one compaction item. No synthetic assistant prose or provider usage. - writer.Header().Set("Content-Type", "text/event-stream") - for sequence, event := range []map[string]any{ - {"type": "response.created", "response": map[string]any{"id": responseID, "status": "in_progress", "output": []any{}}}, - {"type": "response.output_item.added", "output_index": 0, "item": capsule}, - {"type": "response.output_item.done", "output_index": 0, "item": capsule}, - {"type": "response.completed", "response": map[string]any{"id": responseID, "status": "completed", "output": []json.RawMessage{capsule}}}, - } { - event["sequence_number"] = sequence - if _, err := fmt.Fprintf(writer, "event: %s\ndata: %s\n\n", event["type"], mustMarshalJSON(event)); err != nil { - return + _ = json.Unmarshal(metadata.Compaction, &details) + if !parsed.streamResponse || details.Implementation != "responses_compaction_v2" { + return fail(http.StatusUnprocessableEntity, "local compaction requires the standalone compact endpoint or streaming compaction V2; provider summaries are disabled") + } + var last map[string]json.RawMessage + _ = json.Unmarshal(input[len(input)-1], &last) + if jsonString(last, "type") == "compaction_trigger" { + input = input[:len(input)-1] + } + } + reduced := reduceContextCompaction(input) + if slices.EqualFunc(input, reduced, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { + return fail(http.StatusUnprocessableEntity, "no supported context reduction is available for this history; protected context was not discarded and no provider compaction was requested") + } + capsule, err := c.seal(ctx, reduced) + if err != nil { + return fail(http.StatusUnprocessableEntity, err.Error()) + } + parsed.setInput(mustMarshalJSON(reduced)) + return capsule, nil +} + +// The stream framing is shared by HTTP/SSE and the WebSocket output adapter. +func writeContextCompactionResponse(writer io.Writer, capsule, retained json.RawMessage, standalone bool) error { + var sealedItem struct { + ID string `json:"id"` + } + _ = json.Unmarshal(capsule, &sealedItem) + responseID := "resp_" + strings.TrimPrefix(sealedItem.ID, "cmp_") + if standalone { + // Keep real user messages visible to legacy Codex's input handling. + // Historical canonical context stays only inside the capsule. + var input, output []json.RawMessage + _ = json.Unmarshal(retained, &input) + for _, item := range input { + var fields map[string]json.RawMessage + _ = json.Unmarshal(item, &fields) + if jsonString(fields, "type") == "message" && jsonString(fields, "role") == "user" && !contextCompactionFreshContext(item) { + output = append(output, item) } } + output = append(output, capsule) + return json.NewEncoder(writer).Encode(map[string]any{ + "object": "response.compaction", "id": responseID, + "created_at": time.Now().Unix(), "output": output, + }) + } + for sequence, event := range []map[string]any{ + {"type": "response.created", "response": map[string]any{"id": responseID, "status": "in_progress", "output": []any{}}}, + {"type": "response.output_item.added", "output_index": 0, "item": capsule}, + {"type": "response.output_item.done", "output_index": 0, "item": capsule}, + {"type": "response.completed", "response": map[string]any{"id": responseID, "status": "completed", "output": []json.RawMessage{capsule}}}, + } { + event["sequence_number"] = sequence + if _, err := fmt.Fprintf(writer, "event: %s\ndata: %s\n\n", event["type"], mustMarshalJSON(event)); err != nil { + return err + } } + return nil } // Restore the native timeline once. Codex may carry a subset of the original diff --git a/internal/router/context_compaction_websocket_test.go b/internal/router/context_compaction_websocket_test.go new file mode 100644 index 00000000..0b512c9a --- /dev/null +++ b/internal/router/context_compaction_websocket_test.go @@ -0,0 +1,262 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" +) + +func compactionTestSocket(t *testing.T, ctx context.Context, compactor *contextCompactor, upstream http.Handler) *websocket.Conn { + t.Helper() + provider := httptest.NewServer(upstream) + t.Cleanup(provider.Close) + endpoint := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, nil, nil, nil, compactor) + t.Cleanup(endpoint.Close) + router := httptest.NewServer(endpoint) + t.Cleanup(router.Close) + conn, _, err := websocket.Dial(ctx, router.URL, &websocket.DialOptions{HTTPHeader: codexAuthHeaders()}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + return conn +} + +func compactionSocketCompletion(t *testing.T, ctx context.Context, conn *websocket.Conn) map[string]json.RawMessage { + t.Helper() + for { + event := socketRead(t, ctx, conn) + if jsonString(event, "type") == "error" { + t.Fatalf("socket error: %s", mustMarshalJSON(event)) + } + if jsonString(event, "type") == "response.completed" { + var response map[string]json.RawMessage + if err := json.Unmarshal(event["response"], &response); err != nil { + t.Fatal(err) + } + return response + } + } +} + +func TestCompactionWebSocketPreservesPendingSteering(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + startSuccessor := make(chan struct{}) + conn := compactionTestSocket(t, ctx, compactor, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstream, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer upstream.CloseNow() + if _, err := providerSocketRead(ctx, upstream); err != nil { + t.Error(err) + return + } + if err := providerSocketWrite(ctx, upstream, socketEvent("response.created", "parent")); err != nil { + t.Error(err) + return + } + if _, err := providerSocketRead(ctx, upstream); err != nil { + t.Error(err) + return + } + if err := providerSocketWrite(ctx, upstream, map[string]any{ + "type": "response.steer.accepted", + "steer": map[string]string{"id": "steer", "previous_response_id": "parent"}, + }); err != nil { + t.Error(err) + return + } + if err := providerSocketWrite(ctx, upstream, socketEvent("response.completed", "parent")); err != nil { + t.Error(err) + return + } + select { + case <-ctx.Done(): + return + case <-startSuccessor: + } + // No explicit parent: the router must use the last provider response, + // not the intervening local compact response. + if err := providerSocketWrite(ctx, upstream, socketEvent("response.created", "successor")); err != nil { + t.Error(err) + return + } + if err := providerSocketWrite(ctx, upstream, socketEvent("response.completed", "successor")); err != nil { + t.Error(err) + return + } + <-ctx.Done() + })) + socketWrite(t, ctx, conn, map[string]any{"type": "response.create", "model": "gpt-5", "input": compactHTTPHistory()}) + if event := socketRead(t, ctx, conn); jsonString(event, "type") != "response.created" { + t.Fatalf("expected response.created: %s", mustMarshalJSON(event)) + } + direction := "Preserve this accepted direction across the local compact." + socketWrite(t, ctx, conn, map[string]any{"type": "response.steer", "previous_response_id": "parent", "input": direction}) + compactionSocketCompletion(t, ctx, conn) + compact := func(parent string) map[string]json.RawMessage { + socketWrite(t, ctx, conn, map[string]any{ + "type": "response.create", "model": "gpt-5", "previous_response_id": parent, + "input": []any{map[string]string{"type": "compaction_trigger"}}, + "client_metadata": map[string]string{codexTurnMetadataHeader: `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`}, + }) + return compactionSocketCompletion(t, ctx, conn) + } + compact("parent") + close(startSuccessor) + if response := compactionSocketCompletion(t, ctx, conn); jsonString(response, "id") != "successor" { + t.Fatalf("automatic successor was lost: %s", mustMarshalJSON(response)) + } + response := compact("successor") + var output []json.RawMessage + if err := json.Unmarshal(response["output"], &output); err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(ctx, output) + if err != nil { + t.Fatal(err) + } + if strings.Count(string(mustMarshalJSON(restored)), direction) != 1 { + t.Fatalf("accepted steering was lost or duplicated: %s", mustMarshalJSON(restored)) + } +} +func TestCompactionWebSocketRoundTrip(t *testing.T) { + for _, localV2 := range []bool{false, true} { + t.Run(fmt.Sprintf("websocket_compaction=%t", localV2), func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + history := compactHTTPHistory() + // Provider-owned encrypted state must remain byte-for-byte opaque. + history = append(history, json.RawMessage(`{"type":"reasoning","encrypted_content":"provider-owned"}`)) + reduced := reduceContextCompaction(history) + var calls atomic.Int32 + received := make(chan []json.RawMessage, 2) + conn := compactionTestSocket(t, ctx, compactor, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + upstream, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer upstream.CloseNow() + for i := range 2 { + create, err := providerSocketRead(ctx, upstream) + if err != nil { + t.Error(err) + return + } + if len(create["previous_response_id"]) != 0 || bytes.Contains(create["input"], []byte(contextCompactionPrefix)) { + t.Errorf("local state escaped to provider: %s", mustMarshalJSON(create)) + } + var input []json.RawMessage + if err := json.Unmarshal(create["input"], &input); err != nil { + t.Error(err) + return + } + received <- input + id := fmt.Sprintf("provider-%d", i) + if err := providerSocketWrite(ctx, upstream, socketEvent("response.created", id)); err != nil { + t.Error(err) + return + } + if err := providerSocketWrite(ctx, upstream, socketEvent("response.completed", id)); err != nil { + t.Error(err) + return + } + } + })) + var window []json.RawMessage + parent := "" + if localV2 { + socketWrite(t, ctx, conn, map[string]any{ + "type": "response.create", "model": "gpt-5", + "input": append(history, json.RawMessage(`{"type":"compaction_trigger"}`)), + "client_metadata": map[string]string{codexTurnMetadataHeader: `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`}, + }) + response := compactionSocketCompletion(t, ctx, conn) + parent = jsonString(response, "id") + if !strings.Contains(string(response["output"]), contextCompactionPrefix) || calls.Load() != 0 { + t.Fatal("V2 did not complete locally") + } + } else { + // A standalone HTTP compact followed by a new socket models resume. + request := httptest.NewRequest(http.MethodPost, "/v1/responses/compact", bytes.NewReader(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": history}))) + response := httptest.NewRecorder() + compactor.handler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("compact reached provider") }))(response, request) + var result struct { + Output []json.RawMessage `json:"output"` + } + if response.Code != http.StatusOK || json.Unmarshal(response.Body.Bytes(), &result) != nil { + t.Fatalf("compact failed: %s", response.Body.String()) + } + window = result.Output + // Reopen the installation-owned key instead of relying on process state. + *compactor = contextCompactor{keyPath: compactor.keyPath} + } + followup := json.RawMessage(`{"type":"message","role":"user","content":"Continue after compaction."}`) + create := map[string]any{"type": "response.create", "model": "gpt-5", "input": append(window, followup)} + if parent != "" { + create["previous_response_id"] = parent + } + socketWrite(t, ctx, conn, create) + response := compactionSocketCompletion(t, ctx, conn) + want := append(reduced, followup) + if got := <-received; contextCompactionCanonicalJSON(mustMarshalJSON(got)) != contextCompactionCanonicalJSON(mustMarshalJSON(want)) { + t.Fatalf("restoration mismatch:\ngot %s\nwant %s", mustMarshalJSON(got), mustMarshalJSON(want)) + } + next := json.RawMessage(`{"type":"message","role":"user","content":"One more turn."}`) + socketWrite(t, ctx, conn, map[string]any{"type": "response.create", "model": "gpt-5", "previous_response_id": jsonString(response, "id"), "input": []json.RawMessage{next}}) + compactionSocketCompletion(t, ctx, conn) + want = append(want, next) + if got := <-received; contextCompactionCanonicalJSON(mustMarshalJSON(got)) != contextCompactionCanonicalJSON(mustMarshalJSON(want)) { + t.Fatalf("incremental restoration duplicated or lost history:\ngot %s\nwant %s", mustMarshalJSON(got), mustMarshalJSON(want)) + } + }) + } +} + +func TestCompactionWebSocketFailsClosed(t *testing.T) { + for _, tc := range []struct { + name, input, metadata string + status int + }{ + {"damaged", `[{"type":"compaction","encrypted_content":"hpatch\u002ecompaction\u002ev1:broken"}]`, "", 422}, + {"unknown_version", `[{"type":"compaction","encrypted_content":"hpatch.compaction.v999:broken"}]`, "", 422}, + {"unsupported", `[{"type":"message","role":"user","content":"Keep this."}]`, `{"request_kind":"compaction","compaction":{"implementation":"unknown"}}`, 422}, + {"no_reduction", `[{"type":"message","role":"user","content":"Keep this."}]`, `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`, 422}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + var calls atomic.Int32 + conn := compactionTestSocket(t, ctx, compactor, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + http.Error(w, "must not reach provider", 500) + })) + socketWrite(t, ctx, conn, map[string]any{ + "type": "response.create", "model": "gpt-5", "input": json.RawMessage(tc.input), + "client_metadata": map[string]string{codexTurnMetadataHeader: tc.metadata}, + }) + event := socketRead(t, ctx, conn) + if jsonString(event, "type") != "error" || string(event["status"]) != fmt.Sprint(tc.status) || calls.Load() != 0 { + t.Fatalf("not fail-closed: %s, provider calls %d", mustMarshalJSON(event), calls.Load()) + } + }) + } +} diff --git a/internal/router/debug_test.go b/internal/router/debug_test.go index 0407a1a8..8f7ab23f 100644 --- a/internal/router/debug_test.go +++ b/internal/router/debug_test.go @@ -195,7 +195,7 @@ func TestDebugWebSocketInheritedInstructions(t *testing.T) { t.Cleanup(upstream.Close) proxy := newToolPluginTestProxy(t) proxy.customizedInstructions = true - endpoint := responsesWebSocketHandler(ctx, 10*time.Second, newProviderClient(upstream.URL, upstream.Client()), nil, proxy, mustCTP2Codec(t), nil) + endpoint := responsesWebSocketHandler(ctx, 10*time.Second, newProviderClient(upstream.URL, upstream.Client()), nil, proxy, mustCTP2Codec(t), nil, nil) t.Cleanup(endpoint.Close) server := httptest.NewServer(d.handler(endpoint)) t.Cleanup(server.Close) diff --git a/internal/router/server.go b/internal/router/server.go index 0a737a5e..532f800d 100644 --- a/internal/router/server.go +++ b/internal/router/server.go @@ -243,9 +243,6 @@ func RunSession(ctx context.Context, args []string, issues *CriticalErrors, read if mekugiCalls != nil { mux.HandleFunc("POST "+commentaryPublisherPath, mekugiCalls.commentary.serveHTTP) } - webSocketEndpoint := responsesWebSocketHandler(ctx, *flags.timeout, provider, issues, mekugiCalls, compactTokens, mentor) - defer webSocketEndpoint.Close() - mux.Handle("GET /v1/responses", webSocketEndpoint) // The compact boundary also restores router-owned envelopes before any // ordinary request reaches projection or upstream transport. compactionDirectory, err := mekugiDataDirectory() @@ -253,6 +250,9 @@ func RunSession(ctx context.Context, args []string, issues *CriticalErrors, read return err } compaction := &contextCompactor{keyPath: filepath.Join(compactionDirectory, "compaction.key")} + webSocketEndpoint := responsesWebSocketHandler(ctx, *flags.timeout, provider, issues, mekugiCalls, compactTokens, mentor, compaction) + defer webSocketEndpoint.Close() + mux.Handle("GET /v1/responses", webSocketEndpoint) responses := compaction.handler(responsesHandler(ctx, *flags.timeout, provider, issues, mekugiCalls, compactTokens, mentor)) mux.HandleFunc("POST /v1/responses", responses) mux.HandleFunc("POST /v1/responses/compact", responses) diff --git a/internal/router/server_websocket.go b/internal/router/server_websocket.go index 30a0c0c3..b1594339 100644 --- a/internal/router/server_websocket.go +++ b/internal/router/server_websocket.go @@ -23,7 +23,7 @@ import ( // A downstream socket owns a dedicated provider socket. In particular it never // enters the HTTP pool: accepted steering and previous_response_id are scoped // to this connection, including the quiet interval after response.completed. -func responsesWebSocketHandler(lifecycle context.Context, timeout time.Duration, provider *providerClient, issues *CriticalErrors, proxy *mekugiProxy, codec *ctp2Codec, mentor *mentorHandoff) *responsesWebSocketEndpoint { +func responsesWebSocketHandler(lifecycle context.Context, timeout time.Duration, provider *providerClient, issues *CriticalErrors, proxy *mekugiProxy, codec *ctp2Codec, mentor *mentorHandoff, compaction *contextCompactor) *responsesWebSocketEndpoint { lifecycle, cancel := context.WithCancel(lifecycle) endpoint := &responsesWebSocketEndpoint{cancel: cancel} endpoint.handler = func(w http.ResponseWriter, r *http.Request) { @@ -43,7 +43,8 @@ func responsesWebSocketHandler(lifecycle context.Context, timeout time.Duration, defer stop() s := &responsesWebSocket{ ctx: ctx, downstream: conn, provider: provider, headers: r.Header.Clone(), - timeout: timeout, issues: issues, proxy: proxy, codec: codec, mentor: mentor, + compaction: compaction, + timeout: timeout, issues: issues, proxy: proxy, codec: codec, mentor: mentor, clientMessages: readResponsesWebSocket(ctx, conn, cancel), histories: make(map[string]*webSocketHistory), } defer func() { @@ -103,6 +104,9 @@ func (s *responsesWebSocket) writeError(ctx context.Context, err error) ([]byte, if _, ok := errors.AsType[*requestCompatibilityError](err); ok { status = http.StatusBadRequest } + if failure, ok := errors.AsType[*contextCompactionRequestError](err); ok { + status = failure.status + } fields := map[string]any{"type": "error", "status": status, "error": map[string]string{"type": "invalid_request_error", "message": err.Error()}} if upstream, ok := errors.AsType[*webSocketStatusError](err); ok { fields["status"] = upstream.status @@ -246,6 +250,7 @@ type responsesWebSocket struct { proxy *mekugiProxy codec *ctp2Codec mentor *mentorHandoff + compaction *contextCompactor histories map[string]*webSocketHistory lastID string steers []webSocketSteer @@ -352,6 +357,9 @@ func (s *responsesWebSocket) control(body []byte) error { if err != nil { return err } + if hasLocalContextCompaction(input) { + return incompatibleRequest("invalid_websocket_request", "local compaction envelopes require response.create, not steering") + } if err := s.retain(input); err != nil { return err } @@ -560,7 +568,18 @@ func (s *responsesWebSocket) execute(command, firstEvent []byte) error { defer cancel() exchange.ctx = executionCtx output := &webSocketOutput{exchange: exchange} - err = executeRequest(startCtx, executionCtx, parsed, headers, routingSessionID(headers, parsed), exchange, output, s.issues, s.proxy, s.codec, s.mentor) + var capsule json.RawMessage + if s.compaction != nil { + capsule, err = s.compaction.prepare(executionCtx, &parsed, headers, false) + } + if err == nil { + if len(capsule) != 0 { + exchange.local = true + err = writeContextCompactionResponse(output, capsule, parsed.fields["input"], false) + } else { + err = executeRequest(startCtx, executionCtx, parsed, headers, routingSessionID(headers, parsed), exchange, output, s.issues, s.proxy, s.codec, s.mentor) + } + } if err != nil && executionCtx.Err() == nil && !s.errorDelivered { if payload, writeErr := s.writeError(executionCtx, err); writeErr == nil { clientObservation.Message(payload) @@ -575,6 +594,7 @@ type webSocketExchange struct { session *responsesWebSocket ctx context.Context automatic bool + local bool first []byte history *webSocketHistory parentID string @@ -622,7 +642,7 @@ func (e *webSocketExchange) forwardExecution(startCtx, responseCtx context.Conte if err != nil { return nil, err } - if len(previous) != 0 { + if len(previous) != 0 && string(previous) != "null" { payload, err = replaceRawField(payload, "previous_response_id", previous) if err != nil { return nil, err @@ -832,8 +852,18 @@ func (w *webSocketOutput) message(payload []byte) error { if event.Type == "response.created" { // Creation commits queued steering. Never replay accepted input after // this point, even if delivery or the successor subsequently fails. - s.commitSteering(e.history, e.parentID) + if e.local { + // Local completion retains only the capsule, but does not admit a + // provider successor. Its accepted steering remains pending until + // that provider response actually starts. + e.history.parent = nil + e.history.input = nil + } else { + s.commitSteering(e.history, e.parentID) + } + } + switch event.Type { case "response.completed", "response.incomplete", "response.failed": if event.Response.ID == "" { @@ -852,7 +882,12 @@ func (w *webSocketOutput) message(payload []byte) error { e.history.output = event.Response.Output } s.histories[event.Response.ID] = e.history - s.lastID = event.Response.ID + // An automatic provider successor without an explicit parent belongs + // to the last provider response, never a locally generated compact ID. + if !e.local { + s.lastID = event.Response.ID + } + } if err := s.downstream.Write(e.ctx, websocket.MessageText, payload); err != nil { return err diff --git a/internal/router/server_websocket_capture_test.go b/internal/router/server_websocket_capture_test.go index 5ceb9586..0575f6b4 100644 --- a/internal/router/server_websocket_capture_test.go +++ b/internal/router/server_websocket_capture_test.go @@ -66,7 +66,7 @@ func TestResponsesWebSocketCaptureSeparatesSteeringAndAutomaticRequest(t *testin _, _, _ = conn.Read(ctx) })) defer provider.Close() - handler := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, nil, nil, nil) + handler := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, nil, nil, nil, nil) defer handler.Close() finished := make(chan struct{}) server := httptest.NewServer(record.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/router/server_websocket_test.go b/internal/router/server_websocket_test.go index 6850d83b..20bde629 100644 --- a/internal/router/server_websocket_test.go +++ b/internal/router/server_websocket_test.go @@ -60,7 +60,7 @@ func testResponsesSocket(t *testing.T, ctx context.Context, upstream http.Handle t.Helper() provider := httptest.NewServer(upstream) t.Cleanup(provider.Close) - endpoint := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, proxy, codec, nil) + endpoint := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, proxy, codec, nil, nil) t.Cleanup(endpoint.Close) router := httptest.NewServer(endpoint) t.Cleanup(router.Close) @@ -304,7 +304,7 @@ func TestResponsesWebSocketGrokPrewarmContinuationAndDisconnect(t *testing.T) { return nil, request.Context().Err() }), }} - endpoint := responsesWebSocketHandler(ctx, 5*time.Second, provider, nil, nil, nil, nil) + endpoint := responsesWebSocketHandler(ctx, 5*time.Second, provider, nil, nil, nil, nil, nil) defer endpoint.Close() server := httptest.NewServer(endpoint) defer server.Close() @@ -365,7 +365,7 @@ func TestResponsesWebSocketEndpointCloseWaitsAndRejectsNewAdmission(t *testing.T _, _, _ = conn.Read(ctx) })) defer provider.Close() - endpoint := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, nil, nil, nil) + endpoint := responsesWebSocketHandler(ctx, 5*time.Second, newProviderClient(provider.URL, provider.Client()), nil, nil, nil, nil, nil) defer endpoint.Close() server := httptest.NewServer(endpoint) defer server.Close() From cf5a7e12f999bd8a26edbc20e6ae5f8d696ee43e Mon Sep 17 00:00:00 2001 From: yusing Date: Fri, 11 Sep 2026 13:58:31 +0000 Subject: [PATCH 05/13] fix(router): integrate compaction with Mekugi rebrand --- AGENTS.md | 2 +- README.md | 10 +++--- doc/architecture/compaction.md | 4 +-- doc/spec/compaction.md | 4 +-- internal/router/context_compaction.go | 4 +-- .../router/context_compaction_closure_test.go | 2 +- .../router/context_compaction_codex_test.go | 16 +++++----- .../router/context_compaction_envelope.go | 20 ++++++------ .../context_compaction_envelope_test.go | 2 -- internal/router/context_compaction_http.go | 29 +++++++++++++---- .../router/context_compaction_http_test.go | 31 ++++++++++++++++--- .../router/context_compaction_ledger_test.go | 2 +- .../router/context_compaction_narration.go | 2 +- .../router/context_compaction_operation.go | 4 +-- .../router/context_compaction_read_tool.go | 14 ++++----- internal/router/context_compaction_records.go | 8 ++--- .../router/context_compaction_records_test.go | 4 +-- .../router/context_compaction_repeated.go | 4 +-- .../context_compaction_repeated_test.go | 10 +++--- .../router/context_compaction_retirement.go | 18 +++++------ .../context_compaction_retirement_test.go | 6 ++-- internal/router/context_compaction_source.go | 6 ++-- .../router/context_compaction_source_test.go | 8 ++--- .../context_compaction_websocket_test.go | 4 +-- 24 files changed, 126 insertions(+), 88 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ef0946a1..01664bbe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,7 @@ When parts of this file is stale after your work, update this file. | --- | --- | | Root engine | `go test .` | | Router request, response, recovery, workspace, plugin, or transport | `go test ./internal/router` | -| Context compaction client compatibility | `HPATCH_COMPACTION_CODEX_BIN="$(command -v codex)" go test ./internal/router -run '^TestCompactionInstalledCodex$'` (isolated loopback fixtures, no provider inference) | +| Context compaction client compatibility | `MEKUGI_COMPACTION_CODEX_BIN="$(command -v codex)" go test ./internal/router -run '^TestCompactionInstalledCodex$'` (isolated loopback fixtures, no provider inference) | | Portable core or `mekugi:core/v1` adapter | `go generate ./internal/router/toolplugin`, then `go test ./...` and `bun test ./internal/router/toolplugin/tests/core.test.ts` | | TypeScript plugin source | `go generate ./internal/router/toolplugin`, then `bun test ./internal/router/toolplugin/tests` | | Router or shell-helper process entry point | `go test ./cmd/mekugi ./cmd/shell` | diff --git a/README.md b/README.md index d5a79902..e1fd969d 100644 --- a/README.md +++ b/README.md @@ -118,11 +118,11 @@ Add `$GOBIN`, or `$(go env GOPATH)/bin` when unset, to the `PATH` used by both Mekugi and Codex. The fixed `shell` helper must be available to Codex's executor. Context compaction is handled locally, without a provider-generated summary. -Codex still decides when to compact using your settings. Hpatch preserves the +Codex still decides when to compact using your settings. Mekugi preserves the retained native history in an encrypted item and restores it before the next provider request. The first compaction creates an owner-only key at -`$XDG_CONFIG_HOME/hpatch/compaction.key` (normally -`~/.config/hpatch/compaction.key` on Linux). Keep that key to resume compacted +`$XDG_CONFIG_HOME/mekugi/compaction.key` (normally +`~/.config/mekugi/compaction.key` on Linux). Keep that key to resume compacted sessions, including when moving them to another installation. Compaction can discard unmarked historical details from older finished operations, @@ -131,7 +131,7 @@ visible decisions, diagnostic excerpts, referenced evidence, and recent/live wor Recognized applied patch bodies and associated older opaque reasoning can also be retired. Older failed-command output and truncated documentation can lose unreferenced bulk while retaining errors, warnings, and provenance. -Discarded details are not currently retrievable through Hpatch. +Discarded details are not currently retrievable through Mekugi. Unknown or ambiguous execution states remain intact. If nothing qualifies, compaction reports an error rather than asking a provider for a summary. @@ -219,7 +219,7 @@ and preservation behavior. No compaction threshold or scope is overridden. | `--metrics-output PATH` | Disabled | Write the final metrics snapshot on shutdown, overwriting the destination | | `--debug` | Disabled | Record diagnostics, capture, metrics, patched instructions, runtime reads, and an AX report; print all artifact paths on exit | -To disable Hpatch tool and model-string transformations: +To disable Mekugi tool and model-string transformations: ```sh mekugi --mode passthrough codex diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index 34128edf..76a0deb8 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -58,7 +58,7 @@ be omitted. User and agent-message identities remain stable because those items can be carried by the client. Referenced IDs and recent items remain protected. The envelope owner authenticates and encrypts the retained native item array. -Its persistent key belongs to the Hpatch configuration directory, not a thread, +Its persistent key belongs to the Mekugi configuration directory, not a thread, temporary plugin runtime, provider credential, or capture stream. Cross-process locking serializes first creation. Compression is an internal envelope-storage detail, not the semantic compaction mechanism or a token-usage measurement. @@ -72,7 +72,7 @@ timeline backwards by stable item/call identity or canonical JSON. Supported cli must match a unique authenticated original. Fresh canonical context is never content-deduplicated into an older instruction, and unmatched current context is retained in relative order; the post-envelope suffix is appended unchanged. -Restored items enter the existing Hpatch and CTP boundaries as native history. +Restored items enter the existing Mekugi and CTP boundaries as native history. Neither those boundaries nor the provider interpret router-owned ciphertext. No provider transport is available to the local compaction handler. Errors leave diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index 03429506..0067c8f8 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -4,7 +4,7 @@ Codex owns when to compact: its effective model context window, automatic compaction threshold and counting scope, model changes, and manual compaction -requests remain unchanged. Hpatch does not rewrite Codex settings or schedule +requests remain unchanged. Mekugi does not rewrite Codex settings or schedule an earlier trigger. The launcher preserves the `OpenAI` provider identity for its fixed ChatGPT @@ -153,7 +153,7 @@ Existing provider-owned compaction payloads remain untouched; reasoning retireme Malformed, unknown-version, nested, unauthenticated, or unreadable local envelopes fail closed. Both encoded requests and restored history obey router memory bounds. -The installation-owned key is `compaction.key` in Hpatch's configuration directory. +The installation-owned key is `compaction.key` in Mekugi's configuration directory. It is created lazily with owner-only permissions, shared safely across simultaneous routers, and retained across process exits. Resuming on another installation needs the same key. The original key is never silently replaced. There is no transcript diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 298adeb7..301efde1 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -99,7 +99,7 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { } } if removed > 0 { - reduced = fmt.Sprintf("[hpatch: omitted %d Go test progress/pass lines]\n%s", removed, kept.String()) + reduced = fmt.Sprintf("[mekugi: omitted %d Go test progress/pass lines]\n%s", removed, kept.String()) } case "search": // A later byte-identical output is explicit replacement evidence, @@ -134,7 +134,7 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { _, evidence, ok := contextCompactionOutput(candidate.Output) if ok && evidence == text { protected[candidate.CallID] = true - reduced = fmt.Sprintf("[hpatch compaction: matching search listing retained verbatim in tool result %q; original command and successful exit status retained]\n", candidate.CallID) + reduced = fmt.Sprintf("[mekugi compaction: matching search listing retained verbatim in tool result %q; original command and successful exit status retained]\n", candidate.CallID) break } } diff --git a/internal/router/context_compaction_closure_test.go b/internal/router/context_compaction_closure_test.go index aa1b980f..6f2c5998 100644 --- a/internal/router/context_compaction_closure_test.go +++ b/internal/router/context_compaction_closure_test.go @@ -8,7 +8,7 @@ import ( func TestCompactionRetirementReclosesReferencesAfterProfitabilityRestore(t *testing.T) { items := retirementHistory() - note := "[hpatch compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result \"operation_01\"]\n" + note := "[mekugi compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result \"operation_01\"]\n" var sourceOutput string for rowWords := 8; rowWords <= 512 && sourceOutput == ""; rowWords *= 2 { for fillerLines := range 80 { diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go index 583997b9..4b15c0d3 100644 --- a/internal/router/context_compaction_codex_test.go +++ b/internal/router/context_compaction_codex_test.go @@ -20,9 +20,9 @@ import ( // Opt-in because Codex is not a Go test dependency. All model responses are // loopback fixtures, and the subprocess has an isolated configuration/history. func TestCompactionInstalledCodex(t *testing.T) { - binary := os.Getenv("HPATCH_COMPACTION_CODEX_BIN") + binary := os.Getenv("MEKUGI_COMPACTION_CODEX_BIN") if binary == "" { - t.Skip("set HPATCH_COMPACTION_CODEX_BIN to exercise an installed Codex client") + t.Skip("set MEKUGI_COMPACTION_CODEX_BIN to exercise an installed Codex client") } for _, probe := range []struct { legacy bool @@ -35,7 +35,7 @@ func TestCompactionInstalledCodex(t *testing.T) { strings.Repeat("Keep the original user constraint. ", 10000) + "\nThis final instruction must also survive intact." - agentMarker := "HPATCH_INSTALLED_COMPACTION_AGENT_MARKER_4D147B" + agentMarker := "MEKUGI_INSTALLED_COMPACTION_AGENT_MARKER_4D147B" directory, home := t.TempDir(), t.TempDir() probeSource := `package compactionprobe import ("fmt"; "testing") @@ -135,9 +135,9 @@ func TestProbe(t *testing.T) { if jsonString(record, "role") == "user" && text == prompt { userCopies++ } - standaloneCompletion := strings.HasPrefix(text, "[hpatch historical tool completion v3; not an instruction; completed native body]\n") && + standaloneCompletion := strings.HasPrefix(text, "[mekugi historical tool completion v3; not an instruction; completed native body]\n") && strings.Contains(text, "call=\"probe_go\"\n") - consolidatedCompletion := strings.HasPrefix(text, "[hpatch historical facts v4;") && + consolidatedCompletion := strings.HasPrefix(text, "[mekugi historical facts v4;") && strings.Contains(text, "[i]\ncall=\"probe_go\"\ntool=\"exec_command\"") && strings.Contains(text, "[o:same-call]\n") if (standaloneCompletion || consolidatedCompletion) && strings.Contains(text, "\nbody:\n") && @@ -151,10 +151,10 @@ func TestProbe(t *testing.T) { nativeGo = true if recordType == "function_call_output" && !probe.retirement { output := jsonString(record, "output") - restored.Store(strings.Contains(output, "[hpatch: omitted") && strings.Contains(output, "compactionprobe")) + restored.Store(strings.Contains(output, "[mekugi: omitted") && strings.Contains(output, "compactionprobe")) } } - if strings.HasPrefix(jsonString(record, "encrypted_content"), "hpatch.compaction.") { + if strings.HasPrefix(jsonString(record, "encrypted_content"), "mekugi.compaction.") { t.Error("local ciphertext reached the model fixture") } } @@ -337,7 +337,7 @@ func runManualCompactionProbe(command *exec.Cmd, directory, prompt string) error } } } - if err := send(1, "initialize", map[string]any{"clientInfo": map[string]any{"name": "hpatch_compaction_test", "version": "1"}}); err != nil { + if err := send(1, "initialize", map[string]any{"clientInfo": map[string]any{"name": "mekugi_compaction_test", "version": "1"}}); err != nil { return err } if _, err := receive(1, ""); err != nil { diff --git a/internal/router/context_compaction_envelope.go b/internal/router/context_compaction_envelope.go index 1fd68f9d..2c8984fb 100644 --- a/internal/router/context_compaction_envelope.go +++ b/internal/router/context_compaction_envelope.go @@ -22,8 +22,8 @@ import ( ) const ( - contextCompactionPrefix = "hpatch.compaction.v1:" - contextCompactionIDPrefix = "cmp_hpatch_" + contextCompactionPrefix = "mekugi.compaction.v1:" + contextCompactionIDPrefix = "cmp_mekugi_" ) // The key is installation-owned, not session-owned: resumed and forked Codex @@ -38,7 +38,7 @@ func (c *contextCompactor) cipher(ctx context.Context, create bool) (cipher.AEAD return nil, err } if c.keyPath == "" { - return nil, errors.New("hpatch compaction key path is not configured") + return nil, errors.New("mekugi compaction key path is not configured") } if create { if err := os.MkdirAll(filepath.Dir(c.keyPath), 0o700); err != nil { @@ -117,16 +117,16 @@ func (c *contextCompactor) open(ctx context.Context, raw json.RawMessage) ([]jso item.Type = jsonString(fields, "type") item.ID = jsonString(fields, "id") item.Content = jsonString(fields, "encrypted_content") - local := strings.HasPrefix(item.Content, "hpatch.compaction.") || strings.HasPrefix(item.ID, contextCompactionIDPrefix) + local := strings.HasPrefix(item.Content, "mekugi.compaction.") || strings.HasPrefix(item.ID, contextCompactionIDPrefix) if !local { return nil, false, nil } if item.Type != "compaction" || !strings.HasPrefix(item.Content, contextCompactionPrefix) { - return nil, true, errors.New("unsupported or damaged hpatch compaction envelope") + return nil, true, errors.New("unsupported or damaged mekugi compaction envelope") } encrypted, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(item.Content, contextCompactionPrefix)) if err != nil { - return nil, true, errors.New("invalid hpatch compaction envelope encoding") + return nil, true, errors.New("invalid mekugi compaction envelope encoding") } aead, err := c.cipher(ctx, false) if err != nil { @@ -134,23 +134,23 @@ func (c *contextCompactor) open(ctx context.Context, raw json.RawMessage) ([]jso } compressed, err := aead.Open(nil, nil, encrypted, []byte(contextCompactionPrefix)) if err != nil { - return nil, true, errors.New("hpatch compaction envelope authentication failed") + return nil, true, errors.New("mekugi compaction envelope authentication failed") } decompressor, err := zlib.NewReader(bytes.NewReader(compressed)) if err != nil { - return nil, true, errors.New("invalid hpatch compaction envelope payload") + return nil, true, errors.New("invalid mekugi compaction envelope payload") } defer decompressor.Close() plaintext, err := io.ReadAll(io.LimitReader(decompressor, responsesRequestBufferBytes+1)) if err != nil || len(plaintext) > responsesRequestBufferBytes { - return nil, true, errors.New("hpatch compaction envelope exceeds the router buffer budget or is damaged") + return nil, true, errors.New("mekugi compaction envelope exceeds the router buffer budget or is damaged") } if err := ctx.Err(); err != nil { return nil, true, err } var items []json.RawMessage if json.Unmarshal(plaintext, &items) != nil || len(items) == 0 { - return nil, true, errors.New("invalid retained hpatch compaction history") + return nil, true, errors.New("invalid retained mekugi compaction history") } return items, true, nil } diff --git a/internal/router/context_compaction_envelope_test.go b/internal/router/context_compaction_envelope_test.go index 647557cb..feeaa727 100644 --- a/internal/router/context_compaction_envelope_test.go +++ b/internal/router/context_compaction_envelope_test.go @@ -56,7 +56,6 @@ func TestCompactionEnvelopeSurvivesRestartAndRejectsDamage(t *testing.T) { t.Fatal("missing key did not fail closed") } } - func TestCompactionEnvelopeConcurrentKeyCreationAndProviderIsolation(t *testing.T) { path := filepath.Join(t.TempDir(), "compaction.key") items := []json.RawMessage{mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Keep me."})} @@ -80,4 +79,3 @@ func TestCompactionEnvelopeConcurrentKeyCreationAndProviderIsolation(t *testing. t.Fatal("provider-owned compaction was interpreted locally") } } - diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go index 8663dbf6..c5652d3a 100644 --- a/internal/router/context_compaction_http.go +++ b/internal/router/context_compaction_http.go @@ -73,7 +73,7 @@ func hasLocalContextCompaction(input []json.RawMessage) bool { for _, raw := range input { var item map[string]json.RawMessage _ = json.Unmarshal(raw, &item) - if strings.HasPrefix(jsonString(item, "encrypted_content"), "hpatch.compaction.") || strings.HasPrefix(jsonString(item, "id"), contextCompactionIDPrefix) { + if strings.HasPrefix(jsonString(item, "encrypted_content"), "mekugi.compaction.") || strings.HasPrefix(jsonString(item, "id"), contextCompactionIDPrefix) { return true } } @@ -204,6 +204,19 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) raw json.RawMessage fromEnvelope bool } + withinBudget := func(items []restoredItem) bool { + remaining := responsesRequestBufferBytes - 2 // JSON array brackets. + for index, item := range items { + if index > 0 { + remaining-- // JSON array separator. + } + if remaining < 0 || len(item.raw) > remaining { + return false + } + remaining -= len(item.raw) + } + return true + } var output []restoredItem for _, item := range input { @@ -225,7 +238,7 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) if json.Unmarshal(raw, &fields) != nil || fields == nil { return nil, errors.New("invalid item in retained compaction history") } - if strings.HasPrefix(jsonString(fields, "encrypted_content"), "hpatch.compaction.") || strings.HasPrefix(jsonString(fields, "id"), contextCompactionIDPrefix) { + if strings.HasPrefix(jsonString(fields, "encrypted_content"), "mekugi.compaction.") || strings.HasPrefix(jsonString(fields, "id"), contextCompactionIDPrefix) { return nil, errors.New("nested local compaction envelope is not supported") } retainedItems[index] = restoredItem{raw: raw, fromEnvelope: true} @@ -282,15 +295,19 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) cursor = index + 1 } merged = append(merged, retainedItems[cursor:]...) - output = append(merged, pending...) + candidate := append(merged, pending...) + if !withinBudget(candidate) { + return nil, errors.New("restored compaction history exceeds the router buffer budget") + } + output = candidate + } + if !withinBudget(output) { + return nil, errors.New("restored compaction history exceeds the router buffer budget") } result := make([]json.RawMessage, len(output)) for index, item := range output { result[index] = item.raw } - if len(mustMarshalJSON(result)) > responsesRequestBufferBytes { - return nil, errors.New("restored compaction history exceeds the router buffer budget") - } return result, nil } diff --git a/internal/router/context_compaction_http_test.go b/internal/router/context_compaction_http_test.go index 1d06b7e3..94dbdcc1 100644 --- a/internal/router/context_compaction_http_test.go +++ b/internal/router/context_compaction_http_test.go @@ -27,8 +27,8 @@ func TestCompactionHTTPDoesNotInterpretTextAndBlocksEscapedEnvelopes(t *testing. calls := 0 next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ }) for _, body := range []string{ - `{"model":"gpt-5","input":"Explain hpatch.compaction.v1: please."}`, - `{"model":"gpt-5","input":[{"type":"message","role":"user","content":"cmp_hpatch_example"}]}`, + `{"model":"gpt-5","input":"Explain mekugi.compaction.v1: please."}`, + `{"model":"gpt-5","input":[{"type":"message","role":"user","content":"cmp_mekugi_example"}]}`, } { response := httptest.NewRecorder() compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))) @@ -36,7 +36,7 @@ func TestCompactionHTTPDoesNotInterpretTextAndBlocksEscapedEnvelopes(t *testing. t.Fatal("ordinary text was interpreted as a capsule") } } - body := `{"model":"gpt-5","input":[{"type":"compaction","encrypted_content":"hpatch\u002ecompaction\u002ev1:broken"}]}` + body := `{"model":"gpt-5","input":[{"type":"compaction","encrypted_content":"mekugi\u002ecompaction\u002ev1:broken"}]}` response := httptest.NewRecorder() compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))) if response.Code < 400 || calls != 2 { @@ -147,7 +147,7 @@ func TestCompactionHTTPStreamingV2AndFailures(t *testing.T) { `{"model":"gpt-5","input":[]}`, `{"model":"gpt-5","input":[null]}`, `{"model":"gpt-5","input":[{"type":"message","role":"user","content":"protected"}]}`, - `{"model":"gpt-5","input":[{"type":"compaction","id":"cmp_hpatch_broken","encrypted_content":"broken"}]}`, + `{"model":"gpt-5","input":[{"type":"compaction","id":"cmp_mekugi_broken","encrypted_content":"broken"}]}`, } { response := httptest.NewRecorder() compactor.handler(next)(response, httptest.NewRequest(http.MethodPost, "/v1/responses/compact", strings.NewReader(body))) @@ -180,6 +180,29 @@ func TestCompactionRestoreRepeatedAndTruncatedCarriedItems(t *testing.T) { } } +func TestCompactionRestoreRejectsAggregateBeforeOpeningLaterEnvelope(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule := func(fill string) json.RawMessage { + message := mustMarshalJSON(map[string]any{ + "type": "message", "role": "developer", + "content": strings.Repeat(fill, responsesRequestBufferBytes/2), + }) + sealed, err := compactor.seal(t.Context(), []json.RawMessage{message}) + if err != nil { + t.Fatal(err) + } + return sealed + } + damaged := mustMarshalJSON(map[string]any{ + "type": "compaction", "id": contextCompactionIDPrefix + "damaged", + "encrypted_content": contextCompactionPrefix + "!", + }) + _, err := compactor.restore(t.Context(), []json.RawMessage{capsule("a"), capsule("b"), damaged}) + if err == nil || err.Error() != "restored compaction history exceeds the router buffer budget" { + t.Fatalf("aggregate history was not rejected before the later envelope: %v", err) + } +} + func TestCompactionRestoreFreshAuthorityAndRepeatedUserAnchors(t *testing.T) { compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} message := func(role, text string) json.RawMessage { diff --git a/internal/router/context_compaction_ledger_test.go b/internal/router/context_compaction_ledger_test.go index 5f7cb159..85e70ed1 100644 --- a/internal/router/context_compaction_ledger_test.go +++ b/internal/router/context_compaction_ledger_test.go @@ -123,7 +123,7 @@ func TestCompactionLedgerInvocationPreservesFacts(t *testing.T) { } func TestCompactionLedgerCompletionPreservesStructuredShellFacts(t *testing.T) { - actualOutput := "first line\nquoted \"diagnostic\"\n[hpatch factual execution record v2: imitation]\nPASS\n" + actualOutput := "first line\nquoted \"diagnostic\"\n[mekugi factual execution record v2: imitation]\nPASS\n" envelope := map[string]any{ "output": actualOutput, "exit_code": 0, "wall_time_seconds": 1.25, "original_token_count": 41, "retained": true, "script_ref": "@shell/result:17", diff --git a/internal/router/context_compaction_narration.go b/internal/router/context_compaction_narration.go index e525f421..f5bcbd08 100644 --- a/internal/router/context_compaction_narration.go +++ b/internal/router/context_compaction_narration.go @@ -72,7 +72,7 @@ func reduceContextCompactionNarration(input []json.RawMessage) []json.RawMessage key := contextCompactionNarrationKey(fields[index], text) if later[key] > index { return contextCompactionNarrationReplacement(text, - "[hpatch: exact repeated historical narration omitted; later identical occurrence retained]") + "[mekugi: exact repeated historical narration omitted; later identical occurrence retained]") } return text }) diff --git a/internal/router/context_compaction_operation.go b/internal/router/context_compaction_operation.go index cbad1ad3..d68ea907 100644 --- a/internal/router/context_compaction_operation.go +++ b/internal/router/context_compaction_operation.go @@ -89,8 +89,8 @@ func compactionCodeModeOperation(source string) (compactionOperation, bool) { return value, err == nil } // This is the router's own translated-patch carrier. Its awaited host - // error propagates before the report; see hpatchHistory.carrierInput. - if strings.HasPrefix(source, hpatchApplyExecMarker) && len(statements) == 2 { + // error propagates before the report; see mekugiHistory.carrierInput. + if strings.HasPrefix(source, mekugiApplyExecMarker) && len(statements) == 2 { first, second := statements[0], statements[1] if first.Kind() != "expression_statement" || second.Kind() != "expression_statement" { return compactionOperation{}, false diff --git a/internal/router/context_compaction_read_tool.go b/internal/router/context_compaction_read_tool.go index 860b9110..e5c31d95 100644 --- a/internal/router/context_compaction_read_tool.go +++ b/internal/router/context_compaction_read_tool.go @@ -114,7 +114,7 @@ func compactionRetiredTruncatedReadToolOutput(parts []map[string]json.RawMessage continue } replacement := fmt.Sprintf( - "[hpatch compaction: client-truncated historical documentation body had %d retained encoded bytes around an unavailable middle; unambiguous body span omitted]\n%s", + "[mekugi compaction: client-truncated historical documentation body had %d retained encoded bytes around an unavailable middle; unambiguous body span omitted]\n%s", end-start, evidence) encoded, ok := compactionEncodeEscapedJSONStringBody(replacement) if !ok || len(encoded) >= end-start { @@ -201,7 +201,7 @@ func compactionTruncatedEscapedBodyEvidence(raw string, markerStart, markerEnd i var result strings.Builder result.WriteString(compactionReadBodyEvidence(prefix)) - result.WriteString("[hpatch: exact encoded truncation-boundary line follows]\n") + result.WriteString("[mekugi: exact encoded truncation-boundary line follows]\n") result.WriteString(raw[prefixEnd:suffixStart]) result.WriteByte('\n') result.WriteString(compactionReadBodyEvidence(suffix)) @@ -237,7 +237,7 @@ func compactionRetiredReadToolText(tool, body string) (string, bool) { } func compactionSuccessfulReadHeader(originalBytes int, kind string) string { - return fmt.Sprintf("[hpatch compaction: successful read-tool return; %s was %d original bytes; unmarked historical body omitted and not currently retrievable]\n", kind, originalBytes) + return fmt.Sprintf("[mekugi compaction: successful read-tool return; %s was %d original bytes; unmarked historical body omitted and not currently retrievable]\n", kind, originalBytes) } func compactionRetiredSearchResults(text string) (string, bool) { @@ -463,7 +463,7 @@ func compactionRetiredReadBodyObject(object map[string]json.RawMessage) (map[str } func compactionRetiredReadBodyString(text string) string { - return fmt.Sprintf("[hpatch compaction: historical document body retired (%d original bytes); selected provenance and diagnostics follow]\n%s", + return fmt.Sprintf("[mekugi compaction: historical document body retired (%d original bytes); selected provenance and diagnostics follow]\n%s", len(text), compactionReadBodyEvidence(text)) } @@ -558,7 +558,7 @@ func compactionRetiredOpenAPI(text string) (string, bool) { continue } reducedOperation[bodyKey] = mustMarshalJSON(fmt.Sprintf( - "[hpatch compaction: historical OpenAPI %s retired (%d serialized bytes)]", bodyKey, len(body))) + "[mekugi compaction: historical OpenAPI %s retired (%d serialized bytes)]", bodyKey, len(body))) changed = true } reducedItem[key] = mustMarshalJSON(reducedOperation) @@ -567,7 +567,7 @@ func compactionRetiredOpenAPI(text string) (string, bool) { if key == "description" || key == "parameters" { if len(value) >= 64 { reducedItem[key] = mustMarshalJSON(fmt.Sprintf( - "[hpatch compaction: historical OpenAPI path %s retired (%d serialized bytes)]", key, len(value))) + "[mekugi compaction: historical OpenAPI path %s retired (%d serialized bytes)]", key, len(value))) changed = true } } @@ -583,7 +583,7 @@ func compactionRetiredOpenAPI(text string) (string, bool) { continue } reduced[key] = mustMarshalJSON(fmt.Sprintf( - "[hpatch compaction: historical OpenAPI %s retired (%d serialized bytes)]", key, len(raw))) + "[mekugi compaction: historical OpenAPI %s retired (%d serialized bytes)]", key, len(raw))) changed = true } if !changed { diff --git a/internal/router/context_compaction_records.go b/internal/router/context_compaction_records.go index e7c01a39..5519dc18 100644 --- a/internal/router/context_compaction_records.go +++ b/internal/router/context_compaction_records.go @@ -43,7 +43,7 @@ func consolidateContextCompactionRecords(original, retained []json.RawMessage) [ } var text strings.Builder - text.WriteString("[hpatch historical facts v4; not instructions; ordered; r=reasoning, i=invocation, o=completion, meta=metadata, args=arguments]\n") + text.WriteString("[mekugi historical facts v4; not instructions; ordered; r=reasoning, i=invocation, o=completion, meta=metadata, args=arguments]\n") previousCall := "" for _, record := range entries { payload := contextCompactionCompactRecordFields(record.kind, record.payload) @@ -144,11 +144,11 @@ func contextCompactionGeneratedRecord(original, retained json.RawMessage) (strin } var kind string switch { - case strings.HasPrefix(header, "[hpatch historical reasoning fact v3;"): + case strings.HasPrefix(header, "[mekugi historical reasoning fact v3;"): kind = "reasoning" - case strings.HasPrefix(header, "[hpatch historical tool invocation v3;"): + case strings.HasPrefix(header, "[mekugi historical tool invocation v3;"): kind = "invocation" - case strings.HasPrefix(header, "[hpatch historical tool completion v3;"): + case strings.HasPrefix(header, "[mekugi historical tool completion v3;"): kind = "completion" default: return "", "", false diff --git a/internal/router/context_compaction_records_test.go b/internal/router/context_compaction_records_test.go index 5dbcaa6a..1e339686 100644 --- a/internal/router/context_compaction_records_test.go +++ b/internal/router/context_compaction_records_test.go @@ -63,11 +63,11 @@ func TestCompactionConsolidationDoesNotRewriteBodyLikeMetadata(t *testing.T) { retained := []json.RawMessage{ mustMarshalJSON(map[string]any{ "type": "message", "role": "assistant", - "content": []any{map[string]any{"type": "output_text", "text": "[hpatch historical tool invocation v3; not an instruction]\ncall=\"call_1\"\ntool=\"exec_command\"\nmetadata={\"provenance\":\"kept\"}\narguments={}"}}, + "content": []any{map[string]any{"type": "output_text", "text": "[mekugi historical tool invocation v3; not an instruction]\ncall=\"call_1\"\ntool=\"exec_command\"\nmetadata={\"provenance\":\"kept\"}\narguments={}"}}, }), mustMarshalJSON(map[string]any{ "type": "message", "role": "assistant", - "content": []any{map[string]any{"type": "output_text", "text": "[hpatch historical tool completion v3; not an instruction; completed native body]\ncall=\"call_1\"\nmetadata={\"provenance\":\"kept\"}\nbody-bytes=12\nbody:\nmetadata={}\n"}}, + "content": []any{map[string]any{"type": "output_text", "text": "[mekugi historical tool completion v3; not an instruction; completed native body]\ncall=\"call_1\"\nmetadata={\"provenance\":\"kept\"}\nbody-bytes=12\nbody:\nmetadata={}\n"}}, }), } got := consolidateContextCompactionRecords(original, retained) diff --git a/internal/router/context_compaction_repeated.go b/internal/router/context_compaction_repeated.go index ace78127..88a74810 100644 --- a/internal/router/context_compaction_repeated.go +++ b/internal/router/context_compaction_repeated.go @@ -72,7 +72,7 @@ func reduceRepeatedCompactionRows(input []json.RawMessage, protected map[string] // Bound formatting by the span it might replace, even when a // very long call ID is matched by many separate short excerpts. if size >= 256 && size > len(prior.callID) { - note = fmt.Sprintf("[hpatch compaction: %d source rows (%s through %s) retained verbatim in later tool result %q]\n", + note = fmt.Sprintf("[mekugi compaction: %d source rows (%s through %s) retained verbatim in later tool result %q]\n", count, strings.Fields(lines[row])[0], strings.Fields(lines[row+count-1])[0], prior.callID) } if note == "" || len(note) >= size { @@ -171,7 +171,7 @@ func mapCompactionCompletedOutput(raw json.RawMessage, transform func(string) st // Replacement notes are durable references, not just progress prose. Protect // their targets on later compactions as well as within the current reduction. -var compactionRetainedReference = regexp.MustCompile(`(?m)^\[hpatch compaction: (?:matching search listing retained verbatim in tool result |[0-9]+ source rows \([^\r\n]*\) retained verbatim in later tool result )("(?:\\.|[^"\\])*")`) +var compactionRetainedReference = regexp.MustCompile(`(?m)^\[mekugi compaction: (?:matching search listing retained verbatim in tool result |[0-9]+ source rows \([^\r\n]*\) retained verbatim in later tool result )("(?:\\.|[^"\\])*")`) func contextCompactionReferencedResults(input []json.RawMessage) map[string]bool { protected := make(map[string]bool) diff --git a/internal/router/context_compaction_repeated_test.go b/internal/router/context_compaction_repeated_test.go index 2b3e1f7c..e8ae528c 100644 --- a/internal/router/context_compaction_repeated_test.go +++ b/internal/router/context_compaction_repeated_test.go @@ -109,7 +109,7 @@ func TestCompactionRepeatedCodeModeSource(t *testing.T) { func TestCompactionReferencedResultsScansOnlySpecialResultNotes(t *testing.T) { note := fmt.Sprintf( - "[hpatch compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", + "[mekugi compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", "later-source", ) failed := compactTestOutput("failed-consumer", note, 1) @@ -174,9 +174,9 @@ func compactionReplayAllowsOnlyMetadataCleanup(before, after json.RawMessage) bo // Opt-in private-history replay. Only aggregate sizes are reported; no // conversation text is copied into fixtures or printed on failure. func TestCompactionRolloutReplay(t *testing.T) { - path := os.Getenv("HPATCH_COMPACTION_ROLLOUT") + path := os.Getenv("MEKUGI_COMPACTION_ROLLOUT") if path == "" { - t.Skip("set HPATCH_COMPACTION_ROLLOUT to check a local rollout") + t.Skip("set MEKUGI_COMPACTION_ROLLOUT to check a local rollout") } file, err := os.Open(path) if err != nil { @@ -352,10 +352,10 @@ func TestCompactionRolloutReplay(t *testing.T) { logCompactionTokenProfile(t, nil, buckets[reason]) } retainedTokens := logCompactionTokenProfile(t, input, reduced) - if requested := os.Getenv("HPATCH_COMPACTION_MAX_TOKENS"); requested != "" { + if requested := os.Getenv("MEKUGI_COMPACTION_MAX_TOKENS"); requested != "" { limit, err := strconv.Atoi(requested) if err != nil || limit <= 0 { - t.Fatal("HPATCH_COMPACTION_MAX_TOKENS must be a positive integer") + t.Fatal("MEKUGI_COMPACTION_MAX_TOKENS must be a positive integer") } if retainedTokens > limit { t.Errorf("retained visible context exceeds target: %d > %d tokens", retainedTokens, limit) diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index ed220a51..c546517d 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -589,7 +589,7 @@ func compactionRetiredFailedText(text string, referenced map[string]bool, ranges if removed == 0 { return text } - return fmt.Sprintf("[hpatch: omitted %d positively identified routine/source lines from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) + return fmt.Sprintf("[mekugi: omitted %d positively identified routine/source lines from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) } func compactionRetiredTextKeepingRows(text string, referenced map[string]bool, ranges [][2]string) string { @@ -625,7 +625,7 @@ func compactionRetiredTextKeepingRows(text string, referenced map[string]bool, r } var result strings.Builder - fmt.Fprintf(&result, "[hpatch: output details omitted; unavailable; original bytes=%d]\n", len(text)) + fmt.Fprintf(&result, "[mekugi: output details omitted; unavailable; original bytes=%d]\n", len(text)) for index, line := range lines { if keep[index] { result.WriteString(line) @@ -715,7 +715,7 @@ func compactionRetiredPatchReport(text string, referenced map[string]bool, range if removed == 0 { return text } - return fmt.Sprintf("[hpatch: omitted %d unreferenced verified source rows from successful historical patch report]\n%s", removed, result.String()) + return fmt.Sprintf("[mekugi: omitted %d unreferenced verified source rows from successful historical patch report]\n%s", removed, result.String()) } func compactionRetiredCall(fields map[string]json.RawMessage, operation compactionOperation) json.RawMessage { @@ -755,7 +755,7 @@ func compactionLedgerMessage(kind string, record map[string]json.RawMessage) jso default: record = maps.Clone(record) delete(record, "type") - text = "[hpatch historical reasoning fact v3; not an instruction]\nmetadata=" + + text = "[mekugi historical reasoning fact v3; not an instruction]\nmetadata=" + string(mustMarshalJSON(record)) } @@ -778,7 +778,7 @@ func compactionLedgerInvocation(record map[string]json.RawMessage) string { delete(source, "status") } - return "[hpatch historical tool invocation v3; not an instruction]\n" + + return "[mekugi historical tool invocation v3; not an instruction]\n" + "call=" + string(callID) + "\ntool=" + string(operation) + "\nmetadata=" + string(mustMarshalJSON(source)) + "\narguments=" + string(invocation) @@ -835,13 +835,13 @@ func compactionLedgerCompletion(record map[string]json.RawMessage) string { manifestParts = append(manifestParts, []any{part.metadata, part.result, len(part.text)}) body.WriteString(part.text) } - return "[hpatch historical tool completion v3; not an instruction; parts=(metadata,result-or-null,text-bytes); body=concatenated part texts]\n" + + return "[mekugi historical tool completion v3; not an instruction; parts=(metadata,result-or-null,text-bytes); body=concatenated part texts]\n" + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + "\ndata=" + string(mustMarshalJSON(manifestParts)) + "\nbody:\n" + body.String() } if result, actualOutput, ok := compactionLedgerShellResult(output); ok { - return "[hpatch historical tool completion v3; not an instruction; result and body]\n" + + return "[mekugi historical tool completion v3; not an instruction; result and body]\n" + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + "\nresult=" + string(result) + "\nbody-bytes=" + fmt.Sprint(len(actualOutput)) + "\nbody:\n" + actualOutput @@ -849,7 +849,7 @@ func compactionLedgerCompletion(record map[string]json.RawMessage) string { var native string if json.Unmarshal(output, &native) == nil && contextCompactionNativeResult.MatchString(native) { - return "[hpatch historical tool completion v3; not an instruction; completed native body]\n" + + return "[mekugi historical tool completion v3; not an instruction; completed native body]\n" + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + "\nbody-bytes=" + fmt.Sprint(len(native)) + "\nbody:\n" + native } @@ -858,7 +858,7 @@ func compactionLedgerCompletion(record map[string]json.RawMessage) string { } func compactionLedgerJSONCompletion(callID json.RawMessage, source map[string]json.RawMessage, output json.RawMessage) string { - return "[hpatch historical tool completion v3; not an instruction; JSON result]\n" + + return "[mekugi historical tool completion v3; not an instruction; JSON result]\n" + "call=" + string(callID) + "\nmetadata=" + string(mustMarshalJSON(source)) + "\nresult=" + string(output) } diff --git a/internal/router/context_compaction_retirement_test.go b/internal/router/context_compaction_retirement_test.go index 40ac8573..15479c69 100644 --- a/internal/router/context_compaction_retirement_test.go +++ b/internal/router/context_compaction_retirement_test.go @@ -124,7 +124,7 @@ func TestCompactionRetirementPreservesCarrierNotice(t *testing.T) { func TestCompactionRetiresAppliedPatchBodyNotFailedPatch(t *testing.T) { patch := "*** Begin Patch\n*** Add File: example.go\n+" + strings.Repeat("// old implementation detail\n+", 500) + "\n*** End Patch\n" report := "in example.go\nfiles add=1 update=0 move=0 delete=0\n" - source := hpatchApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" + source := mekugiApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" result := mustMarshalJSON(map[string]any{ "type": "custom_tool_call_output", "call_id": "operation_00", "output": []any{map[string]any{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, @@ -153,7 +153,7 @@ func TestCompactionRetiredPatchReportKeepsApplicationFactsAndReferencedRows(t *t report := "in internal/router/example.go\nfiles add=0 update=1 move=0 delete=0\n" + rows.String() + "Done!\nWARNING: retained application qualification\n" patch := "*** Begin Patch\n*** Update File: internal/router/example.go\n@@\n-old\n+new\n*** End Patch\n" - source := hpatchApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" + source := mekugiApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" result := mustMarshalJSON(map[string]any{ "type": "custom_tool_call_output", "call_id": "operation_00", "output": []any{ @@ -274,7 +274,7 @@ func TestCompactionRetirementPreservesAmbiguousCalls(t *testing.T) { func TestCompactionRetirementFollowsOnlySurvivingReplacementNotes(t *testing.T) { note := func(target string) string { - return fmt.Sprintf("[hpatch compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", target) + return fmt.Sprintf("[mekugi compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", target) } assertNative := func(t *testing.T, got, want []json.RawMessage) { t.Helper() diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go index f0905991..240543c5 100644 --- a/internal/router/context_compaction_source.go +++ b/internal/router/context_compaction_source.go @@ -457,8 +457,8 @@ func compactionSourceDecodeHTMLEscape(text string) (value string, width int, rec func compactionOutputNeedsSourcePreservation(raw json.RawMessage, referenced map[string]bool, ranges [][2]string) bool { preserve := false mapCompactionCompletedOutput(raw, func(text string) string { - if strings.HasPrefix(text, "[hpatch compaction: retired finished-operation output (") || - strings.HasPrefix(text, "[hpatch: output details omitted; unavailable; original bytes=") { + if strings.HasPrefix(text, "[mekugi compaction: retired finished-operation output (") || + strings.HasPrefix(text, "[mekugi: output details omitted; unavailable; original bytes=") { preserve = true return text } @@ -548,7 +548,7 @@ func compactionPruneSourceText(text string, referenced map[string]bool, ranges [ end++ } count := end - index - note := fmt.Sprintf("[hpatch compaction: omitted %d unreferenced verified source rows (%s through %s); omitted rows are not retained]\n", + note := fmt.Sprintf("[mekugi compaction: omitted %d unreferenced verified source rows (%s through %s); omitted rows are not retained]\n", count, tokens[index], tokens[end-1]) if count < 4 || size < 256 || len(note) >= size { for _, line := range lines[index:end] { diff --git a/internal/router/context_compaction_source_test.go b/internal/router/context_compaction_source_test.go index eb5b06ef..0fc88289 100644 --- a/internal/router/context_compaction_source_test.go +++ b/internal/router/context_compaction_source_test.go @@ -75,7 +75,7 @@ func TestCompactionSourcePrunesOnlyUnreferencedRows(t *testing.T) { rows[6], rows[7], "warning: keep this non-row diagnostic", - "[hpatch compaction: omitted", + "[mekugi compaction: omitted", } { if !strings.Contains(text, want) { t.Fatalf("source pruning lost %q", want) @@ -159,7 +159,7 @@ func TestCompactionSourceUsesCompletedCodeModeBody(t *testing.T) { got := reduceContextCompaction(items) text := compactionSourceTestOutputText(t, got[1]) - if !strings.Contains(text, "[hpatch compaction: omitted") || strings.Contains(text, "10:000a source declaration") { + if !strings.Contains(text, "[mekugi compaction: omitted") || strings.Contains(text, "10:000a source declaration") { t.Fatal("completed Code Mode source body was not pruned") } } @@ -338,7 +338,7 @@ func TestCompactionSourceGeneratedCarrierReferences(t *testing.T) { got := reduceContextCompaction(base) text := compactionSourceTestOutputText(t, got[2]) if string(got[2]) == string(base[2]) || strings.Contains(text, "10:000a source declaration") || - !strings.Contains(text, "[hpatch:") { + !strings.Contains(text, "[mekugi:") { t.Fatal("generated carrier self metadata pinned its own source output") } @@ -356,7 +356,7 @@ func TestCompactionSourceFailedPatchRetainsOriginalReferences(t *testing.T) { sourceRows := compactionSourceTestRows("", 18) patch := "*** Begin Patch\n*** Update File: source.go\n@@\n-" + strings.TrimSuffix(sourceRows[2], "\n") + "\n+replacement\n*** End Patch\n" report := "in source.go\nfiles add=0 update=1 move=0 delete=0\n" - patchInput := hpatchApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" + patchInput := mekugiApplyExecMarker + "await tools.apply_patch(" + string(mustMarshalJSON(patch)) + ");\ntext(" + string(mustMarshalJSON(report)) + ");" patchCall := mustMarshalJSON(map[string]any{ "type": "custom_tool_call", "name": "exec", "call_id": "patch-consumer", "input": patchInput, }) diff --git a/internal/router/context_compaction_websocket_test.go b/internal/router/context_compaction_websocket_test.go index 0b512c9a..8a77d431 100644 --- a/internal/router/context_compaction_websocket_test.go +++ b/internal/router/context_compaction_websocket_test.go @@ -235,8 +235,8 @@ func TestCompactionWebSocketFailsClosed(t *testing.T) { name, input, metadata string status int }{ - {"damaged", `[{"type":"compaction","encrypted_content":"hpatch\u002ecompaction\u002ev1:broken"}]`, "", 422}, - {"unknown_version", `[{"type":"compaction","encrypted_content":"hpatch.compaction.v999:broken"}]`, "", 422}, + {"damaged", `[{"type":"compaction","encrypted_content":"mekugi\u002ecompaction\u002ev1:broken"}]`, "", 422}, + {"unknown_version", `[{"type":"compaction","encrypted_content":"mekugi.compaction.v999:broken"}]`, "", 422}, {"unsupported", `[{"type":"message","role":"user","content":"Keep this."}]`, `{"request_kind":"compaction","compaction":{"implementation":"unknown"}}`, 422}, {"no_reduction", `[{"type":"message","role":"user","content":"Keep this."}]`, `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`, 422}, } { From 5307d52909523f81bbd0d92739607de6044ac9c8 Mon Sep 17 00:00:00 2001 From: Yuzerion Date: Sat, 12 Sep 2026 01:37:49 +0800 Subject: [PATCH 06/13] feat(compaction): select budgeted native working sets Separate token-budget admission from evidence preservation. Target 50k visible-string tokens with at most 30k overshoot; reduce eligible completed output before relaxing whole-operation recency. Compile every candidate independently, preserve hard protections, and reject inadmissible or no-op results before envelope sealing. Add selector, fuzz, native-preservation and transport-admission tests. Document the budget metric, retention plans and unchanged Codex ownership. Validation: isolated selector tests pass with race detection, 100 repeated runs, 100% statement coverage, go vet, and 213539 fuzz executions. Full router and installed-Codex tests were not run: the local environment has Go 1.23, requires Go 1.26 for this repository, and cannot resolve external hosts to obtain the toolchain and dependencies. --- doc/architecture/compaction.md | 26 ++- doc/spec/compaction.md | 53 ++++- internal/router/context_compaction.go | 10 +- internal/router/context_compaction_budget.go | 107 +++++++++ ...text_compaction_budget_integration_test.go | 215 ++++++++++++++++++ .../router/context_compaction_budget_test.go | 210 +++++++++++++++++ internal/router/context_compaction_http.go | 8 +- .../router/context_compaction_retirement.go | 9 +- internal/router/context_compaction_source.go | 9 +- 9 files changed, 623 insertions(+), 24 deletions(-) create mode 100644 internal/router/context_compaction_budget.go create mode 100644 internal/router/context_compaction_budget_integration_test.go create mode 100644 internal/router/context_compaction_budget_test.go diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index 76a0deb8..3fb001ee 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -8,11 +8,33 @@ restoration of its own input envelopes, and bounded request decoding. Codex owns configuration resolution, trigger timing, retained client-side context, and installation of the returned compaction result. -The selector owns evidence reduction and the approved lossy retirement of finished +Evidence reducers own reduction and the approved lossy retirement of finished operations. Static carrier parsing establishes invocation facts; terminal output establishes completion, not task closure. It does not execute scripts or interpret opaque reasoning. Unknown lifecycle/dependency states stay protected. +The working-set selector owns token pressure and admission, not relevance +inference. It measures candidate native histories before envelope sealing using +the existing visible-string metric. Its 50,000-token target permits at most +30,000 tokens of overshoot; an inadmissible history fails explicitly rather than +relaxing authority, lifecycle, reference, diagnostic or reasoning-group rules. +These budgets do not change Codex's scheduling or model context configuration. + +The default eight-operation continuity buffer is a warm retention preference, +not the definition of live work. Under pressure the selector first reduces +eligible completed output while retaining native calls and reasoning, then may +retire more complete historical groups. The newest operation, all live or unknown +operations, and dependency closure remain protected under every plan. Metadata +and narration reducers keep their existing conservative frontier. + +Each retention plan runs against the same original input. Selection never chains +lossy candidates or assumes that a more aggressive plan must be smaller. It stops +at the first candidate reaching the target; otherwise it selects the lowest-cost +admissible candidate, favoring the earlier plan on ties. An already-small history +cannot justify escalation merely to manufacture successful compaction. The +request-local item-cost cache only avoids duplicate tokenization; it is not an +archive, a shared mutable policy, or a provider token-usage claim. + Retirement records replace complete native call/result pairs in their original relative order. Consecutive newly generated records may share framing after reference closure; authenticated native restoration remains the envelope owner's @@ -44,7 +66,7 @@ User/developer authority and the active frontier do not become historical narrat merely because they are old. No reduction step calls a provider or claims that omitted details can be retrieved. -The selector may remove unreferenced transport turn/time fields from older native +The metadata reducer may remove unreferenced transport turn/time fields from older native items only when unique stable IDs preserve reconciliation, and from factual records only when retained content does not reference their exact values. Content classifications, roles, phases, opaque payloads, and unknown metadata remain with diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index 0067c8f8..f70b2d96 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -51,9 +51,10 @@ Before lossy retirement, evidence reducers can shorten redundant output: structured shell output; it does not require interpreting the script. Finished-operation retirement: -- Keep the newest eight tool invocations and their results, live/unmatched or - duplicate identities, unknown completion states, and explicitly - referenced calls or retained-script producers. Referenced verified rows and +- By default keep the newest eight tool invocations and their results. The + budget policy below may relax this warm recency buffer, never the newest + invocation/result, live/unmatched or duplicate identities, unknown completion + states, or explicitly referenced calls and retained-script producers. Referenced verified rows and numeric ranges may survive exactly in factual records instead of pinning their whole successful group. Reference matching decodes nested argument/result envelopes and supported escaped text; suspicious encodings retain evidence. @@ -100,8 +101,9 @@ Finished-operation retirement: Older completed shell outputs may also shrink without changing native calls or reasoning in a blocked group. Whole-output references stay intact; referenced full or partial verified rows and numeric ranges remain exact. Live, - unknown, and recent output is not made eligible by this output-only pass; - failed output obeys the diagnostic-preservation rule above. + unknown, and output inside the selected recent frontier is not made eligible + by this output-only pass; failed output obeys the diagnostic-preservation rule + above. Existing provider-owned compaction items remain untouched. Factual records use a compact versioned representation with exact invocation values, ordered output parts, and unknown metadata. Unreferenced transport item @@ -126,13 +128,37 @@ Operation completion is not task completion. Execution records report observed facts without inventing scope closure, successful validation, or a workspace version. User corrections and visible reasoning/decision text are not blanket-pruned. -If no supported reduction is available, compaction fails with HTTP 422. It does not -discard protected context just to fit a budget, report a fabricated summary, -or fall back to provider compaction. The target replay must retain 50,000 or fewer -visible-string tokens using the first native replay boundary and `o200k_base`, -excluding opaque reasoning and request/tool framing. This acceptance target is -not a universal cap on arbitrary histories or a complete provider-context count. -Encoded size and downstream projection savings do not establish this target. +The working-set selector targets 50,000 visible-string tokens with a maximum +30,000-token overshoot: no newly completed compaction may retain more than +80,000 tokens under this metric. Count the selected native item array at the +first replay boundary using `o200k_base`, excluding opaque `encrypted_content` +and request/tool framing. Encoded size and downstream projection savings do not +establish this target. This is not a complete provider-context count and does not +cap fresh instructions or subsequent input appended after the selected snapshot. + +Selection evaluates retention plans in this order: keep the newest eight +operations and eight outputs; keep eight operations but only one output from +output-only reduction; then keep four, two, or one operations, still protecting +the newest output. These counts constrain eligibility, not hard protections: +user/developer authority, live/unknown states, references, diagnostic preservation +and complete reasoning/tool-group rules apply to every plan. No plan infers +irrelevance merely from age or completion. Metadata and narration reduction keep +their existing frontier. Output reduction precedes more aggressive group +retirement so native invocation and reasoning context can survive historical bulk. + +Each candidate is compiled independently from the same original history and +measured before sealing. Stop at the first candidate reaching the target; +otherwise retain the smallest candidate within the overshoot allowance, preferring +the earlier plan on equal token counts. Do not escalate an already-small history +when the normal pass cannot shrink it. Repeated items each contribute to the +metric even when tokenization of their identical bytes is cached for the request. + +If no supported token reduction is available, token counting fails, or every +candidate exceeds 80,000 tokens, compaction fails with HTTP 422 before sealing. +Budget failures report the measured before/after counts and target/ceiling. +Removing a V2 trigger is not token savings, and trigger-only input cannot produce +an empty capsule. Never discard protected context to force admission, fabricate +a summary or provider usage, or fall back to provider compaction. The retained native timeline travels inline in an authenticated, encrypted router-owned compaction item. Legacy output also carries original real-user messages @@ -162,6 +188,9 @@ archive, session cache requirement, or model-operated retrieval step. Acceptance checks cover local HTTP completion without provider calls, native restoration with fresh context and suffixes, repeated compaction, restart and concurrent key creation, damaged or missing keys, and conservative output pruning. +Budget checks cover candidate independence, output-first selection, non-monotonic +costs, exact target/overshoot boundaries, no-op and cancellation failures, repeated +item accounting, preserved native evidence, and admission before envelope sealing. Installed Codex 0.153.4 has passed loopback legacy and V2 round trips with synthetic ChatGPT authentication: automatic compaction with both counting scopes, and the manual compact operation used by `/compact`. A large user request is diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 301efde1..9e49c47f 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -13,8 +13,12 @@ import ( // reduceContextCompaction retains authority and the active frontier while // reducing redundant evidence and retiring eligible finished operations under -// the explicit lossy retention policy. It does not guarantee a fixed token cap. +// the explicit lossy retention policy. Token admission belongs to the selector. func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { + return reduceContextCompactionWithPlan(input, compactionRetentionPlan{compactionRecentOperations, compactionRecentOperations}) +} + +func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRetentionPlan) []json.RawMessage { // Clean transport metadata while stable native IDs are still present. // Narration reduction may remove an unreferenced ordinary-assistant ID, // after which the metadata pass must conservatively leave that item alone. @@ -149,8 +153,8 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { fields["output"] = encode(reduced) output[index] = mustMarshalJSON(fields) } - retained := reduceContextCompactionSource(input, - retireCompactionOperations(reduceRepeatedCompactionRows(output, protected))) + retained := reduceContextCompactionSourceWithFrontier(input, + retireCompactionOperationsWithFrontier(reduceRepeatedCompactionRows(output, protected), plan.operations), plan.outputs) return consolidateContextCompactionRecords(input, retained) } diff --git a/internal/router/context_compaction_budget.go b/internal/router/context_compaction_budget.go new file mode 100644 index 00000000..e60587c0 --- /dev/null +++ b/internal/router/context_compaction_budget.go @@ -0,0 +1,107 @@ +package router + +import ( + "context" + "encoding/json" + "fmt" +) + +const ( + compactionTargetTokens = 50_000 + compactionOvershootTokens = 30_000 +) + +// A retention plan changes eligibility, never the evidence-preservation rules. +// Output-only pruning gets the first opportunity to release the warm frontier: +// exact invocations and native reasoning can survive without their old bulk. +type compactionRetentionPlan struct { + operations int + outputs int +} + +type compactionBudgetReport struct { + before int + after int + plan compactionRetentionPlan +} + +// selectCompactionWorkingSet measures the native replay, not the encrypted wire +// envelope. Every candidate is compiled from the same input; a more aggressive +// attempt cannot erode the next attempt's evidence. The reducer owns semantic +// eligibility, reference closure and atomic reasoning/tool groups; this selector +// owns only pressure, candidate ordering and admission. +// +// count is additive across native items (the visible-string token metric). The +// request-local cache avoids retokenizing unchanged authority/frontier items on +// every attempt. It is not a transcript store and never survives the request. +func selectCompactionWorkingSet( + ctx context.Context, + input []json.RawMessage, + target, overshoot int, + reduce func([]json.RawMessage, compactionRetentionPlan) []json.RawMessage, + count func(...json.RawMessage) (int, bool), +) ([]json.RawMessage, compactionBudgetReport, error) { + var report compactionBudgetReport + if len(input) == 0 { + return nil, report, fmt.Errorf("local compaction requires nonempty history after removing the trigger") + } + cache := make(map[string]int) + measure := func(items []json.RawMessage) (int, error) { + total := 0 + for _, raw := range items { + if err := ctx.Err(); err != nil { + return 0, err + } + key := string(raw) + tokens, cached := cache[key] + if !cached { + var ok bool + tokens, ok = count(raw) + if !ok { + return 0, fmt.Errorf("cannot measure native compaction history with the visible-string tokenizer") + } + cache[key] = tokens + } + total += tokens + } + return total, nil + } + before, err := measure(input) + if err != nil { + return nil, report, err + } + report.before, report.after = before, before + best := input + // Preserve the established eight-operation continuity buffer until actual + // token pressure warrants reducing completed output, then whole groups. + // The newest invocation/result is never made eligible by budget pressure. + for _, plan := range []compactionRetentionPlan{{8, 8}, {8, 1}, {4, 1}, {2, 1}, {1, 1}} { + if err := ctx.Err(); err != nil { + return nil, report, err + } + candidate := reduce(input, plan) + if len(candidate) == 0 { + return nil, report, fmt.Errorf("context reduction produced empty native history") + } + tokens, err := measure(candidate) + if err != nil { + return nil, report, err + } + if tokens < report.after { + best, report.after, report.plan = candidate, tokens, plan + } + if report.after <= target { + // No pressure means no escalation, even when the normal pass + // cannot reduce an already-small window. That is a no-op error, + // not permission to discard the warm frontier to force success. + break + } + } + if report.after > target+overshoot { + return nil, report, fmt.Errorf("native compaction history retains %d visible-string tokens (before %d; target %d + overshoot %d = ceiling %d); protected or unsupported context was not discarded", report.after, before, target, overshoot, target+overshoot) + } + if report.after >= before { + return nil, report, fmt.Errorf("no supported token reduction is available for this history (%d visible-string tokens); protected context was not discarded and no provider compaction was requested", before) + } + return best, report, nil +} diff --git a/internal/router/context_compaction_budget_integration_test.go b/internal/router/context_compaction_budget_integration_test.go new file mode 100644 index 00000000..a04cd68b --- /dev/null +++ b/internal/router/context_compaction_budget_integration_test.go @@ -0,0 +1,215 @@ +package router + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func compactionBudgetPressureHistory() []json.RawMessage { + input := []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "developer", "content": "Change only the router; retain exact evidence."}), + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Continue from protected_call. Do not repeat its operation."}), + mustMarshalJSON(map[string]any{"type": "reasoning", "encrypted_content": "opaque-provider-state", "summary": []any{map[string]string{"type": "summary_text", "text": "The failure is unresolved; inspect it before proceeding."}}}), + } + for index := range 4 { + id := fmt.Sprintf("bulk_%d", index) + input = append(input, compactTestCall(id, fmt.Sprintf("cat old-%d.log", index)), + compactTestOutput(id, strings.Repeat("payload ", 25_000), 0)) + } + input = append(input, + compactTestCall("protected_call", "cat evidence.log"), compactTestOutput("protected_call", strings.Repeat("exact evidence ", 500), 0), + compactTestCall("failed_call", "go test ./..."), compactTestOutput("failed_call", "FAIL: assertion remains unresolved\nexpected alpha\nactual beta\nmultiline diagnostic detail\n", 1), + compactTestCall("live_call", "long-running-command"), mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": "live_call", + "output": string(mustMarshalJSON(map[string]any{"output": "still running", "exit_code": nil, "session_id": 1234})), + }), + compactTestCall("frontier_call", "pwd"), compactTestOutput("frontier_call", "/workspace/current\n", 0)) + return input +} + +func TestCompactionBudgetNativeOutputFirstAndRestoration(t *testing.T) { + input := compactionBudgetPressureHistory() + original := mustMarshalJSON(input) + baseline := reduceContextCompaction(input) + baselineTokens, ok := compactionVisibleStringTokens(baseline...) + if !ok || baselineTokens <= compactionTargetTokens+compactionOvershootTokens { + t.Fatalf("fixture must exceed the ceiling with the fixed frontier: %d", baselineTokens) + } + retained, report, err := selectCompactionWorkingSet(t.Context(), input, + compactionTargetTokens, compactionOvershootTokens, reduceContextCompactionWithPlan, compactionVisibleStringTokens) + if err != nil || report.after > compactionTargetTokens || report.plan != (compactionRetentionPlan{8, 1}) { + t.Fatalf("output-only pressure plan = %+v: %v", report, err) + } + if !bytes.Equal(original, mustMarshalJSON(input)) { + t.Fatal("candidate evaluation mutated the original evidence") + } + if len(retained) != len(input) { + t.Fatal("output-only plan retired native operation or reasoning items") + } + for index := range input { + // Only the four unreferenced, completed bulk result bodies may change. + if index >= 4 && index <= 10 && index%2 == 0 { + if bytes.Equal(retained[index], input[index]) || strings.Contains(string(retained[index]), "payload payload") { + t.Fatalf("historical bulk result %d survived", index) + } + continue + } + if !bytes.Equal(retained[index], input[index]) { + t.Fatalf("authority, reference, invocation, reasoning, failure, live state or frontier changed at %d", index) + } + } + measured, ok := compactionVisibleStringTokens(retained...) + if !ok || measured != report.after { + t.Fatalf("budget report differs from native replay: %d != %d", measured, report.after) + } + + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.seal(t.Context(), retained) + if err != nil { + t.Fatal(err) + } + // A different instance must restore selected native history, not the + // pre-compaction bulk. Fresh suffix authority must survive unchanged. + suffix := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Now inspect the unresolved failure."}) + resumed := &contextCompactor{keyPath: compactor.keyPath} + restored, err := resumed.restore(t.Context(), []json.RawMessage{capsule, suffix}) + want := append(slices.Clone(retained), suffix) + if err != nil || !bytes.Equal(mustMarshalJSON(restored), mustMarshalJSON(want)) { + t.Fatalf("budgeted history was not restored exactly: %v", err) + } + // A repeated compaction may decline to reduce this small window further; + // it must not relax the normal frontier just to manufacture progress. + again, againReport, err := selectCompactionWorkingSet(t.Context(), restored, + compactionTargetTokens, compactionOvershootTokens, reduceContextCompactionWithPlan, compactionVisibleStringTokens) + if err != nil { + if !strings.Contains(err.Error(), "no supported token reduction") { + t.Fatalf("repeated compaction failed unexpectedly: %v", err) + } + } else if againReport.plan != (compactionRetentionPlan{8, 8}) || + !bytes.Equal(mustMarshalJSON(again), mustMarshalJSON(reduceContextCompaction(restored))) { + t.Fatal("repeated compaction escalated an already-small working set") + } +} + +func TestCompactionBudgetHTTPAdmission(t *testing.T) { + for _, v2 := range []bool{false, true} { + for _, test := range []struct { + name string + authority int + status int + }{ + {"protected-over-ceiling", 90_000, http.StatusUnprocessableEntity}, + {"protected-within-overshoot", 60_000, http.StatusOK}, + } { + t.Run(fmt.Sprintf("%s/v2=%t", test.name, v2), func(t *testing.T) { + input := append([]json.RawMessage{mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", "content": strings.Repeat("authority ", test.authority), + })}, compactHTTPHistory()...) + path := "/v1/responses/compact" + if v2 { + path = "/v1/responses" + input = append(input, mustMarshalJSON(map[string]any{"type": "compaction_trigger"})) + } + request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(mustMarshalJSON(map[string]any{ + "model": "gpt-5", "input": input, "stream": v2, + }))) + if v2 { + request.Header.Set(codexTurnMetadataHeader, `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`) + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + response := httptest.NewRecorder() + compactor.handler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("budget admission called a provider") + }))(response, request) + if response.Code != test.status { + t.Fatalf("status = %d, want %d: %.300s", response.Code, test.status, response.Body.String()) + } + if test.status != http.StatusOK { + if !strings.Contains(response.Body.String(), "ceiling 80000") { + t.Fatal("budget failure omitted its actual ceiling") + } + if _, err := os.Stat(compactor.keyPath); !os.IsNotExist(err) { + t.Fatalf("inadmissible history reached envelope sealing: %v", err) + } + return + } + var compacted struct { + Output []json.RawMessage `json:"output"` + } + if v2 { + for line := range strings.SplitSeq(response.Body.String(), "\n") { + if !strings.HasPrefix(line, "data: ") { + continue + } + var event struct { + Type string `json:"type"` + Response struct { + Output []json.RawMessage `json:"output"` + } `json:"response"` + } + if json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &event) == nil && event.Type == "response.completed" { + compacted.Output = event.Response.Output + } + } + } else if err := json.Unmarshal(response.Body.Bytes(), &compacted); err != nil { + t.Fatal(err) + } + if len(compacted.Output) == 0 { + t.Fatal("successful admission emitted no capsule") + } + restored, err := compactor.restore(t.Context(), compacted.Output) + if err != nil { + t.Fatal(err) + } + tokens, ok := compactionVisibleStringTokens(restored...) + if !ok || tokens <= compactionTargetTokens || tokens > compactionTargetTokens+compactionOvershootTokens { + t.Fatalf("overshoot not measured at native replay: %d", tokens) + } + if !bytes.Equal(restored[0], input[0]) { + t.Fatal("overshoot was achieved by truncating authority") + } + }) + } + } +} + +func TestCompactionBudgetV2TriggerIsNotSavings(t *testing.T) { + for _, input := range [][]json.RawMessage{ + {mustMarshalJSON(map[string]any{"type": "compaction_trigger"})}, + {mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Protected request"}), mustMarshalJSON(map[string]any{"type": "compaction_trigger"})}, + } { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + request := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(mustMarshalJSON(map[string]any{"model": "gpt-5", "stream": true, "input": input}))) + request.Header.Set(codexTurnMetadataHeader, `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`) + response := httptest.NewRecorder() + compactor.handler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("provider called") }))(response, request) + if response.Code != http.StatusUnprocessableEntity { + t.Fatalf("trigger removal manufactured successful compaction: %d", response.Code) + } + } +} + +func TestCompactionBudgetFrontierFloor(t *testing.T) { + input := []json.RawMessage{ + compactTestCall("old", "cat old.log"), compactTestOutput("old", strings.Repeat("historical ", 1000), 0), + compactTestCall("new", "cat new.log"), compactTestOutput("new", strings.Repeat("current ", 1000), 0), + } + for _, recent := range []int{-1, 0, 1} { + for _, retained := range [][]json.RawMessage{ + retireCompactionOperationsWithFrontier(input, recent), + reduceContextCompactionSourceWithFrontier(input, input, recent), + } { + if !bytes.Equal(retained[2], input[2]) || !bytes.Equal(retained[3], input[3]) { + t.Fatal("budget pressure made the newest invocation or result eligible") + } + } + } +} diff --git a/internal/router/context_compaction_budget_test.go b/internal/router/context_compaction_budget_test.go new file mode 100644 index 00000000..37a602f1 --- /dev/null +++ b/internal/router/context_compaction_budget_test.go @@ -0,0 +1,210 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" +) + +// Synthetic additive costs isolate selection from tokenization and reducers. +// Production-tokenizer and transport coverage lives in the integration tests. +func compactionBudgetItem(name string, tokens int) json.RawMessage { + return json.RawMessage(fmt.Sprintf(`{"name":%q,"tokens":%d}`, name, tokens)) +} + +func compactionBudgetCount(items ...json.RawMessage) (int, bool) { + total := 0 + for _, raw := range items { + var item struct { + Tokens int `json:"tokens"` + } + if json.Unmarshal(raw, &item) != nil { + return 0, false + } + total += item.Tokens + } + return total, true +} + +func TestCompactionBudgetSelection(t *testing.T) { + tests := []struct { + name string + before int + costs []int + want int + wantPlan compactionRetentionPlan + wantError string + wantPasses int + }{ + {"normal target", 120_000, []int{50_000}, 50_000, compactionRetentionPlan{8, 8}, "", 1}, + {"output first", 120_000, []int{95_000, 49_000}, 49_000, compactionRetentionPlan{8, 1}, "", 2}, + {"whole group pressure", 120_000, []int{95_000, 90_000, 70_000, 60_000, 45_000}, 45_000, compactionRetentionPlan{1, 1}, "", 5}, + {"bounded overshoot", 120_000, []int{100_000, 90_000, 80_000, 81_000, 82_000}, 80_000, compactionRetentionPlan{4, 1}, "", 5}, + {"nonmonotonic costs", 120_000, []int{100_000, 65_000, 70_000, 71_000, 66_000}, 65_000, compactionRetentionPlan{8, 1}, "", 5}, + {"ties preserve less aggressive plan", 120_000, []int{70_000, 70_000, 70_000, 70_000, 70_000}, 70_000, compactionRetentionPlan{8, 8}, "", 5}, + {"over ceiling", 120_000, []int{100_000, 95_000, 90_000, 85_000, 80_001}, 80_001, compactionRetentionPlan{1, 1}, "ceiling 80000", 5}, + {"already small no-op", 40_000, []int{40_000}, 40_000, compactionRetentionPlan{}, "no supported token reduction", 1}, + {"already small growth", 40_000, []int{45_000}, 40_000, compactionRetentionPlan{}, "no supported token reduction", 1}, + {"no gain within allowance", 60_000, []int{60_000, 60_000, 60_000, 60_000, 60_000}, 60_000, compactionRetentionPlan{}, "no supported token reduction", 5}, + {"reject growth", 70_000, []int{71_000, 72_000, 73_000, 74_000, 75_000}, 70_000, compactionRetentionPlan{}, "no supported token reduction", 5}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := []json.RawMessage{compactionBudgetItem("original", test.before)} + original := bytes.Clone(input[0]) + passes := 0 + plans := []compactionRetentionPlan{{8, 8}, {8, 1}, {4, 1}, {2, 1}, {1, 1}} + reduce := func(history []json.RawMessage, plan compactionRetentionPlan) []json.RawMessage { + if len(history) != 1 || !bytes.Equal(history[0], original) { + t.Fatal("candidate compiled from a previously reduced history") + } + if passes >= len(test.costs) || plan != plans[passes] { + t.Fatalf("unexpected escalation: pass=%d plan=%+v", passes, plan) + } + item := compactionBudgetItem(fmt.Sprintf("candidate-%d", passes), test.costs[passes]) + passes++ + return []json.RawMessage{item} + } + got, report, err := selectCompactionWorkingSet(context.Background(), input, + compactionTargetTokens, compactionOvershootTokens, reduce, compactionBudgetCount) + if test.wantError == "" { + if err != nil { + t.Fatal(err) + } + tokens, ok := compactionBudgetCount(got...) + if !ok || tokens != test.want { + t.Fatalf("selected tokens=%d, want=%d", tokens, test.want) + } + } else if err == nil || !strings.Contains(err.Error(), test.wantError) || got != nil { + t.Fatalf("unsafe/no-op candidate returned: items=%d err=%v", len(got), err) + } + if report.before != test.before || report.after != test.want || report.plan != test.wantPlan || passes != test.wantPasses { + t.Fatalf("report=%+v passes=%d", report, passes) + } + if !bytes.Equal(input[0], original) { + t.Fatal("selection mutated original history") + } + }) + } +} + +func TestCompactionBudgetCacheCountsOccurrences(t *testing.T) { + authority := compactionBudgetItem("authority", 10) + bulk := compactionBudgetItem("bulk", 200) + input := []json.RawMessage{authority, authority, bulk} + counts := make(map[string]int) + count := func(items ...json.RawMessage) (int, bool) { + for _, item := range items { + counts[string(item)]++ + } + return compactionBudgetCount(items...) + } + pass := 0 + reduce := func(history []json.RawMessage, _ compactionRetentionPlan) []json.RawMessage { + cost := []int{120, 90, 70, 60, 50}[pass] + pass++ + return []json.RawMessage{history[0], history[1], compactionBudgetItem("smaller", cost)} + } + _, report, err := selectCompactionWorkingSet(context.Background(), input, 50, 30, reduce, count) + if err != nil || report.before != 220 || report.after != 70 || pass != 5 { + t.Fatalf("duplicate occurrences miscounted: report=%+v passes=%d err=%v", report, pass, err) + } + for key, calls := range counts { + if calls != 1 { + t.Fatalf("retokenized an unchanged item %q %d times", key, calls) + } + } + // No cached transcript or costs may leak across requests. + _, _, err = selectCompactionWorkingSet(context.Background(), input, 500, 0, + func(items []json.RawMessage, _ compactionRetentionPlan) []json.RawMessage { return items }, count) + if err == nil || counts[string(authority)] != 2 { + t.Fatalf("request-local cache escaped its lifetime: counts=%v err=%v", counts, err) + } +} + +func TestCompactionBudgetErrorsAndCancellation(t *testing.T) { + input := []json.RawMessage{compactionBudgetItem("original", 100)} + for _, name := range []string{"empty input", "empty candidate", "input counting", "candidate counting", "canceled before", "canceled during reduction", "canceled during counting"} { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + history := slices.Clone(input) + passes, counts := 0, 0 + reduce := func([]json.RawMessage, compactionRetentionPlan) []json.RawMessage { + passes++ + if name == "empty candidate" { + return nil + } + if name == "canceled during reduction" { + cancel() + } + return []json.RawMessage{compactionBudgetItem("reduced", 30)} + } + count := func(items ...json.RawMessage) (int, bool) { + counts++ + if name == "input counting" || name == "candidate counting" && counts == 2 { + return 0, false + } + if name == "canceled during counting" { + cancel() + } + return compactionBudgetCount(items...) + } + switch name { + case "empty input": + history = nil + case "canceled before": + cancel() + } + got, _, err := selectCompactionWorkingSet(ctx, history, 50, 30, reduce, count) + if err == nil || got != nil { + t.Fatalf("error produced a replacement history: %v", err) + } + if strings.HasPrefix(name, "canceled") && !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation was hidden: %v", err) + } + if (name == "empty input" || name == "input counting" || name == "canceled before" || name == "canceled during counting") && passes != 0 { + t.Fatal("reduction ran after an input failure") + } + }) + } +} + +func FuzzCompactionBudgetAdmission(f *testing.F) { + f.Add([]byte{200, 180, 140, 110, 90, 50}) + f.Add([]byte{90, 90, 90, 90, 90, 90}) + f.Add([]byte{20, 25, 10, 5, 4, 3}) + f.Fuzz(func(t *testing.T, costs []byte) { + if len(costs) < 6 { + return + } + before := int(costs[0]) + input := []json.RawMessage{compactionBudgetItem("original", before)} + pass, best := 0, before + reduce := func([]json.RawMessage, compactionRetentionPlan) []json.RawMessage { + pass++ + cost := int(costs[pass]) + best = min(best, cost) + return []json.RawMessage{compactionBudgetItem(fmt.Sprint(pass), cost)} + } + got, report, err := selectCompactionWorkingSet(context.Background(), input, 50, 30, reduce, compactionBudgetCount) + if report.after != best || report.before != before || pass < 1 || pass > 5 { + t.Fatalf("invalid selection: report=%+v best=%d passes=%d", report, best, pass) + } + admissible := best < before && best <= 80 + if (err == nil) != admissible || err != nil && got != nil { + t.Fatalf("admission mismatch: report=%+v err=%v", report, err) + } + if err == nil { + tokens, ok := compactionBudgetCount(got...) + if !ok || tokens != best { + t.Fatalf("wrong candidate returned: tokens=%d best=%d", tokens, best) + } + } + }) +} diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go index c5652d3a..ed502adc 100644 --- a/internal/router/context_compaction_http.go +++ b/internal/router/context_compaction_http.go @@ -144,9 +144,11 @@ func (c *contextCompactor) prepare(ctx context.Context, parsed *parsedResponsesR input = input[:len(input)-1] } } - reduced := reduceContextCompaction(input) - if slices.EqualFunc(input, reduced, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { - return fail(http.StatusUnprocessableEntity, "no supported context reduction is available for this history; protected context was not discarded and no provider compaction was requested") + reduced, _, err := selectCompactionWorkingSet(ctx, input, + compactionTargetTokens, compactionOvershootTokens, + reduceContextCompactionWithPlan, compactionVisibleStringTokens) + if err != nil { + return fail(http.StatusUnprocessableEntity, err.Error()) } capsule, err := c.seal(ctx, reduced) if err != nil { diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index c546517d..39121170 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -28,6 +28,11 @@ type compactionRetirement struct { } func retireCompactionOperations(input []json.RawMessage) []json.RawMessage { + return retireCompactionOperationsWithFrontier(input, compactionRecentOperations) +} + +func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) []json.RawMessage { + recent = max(1, recent) fields := make([]map[string]json.RawMessage, len(input)) calls, results := make(map[string][]int), make(map[string][]int) var callOrder []int @@ -42,10 +47,10 @@ func retireCompactionOperations(input []json.RawMessage) []json.RawMessage { results[id] = append(results[id], index) } } - if len(callOrder) <= compactionRecentOperations { + if len(callOrder) <= recent { return input } - cutoff := callOrder[len(callOrder)-compactionRecentOperations] + cutoff := callOrder[len(callOrder)-recent] plans := make(map[string]*compactionRetirement) for id, positions := range calls { if id == "" || len(positions) != 1 || len(results[id]) != 1 || !compactionCallID.MatchString(id) { diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go index 240543c5..88c49b79 100644 --- a/internal/router/context_compaction_source.go +++ b/internal/router/context_compaction_source.go @@ -18,6 +18,11 @@ type compactionSourcePair struct { } func reduceContextCompactionSource(original, retained []json.RawMessage) []json.RawMessage { + return reduceContextCompactionSourceWithFrontier(original, retained, compactionRecentOperations) +} + +func reduceContextCompactionSourceWithFrontier(original, retained []json.RawMessage, recent int) []json.RawMessage { + recent = max(1, recent) if len(original) != len(retained) { return retained } @@ -40,10 +45,10 @@ func reduceContextCompactionSource(original, retained []json.RawMessage) []json. results[id] = append(results[id], index) } } - if len(callOrder) <= compactionRecentOperations { + if len(callOrder) <= recent { return retained } - cutoff := callOrder[len(callOrder)-compactionRecentOperations] + cutoff := callOrder[len(callOrder)-recent] pairs := make(map[string]compactionSourcePair) for id, callPositions := range calls { From 206beee669ecb1a6d45e22f0af6ecf4ac4ea5037 Mon Sep 17 00:00:00 2001 From: Yuzerion Date: Sat, 12 Sep 2026 09:11:01 +0800 Subject: [PATCH 07/13] fix(compaction): preserve identity and legacy octal JS references Decode JavaScript NonEscapeCharacter, legacy octal and non-octal decimal escapes before evidence selection. Honor octal digit boundaries, Unicode identity characters and line continuations. Preserve the original text and a single decoded layer; malformed UTF-8 remains unsafe. Previously 3\:0003 and non-strict 3\720003 were treated as unreferenced, allowing the exact referenced source row to be pruned. Decode them without globally pinning unrelated output or changing the token-budget policy. Add decoder/row-preservation regressions and source-frontier cases for assistant text, pending exec arguments and pending Code Mode input at both the normal and pressure frontiers, including repeated compaction. Validation: reproduced omission with the original production decoder and row-pruning functions in an isolated Go 1.23 harness; regressions pass after the fix. Decoder matches Node.js on 11198 escape cases. Isolated go test -race -count=10 and go vet pass; changed files are gofmt-clean. Full router/frontier integration tests could not run: the environment has Go 1.23, this repository requires Go 1.26, and external DNS is unavailable. --- .../context_compaction_js_escape_test.go | 130 ++++++++++++++++++ internal/router/context_compaction_source.go | 33 ++++- 2 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 internal/router/context_compaction_js_escape_test.go diff --git a/internal/router/context_compaction_js_escape_test.go b/internal/router/context_compaction_js_escape_test.go new file mode 100644 index 00000000..76e59cfb --- /dev/null +++ b/internal/router/context_compaction_js_escape_test.go @@ -0,0 +1,130 @@ +package router + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "testing" +) + +func TestCompactionJSEscapeReferenceForms(t *testing.T) { + // StringLiteral NonEscapeCharacter and non-strict legacy decimal/octal + // escapes. A leading 4..7 consumes at most two octal digits, not three. + for _, test := range []struct { + name, source, want string + }{ + {"identity colon", `3\:0003`, `3:0003`}, + {"identity letters", `source\_\old`, `source_old`}, + {"identity range", `3\:0003\.\.5\:0005`, `3:0003..5:0005`}, + {"identity unicode", `\界\😀`, `界😀`}, + {"octal colon two digits", `3\720003`, `3:0003`}, + {"octal colon three digits", `3\0720003`, `3:0003`}, + {"octal script", `\100shell\57result`, `@shell/result`}, + {"octal one digit", `\7x`, "\ax"}, + {"octal three digit boundary", `\1234`, `S4`}, + {"octal byte maximum", `\3777`, `ÿ7`}, + {"octal four prefix", `\400`, ` 0`}, + {"octal seven prefix", `\777`, `?7`}, + {"octal stops before eight", `\078`, "\a8"}, + {"null before eight", `\08`, "\x008"}, + {"null before nine", `\09`, "\x009"}, + {"non-octal decimal", `\8\9`, `89`}, + {"line separator continuation", "3\\\u2028:0003", `3:0003`}, + {"paragraph separator continuation", "3\\\u2029:0003", `3:0003`}, + {"escaped backslash stays one layer", `3\\720003`, `3\720003`}, + {"octal backslash stays one layer", `3\134720003`, `3\720003`}, + } { + t.Run(test.name, func(t *testing.T) { + var visited []string + unsafeEncoding := false + compactionSourceVisitDecodedReferences(test.source, func(text string) { + visited = append(visited, text) + }, &unsafeEncoding) + if unsafeEncoding || !slices.Equal(visited, []string{test.source, test.want}) { + t.Fatalf("unsafe=%t visits=%q; want raw then %q", unsafeEncoding, visited, test.want) + } + }) + } +} + +func TestCompactionJSEscapeInvalidUTF8IsUnsafe(t *testing.T) { + _, _, unsafe := compactionSourceDecodeReferenceEscapes("\\\xff") + if !unsafe { + t.Fatal("invalid UTF-8 after a backslash was silently treated as a safe identity escape") + } +} + +func TestCompactionJSEscapeKeepsReferencedRows(t *testing.T) { + rows := compactionSourceTestRows("", 18) + for _, reference := range []string{`3\:0003`, `3\720003`, `3\0720003`} { + t.Run(reference, func(t *testing.T) { + referenced := make(map[string]bool) + unsafeEncoding := false + compactionSourceVisitDecodedReferences(reference, func(text string) { + for _, row := range compactionRowReference.FindAllString(text, -1) { + referenced[row] = true + } + }, &unsafeEncoding) + if unsafeEncoding { + t.Fatal("supported reference globally pinned source output") + } + got := compactionPruneSourceText(strings.Join(rows, ""), referenced, nil) + if !strings.Contains(got, rows[2]) { + t.Fatalf("referenced row lost: %s", got) + } + if strings.Contains(got, rows[10]) || !strings.Contains(got, "[mekugi compaction: omitted") { + t.Fatal("unrelated historical source rows were not pruned") + } + }) + } +} + +func TestCompactionJSEscapeSourceFrontier(t *testing.T) { + for _, recent := range []int{1, compactionRecentOperations} { + for _, reference := range []string{`3\:0003`, `3\720003`, `3\0720003`, `3\:0003\.\.5\:0005`, `3\720003\56\0565\720005`} { + for _, consumer := range []string{"message", "function_call", "custom_tool_call"} { + t.Run(fmt.Sprintf("recent=%d/%s/%s", recent, consumer, reference), func(t *testing.T) { + rows := compactionSourceTestRows("source.go", 18) + code := `const target = "` + reference + `"; text(target);` + var retained json.RawMessage + switch consumer { + case "message": + retained = mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": code}) + case "function_call": + retained = compactTestCall("pending-exec", code) + case "custom_tool_call": + retained = mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "pending-script", "input": code}) + } + items := []json.RawMessage{ + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", strings.Join(rows, ""), 0), + retained, + } + items = append(items, compactionSourceTestRecent()...) + before := string(mustMarshalJSON(items)) + got := reduceContextCompactionSourceWithFrontier(items, items, recent) + text := compactionSourceTestOutputText(t, got[1]) + if !strings.Contains(text, rows[2]) || strings.Contains(text, rows[10]) { + t.Fatal("escaped reference lost or unrelated rows pinned") + } + if strings.Contains(reference, "0005") && (!strings.Contains(text, rows[3]) || !strings.Contains(text, rows[4])) { + t.Fatal("escaped range did not retain every row") + } + for index := range items { + if index != 1 && string(got[index]) != string(items[index]) { + t.Fatalf("native item %d changed", index) + } + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("source reduction mutated input") + } + again := reduceContextCompactionSourceWithFrontier(got, got, recent) + if string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { + t.Fatal("repeated compaction changed retained evidence") + } + }) + } + } + } +} diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go index 88c49b79..1a24ff18 100644 --- a/internal/router/context_compaction_source.go +++ b/internal/router/context_compaction_source.go @@ -336,11 +336,23 @@ func compactionSourceDecodeJSEscape(text string) (value string, width int, recog return "", 3, true, true } return "", 2, true, true - case '0': - if len(text) > 2 && text[2] >= '0' && text[2] <= '9' { - return "", 0, true, false - } - return "\x00", 2, true, true + case '0', '1', '2', '3', '4', '5', '6', '7': + // Non-strict legacy octal consumes at most three digits for 0..3, + // but only two for 4..7: \720003 is ':' followed by "0003". + limit := 4 // Backslash plus at most three octal digits. + if text[1] >= '4' { + limit = 3 + } + codePoint := rune(text[1] - '0') + width := 2 + for width < min(len(text), limit) && text[width] >= '0' && text[width] <= '7' { + codePoint = codePoint*8 + rune(text[width]-'0') + width++ + } + return string(codePoint), width, true, true + case '8', '9': + // NonOctalDecimalEscapeSequence in non-strict string literals. + return text[1:2], 2, true, true case 'x': if len(text) < 3 || compactionSourceHexValue(text[2]) < 0 { return "", 0, false, false @@ -355,7 +367,16 @@ func compactionSourceDecodeJSEscape(text string) (value string, width int, recog } return compactionSourceDecodeJSUnicodeEscape(text) default: - return "", 0, false, false + // NonEscapeCharacter is an identity escape, including \: and \_. + // Consume one code point; Unicode line continuations contribute none. + codePoint, size := utf8.DecodeRuneInString(text[1:]) + if codePoint == utf8.RuneError && size == 1 { + return "", 0, true, false + } + if codePoint == '\u2028' || codePoint == '\u2029' { + return "", size + 1, true, true + } + return text[1 : size+1], size + 1, true, true } } From ae7911cc23aa119bca35b70d6a80cb5ebdafdd83 Mon Sep 17 00:00:00 2001 From: yusing Date: Sat, 12 Sep 2026 09:47:57 +0800 Subject: [PATCH 08/13] fix(compaction): preserve replacement evidence and support hcat read alias Add `hcat` to read-command mapping while retaining legacy `hread` compatibility for historical records. Update repeated-context reference collection to include assistant `message` evidence (in addition to function output records), and make retirement pin and keep replacement evidence-derived references before candidate eviction. Add focused compaction tests that verify replacement outputs and dependencies survive repeated compaction and retirement paths, including deduplication and ledger scenarios. --- internal/router/context_compaction.go | 3 +- .../router/context_compaction_repeated.go | 14 +- .../context_compaction_replacement_test.go | 172 ++++++++++++++++++ .../router/context_compaction_retirement.go | 10 +- 4 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 internal/router/context_compaction_replacement_test.go diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 9e49c47f..6970f1cd 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -252,7 +252,8 @@ func contextCompactionCommand(command string) string { } return "search" - case "cat", "hread": + case "cat", "hcat", "hread": + // hread remains recognizable in histories captured before the rename. return "read" } return "" diff --git a/internal/router/context_compaction_repeated.go b/internal/router/context_compaction_repeated.go index 88a74810..e140f6de 100644 --- a/internal/router/context_compaction_repeated.go +++ b/internal/router/context_compaction_repeated.go @@ -180,10 +180,20 @@ func contextCompactionReferencedResults(input []json.RawMessage) map[string]bool if json.Unmarshal(raw, &fields) != nil { continue } - if kind := jsonString(fields, "type"); kind != "function_call_output" && kind != "custom_tool_call_output" { + var evidence json.RawMessage + switch jsonString(fields, "type") { + case "function_call_output", "custom_tool_call_output": + evidence = fields["output"] + case "message": + // Retired consumers carry the same notes in factual assistant records. + if jsonString(fields, "role") != "assistant" { + continue + } + evidence = fields["content"] + default: continue } - compactionVisitReferenceStrings(fields["output"], func(text string) { + compactionVisitReferenceStrings(evidence, func(text string) { for _, match := range compactionRetainedReference.FindAllStringSubmatch(text, -1) { if id, err := strconv.Unquote(match[1]); err == nil { protected[id] = true diff --git a/internal/router/context_compaction_replacement_test.go b/internal/router/context_compaction_replacement_test.go new file mode 100644 index 00000000..b06272b7 --- /dev/null +++ b/internal/router/context_compaction_replacement_test.go @@ -0,0 +1,172 @@ +package router + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "testing" +) + +func compactionReplacementReasoning(id string) json.RawMessage { + return mustMarshalJSON(map[string]any{ + "type": "reasoning", "id": id, "summary": []any{}, + "encrypted_content": strings.Repeat("opaque", 512), + }) +} + +func compactionReplacementRecent(prefix string) []json.RawMessage { + items := []json.RawMessage{mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", "content": "Continue the current task.", + })} + for index := range compactionRecentOperations { + id := fmt.Sprintf("%s_%d", prefix, index) + items = append(items, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + return items +} + +func compactionReplacementAssertNative(t *testing.T, items []json.RawMessage, want json.RawMessage) { + t.Helper() + if !slices.ContainsFunc(items, func(raw json.RawMessage) bool { return string(raw) == string(want) }) { + var fields map[string]json.RawMessage + _ = json.Unmarshal(want, &fields) + t.Fatalf("protected native item changed or retired: type=%q id=%q call=%q", jsonString(fields, "type"), jsonString(fields, "id"), jsonString(fields, "call_id")) + } +} + +func TestCompactionReplacementEvidenceSurvivesEveryPlan(t *testing.T) { + for _, fixture := range []struct { + name, command, read string + rows, native bool + }{ + {"search-cat", "rg needle .", "cat listing.txt", false, false}, + {"search-hcat", "hgrep needle .", "hcat listing.txt", false, false}, + {"search-native-hcat", "rg needle .", "hcat listing.txt", false, true}, + {"legacy-search-hread", "hgrep needle .", "hread listing.txt", false, false}, + {"search-source-rows", "rg needle .", "hcat listing.txt", true, false}, + {"repeated-source-rows", "hcat earlier.txt", "hcat listing.txt", true, false}, + } { + for _, plan := range []compactionRetentionPlan{{8, 8}, {8, 1}, {4, 1}, {2, 1}, {1, 1}} { + t.Run(fmt.Sprintf("%s/%d/%d", fixture.name, plan.operations, plan.outputs), func(t *testing.T) { + listing := strings.Repeat("matched file with exact historical search evidence\n", 512) + if fixture.rows { + listing = strings.Join(compactionSourceTestRows("", 64), "") + } + output := func(id, text string) json.RawMessage { + if !fixture.native { + return compactTestOutput(id, text, 0) + } + return mustMarshalJSON(map[string]any{ + "type": "function_call_output", "call_id": id, + "output": "Wall time: 1 seconds\nProcess exited with code 0\nOutput:\n" + text, + }) + } + // Both copies are old and share one otherwise-profitable group. + // Ordinary intra-group reference scanning must not authorize + // dropping evidence promised by a replacement note. + items := []json.RawMessage{ + compactionReplacementReasoning("rs_replacement"), + compactTestCall("replacement-search", fixture.command), + output("replacement-search", listing), + compactTestCall("replacement-read", fixture.read), + output("replacement-read", listing), + compactionReplacementReasoning("rs_unrelated"), + compactTestCall("unrelated-finished", "make inspect"), + compactTestOutput("unrelated-finished", strings.Repeat("UNREFERENCED_BULK\n", 512), 0), + } + items = append(items, compactionReplacementRecent("recent")...) + before := string(mustMarshalJSON(items)) + got := reduceContextCompactionWithPlan(items, plan) + keptListing := false + for _, raw := range got { + compactionVisitReferenceStrings(raw, func(text string) { + keptListing = keptListing || strings.Contains(text, listing) + }) + } + if !keptListing { + t.Fatal("both verbatim copies of the replacement listing were lost") + } + for _, index := range []int{4, 3, 1, 0} { + compactionReplacementAssertNative(t, got, items[index]) + } + if !contextCompactionReferencedResults(got)["replacement-read"] { + t.Fatal("replacement note was lost during output-only reduction") + } + if slices.ContainsFunc(got, func(raw json.RawMessage) bool { return string(raw) == string(items[2]) }) { + t.Fatal("fixture did not replace the original duplicate output") + } + if strings.Contains(string(mustMarshalJSON(got)), "UNREFERENCED_BULK") { + t.Fatal("replacement protection blocked unrelated retirement") + } + if string(mustMarshalJSON(items)) != before { + t.Fatal("compaction mutated its input") + } + // A newer duplicate must not redirect a still-referenced result + // or make its only verbatim copy eligible on the next pass. + again := append(slices.Clone(got), compactTestCall("newer-copy", fixture.read), output("newer-copy", listing)) + again = append(again, compactionReplacementRecent("next")...) + again = reduceContextCompactionWithPlan(again, plan) + compactionReplacementAssertNative(t, again, items[4]) + if !contextCompactionReferencedResults(again)["replacement-read"] { + t.Fatal("replacement dependency disappeared on repeated compaction") + } + }) + } + } +} + +func TestCompactionReplacementNotesSurviveLedgerRetirement(t *testing.T) { + for _, note := range []string{ + "[mekugi compaction: matching search listing retained verbatim in tool result \"replacement-read\"; original command and successful exit status retained]\n", + "[mekugi compaction: 64 source rows (1:0001 through 64:0040) retained verbatim in later tool result \"replacement-read\"]\n", + } { + listing := strings.Join(compactionSourceTestRows("", 64), "") + items := []json.RawMessage{ + compactionReplacementReasoning("rs_consumer"), + compactTestCall("ledger-consumer", "make inspect"), + compactTestOutput("ledger-consumer", note+strings.Repeat("CONSUMER_BULK\n", 512), 0), + compactionReplacementReasoning("rs_replacement"), + compactTestCall("replacement-read", "cat listing.txt"), + compactTestOutput("replacement-read", listing, 0), + } + items = append(items, compactionReplacementRecent("ledger_recent")...) + retained := retireCompactionOperationsWithFrontier(items, compactionRecentOperations) + var record map[string]json.RawMessage + _ = json.Unmarshal(retained[2], &record) + if jsonString(record, "type") != "message" || strings.Contains(string(retained[2]), "CONSUMER_BULK") { + t.Fatal("eligible consumer did not retire independently of its protected producer") + } + for _, history := range [][]json.RawMessage{retained, consolidateContextCompactionRecords(items, retained)} { + if !contextCompactionReferencedResults(history)["replacement-read"] { + t.Fatal("native replacement reference was lost when its consumer became an assistant ledger record") + } + compactionReplacementAssertNative(t, history, items[5]) + history = append(slices.Clone(history), compactTestCall("ledger-newer-copy", "cat listing.txt"), compactTestOutput("ledger-newer-copy", listing, 0)) + history = append(history, compactionReplacementRecent("ledger_next")...) + again := reduceContextCompactionWithPlan(history, compactionRetentionPlan{1, 1}) + compactionReplacementAssertNative(t, again, items[5]) + if !contextCompactionReferencedResults(again)["replacement-read"] { + t.Fatal("ledger replacement dependency disappeared on repeated compaction") + } + } + } +} + +func TestCompactionReplacementPinFollowsOriginalDependencies(t *testing.T) { + items := []json.RawMessage{ + compactionReplacementReasoning("rs_upstream"), + compactTestCall("upstream-evidence", "cat upstream.txt"), + compactTestOutput("upstream-evidence", strings.Repeat("exact upstream evidence\n", 512), 0), + compactionReplacementReasoning("rs_replacement"), + compactTestCall("replacement-search", "rg needle ."), + compactTestOutput("replacement-search", "[mekugi compaction: matching search listing retained verbatim in tool result \"replacement-read\"; original command and successful exit status retained]\n", 0), + compactTestCall("replacement-read", "cat listing.txt"), + compactTestOutput("replacement-read", "Depends on upstream-evidence.\n"+strings.Repeat("exact replacement evidence\n", 512), 0), + } + items = append(items, compactionReplacementRecent("dependency_recent")...) + got := retireCompactionOperationsWithFrontier(items, 1) + for index := range 8 { + compactionReplacementAssertNative(t, got, items[index]) + } +} diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index 39121170..e3618adb 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -199,6 +199,13 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } } } + // Earlier reducers have already exchanged evidence for these references. + // Pin their targets before candidate-output scanning can erase the notes, + // including references within one otherwise-retirable reasoning/tool group. + // Re-read the reduced input so newly generated source-row notes count too. + for id := range contextCompactionReferencedResults(input) { + pin(id) + } for _, g := range groups { blocked := g.blocked || g.end > cutoff for index := g.start; index < g.end; index++ { @@ -602,7 +609,8 @@ func compactionRetiredTextKeepingRows(text string, referenced map[string]bool, r keep := make([]bool, len(lines)) for index, line := range lines { sourceRow := compactionCompleteSourceRow.MatchString(line) - if compactionTextReferencesRows(line, referenced, ranges) { + // Replacement notes remain dependencies even when their consumer retires. + if compactionRetainedReference.MatchString(line) || compactionTextReferencesRows(line, referenced, ranges) { keep[index] = true } if sourceRow { From 4edad4f9dc9a5692d6a6d54e3b696e5dea5a8f4b Mon Sep 17 00:00:00 2001 From: yusing Date: Fri, 11 Sep 2026 15:13:38 +0000 Subject: [PATCH 09/13] fix(router): harden provider-free context compaction Make local reductions conservative by requiring corroborated Go test progress, validating tool payloads and identities, bounding oversized evidence lines, and following row references exposed by retained tool output. Propagate cancellation, cache the local AEAD safely, preserve WebSocket retention accounting, and document the updated HTTP/WebSocket failure behavior. --- README.md | 5 +- doc/architecture/compaction.md | 4 +- doc/spec/compaction.md | 18 +- internal/router/context_compaction.go | 179 ++++++++++++++---- .../router/context_compaction_codex_test.go | 13 +- .../router/context_compaction_envelope.go | 15 +- .../context_compaction_envelope_test.go | 19 +- internal/router/context_compaction_http.go | 3 + .../router/context_compaction_http_test.go | 23 +++ .../router/context_compaction_operation.go | 6 +- .../router/context_compaction_read_tool.go | 54 +++++- .../context_compaction_read_tool_test.go | 56 ++++++ .../router/context_compaction_retirement.go | 40 ++-- .../context_compaction_retirement_test.go | 72 +++++++ internal/router/context_compaction_source.go | 16 ++ .../router/context_compaction_source_test.go | 67 +++++++ internal/router/context_compaction_test.go | 21 +- .../context_compaction_websocket_test.go | 3 + internal/router/server_websocket.go | 3 + 19 files changed, 532 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index af680958..51fc1a86 100644 --- a/README.md +++ b/README.md @@ -159,8 +159,9 @@ Compaction can discard unmarked historical details from older finished operation even while the task is still open. It keeps factual execution records, requests, visible decisions, diagnostic excerpts, referenced evidence, and recent/live work. Recognized applied patch bodies and associated older opaque reasoning can also be -retired. Older failed-command output and truncated documentation can lose -unreferenced bulk while retaining errors, warnings, and provenance. +retired. Older failed-command output can lose unreferenced bulk while retaining +errors, warnings, and provenance. Truncated or oversized documentation can also +lose unreferenced bulk; a single oversized evidence line retains bounded excerpts. Discarded details are not currently retrievable through Mekugi. Unknown or ambiguous execution states remain intact. If nothing qualifies, diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index 3fb001ee..40bef7a9 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -55,7 +55,9 @@ may shorten historical output while leaving calls and opaque reasoning native. Both paths share reference-preservation rules. Static documentation reducers own only recognized bodies and unambiguous body fragments in truncated documentation; provenance, warnings, references, unknown metadata, and uncertain fragments remain -exact. They do not reconstruct missing structure or infer a successful outcome. +exact except that a single oversized selected evidence line retains bounded prefix +and suffix evidence with an explicit truncation marker. They do not reconstruct +missing structure or infer a successful outcome. Historical ordinary-assistant narration reduction consolidates exact repeated text rather than guessing whether an arbitrary sentence is routine progress. It preserves diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index f70b2d96..cb53b062 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -37,9 +37,9 @@ completion proves irrelevance. Before lossy retirement, evidence reducers can shorten redundant output: -- Successful Go test results with a structured result or native Codex exec header can omit routine run/pause/continue and - pass lines. The exact call, exit status, package summaries, and other output - remain. +- Successful Go test results with a structured result or native Codex exec header can omit routine run, pause, and continue + progress lines. Test outcome lines, the exact call, exit status, package summaries, + and other output remain. - Successful search listings from recognized `rg`, `hgrep`, or `find` calls can be replaced with a reference to a later retained read result containing the exact same complete listing. The original call and completion metadata stay. @@ -64,10 +64,12 @@ Finished-operation retirement: JavaScript literals are parsed without execution; dynamic expressions remain native. A literal shell-carrier progress notice is retained verbatim and checked against the corresponding result part; its evidence references remain protected. - Successful documentation results retain provenance, identities, hierarchy, +- Successful documentation results retain provenance, identities, hierarchy, pagination, annotations, and unknown metadata while recognized unmarked bodies - may shrink. In truncated documentation, only historical body fragments with - unambiguous boundaries may shrink; provenance, warnings, references, and + may shrink. Selected provenance or diagnostic lines that are themselves + oversized retain bounded prefix and suffix evidence with an explicit truncation + marker. In truncated documentation, only historical body fragments with + unambiguous boundaries may shrink; other provenance, warnings, references, and uncertain material remain. Reduction must not fabricate missing structure or represent a truncated result as complete. A known successful result need not shrink individually for its complete group to be profitable. @@ -158,7 +160,9 @@ candidate exceeds 80,000 tokens, compaction fails with HTTP 422 before sealing. Budget failures report the measured before/after counts and target/ceiling. Removing a V2 trigger is not token savings, and trigger-only input cannot produce an empty capsule. Never discard protected context to force admission, fabricate -a summary or provider usage, or fall back to provider compaction. +a summary or provider usage, or fall back to provider compaction. For WebSocket +`response.create`, the same admission failure emits an error event with status +422 and then closes the connection. The retained native timeline travels inline in an authenticated, encrypted router-owned compaction item. Legacy output also carries original real-user messages diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 6970f1cd..09242ea1 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -1,6 +1,8 @@ package router import ( + "cmp" + "context" "encoding/json" "fmt" "regexp" @@ -19,6 +21,22 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { } func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRetentionPlan) []json.RawMessage { + reduced, err := reduceContextCompactionPlan(context.Background(), input, plan) + if err != nil { + return input + } + return reduced +} + +func reduceContextCompactionContext(ctx context.Context, input []json.RawMessage) ([]json.RawMessage, error) { + return reduceContextCompactionPlan(ctx, input, compactionRetentionPlan{compactionRecentOperations, compactionRecentOperations}) +} + +func reduceContextCompactionPlan(ctx context.Context, input []json.RawMessage, plan compactionRetentionPlan) ([]json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return input, err + } + original := input // Clean transport metadata while stable native IDs are still present. // Narration reduction may remove an unreferenced ordinary-assistant ID, // after which the metadata pass must conservatively leave that item alone. @@ -39,6 +57,9 @@ func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRet duplicates := make(map[string]bool) lastResult := -1 for index, raw := range input { + if err := ctx.Err(); err != nil { + return original, err + } if json.Unmarshal(raw, &items[index]) != nil { continue } @@ -53,7 +74,51 @@ func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRet lastResult = index } } + + type readEvidence struct { + callID string + call int + } + replacementEvidence := make(map[string][]readEvidence) + for resultIndex, candidate := range items { + if err := ctx.Err(); err != nil { + return original, err + } + if candidate.Type != "function_call_output" || candidate.CallID == "" || duplicates[candidate.CallID] { + continue + } + sourceIndex, exists := calls[candidate.CallID] + if !exists || sourceIndex >= resultIndex { + continue + } + source := items[sourceIndex] + if source.Type != "function_call" || (source.Name != "exec_command" && source.Name != "functions.exec_command") { + continue + } + var args struct { + Command string `json:"cmd"` + Shell string `json:"shell"` + } + if json.Unmarshal([]byte(source.Arguments), &args) != nil || args.Shell != "" || + contextCompactionCommand(args.Command) != "read" { + continue + } + _, evidence, ok := contextCompactionOutput(candidate.Output) + if ok && len(evidence) >= 256 { + replacementEvidence[evidence] = append(replacementEvidence[evidence], readEvidence{candidate.CallID, sourceIndex}) + } + } + for text, candidates := range replacementEvidence { + slices.SortFunc(candidates, func(a, b readEvidence) int { + return cmp.Compare(a.call, b.call) + }) + replacementEvidence[text] = candidates + } + for index, current := range items { + if err := ctx.Err(); err != nil { + return original, err + } if index == lastResult || protected[current.CallID] || (current.Type != "function_call_output" && current.Type != "custom_tool_call_output") { continue } @@ -93,17 +158,8 @@ func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRet reduced := text switch kind { case "go-test": - var kept strings.Builder - removed := 0 - for line := range strings.SplitAfterSeq(text, "\n") { - if contextCompactionGoRoutine.MatchString(strings.TrimSuffix(line, "\n")) { - removed++ - } else { - kept.WriteString(line) - } - } - if removed > 0 { - reduced = fmt.Sprintf("[mekugi: omitted %d Go test progress/pass lines]\n%s", removed, kept.String()) + if kept, removed := compactionReduceGoTestOutput(text); removed > 0 { + reduced = fmt.Sprintf("[mekugi: omitted %d corroborated Go test runner lines]\n%s", removed, kept) } case "search": // A later byte-identical output is explicit replacement evidence, @@ -112,35 +168,14 @@ func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRet if len(text) < 256 { continue } - for later := index + 1; later < len(items); later++ { - candidate := items[later] - if candidate.Type != "function_call_output" || candidate.CallID == "" || duplicates[candidate.CallID] { - continue - } - sourceIndex, exists := calls[candidate.CallID] - if !exists || sourceIndex <= index || sourceIndex >= later { - continue - } - source := items[sourceIndex] - if source.Type != "function_call" || (source.Name != "exec_command" && source.Name != "functions.exec_command") { - continue - } - var args struct { - Command string `json:"cmd"` - Shell string `json:"shell"` - } - if json.Unmarshal([]byte(source.Arguments), &args) != nil || args.Shell != "" { - continue - } - if contextCompactionCommand(args.Command) != "read" { - continue - } - _, evidence, ok := contextCompactionOutput(candidate.Output) - if ok && evidence == text { - protected[candidate.CallID] = true - reduced = fmt.Sprintf("[mekugi compaction: matching search listing retained verbatim in tool result %q; original command and successful exit status retained]\n", candidate.CallID) - break - } + candidates := replacementEvidence[text] + position, _ := slices.BinarySearchFunc(candidates, index+1, func(candidate readEvidence, target int) int { + return cmp.Compare(candidate.call, target) + }) + if position < len(candidates) { + candidate := candidates[position] + protected[candidate.callID] = true + reduced = fmt.Sprintf("[mekugi compaction: matching search listing retained verbatim in tool result %q; original command and successful exit status retained]\n", candidate.callID) } } if len(reduced) >= len(text) { @@ -153,9 +188,15 @@ func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRet fields["output"] = encode(reduced) output[index] = mustMarshalJSON(fields) } + if err := ctx.Err(); err != nil { + return original, err + } retained := reduceContextCompactionSourceWithFrontier(input, retireCompactionOperationsWithFrontier(reduceRepeatedCompactionRows(output, protected), plan.operations), plan.outputs) - return consolidateContextCompactionRecords(input, retained) + if err := ctx.Err(); err != nil { + return original, err + } + return consolidateContextCompactionRecords(input, retained), nil } // Accept structured results or Codex's native completed-exec header. Unknown @@ -195,7 +236,61 @@ func contextCompactionOutput(raw json.RawMessage) (func(string) json.RawMessage, var contextCompactionNativeResult = regexp.MustCompile(`(?s)\A((?:Chunk ID: [^\r\n]+\n)?Wall time: [0-9]+(?:\.[0-9]+)? seconds\nProcess exited with code 0\n(?:Original token count: [0-9]+\n)?(?:Output|Final output):\n)(.*)\z`) -var contextCompactionGoRoutine = regexp.MustCompile(`^(=== (RUN|PAUSE|CONT) +\S+|[ \t]*--- PASS: \S+ \([0-9]+(\.[0-9]+)?s\))$`) +var ( + contextCompactionGoEvent = regexp.MustCompile(`^=== (RUN|PAUSE|CONT) +(\S+)$`) + contextCompactionGoOutcome = regexp.MustCompile(`^[ \t]*--- (PASS|FAIL|SKIP): (\S+) \([0-9]+(?:\.[0-9]+)?s\)$`) +) + +func compactionReduceGoTestOutput(text string) (string, int) { + type testRun struct { + runs, passes int + otherOutcome bool + } + runs := make(map[string]*testRun) + state := func(name string) *testRun { + if runs[name] == nil { + runs[name] = new(testRun) + } + return runs[name] + } + for line := range strings.SplitAfterSeq(text, "\n") { + trimmed := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") + if match := contextCompactionGoEvent.FindStringSubmatch(trimmed); match != nil { + if match[1] == "RUN" { + state(match[2]).runs++ + } + continue + } + if match := contextCompactionGoOutcome.FindStringSubmatch(trimmed); match != nil { + if match[1] == "PASS" { + state(match[2]).passes++ + } else { + state(match[2]).otherOutcome = true + } + } + } + corroborated := func(name string) bool { + run := runs[name] + return run != nil && run.runs > 0 && run.runs == run.passes && !run.otherOutcome + } + + var kept strings.Builder + removed := 0 + for line := range strings.SplitAfterSeq(text, "\n") { + trimmed := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") + if match := contextCompactionGoEvent.FindStringSubmatch(trimmed); match != nil && corroborated(match[2]) { + removed++ + continue + } + if match := contextCompactionGoOutcome.FindStringSubmatch(trimmed); match != nil && + match[1] == "PASS" && corroborated(match[2]) { + removed++ + continue + } + kept.WriteString(line) + } + return kept.String(), removed +} // Parse, never execute or expand dynamic shell syntax. Compound commands, // redirections, wrappers, assignments, and substitutions are outside this diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go index 4b15c0d3..2dec2a7d 100644 --- a/internal/router/context_compaction_codex_test.go +++ b/internal/router/context_compaction_codex_test.go @@ -323,7 +323,17 @@ func runManualCompactionProbe(command *exec.Cmd, directory, prompt string) error Params json.RawMessage `json:"params"` Error json.RawMessage `json:"error"` } + pending := make([]rpcMessage, 0) + matches := func(message rpcMessage, id int, method string) bool { + return id != 0 && message.ID == id || method != "" && message.Method == method + } receive := func(id int, method string) (rpcMessage, error) { + for index, message := range pending { + if matches(message, id, method) { + pending = append(pending[:index], pending[index+1:]...) + return message, nil + } + } for { var message rpcMessage if err := decoder.Decode(&message); err != nil { @@ -332,9 +342,10 @@ func runManualCompactionProbe(command *exec.Cmd, directory, prompt string) error if len(message.Error) > 0 || message.Method == "error" { return message, fmt.Errorf("app-server error: %+v", message) } - if (id != 0 && message.ID == id) || (method != "" && message.Method == method) { + if matches(message, id, method) { return message, nil } + pending = append(pending, message) } } if err := send(1, "initialize", map[string]any{"clientInfo": map[string]any{"name": "mekugi_compaction_test", "version": "1"}}); err != nil { diff --git a/internal/router/context_compaction_envelope.go b/internal/router/context_compaction_envelope.go index 2c8984fb..538ba252 100644 --- a/internal/router/context_compaction_envelope.go +++ b/internal/router/context_compaction_envelope.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/gofrs/flock" @@ -31,12 +32,19 @@ const ( // history travels in the envelope; no transcript archive or retrieval is used. type contextCompactor struct { keyPath string + aeadMu sync.Mutex + aead cipher.AEAD } func (c *contextCompactor) cipher(ctx context.Context, create bool) (cipher.AEAD, error) { if err := ctx.Err(); err != nil { return nil, err } + c.aeadMu.Lock() + defer c.aeadMu.Unlock() + if c.aead != nil { + return c.aead, nil + } if c.keyPath == "" { return nil, errors.New("mekugi compaction key path is not configured") } @@ -72,7 +80,12 @@ func (c *contextCompactor) cipher(ctx context.Context, create bool) (cipher.AEAD if err != nil { return nil, err } - return cipher.NewGCMWithRandomNonce(block) + aead, err := cipher.NewGCMWithRandomNonce(block) + if err != nil { + return nil, err + } + c.aead = aead + return aead, nil } func (c *contextCompactor) seal(ctx context.Context, items []json.RawMessage) (json.RawMessage, error) { diff --git a/internal/router/context_compaction_envelope_test.go b/internal/router/context_compaction_envelope_test.go index feeaa727..93db9e39 100644 --- a/internal/router/context_compaction_envelope_test.go +++ b/internal/router/context_compaction_envelope_test.go @@ -59,21 +59,34 @@ func TestCompactionEnvelopeSurvivesRestartAndRejectsDamage(t *testing.T) { func TestCompactionEnvelopeConcurrentKeyCreationAndProviderIsolation(t *testing.T) { path := filepath.Join(t.TempDir(), "compaction.key") items := []json.RawMessage{mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "Keep me."})} + sealed := make([]json.RawMessage, 8) var workers sync.WaitGroup - for range 8 { + for index := range sealed { workers.Go(func() { compactor := &contextCompactor{keyPath: path} - sealed, err := compactor.seal(t.Context(), items) + var err error + sealed[index], err = compactor.seal(t.Context(), items) if err != nil { t.Error(err) return } - if _, _, err := (&contextCompactor{keyPath: path}).open(t.Context(), sealed); err != nil { + if _, _, err := compactor.open(t.Context(), sealed[index]); err != nil { t.Error(err) } }) } workers.Wait() + + verifier := &contextCompactor{keyPath: path} + for index, envelope := range sealed { + if len(envelope) == 0 { + continue + } + if _, _, err := verifier.open(t.Context(), envelope); err != nil { + t.Errorf("worker %d used a different installation key: %v", index, err) + } + } + provider := mustMarshalJSON(map[string]any{"type": "compaction", "encrypted_content": "provider-owned"}) if _, local, err := (&contextCompactor{}).open(t.Context(), provider); local || err != nil { t.Fatal("provider-owned compaction was interpreted locally") diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go index ed502adc..8837b92c 100644 --- a/internal/router/context_compaction_http.go +++ b/internal/router/context_compaction_http.go @@ -148,6 +148,9 @@ func (c *contextCompactor) prepare(ctx context.Context, parsed *parsedResponsesR compactionTargetTokens, compactionOvershootTokens, reduceContextCompactionWithPlan, compactionVisibleStringTokens) if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return nil, err + } return fail(http.StatusUnprocessableEntity, err.Error()) } capsule, err := c.seal(ctx, reduced) diff --git a/internal/router/context_compaction_http_test.go b/internal/router/context_compaction_http_test.go index 94dbdcc1..7b085745 100644 --- a/internal/router/context_compaction_http_test.go +++ b/internal/router/context_compaction_http_test.go @@ -143,6 +143,29 @@ func TestCompactionHTTPStreamingV2AndFailures(t *testing.T) { if response.Code != http.StatusOK || strings.Count(response.Body.String(), "event: response.output_item.done\n") != 1 || !strings.Contains(response.Body.String(), "event: response.completed\n") { t.Fatalf("V2 result = %d: %s", response.Code, response.Body.String()) } + for _, test := range []struct { + name string + input []json.RawMessage + }{ + {"trigger_only", []json.RawMessage{mustMarshalJSON(map[string]any{"type": "compaction_trigger"})}}, + {"protected_with_trigger", []json.RawMessage{ + mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "protected"}), + mustMarshalJSON(map[string]any{"type": "compaction_trigger"}), + }}, + } { + t.Run(test.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(string(mustMarshalJSON(map[string]any{ + "model": "gpt-5", "input": test.input, "stream": true, + })))) + request.Header.Set(codexTurnMetadataHeader, `{"request_kind":"compaction","compaction":{"implementation":"responses_compaction_v2"}}`) + response := httptest.NewRecorder() + compactor.handler(next)(response, request) + if response.Code != http.StatusUnprocessableEntity || strings.Contains(response.Body.String(), contextCompactionPrefix) { + t.Fatalf("unsafe V2 compaction accepted: %d: %s", response.Code, response.Body.String()) + } + }) + } + for _, body := range []string{ `{"model":"gpt-5","input":[]}`, `{"model":"gpt-5","input":[null]}`, diff --git a/internal/router/context_compaction_operation.go b/internal/router/context_compaction_operation.go index d68ea907..39ba809b 100644 --- a/internal/router/context_compaction_operation.go +++ b/internal/router/context_compaction_operation.go @@ -34,7 +34,11 @@ func compactionOperationCall(fields map[string]json.RawMessage) (compactionOpera } case "custom_tool_call": if name == "shell" { - return compactionOperation{tool: "shell", arguments: mustMarshalJSON(map[string]any{"input": jsonString(fields, "input")})}, true + var input *string + if json.Unmarshal(fields["input"], &input) != nil || input == nil { + return compactionOperation{}, false + } + return compactionOperation{tool: "shell", arguments: mustMarshalJSON(map[string]any{"input": *input})}, true } if name == "exec" { return compactionCodeModeOperation(jsonString(fields, "input")) diff --git a/internal/router/context_compaction_read_tool.go b/internal/router/context_compaction_read_tool.go index e5c31d95..7bc04ff3 100644 --- a/internal/router/context_compaction_read_tool.go +++ b/internal/router/context_compaction_read_tool.go @@ -7,6 +7,7 @@ import ( "regexp" "slices" "strings" + "unicode/utf8" ) const ( @@ -50,8 +51,8 @@ func compactionRetiredReadToolOutput(raw json.RawMessage, tool string) (json.Raw return compactionRetiredTruncatedReadToolOutput(parts, serialized, tool) } if rawError, exists := result["isError"]; exists { - var isError bool - if json.Unmarshal(rawError, &isError) != nil || isError { + var isError *bool + if json.Unmarshal(rawError, &isError) != nil || isError == nil || *isError { return nil, false } } @@ -287,7 +288,8 @@ func compactionRetiredSearchItemMaps(items []map[string]json.RawMessage) (json.R for index, item := range items { identified := false for _, key := range []string{"url", "source_url", "title", "id", "objectID", "citation", "citation_id", "source"} { - if raw, exists := item[key]; exists && len(raw) > 0 && string(raw) != "null" { + var identity string + if raw, exists := item[key]; exists && json.Unmarshal(raw, &identity) == nil && strings.TrimSpace(identity) != "" { identified = true break } @@ -300,7 +302,15 @@ func compactionRetiredSearchItemMaps(items []map[string]json.RawMessage) (json.R itemChanged := false for _, key := range []string{"snippet", "snippets", "body", "content", "text", "markdown", "highlights"} { raw, exists := item[key] - if !exists || len(raw) < 256 { + if !exists { + continue + } + var text string + if json.Unmarshal(raw, &text) == nil { + if len(text) < 256 { + continue + } + } else if len(raw) < 256 { continue } body, ok := compactionRetiredReadJSONBody(raw) @@ -410,8 +420,16 @@ func compactionRetiredReadJSONBody(raw json.RawMessage) (json.RawMessage, bool) var stringsOnly []string if json.Unmarshal(raw, &stringsOnly) == nil && len(stringsOnly) > 0 { + changed := false for index, text := range stringsOnly { + if len(text) < 256 { + continue + } stringsOnly[index] = compactionRetiredReadBodyString(text) + changed = true + } + if !changed { + return nil, false } return mustMarshalJSON(stringsOnly), true } @@ -478,6 +496,32 @@ func compactionRetiredDocumentText(text string) (string, bool) { return evidence, true } +func compactionBoundReadBodyEvidenceLine(line string) string { + const ( + maxLineBytes = 1024 + edgeBytes = 384 + ) + if len(line) <= maxLineBytes { + return line + } + content, ending := line, "" + if trimmed, ok := strings.CutSuffix(content, "\r\n"); ok { + content, ending = trimmed, "\r\n" + } else if trimmed, ok := strings.CutSuffix(content, "\n"); ok { + content, ending = trimmed, "\n" + } + prefixEnd := edgeBytes + for prefixEnd > 0 && !utf8.ValidString(content[:prefixEnd]) { + prefixEnd-- + } + suffixStart := len(content) - edgeBytes + for suffixStart < len(content) && !utf8.RuneStart(content[suffixStart]) { + suffixStart++ + } + return fmt.Sprintf("%s\n[mekugi compaction: oversized retained documentation evidence line truncated; original bytes=%d; omitted bytes=%d]\n%s%s", + content[:prefixEnd], len(content), len(content)-prefixEnd-(len(content)-suffixStart), content[suffixStart:], ending) +} + func compactionReadBodyEvidence(text string) string { lines := strings.SplitAfter(text, "\n") keep := make([]bool, len(lines)) @@ -516,7 +560,7 @@ func compactionReadBodyEvidence(text string) string { var result strings.Builder for index, line := range lines { if keep[index] { - result.WriteString(line) + result.WriteString(compactionBoundReadBodyEvidenceLine(line)) } } return result.String() diff --git a/internal/router/context_compaction_read_tool_test.go b/internal/router/context_compaction_read_tool_test.go index 19c07646..5c9cda45 100644 --- a/internal/router/context_compaction_read_tool_test.go +++ b/internal/router/context_compaction_read_tool_test.go @@ -296,6 +296,48 @@ func TestCompactionRetiresActualDocumentationSearchSchema(t *testing.T) { } } +func TestCompactionSearchBodyThresholdsAndIdentities(t *testing.T) { + t.Run("decoded short escaped string", func(t *testing.T) { + items := []map[string]json.RawMessage{{ + "id": mustMarshalJSON("result-1"), + "snippet": mustMarshalJSON(strings.Repeat("\"", 200)), + }} + if _, changed := compactionRetiredSearchItemMaps(items); changed { + t.Fatal("short decoded snippet was retired because its JSON encoding was large") + } + }) + + t.Run("mixed string array", func(t *testing.T) { + const short = "keep this short snippet" + items := []map[string]json.RawMessage{{ + "id": mustMarshalJSON("result-1"), + "snippets": mustMarshalJSON([]string{strings.Repeat("large body ", 40), short}), + }} + reduced, changed := compactionRetiredSearchItemMaps(items) + if !changed { + t.Fatal("large snippet was not retired") + } + var decoded []map[string]json.RawMessage + var snippets []string + if json.Unmarshal(reduced, &decoded) != nil || json.Unmarshal(decoded[0]["snippets"], &snippets) != nil || + len(snippets) != 2 || snippets[1] != short || !strings.Contains(snippets[0], "historical document body retired") { + t.Fatalf("mixed snippets were not reduced conservatively: %s", reduced) + } + }) + + t.Run("invalid identity", func(t *testing.T) { + for _, identity := range []any{"", false, map[string]any{"url": "opaque"}} { + items := []map[string]json.RawMessage{{ + "url": mustMarshalJSON(identity), + "snippet": mustMarshalJSON(strings.Repeat("unidentified body ", 40)), + }} + if _, changed := compactionRetiredSearchItemMaps(items); changed { + t.Fatalf("body with unusable identity %v was retired", identity) + } + } + }) +} + func TestCompactionSuccessfulReadWithoutBodyReductionStaysEligible(t *testing.T) { tool := compactionDocsFetchTool source := readToolSource(tool, `{"url":"https://developers.openai.com/api/docs/index"}`) @@ -426,6 +468,9 @@ func TestCompactionDocumentationReadFailuresStayNative(t *testing.T) { {"unknown error flag", goodSource, readToolResultOutput("operation_00", "Script completed\n", map[string]any{ "isError": "false", "content": []any{map[string]any{"type": "text", "text": "ambiguous status"}}, })}, + {"null error flag", goodSource, readToolResultOutput("operation_00", "Script completed\n", map[string]any{ + "isError": nil, "content": []any{map[string]any{"type": "text", "text": "ambiguous null status"}}, + })}, {"media", goodSource, readToolResultOutput("operation_00", "Script completed\n", map[string]any{ "content": []any{map[string]any{"type": "image", "data": "opaque-media"}}, })}, @@ -518,6 +563,17 @@ func TestCompactionTruncatedDocumentationSearchRetiresCompleteBodySpan(t *testin } } +func TestCompactionReadBodyEvidenceBoundsOversizedLines(t *testing.T) { + body := "# " + strings.Repeat("oversized documentation evidence ", 400) + + "https://developers.openai.com END" + evidence, ok := compactionRetiredDocumentText(body) + if !ok || len(evidence) >= len(body)/2 || + !strings.Contains(evidence, "oversized retained documentation evidence line truncated") || + !strings.HasPrefix(evidence, "# ") || !strings.HasSuffix(evidence, " END") { + t.Fatalf("oversized evidence line was not bounded with its edges intact: %d of %d bytes", len(evidence), len(body)) + } +} + func TestCompactionDocumentationReadProtectionAndRecentFrontier(t *testing.T) { result := map[string]any{ "content": []any{map[string]any{ diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index e3618adb..10007400 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -180,7 +180,7 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) enqueueOriginal := func(id string, plan *compactionRetirement) { queue = append(queue, referenceText{fields[plan.call]["input"], id, true}, referenceText{fields[plan.call]["arguments"], id, true}, - referenceText{fields[plan.result]["output"], id, false}) + referenceText{fields[plan.result]["output"], id, true}) } revision := 0 var pin func(string) @@ -309,7 +309,7 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } case "function_call_output", "custom_tool_call_output": if p := plans[id]; p == nil || !p.eligible { - queue = append(queue, referenceText{item["output"], id, false}) + queue = append(queue, referenceText{item["output"], id, true}) } default: queue = append(queue, referenceText{item["content"], "", true}, referenceText{item["summary"], "", true}) @@ -406,13 +406,14 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } } - // Pinning is monotonic. Repeat reference closure only when profitability - // restores original content, which can expose references that were absent - // from the proposed factual record. At most one pass per retired candidate - // can restore content, so this reaches a bounded stable result. + // Pinning and row restoration are monotonic. Repeat reference closure when + // either exposes references that were absent from the proposed factual + // record. At most one pass per retired candidate can restore content, so + // this reaches a bounded stable result. for { + beforeRevision := revision drainReferences() - for _, plan := range plans { + for id, plan := range plans { if !plan.eligible || (len(plan.rows) == 0 && len(plan.ranges) == 0) { continue } @@ -421,10 +422,13 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) if !ok { return input } - plan.output = retained + if string(plan.output) != string(retained) { + plan.output = retained + queue = append(queue, referenceText{retained, id, true}) + revision++ + } } - beforeProfitability := revision for id, plan := range plans { if !plan.eligible || groupAt[plan.call] >= 0 { continue @@ -449,7 +453,7 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } } } - if revision == beforeProfitability { + if revision == beforeRevision { break } } @@ -583,16 +587,14 @@ func compactionFailedOutput(raw json.RawMessage) (func(string) json.RawMessage, return nil, "", false } -// Failed operations retain every unclassified line. Only known routine test -// progress/pass lines and unreferenced verified source rows are positively -// identified as historical bulk and eligible for omission. +// Failed operations retain every unclassified line. Only corroborated runner +// lines and unreferenced verified source rows are positively identified as +// historical bulk and eligible for omission. func compactionRetiredFailedText(text string, referenced map[string]bool, ranges [][2]string) string { + reduced, removed := compactionReduceGoTestOutput(text) var result strings.Builder - removed := 0 - for line := range strings.SplitAfterSeq(text, "\n") { - trimmed := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") - if contextCompactionGoRoutine.MatchString(trimmed) || - (compactionCompleteSourceRow.MatchString(line) && !compactionTextReferencesRows(line, referenced, ranges)) { + for line := range strings.SplitAfterSeq(reduced, "\n") { + if compactionCompleteSourceRow.MatchString(line) && !compactionTextReferencesRows(line, referenced, ranges) { removed++ continue } @@ -601,7 +603,7 @@ func compactionRetiredFailedText(text string, referenced map[string]bool, ranges if removed == 0 { return text } - return fmt.Sprintf("[mekugi: omitted %d positively identified routine/source lines from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) + return fmt.Sprintf("[mekugi: omitted %d positively identified runner/source lines from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) } func compactionRetiredTextKeepingRows(text string, referenced map[string]bool, ranges [][2]string) string { diff --git a/internal/router/context_compaction_retirement_test.go b/internal/router/context_compaction_retirement_test.go index 15479c69..a7d85830 100644 --- a/internal/router/context_compaction_retirement_test.go +++ b/internal/router/context_compaction_retirement_test.go @@ -121,6 +121,25 @@ func TestCompactionRetirementPreservesCarrierNotice(t *testing.T) { } } +func TestCompactionOperationRejectsMalformedShellInput(t *testing.T) { + for _, fields := range []map[string]json.RawMessage{ + {"type": mustMarshalJSON("custom_tool_call"), "name": mustMarshalJSON("shell")}, + {"type": mustMarshalJSON("custom_tool_call"), "name": mustMarshalJSON("shell"), "input": json.RawMessage("null")}, + {"type": mustMarshalJSON("custom_tool_call"), "name": mustMarshalJSON("shell"), "input": json.RawMessage("17")}, + {"type": mustMarshalJSON("custom_tool_call"), "name": mustMarshalJSON("shell"), "input": json.RawMessage(`{"cmd":"pwd"}`)}, + } { + if _, ok := compactionOperationCall(fields); ok { + t.Fatalf("malformed shell input was accepted: %s", mustMarshalJSON(fields)) + } + } + valid := map[string]json.RawMessage{ + "type": mustMarshalJSON("custom_tool_call"), "name": mustMarshalJSON("shell"), "input": mustMarshalJSON(""), + } + if _, ok := compactionOperationCall(valid); !ok { + t.Fatal("valid empty shell input was rejected") + } +} + func TestCompactionRetiresAppliedPatchBodyNotFailedPatch(t *testing.T) { patch := "*** Begin Patch\n*** Add File: example.go\n+" + strings.Repeat("// old implementation detail\n+", 500) + "\n*** End Patch\n" report := "in example.go\nfiles add=1 update=0 move=0 delete=0\n" @@ -371,6 +390,59 @@ func TestCompactionRetirementFollowsOnlySurvivingReplacementNotes(t *testing.T) }) } +func TestCompactionRetirementFollowsReferencesExposedByRetainedResults(t *testing.T) { + t.Run("pinned tool result row", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", + "Continue with exact row 17:abcd.\n"+strings.Repeat("consumer historical detail\n", 500), 0) + referencedLine := "17:abcd required source evidence\n" + items[6] = compactTestOutput("operation_01", + referencedLine+strings.Repeat("unreferenced source detail\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep operation_00.", + })) + + got := retireCompactionOperations(items) + if !strings.Contains(string(mustMarshalJSON(got)), strings.TrimSpace(referencedLine)) { + t.Fatal("row reference in a pinned tool result lost its source evidence") + } + }) + + t.Run("restored factual row references operation", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", + "17:abcd retained evidence requires operation_01\n"+strings.Repeat("consumer historical detail\n", 500), 0) + items[6] = compactTestOutput("operation_01", strings.Repeat("required dependency detail\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep exact row 17:abcd.", + })) + + got := retireCompactionOperations(items) + for _, index := range []int{4, 5, 6} { + if string(got[index]) != string(items[index]) { + t.Fatalf("reference exposed by restored factual row did not pin operation item %d", index) + } + } + }) + + t.Run("restored factual row references another row", func(t *testing.T) { + items := retirementHistory() + items[3] = compactTestOutput("operation_00", + "17:abcd retained evidence requires row 23:beef\n"+strings.Repeat("consumer historical detail\n", 500), 0) + referencedLine := "23:beef required chained source evidence\n" + items[6] = compactTestOutput("operation_01", + referencedLine+strings.Repeat("unreferenced source detail\n", 500), 0) + items = append(items, mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": "Keep exact row 17:abcd.", + })) + + got := retireCompactionOperations(items) + if !strings.Contains(string(mustMarshalJSON(got)), strings.TrimSpace(referencedLine)) { + t.Fatal("row reference exposed by restored factual evidence lost its dependency") + } + }) +} + func TestCompactionRetirementPinsRetainedScriptProducer(t *testing.T) { items := retirementHistory() items[3] = mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "operation_00", diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go index 1a24ff18..386a1595 100644 --- a/internal/router/context_compaction_source.go +++ b/internal/router/context_compaction_source.go @@ -112,6 +112,22 @@ func reduceContextCompactionSourceWithFrontier(original, retained []json.RawMess references = append(references, fields["input"], fields["arguments"]) } case "function_call_output", "custom_tool_call_output": + visitOutput := func(text string) { + compactionSourceVisitDecodedReferences(text, func(decoded string) { + for line := range strings.SplitAfterSeq(decoded, "\n") { + if match := compactionCompleteSourceRow.FindStringSubmatch(line); match != nil { + line = strings.Replace(line, match[1], "", 1) + } + for _, match := range compactionSourceRangeReference.FindAllStringSubmatch(line, -1) { + ranges = append(ranges, [2]string{match[1], match[2]}) + } + for _, row := range compactionRowReference.FindAllString(line, -1) { + rowReferences[row] = true + } + } + }, &unsafeEncoding) + } + compactionVisitReferenceStrings(fields["output"], visitOutput) continue default: references = append(references, fields["content"], fields["summary"]) diff --git a/internal/router/context_compaction_source_test.go b/internal/router/context_compaction_source_test.go index 0fc88289..c88d912a 100644 --- a/internal/router/context_compaction_source_test.go +++ b/internal/router/context_compaction_source_test.go @@ -96,6 +96,73 @@ func TestCompactionSourcePrunesOnlyUnreferencedRows(t *testing.T) { t.Fatal("source pruning was not idempotent") } } + +func TestCompactionSourceFollowsToolOutputRowReferences(t *testing.T) { + rows := compactionSourceTestRows("", 18) + successful := string(mustMarshalJSON(map[string]any{ + "output": "Continue from row 3:0003.", "exit_code": 0, "wall_time_seconds": 1, + })) + live := string(mustMarshalJSON(map[string]any{ + "output": "Continue from row 3:0003.", "exit_code": 0, "session_id": 42, + })) + for _, test := range []struct { + name string + call, result json.RawMessage + }{ + { + name: "function_call_output", + call: compactTestCall("consumer", "pwd"), + result: compactTestOutput("consumer", "Continue from row 3:0003.", 0), + }, + { + name: "failed_function_call_output", + call: compactTestCall("consumer", "pwd"), + result: compactTestOutput("consumer", "Continue from row 3:0003.", 1), + }, + { + name: "custom_tool_call_output", + call: mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "shell", "call_id": "consumer", "input": "pwd", + }), + result: mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "consumer", "output": successful, + }), + }, + { + name: "live_custom_tool_call_output", + call: mustMarshalJSON(map[string]any{ + "type": "custom_tool_call", "name": "shell", "call_id": "consumer", "input": "pwd", + }), + result: mustMarshalJSON(map[string]any{ + "type": "custom_tool_call_output", "call_id": "consumer", "output": live, + }), + }, + } { + t.Run(test.name, func(t *testing.T) { + items := []json.RawMessage{ + mustMarshalJSON(map[string]any{ + "type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Inspect source."}}, + }), + compactTestCall("source-old", "hread source.go"), + compactTestOutput("source-old", strings.Join(rows, ""), 0), + test.call, + test.result, + mustMarshalJSON(map[string]any{ + "type": "function_call", "name": "unknown", "call_id": "unknown-old", "arguments": "{}", + }), + compactTestOutput("unknown-old", "unknown companion\n", 0), + } + items = append(items, compactionSourceTestRecent()...) + + got := reduceContextCompaction(items) + text := compactionSourceTestOutputText(t, got[2]) + if !strings.Contains(text, rows[2]) || strings.Contains(text, rows[10]) { + t.Fatal("tool-produced row reference was not preserved conservatively") + } + }) + } +} + func TestCompactionSourceKeepsProtectedOutputsByteExact(t *testing.T) { rows := strings.Join(compactionSourceTestRows("", 18), "") tests := []struct { diff --git a/internal/router/context_compaction_test.go b/internal/router/context_compaction_test.go index a097ea06..60f3877c 100644 --- a/internal/router/context_compaction_test.go +++ b/internal/router/context_compaction_test.go @@ -1,7 +1,9 @@ package router import ( + "context" "encoding/json" + "errors" "strings" "testing" ) @@ -98,15 +100,28 @@ func TestContextCompactionSearchRequiresRetainedExactEvidence(t *testing.T) { } } +func TestContextCompactionHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := reduceContextCompactionContext(ctx, compactHTTPHistory()); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled reduction returned %v", err) + } +} + func TestContextCompactionKeepsUnknownTestDiagnostics(t *testing.T) { + diagnostic := "--- PASS: retained diagnostic (0.1s)" + runnerLines := "=== RUN TestA\n--- PASS: TestA (0.1s)\n" items := []json.RawMessage{ compactTestCall("tests", "go test -v ./..."), - compactTestOutput("tests", strings.Repeat("=== RUN TestA\n--- PASS: TestA (0.1s)\n", 20)+" test.go:10: important diagnostic\nPASS\nok example 0.1s\n", 0), + compactTestOutput("tests", strings.Repeat(runnerLines, 20)+diagnostic+"\n=== RUN retained run diagnostic\n test.go:10: important diagnostic\nPASS\nok example 0.1s\n", 0), compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0), } got := reduceContextCompaction(items) - if string(got[1]) == string(items[1]) || !strings.Contains(string(got[1]), "important diagnostic") { - t.Fatal("non-routine diagnostic removed") + if string(got[1]) == string(items[1]) || !strings.Contains(string(got[1]), diagnostic) || + !strings.Contains(string(got[1]), "retained run diagnostic") || + !strings.Contains(string(got[1]), "important diagnostic") || + strings.Contains(string(got[1]), "--- PASS: TestA") { + t.Fatal("corroborated runner output was not reduced or unmatched diagnostic was removed") } } diff --git a/internal/router/context_compaction_websocket_test.go b/internal/router/context_compaction_websocket_test.go index 8a77d431..81b3d9eb 100644 --- a/internal/router/context_compaction_websocket_test.go +++ b/internal/router/context_compaction_websocket_test.go @@ -257,6 +257,9 @@ func TestCompactionWebSocketFailsClosed(t *testing.T) { if jsonString(event, "type") != "error" || string(event["status"]) != fmt.Sprint(tc.status) || calls.Load() != 0 { t.Fatalf("not fail-closed: %s, provider calls %d", mustMarshalJSON(event), calls.Load()) } + if _, _, err := conn.Read(ctx); err == nil { + t.Fatal("failed compaction left the WebSocket connection open") + } }) } } diff --git a/internal/router/server_websocket.go b/internal/router/server_websocket.go index b1594339..b234b99e 100644 --- a/internal/router/server_websocket.go +++ b/internal/router/server_websocket.go @@ -857,6 +857,9 @@ func (w *webSocketOutput) message(payload []byte) error { // provider successor. Its accepted steering remains pending until // that provider response actually starts. e.history.parent = nil + for _, item := range e.history.input { + s.retainedBytes -= len(item) + } e.history.input = nil } else { s.commitSteering(e.history, e.parentID) From 2db29fae8dd371f8fb503818cef7bee27a95c726 Mon Sep 17 00:00:00 2001 From: yusing Date: Fri, 11 Sep 2026 15:36:36 +0000 Subject: [PATCH 10/13] fix(router): summarize terminal direct Go test results Reduce recognized terminal `go test` output to pass/fail status and distinct failed test names, including results that are recent or referenced. Preserve live and unknown results while limiting generic failed-operation reduction to unreferenced verified source rows. Update reference traversal, retirement handling, documentation, and regression coverage for idempotent summaries and native execution headers. --- README.md | 13 +- doc/architecture/compaction.md | 17 +-- doc/spec/compaction.md | 36 +++--- internal/router/context_compaction.go | 79 ++++-------- .../router/context_compaction_codex_test.go | 7 +- .../router/context_compaction_retirement.go | 17 ++- .../context_compaction_retirement_test.go | 2 +- internal/router/context_compaction_source.go | 10 +- internal/router/context_compaction_test.go | 115 ++++++++++++------ 9 files changed, 169 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 51fc1a86..b31d389f 100644 --- a/README.md +++ b/README.md @@ -158,11 +158,14 @@ sessions, including when moving them to another installation. Compaction can discard unmarked historical details from older finished operations, even while the task is still open. It keeps factual execution records, requests, visible decisions, diagnostic excerpts, referenced evidence, and recent/live work. -Recognized applied patch bodies and associated older opaque reasoning can also be -retired. Older failed-command output can lose unreferenced bulk while retaining -errors, warnings, and provenance. Truncated or oversized documentation can also -lose unreferenced bulk; a single oversized evidence line retains bounded excerpts. -Discarded details are not currently retrievable through Mekugi. +Terminal results from recognized direct `go test` calls are an exception: even when +recent or referenced, they retain only pass/fail status and reported failed test +names, while detailed runner output is discarded. Recognized applied patch bodies +and associated older opaque reasoning can also be retired. Other older failed-command +output can lose unreferenced bulk while retaining errors, warnings, and provenance. +Truncated or oversized documentation can also lose unreferenced bulk; a single +oversized evidence line retains bounded excerpts. Discarded details are not currently +retrievable through Mekugi. Unknown or ambiguous execution states remain intact. If nothing qualifies, compaction reports an error rather than asking a provider for a summary. diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index 40bef7a9..d631e478 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -42,13 +42,16 @@ responsibility. Reasoning groups are retired atomically with eligible calls, retaining visible summaries. The recent frontier, explicit evidence references, live operations, and unknown completion states constrain selection. Completed failures may lose identified historical bulk, but keep their exact invocation, -status, diagnostic blocks, and unresolved details. Reference closure and complete-group -profitability reach a stable result before replacement; restoring a consumer also -restores every dependency exposed by its original content. Supported text and -JavaScript escapes are decoded for reference matching, while suspicious encodings -retain evidence conservatively. These records are factual history, not new -instructions or an external archive. The envelope contains selected history only; -deferred retrieval must not be implied by a digest or retirement marker. +status, diagnostic blocks, and unresolved details. Recognized terminal direct +`go test` results are the deliberate exception: they keep status and reported +failed test names rather than detailed runner output, even inside the recent +frontier. Reference closure and complete-group profitability reach a stable result +before replacement; restoring a consumer also restores every dependency exposed by +its retained content. Supported text and JavaScript escapes are decoded for reference +matching, while suspicious encodings retain evidence conservatively. These records +are factual history, not new instructions or an external archive. The envelope +contains selected history only; deferred retrieval must not be implied by a digest +or retirement marker. Completed native output reduction is independent of whole-group retirement: it may shorten historical output while leaving calls and opaque reasoning native. diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index cb53b062..2bbb94e5 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -37,9 +37,11 @@ completion proves irrelevance. Before lossy retirement, evidence reducers can shorten redundant output: -- Successful Go test results with a structured result or native Codex exec header can omit routine run, pause, and continue - progress lines. Test outcome lines, the exact call, exit status, package summaries, - and other output remain. +- Terminal results from recognized direct `go test` calls retain pass/fail status + and the distinct failed test names reported by `--- FAIL:` lines. Detailed + runner output, diagnostics, package names, and successful test names are + discarded, including for recent or explicitly referenced completed results. + Live and unknown Go test results remain native. - Successful search listings from recognized `rg`, `hgrep`, or `find` calls can be replaced with a reference to a later retained read result containing the exact same complete listing. The original call and completion metadata stay. @@ -54,8 +56,9 @@ Finished-operation retirement: - By default keep the newest eight tool invocations and their results. The budget policy below may relax this warm recency buffer, never the newest invocation/result, live/unmatched or duplicate identities, unknown completion - states, or explicitly referenced calls and retained-script producers. Referenced verified rows and - numeric ranges may survive exactly in factual records instead of pinning their + states, or explicitly referenced calls and retained-script producers. The + terminal Go test reduction above still applies inside this buffer. Referenced + verified rows and numeric ranges may survive exactly in factual records instead of pinning their whole successful group. Reference matching decodes nested argument/result envelopes and supported escaped text; suspicious encodings retain evidence. - Recognize native shell/exec calls, terminal stdin polls, static result-preserving @@ -77,16 +80,17 @@ Finished-operation retirement: - Replace eligible calls/results with factual, versioned assistant-role records, not executable-looking truncated calls. Preserve relative timeline order; consecutive newly generated historical records may share explanatory framing. - Keep exact shell invocation arguments, call identity, observed completion - metadata, test outcome lines, and diagnostic excerpts. Standard Python - tracebacks preserve the entire remaining output because multiline exceptions - and notes have no reliable generic end marker. + Keep exact shell invocation arguments, call identity, and observed completion + metadata. Except for recognized terminal Go test results, keep test outcome lines + and diagnostic excerpts. Standard Python tracebacks preserve the entire remaining + output because multiline exceptions and notes have no reliable generic end marker. - A completed failed command is eligible for removal of positively identified, - unreferenced historical bulk, not removal of its failure. Keep its exact - invocation, exit status, error and diagnostic blocks, traceback chains, - unresolved details, and referenced evidence. Terminal completion is not - success or proof that a failure was resolved. Live and unknown completion - states remain protected. + unreferenced historical bulk, not removal of its failure. Except for recognized + terminal Go test results, keep its exact invocation, exit status, error and + diagnostic blocks, traceback chains, unresolved details, and referenced evidence. + A terminal Go test keeps its invocation and exit status plus reported failed test + names only. Terminal completion is not success or proof that a failure was resolved. + Live and unknown completion states remain protected. - For successfully applied translated patches, keep affected paths, operation kinds, a patch digest, and exact application facts, diagnostics, and referenced evidence. Unreferenced verified source rows in a successful report may shrink @@ -104,8 +108,8 @@ Finished-operation retirement: reasoning in a blocked group. Whole-output references stay intact; referenced full or partial verified rows and numeric ranges remain exact. Live, unknown, and output inside the selected recent frontier is not made eligible - by this output-only pass; failed output obeys the diagnostic-preservation rule - above. + by this output-only pass except for recognized terminal Go test results; + failed output obeys the diagnostic-preservation rule above. Existing provider-owned compaction items remain untouched. Factual records use a compact versioned representation with exact invocation values, ordered output parts, and unknown metadata. Unreferenced transport item diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 09242ea1..83c54ed7 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -119,7 +119,7 @@ func reduceContextCompactionPlan(ctx context.Context, input []json.RawMessage, p if err := ctx.Err(); err != nil { return original, err } - if index == lastResult || protected[current.CallID] || (current.Type != "function_call_output" && current.Type != "custom_tool_call_output") { + if current.Type != "function_call_output" && current.Type != "custom_tool_call_output" { continue } callIndex, exists := calls[current.CallID] @@ -151,16 +151,21 @@ func reduceContextCompactionPlan(ctx context.Context, input []json.RawMessage, p if kind == "" { continue } + if kind != "go-test" && (index == lastResult || protected[current.CallID]) { + continue + } encode, text, ok := contextCompactionOutput(current.Output) + passed := ok + if !ok && kind == "go-test" { + encode, text, ok = compactionFailedOutput(current.Output) + } if !ok { continue } reduced := text switch kind { case "go-test": - if kept, removed := compactionReduceGoTestOutput(text); removed > 0 { - reduced = fmt.Sprintf("[mekugi: omitted %d corroborated Go test runner lines]\n%s", removed, kept) - } + reduced = contextCompactionGoTestSummary(text, passed) case "search": // A later byte-identical output is explicit replacement evidence, // not an assumption that rerunning a search gives its old answer. @@ -236,60 +241,30 @@ func contextCompactionOutput(raw json.RawMessage) (func(string) json.RawMessage, var contextCompactionNativeResult = regexp.MustCompile(`(?s)\A((?:Chunk ID: [^\r\n]+\n)?Wall time: [0-9]+(?:\.[0-9]+)? seconds\nProcess exited with code 0\n(?:Original token count: [0-9]+\n)?(?:Output|Final output):\n)(.*)\z`) -var ( - contextCompactionGoEvent = regexp.MustCompile(`^=== (RUN|PAUSE|CONT) +(\S+)$`) - contextCompactionGoOutcome = regexp.MustCompile(`^[ \t]*--- (PASS|FAIL|SKIP): (\S+) \([0-9]+(?:\.[0-9]+)?s\)$`) -) +const contextCompactionGoTestSummaryPrefix = "[mekugi compaction: Go test " -func compactionReduceGoTestOutput(text string) (string, int) { - type testRun struct { - runs, passes int - otherOutcome bool - } - runs := make(map[string]*testRun) - state := func(name string) *testRun { - if runs[name] == nil { - runs[name] = new(testRun) - } - return runs[name] - } - for line := range strings.SplitAfterSeq(text, "\n") { - trimmed := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") - if match := contextCompactionGoEvent.FindStringSubmatch(trimmed); match != nil { - if match[1] == "RUN" { - state(match[2]).runs++ - } - continue - } - if match := contextCompactionGoOutcome.FindStringSubmatch(trimmed); match != nil { - if match[1] == "PASS" { - state(match[2]).passes++ - } else { - state(match[2]).otherOutcome = true - } - } +var contextCompactionGoFailure = regexp.MustCompile(`(?m)^[ \t]*--- FAIL: ([^ \t\r\n]+)(?: \([^)]+\))?[ \t]*\r?$`) + +func contextCompactionGoTestSummary(text string, passed bool) string { + if strings.HasPrefix(text, contextCompactionGoTestSummaryPrefix) { + return text } - corroborated := func(name string) bool { - run := runs[name] - return run != nil && run.runs > 0 && run.runs == run.passes && !run.otherOutcome + if passed { + return contextCompactionGoTestSummaryPrefix + "passed; detailed output omitted]\n" } - var kept strings.Builder - removed := 0 - for line := range strings.SplitAfterSeq(text, "\n") { - trimmed := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") - if match := contextCompactionGoEvent.FindStringSubmatch(trimmed); match != nil && corroborated(match[2]) { - removed++ - continue - } - if match := contextCompactionGoOutcome.FindStringSubmatch(trimmed); match != nil && - match[1] == "PASS" && corroborated(match[2]) { - removed++ - continue + seen := make(map[string]bool) + var failed []string + for _, match := range contextCompactionGoFailure.FindAllStringSubmatch(text, -1) { + if !seen[match[1]] { + seen[match[1]] = true + failed = append(failed, match[1]) } - kept.WriteString(line) } - return kept.String(), removed + if len(failed) == 0 { + return contextCompactionGoTestSummaryPrefix + "failed; no failed test name was reported; detailed output omitted]\n" + } + return contextCompactionGoTestSummaryPrefix + "failed; failed tests: " + strings.Join(failed, ", ") + "; detailed output omitted]\n" } // Parse, never execute or expand dynamic shell syntax. Compound commands, diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go index 2dec2a7d..a5b410e0 100644 --- a/internal/router/context_compaction_codex_test.go +++ b/internal/router/context_compaction_codex_test.go @@ -140,10 +140,7 @@ func TestProbe(t *testing.T) { consolidatedCompletion := strings.HasPrefix(text, "[mekugi historical facts v4;") && strings.Contains(text, "[i]\ncall=\"probe_go\"\ntool=\"exec_command\"") && strings.Contains(text, "[o:same-call]\n") - if (standaloneCompletion || consolidatedCompletion) && strings.Contains(text, "\nbody:\n") && - strings.Contains(text, "Process exited with code 0\n") && strings.Contains(text, "compactionprobe") { - retiredGo = true - } + retiredGo = standaloneCompletion || consolidatedCompletion } } if (recordType == "function_call" || recordType == "function_call_output") && @@ -151,7 +148,7 @@ func TestProbe(t *testing.T) { nativeGo = true if recordType == "function_call_output" && !probe.retirement { output := jsonString(record, "output") - restored.Store(strings.Contains(output, "[mekugi: omitted") && strings.Contains(output, "compactionprobe")) + restored.Store(strings.Contains(output, "Go test passed") && !strings.Contains(output, "compactionprobe")) } } if strings.HasPrefix(jsonString(record, "encrypted_content"), "mekugi.compaction.") { diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index 10007400..6b6dec94 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -587,13 +587,12 @@ func compactionFailedOutput(raw json.RawMessage) (func(string) json.RawMessage, return nil, "", false } -// Failed operations retain every unclassified line. Only corroborated runner -// lines and unreferenced verified source rows are positively identified as -// historical bulk and eligible for omission. +// Failed operations retain every unclassified line. Unreferenced verified +// source rows are the only historical bulk removed by the generic reducer. func compactionRetiredFailedText(text string, referenced map[string]bool, ranges [][2]string) string { - reduced, removed := compactionReduceGoTestOutput(text) var result strings.Builder - for line := range strings.SplitAfterSeq(reduced, "\n") { + removed := 0 + for line := range strings.SplitAfterSeq(text, "\n") { if compactionCompleteSourceRow.MatchString(line) && !compactionTextReferencesRows(line, referenced, ranges) { removed++ continue @@ -603,7 +602,7 @@ func compactionRetiredFailedText(text string, referenced map[string]bool, ranges if removed == 0 { return text } - return fmt.Sprintf("[mekugi: omitted %d positively identified runner/source lines from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) + return fmt.Sprintf("[mekugi: omitted %d unreferenced verified source rows from terminal failed output; failure remains unresolved]\n%s", removed, result.String()) } func compactionRetiredTextKeepingRows(text string, referenced map[string]bool, ranges [][2]string) string { @@ -679,6 +678,12 @@ func compactionRetiredOutputKeepingRows(raw json.RawMessage, operation compactio return reduced, ok } + if _, text, ok := contextCompactionOutput(raw); ok && strings.HasPrefix(text, contextCompactionGoTestSummaryPrefix) { + return raw, true + } + if _, text, ok := compactionFailedOutput(raw); ok && strings.HasPrefix(text, contextCompactionGoTestSummaryPrefix) { + return raw, true + } if operation.patchReport == "" && operation.notice == nil { if encode, text, ok := contextCompactionOutput(raw); ok { return encode(compactionRetiredTextKeepingRows(text, referenced, ranges)), true diff --git a/internal/router/context_compaction_retirement_test.go b/internal/router/context_compaction_retirement_test.go index a7d85830..0c102383 100644 --- a/internal/router/context_compaction_retirement_test.go +++ b/internal/router/context_compaction_retirement_test.go @@ -242,7 +242,7 @@ func TestCompactionRetirementRetiresCompletedFailedReasoningGroups(t *testing.T) func TestCompactionRetirementPinsFailureLiveAndReferencedEvidence(t *testing.T) { items := retirementHistory() items[3] = compactTestOutput("operation_00", - strings.Repeat("=== RUN TestHistorical\n--- PASS: TestHistorical (0.1s)\n", 200)+"unique unresolved failure\n", 1) + strings.Join(compactionSourceTestRows("", 200), "")+"unique unresolved failure\n", 1) items[6] = mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "operation_01", "output": string(mustMarshalJSON(map[string]any{"exit_code": 0, "session_id": 42, "output": "still running"}))}) items = append(items, mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "content": "The evidence in operation_02 is needed for the remaining investigation."})) diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go index 386a1595..c377e354 100644 --- a/internal/router/context_compaction_source.go +++ b/internal/router/context_compaction_source.go @@ -127,7 +127,15 @@ func reduceContextCompactionSourceWithFrontier(original, retained []json.RawMess } }, &unsafeEncoding) } - compactionVisitReferenceStrings(fields["output"], visitOutput) + visited := false + mapCompactionCompletedOutput(retainedFields[index]["output"], func(text string) string { + visited = true + visitOutput(text) + return text + }) + if !visited { + compactionVisitReferenceStrings(retainedFields[index]["output"], visitOutput) + } continue default: references = append(references, fields["content"], fields["summary"]) diff --git a/internal/router/context_compaction_test.go b/internal/router/context_compaction_test.go index 60f3877c..423baa1b 100644 --- a/internal/router/context_compaction_test.go +++ b/internal/router/context_compaction_test.go @@ -35,18 +35,23 @@ func TestContextCompactionPreservesIntentAndContinuation(t *testing.T) { compactTestCall("last", "go test -v ./internal/router"), compactTestOutput("last", "=== RUN TestRoute\n--- PASS: TestRoute (0.01s)\nPASS\nok example/router 0.1s\n", 0), } + before := string(mustMarshalJSON(items)) got := reduceContextCompaction(items) if len(got) != len(items) { t.Fatal("compaction removed conversation items") } for index := range items { - if index != 2 && string(got[index]) != string(items[index]) { + if index != 2 && index != 7 && string(got[index]) != string(items[index]) { t.Fatalf("protected item %d changed", index) } } - if string(got[2]) == string(items[2]) || !strings.Contains(string(got[2]), "ok example/router") { - t.Fatalf("routine test detail was not reduced with package evidence retained: %s (command kind %q)", got[2], contextCompactionCommand("go test -v ./internal/router")) + for _, index := range []int{2, 7} { + if string(got[index]) == string(items[index]) || + !strings.Contains(string(got[index]), "Go test passed") || + strings.Contains(string(got[index]), "example/router") { + t.Fatalf("Go test result %d was not reduced to its outcome: %s", index, got[index]) + } } if string(mustMarshalJSON(items)) != before { t.Fatal("compaction modified the input") @@ -61,7 +66,6 @@ func TestContextCompactionKeepsUncertainExecutionEvidence(t *testing.T) { name, command, output string exitCode any }{ - {"failed", "go test -v ./...", "=== RUN TestA\n--- FAIL: TestA (0.1s)\nassertion details\nFAIL\n", 1}, {"running", "go test -v ./...", "=== RUN TestA\n", nil}, {"compound", "go test -v ./...; echo done", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, {"pipeline", "go test -v ./... | tee result", "=== RUN TestA\n--- PASS: TestA (0.1s)\nPASS\n", 0}, @@ -108,20 +112,53 @@ func TestContextCompactionHonorsCancellation(t *testing.T) { } } -func TestContextCompactionKeepsUnknownTestDiagnostics(t *testing.T) { - diagnostic := "--- PASS: retained diagnostic (0.1s)" - runnerLines := "=== RUN TestA\n--- PASS: TestA (0.1s)\n" - items := []json.RawMessage{ - compactTestCall("tests", "go test -v ./..."), - compactTestOutput("tests", strings.Repeat(runnerLines, 20)+diagnostic+"\n=== RUN retained run diagnostic\n test.go:10: important diagnostic\nPASS\nok example 0.1s\n", 0), - compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0), - } - got := reduceContextCompaction(items) - if string(got[1]) == string(items[1]) || !strings.Contains(string(got[1]), diagnostic) || - !strings.Contains(string(got[1]), "retained run diagnostic") || - !strings.Contains(string(got[1]), "important diagnostic") || - strings.Contains(string(got[1]), "--- PASS: TestA") { - t.Fatal("corroborated runner output was not reduced or unmatched diagnostic was removed") +func TestContextCompactionKeepsOnlyGoTestFailureNames(t *testing.T) { + tests := []struct { + name, output string + exitCode int + want, omit string + }{ + { + name: "passed", + output: strings.Repeat("=== RUN TestA\n--- PASS: TestA (0.1s)\n", 20) + + "--- PASS: test-written diagnostic (0.1s)\nimportant diagnostic\nPASS\nok example 0.1s\n", + exitCode: 0, + want: "Go test passed", + omit: "important diagnostic", + }, + { + name: "failed", + output: "=== RUN TestBroken\nassertion detail\n--- FAIL: TestBroken (0.1s)\n--- FAIL: TestSuite/Subcase (0.2s)\nFAIL\n", + exitCode: 1, + want: "failed tests: TestBroken, TestSuite/Subcase", + omit: "assertion detail", + }, + { + name: "failed before test", + output: "example.go:10: undefined: missing\nFAIL example [build failed]\n", + exitCode: 1, + want: "no failed test name was reported", + omit: "undefined: missing", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + items := []json.RawMessage{ + compactTestCall("tests", "go test -v ./..."), + compactTestOutput("tests", strings.Repeat(test.output, 20), test.exitCode), + compactTestCall("last", "pwd"), + compactTestOutput("last", "/workspace\n", 0), + } + got := reduceContextCompaction(items) + if string(got[1]) == string(items[1]) || + !strings.Contains(string(got[1]), test.want) || + strings.Contains(string(got[1]), test.omit) { + t.Fatalf("Go test output was not reduced to failed test names: %s", got[1]) + } + if again := reduceContextCompaction(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { + t.Fatal("Go test reduction was not idempotent") + } + }) } } @@ -144,22 +181,32 @@ func TestContextCompactionKeepsLiveHandlesAndAmbiguousCalls(t *testing.T) { func TestContextCompactionNativeExecOutput(t *testing.T) { header := "Chunk ID: abc\nWall time: 1.2500 seconds\nProcess exited with code 0\nOriginal token count: 900\nOutput:\n" log := strings.Repeat("=== RUN TestNative\n--- PASS: TestNative (0.1s)\n", 30) + "PASS\nok example 0.1s\n" - for _, prefix := range []string{header, strings.Replace(header, "code 0", "code 1", 1), strings.Replace(header, "Process exited with code 0", "Process running with session ID 42", 1)} { - result := mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "tests", "output": prefix + log}) - items := []json.RawMessage{compactTestCall("tests", "go test -v ./..."), result, compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0)} - got := reduceContextCompaction(items) - var fields map[string]json.RawMessage - _ = json.Unmarshal(got[1], &fields) - text := jsonString(fields, "output") - if !strings.HasPrefix(text, prefix) { - t.Fatal("native execution header changed") - } - if prefix == header { - if text == prefix+log || !strings.Contains(text, "ok example") { - t.Fatal("native successful test output not reduced") + tests := []struct { + name, prefix, want string + changed bool + }{ + {name: "passed", prefix: header, want: "Go test passed", changed: true}, + {name: "failed", prefix: strings.Replace(header, "code 0", "code 1", 1), want: "no failed test name was reported", changed: true}, + {name: "running", prefix: strings.Replace(header, "Process exited with code 0", "Process running with session ID 42", 1)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "tests", "output": test.prefix + log}) + items := []json.RawMessage{compactTestCall("tests", "go test -v ./..."), result, compactTestCall("last", "pwd"), compactTestOutput("last", "/workspace\n", 0)} + got := reduceContextCompaction(items) + var fields map[string]json.RawMessage + _ = json.Unmarshal(got[1], &fields) + text := jsonString(fields, "output") + if !strings.HasPrefix(text, test.prefix) { + t.Fatal("native execution header changed") } - } else if string(got[1]) != string(result) { - t.Fatal("native failure or running output changed") - } + if test.changed { + if text == test.prefix+log || !strings.Contains(text, test.want) || strings.Contains(text, "example 0.1s") { + t.Fatal("terminal native Go test output was not reduced") + } + } else if string(got[1]) != string(result) { + t.Fatal("running output changed") + } + }) } } From 9e115c5c786205ef5df5aa1693534847f467a19e Mon Sep 17 00:00:00 2001 From: yusing Date: Sat, 12 Sep 2026 02:28:02 +0000 Subject: [PATCH 11/13] fixup! fix(router): harden provider-free context compaction --- .../context_compaction_repeated_test.go | 9 +- .../router/context_compaction_retirement.go | 24 ++-- .../context_compaction_retirement_test.go | 121 +----------------- internal/router/context_compaction_source.go | 24 ---- .../router/context_compaction_source_test.go | 66 ---------- 5 files changed, 21 insertions(+), 223 deletions(-) diff --git a/internal/router/context_compaction_repeated_test.go b/internal/router/context_compaction_repeated_test.go index e8ae528c..a5f0aa8d 100644 --- a/internal/router/context_compaction_repeated_test.go +++ b/internal/router/context_compaction_repeated_test.go @@ -116,13 +116,16 @@ func TestCompactionReferencedResultsScansOnlySpecialResultNotes(t *testing.T) { if !contextCompactionReferencedResults([]json.RawMessage{failed})["later-source"] { t.Fatal("surviving failed result note did not protect its referenced evidence") } + retired := mustMarshalJSON(map[string]any{ + "type": "message", "role": "assistant", "content": note, + }) + if !contextCompactionReferencedResults([]json.RawMessage{retired})["later-source"] { + t.Fatal("surviving factual assistant note did not protect its referenced evidence") + } decoys := []json.RawMessage{ compactTestOutput("plain-id", "later-source", 1), compactTestOutput("prefixed-note", "ordinary output: "+note, 1), - mustMarshalJSON(map[string]any{ - "type": "message", "role": "assistant", "content": note, - }), } if protected := contextCompactionReferencedResults(decoys); len(protected) != 0 { t.Fatal("non-special or non-result text was treated as a replacement note") diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index 6b6dec94..9c08db3f 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -180,7 +180,7 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) enqueueOriginal := func(id string, plan *compactionRetirement) { queue = append(queue, referenceText{fields[plan.call]["input"], id, true}, referenceText{fields[plan.call]["arguments"], id, true}, - referenceText{fields[plan.result]["output"], id, true}) + referenceText{fields[plan.result]["output"], id, false}) } revision := 0 var pin func(string) @@ -309,7 +309,7 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } case "function_call_output", "custom_tool_call_output": if p := plans[id]; p == nil || !p.eligible { - queue = append(queue, referenceText{item["output"], id, true}) + queue = append(queue, referenceText{item["output"], id, false}) } default: queue = append(queue, referenceText{item["content"], "", true}, referenceText{item["summary"], "", true}) @@ -406,14 +406,13 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } } - // Pinning and row restoration are monotonic. Repeat reference closure when - // either exposes references that were absent from the proposed factual - // record. At most one pass per retired candidate can restore content, so - // this reaches a bounded stable result. + // Pinning is monotonic. Repeat reference closure only when profitability + // restores original content, which can expose references that were absent + // from the proposed factual record. At most one pass per retired candidate + // can restore content, so this reaches a bounded stable result. for { - beforeRevision := revision drainReferences() - for id, plan := range plans { + for _, plan := range plans { if !plan.eligible || (len(plan.rows) == 0 && len(plan.ranges) == 0) { continue } @@ -422,13 +421,10 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) if !ok { return input } - if string(plan.output) != string(retained) { - plan.output = retained - queue = append(queue, referenceText{retained, id, true}) - revision++ - } + plan.output = retained } + beforeProfitability := revision for id, plan := range plans { if !plan.eligible || groupAt[plan.call] >= 0 { continue @@ -453,7 +449,7 @@ func retireCompactionOperationsWithFrontier(input []json.RawMessage, recent int) } } } - if revision == beforeRevision { + if revision == beforeProfitability { break } } diff --git a/internal/router/context_compaction_retirement_test.go b/internal/router/context_compaction_retirement_test.go index 0c102383..324fef09 100644 --- a/internal/router/context_compaction_retirement_test.go +++ b/internal/router/context_compaction_retirement_test.go @@ -291,7 +291,7 @@ func TestCompactionRetirementPreservesAmbiguousCalls(t *testing.T) { } } -func TestCompactionRetirementFollowsOnlySurvivingReplacementNotes(t *testing.T) { +func TestCompactionRetirementPinsReplacementNoteTargets(t *testing.T) { note := func(target string) string { return fmt.Sprintf("[mekugi compaction: 3 source rows (1:0001 through 3:0003) retained verbatim in later tool result %q]\n", target) } @@ -299,58 +299,11 @@ func TestCompactionRetirementFollowsOnlySurvivingReplacementNotes(t *testing.T) t.Helper() for _, index := range []int{1, 2, 3, 4, 5, 6} { if string(got[index]) != string(want[index]) { - t.Fatalf("surviving replacement-note closure lost native item %d", index) + t.Fatalf("replacement-note closure lost native item %d", index) } } } - t.Run("generated repeated excerpt note retires with its consumer", func(t *testing.T) { - items := retirementHistory() - var excerpt strings.Builder - for row := 1; row <= 12; row++ { - fmt.Fprintf(&excerpt, "%d:abcd source declaration with enough exact text to retire the earlier repeated excerpt\n", row) - } - items[3] = compactTestOutput("operation_00", - "older read header\n"+excerpt.String()+"unique older evidence\n"+strings.Repeat("consumer historical detail\n", 500), 0) - items[6] = compactTestOutput("operation_01", excerpt.String(), 0) - - got := reduceContextCompaction(items) - wire := string(mustMarshalJSON(got)) - if !strings.Contains(wire, "historical facts v4") || strings.Contains(wire, "retained verbatim in later tool result") { - t.Fatal("generated replacement note survived its retired consumer") - } - if again := reduceContextCompaction(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { - t.Fatal("generated replacement-note retirement was not stable") - } - }) - - t.Run("retired note releases its target", func(t *testing.T) { - items := retirementHistory() - items[3] = compactTestOutput("operation_00", - note("operation_01")+strings.Repeat("retired consumer detail\n", 500), 0) - items[6] = compactTestOutput("operation_01", strings.Repeat("later source evidence\n", 500), 0) - before := string(mustMarshalJSON(items)) - - got := retireCompactionOperations(items) - for _, index := range []int{1, 2, 3, 4, 5, 6} { - if string(got[index]) == string(items[index]) { - t.Fatalf("discarded replacement note pinned native item %d", index) - } - } - if strings.Contains(string(mustMarshalJSON(got)), "retained verbatim in later tool result") { - t.Fatal("retired replacement note survived in factual records") - } - if len(mustMarshalJSON(got)) > len(mustMarshalJSON(items)) { - t.Fatal("replacement-note retirement increased retained history") - } - if string(mustMarshalJSON(items)) != before { - t.Fatal("replacement-note retirement mutated its input") - } - if again := retireCompactionOperations(got); string(mustMarshalJSON(again)) != string(mustMarshalJSON(got)) { - t.Fatal("replacement-note retirement was not stable") - } - }) - t.Run("failed consumer keeps its target", func(t *testing.T) { items := retirementHistory() items[3] = compactTestOutput("operation_00", note("operation_01"), 1) @@ -370,76 +323,12 @@ func TestCompactionRetirementFollowsOnlySurvivingReplacementNotes(t *testing.T) assertNative(t, reduceContextCompaction(items), items) }) - t.Run("replacement-note cycle follows a surviving root", func(t *testing.T) { + t.Run("input notes pin both sides of a replacement cycle", func(t *testing.T) { items := retirementHistory() items[3] = compactTestOutput("operation_00", note("operation_01")+strings.Repeat("cycle detail\n", 500), 0) items[6] = compactTestOutput("operation_01", note("operation_00")+strings.Repeat("cycle detail\n", 500), 0) - - retired := retireCompactionOperations(items) - for _, index := range []int{1, 2, 3, 4, 5, 6} { - if string(retired[index]) == string(items[index]) { - t.Fatalf("unrooted replacement-note cycle pinned native item %d", index) - } - } - - rooted := append(items, mustMarshalJSON(map[string]any{ - "type": "message", "role": "assistant", "content": "Keep operation_00.", - })) - assertNative(t, retireCompactionOperations(rooted), rooted) - assertNative(t, reduceContextCompaction(rooted), rooted) - }) -} - -func TestCompactionRetirementFollowsReferencesExposedByRetainedResults(t *testing.T) { - t.Run("pinned tool result row", func(t *testing.T) { - items := retirementHistory() - items[3] = compactTestOutput("operation_00", - "Continue with exact row 17:abcd.\n"+strings.Repeat("consumer historical detail\n", 500), 0) - referencedLine := "17:abcd required source evidence\n" - items[6] = compactTestOutput("operation_01", - referencedLine+strings.Repeat("unreferenced source detail\n", 500), 0) - items = append(items, mustMarshalJSON(map[string]any{ - "type": "message", "role": "assistant", "content": "Keep operation_00.", - })) - - got := retireCompactionOperations(items) - if !strings.Contains(string(mustMarshalJSON(got)), strings.TrimSpace(referencedLine)) { - t.Fatal("row reference in a pinned tool result lost its source evidence") - } - }) - - t.Run("restored factual row references operation", func(t *testing.T) { - items := retirementHistory() - items[3] = compactTestOutput("operation_00", - "17:abcd retained evidence requires operation_01\n"+strings.Repeat("consumer historical detail\n", 500), 0) - items[6] = compactTestOutput("operation_01", strings.Repeat("required dependency detail\n", 500), 0) - items = append(items, mustMarshalJSON(map[string]any{ - "type": "message", "role": "assistant", "content": "Keep exact row 17:abcd.", - })) - - got := retireCompactionOperations(items) - for _, index := range []int{4, 5, 6} { - if string(got[index]) != string(items[index]) { - t.Fatalf("reference exposed by restored factual row did not pin operation item %d", index) - } - } - }) - - t.Run("restored factual row references another row", func(t *testing.T) { - items := retirementHistory() - items[3] = compactTestOutput("operation_00", - "17:abcd retained evidence requires row 23:beef\n"+strings.Repeat("consumer historical detail\n", 500), 0) - referencedLine := "23:beef required chained source evidence\n" - items[6] = compactTestOutput("operation_01", - referencedLine+strings.Repeat("unreferenced source detail\n", 500), 0) - items = append(items, mustMarshalJSON(map[string]any{ - "type": "message", "role": "assistant", "content": "Keep exact row 17:abcd.", - })) - - got := retireCompactionOperations(items) - if !strings.Contains(string(mustMarshalJSON(got)), strings.TrimSpace(referencedLine)) { - t.Fatal("row reference exposed by restored factual evidence lost its dependency") - } + assertNative(t, retireCompactionOperations(items), items) + assertNative(t, reduceContextCompaction(items), items) }) } diff --git a/internal/router/context_compaction_source.go b/internal/router/context_compaction_source.go index c377e354..1a24ff18 100644 --- a/internal/router/context_compaction_source.go +++ b/internal/router/context_compaction_source.go @@ -112,30 +112,6 @@ func reduceContextCompactionSourceWithFrontier(original, retained []json.RawMess references = append(references, fields["input"], fields["arguments"]) } case "function_call_output", "custom_tool_call_output": - visitOutput := func(text string) { - compactionSourceVisitDecodedReferences(text, func(decoded string) { - for line := range strings.SplitAfterSeq(decoded, "\n") { - if match := compactionCompleteSourceRow.FindStringSubmatch(line); match != nil { - line = strings.Replace(line, match[1], "", 1) - } - for _, match := range compactionSourceRangeReference.FindAllStringSubmatch(line, -1) { - ranges = append(ranges, [2]string{match[1], match[2]}) - } - for _, row := range compactionRowReference.FindAllString(line, -1) { - rowReferences[row] = true - } - } - }, &unsafeEncoding) - } - visited := false - mapCompactionCompletedOutput(retainedFields[index]["output"], func(text string) string { - visited = true - visitOutput(text) - return text - }) - if !visited { - compactionVisitReferenceStrings(retainedFields[index]["output"], visitOutput) - } continue default: references = append(references, fields["content"], fields["summary"]) diff --git a/internal/router/context_compaction_source_test.go b/internal/router/context_compaction_source_test.go index c88d912a..c1bf4f84 100644 --- a/internal/router/context_compaction_source_test.go +++ b/internal/router/context_compaction_source_test.go @@ -97,72 +97,6 @@ func TestCompactionSourcePrunesOnlyUnreferencedRows(t *testing.T) { } } -func TestCompactionSourceFollowsToolOutputRowReferences(t *testing.T) { - rows := compactionSourceTestRows("", 18) - successful := string(mustMarshalJSON(map[string]any{ - "output": "Continue from row 3:0003.", "exit_code": 0, "wall_time_seconds": 1, - })) - live := string(mustMarshalJSON(map[string]any{ - "output": "Continue from row 3:0003.", "exit_code": 0, "session_id": 42, - })) - for _, test := range []struct { - name string - call, result json.RawMessage - }{ - { - name: "function_call_output", - call: compactTestCall("consumer", "pwd"), - result: compactTestOutput("consumer", "Continue from row 3:0003.", 0), - }, - { - name: "failed_function_call_output", - call: compactTestCall("consumer", "pwd"), - result: compactTestOutput("consumer", "Continue from row 3:0003.", 1), - }, - { - name: "custom_tool_call_output", - call: mustMarshalJSON(map[string]any{ - "type": "custom_tool_call", "name": "shell", "call_id": "consumer", "input": "pwd", - }), - result: mustMarshalJSON(map[string]any{ - "type": "custom_tool_call_output", "call_id": "consumer", "output": successful, - }), - }, - { - name: "live_custom_tool_call_output", - call: mustMarshalJSON(map[string]any{ - "type": "custom_tool_call", "name": "shell", "call_id": "consumer", "input": "pwd", - }), - result: mustMarshalJSON(map[string]any{ - "type": "custom_tool_call_output", "call_id": "consumer", "output": live, - }), - }, - } { - t.Run(test.name, func(t *testing.T) { - items := []json.RawMessage{ - mustMarshalJSON(map[string]any{ - "type": "reasoning", "summary": []any{map[string]string{"type": "summary_text", "text": "Inspect source."}}, - }), - compactTestCall("source-old", "hread source.go"), - compactTestOutput("source-old", strings.Join(rows, ""), 0), - test.call, - test.result, - mustMarshalJSON(map[string]any{ - "type": "function_call", "name": "unknown", "call_id": "unknown-old", "arguments": "{}", - }), - compactTestOutput("unknown-old", "unknown companion\n", 0), - } - items = append(items, compactionSourceTestRecent()...) - - got := reduceContextCompaction(items) - text := compactionSourceTestOutputText(t, got[2]) - if !strings.Contains(text, rows[2]) || strings.Contains(text, rows[10]) { - t.Fatal("tool-produced row reference was not preserved conservatively") - } - }) - } -} - func TestCompactionSourceKeepsProtectedOutputsByteExact(t *testing.T) { rows := strings.Join(compactionSourceTestRows("", 18), "") tests := []struct { From 0ac955b894151b5c8997595524eb8bf4fcf8b139 Mon Sep 17 00:00:00 2001 From: yusing Date: Sat, 12 Sep 2026 06:43:22 +0000 Subject: [PATCH 12/13] feat(compaction): add lossy pressure fallback Add content-independent fallback selection targeting 50,000 visible-string tokens, with prioritized excerpts, repetition reduction, omission notices, and explicit required-instruction floors. Replace ordinary historical images with `[Image]` placeholders while preserving fresh and mandatory images. Add encrypted v2 reconciliation receipts and pressure diagnostics for carried messages, while retaining legacy envelope readability and failing closed on overlapping history. --- README.md | 60 +- doc/architecture/compaction.md | 58 +- doc/spec/compaction.md | 138 +++- internal/router/context_compaction.go | 12 +- internal/router/context_compaction_budget.go | 5 +- ...text_compaction_budget_integration_test.go | 28 +- .../router/context_compaction_codex_test.go | 110 ++- .../router/context_compaction_envelope.go | 89 +- internal/router/context_compaction_http.go | 162 +++- internal/router/context_compaction_images.go | 96 +++ .../router/context_compaction_images_test.go | 101 +++ .../router/context_compaction_pressure.go | 503 ++++++++++++ .../context_compaction_pressure_test.go | 762 ++++++++++++++++++ .../context_compaction_repeated_test.go | 90 ++- .../router/context_compaction_repetition.go | 95 +++ .../router/context_compaction_retirement.go | 18 +- 16 files changed, 2217 insertions(+), 110 deletions(-) create mode 100644 internal/router/context_compaction_images.go create mode 100644 internal/router/context_compaction_images_test.go create mode 100644 internal/router/context_compaction_pressure.go create mode 100644 internal/router/context_compaction_pressure_test.go create mode 100644 internal/router/context_compaction_repetition.go diff --git a/README.md b/README.md index b31d389f..e7799357 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ provider request. The first compaction creates an owner-only key at `~/.config/mekugi/compaction.key` on Linux). Keep that key to resume compacted sessions, including when moving them to another installation. -Compaction can discard unmarked historical details from older finished operations, +Compaction first discards unmarked historical details from older finished operations, even while the task is still open. It keeps factual execution records, requests, visible decisions, diagnostic excerpts, referenced evidence, and recent/live work. Terminal results from recognized direct `go test` calls are an exception: even when @@ -167,9 +167,33 @@ Truncated or oversized documentation can also lose unreferenced bulk; a single oversized evidence line retains bounded excerpts. Discarded details are not currently retrievable through Mekugi. -Unknown or ambiguous execution states remain intact. If nothing qualifies, -compaction reports an error rather than asking a provider for a summary. -Compaction does not guarantee a fixed retained-history size. +The retained history targets 50,000 visible-string tokens, with at most 30,000 +tokens of overshoot. If the initial reductions cannot fit, a budget-first pass +keeps prioritized excerpts and drops older material regardless of tool or content +format. Developer/system/model instructions and canonical `AGENTS.md` context +remain intact. The latest user request and recent/live execution evidence come +next, but user messages and live output can be excerpted when oversized. +Repeated text can collapse to representative occurrences with an omission count. +The selector reserves space for the request chain and discussion before execution +bulk, with an initial preference for the latest assistant or agent report. +Unused shares are redistributed as small items fit. It need not fill the +50,000-token target. +This is deliberately lossy: it cannot guarantee that every omitted detail is +unimportant. The model receives explicit omission markers, not an invented summary. + +During compaction, ordinary historical images become `[Image]` placeholders, +keeping the surrounding text. This deliberately discards visual details rather +than assuming earlier reasoning captured them. Fresh images on normal turns and +mandatory instruction images remain unchanged. Image data is not tokenized as +text, and historical images no longer need a separate allocation budget. + +Original user/agent messages needed to reconcile Codex's carried history remain +encrypted in the capsule, but are not returned to the model or available through +a retrieval tool. The budget measures the restored history, not ciphertext size +or complete provider input; fresh instructions and later messages add to it. +Required instructions alone exceeding 80,000 tokens, invalid input, unreadable +envelopes, or a no-op on an already-small history can still produce an error. +Mekugi never falls back to a provider summary. Then launch: @@ -709,6 +733,34 @@ For focused checks, use `go test .` for the engine, `go test ./internal/router` for routing, or `go test ./cmd/mekugi ./cmd/shell` for process entry points. +To inspect compaction loss using an installed Codex client: + +```sh +MEKUGI_COMPACTION_CODEX_BIN="$(command -v codex)" go test ./internal/router -run '^TestCompactionInstalledCodex$' -count=1 -v +``` + +These loopback fixtures make no provider inference requests. Pressure cases log +per-item token counts, retention decisions, and the next request's token count. +They check specified continuation facts and client compatibility, not model +reasoning quality on real task histories. + +For an offline replay of an existing session's first recorded compaction window, +use the rollout check. The optional audit directory receives original/retained +history and per-item diagnostics, so it must already exist with owner-only +permissions. Original session files are read-only; no historical tool calls or +provider inference are executed. + +```sh +compaction_audit_dir="$(mktemp -d)" +MEKUGI_COMPACTION_ROLLOUT="/absolute/path/rollout.jsonl" \ +MEKUGI_COMPACTION_AUDIT_DIR="$compaction_audit_dir" \ +go test ./internal/router -run '^TestCompactionRolloutReplay$' -count=1 -v +``` + +The replay measures recorded items, not omitted model instructions or request/tool +framing. Passing its budget/restoration checks does not establish task-fact retention; +inspect the exported before/after histories for that assessment. + ## License MIT. See [LICENSE](LICENSE). diff --git a/doc/architecture/compaction.md b/doc/architecture/compaction.md index d631e478..0331e8bc 100644 --- a/doc/architecture/compaction.md +++ b/doc/architecture/compaction.md @@ -8,23 +8,48 @@ restoration of its own input envelopes, and bounded request decoding. Codex owns configuration resolution, trigger timing, retained client-side context, and installation of the returned compaction result. -Evidence reducers own reduction and the approved lossy retirement of finished +The initial evidence reducers own reduction and the approved lossy retirement of finished operations. Static carrier parsing establishes invocation facts; terminal output establishes completion, not task closure. It does not execute scripts or interpret opaque reasoning. Unknown lifecycle/dependency states stay protected. The working-set selector owns token pressure and admission, not relevance inference. It measures candidate native histories before envelope sealing using -the existing visible-string metric. Its 50,000-token target permits at most -30,000 tokens of overshoot; an inadmissible history fails explicitly rather than -relaxing authority, lifecycle, reference, diagnostic or reasoning-group rules. -These budgets do not change Codex's scheduling or model context configuration. +the visible-text string metric, excluding native image URL payloads. Its +50,000-token target permits at most +30,000 tokens of overshoot. If evidence-preserving plans cannot fit, the pressure +selector owns a content-independent, explicitly lossy fallback compiled from the +original history. It distributes bounded excerpts by authority, active frontier, +visible reasoning, and historical evidence priority. Generic bounded exact-run +repetition reduction precedes allocation outside the mandatory instruction floor. +Request-chain and discussion reservations precede execution allocation. +Discussion gives its newest readable assistant or agent report a bounded initial +share before broad coverage. Within each family, weighted shares redistribute +capacity after small items fit; +unused capacity is shared with unfinished items. An acknowledgement cannot crowd +out its request chain, nor can many unknown tool results consume discussion's +initial allocation. Fully represented histories need not fill the target. Required developer/system/model instructions and +canonical context such as `AGENTS.md` form a byte-exact floor. A floor above the +ceiling fails explicitly; other history and lifecycle evidence have priority, +not an unlimited exemption. All native tool/reasoning items become historical +observations together, avoiding partial executable groups. Prefix/suffix and +selected constraint/status excerpts are not a semantic summary or proof of irrelevance. +A compaction-only image pass replaces ordinary native image parts with `[Image]` +before text selection, preserving surrounding text and identity metadata. +Image-only reduction can succeed without a text-token saving or forced retirement +of unrelated native history. Mandatory instruction images stay exact, and normal +turns do not strip fresh images. Carried-message receipts reconcile original images +without returning them to model input. There is no historical image allocation +budget; request/envelope size limits remain with their existing owners. +The placeholder accepts visual-detail loss rather than claiming that prior +reasoning captured the image. Text budgeting does not change Codex's scheduling +or model context configuration. The default eight-operation continuity buffer is a warm retention preference, not the definition of live work. Under pressure the selector first reduces eligible completed output while retaining native calls and reasoning, then may retire more complete historical groups. The newest operation, all live or unknown -operations, and dependency closure remain protected under every plan. Metadata +operations, and dependency closure remain protected under every evidence plan. Metadata and narration reducers keep their existing conservative frontier. Each retention plan runs against the same original input. Selection never chains @@ -49,7 +74,7 @@ frontier. Reference closure and complete-group profitability reach a stable resu before replacement; restoring a consumer also restores every dependency exposed by its retained content. Supported text and JavaScript escapes are decoded for reference matching, while suspicious encodings retain evidence conservatively. These records -are factual history, not new instructions or an external archive. The envelope +are factual history, not new instructions or an external archive. The model replay contains selected history only; deferred retrieval must not be implied by a digest or retirement marker. @@ -84,7 +109,24 @@ supported local Codex compaction flows, so its unreferenced transport item IDs m be omitted. User and agent-message identities remain stable because those items can be carried by the client. Referenced IDs and recent items remain protected. -The envelope owner authenticates and encrypts the retained native item array. +The envelope owner authenticates and encrypts the retained item array. Version 2 +also carries original user/agent messages and replay positions when pressure +selection changes or drops them, plus content-free per-item pressure diagnostics. +Diagnostics report allocation and loss inside the encrypted envelope; restoration +does not add them to model input. The bounded reconciliation receipts let full +or client-truncated carried messages match without resurrecting discarded input. +Neither diagnostics nor receipts are forwarded to the model or exposed as a +retrieval interface. +Restoration carries their anchors across subsequent compaction; version 1 remains +readable for existing sessions. +Receipts are merged in their pre-selection timeline before replay indexes are +remapped, with aliases grouped per logical item. Unmatched content from an older +envelope is not fresh input: overlapping snapshots fail closed rather than +silently restoring discarded history. +Evidence plans with receipts retain one output item per source position, avoiding +identity-based anchor guesses or record consolidation across insertion points. +Excerpt identity metadata stays stable while positional content classifications +are rebuilt for the rendered content. Its persistent key belongs to the Mekugi configuration directory, not a thread, temporary plugin runtime, provider credential, or capture stream. Cross-process locking serializes first creation. Compression is an internal envelope-storage diff --git a/doc/spec/compaction.md b/doc/spec/compaction.md index 2bbb94e5..a94fdbdc 100644 --- a/doc/spec/compaction.md +++ b/doc/spec/compaction.md @@ -26,7 +26,9 @@ A locally completed compaction replaces its WebSocket history with the capsule; it does not retain the unpruned parent beside it. Steering cannot carry local envelopes. -Compaction preserves user/developer instructions, corrections, authorization, +### Evidence-preserving passes + +The initial passes preserve user/developer instructions, corrections, authorization, and the active execution frontier. Older ordinary-assistant narration can omit an earlier byte-identical body when a later occurrence remains. Other prose, including routine progress, stays unchanged; agent-message content remains byte-exact for @@ -134,12 +136,15 @@ Operation completion is not task completion. Execution records report observed facts without inventing scope closure, successful validation, or a workspace version. User corrections and visible reasoning/decision text are not blanket-pruned. +### Budget-first selection + The working-set selector targets 50,000 visible-string tokens with a maximum 30,000-token overshoot: no newly completed compaction may retain more than 80,000 tokens under this metric. Count the selected native item array at the -first replay boundary using `o200k_base`, excluding opaque `encrypted_content` -and request/tool framing. Encoded size and downstream projection savings do not -establish this target. This is not a complete provider-context count and does not +first replay boundary using `o200k_base`, excluding opaque `encrypted_content`, +request/tool framing, and native image URL payloads. Ordinary historical images +become text placeholders under the policy below. Encoded size and downstream +projection savings do not establish this target. This is not a complete provider-context count and does not cap fresh instructions or subsequent input appended after the selected snapshot. Selection evaluates retention plans in this order: keep the newest eight @@ -159,24 +164,115 @@ the earlier plan on equal token counts. Do not escalate an already-small history when the normal pass cannot shrink it. Repeated items each contribute to the metric even when tokenization of their identical bytes is cached for the request. -If no supported token reduction is available, token counting fails, or every -candidate exceeds 80,000 tokens, compaction fails with HTTP 422 before sealing. -Budget failures report the measured before/after counts and target/ceiling. +If every evidence-preserving candidate exceeds 80,000 tokens, a budget-first +pass selects directly from the original history, targeting 50,000 tokens. This +pass is content-independent: unfamiliar tools, unique prose, diagnostic-heavy +output, large newest results, and oversized user text are not exempt from +the text budget. Developer/system instructions, model instructions, and canonical +context such as `AGENTS.md` remain byte-exact in every pass. If those required +items and the omission notice alone exceed 80,000 tokens, compaction fails +explicitly instead of truncating them. Otherwise the lossy policy overrides the +initial passes' exact-preservation rules. It does not prove irrelevance. + +The pass first distributes bounded excerpts across the timeline, prioritizing +the latest real-user request, other user requests and recent/live work, +older decision/reasoning text, then older execution evidence. Before allocation, +eligible text can collapse adjacent byte-identical repetitions of short token +sequences, retaining representative occurrences and an explicit omitted count. +Different text between runs, including corrections, remains distinct. Repetition +mapping preserves the prepared content parts, unknown parts, and their metadata; only +a separately reported budget excerpt may replace the part array. Repetition +markers must save tokens, not just bytes. This is not fuzzy deduplication; +similar diagnostics with different values do not match. +Required instructions never enter repetition reduction. + +Coverage is separated into request-chain, discussion, and execution families. +Before execution bulk can consume capacity, the request chain receives up to a +third of the available working space and discussion receives up to half of the +space then remaining. Weighted shares within each family redistribute capacity +as complete small items fit. The newest readable assistant or agent report first +receives up to a quarter of discussion's share, so a bounded continuation report +is not fragmented merely by a long older discussion. This is a recency preference, +not semantic recognition of a handoff or an unlimited exemption. +Remaining execution capacity and any unused surplus +can serve unfinished items; these reservations are not fixed per-item ceilings. +A latest acknowledgement must not displace the request it answers, and many +unknown tool results cannot crowd out every older decision. The target is not a +quota: a complete working set can be much smaller. +Required instructions are reserved first; the overshoot allowance can provide +working space when that mandatory floor consumes the normal target. +Small high-priority items stay exact. Oversized items keep bounded prefix/suffix +excerpts and selected constraint/status lines. Whole historical items may be +dropped if the excerpt framing itself would exhaust the budget. All rendered +items and omission markers participate in the final token measurement. + +At every compaction, ordinary historical native image parts in messages and +tool outputs become literal `[Image]` text parts before text-budget selection. +Surrounding text, message identity, and non-positional metadata stay intact; +positional classifications reflect the replacement. This image-only pass does +not require unrelated text or opaque reasoning to be retired to manufacture +token savings. Subsequent text-pressure selection may still excerpt or drop +history under the rules above. + +Fresh images on ordinary turns and mandatory instruction/context images remain +unchanged. There is no separate historical image-count or encoded-byte allowance; +the existing request/envelope size limits still apply. Native image payloads are +not tokenized as text. The placeholder explicitly accepts visual-detail loss, +without assuming earlier reasoning recorded a sufficient description. Carried +original user/agent images must not reappear after restoration. + +Each pressure snapshot carries a content-free diagnostic report inside its +encrypted envelope, outside model input. It records original/retained token +counts per source item, priority and allocation reason, exact retention, +repetition reduction, excerpts, historical representation, and whole-item drops. +The aggregate includes the new omission notice and separate original/retained +image counts and encoded-byte totals, never fabricated vision-token usage. Installed-client probes log +these reports and the next request's visible-string count; the latter may also +include fresh context and later input. Reports are diagnostic evidence, not +claims that omitted content was irrelevant. + +Native tool invocations, results, and reasoning become non-executable historical +observations together, avoiding truncated executable calls or partial native +reasoning/tool groups. Observed arguments, identities, failures, and live handles +have retention priority but are not an unlimited exemption. Opaque reasoning is +not interpreted. A visible notice identifies the lossy selection and warns that +older reference-retention notes may point to evidence no longer present. This +is extractive compaction, not a fabricated semantic summary; it cannot guarantee +task-critical semantic preservation for arbitrary content. + +If neither text-token reduction nor historical image replacement is available for +an already-small history, compaction fails before sealing. Token-counting and +input/envelope-validation failures also prevent sealing. Removing a V2 trigger is not token savings, and trigger-only input cannot produce -an empty capsule. Never discard protected context to force admission, fabricate -a summary or provider usage, or fall back to provider compaction. For WebSocket +an empty capsule. Never fabricate a summary or provider usage, or fall back to +provider compaction. For WebSocket `response.create`, the same admission failure emits an error event with status 422 and then closes the connection. -The retained native timeline travels inline in an authenticated, encrypted -router-owned compaction item. Legacy output also carries original real-user messages +The retained timeline travels inline in an authenticated, encrypted +router-owned compaction item. Legacy output also carries selected real-user messages for Codex's own user-input handling. Historical canonical instructions and environment context stay only in the envelope, avoiding false fresh injections. V2 emits exactly one completed compaction item. On subsequent requests, the router restores the timeline before any projection or forwarding. It reconciles carried native items without duplicating matched messages, preserves newly injected context and post-compaction input, -and restores full items when Codex retained truncated versions. No-ID truncations -must uniquely match the authenticated original; ambiguous matches fail explicitly. +and restores selected items when Codex retained truncated versions. No-ID truncations +must uniquely match an authenticated original; ambiguous matches fail explicitly. +When budget pressure changes or drops carried user/agent messages, version 2 +envelopes include their originals and replay positions solely for client +reconciliation. Those originals are never restored into model input or exposed +as a retrieval facility. They let full and client-truncated messages match without +resurrecting deliberately omitted content, including across repeated compaction. +Version 1 envelopes remain readable. Reconciliation evidence is subject to the +same bounded envelope decoding as the selected timeline. +Receipts preserve original timeline order and group aliases for the same item, +including when consecutive messages are dropped in different compactions. +Evidence plans carrying receipts keep original item positions instead of +consolidating adjacent records across possible instruction insertion points. +Message excerpts preserve native identity and non-positional metadata; content +classifications are rebuilt to match the selected part array. +Unmatched history from an earlier capsule fails closed when multiple envelopes +overlap; it must not be appended as if it were fresh context. Unreconciled carried user content, including unsupported multimodal truncation, also fails rather than becoming a duplicated request. Fresh canonical instructions retain their current position even when their text @@ -199,10 +295,18 @@ concurrent key creation, damaged or missing keys, and conservative output prunin Budget checks cover candidate independence, output-first selection, non-monotonic costs, exact target/overshoot boundaries, no-op and cancellation failures, repeated item accounting, preserved native evidence, and admission before envelope sealing. -Installed Codex 0.153.4 has passed loopback legacy and V2 round trips with +Pressure checks cover large unfamiliar output, live/failed exec results, unique +prose, user/agent messages, dynamic scripts, many small messages, required +instruction floors, carried truncation, and repeated compaction. +Installed Codex 0.154.0 has passed loopback legacy and V2 round trips with synthetic ChatGPT authentication: automatic compaction with both counting scopes, -and the manual compact operation used by `/compact`. A large user request is -truncated by the client's V2 retention step, then restored in full without -duplication. Other versions and resumed client flows need corresponding runtime +and the manual compact operation used by `/compact`, including pressure selection +for oversized requests. Pressure probes require the buried correction and complete +test-case output to survive repetitive user bulk within a small working set. +These scripted model fixtures establish client compatibility and specified fact +retention, not model reasoning quality. Dense, distinct histories also check +coverage of decisions, multiple failures, live handles, and current corrections. +A client-truncated user request is restored to the selected +full message or budget excerpt without duplication. Other versions and resumed client flows need corresponding runtime coverage. Paired outcome evaluation is still required before claiming an optimal output budget or task-critical semantic preservation on real histories. diff --git a/internal/router/context_compaction.go b/internal/router/context_compaction.go index 83c54ed7..1e283973 100644 --- a/internal/router/context_compaction.go +++ b/internal/router/context_compaction.go @@ -21,7 +21,7 @@ func reduceContextCompaction(input []json.RawMessage) []json.RawMessage { } func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRetentionPlan) []json.RawMessage { - reduced, err := reduceContextCompactionPlan(context.Background(), input, plan) + reduced, err := reduceContextCompactionPlan(context.Background(), input, plan, false) if err != nil { return input } @@ -29,10 +29,10 @@ func reduceContextCompactionWithPlan(input []json.RawMessage, plan compactionRet } func reduceContextCompactionContext(ctx context.Context, input []json.RawMessage) ([]json.RawMessage, error) { - return reduceContextCompactionPlan(ctx, input, compactionRetentionPlan{compactionRecentOperations, compactionRecentOperations}) + return reduceContextCompactionPlan(ctx, input, compactionRetentionPlan{compactionRecentOperations, compactionRecentOperations}, false) } -func reduceContextCompactionPlan(ctx context.Context, input []json.RawMessage, plan compactionRetentionPlan) ([]json.RawMessage, error) { +func reduceContextCompactionPlan(ctx context.Context, input []json.RawMessage, plan compactionRetentionPlan, preservePositions bool) ([]json.RawMessage, error) { if err := ctx.Err(); err != nil { return input, err } @@ -201,6 +201,12 @@ func reduceContextCompactionPlan(ctx context.Context, input []json.RawMessage, p if err := ctx.Err(); err != nil { return original, err } + // Carried-message receipts can anchor fresh instructions between any two + // original items. All reducers above are position-preserving; do not merge + // records across those insertion points when receipts are being retained. + if preservePositions { + return retained, nil + } return consolidateContextCompactionRecords(input, retained), nil } diff --git a/internal/router/context_compaction_budget.go b/internal/router/context_compaction_budget.go index e60587c0..1ed0f655 100644 --- a/internal/router/context_compaction_budget.go +++ b/internal/router/context_compaction_budget.go @@ -3,6 +3,7 @@ package router import ( "context" "encoding/json" + "errors" "fmt" ) @@ -11,6 +12,8 @@ const ( compactionOvershootTokens = 30_000 ) +var errCompactionNoReduction = errors.New("no supported token reduction is available for this history") + // A retention plan changes eligibility, never the evidence-preservation rules. // Output-only pruning gets the first opportunity to release the warm frontier: // exact invocations and native reasoning can survive without their old bulk. @@ -101,7 +104,7 @@ func selectCompactionWorkingSet( return nil, report, fmt.Errorf("native compaction history retains %d visible-string tokens (before %d; target %d + overshoot %d = ceiling %d); protected or unsupported context was not discarded", report.after, before, target, overshoot, target+overshoot) } if report.after >= before { - return nil, report, fmt.Errorf("no supported token reduction is available for this history (%d visible-string tokens); protected context was not discarded and no provider compaction was requested", before) + return nil, report, fmt.Errorf("%w (%d visible-string tokens); protected context was not discarded and no provider compaction was requested", errCompactionNoReduction, before) } return best, report, nil } diff --git a/internal/router/context_compaction_budget_integration_test.go b/internal/router/context_compaction_budget_integration_test.go index a04cd68b..12f6aedd 100644 --- a/internal/router/context_compaction_budget_integration_test.go +++ b/internal/router/context_compaction_budget_integration_test.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "os" "path/filepath" "slices" "strings" @@ -106,12 +105,12 @@ func TestCompactionBudgetHTTPAdmission(t *testing.T) { authority int status int }{ - {"protected-over-ceiling", 90_000, http.StatusUnprocessableEntity}, - {"protected-within-overshoot", 60_000, http.StatusOK}, + {"user-over-ceiling", 90_000, http.StatusOK}, + {"user-within-overshoot", 60_000, http.StatusOK}, } { t.Run(fmt.Sprintf("%s/v2=%t", test.name, v2), func(t *testing.T) { input := append([]json.RawMessage{mustMarshalJSON(map[string]any{ - "type": "message", "role": "user", "content": strings.Repeat("authority ", test.authority), + "type": "message", "role": "user", "content": "Continue the router task.\n" + strings.Repeat("detail ", test.authority) + "\nDo not deploy.", })}, compactHTTPHistory()...) path := "/v1/responses/compact" if v2 { @@ -132,15 +131,6 @@ func TestCompactionBudgetHTTPAdmission(t *testing.T) { if response.Code != test.status { t.Fatalf("status = %d, want %d: %.300s", response.Code, test.status, response.Body.String()) } - if test.status != http.StatusOK { - if !strings.Contains(response.Body.String(), "ceiling 80000") { - t.Fatal("budget failure omitted its actual ceiling") - } - if _, err := os.Stat(compactor.keyPath); !os.IsNotExist(err) { - t.Fatalf("inadmissible history reached envelope sealing: %v", err) - } - return - } var compacted struct { Output []json.RawMessage `json:"output"` } @@ -170,11 +160,17 @@ func TestCompactionBudgetHTTPAdmission(t *testing.T) { t.Fatal(err) } tokens, ok := compactionVisibleStringTokens(restored...) - if !ok || tokens <= compactionTargetTokens || tokens > compactionTargetTokens+compactionOvershootTokens { + if !ok || tokens > compactionTargetTokens+compactionOvershootTokens { t.Fatalf("overshoot not measured at native replay: %d", tokens) } - if !bytes.Equal(restored[0], input[0]) { - t.Fatal("overshoot was achieved by truncating authority") + if test.authority > compactionTargetTokens+compactionOvershootTokens { + visible := mustMarshalJSON(restored) + if tokens > compactionTargetTokens || !bytes.Contains(visible, []byte("mekugi repetition:")) || + !bytes.Contains(visible, []byte("Continue the router task.")) || !bytes.Contains(visible, []byte("Do not deploy.")) { + t.Fatal("oversized user message lost instructions or retained repetitive bulk") + } + } else if tokens <= compactionTargetTokens || !bytes.Equal(restored[0], input[0]) { + t.Fatal("within-allowance user message was changed") } }) } diff --git a/internal/router/context_compaction_codex_test.go b/internal/router/context_compaction_codex_test.go index a5b410e0..7d60912b 100644 --- a/internal/router/context_compaction_codex_test.go +++ b/internal/router/context_compaction_codex_test.go @@ -1,10 +1,14 @@ package router import ( + "bytes" "context" "encoding/base64" "encoding/json" "fmt" + "image" + "image/color" + "image/png" "io" "net/http" "net/http/httptest" @@ -29,10 +33,16 @@ func TestCompactionInstalledCodex(t *testing.T) { scope string manual bool retirement bool - }{{false, "total", false, false}, {false, "body_after_prefix", false, false}, {true, "total", false, false}, {true, "body_after_prefix", false, false}, {false, "total", true, false}, {true, "total", true, false}, {false, "total", false, true}, {true, "total", false, true}, {false, "total", true, true}, {true, "total", true, true}} { - t.Run(fmt.Sprintf("legacy=%v/scope=%s/manual=%v/retirement=%v", probe.legacy, probe.scope, probe.manual, probe.retirement), func(t *testing.T) { + pressure bool + }{{false, "total", false, false, false}, {false, "body_after_prefix", false, false, false}, {true, "total", false, false, false}, {true, "body_after_prefix", false, false, false}, {false, "total", true, false, false}, {true, "total", true, false, false}, {false, "total", false, true, false}, {true, "total", false, true, false}, {false, "total", true, true, false}, {true, "total", true, true, false}, {false, "total", false, false, true}, {true, "total", false, false, true}, {false, "total", true, false, true}, {true, "total", true, false, true}} { + t.Run(fmt.Sprintf("legacy=%v/scope=%s/manual=%v/retirement=%v/pressure=%v", probe.legacy, probe.scope, probe.manual, probe.retirement, probe.pressure), func(t *testing.T) { + bulk := strings.Repeat("Keep the original user constraint. ", 10000) + if probe.pressure { + bulk = strings.Repeat("x ", 126000) + + "\nCorrection: report every test case; never deploy.\n" + strings.Repeat("x ", 126000) + } prompt := "Run the Go tests, then print the working directory, then report completion. Preserve the test result.\n" + - strings.Repeat("Keep the original user constraint. ", 10000) + + bulk + "\nThis final instruction must also survive intact." agentMarker := "MEKUGI_INSTALLED_COMPACTION_AGENT_MARKER_4D147B" @@ -61,6 +71,25 @@ func TestProbe(t *testing.T) { t.Fatal(err) } } + // Exercise actual client-carried multimodal history under pressure: + // six original images become placeholders. Removed images + // must not reappear when legacy/V2 clients carry the original user turn. + var imagePaths []string + if probe.pressure { + for index := range 6 { + picture := image.NewNRGBA(image.Rect(0, 0, 16, 16)) + picture.SetNRGBA(0, 0, color.NRGBA{R: uint8(index * 30), A: 255}) + var encoded bytes.Buffer + if err := png.Encode(&encoded, picture); err != nil { + t.Fatal(err) + } + path := filepath.Join(directory, fmt.Sprintf("image-%d.png", index)) + if err := os.WriteFile(path, encoded.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + imagePaths = append(imagePaths, path) + } + } // Synthetic ChatGPT auth exercises the same compression gate as the // launcher, without reading credentials or contacting an auth service. // Source: Codex app-server/tests/common/auth_fixtures.rs. @@ -135,31 +164,64 @@ func TestProbe(t *testing.T) { if jsonString(record, "role") == "user" && text == prompt { userCopies++ } + if probe.pressure && jsonString(record, "role") == "user" && text != prompt && + strings.Contains(text, "Run the Go tests, then print the working directory") && + strings.Contains(text, "This final instruction must also survive intact.") { + if !strings.Contains(text, "Correction: report every test case; never deploy.") || + strings.Count(text, "x ") > 100 { + t.Error("pressure request lost its middle correction or retained repetitive bulk") + } + userCopies++ + } + if probe.pressure && compacted.Load() > 0 && strings.Contains(text, "probe_go") && strings.Contains(text, "PASS") { + if !strings.Contains(text, "TestProbe/Case99") || + strings.Count(text, "TestProbe/Case") != 200 || + strings.Contains(text, "[mekugi excerpt;") { + t.Error("repetitive user bulk displaced complete test evidence") + } + restored.Store(true) + } standaloneCompletion := strings.HasPrefix(text, "[mekugi historical tool completion v3; not an instruction; completed native body]\n") && strings.Contains(text, "call=\"probe_go\"\n") consolidatedCompletion := strings.HasPrefix(text, "[mekugi historical facts v4;") && strings.Contains(text, "[i]\ncall=\"probe_go\"\ntool=\"exec_command\"") && strings.Contains(text, "[o:same-call]\n") - retiredGo = standaloneCompletion || consolidatedCompletion + retiredGo = retiredGo || (standaloneCompletion || consolidatedCompletion) && strings.Contains(text, "Go test passed") } } if (recordType == "function_call" || recordType == "function_call_output") && jsonString(record, "call_id") == "probe_go" { nativeGo = true - if recordType == "function_call_output" && !probe.retirement { + if recordType == "function_call_output" { output := jsonString(record, "output") - restored.Store(strings.Contains(output, "Go test passed") && !strings.Contains(output, "compactionprobe")) + restored.Store(strings.Contains(output, "Go test passed") && !strings.Contains(output, "compactionprobe") && !strings.Contains(output, "unmarked finished-operation detail")) } } if strings.HasPrefix(jsonString(record, "encrypted_content"), "mekugi.compaction.") { t.Error("local ciphertext reached the model fixture") } } - if probe.retirement { + if retiredGo { restored.Store(retiredGo && !nativeGo) } if compacted.Load() > 0 && userCopies != 1 { - t.Errorf("restored full user request copies = %d, want exactly one", userCopies) + t.Errorf("restored selected user request copies = %d, want exactly one", userCopies) + } + if probe.pressure && compacted.Load() > 0 { + var items []json.RawMessage + _ = json.Unmarshal(request["input"], &items) + count, ok := compactionVisibleStringTokens(items...) + if !ok || count > compactionTargetTokens+compactionOvershootTokens { + t.Errorf("pressure continuation exceeds the text budget: %d", count) + } + images, imageBytes := compactionImageUsage(items) + if images != 0 || imageBytes != 0 { + t.Errorf("multimodal continuation: images=%d encoded bytes=%d", images, imageBytes) + } + if strings.Count(string(request["input"]), "[Image]") != len(imagePaths) { + t.Error("omitted client-carried images lack an explicit notice") + } + t.Logf("pressure continuation visible-string tokens: %d", count) } if compacted.Load() > 0 && len(agentIDs) == 0 { t.Error("fresh canonical AGENTS marker was lost") @@ -204,6 +266,23 @@ func TestProbe(t *testing.T) { compacted.Add(1) } body, _ := io.ReadAll(r.Body) + if probe.pressure { + var fields map[string]json.RawMessage + _ = json.Unmarshal(body, &fields) + var items []json.RawMessage + _ = json.Unmarshal(fields["input"], &items) + for _, raw := range items { + snapshot, local, err := compactor.openSnapshot(r.Context(), raw) + if err != nil { + t.Errorf("pressure diagnostic envelope: %v", err) + http.Error(w, "invalid pressure diagnostic envelope", http.StatusUnprocessableEntity) + return + } + if local && snapshot.Report != nil { + t.Logf("pressure selection report: %s", mustMarshalJSON(snapshot.Report)) + } + } + } r.Body = io.NopCloser(strings.NewReader(string(body))) tracked := &trackedResponseWriter{ResponseWriter: w} compactor.handler(model)(tracked, r) @@ -265,6 +344,9 @@ metrics_exporter = "none" args = append(args, "--disable", "remote_compaction_v2") } if !probe.manual { + for _, path := range imagePaths { + args = append(args, "--image", path) + } args = append(args, "-") } ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second) @@ -277,7 +359,7 @@ metrics_exporter = "none" wantNormal := operationCount + 1 if probe.manual { wantNormal = operationCount + 2 - err = runManualCompactionProbe(command, directory, prompt) + err = runManualCompactionProbe(command, directory, prompt, imagePaths) } else { command.Stdin = strings.NewReader(prompt) output, err = command.CombinedOutput() @@ -292,7 +374,7 @@ metrics_exporter = "none" // The app-server operation uses the same Op::Compact as the TUI's /compact. // Source: Codex app-server/tests/suite/v2/compaction.rs. -func runManualCompactionProbe(command *exec.Cmd, directory, prompt string) error { +func runManualCompactionProbe(command *exec.Cmd, directory, prompt string, imagePaths []string) error { stdin, err := command.StdinPipe() if err != nil { return err @@ -375,7 +457,13 @@ func runManualCompactionProbe(command *exec.Cmd, directory, prompt string) error if index == 1 { method = "thread/compact/start" } else { - params["input"] = []any{map[string]any{"type": "text", "text": text, "textElements": []any{}}} + input := []any{map[string]any{"type": "text", "text": text, "textElements": []any{}}} + if index == 0 { + for _, path := range imagePaths { + input = append(input, map[string]any{"type": "localImage", "path": path}) + } + } + params["input"] = input } if err := send(index+3, method, params); err != nil { return err diff --git a/internal/router/context_compaction_envelope.go b/internal/router/context_compaction_envelope.go index 538ba252..40fbb902 100644 --- a/internal/router/context_compaction_envelope.go +++ b/internal/router/context_compaction_envelope.go @@ -15,6 +15,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -24,12 +25,31 @@ import ( const ( contextCompactionPrefix = "mekugi.compaction.v1:" + contextCompactionV2Prefix = "mekugi.compaction.v2:" contextCompactionIDPrefix = "cmp_mekugi_" ) +type compactionSnapshot struct { + Items []json.RawMessage `json:"items"` + Report *compactionPressureReport `json:"pressure_report,omitempty"` + Carried []compactionCarriedItem `json:"carried,omitempty"` +} + +// Originals are authenticated reconciliation evidence, never model input. Codex +// can carry a no-ID message or an arbitrary client-truncated version alongside +// the capsule. Retaining its original here lets us match it without resurrecting +// its omitted content. Index is a replay item or insertion point when Removed. +type compactionCarriedItem struct { + Originals []json.RawMessage `json:"originals"` + Index int `json:"index"` + Removed bool `json:"removed,omitzero"` + source int +} + // The key is installation-owned, not session-owned: resumed and forked Codex -// histories must remain readable after a router restart. Only encrypted retained -// history travels in the envelope; no transcript archive or retrieval is used. +// histories must remain readable after a router restart. Retained history and +// client-reconciliation receipts travel encrypted; no external archive or model +// retrieval interface is used. type contextCompactor struct { keyPath string aeadMu sync.Mutex @@ -89,7 +109,16 @@ func (c *contextCompactor) cipher(ctx context.Context, create bool) (cipher.AEAD } func (c *contextCompactor) seal(ctx context.Context, items []json.RawMessage) (json.RawMessage, error) { - plaintext, err := marshalProtocolJSON(items) + return c.sealSnapshot(ctx, compactionSnapshot{Items: items}) +} + +func (c *contextCompactor) sealSnapshot(ctx context.Context, snapshot compactionSnapshot) (json.RawMessage, error) { + prefix := contextCompactionPrefix + var payload any = snapshot.Items + if len(snapshot.Carried) > 0 || snapshot.Report != nil { + prefix, payload = contextCompactionV2Prefix, snapshot + } + plaintext, err := marshalProtocolJSON(payload) if err != nil { return nil, err } @@ -108,7 +137,7 @@ func (c *contextCompactor) seal(ctx context.Context, items []json.RawMessage) (j if err != nil { return nil, err } - encoded := contextCompactionPrefix + base64.RawStdEncoding.EncodeToString(aead.Seal(nil, nil, compressed.Bytes(), []byte(contextCompactionPrefix))) + encoded := prefix + base64.RawStdEncoding.EncodeToString(aead.Seal(nil, nil, compressed.Bytes(), []byte(prefix))) digest := sha256.Sum256([]byte(encoded)) return marshalProtocolJSON(map[string]any{ "type": "compaction", @@ -118,6 +147,12 @@ func (c *contextCompactor) seal(ctx context.Context, items []json.RawMessage) (j } func (c *contextCompactor) open(ctx context.Context, raw json.RawMessage) ([]json.RawMessage, bool, error) { + snapshot, local, err := c.openSnapshot(ctx, raw) + return snapshot.Items, local, err +} + +func (c *contextCompactor) openSnapshot(ctx context.Context, raw json.RawMessage) (compactionSnapshot, bool, error) { + var snapshot compactionSnapshot var item struct { Type string `json:"type"` ID string `json:"id"` @@ -125,45 +160,59 @@ func (c *contextCompactor) open(ctx context.Context, raw json.RawMessage) ([]jso } var fields map[string]json.RawMessage if json.Unmarshal(raw, &fields) != nil { - return nil, false, nil + return snapshot, false, nil } item.Type = jsonString(fields, "type") item.ID = jsonString(fields, "id") item.Content = jsonString(fields, "encrypted_content") local := strings.HasPrefix(item.Content, "mekugi.compaction.") || strings.HasPrefix(item.ID, contextCompactionIDPrefix) if !local { - return nil, false, nil + return snapshot, false, nil + } + prefix := contextCompactionPrefix + if strings.HasPrefix(item.Content, contextCompactionV2Prefix) { + prefix = contextCompactionV2Prefix } - if item.Type != "compaction" || !strings.HasPrefix(item.Content, contextCompactionPrefix) { - return nil, true, errors.New("unsupported or damaged mekugi compaction envelope") + if item.Type != "compaction" || !strings.HasPrefix(item.Content, prefix) { + return snapshot, true, errors.New("unsupported or damaged mekugi compaction envelope") } - encrypted, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(item.Content, contextCompactionPrefix)) + encrypted, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(item.Content, prefix)) if err != nil { - return nil, true, errors.New("invalid mekugi compaction envelope encoding") + return snapshot, true, errors.New("invalid mekugi compaction envelope encoding") } aead, err := c.cipher(ctx, false) if err != nil { - return nil, true, err + return snapshot, true, err } - compressed, err := aead.Open(nil, nil, encrypted, []byte(contextCompactionPrefix)) + compressed, err := aead.Open(nil, nil, encrypted, []byte(prefix)) if err != nil { - return nil, true, errors.New("mekugi compaction envelope authentication failed") + return snapshot, true, errors.New("mekugi compaction envelope authentication failed") } decompressor, err := zlib.NewReader(bytes.NewReader(compressed)) if err != nil { - return nil, true, errors.New("invalid mekugi compaction envelope payload") + return snapshot, true, errors.New("invalid mekugi compaction envelope payload") } defer decompressor.Close() plaintext, err := io.ReadAll(io.LimitReader(decompressor, responsesRequestBufferBytes+1)) if err != nil || len(plaintext) > responsesRequestBufferBytes { - return nil, true, errors.New("mekugi compaction envelope exceeds the router buffer budget or is damaged") + return snapshot, true, errors.New("mekugi compaction envelope exceeds the router buffer budget or is damaged") } if err := ctx.Err(); err != nil { - return nil, true, err + return snapshot, true, err } - var items []json.RawMessage - if json.Unmarshal(plaintext, &items) != nil || len(items) == 0 { - return nil, true, errors.New("invalid retained mekugi compaction history") + if prefix == contextCompactionPrefix { + err = json.Unmarshal(plaintext, &snapshot.Items) + } else { + err = json.Unmarshal(plaintext, &snapshot) + } + if err != nil || len(snapshot.Items) == 0 { + return compactionSnapshot{}, true, errors.New("invalid retained mekugi compaction history") + } + for _, carried := range snapshot.Carried { + if len(carried.Originals) == 0 || slices.ContainsFunc(carried.Originals, func(raw json.RawMessage) bool { return !compactionCarriedMessage(raw) }) || carried.Index < 0 || carried.Index > len(snapshot.Items) || + (!carried.Removed && carried.Index == len(snapshot.Items)) { + return compactionSnapshot{}, true, errors.New("invalid compaction carried-message receipt") + } } - return items, true, nil + return snapshot, true, nil } diff --git a/internal/router/context_compaction_http.go b/internal/router/context_compaction_http.go index 8837b92c..02b876f1 100644 --- a/internal/router/context_compaction_http.go +++ b/internal/router/context_compaction_http.go @@ -105,7 +105,8 @@ func (c *contextCompactor) prepare(ctx context.Context, parsed *parsedResponsesR return fail(http.StatusBadRequest, "compaction input items must be objects") } } - input, err := c.restore(ctx, input) + var carried []compactionCarriedItem + input, err := c.restoreWithCarried(ctx, input, &carried) if err != nil { return fail(http.StatusUnprocessableEntity, err.Error()) } @@ -144,20 +145,93 @@ func (c *contextCompactor) prepare(ctx context.Context, parsed *parsedResponsesR input = input[:len(input)-1] } } - reduced, _, err := selectCompactionWorkingSet(ctx, input, + prepared, imagesChanged, err := stripCompactionImages(ctx, input) + if err != nil { + return nil, err + } + var imageCarried []compactionCarriedItem + for index, original := range input { + if compactionCarriedMessage(original) && string(original) != string(prepared[index]) { + imageCarried = append(imageCarried, compactionCarriedItem{ + Originals: []json.RawMessage{original}, Index: index, source: index, + }) + } + } + reduce := reduceContextCompactionWithPlan + if len(carried) > 0 || len(imageCarried) > 0 { + reduce = func(items []json.RawMessage, plan compactionRetentionPlan) []json.RawMessage { + retained, err := reduceContextCompactionPlan(ctx, items, plan, true) + if err != nil { + return items + } + return retained + } + } + reduced, report, err := selectCompactionWorkingSet(ctx, prepared, compactionTargetTokens, compactionOvershootTokens, - reduceContextCompactionWithPlan, compactionVisibleStringTokens) + reduce, compactionVisibleStringTokens) + // Replacing an image with a text marker is useful compaction even when + // the text-only metric cannot shrink. Do not escalate unrelated history. + if imagesChanged && errors.Is(err, errCompactionNoReduction) { + reduced, err = prepared, nil + } + snapshot := compactionSnapshot{Items: reduced, Carried: imageCarried} + var positions []int + if ctx.Err() == nil && err != nil && report.after > compactionTargetTokens+compactionOvershootTokens { + snapshot, positions, err = pressureCompactionWorkingSet(ctx, input, compactionTargetTokens) + } if err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil, err } return fail(http.StatusUnprocessableEntity, err.Error()) } - capsule, err := c.seal(ctx, reduced) + if len(carried) > 0 { + if positions == nil { + // Receipt-bearing evidence plans preserve source positions even + // when metadata cleanup changes an item's reconciliation identity. + if len(input) != len(snapshot.Items) { + return fail(http.StatusUnprocessableEntity, "compaction lost carried-message source positions") + } + positions = make([]int, len(input)+1) + for index := range positions { + positions[index] = index + } + } + // Merge in the pre-selection timeline, not by final insertion point: + // several consecutive dropped items can share that final position. + generated := make(map[int]compactionCarriedItem) + for _, receipt := range snapshot.Carried { + generated[receipt.source] = receipt + } + inherited := make(map[int][]compactionCarriedItem) + for _, receipt := range carried { + inherited[receipt.Index] = append(inherited[receipt.Index], receipt) + } + snapshot.Carried = nil + for source := range len(input) + 1 { + next, changed := generated[source] + for _, receipt := range inherited[source] { + receipt.Index = positions[source] + if receipt.Removed { + snapshot.Carried = append(snapshot.Carried, receipt) + } else if changed { + next.Originals = append(next.Originals, receipt.Originals...) + } else { + receipt.Removed = positions[source] == positions[source+1] + next, changed = receipt, true + } + } + if changed { + snapshot.Carried = append(snapshot.Carried, next) + } + } + } + capsule, err := c.sealSnapshot(ctx, snapshot) if err != nil { return fail(http.StatusUnprocessableEntity, err.Error()) } - parsed.setInput(mustMarshalJSON(reduced)) + parsed.setInput(mustMarshalJSON(snapshot.Items)) return capsule, nil } @@ -205,9 +279,14 @@ func writeContextCompactionResponse(writer io.Writer, capsule, retained json.Raw // them. Align those carried items with the snapshot rather than blindly dropping // the prefix or duplicating every user message. Unmatched current context stays. func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) ([]json.RawMessage, error) { + return c.restoreWithCarried(ctx, input, nil) +} + +func (c *contextCompactor) restoreWithCarried(ctx context.Context, input []json.RawMessage, receipts *[]compactionCarriedItem) ([]json.RawMessage, error) { type restoredItem struct { raw json.RawMessage fromEnvelope bool + originals []json.RawMessage } withinBudget := func(items []restoredItem) bool { remaining := responsesRequestBufferBytes - 2 // JSON array brackets. @@ -219,6 +298,12 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) return false } remaining -= len(item.raw) + for _, original := range item.originals { + remaining -= len(original) + } + if remaining < 0 { + return false + } } return true } @@ -228,7 +313,7 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) if err := ctx.Err(); err != nil { return nil, err } - retained, local, err := c.open(ctx, item) + snapshot, local, err := c.openSnapshot(ctx, item) if err != nil { return nil, err } @@ -236,9 +321,8 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) output = append(output, restoredItem{raw: item}) continue } - retainedItems := make([]restoredItem, len(retained)) - positions := make(map[string][]int, len(retained)) - for index, raw := range retained { + retainedItems := make([]restoredItem, len(snapshot.Items)) + for index, raw := range snapshot.Items { var fields map[string]json.RawMessage if json.Unmarshal(raw, &fields) != nil || fields == nil { return nil, errors.New("invalid item in retained compaction history") @@ -247,17 +331,47 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) return nil, errors.New("nested local compaction envelope is not supported") } retainedItems[index] = restoredItem{raw: raw, fromEnvelope: true} - identity := contextCompactionItemIdentity(raw) - positions[identity] = append(positions[identity], index) + } + removed := make(map[int][]restoredItem) + for _, receipt := range snapshot.Carried { + if receipt.Removed { + removed[receipt.Index] = append(removed[receipt.Index], restoredItem{fromEnvelope: true, originals: receipt.Originals}) + } else { + retainedItems[receipt.Index].originals = append(retainedItems[receipt.Index].originals, receipt.Originals...) + } + } + var expanded []restoredItem + for index := range len(retainedItems) + 1 { + expanded = append(expanded, removed[index]...) + if index < len(retainedItems) { + expanded = append(expanded, retainedItems[index]) + } + } + retainedItems = expanded + positions := make(map[string][]int, len(retainedItems)) + for index, retained := range retainedItems { + for _, raw := range append(slices.Clone(retained.originals), retained.raw) { + if len(raw) == 0 { + continue + } + identity := contextCompactionItemIdentity(raw) + matches := positions[identity] + if len(matches) == 0 || matches[len(matches)-1] != index { + positions[identity] = append(matches, index) + } + } } // Codex retains the newest end of history. Match backwards so a // repeated no-ID user message anchors fresh context at its latest // occurrence, rather than before an older conflicting instruction. matched := make([]int, len(output)) - limit := len(retained) + limit := len(retainedItems) for index := len(output) - 1; index >= 0; index-- { matched[index] = -1 carried := output[index].raw + if len(carried) == 0 && len(output[index].originals) > 0 { + carried = output[index].originals[0] + } if !output[index].fromEnvelope && contextCompactionFreshContext(carried) { continue } @@ -267,7 +381,11 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) matched[index] = matches[match-1] } else { for candidate := range limit { - if contextCompactionTruncatedMatch(carried, retained[candidate]) { + retained := retainedItems[candidate] + originals := append(slices.Clone(retained.originals), retained.raw) + if slices.ContainsFunc(originals, func(original json.RawMessage) bool { + return contextCompactionTruncatedMatch(carried, original) + }) { if matched[index] >= 0 { return nil, errors.New("ambiguous truncated message in compacted history") } @@ -285,10 +403,12 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) } } var merged, pending []restoredItem + unmatchedEnvelope := false cursor := 0 for position, carried := range output { index := matched[position] if index < 0 { + unmatchedEnvelope = unmatchedEnvelope || carried.fromEnvelope pending = append(pending, carried) continue } @@ -304,14 +424,24 @@ func (c *contextCompactor) restore(ctx context.Context, input []json.RawMessage) if !withinBudget(candidate) { return nil, errors.New("restored compaction history exceeds the router buffer budget") } + if unmatchedEnvelope { + return nil, errors.New("cannot safely reconcile overlapping compaction envelopes; discarded history was not restored") + } output = candidate } if !withinBudget(output) { return nil, errors.New("restored compaction history exceeds the router buffer budget") } - result := make([]json.RawMessage, len(output)) - for index, item := range output { - result[index] = item.raw + result := make([]json.RawMessage, 0, len(output)) + for _, item := range output { + if receipts != nil { + if len(item.originals) > 0 { + *receipts = append(*receipts, compactionCarriedItem{Originals: item.originals, Index: len(result), Removed: len(item.raw) == 0}) + } + } + if len(item.raw) > 0 { + result = append(result, item.raw) + } } return result, nil diff --git a/internal/router/context_compaction_images.go b/internal/router/context_compaction_images.go new file mode 100644 index 00000000..441ae7a6 --- /dev/null +++ b/internal/router/context_compaction_images.go @@ -0,0 +1,96 @@ +package router + +import ( + "context" + "encoding/json" + "fmt" + "slices" +) + +// Ordinary historical images are intentionally discarded during compaction. +// Fresh request images and mandatory instruction content are not rewritten. +const compactionImagePlaceholder = "[Image]" + +func compactionImageParts(item map[string]json.RawMessage) (string, []json.RawMessage) { + key := "" + switch jsonString(item, "type") { + case "message", "agent_message": + key = "content" + case "function_call_output", "custom_tool_call_output": + key = "output" + } + var parts []json.RawMessage + if key != "" { + _ = json.Unmarshal(item[key], &parts) + } + return key, parts +} + +func compactionImageURL(raw json.RawMessage) string { + var part map[string]json.RawMessage + if json.Unmarshal(raw, &part) == nil && jsonString(part, "type") == "input_image" { + return jsonString(part, "image_url") + } + return "" +} + +func compactionImageUsage(items []json.RawMessage) (count, bytes int) { + for _, raw := range items { + var item map[string]json.RawMessage + _ = json.Unmarshal(raw, &item) + _, parts := compactionImageParts(item) + for _, part := range parts { + if url := compactionImageURL(part); url != "" { + count++ + bytes += len(url) + } + } + } + return count, bytes +} + +func stripCompactionImages(ctx context.Context, input []json.RawMessage) ([]json.RawMessage, bool, error) { + result := slices.Clone(input) + changed := false + for index, raw := range input { + if err := ctx.Err(); err != nil { + return nil, false, err + } + var item map[string]json.RawMessage + if json.Unmarshal(raw, &item) != nil || item == nil { + return nil, false, fmt.Errorf("invalid image history item") + } + if contextCompactionFreshContext(raw) { + continue + } + key, parts := compactionImageParts(item) + itemChanged := false + for partIndex, raw := range parts { + var part map[string]json.RawMessage + if json.Unmarshal(raw, &part) != nil || jsonString(part, "type") != "input_image" { + continue + } + textType := "input_text" + if jsonString(item, "type") == "message" && jsonString(item, "role") == "assistant" { + textType = "output_text" + } + parts[partIndex] = mustMarshalJSON(map[string]string{"type": textType, "text": compactionImagePlaceholder}) + var metadata map[string]json.RawMessage + if json.Unmarshal(item["internal_chat_message_metadata_passthrough"], &metadata) == nil && metadata != nil { + var kinds []string + if json.Unmarshal(metadata["content_item_kinds"], &kinds) == nil && len(kinds) == len(parts) { + kinds[partIndex] = "unknown" + metadata["content_item_kinds"] = mustMarshalJSON(kinds) + item["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(metadata) + } + } + itemChanged = true + } + if itemChanged { + item[key] = mustMarshalJSON(parts) + result[index] = mustMarshalJSON(item) + changed = true + } + } + return result, changed, nil +} diff --git a/internal/router/context_compaction_images_test.go b/internal/router/context_compaction_images_test.go new file mode 100644 index 00000000..229d7b21 --- /dev/null +++ b/internal/router/context_compaction_images_test.go @@ -0,0 +1,101 @@ +package router + +import ( + "bytes" + "encoding/json" + "net/http" + "path/filepath" + "strings" + "testing" +) + +func TestCompactionImageOnlyPreservesSurroundingHistory(t *testing.T) { + for _, kind := range []string{"user", "assistant", "agent_message", "function_call_output", "custom_tool_call_output"} { + t.Run(kind, func(t *testing.T) { + fields := map[string]any{"type": "message", "role": kind, "id": "picture-message", + "metadata": map[string]string{"trace": "unchanged"}} + key, textType := "content", "input_text" + if kind == "assistant" { + textType = "output_text" + } else if kind == "agent_message" { + fields["type"] = kind + delete(fields, "role") + fields["author"], fields["recipient"] = "reviewer", "main" + } + reasoning := mustMarshalJSON(map[string]any{"type": "reasoning", "id": "opaque-reasoning", "encrypted_content": "opaque", "summary": []any{}}) + input := []json.RawMessage{reasoning} + if strings.HasSuffix(kind, "_output") { + key, fields["type"], fields["call_id"] = "output", kind, "picture-call" + delete(fields, "role") + call := map[string]any{"type": strings.TrimSuffix(kind, "_output"), "name": "unfamiliar_view", "call_id": "picture-call"} + if kind == "function_call_output" { + call["arguments"] = "{}" + } else { + call["input"] = "inspect" + } + input = append(input, mustMarshalJSON(call)) + } + left := mustMarshalJSON(map[string]string{"type": textType, "text": "Exact request before the picture.", "annotation": "unchanged"}) + right := mustMarshalJSON(map[string]string{"type": textType, "text": "Exact correction after the picture."}) + unknown := mustMarshalJSON(map[string]string{"type": "future_part", "value": "unchanged"}) + image := mustMarshalJSON(map[string]string{"type": "input_image", "image_url": "data:image/png;base64,cGljdHVyZQ==", "detail": "high"}) + fields[key] = []json.RawMessage{left, image, right, unknown} + original := mustMarshalJSON(fields) + input = append(input, original) + fields[key] = []json.RawMessage{left, mustMarshalJSON(map[string]string{"type": textType, "text": "[Image]"}), right, unknown} + want := append([]json.RawMessage(nil), input...) + want[len(want)-1] = mustMarshalJSON(fields) + baseline := mustMarshalJSON(input) + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "loopback", "input": input})) + if err != nil { + t.Fatal(err) + } + // Ordinary requests are not compaction: their fresh images stay native. + if capsule, err := compactor.prepare(t.Context(), &parsed, http.Header{}, false); err != nil || len(capsule) != 0 || + !bytes.Equal(parsed.fields["input"], baseline) { + t.Fatal("ordinary request changed a fresh image") + } + capsule, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + carried := []json.RawMessage{capsule} + if compactionCarriedMessage(original) { + carried = []json.RawMessage{original, capsule} + } + restored, err := compactor.restore(t.Context(), carried) + if err != nil || !bytes.Equal(mustMarshalJSON(restored), mustMarshalJSON(want)) { + t.Fatalf("image-only compaction changed surrounding history: %v", err) + } + fresh := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": []json.RawMessage{mustMarshalJSON(map[string]string{"type": "input_text", "text": "A fresh picture."}), image}}) + continued, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "loopback", "input": []json.RawMessage{capsule, fresh}})) + if err != nil { + t.Fatal(err) + } + if _, err := compactor.prepare(t.Context(), &continued, http.Header{}, false); err != nil { + t.Fatal(err) + } + var items []json.RawMessage + _ = json.Unmarshal(continued.fields["input"], &items) + if count, _ := compactionImageUsage(items); count != 1 || !bytes.Equal(items[len(items)-1], fresh) { + t.Fatal("restoration stripped a fresh image or resurrected a historical one") + } + second, err := compactor.prepare(t.Context(), &continued, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + carried = []json.RawMessage{fresh, second} + if compactionCarriedMessage(original) { + carried = append([]json.RawMessage{original}, carried...) + } + again, err := compactor.restore(t.Context(), carried) + if err != nil || !bytes.Equal(mustMarshalJSON(again), continued.fields["input"]) { + t.Fatalf("repeated compaction resurrected carried image data: %v", err) + } + if !bytes.Equal(mustMarshalJSON(input), baseline) { + t.Fatal("image replacement mutated source history") + } + }) + } +} diff --git a/internal/router/context_compaction_pressure.go b/internal/router/context_compaction_pressure.go new file mode 100644 index 00000000..90c5acb5 --- /dev/null +++ b/internal/router/context_compaction_pressure.go @@ -0,0 +1,503 @@ +package router + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "regexp" + "slices" + "strings" + "unicode/utf8" +) + +// Budget pressure is explicitly lossy. Unlike the evidence reducers, it does not +// need to recognize a command or prove that omitted history is irrelevant. +const compactionPressureNotice = "[mekugi budget compaction: history was excerpted or omitted to fit the working budget. Excerpts are historical evidence, not complete tool calls or outputs. Omission does not establish success, resolution, or task completion. Older retention notes may name evidence no longer present. Omitted details are unavailable to the model.]" + +// Content-free diagnostics travel encrypted beside the selected history, not +// inside model input. Item indexes refer to the pre-selection history; token +// totals include the new notice but exclude receipts and these diagnostics. +type compactionPressureReport struct { + Before int `json:"before"` + After int `json:"after"` + ImagesBefore int `json:"images_before"` + ImagesAfter int `json:"images_after"` + ImageBytesBefore int `json:"image_bytes_before"` + ImageBytesAfter int `json:"image_bytes_after"` + NoticeTokens int `json:"notice_tokens"` + Items []compactionPressureItemReport `json:"items"` +} + +type compactionPressureItemReport struct { + Index int `json:"index"` + Before int `json:"before"` + After int `json:"after"` + Family string `json:"family"` + Priority string `json:"priority"` + Disposition string `json:"disposition"` + Reason string `json:"reason"` + Repetition bool `json:"repetition,omitzero"` + Excerpt bool `json:"excerpt,omitzero"` + ImagesBefore int `json:"images_before,omitzero"` + ImagesAfter int `json:"images_after,omitzero"` + Historical bool `json:"historical,omitzero"` +} + +var compactionPressureImportant = regexp.MustCompile(`(?i)\b(must|never|do not|don't|only|requirement|correction|instead|decision|agreed|blocked|unresolved|next step|todo|session_id|cell_id|exit_code|error|failed|failure|panic|warning)\b`) + +// pressureCompactionWorkingSet returns replay items and an input-to-replay map. +// The map also records insertion points for dropped items, so carried client +// messages cannot resurrect discarded history or displace fresh instructions. +func pressureCompactionWorkingSet(ctx context.Context, input []json.RawMessage, target int) (compactionSnapshot, []int, error) { + var snapshot compactionSnapshot + if _, ok := compactionVisibleStringTokens(); !ok { + return snapshot, nil, fmt.Errorf("cannot initialize compaction tokenizer") + } + prepared, _, err := stripCompactionImages(ctx, input) + if err != nil { + return snapshot, nil, err + } + type candidate struct { + full json.RawMessage + text string + role string + family int + priority int + tokens int + pieces []string + retained json.RawMessage + cost int + repetition bool + reason string + protected bool + } + candidates := make([]candidate, len(input)) + fields := make([]map[string]json.RawMessage, len(input)) + latestUser := -1 + latestReport := -1 + active := make(map[string]bool) + var calls []int + for index, raw := range prepared { + if err := ctx.Err(); err != nil { + return snapshot, nil, err + } + if err := json.Unmarshal(raw, &fields[index]); err != nil || fields[index] == nil { + return snapshot, nil, fmt.Errorf("invalid history item at %d", index) + } + item := fields[index] + kind, role := jsonString(item, "type"), jsonString(item, "role") + if kind == "message" { + if role == "user" && !contextCompactionFreshContext(raw) { + latestUser = index + } + } + if kind == "function_call" || kind == "custom_tool_call" { + calls = append(calls, index) + active[jsonString(item, "call_id")] = true + } + if kind == "function_call_output" || kind == "custom_tool_call_output" { + _, _, success := contextCompactionOutput(item["output"]) + _, _, failure := compactionFailedOutput(item["output"]) + if success || failure { + delete(active, jsonString(item, "call_id")) + } + } + } + frontier := len(input) + if len(calls) > 0 { + frontier = calls[max(0, len(calls)-compactionRecentOperations)] + } + for index, item := range fields { + if err := ctx.Err(); err != nil { + return snapshot, nil, err + } + kind, role := jsonString(item, "type"), jsonString(item, "role") + c := &candidates[index] + c.full, c.role, c.priority, c.family = prepared[index], role, 3, 2 + c.protected = contextCompactionFreshContext(input[index]) + switch { + case kind == "message" && (role == "user" || role == "developer" || role == "system"): + c.priority, c.family = 1, 0 + case kind == "message" || kind == "agent_message" || kind == "reasoning": + c.priority, c.family = 2, 1 + } + if index >= frontier || index >= len(input)-8 { + c.priority = min(c.priority, 1) + } + if id := jsonString(item, "call_id"); id != "" && active[id] { + c.priority = min(c.priority, 1) + } + if index == latestUser { + c.priority = 0 + } + if !c.protected && (kind == "agent_message" || kind == "message" && role == "assistant") && + strings.TrimSpace(strings.Join(contextCompactionNarrationTexts(item["content"]), "\n")) != "" { + latestReport = index + } + // Native tool/reasoning groups must not be partially truncated. The + // pressure pass represents all of them as non-executable observations, + // retaining identities, arguments and lifecycle facts as budget permits. + if kind != "message" && kind != "agent_message" && kind != "compaction" { + copyItem := maps.Clone(item) + delete(copyItem, "encrypted_content") + c.text = fmt.Sprintf("[mekugi historical %s; not an executable invocation]\n%s", kind, mustMarshalJSON(copyItem)) + if kind == "function_call_output" || kind == "custom_tool_call_output" { + var body string + if json.Unmarshal(item["output"], &body) == nil { + delete(copyItem, "output") + c.text = fmt.Sprintf("[mekugi historical %s; observed output, not completion inferred]\n%s\n", kind, mustMarshalJSON(copyItem)) + var result map[string]json.RawMessage + var output string + if json.Unmarshal([]byte(body), &result) == nil && result != nil && json.Unmarshal(result["output"], &output) == nil { + delete(result, "output") + c.text += "observed metadata=" + string(mustMarshalJSON(result)) + "\n" + body = output + } + c.text += body + } + } + c.role = "assistant" + c.full = compactionPressureRender(item, c.role, c.text) + } else { + c.text = strings.Join(contextCompactionNarrationTexts(item["content"]), "\n") + if c.text == "" { + copyItem := maps.Clone(item) + delete(copyItem, "encrypted_content") + c.text = string(mustMarshalJSON(copyItem)) + } + } + if !c.protected && kind != "compaction" { + if kind == "message" || kind == "agent_message" { + var reductionErr error + content, changed := contextCompactionMapNarration(item["content"], func(text string) string { + if reductionErr != nil { + return text + } + reduced, err := compactionPressureRepetitions(ctx, text) + reductionErr = err + return reduced + }) + if reductionErr != nil { + return snapshot, nil, reductionErr + } + if changed { + message := maps.Clone(item) + message["content"] = content + c.full = mustMarshalJSON(message) + c.text = strings.Join(contextCompactionNarrationTexts(content), "\n") + c.repetition = true + } + } else { + text, err := compactionPressureRepetitions(ctx, c.text) + if err != nil { + return snapshot, nil, err + } + if text != c.text { + c.repetition = true + c.text = text + c.full = compactionPressureRender(item, c.role, text) + } + } + } + + var ok bool + c.tokens, ok = compactionVisibleStringTokens(c.full) + if !ok { + return snapshot, nil, fmt.Errorf("cannot measure pressure compaction item") + } + if c.protected { + c.retained, c.cost = c.full, c.tokens + } + } + order := make([]int, len(input)) + for index := range order { + order[index] = index + } + slices.SortStableFunc(order, func(a, b int) int { + if delta := candidates[a].priority - candidates[b].priority; delta != 0 { + return delta + } + return b - a + }) + notice := compactionPressureMessage("assistant", compactionPressureNotice) + noticeTokens, _ := compactionVisibleStringTokens(notice) + used := noticeTokens + for _, c := range candidates { + used += c.cost + } + ceiling := target + compactionOvershootTokens + if used > ceiling { + return snapshot, nil, fmt.Errorf("required developer/system instructions and canonical context plus compaction notice retain %d visible-string tokens, exceeding ceiling %d; required instructions were not truncated", used, ceiling) + } + // Required instructions have a hard floor. Use the overshoot allowance + // when needed to leave room for the current request and execution state. + target = min(ceiling, max(target, used+target/3)) + // Cover the request chain and discussion before allocating execution bulk. + // A recent acknowledgement is not a replacement for the request it answers, + // and unknown tool lifecycle states must not displace all older decisions. + allocationEnd := target + retain := func(index, allowance int, reason string) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + c := &candidates[index] + if c.protected || string(c.retained) == string(c.full) { + return false, nil + } + available := min(allocationEnd-used+c.cost, allowance) + if available <= c.cost { + return false, nil + } + retained, cost := c.full, c.tokens + if cost > available { + if available < 96 { + return false, nil + } + if c.pieces == nil { + _, pieces, err := compactionRetirementTokenCodec.Encode(c.text) + if err != nil { + return false, err + } + c.pieces = pieces + } + // Measure the rendered item, including identity metadata and markers. + for keep := available - 64; keep > 0; keep -= max(16, cost-available) { + text := compactionPressureExcerpt(c.text, c.pieces, keep) + retained = compactionPressureRender(fields[index], c.role, text) + var ok bool + cost, ok = compactionVisibleStringTokens(retained) + if !ok { + return false, fmt.Errorf("cannot measure compaction excerpt") + } + if cost <= available { + break + } + } + } + if cost <= available && cost > c.cost { + used += cost - c.cost + c.retained, c.cost, c.reason = retained, cost, reason + return true, nil + } + return false, nil + } + allocate := func(indices []int, budget int, reason string) error { + allocationEnd = min(target, used+budget) + for { + weight := 0 + for _, index := range indices { + c := &candidates[index] + if !c.protected && string(c.retained) != string(c.full) { + weight += []int{4, 2, 1, 1}[c.priority] + } + } + remaining := allocationEnd - used + if weight == 0 || remaining <= 0 { + return nil + } + progress := false + for _, index := range indices { + c := &candidates[index] + // Redistribute unused shares after complete small items fit. + // Minimum framing can force omissions in very long timelines, + // but never lets execution consume the request/discussion shares. + share := max(96, remaining*[]int{4, 2, 1, 1}[c.priority]/weight) + changed, err := retain(index, c.cost+share, reason) + if err != nil { + return err + } + progress = progress || changed + } + if !progress { + return nil + } + } + } + var families [3][]int + for _, index := range order { + families[candidates[index].family] = append(families[candidates[index].family], index) + } + // These are reserved opportunities, not ceilings on an item's final size. + // Unspent capacity flows onward and then back to unfinished items. + if err := allocate(families[0], (target-used)/3, "request_chain"); err != nil { + return snapshot, nil, err + } + discussionBudget := (target - used) / 2 + discussionStart := used + // The newest readable report often carries continuation state. Give it a + // bounded head start before broad timeline coverage, without guessing from + // words such as "handoff" or granting an unlimited exact-retention exemption. + if latestReport >= 0 { + if err := allocate([]int{latestReport}, discussionBudget/4, "latest_report"); err != nil { + return snapshot, nil, err + } + } + if err := allocate(families[1], discussionBudget-(used-discussionStart), "discussion_coverage"); err != nil { + return snapshot, nil, err + } + if err := allocate(families[2], target-used, "execution_coverage"); err != nil { + return snapshot, nil, err + } + if err := allocate(order, target-used, "shared_surplus"); err != nil { + return snapshot, nil, err + } + snapshot.Items = append(snapshot.Items, notice) + snapshot.Report = &compactionPressureReport{NoticeTokens: noticeTokens} + snapshot.Report.After = snapshot.Report.NoticeTokens + positions := make([]int, len(input)+1) + for index, c := range candidates { + before, ok := compactionVisibleStringTokens(input[index]) + if !ok { + return compactionSnapshot{}, nil, fmt.Errorf("cannot measure original pressure item") + } + imagesBefore, _ := compactionImageUsage([]json.RawMessage{input[index]}) + imagesAfter, _ := compactionImageUsage([]json.RawMessage{c.retained}) + itemReport := compactionPressureItemReport{ + Index: index, Before: before, After: c.cost, + Priority: []string{"latest_request", "request_or_active", "decision_or_reasoning", "older_evidence"}[c.priority], + Family: []string{"requests", "discussion", "execution"}[c.family], + ImagesBefore: imagesBefore, ImagesAfter: imagesAfter, + Reason: c.reason, Repetition: c.repetition, + } + switch { + case c.protected: + itemReport.Priority, itemReport.Reason, itemReport.Disposition = "mandatory", "mandatory_floor", "exact" + case len(c.retained) == 0: + itemReport.Disposition, itemReport.Reason, itemReport.Repetition = "dropped", "budget_exhausted", false + case string(c.retained) == string(input[index]): + itemReport.Disposition = "exact" + default: + itemReport.Excerpt = string(c.retained) != string(c.full) + kind := jsonString(fields[index], "type") + itemReport.Historical = kind != "message" && kind != "agent_message" && kind != "compaction" + switch { + case itemReport.Excerpt: + itemReport.Disposition = "excerpt" + case itemReport.Repetition: + itemReport.Disposition = "repetition_reduced" + case imagesAfter < imagesBefore && !itemReport.Historical: + itemReport.Disposition = "image_omitted" + default: + itemReport.Disposition = "historical" + } + } + snapshot.Report.Before += before + snapshot.Report.After += c.cost + snapshot.Report.Items = append(snapshot.Report.Items, itemReport) + positions[index] = len(snapshot.Items) + if compactionCarriedMessage(input[index]) && string(c.retained) != string(input[index]) { + snapshot.Carried = append(snapshot.Carried, compactionCarriedItem{Originals: []json.RawMessage{input[index]}, Index: len(snapshot.Items), Removed: len(c.retained) == 0, source: index}) + } + if len(c.retained) > 0 { + snapshot.Items = append(snapshot.Items, c.retained) + } + } + snapshot.Report.ImagesBefore, snapshot.Report.ImageBytesBefore = compactionImageUsage(input) + snapshot.Report.ImagesAfter, snapshot.Report.ImageBytesAfter = compactionImageUsage(snapshot.Items) + positions[len(input)] = len(snapshot.Items) + if measured, ok := compactionVisibleStringTokens(snapshot.Items...); !ok || measured > target { + return compactionSnapshot{}, nil, fmt.Errorf("pressure compaction exceeded target %d", target) + } + return snapshot, positions, nil +} + +func compactionPressureMessage(role, text string) json.RawMessage { + textType := "input_text" + switch role { + case "user", "developer", "system": + default: + role, textType = "assistant", "output_text" + } + return mustMarshalJSON(map[string]any{"type": "message", "role": role, + "content": []any{map[string]string{"type": textType, "text": text}}}) +} + +// Budget excerpts preserve message identity and non-positional metadata; +// positional classifications are rebuilt for the excerpt. +func compactionPressureRender(item map[string]json.RawMessage, role, text string) json.RawMessage { + kind := jsonString(item, "type") + // Agent reports are native messages with their own author and recipient, not + // unattributed assistant prose. Their content uses input_text. + if kind == "agent_message" { + role = "user" + } + rendered := compactionPressureMessage(role, text) + var excerpt map[string]json.RawMessage + _ = json.Unmarshal(rendered, &excerpt) + var content []json.RawMessage + _ = json.Unmarshal(excerpt["content"], &content) + if kind != "message" && kind != "agent_message" { + return mustMarshalJSON(excerpt) + } + message := maps.Clone(item) + message["content"] = excerpt["content"] + var metadata map[string]json.RawMessage + if json.Unmarshal(message["internal_chat_message_metadata_passthrough"], &metadata) == nil && metadata != nil { + if raw, exists := metadata["content_item_kinds"]; exists { + var kinds []string + var parts []json.RawMessage + _ = json.Unmarshal(item["content"], &parts) + kind := "unknown" + if json.Unmarshal(raw, &kinds) == nil && len(kinds) > 0 && len(kinds) == len(parts) && + !slices.ContainsFunc(kinds, func(value string) bool { return value == "" || value != kinds[0] }) { + kind = kinds[0] + } + selectedKinds := make([]string, len(content)) + for index := range selectedKinds { + selectedKinds[index] = "unknown" + } + if len(content) == 1 { + selectedKinds[0] = kind + } + metadata["content_item_kinds"] = mustMarshalJSON(selectedKinds) + message["internal_chat_message_metadata_passthrough"] = mustMarshalJSON(metadata) + } + } + return mustMarshalJSON(message) +} + +func compactionPressureExcerpt(text string, pieces []string, budget int) string { + // Keep bounded exact excerpts, not an invented semantic summary. Important + // lines in the middle compete for a third of the allowance; prefix/suffix + // preserve setup and the latest conclusion even for unrecognized content. + valid := func(value string) string { + for len(value) > 0 && !utf8.ValidString(value) { + if !utf8.RuneStart(value[0]) { + value = value[1:] + } else { + value = value[:len(value)-1] + } + } + return value + } + var evidence strings.Builder + remaining := budget / 3 + for line := range strings.SplitSeq(text, "\n") { + if remaining <= 0 || !compactionPressureImportant.MatchString(line) { + continue + } + _, tokens, err := compactionRetirementTokenCodec.Encode(line + "\n") + if err != nil { + continue + } + keep := min(len(tokens), remaining) + evidence.WriteString(valid(strings.Join(tokens[:keep], ""))) + remaining -= keep + } + ends := budget - (budget/3 - remaining) + head := min(len(pieces), ends/2) + tail := min(len(pieces)-head, ends-head) + result := "[mekugi excerpt; omitted text is unavailable]\n" + valid(strings.Join(pieces[:head], "")) + if evidence.Len() > 0 { + result += "\n[... selected constraint/status excerpts ...]\n" + evidence.String() + } + return result + "\n[... omitted ...]\n" + valid(strings.Join(pieces[len(pieces)-tail:], "")) +} + +func compactionCarriedMessage(raw json.RawMessage) bool { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + return jsonString(fields, "type") == "agent_message" || + (jsonString(fields, "type") == "message" && jsonString(fields, "role") == "user") +} diff --git a/internal/router/context_compaction_pressure_test.go b/internal/router/context_compaction_pressure_test.go new file mode 100644 index 00000000..1e4e5b50 --- /dev/null +++ b/internal/router/context_compaction_pressure_test.go @@ -0,0 +1,762 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "path/filepath" + "slices" + "strings" + "testing" + "unicode/utf8" +) + +func TestCompactionPressureContentIndependent(t *testing.T) { + for _, size := range []int{252_000, 282_000} { + for _, shape := range []string{"newest-unknown-output", "live-exec", "failed-exec", "unique-prose", "user", "agent", "dynamic-script"} { + t.Run(fmt.Sprintf("%s/%d", shape, size), func(t *testing.T) { + var text strings.Builder + for index := range size / 8 { + fmt.Fprintf(&text, "Historical observation %d: distinct ordinary content.\n", index) + } + _, _ = compactionVisibleStringTokens() + _, pieces, err := compactionRetirementTokenCodec.Encode(text.String()) + if err != nil || len(pieces) < size { + t.Fatalf("fixture tokenization: %d: %v", len(pieces), err) + } + bulk := strings.Join(pieces[:size], "") + authority := compactionPressureMessage("developer", "Only change the router. Never run deployments.") + request := compactionPressureMessage("user", "Continue the current task. The migration remains unresolved.") + history := func(bulk string) []json.RawMessage { + input := []json.RawMessage{authority, request} + switch shape { + case "newest-unknown-output": + input = append(input, mustMarshalJSON(map[string]any{"type": "function_call", "name": "unfamiliar_tool", "call_id": "unknown", "arguments": `{"path":"current.go"}`}), + mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "unknown", "output": bulk})) + case "live-exec": + input = append(input, compactTestCall("live", "long-running-command"), mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "live", "output": string(mustMarshalJSON(map[string]any{"output": bulk, "session_id": 1234, "exit_code": nil}))})) + case "failed-exec": + input = append(input, compactTestCall("failed", "python job.py"), compactTestOutput("failed", "Traceback (most recent call last):\n"+bulk+"\nRuntimeError: transaction remains unresolved", 1)) + case "unique-prose": + for index := range 20 { + start, end := index*len(bulk)/20, (index+1)*len(bulk)/20 + input = append(input, compactionPressureMessage("assistant", bulk[start:end])) + } + case "user": + input = append(input, compactionPressureMessage("user", "Current task constraints:\n"+bulk+"\nDo not change the public API.")) + case "agent": + input = append(input, mustMarshalJSON(map[string]any{"type": "agent_message", "content": bulk})) + case "dynamic-script": + input = append(input, mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "exec", "call_id": "dynamic", "input": "const result = await tools.chooseDynamically();\n" + bulk}), + mustMarshalJSON(map[string]any{"type": "custom_tool_call_output", "call_id": "dynamic", "output": "Status unknown; do not assume success"})) + } + return input + } + input := history(bulk) + before, ok := compactionVisibleStringTokens(input...) + if !ok { + t.Fatal("cannot measure input history") + } + original := mustMarshalJSON(input) + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": input})) + if err != nil { + t.Fatal(err) + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + // A fresh router instance must restore only the selected history, + // including when the client also carries original no-ID messages. + var carried []json.RawMessage + for _, item := range input { + if compactionCarriedMessage(item) { + carried = append(carried, item) + } + } + resumed := &contextCompactor{keyPath: compactor.keyPath} + restored, err := resumed.restore(t.Context(), append(carried, capsule)) + if err != nil { + t.Fatal(err) + } + after, ok := compactionVisibleStringTokens(restored...) + if !ok || after < 20_000 || after > compactionTargetTokens+compactionOvershootTokens { + t.Fatalf("budget missed: %d -> %d", before, after) + } + if !bytes.Equal(mustMarshalJSON(restored), parsed.fields["input"]) { + t.Fatal("carried messages resurrected discarded content") + } + if !slices.ContainsFunc(restored, func(raw json.RawMessage) bool { return bytes.Equal(raw, authority) }) { + t.Fatal("small current authority was not retained exactly") + } + if shape != "user" && !slices.ContainsFunc(restored, func(raw json.RawMessage) bool { return bytes.Equal(raw, request) }) { + t.Fatal("small current task was not retained exactly") + } + if shape == "live-exec" && !bytes.Contains(mustMarshalJSON(restored), []byte("1234")) { + t.Fatal("live session handle was lost") + } + if shape == "failed-exec" && !bytes.Contains(mustMarshalJSON(restored), []byte("transaction remains unresolved")) { + t.Fatal("failure conclusion was lost") + } + if !bytes.Equal(original, mustMarshalJSON(input)) { + t.Fatal("pressure selection mutated original history") + } + t.Logf("visible-string tokens: %d -> %d", before, after) + }) + } + } +} + +func TestCompactionPressureCarriedTruncationAndRepeatedCompaction(t *testing.T) { + user := compactionPressureMessage("user", "user-prefix "+strings.Repeat("large request ", 140_000)+" user-suffix") + agent := mustMarshalJSON(map[string]any{"type": "agent_message", "content": "agent-prefix " + strings.Repeat("large report ", 50_000) + " agent-suffix"}) + input := []json.RawMessage{user, agent} + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.sealSnapshot(t.Context(), snapshot) + if err != nil { + t.Fatal(err) + } + truncated := compactionPressureMessage("user", "user-prefix …200000 tokens truncated… user-suffix") + fresh := compactionPressureMessage("developer", "Fresh restriction: do not deploy.") + for iteration := range 3 { + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": []json.RawMessage{truncated, fresh, agent, capsule, compactionPressureMessage("assistant", strings.Repeat("more unique history ", 90_000))}})) + if err != nil { + t.Fatal(err) + } + capsule, err = compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatalf("iteration %d: %v", iteration, err) + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{truncated, agent, capsule}) + if err != nil { + t.Fatalf("iteration %d continuation: %v", iteration, err) + } + count, ok := compactionVisibleStringTokens(restored...) + if !ok || count > compactionTargetTokens+compactionOvershootTokens { + t.Fatalf("iteration %d resurrected history: %d", iteration, count) + } + if !bytes.Contains(mustMarshalJSON(restored), []byte("Fresh restriction: do not deploy.")) { + t.Fatal("fresh authority disappeared") + } + } +} + +func TestCompactionPressureManySmallMessages(t *testing.T) { + var input []json.RawMessage + for index := range 5000 { + input = append(input, compactionPressureMessage("user", fmt.Sprintf("Historical instruction %d %s", index, strings.Repeat("detail ", 60)))) + } + input = append(input, compactionPressureMessage("user", "Current request: finish the router change.")) + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Items) >= len(input) || len(snapshot.Carried) == 0 { + t.Fatal("pressure did not drop old items") + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.sealSnapshot(t.Context(), snapshot) + if err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(t.Context(), append(slices.Clone(input), capsule)) + if err != nil || !bytes.Equal(mustMarshalJSON(restored), mustMarshalJSON(snapshot.Items)) { + t.Fatalf("dropped carried messages were not reconciled: %v", err) + } +} + +func TestCompactionPressureCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, _, err := pressureCompactionWorkingSet(ctx, []json.RawMessage{compactionPressureMessage("user", "request")}, compactionTargetTokens); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled pressure compaction: %v", err) + } +} + +func TestCompactionPressureRequiredInstructions(t *testing.T) { + for _, role := range []string{"system", "developer", "agents"} { + t.Run(role, func(t *testing.T) { + instruction := func(size int) json.RawMessage { + text := strings.Repeat("required ", size) + if role == "agents" { + return compactionPressureMessage("user", "# AGENTS.md instructions\n\n"+text+"\n") + } + return compactionPressureMessage(role, text) + } + required := instruction(60_000) + input := []json.RawMessage{required, compactionPressureMessage("user", "Current task "+strings.Repeat("history ", 252_000))} + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(snapshot.Items, func(raw json.RawMessage) bool { return bytes.Equal(raw, required) }) { + t.Fatal("required instructions were not retained byte-exact") + } + if count, ok := compactionVisibleStringTokens(snapshot.Items...); !ok || count > compactionTargetTokens+compactionOvershootTokens { + t.Fatalf("instruction floor escaped ceiling: %d", count) + } + input[0] = instruction(90_000) + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": input})) + if err != nil { + t.Fatal(err) + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + if _, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true); err == nil || !strings.Contains(err.Error(), "required instructions were not truncated") { + t.Fatalf("oversized required instructions: %v", err) + } + if compactor.aead != nil { + t.Fatal("inadmissible required instructions reached sealing") + } + }) + } +} + +func TestCompactionPressureDroppedReceiptOrder(t *testing.T) { + a := compactionPressureMessage("user", "Old request A") + b := compactionPressureMessage("user", "Old request B") + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + first, err := compactor.sealSnapshot(t.Context(), compactionSnapshot{ + Items: []json.RawMessage{b}, + Carried: []compactionCarriedItem{{Originals: []json.RawMessage{a}, Index: 0, Removed: true}}, + }) + if err != nil { + t.Fatal(err) + } + input := []json.RawMessage{first} + for index := range 1000 { + input = append(input, compactionPressureMessage("user", fmt.Sprintf("New request %d %s", index, strings.Repeat("detail ", 200)))) + } + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": input})) + if err != nil { + t.Fatal(err) + } + second, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{a, b, second}) + if err != nil || !bytes.Equal(mustMarshalJSON(restored), parsed.fields["input"]) { + t.Fatalf("consecutive dropped receipts reordered or resurrected history: %v", err) + } +} + +func TestCompactionPressureRejectsUnreconciledOlderCapsule(t *testing.T) { + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + first, err := compactor.seal(t.Context(), []json.RawMessage{compactionPressureMessage("assistant", strings.Repeat("previous reasoning ", 20_000))}) + if err != nil { + t.Fatal(err) + } + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": []json.RawMessage{first, compactionPressureMessage("user", strings.Repeat("current task ", 140_000))}})) + if err != nil { + t.Fatal(err) + } + second, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + if restored, err := compactor.restore(t.Context(), []json.RawMessage{first, second}); err == nil || len(restored) != 0 || !strings.Contains(err.Error(), "overlapping compaction envelopes") { + t.Fatalf("old capsule resurrected discarded history: %v", err) + } +} + +func TestCompactionEvidencePlanPreservesDroppedAnchor(t *testing.T) { + a := compactionPressureMessage("user", "Dropped historical user request") + b := mustMarshalJSON(map[string]any{"type": "message", "role": "assistant", "id": "assistant_transport_old", "content": "A historical decision."}) + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + first, err := compactor.sealSnapshot(t.Context(), compactionSnapshot{ + Items: []json.RawMessage{b}, + Carried: []compactionCarriedItem{{Originals: []json.RawMessage{a}, Index: 0, Removed: true}}, + }) + if err != nil { + t.Fatal(err) + } + input := []json.RawMessage{first} + for index := range 9 { + id := fmt.Sprintf("pwd_%d", index) + input = append(input, compactTestCall(id, "pwd"), compactTestOutput(id, "/workspace\n", 0)) + } + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "gpt-5", "input": input})) + if err != nil { + t.Fatal(err) + } + second, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + fresh := compactionPressureMessage("developer", "Fresh restriction before the historical decision.") + restored, err := compactor.restore(t.Context(), []json.RawMessage{fresh, a, second}) + if err != nil || len(restored) < 2 || !bytes.Equal(restored[0], fresh) || !bytes.Contains(restored[1], []byte("A historical decision.")) { + t.Fatalf("metadata cleanup displaced the fresh instruction anchor: %v", err) + } +} + +func TestCompactionPressureMultipartClassification(t *testing.T) { + var dense strings.Builder + for index := range 20_000 { + fmt.Fprintf(&dense, "Distinct message detail %d has value %x.\n", index, index*7919) + } + for _, kinds := range [][]string{{"user.text", "user.text"}, {"user.text", "unknown"}} { + t.Run(strings.Join(kinds, "+"), func(t *testing.T) { + input := []json.RawMessage{mustMarshalJSON(map[string]any{ + "type": "message", "role": "user", + "content": []any{map[string]string{"type": "input_text", "text": "first\n" + dense.String()}, map[string]string{"type": "input_text", "text": "second\n" + dense.String()}}, + "internal_chat_message_metadata_passthrough": map[string]any{"content_item_kinds": kinds, "trace": "preserved"}, + })} + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + selected := snapshot.Items[len(snapshot.Items)-1] + var message, metadata map[string]json.RawMessage + _ = json.Unmarshal(selected, &message) + _ = json.Unmarshal(message["internal_chat_message_metadata_passthrough"], &metadata) + var selectedKinds []string + _ = json.Unmarshal(metadata["content_item_kinds"], &selectedKinds) + wantKind := "user.text" + if kinds[1] == "unknown" { + wantKind = "unknown" + } + if len(selectedKinds) != 1 || selectedKinds[0] != wantKind || jsonString(metadata, "trace") != "preserved" { + t.Fatal("excerpt classifications no longer describe its content") + } + text := contextCompactionNarrationTexts(message["content"])[0] + message["content"] = mustMarshalJSON([]any{map[string]string{"type": "input_text", "text": text[:80] + "…40000 tokens truncated…" + text[len(text)-80:]}}) + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.sealSnapshot(t.Context(), snapshot) + if err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{mustMarshalJSON(message), capsule}) + if err != nil || !bytes.Equal(mustMarshalJSON(restored), mustMarshalJSON(snapshot.Items)) { + t.Fatalf("client-truncated multipart excerpt did not reconcile: %v", err) + } + }) + } +} + +func TestCompactionPressureRepetitionPreservesContinuationEvidence(t *testing.T) { + middle := "Correction: use the staging database, not production." + request := compactionPressureMessage("user", "Run the tests and report the unresolved failures.\n"+ + strings.Repeat("ordinary repeated payload ", 60_000)+"\n"+middle+"\n"+ + strings.Repeat("different repeated payload ", 60_000)+"\nNever deploy.") + var testOutput strings.Builder + for index := range 100 { + fmt.Fprintf(&testOutput, "case %d: checked distinct assertion %d\n", index, index*13) + } + testOutput.WriteString("FAIL: migration_schema remains unresolved\nFAIL: migration_lock remains unresolved\n") + input := []json.RawMessage{ + compactionPressureMessage("developer", "Only change the router."), + request, + compactTestCall("tests", "unfamiliar-test-runner"), + compactTestOutput("tests", testOutput.String(), 1), + compactTestCall("live", "unfamiliar-worker"), + mustMarshalJSON(map[string]any{"type": "function_call_output", "call_id": "live", + "output": string(mustMarshalJSON(map[string]any{"output": "Still running; next poll owns the result.", "session_id": 4567, "exit_code": nil}))}), + } + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + if snapshot.Report == nil || len(snapshot.Report.Items) != len(input) || + snapshot.Report.Items[0].Disposition != "exact" || snapshot.Report.Items[1].Disposition != "repetition_reduced" || + snapshot.Report.Items[3].Disposition != "historical" || snapshot.Report.Items[3].Excerpt { + t.Fatalf("incorrect loss report: %+v", snapshot.Report) + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + capsule, err := compactor.sealSnapshot(t.Context(), snapshot) + if err != nil { + t.Fatal(err) + } + opened, local, err := compactor.openSnapshot(t.Context(), capsule) + if err != nil || !local || !bytes.Equal(mustMarshalJSON(opened.Report), mustMarshalJSON(snapshot.Report)) { + t.Fatalf("loss report did not survive encrypted envelope: %v", err) + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{capsule}) + if err != nil || !bytes.Equal(mustMarshalJSON(restored), mustMarshalJSON(snapshot.Items)) { + t.Fatalf("diagnostics changed model input: %v", err) + } + t.Logf("pressure selection report: %s", mustMarshalJSON(snapshot.Report)) + count, _ := compactionVisibleStringTokens(snapshot.Items...) + if count != snapshot.Report.After { + t.Fatalf("loss report count %d differs from replay %d", snapshot.Report.After, count) + } + if count > 8000 { + t.Fatalf("repetition crowded out evidence: %d tokens", count) + } + var texts []string + for _, raw := range snapshot.Items { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + texts = append(texts, contextCompactionNarrationTexts(fields["content"])...) + } + visible := strings.Join(texts, "\n") + for _, fact := range []string{middle, "Never deploy.", testOutput.String(), "4567", "Still running; next poll owns the result."} { + if !strings.Contains(visible, fact) { + t.Fatalf("lost continuation fact %q", fact) + } + } + if !bytes.Equal(snapshot.Items[1], input[0]) || !strings.Contains(visible, "[mekugi repetition:") || + strings.Count(visible, "ordinary repeated payload") > 4 || strings.Count(visible, "different repeated payload") > 4 { + t.Fatal("required instructions changed or repetitive bulk survived") + } +} + +func TestCompactionPressureDenseFairAllocation(t *testing.T) { + var dense strings.Builder + for index := range 25_000 { + fmt.Fprintf(&dense, "observation %d has distinct value %x\n", index, index*7919) + } + current := compactionPressureMessage("user", "Current task: reconcile these observations.\n"+dense.String()+ + "\nCorrection: preserve the public API.\n"+dense.String()+"\nNext step: investigate the two failures.") + input := []json.RawMessage{current} + for index := range 12 { + input = append(input, compactionPressureMessage("assistant", fmt.Sprintf("Decision %d remains unresolved.\n", index)+dense.String())) + } + var evidence strings.Builder + for index := range 100 { + fmt.Fprintf(&evidence, "assertion %d checked path %d\n", index, index*17) + } + evidence.WriteString("FAIL: parser_boundary\nFAIL: lock_timeout\nsession_id=9876 remains live\n") + input = append(input, compactTestCall("check", "custom-check"), compactTestOutput("check", evidence.String(), 1)) + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + var visible strings.Builder + for _, raw := range snapshot.Items { + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + for _, text := range contextCompactionNarrationTexts(fields["content"]) { + visible.WriteString(text) + } + } + for _, fact := range []string{"Correction: preserve the public API.", "Next step: investigate the two failures.", evidence.String()} { + if !strings.Contains(visible.String(), fact) { + t.Fatalf("large latest request displaced useful evidence: missing %q", fact) + } + } + for index := range 12 { + if !strings.Contains(visible.String(), fmt.Sprintf("Decision %d remains unresolved.", index)) { + t.Fatalf("decision %d received no coverage", index) + } + } + count, _ := compactionVisibleStringTokens(snapshot.Items...) + if count > compactionTargetTokens { + t.Fatalf("fair allocation escaped target: %d", count) + } +} + +func TestCompactionPressureRepetitions(t *testing.T) { + for _, unit := range []string{"x ", "ordinary repeated phrase\n", "前提条件を保つ。", "failure: alpha\nwarning: beta\n"} { + t.Run(unit, func(t *testing.T) { + input := "Before\n" + strings.Repeat(unit, 2000) + "\nCorrection: keep this unique middle fact.\n" + + strings.Repeat(unit, 2000) + "\nAfter" + output, err := compactionPressureRepetitions(t.Context(), input) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(output, "Before\n") || !strings.HasSuffix(output, "\nAfter") || + !strings.Contains(output, "Correction: keep this unique middle fact.") || + len(output) > len(input)/4 || !utf8.ValidString(output) { + t.Fatal("repetition reduction lost distinct text or retained bulk") + } + }) + } + var distinct strings.Builder + for index := range 2000 { + fmt.Fprintf(&distinct, "failure: alpha_%d\nwarning: beta_%d\n", index, index) + } + output, err := compactionPressureRepetitions(t.Context(), distinct.String()) + if err != nil || output != distinct.String() { + t.Fatal("near-duplicate diagnostics were treated as repetitions") + } +} + +func TestCompactionPressureRepetitionPreservesParts(t *testing.T) { + for _, kind := range []string{"message", "agent_message"} { + t.Run(kind, func(t *testing.T) { + imagePart := mustMarshalJSON(map[string]string{"type": "input_image", "image_url": "data:image/png;base64,cHJvYmU="}) + unknownPart := mustMarshalJSON(map[string]string{"type": "future_part", "payload": "distinct evidence"}) + metadata := mustMarshalJSON(map[string]any{"content_item_kinds": []string{"user.text", "user.image", "unknown"}, "trace": "unchanged"}) + placeholder := mustMarshalJSON(map[string]string{"type": "input_text", "text": "[Image]"}) + selectedMetadata := mustMarshalJSON(map[string]any{"content_item_kinds": []string{"user.text", "unknown", "unknown"}, "trace": "unchanged"}) + input := []json.RawMessage{mustMarshalJSON(map[string]any{ + "type": kind, "role": "user", "id": "preserved-message-id", + "content": []json.RawMessage{mustMarshalJSON(map[string]string{"type": "input_text", "text": strings.Repeat("repeated text ", 140_000), "annotation": "unchanged"}), imagePart, unknownPart}, + "internal_chat_message_metadata_passthrough": metadata, + })} + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + var message map[string]json.RawMessage + _ = json.Unmarshal(snapshot.Items[1], &message) + var parts []json.RawMessage + _ = json.Unmarshal(message["content"], &parts) + if len(parts) != 3 || !bytes.Equal(parts[1], placeholder) || !bytes.Equal(parts[2], unknownPart) || + !bytes.Equal(message["internal_chat_message_metadata_passthrough"], selectedMetadata) || + jsonString(message, "id") != "preserved-message-id" { + t.Fatal("repetition reduction lost distinct parts or identity metadata") + } + var first map[string]json.RawMessage + _ = json.Unmarshal(parts[0], &first) + if jsonString(first, "annotation") != "unchanged" || !snapshot.Report.Items[0].Repetition || snapshot.Report.Items[0].Excerpt { + t.Fatal("repetition-only metadata or diagnostics are incorrect") + } + }) + } +} + +func TestCompactionPressureRepetitionRequiresTokenSavings(t *testing.T) { + input := strings.Repeat(" ", 1024) + output, err := compactionPressureRepetitions(t.Context(), input) + if err != nil || output != input { + t.Fatalf("already token-efficient whitespace was changed: %v", err) + } +} + +func TestCompactionPressureRequestChainSurvivesToolFlood(t *testing.T) { + request := compactionPressureMessage("user", "Add a stash-message popup for the selected file. Open the external editor at the selected changed line, not at the file beginning.") + decision := compactionPressureMessage("assistant", "Agreed plan: use a status-aware sidebar and Unstaged/Staged tabs. Keep Git-changing actions blocked until status and diff refresh finish.") + correction := compactionPressureMessage("user", "Study the reference implementation first. Only local commits are approved.") + input := []json.RawMessage{compactionPressureMessage("developer", "Preserve the current public API."), request, decision, correction} + var output strings.Builder + for index := range 250 { + fmt.Fprintf(&output, "result row %d has unique value %x\n", index, index*7919) + } + for index := range 180 { + id := fmt.Sprintf("unknown-%d", index) + input = append(input, + mustMarshalJSON(map[string]any{"type": "custom_tool_call", "name": "unfamiliar", "call_id": id, "input": "inspect the current code"}), + mustMarshalJSON(map[string]any{"type": "custom_tool_call_output", "call_id": id, "output": output.String()})) + } + ack := compactionPressureMessage("user", "yea") + input = append(input, ack) + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + for _, required := range []json.RawMessage{request, decision, correction, ack} { + if !slices.ContainsFunc(snapshot.Items, func(raw json.RawMessage) bool { return bytes.Equal(raw, required) }) { + t.Fatal("tool history displaced a small request, correction, decision, or acknowledgement") + } + } + if count, ok := compactionVisibleStringTokens(snapshot.Items...); !ok || count > compactionTargetTokens { + t.Fatalf("request-chain coverage escaped budget: %d", count) + } + if snapshot.Report.Items[1].Family != "requests" || snapshot.Report.Items[2].Family != "discussion" || + snapshot.Report.Items[4].Family != "execution" { + t.Fatal("loss report does not distinguish task context from execution") + } +} + +func TestCompactionNativeImageMetricAndPlaceholders(t *testing.T) { + image := func(url string) json.RawMessage { + return mustMarshalJSON(map[string]string{"type": "input_image", "image_url": url}) + } + message := func(role, text, url string) json.RawMessage { + return mustMarshalJSON(map[string]any{"type": "message", "role": role, + "content": []json.RawMessage{mustMarshalJSON(map[string]string{"type": "input_text", "text": text}), image(url)}}) + } + short := message("user", "Interpret the screenshot.", "data:image/png;base64,YQ==") + long := message("user", "Interpret the screenshot.", "data:image/png;base64,"+strings.Repeat("YQ==", 10000)) + a, _ := compactionVisibleStringTokens(short) + b, _ := compactionVisibleStringTokens(long) + if a != b { + t.Fatal("image transport bytes were counted as text tokens") + } + plain, _ := compactionVisibleStringTokens(compactionPressureMessage("user", strings.Repeat("YQ==", 10000))) + if plain <= b { + t.Fatal("ordinary visible text bypassed the text budget") + } + // A lookalike image object in arbitrary metadata is not a native image part. + lookalike := mustMarshalJSON(map[string]any{"type": "message", "role": "user", "content": "request", + "unknown_metadata": map[string]any{"type": "message", "content": []json.RawMessage{image("data:image/png;base64," + strings.Repeat("YQ==", 10000))}}}) + lookalikeCost, _ := compactionVisibleStringTokens(lookalike) + if lookalikeCost <= b { + t.Fatal("unknown metadata was mistaken for native image transport") + } + + var input []json.RawMessage + for index := range 6 { + input = append(input, message("user", fmt.Sprintf("Requirement %d remains in force.", index), fmt.Sprintf("https://example.invalid/image-%d.png", index))) + } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "loopback", "input": input})) + if err != nil { + t.Fatal(err) + } + capsule, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(t.Context(), append(slices.Clone(input), capsule)) + if err != nil { + t.Fatal(err) + } + if count, _ := compactionImageUsage(restored); count != 0 { + t.Fatalf("historical images were restored: %d", count) + } + visible := string(mustMarshalJSON(restored)) + for index := range 6 { + if !strings.Contains(visible, fmt.Sprintf("Requirement %d remains in force.", index)) { + t.Fatalf("image pruning removed request %d", index) + } + } + if strings.Count(visible, "[Image]") != 6 || strings.Contains(visible, "example.invalid/image-") { + t.Fatal("historical images were not replaced with one marker each") + } +} + +func TestCompactionRequiredImagesRemainExact(t *testing.T) { + for _, role := range []string{"developer", "system", "user"} { + t.Run(role, func(t *testing.T) { + var parts []map[string]string + var kinds []string + for range 5 { + parts = append(parts, map[string]string{"type": "input_image", "image_url": "required-image"}) + kinds = append(kinds, "context.instructions") + } + if role == "developer" { + parts[0]["image_url"] = strings.Repeat("x", (8<<20)+1) + } + required := mustMarshalJSON(map[string]any{"type": "message", "role": role, "content": parts, + "internal_chat_message_metadata_passthrough": map[string]any{"content_item_kinds": kinds}}) + ordinary := mustMarshalJSON(map[string]any{"type": "message", "role": "user", + "content": []any{map[string]string{"type": "input_image", "image_url": "ordinary-image"}}}) + input := []json.RawMessage{required, ordinary} + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "loopback", "input": input})) + if err != nil { + t.Fatal(err) + } + capsule, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + restored, err := compactor.restore(t.Context(), []json.RawMessage{capsule}) + if err != nil || len(restored) != 2 || !bytes.Equal(restored[0], required) { + t.Fatalf("mandatory instruction images changed: %v", err) + } + if count, _ := compactionImageUsage(restored); count != 5 { + t.Fatal("required images were limited or the ordinary image survived") + } + }) + } +} + +func TestCompactionPressureImagesBecomePlaceholdersInTextExcerpts(t *testing.T) { + var dense strings.Builder + for index := range 25000 { + fmt.Fprintf(&dense, "distinct observation %d has value %x\n", index, index*7919) + } + image := mustMarshalJSON(map[string]string{"type": "input_image", "image_url": "data:image/png;base64,cHJvYmU=", "detail": "original"}) + for _, kind := range []string{"message", "custom_tool_call_output", "function_call_output"} { + t.Run(kind, func(t *testing.T) { + key := "output" + if kind == "message" { + key = "content" + } + input := []json.RawMessage{mustMarshalJSON(map[string]any{"type": kind, "role": "user", "call_id": "picture", + key: []json.RawMessage{mustMarshalJSON(map[string]string{"type": "input_text", "text": dense.String()}), image}})} + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + if count, _ := compactionImageUsage(snapshot.Items); count != 0 { + t.Fatal("text pressure retained a historical image") + } + var selected map[string]json.RawMessage + _ = json.Unmarshal(snapshot.Items[1], &selected) + if !strings.Contains(strings.Join(contextCompactionNarrationTexts(selected["content"]), "\n"), "[Image]") { + t.Fatal("image placeholder was lost from the retained excerpt") + } + if kind != "message" && jsonString(selected, "role") != "assistant" { + t.Fatal("historical tool observation changed attribution") + } + if strings.Contains(strings.Join(contextCompactionNarrationTexts(selected["content"]), "\n"), "cHJvYmU=") { + t.Fatal("historical wrapping leaked base64 into visible text") + } + if snapshot.Report.ImagesBefore != 1 || snapshot.Report.ImagesAfter != 0 || !snapshot.Report.Items[0].Excerpt { + t.Fatal("image/text diagnostics are incorrect") + } + }) + } +} + +func TestCompactionPressureAgentExcerptPreservesAttribution(t *testing.T) { + var body strings.Builder + body.WriteString("Review of the storage boundary.\n") + for index := range 5000 { + fmt.Fprintf(&body, "Finding %d has observed offset %d and checksum %x.\n", index, index*7, index*7919) + } + body.WriteString("Unresolved: cancellation still leaves the archive process running.") + original := map[string]json.RawMessage{ + "type": mustMarshalJSON("agent_message"), "id": mustMarshalJSON("report-storage"), + "author": mustMarshalJSON("storage-reviewer"), "recipient": mustMarshalJSON("main"), + "metadata": mustMarshalJSON(map[string]string{"source": "independent-inspection"}), + "content": mustMarshalJSON([]any{map[string]string{"type": "input_text", "text": body.String()}}), + } + input := []json.RawMessage{mustMarshalJSON(original)} + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, 2000) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Items) != 2 || !snapshot.Report.Items[0].Excerpt { + t.Fatal("oversized agent report did not produce a retained excerpt") + } + var retained map[string]json.RawMessage + _ = json.Unmarshal(snapshot.Items[1], &retained) + for key, value := range original { + if key != "content" && !bytes.Equal(value, retained[key]) { + t.Errorf("agent envelope field %q changed", key) + } + } + var parts []map[string]json.RawMessage + _ = json.Unmarshal(retained["content"], &parts) + if len(parts) != 1 || jsonString(parts[0], "type") != "input_text" || + !strings.Contains(jsonString(parts[0], "text"), "Unresolved: cancellation still leaves the archive process running.") { + t.Fatal("agent excerpt lost its native content type or unresolved conclusion") + } +} + +func TestCompactionPressureLatestReportSurvivesDiscussionFlood(t *testing.T) { + var older, current strings.Builder + for index := range 500 { + fmt.Fprintf(&older, "Historical investigation %d observed revision %x.\n", index, index*7919) + } + for index := range 80 { + fmt.Fprintf(¤t, "Current checkpoint %d retains artifact revision %x.\n", index, index*97) + } + current.WriteString("Do not rerun either agent or perform inference. No regrade has started. Cancellation remains unresolved.") + for _, kind := range []string{"message", "agent_message"} { + t.Run(kind, func(t *testing.T) { + input := []json.RawMessage{compactionPressureMessage("user", "Regrade the saved candidates, preserving their measured timing.")} + for range 200 { + input = append(input, compactionPressureMessage("assistant", older.String())) + } + report := mustMarshalJSON(map[string]any{"type": kind, "role": "assistant", "author": "reviewer", "content": current.String()}) + input = append(input, report, + mustMarshalJSON(map[string]any{"type": "agent_message", "author": "opaque-reviewer", "recipient": "main", + "content": []any{map[string]string{"type": "encrypted_content", "encrypted_content": "opaque-report"}}})) + snapshot, _, err := pressureCompactionWorkingSet(t.Context(), input, compactionTargetTokens) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(snapshot.Items, func(raw json.RawMessage) bool { return bytes.Equal(raw, report) }) { + t.Fatal("historical discussion displaced the bounded current continuation report") + } + if count, ok := compactionVisibleStringTokens(snapshot.Items...); !ok || count > compactionTargetTokens { + t.Fatal("latest-report reservation escaped the text budget") + } + }) + } +} diff --git a/internal/router/context_compaction_repeated_test.go b/internal/router/context_compaction_repeated_test.go index a5f0aa8d..31796fed 100644 --- a/internal/router/context_compaction_repeated_test.go +++ b/internal/router/context_compaction_repeated_test.go @@ -209,17 +209,42 @@ func TestCompactionRolloutReplay(t *testing.T) { if err := scanner.Err(); err != nil { t.Fatal(err) } - reduced := reduceContextCompaction(input) - changed := len(input) - len(reduced) - if changed == 0 { - for index := range input { - if string(input[index]) != string(reduced[index]) { - changed++ - } + compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} + parsed, err := parseResponsesRequest(mustMarshalJSON(map[string]any{"model": "loopback", "input": input})) + if err != nil { + t.Fatal(err) + } + preparedCapsule, err := compactor.prepare(t.Context(), &parsed, http.Header{}, true) + if err != nil { + t.Fatal(err) + } + selected, local, err := compactor.openSnapshot(t.Context(), preparedCapsule) + if err != nil || !local { + t.Fatalf("cannot inspect the selected replay: %v", err) + } + reduced, pressure := selected.Items, selected.Report != nil + expected, _, err := stripCompactionImages(t.Context(), input) + if err != nil { + t.Fatal(err) + } + exactRetained := make(map[string]int, len(reduced)) + for _, raw := range reduced { + exactRetained[string(raw)]++ + } + changed := 0 + for _, raw := range input { + if exactRetained[string(raw)] > 0 { + exactRetained[string(raw)]-- + } else { + changed++ } } + retainedCursor := 0 for index := range input { + if pressure && !contextCompactionFreshContext(input[index]) { + continue + } var fields map[string]json.RawMessage _ = json.Unmarshal(input[index], &fields) kind := jsonString(fields, "type") @@ -233,7 +258,7 @@ func TestCompactionRolloutReplay(t *testing.T) { for retainedCursor < len(reduced) { candidate := reduced[retainedCursor] retainedCursor++ - if string(input[index]) == string(candidate) || compactionReplayAllowsOnlyMetadataCleanup(input[index], candidate) { + if string(expected[index]) == string(candidate) || compactionReplayAllowsOnlyMetadataCleanup(expected[index], candidate) { found = true break } @@ -245,7 +270,6 @@ func TestCompactionRolloutReplay(t *testing.T) { if changed == 0 { t.Fatal("rollout has no supported reduction") } - compactor := &contextCompactor{keyPath: filepath.Join(t.TempDir(), "compaction.key")} response := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(string(mustMarshalJSON(map[string]any{ "model": "loopback", "input": input, "stream": true, @@ -316,6 +340,51 @@ func TestCompactionRolloutReplay(t *testing.T) { if err != nil || contextCompactionCanonicalJSON(mustMarshalJSON(legacyRestored)) != contextCompactionCanonicalJSON(mustMarshalJSON(reduced)) { t.Fatalf("legacy replay duplicated or lost selected context: %v", err) } + // Opt-in local artifacts allow loss inspection against actual source items. + // Never put private transcript content in test output or modify the rollout. + if directory := os.Getenv("MEKUGI_COMPACTION_AUDIT_DIR"); directory != "" { + info, err := os.Stat(directory) + if err != nil || !info.IsDir() || !filepath.IsAbs(directory) || info.Mode().Perm()&0077 != 0 { + t.Fatal("MEKUGI_COMPACTION_AUDIT_DIR must be an existing absolute owner-only directory") + } + sealed, local, err := compactor.openSnapshot(t.Context(), capsule) + if err != nil || !local { + t.Fatal("cannot inspect selected local snapshot") + } + before, _ := compactionVisibleStringTokens(input...) + after, _ := compactionVisibleStringTokens(restored...) + requiredItems, requiredTokens := 0, 0 + for _, raw := range input { + if contextCompactionFreshContext(raw) { + requiredItems++ + count, _ := compactionVisibleStringTokens(raw) + requiredTokens += count + } + } + summary := map[string]any{ + "source": path, "boundary": "recorded response items before first compaction, or EOF", + "before_tokens": before, "after_tokens": after, "before_items": len(input), "after_items": len(restored), + "changed_original_items": changed, "required_items": requiredItems, "required_tokens": requiredTokens, + "pressure": pressure, "pressure_report": sealed.Report, + } + for name, content := range map[string][]byte{ + "before.json": mustMarshalJSON(input), + "after.json": mustMarshalJSON(restored), + "summary.json": mustMarshalJSON(summary), + } { + file, err := os.OpenFile(filepath.Join(directory, name), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + t.Fatal("cannot create private compaction audit artifact") + } + _, writeErr := file.Write(content) + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + t.Fatal("cannot write private compaction audit artifact") + } + } + t.Logf("real rollout replay: tokens=%d -> %d; items=%d -> %d; required=%d items/%d tokens; pressure=%v", + before, after, len(input), len(restored), requiredItems, requiredTokens, pressure) + } // Report only aggregate structural reasons, never private transcript text. calls := make(map[string]map[string]json.RawMessage) originalResults := make(map[string]json.RawMessage) @@ -414,6 +483,9 @@ func logCompactionTokenProfile(t *testing.T, before, after []json.RawMessage) in } case map[string]any: for key, part := range value { + if key == "image_url" && value["type"] == "input_image" { + continue + } if key == "encrypted_content" { if text, ok := part.(string); ok { m.opaqueBytes += len(text) diff --git a/internal/router/context_compaction_repetition.go b/internal/router/context_compaction_repetition.go new file mode 100644 index 00000000..24d681c0 --- /dev/null +++ b/internal/router/context_compaction_repetition.go @@ -0,0 +1,95 @@ +package router + +import ( + "context" + "fmt" + "slices" + "strings" + "unicode/utf8" +) + +// Collapse only adjacent, byte-identical runs of short token sequences. The +// bounded period search is independent of roles, commands and output formats; +// near-duplicates and intervening corrections are never treated as repetitions. +// Keep the first and last occurrence and label the omitted multiplicity. This +// remains lossy: positional occurrences may matter even when their bytes match. +func compactionPressureRepetitions(ctx context.Context, text string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if len(text) < 256 { + return text, nil + } + if _, ok := compactionVisibleStringTokens(); !ok { + return "", fmt.Errorf("cannot initialize compaction tokenizer") + } + _, pieces, err := compactionRetirementTokenCodec.Encode(text) + if err != nil { + return "", err + } + var out strings.Builder + start := 0 + for index := 0; index < len(pieces); { + if index%1024 == 0 { + if err := ctx.Err(); err != nil { + return "", err + } + } + end := index + for period := 1; period <= 64 && index+period*8 <= len(pieces); period++ { + unit := pieces[index : index+period] + if !slices.Equal(unit, pieces[index+period:index+period*2]) { + continue + } + count := 2 + for index+(count+1)*period <= len(pieces) && + slices.Equal(unit, pieces[index+count*period:index+(count+1)*period]) { + count++ + if count%1024 == 0 { + if err := ctx.Err(); err != nil { + return "", err + } + } + } + if count < 8 { + continue + } + sample := strings.Join(unit, "") + if len(sample)*(count-2) < 256 || !utf8.ValidString(sample) { + continue + } + marker := fmt.Sprintf("\n[mekugi repetition: %d identical adjacent occurrences omitted]\n", count-2) + _, markerPieces, err := compactionRetirementTokenCodec.Encode(marker) + if err != nil { + return "", err + } + if period*(count-2) <= len(markerPieces)+4 { + continue + } + out.WriteString(strings.Join(pieces[start:index+period], "")) + out.WriteString(marker) + out.WriteString(sample) + end = index + count*period + start = end + break + } + if end > index { + index = end + } else { + index++ + } + } + if start == 0 { + return text, nil + } + out.WriteString(strings.Join(pieces[start:], "")) + result := out.String() + _, resultPieces, err := compactionRetirementTokenCodec.Encode(result) + if err != nil { + return "", err + } + if len(resultPieces) >= len(pieces) { + return text, nil + } + return result, nil +} diff --git a/internal/router/context_compaction_retirement.go b/internal/router/context_compaction_retirement.go index 9c08db3f..53e21ab8 100644 --- a/internal/router/context_compaction_retirement.go +++ b/internal/router/context_compaction_retirement.go @@ -1016,8 +1016,8 @@ func compactionVisibleStringTokens(items ...json.RawMessage) (int, bool) { if json.Unmarshal(raw, &value) != nil { return 0, false } - var visit func(any) bool - visit = func(current any) bool { + var visit func(any, bool, bool) bool + visit = func(current any, nativeItem, contentPart bool) bool { switch current := current.(type) { case string: count, err := compactionRetirementTokenCodec.Count(current) @@ -1027,20 +1027,28 @@ func compactionVisibleStringTokens(items ...json.RawMessage) (int, bool) { total += count case []any: for _, nested := range current { - if !visit(nested) { + if !visit(nested, nativeItem, contentPart) { return false } } case map[string]any: for key, nested := range current { - if key != "encrypted_content" && !visit(nested) { + // Native image URLs are transport, not visible prose. Vision + // retention is bounded separately by count and encoded bytes. + if key == "encrypted_content" || contentPart && key == "image_url" && current["type"] == "input_image" { + continue + } + kind, _ := current["type"].(string) + parts := nativeItem && (key == "content" && (kind == "message" || kind == "agent_message") || + key == "output" && (kind == "function_call_output" || kind == "custom_tool_call_output")) + if !visit(nested, false, parts) { return false } } } return true } - if !visit(value) { + if !visit(value, true, false) { return 0, false } } From d226f9dcadd9eb58f53949c71fcfe27cb2c7ae42 Mon Sep 17 00:00:00 2001 From: Yuzerion Date: Sat, 12 Sep 2026 14:59:03 +0800 Subject: [PATCH 13/13] Update internal/router/server_websocket.go Co-authored-by: macroscopeapp[bot] <170038800+macroscopeapp[bot]@users.noreply.github.com> --- internal/router/server_websocket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/router/server_websocket.go b/internal/router/server_websocket.go index b234b99e..95a196b3 100644 --- a/internal/router/server_websocket.go +++ b/internal/router/server_websocket.go @@ -856,7 +856,7 @@ func (w *webSocketOutput) message(payload []byte) error { // Local completion retains only the capsule, but does not admit a // provider successor. Its accepted steering remains pending until // that provider response actually starts. - e.history.parent = nil + e.history.parent = event.Response.ID for _, item := range e.history.input { s.retainedBytes -= len(item) }