From f6d021133fa202a438ae4fcdeac2f2565f43ba2e Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 13:51:15 -0400 Subject: [PATCH 1/2] fix(claude): group token accounting must not depend on line adjacency canonicalize_message_usage and sum_usage assumed a group_id's lines were contiguous. Multi-terminal writers can interleave a split message's lines with another message's (docs/agents/formats/claude-code/ known-issues.md), splitting a group into two contiguous runs and double-counting the message total. Separately, group_id was None whenever message.id was absent, turning every content-block line of an id-less assistant message into its own accounting unit. Both functions now group by group_id globally instead of by adjacency, and group_id falls back to the entry-level request_id for assistant entries when message.id is missing (Anthropic's request ID identifies one API request per assistant message; user entries never use this fallback). toolpath-claude 0.12.0 -> 0.12.1. --- CHANGELOG.md | 21 +++ Cargo.lock | 2 +- Cargo.toml | 2 +- crates/toolpath-claude/Cargo.toml | 2 +- crates/toolpath-claude/src/provider.rs | 216 +++++++++++++++++++---- docs/agents/formats/claude-code/usage.md | 19 +- site/_data/crates.json | 2 +- 7 files changed, 220 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ce2d196..26dee7bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ All notable changes to the Toolpath workspace are documented here. +## Claude: group token accounting must not depend on line adjacency — 2026-07-08 + +- **`toolpath-claude`** (0.12.1): the group-usage canonicalization added in + 0.12.0 assumed a `group_id`'s lines were contiguous, and `group_id` itself + was `None` whenever `message.id` was absent. Two holes: + - Multi-terminal writers can interleave a split message's lines + non-contiguously (see + `docs/agents/formats/claude-code/known-issues.md`, "Multi-terminal + writes to the same project"); an interleaved group split into two + contiguous runs, each contributing a full message total — double + counting. + - An id-less assistant message (no `message.id`) made every content-block + line its own accounting unit, with the same usage summed once per line. + `canonicalize_message_usage` and `sum_usage` now group `group_id`s + **globally** rather than by adjacency, and `group_id` falls back to the + entry-level `request_id` for assistant entries when `message.id` is + absent (Anthropic's request ID is "useful for deduping streamed + messages" — one assistant message per request — per + `docs/agents/formats/claude-code/jsonl-envelope.md`). User entries never + fall back to `request_id`. + ## Derive: resolve duplicate step ids — 2026-07-01 - **`toolpath-convo`** (0.11.1): `derive_path` now guarantees the derived diff --git a/Cargo.lock b/Cargo.lock index a799d3c7..9320d43a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3893,7 +3893,7 @@ dependencies = [ [[package]] name = "toolpath-claude" -version = "0.12.0" +version = "0.12.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index cf3d6237..7065aeda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ license = "Apache-2.0" toolpath = { version = "0.7.0", path = "crates/toolpath" } toolpath-convo = { version = "0.11.1", path = "crates/toolpath-convo" } toolpath-git = { version = "0.6.0", path = "crates/toolpath-git" } -toolpath-claude = { version = "0.12.0", path = "crates/toolpath-claude", default-features = false } +toolpath-claude = { version = "0.12.1", path = "crates/toolpath-claude", default-features = false } toolpath-gemini = { version = "0.6.0", path = "crates/toolpath-gemini", default-features = false } toolpath-codex = { version = "0.6.0", path = "crates/toolpath-codex" } toolpath-opencode = { version = "0.5.0", path = "crates/toolpath-opencode" } diff --git a/crates/toolpath-claude/Cargo.toml b/crates/toolpath-claude/Cargo.toml index 5f463412..0ed4b496 100644 --- a/crates/toolpath-claude/Cargo.toml +++ b/crates/toolpath-claude/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toolpath-claude" -version = "0.12.0" +version = "0.12.1" edition.workspace = true license.workspace = true repository = "https://github.com/empathic/toolpath" diff --git a/crates/toolpath-claude/src/provider.rs b/crates/toolpath-claude/src/provider.rs index 1896d1e3..aaec1602 100644 --- a/crates/toolpath-claude/src/provider.rs +++ b/crates/toolpath-claude/src/provider.rs @@ -123,11 +123,23 @@ fn message_to_turn(entry: &ConversationEntry, msg: &Message) -> Turn { Turn { id: entry.uuid.clone(), parent_id: entry.parent_uuid.clone(), - // The API message ID (`msg_…`). Claude Code writes one JSONL line - // per content block, so several turns can share one group_id — - // and each repeats the message-level `usage`. Downstream accounting - // (sum_usage, derive_path) counts a message group once. - group_id: msg.id.clone(), + // Group key: the API message ID (`msg_…`). Claude Code writes one + // JSONL line per content block, so several turns can share one + // group_id — and each repeats the message-level `usage`. Downstream + // accounting (sum_usage, derive_path) counts a message group once. + // + // Some captures omit `message.id`. The entry-level `requestId` still + // identifies the API request for assistant entries — Anthropic's + // request ID is "useful for deduping streamed messages" (one + // assistant message per request; see + // docs/agents/formats/claude-code/jsonl-envelope.md) — so it's the + // natural fallback: split lines of an id-less message still dedupe + // their repeated usage. User entries never group. + group_id: msg.id.clone().or_else(|| { + (msg.role == MessageRole::Assistant) + .then(|| entry.request_id.clone()) + .flatten() + }), role: claude_role_to_role(&msg.role), timestamp: entry.timestamp.clone(), text, @@ -561,55 +573,68 @@ pub(crate) fn max_usage(a: &TokenUsage, b: &TokenUsage) -> TokenUsage { /// per-step attribution from them, and — the format being undocumented — we /// do not trust line order. /// -/// For each consecutive `group_id` run this sets `token_usage` on the run's -/// **final** turn to the field-wise **maximum** across the run (the message -/// total — never under-counts whatever the stream order) and clears it from -/// the others, so summing `token_usage` over turns yields session totals. +/// Groups by `group_id` **globally**, not by consecutive run — multi-terminal +/// writers can interleave a split message's lines non-contiguously (see +/// docs/agents/formats/claude-code/known-issues.md, "Multi-terminal writes to +/// the same project"). Treating an interleaved group as two contiguous runs +/// would count the message total once per fragment; this sets `token_usage` +/// on the group's **last occurrence** (by turn order) to the field-wise +/// **maximum** across *all* the group's turns and clears it from the rest, so +/// summing `token_usage` over turns yields session totals regardless of +/// interleaving. fn canonicalize_message_usage(turns: &mut [Turn]) { - let mut i = 0; - while i < turns.len() { - let Some(mid) = turns[i].group_id.clone() else { - i += 1; - continue; - }; - let mut j = i; - while j < turns.len() && turns[j].group_id.as_deref() == Some(mid.as_str()) { - j += 1; - } - - // Message total = field-wise max across the run (the final streaming - // snapshot, found without trusting line order). - let mut total: Option = None; - for t in &turns[i..j] { - if let Some(u) = &t.token_usage { - total = Some(match total { - Some(acc) => max_usage(&acc, u), - None => u.clone(), - }); - } + use std::collections::HashMap; + + // gid -> (field-wise max across ALL occurrences, index of last occurrence). + let mut groups: HashMap, usize)> = HashMap::new(); + for (i, t) in turns.iter().enumerate() { + let Some(gid) = &t.group_id else { continue }; + let entry = groups.entry(gid.clone()).or_insert((None, i)); + if let Some(u) = &t.token_usage { + entry.0 = Some(match &entry.0 { + Some(acc) => max_usage(acc, u), + None => u.clone(), + }); } + entry.1 = i; + } - for t in &mut turns[i..j] { + for t in turns.iter_mut() { + if t.group_id.is_some() { t.token_usage = None; } + } + for (total, last) in groups.into_values() { if let Some(total) = total { - turns[j - 1].token_usage = Some(total); + turns[last].token_usage = Some(total); } - i = j; } } /// Sum token usage across all turns. +/// +/// Adjacency-free: a turn's usage counts only when it has no `group_id`, or +/// when it's the **last** turn (by index) carrying its `group_id` — computed +/// by scanning for each gid's max index, not by checking the next turn. This +/// stays correct on non-canonicalized input (e.g. interleaved groups) as well +/// as on canonicalized input. fn sum_usage(turns: &[Turn]) -> Option { + use std::collections::HashMap; + + let mut last_occurrence: HashMap<&str, usize> = HashMap::new(); + for (idx, turn) in turns.iter().enumerate() { + if let Some(gid) = &turn.group_id { + last_occurrence.insert(gid.as_str(), idx); + } + } + let mut total = TokenUsage::default(); let mut any = false; for (idx, turn) in turns.iter().enumerate() { - // Turns split from one provider message all repeat that message's - // usage; count it once, on the run's last turn. - if let Some(mid) = &turn.group_id - && turns - .get(idx + 1) - .is_some_and(|next| next.group_id.as_ref() == Some(mid)) + // Turns sharing a group_id all repeat that message's usage; count it + // once, on the group's last occurrence. + if let Some(gid) = &turn.group_id + && last_occurrence.get(gid.as_str()) != Some(&idx) { continue; } @@ -925,6 +950,119 @@ mod tests { } } + #[test] + fn interleaved_group_ids_still_count_message_usage_once() { + // Two terminals interleaving writes to the same project (see + // docs/agents/formats/claude-code/known-issues.md, "Multi-terminal + // writes to the same project") can split one message's lines + // non-contiguously: A(gid=m1), B(gid=m2), C(gid=m1). Adjacency-based + // grouping treats A and C as two independent one-line "runs" and + // counts msg_A's usage twice. + let turns_before = vec![ + grp_turn("t1", "msg_A", 100), + grp_turn("t2", "msg_B", 50), + grp_turn("t3", "msg_A", 100), + ]; + + // sum_usage must be adjacency-free: correct even before + // canonicalization runs. + let total = sum_usage(&turns_before).unwrap(); + assert_eq!(total.output_tokens, Some(150), "X + Y, not 2X + Y"); + + let mut turns = turns_before; + canonicalize_message_usage(&mut turns); + + assert!( + turns[0].token_usage.is_none(), + "msg_A's first (non-last) fragment must not carry the total" + ); + assert_eq!( + turns[1].token_usage.as_ref().unwrap().output_tokens, + Some(50), + "msg_B's only line keeps its usage" + ); + assert_eq!( + turns[2].token_usage.as_ref().unwrap().output_tokens, + Some(100), + "msg_A's total lands on its LAST occurrence, not its first" + ); + + let total_after = sum_usage(&turns).unwrap(); + assert_eq!(total_after.output_tokens, Some(150), "still X + Y after canonicalization"); + } + + /// An id-less assistant message split across content-block lines: two + /// entries share `requestId` but `message.id` is absent from both. + fn setup_idless_message_provider() -> (TempDir, ClaudeConvo) { + let temp = TempDir::new().unwrap(); + let claude_dir = temp.path().join(".claude"); + let project_dir = claude_dir.join("projects/-test-project"); + fs::create_dir_all(&project_dir).unwrap(); + + let entries = [ + r#"{"uuid":"uuid-1","type":"user","timestamp":"2024-01-01T00:00:00Z","message":{"role":"user","content":"Fix the bug"}}"#.to_string(), + r#"{"uuid":"uuid-2","type":"assistant","parentUuid":"uuid-1","timestamp":"2024-01-01T00:00:01Z","requestId":"req_1","message":{"role":"assistant","content":[{"type":"text","text":"Working on it."}],"model":"claude-opus-4-7","stop_reason":null,"usage":{"input_tokens":5,"output_tokens":10}}}"#.to_string(), + r#"{"uuid":"uuid-3","type":"assistant","parentUuid":"uuid-2","timestamp":"2024-01-01T00:00:02Z","requestId":"req_1","message":{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"Read","input":{"file_path":"a.rs"}}],"model":"claude-opus-4-7","stop_reason":"tool_use","usage":{"input_tokens":5,"output_tokens":10}}}"#.to_string(), + ]; + fs::write(project_dir.join("session-3.jsonl"), entries.join("\n")).unwrap(); + + let resolver = PathResolver::new().with_claude_dir(&claude_dir); + (temp, ClaudeConvo::with_resolver(resolver)) + } + + #[test] + fn idless_assistant_message_groups_by_request_id() { + let (_temp, provider) = setup_idless_message_provider(); + let view = ConversationProvider::load_conversation(&provider, "/test/project", "session-3") + .unwrap(); + + assert_eq!(view.turns.len(), 3); + assert!(view.turns[0].group_id.is_none(), "user line carries no group id"); + assert!( + view.turns[1].group_id.is_some(), + "id-less assistant lines still get a group id (from requestId)" + ); + assert_eq!( + view.turns[1].group_id, view.turns[2].group_id, + "both lines of the split id-less message share one group id" + ); + + let total = view.total_usage.as_ref().unwrap(); + assert_eq!( + total.output_tokens, + Some(10), + "one message's usage counted once, not once per content-block line" + ); + } + + #[test] + fn user_entries_never_group_by_request_id() { + let temp = TempDir::new().unwrap(); + let claude_dir = temp.path().join(".claude"); + let project_dir = claude_dir.join("projects/-test-project"); + fs::create_dir_all(&project_dir).unwrap(); + + // A user entry carrying `requestId` (not something real Claude Code + // emits per docs/agents/formats/claude-code/jsonl-envelope.md, which + // scopes `requestId` to assistant entries — but the grouping logic + // must not rely on that being enforced upstream). + let entries = [ + r#"{"uuid":"uuid-1","type":"user","timestamp":"2024-01-01T00:00:00Z","requestId":"req_x","message":{"role":"user","content":"Fix the bug"}}"#, + ]; + fs::write(project_dir.join("session-4.jsonl"), entries.join("\n")).unwrap(); + + let resolver = PathResolver::new().with_claude_dir(&claude_dir); + let provider = ClaudeConvo::with_resolver(resolver); + let view = ConversationProvider::load_conversation(&provider, "/test/project", "session-4") + .unwrap(); + + assert_eq!(view.turns.len(), 1); + assert!( + view.turns[0].group_id.is_none(), + "user entries never group by request_id" + ); + } + fn setup_provider() -> (TempDir, ClaudeConvo) { let temp = TempDir::new().unwrap(); let claude_dir = temp.path().join(".claude"); diff --git a/docs/agents/formats/claude-code/usage.md b/docs/agents/formats/claude-code/usage.md index c799ff34..e950a94b 100644 --- a/docs/agents/formats/claude-code/usage.md +++ b/docs/agents/formats/claude-code/usage.md @@ -15,7 +15,24 @@ over-counts (~3× on typical sessions) — the values are **cumulative snapshots of one message, not per-line bills.** The grouping key is **`message.id`** (`msg_…`), identical on every line -of the split. Empirically, across every session sampled: +of the split. Some captures omit `message.id`; for assistant entries +`toolpath-claude` falls back to the entry-level `requestId` +([jsonl-envelope.md](jsonl-envelope.md#api-correlation)), which is +"useful for deduping streamed messages" per Anthropic — one assistant +message per API request — so id-less split lines still dedupe their +repeated usage under one group. User entries never fall back to +`requestId`, even if one happened to carry it. + +Grouping is **not** adjacency-based: multi-terminal writers can +interleave a split message's lines with another message's lines (see +[known-issues.md](known-issues.md), "Multi-terminal writes to the same +project"), so a group's lines aren't guaranteed to be contiguous. +`toolpath-claude` groups by `group_id` globally — across the whole +turn sequence, not just consecutive runs — before taking the max; +adjacency-based grouping would double-count an interleaved group's +usage once per contiguous fragment. + +Empirically, across every session sampled: - `input_tokens` and the cache counters are **constant** across a message's lines (prompt-side cost, paid once for the message). diff --git a/site/_data/crates.json b/site/_data/crates.json index 6b730be1..ec5476c5 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -33,7 +33,7 @@ }, { "name": "toolpath-claude", - "version": "0.12.0", + "version": "0.12.1", "description": "Derive from Claude conversation logs", "docs": "https://docs.rs/toolpath-claude", "crate": "https://crates.io/crates/toolpath-claude", From 4ec1459a2106f032fe6b214bcec6e832adb9b793 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 13:56:52 -0400 Subject: [PATCH 2/2] docs(claude): note request_id fallback in CLAUDE.md; correct sum_usage comment --- CLAUDE.md | 2 +- crates/toolpath-claude/src/provider.rs | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 01c70e9f..e360ce3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -243,7 +243,7 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - Provider-specific extras convention: `Turn.extra` and `WatcherEvent::Progress.data` use provider-namespaced keys (e.g. `extra["claude"]`, `extra["gemini"]`). `toolpath-claude` populates `Turn.extra["claude"]` from `ConversationEntry.extra`; `toolpath-gemini` populates `Turn.extra["gemini"]` with the full `tokens` struct, per-thought metadata, and tool-call status. This lets trait-only consumers access provider metadata without importing provider types. - Shared derivation: `toolpath-convo` provides a provider-agnostic `ConversationView → Path` mapping via `toolpath_convo::derive_path`. New conversation providers should build on it rather than re-implementing the mapping. - Path kinds: `toolpath::v1::PathMeta.kind` is an optional URI naming a hosted kind spec; URIs are immutable and semver-versioned. The only one defined so far is `https://toolpath.net/kinds/agent-coding-session/v1.1.0` (constant `toolpath::v1::PATH_KIND_AGENT_CODING_SESSION`; `…_V1_0_0` names the superseded URI); every conversation → `Path` derivation sets it via the shared `toolpath_convo::derive_path` or each provider crate's own. Carried through the JSONL form via `PathOpen.meta` and `PathMeta` patch lines. Spec sources live in `site/kinds///{index.md,schema.json}` (schema.json is a symlink into `crates/path-cli/kinds/`, which `path p validate` bundles — both versions) and publish under `https://toolpath.net/kinds/`; the registry index is `site/kinds/index.md`. RFC: "Document Kind". JSON Schema: `$defs/pathMeta`. -- Token accounting (kind v1.1.0): two keys on `conversation.append`/`Turn`, both optional. `token_usage` = "the total for a message" (on the group's final step; `Σ` over a path = session total). `attributed_token_usage` = "this step's own attributed spend", populated only where the source genuinely reports per-step spend (its own key, so the sum is unaffected; remainder = group total − Σ attributed, computed not stored). One provider message can span several steps (Claude writes one JSONL line per content block); `Turn.group_id` groups them. `toolpath-claude` fills `group_id` from `message.id` and takes the **field-wise-max** group total (line order not trusted). Claude's per-line `usage` is a cumulative *streaming snapshot* (Anthropic streaming API: `message_start` seeds output near 0, `message_delta` is cumulative), NOT a per-block cost — so Claude emits no `attributed_token_usage`; the projector re-expands the total onto every line. `toolpath-codex` differences the cumulative `total_token_usage` (dedup-safe: never sum `last_token_usage` — Codex re-emits it stale; openai/codex #14489), attributes each per-call delta to the step it follows, and derives the round total from those attributions. pi/opencode decode all-zero wire counters as `None`. Never stamp a cumulative counter, a repeated message total, or zero-filled placeholders onto a step; never derive attribution from Claude's streaming snapshots. +- Token accounting (kind v1.1.0): two keys on `conversation.append`/`Turn`, both optional. `token_usage` = "the total for a message" (on the group's final step; `Σ` over a path = session total). `attributed_token_usage` = "this step's own attributed spend", populated only where the source genuinely reports per-step spend (its own key, so the sum is unaffected; remainder = group total − Σ attributed, computed not stored). One provider message can span several steps (Claude writes one JSONL line per content block); `Turn.group_id` groups them. `toolpath-claude` fills `group_id` from `message.id` (falling back to the entry-level `requestId` for assistant entries when `message.id` is absent; user entries never group) and takes the **field-wise-max** group total across all of a group's lines (line order and adjacency not trusted — interleaved multi-terminal writes can split a group). Claude's per-line `usage` is a cumulative *streaming snapshot* (Anthropic streaming API: `message_start` seeds output near 0, `message_delta` is cumulative), NOT a per-block cost — so Claude emits no `attributed_token_usage`; the projector re-expands the total onto every line. `toolpath-codex` differences the cumulative `total_token_usage` (dedup-safe: never sum `last_token_usage` — Codex re-emits it stale; openai/codex #14489), attributes each per-call delta to the step it follows, and derives the round total from those attributions. pi/opencode decode all-zero wire counters as `None`. Never stamp a cumulative counter, a repeated message total, or zero-filled placeholders onto a step; never derive attribution from Claude's streaming snapshots. - Token usage `breakdowns` (kind v1.1.0, additive): an optional third key on `TokenUsage` — a decomposition of a top-level class into named sub-classes, keyed by class (e.g. `"output"`), inner map sub-class → tokens (e.g. `breakdowns["output"]["reasoning"] = 243`). INFORMATIONAL ONLY: **never summed into any total** (the parent class already counts those tokens, so the session-total guarantee is untouched); invariant `Σ(inner) ≤ parent`; omitted when empty; rides both `token_usage` and `attributed_token_usage`. Per-provider reality: **Gemini** reports `thoughts` (reasoning) as an additive sibling that the derivation used to **drop** (under-counting output) — it's now folded into `output_tokens` *and* recorded as `breakdowns["output"]["reasoning"]`, with the projector un-folding it on the reverse path for a lossless round-trip (`Some(0)` preserved as a real Gemini-3 zero-reasoning signal). **OpenCode** folds `reasoning` into output and records the same breakdown. **Codex** differences `reasoning_output_tokens` (⊆ output, cumulative) into `breakdowns["output"]["reasoning"]` on both per-step `attributed_token_usage` and per-round `token_usage`. **Claude** records no breakdown (its JSONL `usage` doesn't itemize thinking tokens). - Pi provider: `toolpath-pi` reads Pi session JSONL from `~/.pi/agent/sessions/`. Sessions use a tree (id/parentId) in a single file, and may link to a parent file via `parentSession` in the header. The tree is preserved as a DAG in the derived `Path`. - Codex provider: `toolpath-codex` reads Codex CLI rollout files from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl`. Sessions are date-bucketed (not project-keyed). File-change fidelity is excellent — Codex's `patch_apply_end` events carry either the unified diff (for updates) or the full file content (for adds), so the derived `Path` gets a real `raw` perspective on every file artifact. See `docs/agents/formats/codex.md` for the full format reference. diff --git a/crates/toolpath-claude/src/provider.rs b/crates/toolpath-claude/src/provider.rs index aaec1602..9a822863 100644 --- a/crates/toolpath-claude/src/provider.rs +++ b/crates/toolpath-claude/src/provider.rs @@ -615,9 +615,13 @@ fn canonicalize_message_usage(turns: &mut [Turn]) { /// /// Adjacency-free: a turn's usage counts only when it has no `group_id`, or /// when it's the **last** turn (by index) carrying its `group_id` — computed -/// by scanning for each gid's max index, not by checking the next turn. This -/// stays correct on non-canonicalized input (e.g. interleaved groups) as well -/// as on canonicalized input. +/// by scanning for each gid's max index, not by checking the next turn, so +/// interleaved (non-contiguous) groups still count once. Note the counted +/// value is that last turn's **own** `token_usage`: it equals the message +/// total only when the group's usage is repeated on every line or carried on +/// the last line (or was already canonicalized to the field-wise max). +/// Production always runs [`canonicalize_message_usage`] first, which +/// guarantees that precondition. fn sum_usage(turns: &[Turn]) -> Option { use std::collections::HashMap;