From a838f466d671fbd95e1e75f5337ef33cd6a289f9 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:08:23 -0400 Subject: [PATCH 01/22] feat(convo): session-level user_actor override in DeriveConfig Adds DeriveConfig.user_actor: Option so providers can set a channel-aware human actor (e.g. human:whatsapp/) for user turns instead of the default human:user. Additive; existing callers via ..Default::default() unaffected. Bumps toolpath-convo 0.11.0 -> 0.12.0. --- CHANGELOG.md | 9 + Cargo.toml | 2 +- crates/toolpath-convo/Cargo.toml | 2 +- crates/toolpath-convo/src/derive.rs | 34 +- docs/agents/formats/README.md | 16 +- docs/agents/formats/openclaw/README.md | 153 +++++++++ .../formats/openclaw/channels-and-actors.md | 101 ++++++ .../formats/openclaw/directory-layout.md | 137 ++++++++ docs/agents/formats/openclaw/entry-types.md | 123 +++++++ .../formats/openclaw/format-changelog.md | 52 +++ .../agents/formats/openclaw/jsonl-envelope.md | 116 +++++++ docs/agents/formats/openclaw/known-issues.md | 108 +++++++ docs/agents/formats/openclaw/lineage.md | 126 ++++++++ docs/agents/formats/openclaw/messages.md | 123 +++++++ docs/agents/formats/openclaw/tools.md | 85 +++++ docs/agents/formats/openclaw/usage.md | 97 ++++++ docs/agents/formats/openclaw/walkthrough.md | 153 +++++++++ .../2026-06-30-openclaw-format-docs-design.md | 301 ++++++++++++++++++ site/_data/crates.json | 2 +- 19 files changed, 1731 insertions(+), 9 deletions(-) create mode 100644 docs/agents/formats/openclaw/README.md create mode 100644 docs/agents/formats/openclaw/channels-and-actors.md create mode 100644 docs/agents/formats/openclaw/directory-layout.md create mode 100644 docs/agents/formats/openclaw/entry-types.md create mode 100644 docs/agents/formats/openclaw/format-changelog.md create mode 100644 docs/agents/formats/openclaw/jsonl-envelope.md create mode 100644 docs/agents/formats/openclaw/known-issues.md create mode 100644 docs/agents/formats/openclaw/lineage.md create mode 100644 docs/agents/formats/openclaw/messages.md create mode 100644 docs/agents/formats/openclaw/tools.md create mode 100644 docs/agents/formats/openclaw/usage.md create mode 100644 docs/agents/formats/openclaw/walkthrough.md create mode 100644 docs/superpowers/specs/2026-06-30-openclaw-format-docs-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 895f5e07..00545273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to the Toolpath workspace are documented here. +## toolpath-convo 0.12.0: session-level `user_actor` override — 2026-06-30 + +Adds `DeriveConfig.user_actor: Option` so a provider can set a +session-level actor string for user turns (e.g. a channel-aware +`human:whatsapp/`) instead of the default `human:user`. Additive +and backward-compatible — existing callers using `..Default::default()` +are unaffected. Enables the multi-channel human identity in the new +`toolpath-openclaw` provider. + ## Token usage: once per message, with per-step attribution + kind v1.1.0 — 2026-06-17 Fixes token over-counting in derived documents (~3× output-token diff --git a/Cargo.toml b/Cargo.toml index b03c2d21..5e4e8e3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ license = "Apache-2.0" [workspace.dependencies] toolpath = { version = "0.7.0", path = "crates/toolpath" } -toolpath-convo = { version = "0.11.0", path = "crates/toolpath-convo" } +toolpath-convo = { version = "0.12.0", path = "crates/toolpath-convo" } toolpath-git = { version = "0.6.0", path = "crates/toolpath-git" } toolpath-claude = { version = "0.12.0", path = "crates/toolpath-claude", default-features = false } toolpath-gemini = { version = "0.6.0", path = "crates/toolpath-gemini", default-features = false } diff --git a/crates/toolpath-convo/Cargo.toml b/crates/toolpath-convo/Cargo.toml index e1009ede..6ec4f1d0 100644 --- a/crates/toolpath-convo/Cargo.toml +++ b/crates/toolpath-convo/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toolpath-convo" -version = "0.11.0" +version = "0.12.0" edition.workspace = true license.workspace = true repository = "https://github.com/empathic/toolpath" diff --git a/crates/toolpath-convo/src/derive.rs b/crates/toolpath-convo/src/derive.rs index ad66857c..ca53202f 100644 --- a/crates/toolpath-convo/src/derive.rs +++ b/crates/toolpath-convo/src/derive.rs @@ -29,6 +29,11 @@ pub struct DeriveConfig { pub include_thinking: bool, /// Include `Turn.tool_uses` in the structural change extras. pub include_tool_uses: bool, + /// Session-level actor string for user turns (e.g. a channel-aware + /// `human:whatsapp/`). When `None`, user turns get the default + /// `human:user`. Used by providers (OpenClaw) whose human is a known + /// per-session identity rather than the local shell user. + pub user_actor: Option, } impl Default for DeriveConfig { @@ -39,6 +44,7 @@ impl Default for DeriveConfig { title: None, include_thinking: true, include_tool_uses: true, + user_actor: None, } } } @@ -115,7 +121,7 @@ pub fn derive_path(view: &ConversationView, config: &DeriveConfig) -> Path { }; turn_to_step.insert(turn.id.clone(), step_id.clone()); - let actor = actor_for_turn(turn, provider); + let actor = actor_for_turn(turn, provider, config.user_actor.as_deref()); record_actor(&mut actors, &actor, turn, provider, view); let mut step = Step { @@ -473,9 +479,11 @@ pub fn derive_path(view: &ConversationView, config: &DeriveConfig) -> Path { } } -fn actor_for_turn(turn: &Turn, provider: &str) -> String { +fn actor_for_turn(turn: &Turn, provider: &str, user_actor: Option<&str>) -> String { match &turn.role { - Role::User => "human:user".to_string(), + Role::User => user_actor + .map(str::to_string) + .unwrap_or_else(|| "human:user".to_string()), Role::Assistant => { let model = turn.model.as_deref().unwrap_or("unknown"); format!("agent:{}", model) @@ -804,6 +812,26 @@ mod tests { assert_eq!(path.steps[0].step.id, "t1"); } + #[test] + fn user_actor_override_sets_human_actor() { + let turn = base_turn("t1", Role::User); + let view = view_with(vec![turn]); + let cfg = DeriveConfig { + user_actor: Some("human:whatsapp/15555550123".into()), + ..Default::default() + }; + let path = derive_path(&view, &cfg); + assert_eq!(path.steps[0].step.actor, "human:whatsapp/15555550123"); + } + + #[test] + fn user_actor_default_is_human_user() { + let turn = base_turn("t1", Role::User); + let view = view_with(vec![turn]); + let path = derive_path(&view, &DeriveConfig::default()); + assert_eq!(path.steps[0].step.actor, "human:user"); + } + #[test] fn test_single_assistant_turn() { let mut turn = base_turn("t1", Role::Assistant); diff --git a/docs/agents/formats/README.md b/docs/agents/formats/README.md index cc01167b..86abe527 100644 --- a/docs/agents/formats/README.md +++ b/docs/agents/formats/README.md @@ -35,6 +35,14 @@ culting. - **[`gemini.md`](gemini.md)** — Gemini CLI chats under `~/.gemini/tmp//chats/`, including the main-file + sibling sub-agent UUID directory layout. +- **[`openclaw/`](openclaw/README.md)** — OpenClaw, a multi-channel + local-first assistant. JSONL transcripts under + `~/.openclaw/agents//sessions/`, with a tree/visible-leaf entry + model and channel/peer-keyed sessions. Split into focused docs covering + directory layout, the version-3 envelope, the ten entry types, messages + and content blocks, tools and file-ops, token usage, lineage, the + channel/actor identity model, known issues, a walkthrough, and a format + changelog. Sourced from upstream code only — no first-hand sample yet. - **[`opencode.md`](opencode.md)** — opencode's SQLite database (`~/.local/share/opencode/opencode.db`), its 12 typed message-part variants, and the sibling bare-git snapshot repo used for file diffs. @@ -45,9 +53,11 @@ culting. The Claude Code reference is the most detailed because it's the longest-standing provider and has the most moving parts (JSONL envelope variants, session chaining, compaction, sidechains, and the -loader's own undocumented strictness on what it will accept). The -other three sit in single files because their formats are either -simpler or sufficiently covered there. +loader's own undocumented strictness on what it will accept). OpenClaw +also gets a folder — its tree/visible-leaf entry model and +multi-channel actor identity have enough moving parts to warrant the +breakout. The rest sit in single files because their formats are +either simpler or sufficiently covered there. ## Conventions used in these docs diff --git a/docs/agents/formats/openclaw/README.md b/docs/agents/formats/openclaw/README.md new file mode 100644 index 00000000..656f0f3d --- /dev/null +++ b/docs/agents/formats/openclaw/README.md @@ -0,0 +1,153 @@ +# OpenClaw on-disk format + +> **Reference revision:** 2026-06-30 +> **Tracks:** OpenClaw package `2026.6.10`, session format **version 3** +> **Sourced from:** upstream code at +> `openclaw/openclaw @ 68c533cfb339cbb8650832cb2a4bf38dba7022fa` — **no +> first-hand on-disk sample yet** (see [Sourcing](#sourcing-and-confidence)). +> +> When you change anything in this directory, bump the revision date here +> and add a note to [format-changelog.md](format-changelog.md). + +[OpenClaw](https://github.com/openclaw/openclaw) is a local-first personal +AI assistant — "Your own personal AI assistant. Any OS. Any Platform. The +lobster way. 🦞" It runs agent sessions on your own device and bridges them +to messaging channels (WhatsApp, Telegram, Slack, Discord, Matrix, Signal, +…). It persists every session as an append-only JSONL transcript under +`~/.openclaw/`. + +This is different from the other providers we derive from, which are all +single-user coding-agent CLIs. An OpenClaw session is scoped to an *agent +persona × channel × peer/thread*, and the human is a messaging contact, not +a local shell user — so identity, lineage, and "what is one session" all +work differently here. These docs are the working reference for what +OpenClaw writes to disk, for anyone building a tool that reads or writes it +(such as a future `toolpath-openclaw` provider). + +## How the docs are organized + +Read in this order if you're new; otherwise skip to what you need. If you +prefer a concrete example, start with the **walkthrough** (#10). + +1. **[directory-layout.md](directory-layout.md)** — the `~/.openclaw/` tree, + how the state directory is resolved (env overrides, no XDG), the + per-agent `sessions/` directory, the `sessions.json` index, and the + telemetry/SQLite stores that are *not* the transcript. +2. **[jsonl-envelope.md](jsonl-envelope.md)** — the version-3 header line, + the entry base (`type`/`id`/`parentId`/`timestamp`), and the tree + + visible-leaf mechanics that decide which branch is live. +3. **[entry-types.md](entry-types.md)** — the ten `type` variants in detail. +4. **[messages.md](messages.md)** — message roles, the `text`/`thinking`/ + `image`/`toolCall` content blocks, assistant metadata, and the two + timestamp encodings. +5. **[tools.md](tools.md)** — `toolCall` ↔ `toolResult` correlation, + `isError`, and why there's no stored diff (file ops are tool-input only). +6. **[usage.md](usage.md)** — persisted `Usage` vs runtime `NormalizedUsage`, + per-step deltas, compaction-zeroing, and why the reasoning token count + isn't on disk. +7. **[lineage.md](lineage.md)** — the intra-session entry DAG, cross-session + forks/sub-agents, compaction branches, and the session-kind classifier. +8. **[channels-and-actors.md](channels-and-actors.md)** — the session-key + grammar, channels, DM vs group identity, `[from:]` markers, and persona + vs model as the agent actor. The OpenClaw-specific axis. +9. **[known-issues.md](known-issues.md)** — format-level gotchas to defend + against. +10. **[walkthrough.md](walkthrough.md)** — a session read line by line, with + cross-links back to the reference docs. +11. **[format-changelog.md](format-changelog.md)** — version-keyed record of + field/behavior changes. + +## Sourcing and confidence + +Unlike the other references in this directory (claude-code, codex, cursor, +gemini, opencode, pi), this one is **not** backed by first-hand on-disk +samples or by our own parser: + +- **No installed OpenClaw / no sample file.** Every claim is derived from + **upstream source code** at the pinned commit, not from observed bytes. +- **No `toolpath-openclaw` crate yet.** We can't corroborate with our own + tests. This is the same posture `opencode.md` was written in before its + crate existed. + +Consequences for how to read these docs: + +- Field tables here mean "**from source**" (the producer's type), not + "observed in the wild." Where a serialized form could not be confirmed + from the code, the doc says so explicitly. +- The transcript is touched by **two code layers** (agent-core harness + storage and the gateway session manager) with slightly different type + names for the same JSON; we use the agent-core names and flag the + reconciliation in + [known-issues.md](known-issues.md#two-code-layers-for-one-format). + +When the crate is built, re-verify against a real session, upgrade claims +from "from source" to "observed," and record it in the changelog. + +## Conventions + +- **Field names** are shown as they appear in JSON (camelCase envelope keys + like `parentId`; provider-style keys like `provider`/`modelId` where the + source uses them). +- **"From source"** = read from upstream TypeScript at the pinned commit. + **"Observed"** is reserved for claims confirmed against a real file (none + yet). +- **Citations** point to `openclaw/openclaw path:line` at commit + `68c533cf`, or to a doc in this folder. +- **Keep headings anchor-stable.** Cross-links use GitHub auto-anchors + (lowercased, punctuation stripped, spaces to hyphens). Avoid em-dashes in + linkable headings. + +## Field index + +Quick lookup: which doc defines a given field? + +| Field | Defined in | +|---|---| +| `agentId` | [directory-layout.md](directory-layout.md), [channels-and-actors.md](channels-and-actors.md) | +| `api` | [messages.md](messages.md) | +| `appendMode` / `appendParentId` | [jsonl-envelope.md](jsonl-envelope.md) | +| `arguments` (on `toolCall`) | [messages.md](messages.md), [tools.md](tools.md) | +| `branch_summary` / `fromId` | [entry-types.md](entry-types.md), [lineage.md](lineage.md) | +| `cacheRead` / `cacheWrite` | [usage.md](usage.md) | +| `channel` / `peerKind` / `peerId` | [channels-and-actors.md](channels-and-actors.md) | +| `compaction` / `firstKeptEntryId` / `tokensBefore` | [entry-types.md](entry-types.md), [usage.md](usage.md), [lineage.md](lineage.md) | +| `content` (message / block) | [messages.md](messages.md) | +| `cost` | [usage.md](usage.md) | +| `custom` / `custom_message` / `customType` | [entry-types.md](entry-types.md) | +| `cwd` | [jsonl-envelope.md](jsonl-envelope.md), [directory-layout.md](directory-layout.md) | +| `details` (tool result / summaries) | [tools.md](tools.md), [entry-types.md](entry-types.md) | +| `id` (entry) | [jsonl-envelope.md](jsonl-envelope.md) | +| `id` (session header) | [jsonl-envelope.md](jsonl-envelope.md) | +| `id` (on `toolCall`) | [messages.md](messages.md), [tools.md](tools.md) | +| `input` / `output` / `totalTokens` | [usage.md](usage.md) | +| `InputProvenance` / `kind` | [channels-and-actors.md](channels-and-actors.md) | +| `isError` | [tools.md](tools.md) | +| `label` (on `label` entry) | [entry-types.md](entry-types.md) | +| `leaf` / `targetId` | [jsonl-envelope.md](jsonl-envelope.md), [entry-types.md](entry-types.md) | +| `model` / `modelId` / `model_change` | [messages.md](messages.md), [entry-types.md](entry-types.md) | +| `parentId` | [jsonl-envelope.md](jsonl-envelope.md), [lineage.md](lineage.md) | +| `parentSession` (header) | [jsonl-envelope.md](jsonl-envelope.md), [lineage.md](lineage.md) | +| `parentSessionKey` / `spawnedBy` / `spawnDepth` / `subagentRole` | [lineage.md](lineage.md) | +| `provider` | [messages.md](messages.md), [channels-and-actors.md](channels-and-actors.md) | +| `redacted` (on `thinking`) | [messages.md](messages.md) | +| `role` | [messages.md](messages.md) | +| `session_info` / `name` | [entry-types.md](entry-types.md) | +| `stopReason` | [messages.md](messages.md) | +| `summary` (compaction / branch) | [entry-types.md](entry-types.md), [lineage.md](lineage.md) | +| `textSignature` / `thinkingSignature` / `thoughtSignature` | [messages.md](messages.md) | +| `thinking` (content block) | [messages.md](messages.md) | +| `thinking_level_change` / `thinkingLevel` | [entry-types.md](entry-types.md) | +| `timestamp` (entry, ISO) | [jsonl-envelope.md](jsonl-envelope.md) | +| `timestamp` (inner message, epoch ms) | [messages.md](messages.md) | +| `toolCall` / `toolCallId` / `toolName` | [messages.md](messages.md), [tools.md](tools.md) | +| `type` (header / entry) | [jsonl-envelope.md](jsonl-envelope.md), [entry-types.md](entry-types.md) | +| `usage` | [usage.md](usage.md) | +| `version` | [jsonl-envelope.md](jsonl-envelope.md), [format-changelog.md](format-changelog.md) | + +## Maintenance + +When a field, entry type, or behavior changes (or a real sample +contradicts something here), update the relevant doc in the same change, +keep this index and the doc map in sync, and add a +[format-changelog.md](format-changelog.md) entry. The point of this folder +is to be the single place OpenClaw format knowledge accumulates. diff --git a/docs/agents/formats/openclaw/channels-and-actors.md b/docs/agents/formats/openclaw/channels-and-actors.md new file mode 100644 index 00000000..aedabf9e --- /dev/null +++ b/docs/agents/formats/openclaw/channels-and-actors.md @@ -0,0 +1,101 @@ +# Channels and actors + +This is the axis that sets OpenClaw apart from every other provider we +derive from. The other agents are single-user coding CLIs: the human is +whoever is at the terminal. OpenClaw is a multi-channel assistant — the same +agent persona talks to many people across WhatsApp, Telegram, Slack, +Discord, Matrix, Signal, and more — so "who is the human" is a real +question, and the answer is **not a field on the message**. It is encoded +structurally in the session key. + +## The session key grammar + +Built by `buildAgentPeerSessionKey` (`src/routing/session-key.ts:222-273`): + +``` +agent:::: +agent::::direct: # per-account-channel-peer DM scope +agent::direct: # per-peer DM scope +agent::main # CLI / main session +``` + +| Segment | Values / meaning | +|---|---| +| `agentId` | The OpenClaw agent persona (default `main`). Also the directory bucket ([directory-layout.md](directory-layout.md)). | +| `channel` | `whatsapp` \| `telegram` \| `slack` \| `discord` \| `matrix` \| `signal` \| … | +| `peerKind` | `direct` \| `dm` \| `group` \| `channel` (`ParsedSessionDeliveryRoute`, `src/sessions/session-key-utils.ts`). | +| `peerId` | The channel-native id. For a **DM** this is the human's channel user id; for a **group/channel** it is the room/group id, *not* an individual person. | + +Threads append `:thread:` (`resolveThreadSessionKeys`). Opaque, +case-sensitive peer ids (Signal groups, Matrix rooms) are preserved verbatim +via `CASE_PRESERVING_PEERS`; everything else is lowercased. The key is a +stable, parseable identity string, and it is what `sessions.json` is keyed +on. + +## Who is the human? + +There is **no `sender` field** on a transcript `UserMessage` +(`llm-core/src/types.ts:280-284` — just `role`/`content`/`timestamp`). +Identity comes from the key: + +- **Direct messages:** `peerId` *is* the human's channel user id. You can + form a reasonable actor string like `whatsapp:15555550123`. +- **Groups / channels:** the key only gives you `group:`. The + **individual speaker is text-only** — OpenClaw injects a + `[from: Sender Name (+E164)]` marker into the prompt text at the end of + each group batch (`docs/channels/group-messages.md`), and lists members in + the system prompt. The inbound layer *does* have structured `senderId` / + `senderName` / `pushName` (e.g. `extensions/whatsapp/src/inbound/types.ts`), + but those are consumed at routing/policy time and **flattened into message + text** — they do **not** persist as transcript fields. + +So per-message individual identity in a group is only recoverable by +**parsing the `[from: …]` marker** out of the user text. Treat that as +best-effort. + +### `InputProvenance` + +The closest structured "who / where from" kept on a user message is +`InputProvenance` (`src/sessions/input-provenance.ts:14-21`): + +```ts +type InputProvenance = { + kind: "external_user" | "inter_session" | "internal_system"; + originSessionId?: string; + sourceSessionKey?: string; + sourceChannel?: string; + sourceTool?: string; +}; +``` + +Use `kind` to distinguish a real channel user from an inter-session message +(one agent/session driving another) or an internal system prompt. + +## Who is the agent? + +Two different axes, and a toolpath derivation must decide which is its +`agent:`: + +- **The persona:** `agentId` (plus a public identity blob + `AgentIdentityResultSchema` — `agentId`, `name`, `avatar`, `emoji` — + `gateway-protocol/src/schema/agent.ts:955-966`). Stable across turns and + models. +- **The model:** every `AssistantMessage` carries `provider` / `model` / + `api` ([messages.md](messages.md#assistant-message-metadata)), and + `model_change` entries mark switches. This can change mid-session. + +## Closest `type:name` actor strings + +toolpath actors are `type:name`. The honest mapping: + +| Side | Suggested actor string | Caveat | +|---|---|---| +| Human (DM) | `:` e.g. `whatsapp:15555550123` | Clean. | +| Human (group) | `:group:` + `[from:]` parse for the person | The group id is structural; the person is text-only. | +| Agent (persona) | `agent:` | Stable; loses the model. | +| Agent (model) | `:` e.g. `anthropic:claude-…` | Per-message; changes on `model_change`. | + +This is a **decision for the eventual `toolpath-openclaw` crate**, not +something the format dictates — the format gives you both axes and a +multi-party group reality that none of the single-user providers have. Pick +deliberately and document it. diff --git a/docs/agents/formats/openclaw/directory-layout.md b/docs/agents/formats/openclaw/directory-layout.md new file mode 100644 index 00000000..06a46065 --- /dev/null +++ b/docs/agents/formats/openclaw/directory-layout.md @@ -0,0 +1,137 @@ +# `~/.openclaw/` directory layout + +Everything OpenClaw persists lives under a single state directory, by +default `~/.openclaw/`. There is **no XDG split and no per-OS special +directory** — the layout is identical on macOS, Linux, and Windows; the +only platform difference is how the home directory itself is found. The +directory is created on first run and populated lazily. + +## Resolving the state directory + +The root is chosen by `resolveStateDir` +(`src/config/paths.ts:209-273`), in this precedence: + +1. `OPENCLAW_STATE_DIR` (explicit override; `~` is expanded). +2. An existing `~/.openclaw` (the current default name). +3. An existing legacy `~/.clawdbot` (the project's former name). +4. Default `~/.openclaw` (created if nothing above matched). + +The `~` used above is itself resolved by `resolveOsHomeDir` +(`src/infra/home-dir.ts:35-54`): `OPENCLAW_HOME` wins (also `~`-expanding), +otherwise `HOME` → `USERPROFILE` → Termux home → `os.homedir()`. + +Literal defaults for a default user with no overrides: + +| OS | State directory | +|---------|---------------------------------------| +| macOS | `/Users//.openclaw` | +| Linux | `/home//.openclaw` | +| Windows | `C:\Users\\.openclaw` | + +Related overrides: `OPENCLAW_CONFIG_PATH` (the config file, default +`/openclaw.json`; legacy `clawdbot.json` is tolerated), +`OPENCLAW_OAUTH_DIR` (default `/credentials`), and +`OPENCLAW_TRAJECTORY_DIR` (see [known-issues.md](known-issues.md) and +below). There is **no** `OPENCLAW_CONFIG_DIR` and **no** `XDG_*` lookup. + +### Daemon path divergence + +The managed-service daemon has its own resolver +(`src/daemon/paths.ts:116-127`) that reads `HOME`/`USERPROFILE` directly +(ignoring `OPENCLAW_HOME`) and appends a profile suffix: a non-default +`OPENCLAW_PROFILE` yields `~/.openclaw-`. The daemon then launches +the gateway with `OPENCLAW_STATE_DIR` pointed at that profiled directory. +With the default profile both resolvers agree on `~/.openclaw`. If you ever +find sessions missing, check whether a profile is in play. + +## The tree + +``` +~/.openclaw/ # = state dir (override: OPENCLAW_STATE_DIR) +├── openclaw.json # user config (override: OPENCLAW_CONFIG_PATH) +├── credentials/ +│ └── oauth.json # OAuth creds (override dir: OPENCLAW_OAUTH_DIR) +├── state/ +│ └── openclaw.sqlite # shared state DB — NOT conversation content +└── agents/ + └── / # default agentId = "main" + ├── agent/ + │ └── openclaw-agent.sqlite # per-agent DB (caches, RAG index) — NOT content + └── sessions/ # ← canonical transcripts live here + ├── sessions.json # index: sessionKey -> { sessionId, sessionFile, … } + ├── .jsonl # canonical session transcript + ├── -topic-.jsonl # topic-scoped transcript + ├── _.jsonl # forked / rotated transcript + ├── .trajectory.jsonl # runtime telemetry sidecar — NOT canonical + └── .trajectory-path.json # pointer to the runtime trace file +``` + +Only the per-session `*.jsonl` transcript under `agents//sessions/` +is needed to reconstruct a conversation. Everything else is supporting +state (see [Stores that are not the transcript](#stores-that-are-not-the-transcript)). + +Path builders: + +- Sessions directory — `src/config/sessions/paths.ts:653-661`: + `/agents//sessions`, default agent id + `"main"` (`src/routing/session-key.ts`, `DEFAULT_AGENT_ID`; + `normalizeAgentId` lowercases and path-sanitizes). +- Transcript filename — `src/config/sessions/paths.ts:898-902`: + `.jsonl`, or `-topic-.jsonl` when a topic + is set. +- Session index — `src/config/sessions/paths.ts:678-680`: + `/sessions.json`. + +## Naming and keying + +Sessions are **bucketed by agent** (the directory) and **keyed by +channel/peer/thread** (the routing key, used as the key in `sessions.json`). +They are **not** bucketed by project (like Claude/Gemini) or by date (like +Codex). + +- **Directory bucket** = `agentId` (default `main`). +- **Logical key** = a composite routing key such as + `agent:main:whatsapp:group:12345` — see + [channels-and-actors.md](channels-and-actors.md) for the full grammar. +- **Filename** = the generated `sessionId`, a UUID. A forked or rotated + transcript prefixes an ISO timestamp (`_.jsonl`); a + topic-scoped one suffixes `-topic-`. (Entry ids *inside* a file + are only 8-char UUIDv7 prefixes — distinct from the file's full UUID; see + [jsonl-envelope.md](jsonl-envelope.md).) + +The `sessions.json` index is a JSON object mapping each routing key to an +entry of roughly `{ sessionId, sessionFile, updatedAt, sessionStartedAt, … +delivery state }`. A reader should resolve keys to concrete files through +this index rather than guessing filenames. + +## Stores that are not the transcript + +Three other on-disk stores sit next to the transcript. None of them is the +canonical conversation record; do not reconstruct conversations from them. + +| Store | Path | What it holds | +|---|---|---| +| Trajectory trace | `…/sessions/.trajectory.jsonl` (+ `.trajectory-path.json` pointer) | Append-only `openclaw-trajectory` runtime telemetry (prompts, compiled context, tool calls, richer usage). The diagnostic exporter *joins* this with the transcript to build a support bundle, so the transcript is the source of truth and this is auxiliary. Advisory sidecar. | +| Shared state DB | `~/.openclaw/state/openclaw.sqlite` | Auth profiles, device/node pairing, push, model-capability cache, cron jobs/logs, task/subagent/flow runs, delivery/ingress queues, `current_conversation_bindings` (a routing pointer, no message text). The only message-shaped table, `acp_replay_events`, is a transient ACP replay buffer, not the agent's own transcript. | +| Per-agent DB | `~/.openclaw/agents//agent/openclaw-agent.sqlite` | `cache_entries`, auth-profile rows, a `memory_index_*` / embedding RAG index. No transcript table. | + +A diagnostic **trajectory export** (a different root entirely) lands under +`/.openclaw/trajectory-exports/openclaw-trajectory--/` +with `events.jsonl`, `session-branch.json`, `manifest.json`, etc. +(`src/trajectory/export.ts`). That is a redacted support bundle, not part +of the live store. + +## Permissions + +| Path | Mode | +|---|---| +| `…/sessions/.jsonl` | `0600` (append opens `a+`, `0o600`; `src/config/sessions/transcript-jsonl.ts:98`) | +| `…/sessions/sessions.json` | `0600` (atomic write; dir `0o777 & ~umask`) | +| `…/sessions/.trajectory.jsonl` | `0600` | +| `~/.openclaw/state/openclaw.sqlite` | dir `0700`, file `0600` | +| `~/.openclaw/agents//agent/openclaw-agent.sqlite` | dir `0700`, file `0600` | +| `~/.openclaw/openclaw.json`, `credentials/oauth.json` | `0600` | + +No protobuf anywhere in these paths. The session transcript is JSONL; +indexes and pointers are JSON; the two databases are SQLite (node:sqlite, +WAL). diff --git a/docs/agents/formats/openclaw/entry-types.md b/docs/agents/formats/openclaw/entry-types.md new file mode 100644 index 00000000..ef1b6945 --- /dev/null +++ b/docs/agents/formats/openclaw/entry-types.md @@ -0,0 +1,123 @@ +# Entry types + +Every non-header line carries a `type` discriminant. There are **ten** +known entry types (`harness/types.ts:443-454`). A reader that handles only +`message` will silently drop model changes, compaction boundaries, branch +summaries, labels, and the leaf pointers that tell it which branch is even +live. + +## Summary table + +| `type` | Carries a message? | Purpose | +|---|---|---| +| `message` | yes (`message`) | A conversational turn: user / assistant / toolResult / bashExecution. The dominant type. | +| `model_change` | no | Records a model/provider switch mid-session. | +| `thinking_level_change` | no | OpenClaw-specific reasoning-budget toggle. | +| `compaction` | no | Context-compaction boundary: history before `firstKeptEntryId` is replaced by `summary`. | +| `branch_summary` | no | Summary of an abandoned branch when navigating away from it. | +| `custom` | no | Harness/app marker **not** replayed into model context. | +| `custom_message` | yes (`content`) | Harness/app content that **is** replayable into context. | +| `label` | no | Display label for a target entry (last write wins). | +| `session_info` | no | Session name/title (last write wins). | +| `leaf` | no | Visible-head pointer; see [jsonl-envelope.md](jsonl-envelope.md#the-tree-and-the-visible-leaf). | + +All ten extend [`SessionTreeEntryBase`](jsonl-envelope.md#the-entry-base) +(`type` / `id` / `parentId` / `timestamp` / `appendMode?`). Fields below are +the *additional* ones each variant carries. + +--- + +## `message` + +`{ type:"message", message: AgentMessage }` (`harness/types.ts:367-370`). +The `message` object holds the role, content, and (for assistant turns) the +provider/usage metadata. Roles realistically seen on disk: `user`, +`assistant`, `toolResult`, `bashExecution`. Full treatment in +[messages.md](messages.md); tool-call/result correlation in +[tools.md](tools.md). + +> The role union also names `custom` / `branchSummary` / `compactionSummary`, +> but those are **not** stored as `message` entries — they live as their own +> entry types (`custom_message`, `branch_summary`, `compaction`) and are +> reconstructed into role-bearing messages at read time +> (`session.ts:46-99`). + +## `model_change` + +`{ type:"model_change", provider, modelId }` (`harness/types.ts:379-382`). +A mid-session model/provider switch. The active model is also redundantly +recorded on every `assistant` message (`provider`/`model`/`api`), so a +reader can track the model without relying on these markers — but they're +the explicit signal. + +## `thinking_level_change` + +`{ type:"thinking_level_change", thinkingLevel }` (`harness/types.ts:373-376`). +OpenClaw's reasoning-budget toggle (analogous to a thinking-effort setting). +Informational for provenance. + +## `compaction` + +`{ type:"compaction", summary, firstKeptEntryId, tokensBefore, details?, fromHook? }` +(`harness/types.ts:386-393`). + +| Field | Shape | Notes | +|---|---|---| +| `summary` | string | Structured markdown summary of the dropped history. | +| `firstKeptEntryId` | string | Entries strictly before this id are represented by `summary`; the tail is replayed. | +| `tokensBefore` | int | Estimated context tokens before compaction. | +| `details` | `{ readFiles: string[], modifiedFiles: string[] }` | Optional; the concrete shape `CompactionDetails`. | +| `fromHook` | bool | Optional; true if produced by an app hook rather than the built-in summarizer. | + +At replay, history before `firstKeptEntryId` is dropped and replaced by a +synthetic `compactionSummary` message. See [lineage.md](lineage.md) and the +usage caveat in [usage.md](usage.md#compaction-zeroes-stale-usage). + +## `branch_summary` + +`{ type:"branch_summary", fromId, summary, details?, fromHook? }` +(`harness/types.ts:396-402`). + +| Field | Shape | Notes | +|---|---|---| +| `fromId` | string | Entry id of the abandoned branch's source leaf (`"root"` when null). | +| `summary` | string | Summary of the abandoned branch. | +| `details` | `{ readFiles: string[], modifiedFiles: string[] }` | Optional; `BranchSummaryDetails`. | +| `fromHook` | bool | Optional. | + +Appended by `Session.moveTo(...)` when navigating away from a branch +(`session.ts:268-289`). Reconstituted into context as a synthetic +`branchSummary` message wrapped in `…`. + +## `custom` + +`{ type:"custom", customType, data? }` (`harness/types.ts:405-409`). An +arbitrary harness/app marker that is **not** replayed into model context. +`customType` namespaces it; `data` is an opaque payload. + +## `custom_message` + +`{ type:"custom_message", customType, content, details?, display }` +(`harness/types.ts:412-418`). Like `custom`, but its `content` (string or +`(text|image)[]`) **is** replayable into context. `display` controls UI +visibility. + +## `label` + +`{ type:"label", targetId, label }` (`harness/types.ts:421-425`). A display +label for the entry `targetId`. **Last write wins** per target; an +`undefined`/empty `label` clears it. + +## `session_info` + +`{ type:"session_info", name? }` (`harness/types.ts:428-432`). The session +name/title. **Last write wins** (`session.ts getSessionName`). The +persisted discriminant `session_info` predates the "session name" wording, +so don't expect the field to be called `title`. + +## `leaf` + +`{ type:"leaf", targetId, appendParentId? }` (`harness/types.ts:435-440`). +Moves the visible-head pointer. Carries no message. Covered in detail in +[jsonl-envelope.md](jsonl-envelope.md#the-tree-and-the-visible-leaf) because +you cannot determine the live conversation without it. diff --git a/docs/agents/formats/openclaw/format-changelog.md b/docs/agents/formats/openclaw/format-changelog.md new file mode 100644 index 00000000..e6a8844d --- /dev/null +++ b/docs/agents/formats/openclaw/format-changelog.md @@ -0,0 +1,52 @@ +# Format changelog + +A version-keyed record of OpenClaw session-format fields and behaviors, so +downstream readers can tell whether a cited rule is current. Entries are +newest-first. + +The format self-identifies via the header `version` field +([jsonl-envelope.md](jsonl-envelope.md#the-header-line)); the reader +hard-rejects anything other than `3`. The OpenClaw package version (CalVer, +e.g. `2026.6.10`) is separate from this on-disk format version. + +## Format version 3 + +**Status as of this reference:** the only format version this doc set +covers. Established by reading upstream source at +`openclaw/openclaw @ 68c533cfb339cbb8650832cb2a4bf38dba7022fa` (package +`2026.6.10`); **not** yet confirmed against a first-hand on-disk sample. + +Salient v3 facts (all detailed elsewhere in this folder): + +- Header line `{ type:"session", version:3, id, timestamp, cwd, + parentSession? }`. +- Ten entry types: `message`, `model_change`, `thinking_level_change`, + `compaction`, `branch_summary`, `custom`, `custom_message`, `label`, + `session_info`, `leaf`. +- Tree/DAG via `id`/`parentId` with a separate visible-leaf pointer + (`leaf` rows, `appendMode:"side"`). +- Message roles `user`/`assistant`/`toolResult`/`bashExecution`; content + blocks `text`/`thinking`/`image`/`toolCall`; tool results are separate + entries linked by `toolCallId`. +- Per-message `Usage` (input/output/cacheRead/cacheWrite/totalTokens/cost), + per-step delta, no reasoning token field; zeroed across compaction. +- Dual timestamp encodings (entry ISO string vs inner message epoch-ms). + +### Pre-v3 / legacy + +Logs predating entry ids exist and are migrated with synthetic `parentId` +chains by `migrateLegacySessionEntries` (`src/trajectory/export.ts:946-989`). +The legacy state directory name was `~/.clawdbot` (still tolerated; config +`clawdbot.json`). We have not characterized the pre-v3 on-disk shape — treat +migrated files as v3-shaped with back-filled lineage. + +## Maintenance + +When the `toolpath-openclaw` crate lands, or when a new OpenClaw release +changes a field or bumps the header `version`: + +1. Add a new section at the top here with the version and what changed. +2. Update the affected reference doc(s) in the same change. +3. Upgrade claims from "from source" to "observed" once a real sample + confirms them, and bump the revision date in + [README.md](README.md). diff --git a/docs/agents/formats/openclaw/jsonl-envelope.md b/docs/agents/formats/openclaw/jsonl-envelope.md new file mode 100644 index 00000000..a1580caa --- /dev/null +++ b/docs/agents/formats/openclaw/jsonl-envelope.md @@ -0,0 +1,116 @@ +# The JSONL envelope + +An OpenClaw session is one **append-only JSONL file**: the first non-blank +line is a session header, and every subsequent non-blank line is one +**session-tree entry**. Crucially, it is **not a flat message log** — the +entries form a tree (a DAG) via `parentId`, and the "current" conversation +is the path from a leaf back to the root. Branching, the visible-leaf +pointer, model changes, and compaction are all just additional entry types +appended to the same file. + +The format self-identifies as **version 3**. + +> **Two code layers, one file.** The transcript is read/written by the +> agent-core harness storage (`packages/agent-core/src/harness/session/jsonl-storage.ts`, +> types in `harness/types.ts`) and mirrored by the gateway session manager +> (`src/agents/sessions/session-manager.ts`, writer +> `src/config/sessions/transcript-jsonl.ts`). They use slightly different +> type names (`SessionTreeEntry`/`MessageEntry` vs +> `SessionEntry`/`SessionMessageEntry`) for what serializes to the same +> JSON. This doc uses the agent-core names. See +> [known-issues.md](known-issues.md#two-code-layers-for-one-format). + +## The header line + +Always the first line. From `jsonl-storage.ts:22-29`: + +```json +{ + "type": "session", + "version": 3, + "id": "0190ab00-aaaa-7bbb-8ccc-000000000001", + "timestamp": "2026-06-30T12:00:00.000Z", + "cwd": "/home/u/proj", + "parentSession": "/home/u/.openclaw/agents/main/sessions/.jsonl" +} +``` + +| Field | Shape | Req? | Notes | +|---|---|---|---| +| `type` | `"session"` | required | Header discriminant. | +| `version` | int | required | Format version. **Reader hard-rejects `version !== 3`** (`jsonl-storage.ts:80-82`). | +| `id` | string (UUID) | required | Session id. Matches the transcript filename stem. | +| `timestamp` | ISO-8601 string | required | Session creation time (`new Date().toISOString()`). | +| `cwd` | string | required | Working directory the session ran in. | +| `parentSession` | string (path) | optional | Path to a parent session **file** when this session was forked. Cross-file lineage; see [lineage.md](lineage.md). | + +The header maps to `JsonlSessionMetadata { id, createdAt, cwd, path, +parentSessionPath? }`; `loadJsonlSessionMetadata` reads only this first line +(`jsonl-storage.ts:150-173`). Notably **absent** from the header: model, +title, and channel — those are carried elsewhere (see +[entry-types.md](entry-types.md) and +[channels-and-actors.md](channels-and-actors.md)). + +## The entry base + +Every non-header line shares `SessionTreeEntryBase` +(`harness/types.ts:353-364`): + +```ts +interface SessionTreeEntryBase { + type: string; // discriminator (see entry-types.md) + id: string; // entry id, unique within this file + parentId: string | null; // parent entry id; null for a root entry + timestamp: string; // ISO-8601 string (NB: differs from inner message.timestamp) + appendMode?: "side"; // this row advances the raw cursor, not the visible leaf +} +``` + +| Field | Shape | Req? | Notes | +|---|---|---|---| +| `type` | string | required | Entry discriminant. Ten known values; see [entry-types.md](entry-types.md). Unknown future values are tolerated. | +| `id` | string | required | **8-char prefix of a UUIDv7** (`uuidv7().slice(0,8)`, `storage-base.ts:35-43`), with collision retry. Time-sortable, **file-scoped only** — do not assume global uniqueness. | +| `parentId` | string \| null | required | Parent entry's `id`; `null` marks a root. The DAG backbone. | +| `timestamp` | ISO-8601 string | required | Entry time. Validated with `Date.parse`, but writers emit ISO-8601. **This is a string**; the inner `message.timestamp` is epoch-ms — see [messages.md](messages.md). | +| `appendMode` | `"side"` | optional | Marks a side-branch row that moves the raw append cursor without selecting a model-visible branch (`storage-base.ts:46-50`). | + +## The tree and the visible leaf + +`parentId` forms a tree/DAG. The active conversation is reconstructed by +walking from the current leaf to the root (`getPathToRoot`, +`storage-base.ts`). Anything not on the leaf's ancestry is a **dead end** — +which lines up exactly with toolpath's implicit dead-end model (steps not on +`path.head`'s ancestry). + +The **visible leaf is a separate pointer**, not simply "the last line." +It is maintained by `leaf` control rows (`LeafEntry`, +`harness/types.ts:435-440`): + +```json +{ "type": "leaf", "id": "0190ab14", "parentId": "0190ab13", + "timestamp": "2026-06-30T12:00:05.600Z", "targetId": "0190ab13" } +``` + +| Field | Shape | Notes | +|---|---|---| +| `targetId` | string \| null | The entry the branch currently points at (the visible head). | +| `appendParentId` | string \| null (optional) | Overrides the raw parent for the next append when it differs from the visible leaf. | + +So to find the live conversation you (1) replay the entries, tracking the +leaf via `leaf` rows and `appendMode:"side"`, then (2) walk `targetId → +root` over `parentId`. The helpers `leafIdUpdateAfterEntry` / +`appendParentIdAfterEntry` (`storage-base.ts:46-78`) encode the exact rules. + +## Robustness notes + +- **Blank lines are tolerated.** The reader splits on `\n` and filters empty + lines (`jsonl-storage.ts:184`); do not assume one record per physical line + with no gaps. +- **The inner message body is not validated on read.** `parseEntryLine` + validates only the envelope (`type`, `id`, `parentId`, `timestamp`, and + the `leaf`-specific fields), then casts the rest through + (`jsonl-storage.ts:108-147`). The `message`/content-block shapes in + [messages.md](messages.md) are the **producer** contract — trusted on + read, not runtime-checked. There is no Zod/TypeBox over persisted lines. +- **Unknown entry `type`s are skipped gracefully**, not fatal + (`storage-base.ts:46-69`) — OpenClaw expects to add entry types over time. diff --git a/docs/agents/formats/openclaw/known-issues.md b/docs/agents/formats/openclaw/known-issues.md new file mode 100644 index 00000000..7b789d6c --- /dev/null +++ b/docs/agents/formats/openclaw/known-issues.md @@ -0,0 +1,108 @@ +# Known issues and gotchas + +Format-level quirks, ambiguities, and things to defend against. Because we +have **no first-hand on-disk sample** yet (see +[README.md §Sourcing](README.md#sourcing-and-confidence)), some of these are +"from the producer type, not confirmed against a real file" — flagged where +so. + +## Two timestamp encodings per line + +A `message` entry carries an **ISO-8601 string** `timestamp` at the entry +level and an **epoch-milliseconds number** `timestamp` inside the message +object. Mixing them up yields 1970-era or unparseable dates. See +[messages.md](messages.md#the-two-timestamp-encodings). + +## The visible head is not the last line + +You cannot take "the last entry" as the conversation tip. The visible leaf +is a separate pointer moved by `leaf` rows, and `appendMode:"side"` rows +advance the raw cursor without changing the visible branch. Reconstruct the +conversation by tracking the leaf and walking `parentId` to the root, not by +file order. See +[jsonl-envelope.md](jsonl-envelope.md#the-tree-and-the-visible-leaf). + +## Entry ids are file-scoped and only 8 chars + +Entry `id`s are `uuidv7().slice(0,8)` — 8 hex chars, unique **within one +file** via collision retry, not globally. A consumer merging multiple +sessions must namespace ids by file; do not assume cross-file uniqueness. +The session id (the filename stem) is a full UUID — a different format from +entry ids. + +## Usage undercounts across a compaction + +Assistant `usage` at or under the latest compaction boundary is **zeroed** +on disk (`stripStaleAssistantUsageBeforeLatestCompaction`). Summing +transcript usage without accounting for compaction boundaries undercounts +the session total. See [usage.md](usage.md#compaction-zeroes-stale-usage). + +## No reasoning token count on disk + +The persisted `Usage` has input/output/cache/total only. Reasoning/thinking +**token counts** live solely in the runtime accumulator / stream events / +trajectory, never in the transcript — even though a `thinking` **content** +block is persisted. Don't try to populate a reasoning breakdown from the +transcript. See [usage.md](usage.md#reasoning-tokens-are-runtime-only). + +## No stored diffs + +File edits are tool calls with arguments; there is no unified diff or +before/after content in the transcript. A derived `Path` gets structural, +tool-input-derived changes only — no `raw` perspective. See +[tools.md](tools.md#file-operations-tool-input-only-no-raw-diff). + +## Group-sender identity is text-only + +In a group/channel session the key identifies the room, not the speaker. +The individual sender is injected as a `[from: Name (+E164)]` marker in the +prompt text; structured `senderId`/`senderName` are not persisted as +transcript fields. Per-message human identity in groups is best-effort +string parsing. See +[channels-and-actors.md](channels-and-actors.md#who-is-the-human). + +## `version` is hard-rejected if not 3 + +The reader throws on `version !== 3` (`jsonl-storage.ts:80-82`). A +forward-compatible consumer should treat a different version as "re-read +this reference," not silently parse it as v3. + +## The inner message body is unvalidated + +`parseEntryLine` validates only the envelope; the `message` object and its +content blocks are cast through unchecked (`jsonl-storage.ts:108-147`). The +shapes in [messages.md](messages.md) are the **producer** contract. A robust +reader should tolerate missing/extra fields inside `message` rather than +assume the documented shape holds for every line. + +## Blank lines between records + +The reader filters empty lines before parsing (`jsonl-storage.ts:184`). +Don't assume strictly one record per physical line with no gaps. + +## Legacy v1 logs get back-filled ids + +Old logs predating entry ids are migrated with synthetic `parentId` chains +(`migrateLegacySessionEntries`, `src/trajectory/export.ts:946-989`). Ids in +such files may be derived, not original. + +## Two code layers for one format + +The transcript is produced/consumed by both the agent-core harness storage +(`packages/agent-core/src/harness/session/jsonl-storage.ts`, types in +`harness/types.ts`: `SessionTreeEntry`, `MessageEntry`, `LeafEntry`, …) and +the gateway session manager (`src/agents/sessions/session-manager.ts`, +writer `src/config/sessions/transcript-jsonl.ts`: `SessionEntry`, +`SessionMessageEntry`, `CompactionEntry`, `FileEntry`). These appear to be +two views of the **same** on-disk JSONL, but they use different type names. +This reference uses the agent-core names. **Unconfirmed:** that the two +serialize byte-identically in every case — verify against a real file (and +against whichever layer actually writes the user's transcript) when the +`toolpath-openclaw` crate is built. + +## Daemon profile can relocate the whole store + +A non-default `OPENCLAW_PROFILE` moves everything to `~/.openclaw-` +via the daemon's own path resolver, which ignores `OPENCLAW_HOME`. If +sessions seem to be "missing," check for a profile. See +[directory-layout.md](directory-layout.md#daemon-path-divergence). diff --git a/docs/agents/formats/openclaw/lineage.md b/docs/agents/formats/openclaw/lineage.md new file mode 100644 index 00000000..1a6e6424 --- /dev/null +++ b/docs/agents/formats/openclaw/lineage.md @@ -0,0 +1,126 @@ +# Lineage and session kinds + +OpenClaw has lineage at **two scales** — within a session (the entry DAG) +and across sessions (forks, sub-agents, compaction branches) — plus a +session-kind classifier that separates user chats from cron/automation and +sub-agent runs. A toolpath derivation that wants the full provenance graph +must union all of these. + +## 1. Intra-session entry DAG + +Every entry carries `id` + `parentId` +([jsonl-envelope.md](jsonl-envelope.md#the-tree-and-the-visible-leaf)), +forming a tree/DAG inside one file. The leaf is found via `getLeafId()`; +branches are real (`getChildren`, `getTree`); side branches use +`appendMode:"side"`. Dead ends are entries not on the leaf's ancestry. + +This maps to toolpath's Step DAG **verbatim**: `id` → step id, `parentId` → +parent reference, leaf ancestry → `path.head`, off-ancestry entries → +implicit dead ends. + +> Legacy v1 logs that predate entry ids get synthetic `parentId` chains via +> `migrateLegacySessionEntries` (`src/trajectory/export.ts:946-989`). A +> reader ingesting old files should expect ids/parents to have been +> back-filled rather than original. + +## 2. Cross-session lineage (forks and sub-agents) + +Three independent primitives connect *sessions* to each other: + +### `parentSession` on the header + +The session header's optional `parentSession` is a **path to a parent +session file** ([jsonl-envelope.md](jsonl-envelope.md#the-header-line)). The +simplest cross-file fork edge. + +### `AcpSessionLineageMeta` + +`packages/acp-core/src/session-lineage-meta.ts:10-22`: + +```ts +type AcpSessionLineageMeta = { + sessionKey: string; + kind?: string; + channel?: string; + parentSessionId?: string; // = parentSessionKey ?? spawnedBy + spawnedBy?: string; + spawnDepth?: number; + subagentRole?: "orchestrator" | "leaf"; + subagentControlScope?: "children" | "none"; + spawnedWorkspaceDir?: string; + spawnedCwd?: string; +}; +``` + +`parentSessionKey` (new) / `spawnedBy` (legacy) is the parent-session +pointer; `parentSessionId` normalizes the two. `spawnDepth` and +`subagentRole` describe the sub-agent tree. This is persisted with the +session record (`SessionsPatchParamsSchema`, +`gateway-protocol/src/schema/sessions.ts:300-346`). + +### Sub-agent completion events + +Sub-agent results flow back via `AgentInternalEventSchema` +(`gateway-protocol/src/schema/agent.ts:741-758`): +`{ type:"task_completion", source:"subagent"|"cron"|"image_generation"|…, +childSessionKey, childSessionId, status }`. Stream events also carry +`spawnedBy`. + +> **Keys vs ids.** `spawnedBy` / `parentSessionKey` are session **keys**, +> while `childSessionId` / `originSessionId` are session **ids**. To stitch a +> cross-session DAG you need the key↔id resolver +> (`src/sessions/session-id-resolution.ts`). Sub-agent keys are also +> detectable from key shape (`isSubagentSessionKey`, +> `src/sessions/session-key-utils.ts`). + +**toolpath mapping:** a session-of-sessions DAG. Model the union as a +`Graph` of `Path`s, or as cross-`Path` parent edges, using +`parentSessionId`/`spawnedBy` for the edges and `subagentRole` to tell +orchestrators from leaves. + +## 3. Compaction as a new branch + +A `compaction` entry ([entry-types.md §compaction](entry-types.md#compaction)) +truncates history in place. But OpenClaw can also turn a compaction into a +**new session**: `sessions.compaction.branch` +(`SessionsCompactionBranchResultSchema`, +`gateway-protocol/src/schema/sessions.ts:462-478`) creates a **new session +key + new sessionId** from a checkpoint, with `sourceKey` linking back to +the predecessor. Checkpoints (`SessionCompactionCheckpointSchema`, +`sessions.ts:50-65`) record `tokensBefore`, `tokensAfter`, +`firstKeptEntryId`, and `preCompaction`/`postCompaction` transcript +references (`{ sessionId, sessionFile, leafId, entryId }`). + +**toolpath mapping:** a compaction branch is a new `Path` whose root has a +parent reference into the predecessor (`sourceKey` + +`preCompaction.entryId`). + +## Session kinds + +`classifySessionKind(key, entry)` (`src/sessions/classify-session-kind.ts`): + +```ts +type SessionKind = "cron" | "direct" | "group" | "global" | "spawn-child" | "unknown"; +``` + +Priority (most specific first): + +1. Sentinel keys `"global"` / `"unknown"`. +2. **`cron`** — `isCronSessionKey(key)` (key rest begins `cron:`; + `cron::run:` for a specific run). +3. **`spawn-child`** — `entry.spawnedBy` is set (checked *before* key shape + so ACP spawn-children with opaque keys aren't mislabeled `direct`). +4. **`group`** — `entry.chatType === "group"|"channel"`, or the key contains + `:group:` / `:channel:`. +5. Fallback **`direct`**. + +Complementary: `deriveSessionChatType` (`src/sessions/session-chat-type.ts`) +→ `direct|group|channel|…` from the key. Automation run-kind is also +signaled on the request (`AgentParamsSchema.bootstrapContextRunKind: +"default"|"heartbeat"|"cron"`, `acpTurnSource: "manual_spawn"`). + +**toolpath mapping:** a clean discriminator for a `Path` `kind`/`meta` tag — +`cron`/`spawn-child` are automation/sub-agent, `direct`/`group` are user +sessions, `global` is the agent-wide session. `spawn-child` detection +depends on `entry.spawnedBy` being populated; `subagentRole` refines it +further. diff --git a/docs/agents/formats/openclaw/messages.md b/docs/agents/formats/openclaw/messages.md new file mode 100644 index 00000000..ee2b831e --- /dev/null +++ b/docs/agents/formats/openclaw/messages.md @@ -0,0 +1,123 @@ +# Messages and content blocks + +A `message` entry wraps an `AgentMessage` +([entry-types.md §message](entry-types.md#message)). This doc covers the +role model, the content-block variants, and the two timestamp encodings +that bite naive readers. Tool-call/result linkage is in +[tools.md](tools.md); token usage on assistant messages is in +[usage.md](usage.md). + +The message types live in `packages/llm-core/src/types.ts`; the harness +extends them with extra roles in `packages/agent-core/src/types.ts`. + +## Roles + +`AgentMessage` is the LLM `Message` union (`user | assistant | toolResult`) +plus harness-only roles (`bashExecution`, and the read-time-only `custom` / +`branchSummary` / `compactionSummary`). On disk inside a `message` entry, +the realistic set is: + +| `role` | Meaning | +|---|---| +| `user` | Human / channel input. `content` is a bare string **or** an array of `text`/`image` blocks. | +| `assistant` | Model output. `content` is always a block array; carries `api`/`provider`/`model`/`usage`/`stopReason`. | +| `toolResult` | Result of a prior `toolCall`. Carries `toolCallId`, `toolName`, `content`, `isError`. A **separate** entry from the call. See [tools.md](tools.md). | +| `bashExecution` | Harness shell-execution message (distinct shape). | + +The `custom` / `branchSummary` / `compactionSummary` roles are +**reconstructed at read time** from `custom_message` / `branch_summary` / +`compaction` entries; you will not find them serialized as a `message` +entry's `role` (`session.ts:46-99`). + +## Content blocks + +Block discriminants are `text`, `thinking`, `image`, `toolCall`. There is +**no `tool_result` content block** — tool results are whole messages +(see [tools.md](tools.md)). + +### `text` (`llm-core/src/types.ts:226-230`) + +```json +{ "type": "text", "text": "I'll read x.ts first.", "textSignature": "…optional…" } +``` + +`textSignature` is optional; it may be a legacy id string or a +`TextSignatureV1` JSON blob `{ v:1, id, phase?: "commentary"|"final_answer" }`. + +### `thinking` (`llm-core/src/types.ts:233-241`) + +```json +{ "type": "thinking", "thinking": "Let me read the file", "thinkingSignature": "…opaque…", "redacted": false } +``` + +| Field | Shape | Notes | +|---|---|---| +| `thinking` | string | The reasoning text. | +| `thinkingSignature` | string (optional) | Opaque replay signature / reasoning-item id. | +| `redacted` | bool (optional) | Safety-redacted; the payload is kept in `thinkingSignature`. | + +Note: this is reasoning **content**. The reasoning **token count** is a +different thing and is *not* in the persisted usage — see +[usage.md](usage.md#reasoning-tokens-are-runtime-only). + +### `image` (`llm-core/src/types.ts:244-248`) + +```json +{ "type": "image", "data": "", "mimeType": "image/png" } +``` + +### `toolCall` (`llm-core/src/types.ts:251-258`) + +```json +{ "type": "toolCall", "id": "call_1", "name": "read_file", + "arguments": { "path": "src/x.ts" }, + "thoughtSignature": "…", "executionMode": "sequential" } +``` + +| Field | Shape | Notes | +|---|---|---| +| `id` | string | Correlates with the later `toolResult.toolCallId`. | +| `name` | string | Tool name. | +| `arguments` | object | Free-form JSON args (the only place file paths etc. appear; [tools.md](tools.md)). | +| `thoughtSignature` | string (optional) | Google-specific opaque thought-context signature. | +| `executionMode` | `"sequential"`\|`"parallel"` (optional) | Scheduling hint. | + +## Assistant message metadata + +`AssistantMessage` (`llm-core/src/types.ts:287-304`) carries, beyond +`content`: + +| Field | Shape | Notes | +|---|---|---| +| `api` | string | e.g. `"anthropic-messages"`, `"openai-responses"`. | +| `provider` | string | e.g. `"anthropic"`, `"openai"`. | +| `model` | string | Requested model. | +| `responseModel` | string (optional) | Concrete served model when it differs from requested (e.g. OpenRouter `auto`). | +| `responseId` | string (optional) | Provider response id. | +| `diagnostics` | array (optional) | `AssistantMessageDiagnostic[]`; serialized fields not confirmed against a sample. | +| `usage` | `Usage` | See [usage.md](usage.md). | +| `stopReason` | string | See below. | +| `errorMessage` / `errorCode` / `errorType` / `errorBody` | string (optional) | Error detail when the turn failed. | +| `timestamp` | int (epoch ms) | **Number, not ISO** — see below. | + +### `stopReason` + +Observed values: `stop`, `length`, `toolUse`, `error`, `aborted`. Treat +unknown values as forward-compatible (round-trip rather than reject). + +## The two timestamp encodings + +This is the single easiest thing to get wrong: + +- The **entry-level** `timestamp` (header and every entry) is an **ISO-8601 + string** (`new Date().toISOString()`). +- The **inner message** `timestamp` (`UserMessage` / `AssistantMessage` / + `ToolResultMessage`) is **epoch milliseconds (a number)** — + `timestamp: number; // Unix timestamp in milliseconds`. + +So a single `message` line has *both* a string entry-timestamp and a numeric +message-timestamp. Keep them in sync on round-trip. (The one exception, +`CompactionSummaryMessage.timestamp`, is typed `number | string` for +backward-compat, but that role isn't normally persisted as a `message` +entry.) This mirrors Pi's format exactly; see +[known-issues.md](known-issues.md). diff --git a/docs/agents/formats/openclaw/tools.md b/docs/agents/formats/openclaw/tools.md new file mode 100644 index 00000000..5a799af3 --- /dev/null +++ b/docs/agents/formats/openclaw/tools.md @@ -0,0 +1,85 @@ +# Tool calls, results, and file operations + +OpenClaw records a tool invocation as a `toolCall` content block inside an +assistant message, and the result as a **separate** `toolResult` message +entry. There is no inline result block and — importantly — **no stored +diff**. This shapes how a derived `Path` can represent file changes. + +## Call and result are separate entries + +The call is an assistant content block +([messages.md §toolCall](messages.md#toolcall-llm-coresrctypests251-258)): + +```json +{ "type": "toolCall", "id": "call_1", "name": "edit_file", + "arguments": { "path": "src/x.ts", "old": "…", "new": "…" } } +``` + +The result is its own `message` entry of role `toolResult` +(`llm-core/src/types.ts:306-314`): + +```json +{ "type": "message", "id": "0190ab13", "parentId": "0190ab12", + "timestamp": "2026-06-30T12:00:05.500Z", + "message": { "role": "toolResult", "toolCallId": "call_1", + "toolName": "edit_file", + "content": [ { "type": "text", "text": "edited 1 file" } ], + "isError": false, "timestamp": 1751284805500 } } +``` + +| Field | Shape | Notes | +|---|---|---| +| `toolCallId` | string | Links back to `toolCall.id`. The correlation key. | +| `toolName` | string | Tool name (mirrors the call's `name`). | +| `content` | `(text\|image)[]` | What the model sees as the result. | +| `details` | `unknown` (optional) | Arbitrary structured payload; per-tool, **unconstrained on the wire**. A tool *could* stash structured data (even a diff) here, but the core type defines nothing. | +| `isError` | bool | **The only error signal.** On failure, `isError:true` with the error text placed in `content`; there is no separate error-string field. | +| `timestamp` | int (epoch ms) | Inner message timestamp ([messages.md](messages.md#the-two-timestamp-encodings)). | + +This call-in-assistant / result-as-separate-entry split is the same design +as Pi, and different from Claude (`tool_use`/`tool_result` paired inside one +turn) and Gemini (inline result). A projector writing OpenClaw JSONL must +emit the `toolCall` block **and** a separate `toolResult` entry correlated +by id. + +## File operations: tool-input only, no raw diff + +**OpenClaw does not persist a structured patch or before/after content for +file edits.** A file edit is just a tool call whose `arguments` name the +path (and, for an edit tool, the old/new text). To know *what changed* you +must interpret each tool's argument schema, which is tool-specific. + +The diagnostic trajectory exporter confirms this: `buildTranscriptEvents` +(`src/trajectory/export.ts`) emits `tool.call` events as +`{ toolCallId, name, arguments, blockIndex }` straight from the assistant +content blocks — there is **no diff extraction** anywhere. + +The only structured "files touched" view is a **server-side derivation**, +not something stored on entries — `SessionFileEntrySchema` +(`gateway-protocol/src/schema/sessions.ts:78-89`), served by +`sessions.files.list`: + +```ts +{ path, name, kind: "modified" | "read", missing: boolean, size?, updatedAtMs?, content? } +``` + +It classifies each touched path as `modified` vs `read` (a browser rollup +adds `mixed`) but provides **no hunks** — at most the file's current +`content`, never a before/after pair. + +### Consequence for toolpath derivation + +A derived `Path` can carry **structural / tool-input-derived** file changes +only — there is **no `raw` (unified-diff) perspective** available from the +transcript. This is materially weaker than Codex (whose `patch_apply_end` +carries the diff or full content) and opencode (git tree↔tree diffs), and +parallels opencode's *fallback* behavior for gitignored paths. Recover the +touched-file list from `toolCall.arguments` (or `sessions.files.list` if the +gateway is queried), and mark changes structural with no raw perspective. + +## Tool name classification + +Tool names are free-form strings. A toolpath provider will want a +classifier (read / write / shell / search / network / delegation) keyed on +the lowercased `name`, the way `toolpath-pi` does — OpenClaw does not ship a +canonical category enum in the transcript. diff --git a/docs/agents/formats/openclaw/usage.md b/docs/agents/formats/openclaw/usage.md new file mode 100644 index 00000000..f0af5630 --- /dev/null +++ b/docs/agents/formats/openclaw/usage.md @@ -0,0 +1,97 @@ +# Token usage + +OpenClaw records token usage in **two different shapes**, and they don't +carry the same information. Getting this right matters for toolpath's +token-accounting invariants (kind v1.1.0): the persisted transcript is the +source of truth, but it lacks the reasoning breakdown. + +## Shape A: persisted per-message `Usage` (source of truth) + +Stored on every `AssistantMessage.usage` (`llm-core/src/types.ts:261-275`): + +```json +"usage": { + "input": 1200, "output": 340, "cacheRead": 0, "cacheWrite": 0, + "totalTokens": 1540, + "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0, "total": 0 } +} +``` + +| Field | Shape | Notes | +|---|---|---| +| `input` / `output` | int | Prompt / completion tokens for **this call**. | +| `cacheRead` / `cacheWrite` | int | Prompt-cache read / write tokens. | +| `totalTokens` | int | Headline total (= `Usage.total`). | +| `cost` | object | Per-class cost breakdown; provider-specific, often all-zero when cost can't be computed. | + +Key properties: + +- **Per-step delta, not cumulative.** One `Usage` per assistant message = + one provider call. The runtime accumulator sums them with `+=`, confirming + each is a per-call delta. So — unlike Codex's cumulative counters — you do + **not** difference these; a session total is `Σ` of per-message totals. +- **No reasoning/thinking field.** This shape has input/output/cache/total + only (see below). + +### `totalTokens` / `prompt_tokens` conventions + +`Usage.total` is `totalTokens`. OpenClaw's notion of `prompt_tokens` is +`input + cacheRead` (cacheWrite excluded). For toolpath, prefer summing the +raw `input`/`output`/`cacheRead`/`cacheWrite` independently rather than +trusting a single headline number, the way `toolpath-pi` does — it stays +correct regardless of convention drift. + +> **Do not confuse with `SessionEntry.totalTokens`.** A separate +> `deriveSessionTotalTokens` (`src/agents/usage.ts`) produces a +> prompt/context-size snapshot that **excludes output tokens**. That is a +> context-window gauge, not a turn cost. Only `AssistantMessage.usage` is a +> per-turn spend. + +## Compaction zeroes stale usage + +`stripStaleAssistantUsageBeforeLatestCompaction` +(`src/agents/compaction-usage.ts`) **zeroes** the `usage` on assistant +messages at or under the latest compaction boundary (via +`makeZeroUsageSnapshot`). So **summing transcript usage naively across a +compaction undercounts** — the pre-compaction turns may carry zeroed usage. +A reader reconstructing a session total must account for compaction +boundaries (see [entry-types.md §compaction](entry-types.md#compaction) and +[lineage.md](lineage.md)). + +## Reasoning tokens are runtime-only + +A richer `NormalizedUsage` exists at runtime (`src/agents/usage.ts:52-60`): + +```ts +type NormalizedUsage = { + input?; output?; cacheRead?; cacheWrite?; + reasoningTokens?: number; // the reasoning / thinking breakdown + total?; +}; +``` + +`normalizeUsage` maps ~20 provider aliases into these buckets and pulls +reasoning from `reasoning_tokens` / +`completion_tokens_details.reasoning_tokens` / +`output_tokens_details.reasoning_tokens`; cache-read is de-double-counted +out of OpenAI-style prompt totals. `UsageAccumulator` +(`src/agents/embedded-agent-runner/usage-accumulator.ts`) keeps both a +running total and the last call. + +But this richer usage is surfaced only on **chat stream events** and in the +**trajectory artifacts** (`src/trajectory/metadata.ts`) — it is **not** in +the persisted transcript `Usage`. A reasoning **content** block exists per +message ([messages.md §thinking](messages.md#thinking-llm-coresrctypests233-241)), +but not a reasoning **token count**. + +### Consequence for toolpath derivation + +- Per-step `attributed_token_usage` and per-session `token_usage` map + cleanly from `AssistantMessage.usage` (already a per-step delta — no + differencing). +- A reasoning `breakdowns["output"]["reasoning"]` sub-class **cannot** be + populated from the transcript — that data only exists in the runtime + accumulator / stream events / trajectory. Unlike Gemini/OpenCode/Codex + (which all expose a reasoning sub-count the projector can record), + OpenClaw records no per-step reasoning token count on disk. Omit the + breakdown rather than fabricate it. diff --git a/docs/agents/formats/openclaw/walkthrough.md b/docs/agents/formats/openclaw/walkthrough.md new file mode 100644 index 00000000..f4fd35d3 --- /dev/null +++ b/docs/agents/formats/openclaw/walkthrough.md @@ -0,0 +1,153 @@ +# A session, line by line + +This is a representative OpenClaw session read top to bottom, with each line +cross-linked back to the reference docs. Because we have **no first-hand +on-disk sample** yet, the JSON below is **reconstructed from the upstream +types**, not a captured fixture — ids and timestamps are invented and it is +illustrative only. Real files are one JSON object per physical line; lines +are pretty-printed here for readability. + +The scenario: a WhatsApp DM where the user asks the agent to fix a bug, the +model reads a file, and the context later compacts. + +--- + +## Line 1 — session header + +```json +{ "type": "session", "version": 3, + "id": "0190ab00-aaaa-7bbb-8ccc-000000000001", + "timestamp": "2026-06-30T12:00:00.000Z", + "cwd": "/home/u/proj", + "parentSession": null } +``` + +The first line is always the header +([jsonl-envelope.md](jsonl-envelope.md#the-header-line)). `version` must be +`3` or the reader rejects the file. `id` matches the filename stem; this is +a fresh session, so there's no `parentSession`. The session's channel/peer +is **not** here — it's in the routing key under which `sessions.json` +filed this file (`agent:main:whatsapp:direct:15555550123`, say); see +[channels-and-actors.md](channels-and-actors.md). + +## Line 2 — the user's message + +```json +{ "type": "message", "id": "0190ab10", "parentId": null, + "timestamp": "2026-06-30T12:00:01.000Z", + "message": { "role": "user", + "content": [ { "type": "text", "text": "Fix the bug in x.ts" } ], + "timestamp": 1751284801000 } } +``` + +A `message` entry ([entry-types.md §message](entry-types.md#message)) with +`role:"user"`. `parentId:null` makes it a root of the DAG. Note the **two +timestamps**: the entry's is an ISO string, the inner message's is epoch ms +([messages.md](messages.md#the-two-timestamp-encodings)). The human is the +WhatsApp peer from the key — there is no sender field on the message itself +([channels-and-actors.md](channels-and-actors.md#who-is-the-human)). + +## Line 3 — model selection + +```json +{ "type": "model_change", "id": "0190ab11", "parentId": "0190ab10", + "timestamp": "2026-06-30T12:00:01.500Z", + "provider": "anthropic", "modelId": "claude-..." } +``` + +A `model_change` marker ([entry-types.md §model_change](entry-types.md#model_change)). +It chains off the user message via `parentId`. The same model also appears +on each assistant message, so this is the explicit signal but not the only +record of it. + +## Line 4 — the assistant turn + +```json +{ "type": "message", "id": "0190ab12", "parentId": "0190ab11", + "timestamp": "2026-06-30T12:00:05.000Z", + "message": { "role": "assistant", + "content": [ + { "type": "thinking", "thinking": "Let me read the file", "thinkingSignature": "opaque..." }, + { "type": "text", "text": "I'll read x.ts first." }, + { "type": "toolCall", "id": "call_1", "name": "read_file", "arguments": { "path": "src/x.ts" } } ], + "api": "anthropic-messages", "provider": "anthropic", "model": "claude-...", + "usage": { "input": 1200, "output": 340, "cacheRead": 0, "cacheWrite": 0, "totalTokens": 1540, + "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0, "total": 0 } }, + "stopReason": "toolUse", "timestamp": 1751284805000 } } +``` + +One assistant message with **three content blocks in order**: a `thinking` +block, a `text` block, and a `toolCall` +([messages.md §content-blocks](messages.md#content-blocks)). The `usage` +here is this turn's spend, a per-step delta — sum these for a session total +([usage.md](usage.md#shape-a-persisted-per-message-usage-source-of-truth)). +The `thinking` text is present but its **token count is not** in `usage` +([usage.md](usage.md#reasoning-tokens-are-runtime-only)). `stopReason` is +`toolUse` because the turn ended on a tool call. + +## Line 5 — the tool result + +```json +{ "type": "message", "id": "0190ab13", "parentId": "0190ab12", + "timestamp": "2026-06-30T12:00:05.500Z", + "message": { "role": "toolResult", "toolCallId": "call_1", "toolName": "read_file", + "content": [ { "type": "text", "text": "" } ], + "isError": false, "timestamp": 1751284805500 } } +``` + +The result is a **separate** `message` entry of role `toolResult`, linked to +the call by `toolCallId == "call_1"` +([tools.md](tools.md#call-and-result-are-separate-entries)). Had the read +failed, `isError` would be `true` with the error text in `content` — there's +no dedicated error field. Note this was a *read*; had it been an edit, there +would still be **no diff** on disk, only the tool arguments +([tools.md](tools.md#file-operations-tool-input-only-no-raw-diff)). + +## Line 6 — the visible-leaf pointer + +```json +{ "type": "leaf", "id": "0190ab14", "parentId": "0190ab13", + "timestamp": "2026-06-30T12:00:05.600Z", "targetId": "0190ab13" } +``` + +A `leaf` control row ([entry-types.md §leaf](entry-types.md#leaf)) sets the +visible head to `0190ab13`. This is why you can't just take the last line as +the tip — the live branch is whatever the latest `leaf` points at, walked +back to root over `parentId` +([jsonl-envelope.md](jsonl-envelope.md#the-tree-and-the-visible-leaf)). + +## Line 7 — a compaction boundary + +```json +{ "type": "compaction", "id": "0190ab15", "parentId": "0190ab13", + "timestamp": "2026-06-30T12:10:00.000Z", + "summary": "## Goal\n...", "firstKeptEntryId": "0190ab12", + "tokensBefore": 54000, + "details": { "readFiles": [ "src/x.ts" ], "modifiedFiles": [] }, + "fromHook": false } +``` + +Later, context compacts ([entry-types.md §compaction](entry-types.md#compaction)). +Everything before `firstKeptEntryId` is replaced by `summary` at replay, and +assistant `usage` at/under this boundary is **zeroed on disk** — so a naive +sum across this point undercounts +([usage.md](usage.md#compaction-zeroes-stale-usage)). If this had instead +been turned into a *new* session via `sessions.compaction.branch`, you'd get +a fresh file whose root references this one +([lineage.md](lineage.md#3-compaction-as-a-new-branch)). + +--- + +## Reconstructing the conversation + +To turn this file into a linear transcript: + +1. Read line 1 for session metadata; reject if `version != 3`. +2. Replay lines 2..N, tracking the visible leaf via `leaf` rows and skipping + `appendMode:"side"` side-branch rows for the visible thread. +3. Walk the final `targetId` to the root over `parentId`; that ancestry is + the live conversation. Off-ancestry entries are dead ends. +4. Apply any `compaction` boundary: drop pre-`firstKeptEntryId` history in + favor of `summary`. +5. For the channel/peer actors, parse the routing key from `sessions.json`, + not the transcript. diff --git a/docs/superpowers/specs/2026-06-30-openclaw-format-docs-design.md b/docs/superpowers/specs/2026-06-30-openclaw-format-docs-design.md new file mode 100644 index 00000000..e93764bd --- /dev/null +++ b/docs/superpowers/specs/2026-06-30-openclaw-format-docs-design.md @@ -0,0 +1,301 @@ +# Design: OpenClaw on-disk format reference docs + +> **Status:** approved outline, pre-authoring +> **Date:** 2026-06-30 +> **Author:** Alex (with Claude) +> **Scope:** Write `docs/agents/formats/openclaw/` — a claude-code-style +> folder of focused reference docs for OpenClaw's on-disk session format. +> This is step one of the larger "add OpenClaw as a provider" effort; the +> `toolpath-openclaw` crate itself is **out of scope** for this spec. + +## Why + +We want to add OpenClaw (`github.com/openclaw/openclaw`) as a provider we +derive `toolpath` documents from, the way we already do for Claude Code, +Codex, Gemini, opencode, Cursor, and Pi. Every provider starts from an +empirical on-disk-format reference under `docs/agents/formats/`. This spec +defines that reference for OpenClaw so the eventual crate can be built +without re-reverse-engineering the format. + +## What OpenClaw is + +A local-first personal AI assistant — "Your own personal AI assistant. Any +OS. Any Platform. The lobster way. 🦞" It runs agent sessions locally and +bridges them to messaging channels (WhatsApp, Telegram, Slack, Discord, +Matrix, Signal, …). TypeScript/Node pnpm monorepo. This makes it different +from the other providers, which are all single-user coding-agent CLIs: an +OpenClaw "session" is scoped to an *agent persona × channel × peer/thread*, +and the human on the other end is a messaging contact, not a local shell +user. + +## Sourcing and provenance (read this — it differs from the other docs) + +- **Pinned to** `openclaw/openclaw @ 68c533cfb339cbb8650832cb2a4bf38dba7022fa` + (branch `main`), package version `2026.6.10` (CalVer), license + NOASSERTION (custom). +- **No first-hand on-disk sample.** OpenClaw is not installed on the + authoring machine, so unlike claude-code/codex/cursor we have **no real + session file** to inspect. Every claim is derived from **upstream source + code**, not observed bytes. +- **No `toolpath-openclaw` parser yet.** Unlike the other refs, we can't + cite our own tests/parser as corroboration. +- Therefore these docs are in the same posture `opencode.md` was written in: + "intended input for a future provider," compiled from upstream types. + Every doc's header must say so, and any field whose **serialized** form we + could not confirm from a real file must be flagged ("from the producer + type; not confirmed against a sample"). +- When the crate lands, revisit and upgrade claims from "from source" to + "observed" with a real sample, and add a `format-changelog.md` entry. + +### Reconciliation item (must be resolved while authoring) + +The session-transcript format is produced/consumed across **two code +layers** with slightly different type names. The docs must present one +coherent format and note the divergence: + +- **agent-core harness storage** — `packages/agent-core/src/harness/session/jsonl-storage.ts` + (+ `storage-base.ts`, `session.ts`), types in `packages/agent-core/src/harness/types.ts`: + `SessionHeader`, `SessionTreeEntry`, `MessageEntry`, `LeafEntry`, etc. +- **gateway session manager** — `src/agents/sessions/session-manager.ts` + (+ writer `src/config/sessions/transcript-jsonl.ts`): `SessionHeader`, + `SessionEntry`, `SessionMessageEntry`, `CompactionEntry`, `FileEntry`. + +Working hypothesis: these are two views of the **same on-disk JSONL** (the +gateway wraps/mirrors agent-core's storage). Authoring task: confirm they +serialize identically (same `type` strings, same field names) and, if they +diverge anywhere, document which one writes the file users actually have. +Cite agent-core types as the canonical shape unless proven otherwise. + +## Format synthesis (the durable research) + +All file:line citations below are at the pinned commit. + +### 1. Storage roots and layout + +- Root data dir precedence: `OPENCLAW_STATE_DIR` → existing `~/.openclaw` + → existing legacy `~/.clawdbot` → default `~/.openclaw`. The `~` itself is + resolved via `OPENCLAW_HOME` → `HOME`/`USERPROFILE`/Termux → `os.homedir()`. + No XDG, no per-OS special dir; identical layout on macOS/Linux/Windows. + (`src/config/paths.ts:209-273`, `src/infra/home-dir.ts:35-54`.) +- Config file: `OPENCLAW_CONFIG_PATH` → `/openclaw.json` + (legacy `clawdbot.json` tolerated). OAuth: `OPENCLAW_OAUTH_DIR` → + `/credentials/oauth.json`. +- Canonical transcript path: + `~/.openclaw/agents//sessions/.jsonl`, default + `agentId = "main"` (`src/config/sessions/paths.ts:653-661, 898-902`; + `src/routing/session-key.ts` `DEFAULT_AGENT_ID`). +- Filename variants in the same dir: `.jsonl`, + `-topic-.jsonl`, forked/rotated + `_.jsonl`. `` is a UUID; entry ids are + 8-char UUIDv7 prefixes. +- Index sidecar: `sessions.json` = `Record` + (`src/config/sessions/paths.ts:678-680`). Maps routing keys → files. +- **Not canonical:** `.trajectory.jsonl` + + `.trajectory-path.json` (runtime telemetry sidecars); + `~/.openclaw/state/openclaw.sqlite` and + `~/.openclaw/agents//agent/openclaw-agent.sqlite` (routing/auth/cron/ + queue/cache/embeddings — **no transcript table**; the one message-shaped + table, `acp_replay_events`, is a transient ACP replay buffer). +- Permissions: transcript/JSON files `0600`; SQLite dirs `0700`, files + `0600`. Daemon path divergence: `src/daemon/paths.ts:116-127` resolves its + own `~/.openclaw[-]` from `HOME`/`USERPROFILE` only (ignores + `OPENCLAW_HOME`); non-default `OPENCLAW_PROFILE` ⇒ `~/.openclaw-`. + +### 2. JSONL envelope and entry types + +- Line 1 = header `SessionHeader` (`jsonl-storage.ts:22-29`): + `{type:"session", version:3, id, timestamp (ISO), cwd, parentSession?}`. + Reader hard-rejects `version !== 3` (`jsonl-storage.ts:80-82`). + `parentSession` = path to a parent session file (cross-file chaining). +- Entry base `SessionTreeEntryBase` (`harness/types.ts:353-364`): + `{type:string, id, parentId:string|null, timestamp:ISO, appendMode?:"side"}`. +- 10 entry `type` values (`harness/types.ts:443-454`): `message`, + `thinking_level_change`, `model_change`, `compaction`, `branch_summary`, + `custom`, `custom_message`, `label`, `session_info`, `leaf`. Reader + tolerates unknown future types (`storage-base.ts:46-69`). +- **Tree, not a flat log.** `parentId` forms a DAG (`null` = root); the + active conversation is `leafId → root` (`storage-base.ts getPathToRoot`). + The visible head is a *separate* pointer maintained by `LeafEntry` + (`{type:"leaf", targetId, appendParentId?}`, `harness/types.ts:435-440`); + `appendMode:"side"` advances the raw cursor without selecting a visible + branch. Dead ends = entries not on the leaf's ancestry — maps directly to + toolpath's implicit dead-end model. +- Blank lines between records are tolerated/skipped + (`jsonl-storage.ts:184`). Inner `message` body is **not** schema-validated + on read — only the envelope is (`jsonl-storage.ts:108-147`). Entry ids are + file-scoped and only 8 hex chars. + +### 3. Messages and content blocks + +- `MessageEntry.message: AgentMessage`. On-disk roles realistically: + `user`, `assistant`, `toolResult`, `bashExecution`. (`custom`, + `branchSummary`, `compactionSummary` roles are *reconstructed at read* + from their own entry types, not stored as `message` entries — + `session.ts:46-99`, `messages.ts`.) +- Content blocks (`packages/llm-core/src/types.ts`): `text` + (`text`, `textSignature?`), `thinking` (`thinking`, `thinkingSignature?`, + `redacted?`), `image` (`data` base64, `mimeType`), `toolCall` + (`id`, `name`, `arguments`, `thoughtSignature?`, `executionMode?`). +- `UserMessage.content`: string OR `(text|image)[]`. `AssistantMessage`: + always a block array, plus `api`, `provider`, `model`, `responseModel?`, + `responseId?`, `diagnostics?`, `usage`, `stopReason`, error fields, + `timestamp` (epoch ms). +- **Two timestamp encodings (footgun):** entry-level `timestamp` is an ISO + string; inner `message.timestamp` is epoch-ms number. +- `stopReason` ∈ `stop|length|toolUse|error|aborted`. + +### 4. Tools and file operations + +- `toolCall` (assistant content block) ↔ `toolResult` (separate `message` + entry) linked by `toolResultMessage.toolCallId == toolCall.id`. + `ToolResultMessage` (`llm-core/src/types.ts:306-314`): `{toolCallId, + toolName, content:(text|image)[], details?:unknown, isError, timestamp}`. + Errors = `isError:true` with text in `content`; no separate error field. +- **No stored diffs.** File edits are just tool calls; recover a + *touched-files list* from tool-call `arguments`, not a patch. The + server-side `sessions.files.list` (`SessionFileEntrySchema`, + `gateway-protocol/.../sessions.ts:78-89`) classifies paths as + `modified|read` but has no hunks. → derived `Path` should carry + **structural / tool-input-derived** changes only, **no `raw` perspective** + (parallels opencode's gitignored fallback). + +### 5. Token usage (two shapes) + +- **Persisted (source of truth):** `Usage` on each `AssistantMessage` + (`llm-core/src/types.ts:261-275`): `{input, output, cacheRead, cacheWrite, + totalTokens, cost{…}}`. **Per-step delta**, not cumulative (accumulator + uses `+=`). **No reasoning field.** Compaction zeroes stale assistant + usage (`compaction-usage.ts stripStaleAssistantUsageBeforeLatestCompaction`) + — summing naively after a compaction undercounts. +- **Runtime only:** `NormalizedUsage` (`src/agents/usage.ts:52-60`) adds + `reasoningTokens`; surfaced on stream events and trajectory artifacts but + **not** in the persisted transcript. So a transcript reader gets + total/input/output/cache but **cannot** populate a reasoning + `breakdowns["output"]["reasoning"]` sub-class. +- Convention flags: `Usage.total` = `totalTokens`; OpenClaw's + `prompt_tokens` = input + cacheRead (cacheWrite excluded). A separate + `SessionEntry.totalTokens` / `deriveSessionTotalTokens` is a + prompt/context snapshot that **excludes output** — do not confuse with a + turn total. + +### 6. Lineage and session kinds + +- Three lineage mechanisms (union them): + - (a) **intra-session entry DAG** — `id`/`parentId`. → toolpath Step DAG + verbatim. + - (b) **cross-session fork / sub-agents** — `AcpSessionLineageMeta` + (`packages/acp-core/src/session-lineage-meta.ts:10-22`): + `parentSessionKey`/`spawnedBy`, `spawnDepth`, `subagentRole`, + `subagentControlScope`; `SessionHeader.parentSession`; sub-agent + completion `AgentInternalEventSchema` `task_completion` with + `childSessionKey`/`childSessionId`. → Graph-of-Paths / cross-Path edge. + - (c) **compaction → new branch** — `CompactionEntry` (`summary`, + `firstKeptEntryId`, `tokensBefore`, `fromHook`); `sessions.compaction.branch` + creates a new key+id with `sourceKey` back-link. → new Path with parent ref. + - Caveat: `spawnedBy`/`parentSessionKey` are session **keys**; + `childSessionId`/`originSessionId` are session **ids** — needs the + key↔id resolver (`session-id-resolution.ts`). +- `BranchSummaryEntry` (`fromId`, `summary`, `details:{readFiles, + modifiedFiles}`, `fromHook`) marks an abandoned branch. +- Session kinds (`classify-session-kind.ts`): `cron | direct | group | + global | spawn-child | unknown`. → Path `kind`/`meta` tag. + +### 7. Channels and actors (the OpenClaw-specific axis) + +- **No `sender` field on transcript messages.** Human identity is + structural in the session **key** (`buildAgentPeerSessionKey`, + `src/routing/session-key.ts:222-273`): + `agent::::` (+ DM-scope and + `:thread:` variants). `channel` ∈ whatsapp/telegram/slack/discord/ + matrix/signal/…; `peerKind` ∈ direct/dm/group/channel. +- DM: `peerId` *is* the human's channel user id. Group/channel: `peerId` is + the room id; **individual sender is text-only** — `[from: Name (+E164)]` + markers injected into prompt text (`docs/channels/group-messages.md`). + Structured `senderId`/`senderName`/`pushName` exist at the inbound layer + but are flattened into text, not persisted as transcript fields. +- `InputProvenance` (`src/sessions/input-provenance.ts:14-21`): + `{kind:"external_user"|"inter_session"|"internal_system", + originSessionId?, sourceSessionKey?, sourceChannel?, sourceTool?}` — the + closest structured who/where on a user message. +- Agent identity: per-`AssistantMessage` `provider`/`model`/`api`; + `model_change` entries; agent persona `agentId`/name/avatar/emoji. +- Closest `type:name` actor strings: human `:` (DM) or + `group:` (+`[from:]` parse); agent `agent:` or + `:`. **Decision for the crate (not this doc):** which + axis is `agent:name` — the persona or the model. + +## Doc folder structure (12 files) + +Mirror the claude-code house style: each doc opens with a dated provenance +header; field tables show the JSON name / shape / optional|required; +"Observed" vs (here, mostly) "From source"; keep headings anchor-stable +(no em-dashes in linkable headings). + +1. `README.md` — overview, sourcing/provenance header, doc map (ordered + reading list), conventions, field index table, maintenance note. +2. `directory-layout.md` — §1 above. +3. `jsonl-envelope.md` — header + entry base + tree/`leaf`/`appendMode` + mechanics + blank-line tolerance + no-inner-validation. +4. `entry-types.md` — the 10 `type` variants in detail, each with a field + table. +5. `messages.md` — roles + content blocks; the dual-timestamp note; + `stopReason` values. +6. `tools.md` — `toolCall`↔`toolResult` correlation; `isError`; file-ops + story (tool-input only, no raw diff, `sessions.files.list`). +7. `usage.md` — persisted `Usage` vs runtime `NormalizedUsage`; per-step + deltas; reasoning-only-at-runtime; compaction zeroing; `totalTokens` + conventions + `deriveSessionTotalTokens` gotcha. +8. `lineage.md` — three lineage mechanisms; `BranchSummaryEntry`; session + kinds; key↔id resolution. +9. `channels-and-actors.md` — §7 above: session-key grammar, channel list, + DM vs group identity, `[from:]` markers, `InputProvenance`, persona vs + model. +10. `known-issues.md` — dual timestamps; file-scoped 8-char ids; usage + undercount after compaction; no raw diff; group-sender text-only; + `version==3` hard-reject; legacy v1 migration + (`migrateLegacySessionEntries`); the two-code-layer reconciliation. +11. `walkthrough.md` — a linear, annotated example session (header → user + message → model_change → assistant turn with thinking/text/toolCall → + toolResult → leaf → compaction), cross-linking each line back to the + reference docs. Example is **reconstructed from the types** and clearly + labeled as illustrative, not a captured fixture. +12. `format-changelog.md` — seeded with "version 3 (observed in source at + `2026.6.10`)"; a place to record future field/behavior drift. + +Also update `docs/agents/formats/README.md` to add an `openclaw/` entry in +the Contents list (after `gemini.md`/`opencode.md`, keeping the existing +ordering style). + +## Conventions to follow + +- Field tables: name as it appears in JSON (camelCase envelope keys like + `parentId`; `snake_case`/provider-style where the source uses it), + shape, optional|required (here: "from source" rather than "observed"). +- Mark every unconfirmed serialized field explicitly. +- Cite upstream as `openclaw/openclaw path:line @ 68c533cf`; cite our future + crate only once it exists. +- Keep linkable headings free of em-dashes and decorative punctuation. + +## Authoring checklist (ordered) + +1. `directory-layout.md`, `jsonl-envelope.md`, `entry-types.md` (the + structural core). +2. `messages.md`, `tools.md`, `usage.md` (record content). +3. `lineage.md`, `channels-and-actors.md` (the OpenClaw-specific axes). +4. `known-issues.md`, `walkthrough.md`, `format-changelog.md`. +5. `README.md` last (its field index and doc map reference the others). +6. Update `docs/agents/formats/README.md` Contents. +7. While authoring, resolve the two-code-layer reconciliation item by + re-reading `jsonl-storage.ts` + `session-manager.ts` + `transcript-jsonl.ts` + and confirming identical serialization. + +## Out of scope (future, separate spec) + +- The `toolpath-openclaw` crate (reader, provider, `derive_path` mapping, + projector, CLI wiring under `path p import/export openclaw`, + `share`/`resume` integration, tests). +- All `CLAUDE.md` / `Cargo.toml` / `site/_data/crates.json` / release-script + updates that come with a new crate. +- Decisions the docs only *surface* (persona vs model as `agent:name`; + how to represent multi-channel human actors in a `type:name` string). diff --git a/site/_data/crates.json b/site/_data/crates.json index abf80299..a9e3efa9 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -9,7 +9,7 @@ }, { "name": "toolpath-convo", - "version": "0.11.0", + "version": "0.12.0", "description": "Provider-agnostic conversation types, traits, and Toolpath-Path derivation", "docs": "https://docs.rs/toolpath-convo", "crate": "https://crates.io/crates/toolpath-convo", From b3333b9108c83612fcf630e4aa1e02043b26be31 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:08:38 -0400 Subject: [PATCH 02/22] docs(openclaw): implementation plan for toolpath-openclaw crate --- .../plans/2026-06-30-toolpath-openclaw.md | 496 ++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-30-toolpath-openclaw.md diff --git a/docs/superpowers/plans/2026-06-30-toolpath-openclaw.md b/docs/superpowers/plans/2026-06-30-toolpath-openclaw.md new file mode 100644 index 00000000..9237f04a --- /dev/null +++ b/docs/superpowers/plans/2026-06-30-toolpath-openclaw.md @@ -0,0 +1,496 @@ +# toolpath-openclaw Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `toolpath-openclaw` provider crate that derives toolpath `Path` documents from OpenClaw agent-session JSONL (forward) and projects/incepts a `Path` back into an OpenClaw on-disk session (reverse), wired into the `path` CLI like the other providers. + +**Architecture:** Mirror `toolpath-pi` (the closest structural twin: v3 header + `id`/`parentId` entry tree + `toolCall` blocks + separate `toolResult` messages + dual ISO/epoch timestamps). The forward path is purely native→`ConversationView`; the provider-agnostic `toolpath_convo::derive_path` builds the step DAG and token grouping. OpenClaw's one novel axis — multi-channel human identity — is handled by a small additive change to `toolpath-convo` (`DeriveConfig.user_actor`) plus channel/peer metadata, because an OpenClaw session is single-peer so the human actor is session-level. + +**Tech Stack:** Rust (edition 2024), `serde`/`serde_json`, `chrono`, `thiserror`, `anyhow`; deps `toolpath` + `toolpath-convo`. Test helper: `tempfile`. + +## Global Constraints + +- Rust edition 2024; workspace pinned to 1.94.0. Build clean under `cargo clippy --workspace -- -D warnings`. +- New crate version `0.1.0`. Update all 11 new-crate checklist sites (Phase 9). +- Provider id / `meta.source` string: **`openclaw`**. Path kind: `toolpath::v1::PATH_KIND_AGENT_CODING_SESSION`. +- Format reference of record: `docs/agents/formats/openclaw/` (v3, sourced from `openclaw/openclaw @ 68c533cf`). +- No first-hand OpenClaw install: all fixtures are **synthesized** from the documented v3 format and committed under `crates/toolpath-openclaw/tests/fixtures/` and `test-fixtures/openclaw/`. +- Commits are signed (`git commit -S`); in this background session signing may need the user's 1Password — accumulate and hand off if it fails. +- Actor strings: assistant `agent:` (shared default), human session-level `human:/` for DMs, `human:/group/` for groups, fallback `human:user`. + +--- + +## File Structure + +New crate `crates/toolpath-openclaw/`: + +| File | Responsibility | +|---|---| +| `Cargo.toml` | Package + deps (mirror pi). | +| `README.md` | Crate doc (wired via `#![doc = include_str!("../README.md")]`). | +| `src/lib.rs` | `OpenClawConvo` manager + re-exports. | +| `src/error.rs` | `OpenClawError` + `Result`. | +| `src/types.rs` | v3 JSONL schema: `SessionHeader`, `Entry` (10 variants), `AgentMessage`, `ContentBlock`, `Usage`, `StopReason`, `EntryBase`. | +| `src/paths.rs` | `PathResolver`: state-dir resolution, agent sessions dir, `sessions.json` index, session-id resolution + routing-key lookup. | +| `src/reader.rs` | `OpenClawSession`, `read_session_from_file`, parentSession chaining, leaf/tree assembly, `main_thread`. | +| `src/io.rs` | Lightweight listing: `list_agents`, `list_sessions`, `first_user_message`, header peek. | +| `src/provider.rs` | `session_to_view`, `classify_tool`, `native_name`, channel/peer parsing, `impl ConversationProvider`. | +| `src/derive.rs` | `derive_path`/`derive_graph` wrappers: compute `user_actor`, inject `meta.extra["openclaw"]`. | +| `src/project.rs` | `OpenClawProjector` (ConversationView→`OpenClawSession`) + `sessions.json` inception entry. | +| `tests/fixtures/*.jsonl` + `tests/*.rs` | Synthesized fixtures + reader/provider/roundtrip integration tests. | + +Shared change: `crates/toolpath-convo/src/derive.rs` (+ its tests). + +CLI wiring (Phase 8): `crates/path-cli/src/{cmd_import,cmd_list,cmd_show,cmd_share,cmd_resume,cmd_export}.rs` + `crates/path-cli/Cargo.toml`. + +Checklist (Phase 9): root `Cargo.toml`, `CLAUDE.md`, `README.md`, `site/_data/crates.json`, `site/pages/crates.md`, `scripts/release.sh`, the `docs/agents/formats/openclaw/` caveats + changelog. + +--- + +## Phase 0: Shared IR — session-level `user_actor` + +### Task 0: Add `DeriveConfig.user_actor` and honor it in `actor_for_turn` + +**Files:** +- Modify: `crates/toolpath-convo/src/derive.rs` (`DeriveConfig` struct, `Default`, `actor_for_turn` ~476, call site ~118) +- Test: `crates/toolpath-convo/src/derive.rs` (`#[cfg(test)]`) + +**Interfaces:** +- Produces: `DeriveConfig.user_actor: Option`; `actor_for_turn(turn, provider, user_actor: Option<&str>) -> String`. + +- [ ] **Step 1: Failing test** — in derive.rs tests: + +```rust +#[test] +fn user_actor_override_sets_human_actor() { + let view = view_with(vec![base_turn("t1", Role::User)]); + let cfg = DeriveConfig { user_actor: Some("human:whatsapp/15555550123".into()), ..Default::default() }; + let path = derive_path(&view, &cfg); + assert_eq!(path.steps[0].step.actor, "human:whatsapp/15555550123"); +} + +#[test] +fn user_actor_default_is_human_user() { + let view = view_with(vec![base_turn("t1", Role::User)]); + let path = derive_path(&view, &DeriveConfig::default()); + assert_eq!(path.steps[0].step.actor, "human:user"); +} +``` + +- [ ] **Step 2: Run, expect FAIL** (`cargo test -p toolpath-convo user_actor`) — `user_actor` field missing. + +- [ ] **Step 3: Implement.** Add to `DeriveConfig` (keep `Default` derive): + +```rust +/// Session-level actor string for user turns (e.g. a channel-aware +/// `human:whatsapp/`). When `None`, user turns get the default +/// `human:user`. Used by providers (OpenClaw) whose human is a known +/// per-session identity rather than the local shell user. +pub user_actor: Option, +``` + +Change the call site (~118) from `actor_for_turn(turn, provider)` to `actor_for_turn(turn, provider, config.user_actor.as_deref())` and: + +```rust +fn actor_for_turn(turn: &Turn, provider: &str, user_actor: Option<&str>) -> String { + match &turn.role { + Role::User => user_actor.map(str::to_string).unwrap_or_else(|| "human:user".to_string()), + Role::Assistant => { + let model = turn.model.as_deref().unwrap_or("unknown"); + format!("agent:{}", model) + } + Role::System | Role::Other(_) => format!("tool:{}", provider), + } +} +``` + +- [ ] **Step 4: Run, expect PASS.** Also run full `cargo test -p toolpath-convo` to confirm no regressions (all existing `DeriveConfig{..}` callers use `..Default::default()` or the struct's Default, so they're unaffected — verify). + +- [ ] **Step 5: Bump `toolpath-convo` version** (minor: additive public field). Update `crates/toolpath-convo/Cargo.toml`, root `Cargo.toml` `[workspace.dependencies]`, `site/_data/crates.json`, `CHANGELOG.md`. + +- [ ] **Step 6: Commit** `feat(convo): session-level user_actor override in DeriveConfig`. + +--- + +## Phase 1: Crate scaffold + types + +### Task 1: Crate skeleton + +**Files:** Create `crates/toolpath-openclaw/Cargo.toml`, `src/lib.rs`, `src/error.rs`, `README.md`. + +- [ ] **Step 1:** `Cargo.toml` — copy pi's, rename to `toolpath-openclaw`, version `0.1.0`, description "Derive Toolpath provenance documents from OpenClaw agent-session logs", keywords `["openclaw","provenance","toolpath","audit","ai"]`. Deps: `toolpath`, `toolpath-convo`, `anyhow`, `chrono`, `serde`, `serde_json`, `thiserror`; dev-dep `tempfile`. +- [ ] **Step 2:** `src/error.rs` — mirror pi's `PiError` as `OpenClawError` (`Io`, `Json`, `SessionNotFound`, `AgentNotFound`, `InvalidSessionFile`, `MalformedHeader`, `UnsupportedVersion(u32)`, `Convo`, `Other`) + `Result` alias. `thiserror`. +- [ ] **Step 3:** `src/lib.rs` — `#![doc = include_str!("../README.md")]`, `pub mod {error,types,paths,reader,io,provider,derive,project};`, re-exports placeholder (filled later). Minimal `README.md` (expand in Phase 9). +- [ ] **Step 4:** Add `crates/toolpath-openclaw` to root `Cargo.toml` `members` + a `[workspace.dependencies]` entry so it builds. `cargo build -p toolpath-openclaw` (empty modules → may need stub `pub fn` or empty files; create empty module files now). +- [ ] **Step 5: Commit** `feat(openclaw): crate skeleton`. + +### Task 2: `types.rs` — v3 schema + serde round-trip + +**Files:** Create `crates/toolpath-openclaw/src/types.rs`. Reference: [`docs/agents/formats/openclaw/{jsonl-envelope,entry-types,messages,usage}.md`] and pi `types.rs`. + +**Interfaces — Produces:** +- `SessionHeader { type_="session", version:u32, id:String, timestamp:String, cwd:String, parent_session:Option, #[serde(flatten)] extra }`. +- `EntryBase { id:String, parent_id:Option, timestamp:String, append_mode:Option }` (flattened into entries). +- `Entry` enum tagged by `type`: `Message{base, message:AgentMessage}`, `ModelChange{base, provider, model_id}`, `ThinkingLevelChange{base, thinking_level}`, `Compaction{base, summary, first_kept_entry_id, tokens_before, details:Option, from_hook:Option}`, `BranchSummary{base, from_id, summary, details, from_hook}`, `Custom{base, custom_type, data:Option}`, `CustomMessage{base, custom_type, content, details, display:bool}`, `Label{base, target_id, label:Option}`, `SessionInfo{base, name:Option}`, `Leaf{base, target_id:Option, append_parent_id:Option}`. Unknown `type` → tolerant `Other` variant (serde `#[serde(other)]` or untagged fallback) so future entry types don't break parsing. +- `AgentMessage` tagged by `role`: `User{content:Content, timestamp:i64}`, `Assistant{content:Vec, api, provider, model, response_model:Option, usage:Usage, stop_reason:StopReason, error_*:Option, timestamp:i64}`, `ToolResult{tool_call_id, tool_name, content:Vec, details:Option, is_error:bool, timestamp:i64}`, `BashExecution{...}`. +- `Content` = `enum { Text(String), Blocks(Vec) }` (untagged) for the user string-or-array case. +- `ContentBlock` tagged by `type`: `Text{text, text_signature:Option}`, `Thinking{thinking, thinking_signature:Option, redacted:Option}`, `Image{data, mime_type}`, `ToolCall{id, name, arguments:Value, thought_signature:Option, execution_mode:Option}`. +- `Usage { input:u32, output:u32, cache_read:u32, cache_write:u32, total_tokens:u32, cost:Option }`. +- `StopReason` enum with `Other(String)` fallback for `stop|length|toolUse|error|aborted`. + +- [ ] **Step 1: Failing tests** — serde round-trip for header + one of each entry type. Example: + +```rust +#[test] +fn header_roundtrip() { + let j = r#"{"type":"session","version":3,"id":"s1","timestamp":"2026-06-30T12:00:00Z","cwd":"/p"}"#; + let h: SessionHeader = serde_json::from_str(j).unwrap(); + assert_eq!(h.version, 3); + assert_eq!(serde_json::from_str::(&serde_json::to_string(&h).unwrap()).unwrap().id, "s1"); +} + +#[test] +fn assistant_message_with_blocks_roundtrip() { + let j = r#"{"type":"message","id":"e1","parentId":"e0","timestamp":"2026-06-30T12:00:05Z", + "message":{"role":"assistant","content":[ + {"type":"thinking","thinking":"hm"}, + {"type":"text","text":"hi"}, + {"type":"toolCall","id":"c1","name":"read_file","arguments":{"path":"x"}}], + "api":"anthropic-messages","provider":"anthropic","model":"claude-x", + "usage":{"input":1,"output":2,"cacheRead":0,"cacheWrite":0,"totalTokens":3}, + "stopReason":"toolUse","timestamp":1751284805000}}"#; + let e: Entry = serde_json::from_str(j).unwrap(); + match e { Entry::Message{message: AgentMessage::Assistant{content, usage, ..}, ..} => { + assert_eq!(content.len(), 3); assert_eq!(usage.total_tokens, 3); + } _ => panic!("wrong variant") } +} + +#[test] +fn user_content_string_or_array() { + let s: AgentMessage = serde_json::from_str(r#"{"role":"user","content":"hi","timestamp":1}"#).unwrap(); + let a: AgentMessage = serde_json::from_str(r#"{"role":"user","content":[{"type":"text","text":"hi"}],"timestamp":1}"#).unwrap(); + // both deserialize; text extraction yields "hi" +} + +#[test] +fn leaf_and_compaction_roundtrip() { /* leaf targetId/appendParentId; compaction firstKeptEntryId/tokensBefore/details */ } + +#[test] +fn unknown_entry_type_tolerated() { + let e: Entry = serde_json::from_str(r#"{"type":"future_thing","id":"e","parentId":null,"timestamp":"t"}"#).unwrap(); + matches!(e, Entry::Other{..}); +} +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement** `types.rs` with `#[serde(rename_all = "camelCase")]`, `#[serde(tag = "type")]` on `Entry`/`ContentBlock`, `#[serde(tag = "role")]` on `AgentMessage`, `#[serde(untagged)]` on `Content`, `#[serde(other)]` `Other` on `Entry`/`StopReason`. Flatten `EntryBase` into each entry variant. Add `text()`/`thinking()`/`tool_calls()` helper methods on `AgentMessage` (mirror pi). +- [ ] **Step 4: Run, expect PASS.** +- [ ] **Step 5: Commit** `feat(openclaw): v3 JSONL schema types + serde round-trip`. + +--- + +## Phase 2: paths.rs + +### Task 3: State-dir resolution + session-id resolution + routing key + +**Files:** Create `crates/toolpath-openclaw/src/paths.rs`. Reference: [`docs/agents/formats/openclaw/directory-layout.md`]. + +**Interfaces — Produces:** +- `PathResolver { home: Option, state_dir: PathBuf }` with `new()` (reads `OPENCLAW_STATE_DIR`→existing `~/.openclaw`→existing `~/.clawdbot`→`~/.openclaw`; `~` via `OPENCLAW_HOME`→`HOME`→`USERPROFILE`), `with_state_dir(p)`, `with_home(p)`, `agent_sessions_dir(agent_id:&str) -> PathBuf` (default agent `"main"`), `list_agent_ids() -> io::Result>`. +- `SessionsIndex` = parsed `sessions.json`: `BTreeMap`; `load(dir) -> Option`, `routing_key_for(session_id) -> Option<(String /*key*/, ParsedKey)>`. +- `ParsedKey { agent_id, channel:Option, peer_kind:Option, peer_id:Option, thread_id:Option }` from `parse_session_key("agent:main:whatsapp:group:123")`. +- `resolve_session_file(agent_id, session_id) -> Result` (match `sessions.json` sessionId, then filename stems `.jsonl` / `_.jsonl` / `-topic-*.jsonl`). +- `DEFAULT_AGENT_ID: &str = "main"`. + +- [ ] **Step 1: Failing tests** (temp dirs): + +```rust +#[test] +fn resolves_state_dir_from_env() { + let tmp = tempfile::tempdir().unwrap(); + let r = PathResolver::new_with_env(Some(tmp.path()), None); // test seam: explicit state-dir + assert_eq!(r.state_dir(), tmp.path()); + assert_eq!(r.agent_sessions_dir("main"), tmp.path().join("agents/main/sessions")); +} + +#[test] +fn parses_session_key_dm_and_group() { + let dm = parse_session_key("agent:main:whatsapp:direct:15555550123"); + assert_eq!(dm.channel.as_deref(), Some("whatsapp")); + assert_eq!(dm.peer_kind.as_deref(), Some("direct")); + assert_eq!(dm.peer_id.as_deref(), Some("15555550123")); + let grp = parse_session_key("agent:main:slack:group:T42"); + assert_eq!(grp.peer_kind.as_deref(), Some("group")); + assert_eq!(grp.peer_id.as_deref(), Some("T42")); + let main = parse_session_key("agent:main:main"); + assert!(main.channel.is_none()); +} + +#[test] +fn resolve_session_file_by_stem_and_index() { /* write a sessions/ dir with .jsonl + sessions.json; resolve both ways */ } +``` + +- [ ] **Step 2: Run, expect FAIL.** +- [ ] **Step 3: Implement.** Provide a test seam (`new_with_env` or builder) so tests don't touch real `$HOME`. `parse_session_key`: split on `:`, first seg `agent`, second `agentId`; if 3rd seg is `main`/`direct` handle DM-scope forms; else `channel:peerKind:peerId` with optional trailing `thread:`. Be lenient (return `None` parts on unexpected shapes). +- [ ] **Step 4: Run, expect PASS.** +- [ ] **Step 5: Commit** `feat(openclaw): path + session-key resolution`. + +--- + +## Phase 3: reader.rs + io.rs + +### Task 4: `reader.rs` — parse session, build tree, follow parentSession, main thread + +**Files:** Create `crates/toolpath-openclaw/src/reader.rs`. Create fixtures `tests/fixtures/dm_session.jsonl` (header + user + model_change + assistant[thinking,text,toolCall] + toolResult + leaf + compaction) and `tests/fixtures/sessions.json` (key→file). Reference: [`docs/agents/formats/openclaw/walkthrough.md`] (its lines are a ready fixture). + +**Interfaces — Produces:** +- `OpenClawSession { header:SessionHeader, entries:Vec, file_path:PathBuf, parent:Option>, session_key:Option, parsed_key:Option }`. +- `read_session_from_file(&Path) -> Result` (parse header line, reject `version != 3` → `UnsupportedVersion`, skip blank lines, tolerant entry parse). +- `read_session_with_parent(&Path, max_depth) -> Result` (follow `parentSession`). +- `OpenClawSession::main_thread(&self) -> Vec<&Entry>` (resolve visible leaf via last `Leaf.target_id`, else last message entry; walk `parentId → root`, reverse). +- `attach_routing_key(&mut self, resolver)` (lookup `sessions.json` in the file's dir; set `session_key`/`parsed_key`). + +- [ ] **Step 1: Failing tests:** + +```rust +#[test] +fn reads_dm_fixture() { + let s = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + assert_eq!(s.header.version, 3); + assert!(s.entries.iter().any(|e| matches!(e, Entry::Message{..}))); +} +#[test] +fn rejects_non_v3() { /* version:2 fixture → UnsupportedVersion */ } +#[test] +fn main_thread_follows_leaf_to_root() { + let s = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + let thread = s.main_thread(); + // first entry is the root user message; tool result present; ordering root→leaf + assert!(matches!(thread.first(), Some(Entry::Message{..}))); +} +``` + +- [ ] **Step 2: FAIL → Step 3: implement** (mirror pi `reader.rs`; the only real delta is `Leaf`-row-driven head selection in `main_thread` instead of pi's newest-leaf assumption). → **Step 4: PASS.** +- [ ] **Step 5: Commit** `feat(openclaw): session reader + tree assembly`. + +### Task 5: `io.rs` — listing + first_user_message + +**Files:** Create `crates/toolpath-openclaw/src/io.rs`. Mirror pi `io.rs`. + +**Interfaces — Produces:** `list_sessions(resolver, agent_id) -> Result>`, `list_all_sessions(resolver) -> Result>` (across agents), `SessionMeta { id, timestamp, file_path, entry_count, first_user_message, cwd:Option, session_key:Option }`, `extract_first_user_message(&Path)`. + +- [ ] **Step 1: Failing tests** — list the fixtures dir; assert one `SessionMeta` with `first_user_message == "Fix the bug in x.ts"` and `cwd` from header. **2: FAIL → 3: implement** (`extract_first_user_message` walks lines for `type=="message"`, `message.role=="user"`, content string-or-array text). **4: PASS. → 5: Commit** `feat(openclaw): session listing + metadata`. + +--- + +## Phase 4: provider.rs (forward) + +### Task 6: `session_to_view` + tool classification + channel/peer + +**Files:** Create `crates/toolpath-openclaw/src/provider.rs`. Reference: pi `provider.rs`; [`docs/agents/formats/openclaw/{tools,usage,channels-and-actors,lineage}.md`]. + +**Interfaces — Produces:** +- `session_to_view(&OpenClawSession) -> ConversationView` (two-pass: turns then tool-result correlation; `provider_id = Some("openclaw")`). +- `classify_tool(name:&str) -> Option` and `native_name(category, args) -> Option<&'static str>` (OpenClaw tool vocabulary; lenient like pi). +- `pub const PROVIDER_ID: &str = "openclaw";` +- `user_actor_for(parsed: Option<&ParsedKey>) -> Option` (DM → `human:/`; group/channel → `human:/group/`; else `None`). +- `openclaw_meta_extra(session) -> serde_json::Map` (`channel`, `peerKind`, `peerId`, `sessionKey`, `sessionKind`, `agentId`). + +Mapping rules (the deltas from pi): +1. `Entry::Message` → `Turn { id: base.id, parent_id: base.parent_id, role, text, thinking, tool_uses, model (assistant), token_usage: Some(usage→TokenUsage) for assistant, timestamp: entry ISO, stop_reason }`. Each assistant message is its own step → set `token_usage` directly (no `group_id`; no `attributed_token_usage`). +2. `toolCall` blocks → `ToolInvocation{ id, name, input: arguments, category: classify_tool(name), result: None }`; record `(turn_idx, tool_idx)` by call id. +3. `Entry::Message{ToolResult}` → pass-2 correlate by `tool_call_id`; set `inv.result = Some(ToolResult{ content: joined text, is_error })`. Don't emit a standalone turn. +4. Write/edit tool calls → push `FileMutation{ path: extract_path(args), tool_id: Some(call_id), operation: Some(op_from_name), raw_diff: None, before: None, after: None, rename_to: None }`. **No raw perspective.** +5. `Entry::Compaction`/`BranchSummary` → synthetic `Role::System` turn carrying `text = summary` (and stash markers — drop if not needed). `ModelChange`/`ThinkingLevelChange`/`Label`/`Leaf`/`Custom` → drop from turns (Leaf only affects ordering). +6. Delegations: OpenClaw sub-agents are cross-session (not inline) → v1 leaves `delegations` empty; lineage captured in meta (Phase 5). Note as a known gap. +7. `ConversationView`: `id = header.id`, `session_ids` = chain, `base = SessionBase{ working_dir: header.cwd, ... }`, `total_usage` summed, `files_changed` from file_mutations. + +- [ ] **Step 1: Failing tests** (against `dm_session.jsonl` with `attach_routing_key`): + +```rust +#[test] +fn view_has_user_assistant_and_tool_result() { + let s = load_fixture_with_key(); + let v = session_to_view(&s); + assert_eq!(v.provider_id.as_deref(), Some("openclaw")); + let asst = v.turns.iter().find(|t| t.role == Role::Assistant).unwrap(); + assert!(asst.tool_uses[0].result.is_some()); // correlated + assert_eq!(asst.tool_uses[0].category, Some(ToolCategory::FileRead)); + assert!(asst.token_usage.is_some()); +} +#[test] +fn user_actor_from_dm_key() { + assert_eq!(user_actor_for(Some(&parse_session_key("agent:main:whatsapp:direct:155"))).as_deref(), + Some("human:whatsapp/155")); +} +#[test] +fn edit_tool_emits_structural_file_mutation_without_raw() { + // assistant with an edit_file toolCall {path} → FileMutation{path, raw_diff:None} +} +``` + +- [ ] **Step 2: FAIL → Step 3: implement** (mirror pi two-pass; add channel/file-mutation/usage deltas). → **Step 4: PASS.** +- [ ] **Step 5: Commit** `feat(openclaw): session_to_view + tool/file/usage mapping`. + +--- + +## Phase 5: derive.rs + lib.rs manager + +### Task 7: `derive.rs` — config wrappers with channel-aware actor + meta extra + +**Files:** Create `crates/toolpath-openclaw/src/derive.rs`. Reference pi `derive.rs`. + +**Interfaces — Produces:** +- `DeriveConfig` (re-export `toolpath_convo::DeriveConfig`). +- `derive_path(&OpenClawSession, &DeriveConfig) -> toolpath::v1::Path`: build view, set `cfg.user_actor = user_actor_for(session.parsed_key)`, call `toolpath_convo::derive_path`, then inject `meta.extra["openclaw"] = openclaw_meta_extra(session)` and ensure `meta.source == "openclaw"`, `meta.kind == PATH_KIND_AGENT_CODING_SESSION`. +- `derive_graph(&[OpenClawSession], title, &DeriveConfig) -> Graph`. + +- [ ] **Step 1: Failing test:** + +```rust +#[test] +fn derive_sets_channel_actor_kind_and_meta() { + let s = load_fixture_with_key(); // key agent:main:whatsapp:direct:155 + let p = derive_path(&s, &DeriveConfig::default()); + let user_step = p.steps.iter().find(|s| s.step.actor.starts_with("human:")).unwrap(); + assert_eq!(user_step.step.actor, "human:whatsapp/155"); + let meta = p.meta.unwrap(); + assert_eq!(meta.source.as_deref(), Some("openclaw")); + assert_eq!(meta.kind.as_deref(), Some(PATH_KIND_AGENT_CODING_SESSION)); + assert_eq!(meta.extra["openclaw"]["channel"], "whatsapp"); +} +``` + +- [ ] **2: FAIL → 3: implement → 4: PASS. → 5: Commit** `feat(openclaw): derive_path with channel-aware actor + meta`. + +### Task 8: `lib.rs` — `OpenClawConvo` manager + ConversationProvider + +**Files:** Modify `crates/toolpath-openclaw/src/lib.rs` + `provider.rs` (`impl ConversationProvider`). + +**Interfaces — Produces:** `OpenClawConvo { resolver }` with `new()`, `with_resolver(r)`, `list_agents()`, `list_sessions(agent_id)`, `read_session(agent_id, session_id) -> Result` (attaches routing key), `most_recent_session(agent_id)`, `to_view(&session)`. `impl ConversationProvider for OpenClawConvo` (project param = agentId). + +- [ ] **Step 1: Failing test** — `OpenClawConvo::with_resolver(PathResolver::with_state_dir(tmp)).read_session("main", "")` returns the session; `to_view` non-empty. **2: FAIL → 3: implement → 4: PASS → 5: Commit** `feat(openclaw): OpenClawConvo manager + ConversationProvider`. + +--- + +## Phase 6: project.rs (reverse / inception) + +### Task 9: `OpenClawProjector` — ConversationView → OpenClawSession + sessions.json + +**Files:** Create `crates/toolpath-openclaw/src/project.rs`. Reference: pi `project.rs`, [`docs/agents/adding-a-projector.md`], [`docs/agents/formats/openclaw/{jsonl-envelope,entry-types,messages,tools}.md`]. + +**Interfaces — Produces:** +- `OpenClawProjector { default_api, default_provider, cwd:Option, agent_id:String, channel:Option, peer_id:Option }` + builders. +- `impl ConversationProjector` → `OpenClawSession` (header from `cwd`/env, entries: per user turn a `Message{User}`; per assistant turn a `Message{Assistant}` with `thinking`+`text`+`toolCall` blocks AND, for each tool result, a separate `Message{ToolResult}` entry; `parentId` chain preserved from `Turn.parent_id`/order; append a final `Leaf` row pointing at the last entry). Drop foreign data; remap tool names via `category`+`native_name(args)` when source name isn't OpenClaw-native. +- `write_session(&OpenClawSession, dir, agent_id) -> Result` — writes `.jsonl` (0600) **and** upserts a `sessions.json` routing entry `agent:::direct:` (or `agent::main` when no channel), so an OpenClaw instance can route the incepted session. + +- [ ] **Step 1: Failing tests** — round-trip contract: + +```rust +#[test] +fn roundtrip_preserves_messages_tools_usage() { + let src = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + let view = session_to_view(&src); + let projected = OpenClawProjector::default().project(&view).unwrap(); + // header version 3; user+assistant+toolResult entries present; toolCall id matches toolResult.toolCallId + assert_eq!(projected.header.version, 3); + // serialize each entry → reparse via Entry: lossless for text/role/toolCall/usage +} +#[test] +fn inception_writes_session_and_routing_entry() { + let tmp = tempfile::tempdir().unwrap(); + let view = session_to_view(&read_session_from_file(...).unwrap()); + let proj = OpenClawProjector::default().with_channel("whatsapp").with_peer("155"); + let path = proj.write_session(&proj.project(&view).unwrap(), tmp.path(), "main").unwrap(); + assert!(path.exists()); + let idx: serde_json::Value = serde_json::from_reader(File::open(tmp.path().join("sessions.json")).unwrap()).unwrap(); + assert!(idx.as_object().unwrap().keys().any(|k| k.contains("whatsapp"))); +} +``` + +- [ ] **2: FAIL → 3: implement → 4: PASS → 5: Commit** `feat(openclaw): projector + sessions.json inception`. + +### Task 10: Round-trip integration test + +**Files:** Create `crates/toolpath-openclaw/tests/projection_roundtrip.rs`. Walk `fixture → to_view → derive_path → serialize+reparse Path → extract_conversation → project → OpenClawSession`; assert field-by-field (messages, roles, content text, toolCall name/args/result/error, usage, channel meta survives on the Path). + +- [ ] **1: write test → 2: run (FAIL where gaps) → 3: fix provider/projector → 4: PASS → 5: Commit** `test(openclaw): full projection round-trip`. + +--- + +## Phase 7: CLI — import / list / show + +### Task 11: `p import openclaw` + +**Files:** Modify `crates/path-cli/Cargo.toml` (add `toolpath-openclaw` dep in the native target deps block), `crates/path-cli/src/cmd_import.rs`. Reference: pi/opencode variants. + +**Interfaces — Produces:** `ImportSource::Openclaw { agent:Option, session:Option, all:bool, base:Option }`; `derive_openclaw(...)`, `derive_openclaw_session(agent, session, base) -> DerivedDoc`, `pick_openclaw(...)`; cache id `make_id("openclaw", &doc_inner_id(&doc))`. + +- [ ] **Step 1:** Add enum variant + match arm + `derive_openclaw` (mirror `derive_pi`; "project"→agentId, default `main`; `--all` → Graph). Picker `pick_openclaw` lists via `OpenClawConvo`. +- [ ] **Step 2:** Integration test in `cmd_import.rs::tests` — temp state-dir with a fixture session, `derive_openclaw(Some("main"), Some(""), Some(base))` returns a `Path` with `meta.source=="openclaw"` and full message count. +- [ ] **Step 3: Run → PASS. Step 4: Commit** `feat(cli): path p import openclaw`. + +### Task 12: `p list openclaw` + `show openclaw` + +**Files:** Modify `crates/path-cli/src/cmd_list.rs`, `cmd_show.rs`. + +- [ ] **Step 1:** `ListSource::Openclaw { agent:Option, base:Option }` + `run_openclaw` (tsv: `agent\tsession_id\trender_row(...,first_user_message)`). `ShowSource::Openclaw { agent:String, session:String, base:Option }` + `derive_one` arm (mirror pi). +- [ ] **Step 2:** tests: list emits a tsv row; show returns markdown-able Path. **Step 3: PASS → Commit** `feat(cli): path p list/show openclaw`. + +--- + +## Phase 8: CLI — share / resume / export + +### Task 13: `share` integration + +**Files:** Modify `crates/path-cli/src/cmd_share.rs`. + +- [ ] **Step 1:** Add `Openclaw` to `HarnessArg`, `Harness` (`name()="openclaw"`, `symbol()="openclaw"`, `project_keyed()=false` — rank by header cwd, `parse("openclaw")`), `HarnessBundle.openclaw` + `from_environment`, and `collect_openclaw` (list all sessions across agents; `SessionRow{ harness:Openclaw, cwd: Some(header_cwd), session_id, title: first_user_message, matches_cwd }`). Wire into `gather_sessions`. +- [ ] **Step 2:** test: `collect_openclaw` over a temp state-dir yields a row with `matches_cwd` true when header cwd == canonical cwd. **Step 3: PASS → Commit** `feat(cli): openclaw in path share`. + +### Task 14: `resume` source recognition + `p export openclaw` + +**Files:** Modify `crates/path-cli/src/cmd_resume.rs`, `cmd_export.rs`. + +- [ ] **Step 1 (resume):** `infer_source_harness`: add `"openclaw" => Some(Harness::Openclaw)` and actor fallback `agent:openclaw`/`human:` … but **OpenClaw is not an interactive resume target** (no `--resume` CLI). In the resume harness picker, exclude Openclaw from selectable targets (or map it to "incept-only"): document that resuming an openclaw-sourced Path defaults to another harness; OpenClaw inception is via `p export openclaw --project`. Add a guard returning a clear error if a user forces `--harness openclaw` for resume. +- [ ] **Step 2 (export):** `ExportTarget::Openclaw { input, agent:Option, project:Option, output:Option }`; `build_openclaw_conversation(path) -> OpenClawSession` shared by three modes: `--project DIR` writes resume-ready layout under `DIR/.openclaw`-style or the real `~/.openclaw` (use `OpenClawProjector::write_session`, reading channel/peer from `path.meta.extra["openclaw"]`); `--output FILE` writes the `.jsonl`; neither → stdout. Mirror the Claude/Gemini three-mode dispatch. +- [ ] **Step 3:** integration test in `cmd_export.rs::tests`: export a fixture Path with `--project tmp`; assert the written `.jsonl` is readable by `OpenClawConvo`/`read_session_from_file` under the same session id, and `sessions.json` got a routing entry. **Step 4: PASS → Commit** `feat(cli): path p export openclaw + resume source recognition`. + +--- + +## Phase 9: New-crate checklist + docs + verification + +### Task 15: Workspace + metadata wiring + +**Files:** root `Cargo.toml` (members + `[workspace.dependencies]` `toolpath-openclaw = { version="0.1.0", path="crates/toolpath-openclaw" }`), `CLAUDE.md` (repo layout, dep graph, satellite cross-dep line, provider notes, test counts), `README.md` (crate line), `site/_data/crates.json` (full entry), `site/pages/crates.md` (diagram), `scripts/release.sh` (`_all_crates` Tier-2 + dep-order comment), `crates/toolpath-openclaw/README.md` (full), `lib.rs` re-exports. + +- [ ] **Step 1:** Make all edits (copy pi/opencode entries as templates). **Step 2:** `cargo build --workspace` + `cargo clippy --workspace -- -D warnings` clean. **Step 3:** `cd site && pnpm run build` (expect page count +0 or as appropriate). **Step 4: Commit** `chore(openclaw): wire crate into workspace + site + release`. + +### Task 16: Format-doc upgrade + changelog + +**Files:** `docs/agents/formats/openclaw/{README,format-changelog,known-issues}.md`. + +- [ ] **Step 1:** Now that a parser exists, upgrade the "no crate yet / source-only" caveats: cite `crates/toolpath-openclaw` as corroboration where the parser confirms a field; add a `format-changelog.md` maintenance note; resolve the "two code layers" reconciliation item with whatever the parser proved. **Step 2: Commit** `docs(openclaw): cross-link the new toolpath-openclaw crate`. + +### Task 17: Final verification + +- [ ] **Step 1:** `cargo test --workspace` green; record per-crate test counts; update `CLAUDE.md` Testing section with `toolpath-openclaw` counts. +- [ ] **Step 2 (live-ish):** project a real Claude session into OpenClaw format and validate shape: + +```bash +cargo run -q -p path-cli -- p import claude --project "$PWD" --session --no-cache --pretty > /tmp/src.path.json +cargo run -q -p path-cli -- p export openclaw --input /tmp/src.path.json --output /tmp/out.jsonl +# assert: /tmp/out.jsonl line 1 is a v3 session header; every line re-parses via Entry; tool names remapped +``` + +- [ ] **Step 3:** `path p validate` any derived doc; confirm `meta.kind`/`source`. **Step 4: Commit** `test(openclaw): workspace test pass + live projection check`. + +--- + +## Self-Review + +- **Spec coverage:** forward (Tasks 4–8), reverse/inception (9–10, 14), channel-aware actor (0, 6–7), CLI import/list/show/share/resume/export (11–14), new-crate checklist (15), docs (16), verification (17). ✓ +- **Decisions honored:** build both derive+project ✓; channel-aware human actors via `DeriveConfig.user_actor` ✓; inception via projector+sessions.json ✓. +- **Known gaps (documented, not silent):** no `raw` diff perspective (Task 6.4); cross-session sub-agent delegations not inlined in v1 (Task 6.6); OpenClaw not an interactive resume target (Task 14.1). +- **Type consistency:** `session_to_view`/`user_actor_for`/`PROVIDER_ID`/`OpenClawConvo`/`OpenClawProjector`/`PathResolver`/`OpenClawSession` used consistently across tasks. +- **Placeholder scan:** none (mechanical mirrors explicitly point at the pi file + the documented delta). From 10b90b8553b074ddd8206ac1313a7854ef7bf617 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:10:44 -0400 Subject: [PATCH 03/22] feat(openclaw): crate skeleton (Cargo, error types, module stubs) --- Cargo.toml | 4 +- crates/toolpath-openclaw/Cargo.toml | 21 +++ crates/toolpath-openclaw/README.md | 22 ++++ crates/toolpath-openclaw/src/derive.rs | 3 + crates/toolpath-openclaw/src/error.rs | 158 +++++++++++++++++++++++ crates/toolpath-openclaw/src/io.rs | 3 + crates/toolpath-openclaw/src/lib.rs | 13 ++ crates/toolpath-openclaw/src/paths.rs | 3 + crates/toolpath-openclaw/src/project.rs | 3 + crates/toolpath-openclaw/src/provider.rs | 6 + crates/toolpath-openclaw/src/reader.rs | 3 + crates/toolpath-openclaw/src/types.rs | 3 + 12 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 crates/toolpath-openclaw/Cargo.toml create mode 100644 crates/toolpath-openclaw/README.md create mode 100644 crates/toolpath-openclaw/src/derive.rs create mode 100644 crates/toolpath-openclaw/src/error.rs create mode 100644 crates/toolpath-openclaw/src/io.rs create mode 100644 crates/toolpath-openclaw/src/lib.rs create mode 100644 crates/toolpath-openclaw/src/paths.rs create mode 100644 crates/toolpath-openclaw/src/project.rs create mode 100644 crates/toolpath-openclaw/src/provider.rs create mode 100644 crates/toolpath-openclaw/src/reader.rs create mode 100644 crates/toolpath-openclaw/src/types.rs diff --git a/Cargo.toml b/Cargo.toml index 5e4e8e3d..593ce60d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,9 +9,10 @@ members = [ "crates/toolpath-codex", "crates/toolpath-opencode", "crates/toolpath-cursor", + "crates/toolpath-pi", + "crates/toolpath-openclaw", "crates/toolpath-dot", "crates/toolpath-md", - "crates/toolpath-pi", "crates/pathbase-client", "crates/path-cli", ] @@ -35,6 +36,7 @@ toolpath-github = { version = "0.6.0", path = "crates/toolpath-github" } toolpath-dot = { version = "0.5.0", path = "crates/toolpath-dot" } toolpath-md = { version = "0.7.0", path = "crates/toolpath-md" } toolpath-pi = { version = "0.6.0", path = "crates/toolpath-pi" } +toolpath-openclaw = { version = "0.1.0", path = "crates/toolpath-openclaw" } path-cli = { version = "0.14.0", path = "crates/path-cli" } pathbase-client = { version = "0.2.0", path = "crates/pathbase-client" } diff --git a/crates/toolpath-openclaw/Cargo.toml b/crates/toolpath-openclaw/Cargo.toml new file mode 100644 index 00000000..2dfb05d3 --- /dev/null +++ b/crates/toolpath-openclaw/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "toolpath-openclaw" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository = "https://github.com/empathic/toolpath" +description = "Derive Toolpath provenance documents from OpenClaw agent-session logs" +keywords = ["openclaw", "provenance", "toolpath", "audit", "ai"] +categories = ["development-tools"] + +[dependencies] +toolpath = { workspace = true } +toolpath-convo = { workspace = true } +anyhow = { workspace = true } +chrono = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/toolpath-openclaw/README.md b/crates/toolpath-openclaw/README.md new file mode 100644 index 00000000..f6834524 --- /dev/null +++ b/crates/toolpath-openclaw/README.md @@ -0,0 +1,22 @@ +# toolpath-openclaw + +Read [OpenClaw](https://github.com/openclaw/openclaw) agent-session logs and +derive [Toolpath](https://toolpath.net) provenance documents. + +OpenClaw is a local-first, multi-channel personal AI assistant. It stores +each session as an append-only JSONL transcript at +`~/.openclaw/agents//sessions/.jsonl` (format version 3: +a session header line followed by an `id`/`parentId` tree of entries). + +This crate implements the [`toolpath_convo::ConversationProvider`] trait and +a `derive_path` wrapper that produces a [`toolpath::v1::Path`], plus an +[`crate::project::OpenClawProjector`] that projects a `Path` back into an +OpenClaw session on disk (inception). + +Because an OpenClaw session is scoped to one channel peer, the human actor +is recovered from the `sessions.json` routing key (`agent:::…`) +and set as a channel-aware `human:/` actor; the channel, +peer, and session-kind metadata ride on `path.meta.extra["openclaw"]`. + +The on-disk format is documented in detail at +[`docs/agents/formats/openclaw/`](https://github.com/empathic/toolpath/tree/main/docs/agents/formats/openclaw). diff --git a/crates/toolpath-openclaw/src/derive.rs b/crates/toolpath-openclaw/src/derive.rs new file mode 100644 index 00000000..cfc8e18b --- /dev/null +++ b/crates/toolpath-openclaw/src/derive.rs @@ -0,0 +1,3 @@ +//! Thin `derive_path` / `derive_graph` wrappers over `toolpath_convo`. +//! +//! Filled in Phase 5, Task 7. diff --git a/crates/toolpath-openclaw/src/error.rs b/crates/toolpath-openclaw/src/error.rs new file mode 100644 index 00000000..25a25bbd --- /dev/null +++ b/crates/toolpath-openclaw/src/error.rs @@ -0,0 +1,158 @@ +//! Error types for `toolpath-openclaw`. + +use std::path::PathBuf; +use thiserror::Error; + +/// Errors produced by the `toolpath-openclaw` crate. +#[derive(Debug, Error)] +pub enum OpenClawError { + /// Underlying I/O failure. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + /// JSON (de)serialization failure. + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + /// A session file was expected but could not be located. + #[error("session not found: {0}")] + SessionNotFound(String), + + /// An agent directory (`agents//sessions`) was expected but not + /// found on disk. + #[error("agent not found: {0}")] + AgentNotFound(String), + + /// A session JSONL file exists but cannot be interpreted. + /// + /// Carries the offending path and a short human-readable reason. + #[error("invalid session file {path}: {reason}")] + InvalidSessionFile { + /// Path to the offending file. + path: PathBuf, + /// Short human-readable reason. + reason: String, + }, + + /// A session header line was present but malformed (missing required + /// fields, unexpected shape, etc.). + #[error("malformed session header: {0}")] + MalformedHeader(String), + + /// The session header declares a format version this crate does not + /// understand. OpenClaw's reader hard-rejects anything but version 3. + #[error("unsupported session format version: {0} (expected 3)")] + UnsupportedVersion(u32), + + /// Wrapped error from `toolpath-convo`. + #[error("conversation error: {0}")] + Convo(#[from] toolpath_convo::ConvoError), + + /// Catch-all for arbitrary `anyhow` errors bubbling up from dependencies. + #[error("{0}")] + Anyhow(#[from] anyhow::Error), + + /// Generic free-form error. + #[error("{0}")] + Other(String), +} + +impl OpenClawError { + /// Construct a `SessionNotFound` error. + pub fn session_not_found(id: impl Into) -> Self { + Self::SessionNotFound(id.into()) + } + + /// Construct an `AgentNotFound` error. + pub fn agent_not_found(agent_id: impl Into) -> Self { + Self::AgentNotFound(agent_id.into()) + } + + /// Construct an `InvalidSessionFile` error. + pub fn invalid_session_file(path: impl Into, reason: impl Into) -> Self { + Self::InvalidSessionFile { + path: path.into(), + reason: reason.into(), + } + } + + /// Construct a `MalformedHeader` error. + pub fn malformed_header(reason: impl Into) -> Self { + Self::MalformedHeader(reason.into()) + } + + /// Construct an `Other` error. + pub fn other(msg: impl Into) -> Self { + Self::Other(msg.into()) + } +} + +/// Convenience result alias. +pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + use std::io; + + #[test] + fn io_error_conversion() { + let io_err = io::Error::new(io::ErrorKind::NotFound, "missing"); + let err: OpenClawError = io_err.into(); + assert!(matches!(err, OpenClawError::Io(_))); + } + + #[test] + fn json_error_display() { + let json_err = serde_json::from_str::("x").unwrap_err(); + let err: OpenClawError = json_err.into(); + assert!(err.to_string().to_lowercase().contains("json")); + } + + #[test] + fn session_not_found_display() { + assert!( + OpenClawError::SessionNotFound("abc".into()) + .to_string() + .contains("abc") + ); + } + + #[test] + fn agent_not_found_display() { + assert!( + OpenClawError::agent_not_found("main") + .to_string() + .contains("main") + ); + } + + #[test] + fn unsupported_version_display() { + let msg = OpenClawError::UnsupportedVersion(2).to_string(); + assert!(msg.contains('2') && msg.contains('3')); + } + + #[test] + fn invalid_session_file_display() { + let err = OpenClawError::invalid_session_file(PathBuf::from("/tmp/a.jsonl"), "bad line 3"); + let msg = err.to_string(); + assert!(msg.contains("/tmp/a.jsonl") && msg.contains("bad line 3")); + } + + #[test] + fn helper_constructors() { + assert!(matches!( + OpenClawError::session_not_found("s"), + OpenClawError::SessionNotFound(_) + )); + assert!(matches!( + OpenClawError::agent_not_found("a"), + OpenClawError::AgentNotFound(_) + )); + assert!(matches!( + OpenClawError::other("o"), + OpenClawError::Other(_) + )); + } +} diff --git a/crates/toolpath-openclaw/src/io.rs b/crates/toolpath-openclaw/src/io.rs new file mode 100644 index 00000000..d368ac08 --- /dev/null +++ b/crates/toolpath-openclaw/src/io.rs @@ -0,0 +1,3 @@ +//! Lightweight session listing and metadata extraction. +//! +//! Filled in Phase 3, Task 5. diff --git a/crates/toolpath-openclaw/src/lib.rs b/crates/toolpath-openclaw/src/lib.rs new file mode 100644 index 00000000..7d79acc9 --- /dev/null +++ b/crates/toolpath-openclaw/src/lib.rs @@ -0,0 +1,13 @@ +#![doc = include_str!("../README.md")] +#![warn(missing_docs)] + +pub mod derive; +pub mod error; +pub mod io; +pub mod paths; +pub mod project; +pub mod provider; +pub mod reader; +pub mod types; + +pub use error::{OpenClawError, Result}; diff --git a/crates/toolpath-openclaw/src/paths.rs b/crates/toolpath-openclaw/src/paths.rs new file mode 100644 index 00000000..ceab5363 --- /dev/null +++ b/crates/toolpath-openclaw/src/paths.rs @@ -0,0 +1,3 @@ +//! On-disk layout + state-directory + session-id resolution. +//! +//! Filled in Phase 2, Task 3. diff --git a/crates/toolpath-openclaw/src/project.rs b/crates/toolpath-openclaw/src/project.rs new file mode 100644 index 00000000..57e3005d --- /dev/null +++ b/crates/toolpath-openclaw/src/project.rs @@ -0,0 +1,3 @@ +//! Projector: `ConversationView` -> OpenClaw session JSONL (inception). +//! +//! Filled in Phase 6, Task 9. diff --git a/crates/toolpath-openclaw/src/provider.rs b/crates/toolpath-openclaw/src/provider.rs new file mode 100644 index 00000000..5a692a8d --- /dev/null +++ b/crates/toolpath-openclaw/src/provider.rs @@ -0,0 +1,6 @@ +//! `ConversationView` derivation (`session_to_view`) + tool classification. +//! +//! Filled in Phase 4, Task 6. + +/// Provider identifier written to `path.meta.source`. +pub const PROVIDER_ID: &str = "openclaw"; diff --git a/crates/toolpath-openclaw/src/reader.rs b/crates/toolpath-openclaw/src/reader.rs new file mode 100644 index 00000000..9e8b11dc --- /dev/null +++ b/crates/toolpath-openclaw/src/reader.rs @@ -0,0 +1,3 @@ +//! Session reading: parse JSONL, assemble the entry tree, follow parents. +//! +//! Filled in Phase 3, Task 4. diff --git a/crates/toolpath-openclaw/src/types.rs b/crates/toolpath-openclaw/src/types.rs new file mode 100644 index 00000000..66d7357d --- /dev/null +++ b/crates/toolpath-openclaw/src/types.rs @@ -0,0 +1,3 @@ +//! OpenClaw v3 session-JSONL schema (header + entry tree + content blocks). +//! +//! Filled in Phase 1, Task 2. From e65c2b6971fcf48f9670cb1952e909366b410470 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:14:58 -0400 Subject: [PATCH 04/22] feat(openclaw): v3 JSONL schema types + serde round-trip --- crates/toolpath-openclaw/src/types.rs | 758 +++++++++++++++++++++++++- 1 file changed, 756 insertions(+), 2 deletions(-) diff --git a/crates/toolpath-openclaw/src/types.rs b/crates/toolpath-openclaw/src/types.rs index 66d7357d..0c4a6f7f 100644 --- a/crates/toolpath-openclaw/src/types.rs +++ b/crates/toolpath-openclaw/src/types.rs @@ -1,3 +1,757 @@ -//! OpenClaw v3 session-JSONL schema (header + entry tree + content blocks). +//! OpenClaw session schema (format version 3). //! -//! Filled in Phase 1, Task 2. +//! OpenClaw stores each session as JSONL: the first line is a +//! [`SessionHeader`]; every subsequent line is an [`Entry`], forming a tree +//! via `id` / `parentId`. The visible head is moved by `leaf` entries. +//! +//! All structs carry an `extra` catch-all to preserve unknown fields for +//! forward compatibility. See `docs/agents/formats/openclaw/` for the +//! field-level reference. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The only session-format version this crate understands. +pub const SUPPORTED_VERSION: u32 = 3; + +/// Shared fields on every non-header entry. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct EntryBase { + /// Entry id, unique within the file (an 8-char UUIDv7 prefix). + pub id: String, + /// Parent entry id; `None` for a root entry. + #[serde(rename = "parentId", default)] + pub parent_id: Option, + /// Entry time (ISO-8601 string). + pub timestamp: String, + /// `"side"` marks a row that advances the raw cursor without selecting a + /// model-visible branch. + #[serde( + rename = "appendMode", + default, + skip_serializing_if = "Option::is_none" + )] + pub append_mode: Option, +} + +/// OpenClaw session file header (first line of a `.jsonl` file). +/// +/// ```json +/// {"type":"session","version":3,"id":"...","timestamp":"...","cwd":"...","parentSession":"..."} +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SessionHeader { + /// Format version. The reader hard-rejects anything but `3`. + #[serde(default)] + pub version: u32, + /// Session id (a UUID); matches the transcript filename stem. + pub id: String, + /// Session creation time (ISO-8601 string). + pub timestamp: String, + /// Working directory the session ran in. + pub cwd: String, + /// Path to a parent session file when this session was forked. + #[serde( + rename = "parentSession", + default, + skip_serializing_if = "Option::is_none" + )] + pub parent_session: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + pub extra: HashMap, +} + +/// A single entry in an OpenClaw session JSONL. +/// +/// Tagged by the `type` discriminant. The `session` variant matches the file +/// header; every other variant carries an [`EntryBase`] (id / parentId / +/// timestamp) flattened into the payload. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Entry { + /// The header line (see [`SessionHeader`]). + Session(SessionHeader), + /// A conversational turn. + Message { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// The role-tagged message payload. + message: AgentMessage, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// A model / provider switch. + ModelChange { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// Provider id (e.g. `anthropic`). + provider: String, + /// Model id. + #[serde(rename = "modelId")] + model_id: String, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// A reasoning-budget toggle. + ThinkingLevelChange { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// The new thinking level. + #[serde(rename = "thinkingLevel")] + thinking_level: String, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// A context-compaction boundary. + Compaction { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// Markdown summary of the dropped history. + summary: String, + /// Entries before this id are represented by `summary`. + #[serde(rename = "firstKeptEntryId")] + first_kept_entry_id: String, + /// Estimated context tokens before compaction. + #[serde(rename = "tokensBefore")] + tokens_before: u64, + /// `{ readFiles, modifiedFiles }`. + #[serde(default, skip_serializing_if = "Option::is_none")] + details: Option, + /// True if produced by an app hook. + #[serde(rename = "fromHook", default, skip_serializing_if = "Option::is_none")] + from_hook: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// A summary of an abandoned branch. + BranchSummary { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// Entry id of the abandoned branch's source (`"root"` when null). + #[serde(rename = "fromId")] + from_id: String, + /// Markdown summary of the abandoned branch. + summary: String, + /// `{ readFiles, modifiedFiles }`. + #[serde(default, skip_serializing_if = "Option::is_none")] + details: Option, + /// True if produced by an app hook. + #[serde(rename = "fromHook", default, skip_serializing_if = "Option::is_none")] + from_hook: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// A harness/app marker not replayed into model context. + Custom { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// Namespacing type. + #[serde(rename = "customType")] + custom_type: String, + /// Opaque payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + data: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// Harness/app content that IS replayable into context. + CustomMessage { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// Namespacing type. + #[serde(rename = "customType")] + custom_type: String, + /// The replayable content. + content: MessageContent, + /// UI visibility. + display: bool, + /// Opaque structured payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + details: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// A display label for a target entry (last write wins). + Label { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// The entry this label applies to. + #[serde(rename = "targetId", default, skip_serializing_if = "Option::is_none")] + target_id: Option, + /// The label text (`None`/empty clears). + #[serde(default, skip_serializing_if = "Option::is_none")] + label: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// The session name/title (last write wins). + SessionInfo { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// The session name. + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// Moves the visible-head pointer. + Leaf { + /// id / parentId / timestamp. + #[serde(flatten)] + base: EntryBase, + /// The entry the branch currently points at (the visible head). + #[serde(rename = "targetId", default, skip_serializing_if = "Option::is_none")] + target_id: Option, + /// Overrides the raw parent for the next append. + #[serde( + rename = "appendParentId", + default, + skip_serializing_if = "Option::is_none" + )] + append_parent_id: Option, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, +} + +impl Entry { + /// The `EntryBase` (id / parentId / timestamp) for non-header entries. + pub fn base(&self) -> Option<&EntryBase> { + match self { + Entry::Session(_) => None, + Entry::Message { base, .. } + | Entry::ModelChange { base, .. } + | Entry::ThinkingLevelChange { base, .. } + | Entry::Compaction { base, .. } + | Entry::BranchSummary { base, .. } + | Entry::Custom { base, .. } + | Entry::CustomMessage { base, .. } + | Entry::Label { base, .. } + | Entry::SessionInfo { base, .. } + | Entry::Leaf { base, .. } => Some(base), + } + } +} + +/// Content field for user / custom-role messages: either a bare string or a +/// list of content blocks. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum MessageContent { + /// A bare string. + Text(String), + /// A list of content blocks. + Blocks(Vec), +} + +impl MessageContent { + /// Collapse to a single text string (joining text blocks, ignoring others). + pub fn text(&self) -> String { + match self { + MessageContent::Text(s) => s.clone(), + MessageContent::Blocks(blocks) => blocks + .iter() + .filter_map(|b| match b { + ContentBlock::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"), + } + } +} + +/// One element of a message's `content` array. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum ContentBlock { + /// Visible text. + Text { + /// The text. + text: String, + /// Forward-compat catch-all (e.g. `textSignature`). + #[serde(default, flatten)] + extra: HashMap, + }, + /// Inline image. + Image { + /// Base64 image data. + data: String, + /// MIME type. + #[serde(rename = "mimeType")] + mime_type: String, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// Model reasoning. + Thinking { + /// The reasoning text. + thinking: String, + /// Forward-compat catch-all (e.g. `thinkingSignature`, `redacted`). + #[serde(default, flatten)] + extra: HashMap, + }, + /// A tool invocation. + ToolCall { + /// Correlates with the later `toolResult.toolCallId`. + id: String, + /// Tool name. + name: String, + /// Free-form JSON args. + arguments: serde_json::Value, + /// Forward-compat catch-all (e.g. `thoughtSignature`, `executionMode`). + #[serde(default, flatten)] + extra: HashMap, + }, +} + +/// Restricted content block set for `toolResult` messages (text or image only). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum ToolResultContent { + /// Text result. + Text { + /// The text. + text: String, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// Image result. + Image { + /// Base64 image data. + data: String, + /// MIME type. + #[serde(rename = "mimeType")] + mime_type: String, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, +} + +impl ToolResultContent { + /// The text of this block, if it is a text block. + pub fn as_text(&self) -> Option<&str> { + match self { + ToolResultContent::Text { text, .. } => Some(text), + ToolResultContent::Image { .. } => None, + } + } +} + +/// Assistant stop reason. Unknown values round-trip through [`StopReason::Other`]. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", untagged)] +pub enum StopReason { + /// A reason OpenClaw enumerates. + Known(KnownStopReason), + /// An unrecognized reason string. + Other(String), +} + +/// Enumerated stop reasons defined by OpenClaw. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum KnownStopReason { + /// Normal completion. + Stop, + /// Hit the output length limit. + Length, + /// Ended on a tool call. + ToolUse, + /// Errored. + Error, + /// Aborted. + Aborted, +} + +/// A message inside a `message` entry. Role-tagged union. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "role", rename_all = "camelCase")] +pub enum AgentMessage { + /// Human / channel input. + User { + /// String or block content. + content: MessageContent, + /// Epoch-ms timestamp. + timestamp: u64, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// Model output. + Assistant { + /// Block content (text / thinking / toolCall). + content: Vec, + /// API id (e.g. `anthropic-messages`). + api: String, + /// Provider id. + provider: String, + /// Requested model. + model: String, + /// Concrete served model when it differs. + #[serde( + rename = "responseModel", + default, + skip_serializing_if = "Option::is_none" + )] + response_model: Option, + /// Token usage for this call. + usage: Usage, + /// Why the turn ended. + #[serde(rename = "stopReason")] + stop_reason: StopReason, + /// Error text when the turn failed. + #[serde( + rename = "errorMessage", + default, + skip_serializing_if = "Option::is_none" + )] + error_message: Option, + /// Epoch-ms timestamp. + timestamp: u64, + /// Forward-compat catch-all (errorCode/Type/Body, diagnostics, …). + #[serde(default, flatten)] + extra: HashMap, + }, + /// Result of a prior `toolCall`. + ToolResult { + /// Links back to `toolCall.id`. + #[serde(rename = "toolCallId")] + tool_call_id: String, + /// Tool name. + #[serde(rename = "toolName")] + tool_name: String, + /// Text / image result content. + content: Vec, + /// Opaque structured payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + details: Option, + /// Error flag (error text rides in `content`). + #[serde(rename = "isError")] + is_error: bool, + /// Epoch-ms timestamp. + timestamp: u64, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, + /// Harness shell execution. + BashExecution { + /// The command. + command: String, + /// Captured output. + output: String, + /// Exit code. + #[serde(rename = "exitCode", default, skip_serializing_if = "Option::is_none")] + exit_code: Option, + /// Epoch-ms timestamp. + timestamp: u64, + /// Forward-compat catch-all. + #[serde(default, flatten)] + extra: HashMap, + }, +} + +impl AgentMessage { + /// The visible text of this message. + pub fn text(&self) -> String { + match self { + AgentMessage::User { content, .. } => content.text(), + AgentMessage::Assistant { content, .. } => content + .iter() + .filter_map(|b| match b { + ContentBlock::Text { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"), + AgentMessage::ToolResult { content, .. } => content + .iter() + .filter_map(ToolResultContent::as_text) + .collect::>() + .join("\n"), + AgentMessage::BashExecution { output, .. } => output.clone(), + } + } + + /// The concatenated reasoning text, if any. + pub fn thinking(&self) -> Option { + if let AgentMessage::Assistant { content, .. } = self { + let t = content + .iter() + .filter_map(|b| match b { + ContentBlock::Thinking { thinking, .. } => Some(thinking.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + if !t.is_empty() { + return Some(t); + } + } + None + } + + /// The tool-call blocks `(id, name, arguments)` in an assistant message. + pub fn tool_calls(&self) -> Vec<(&str, &str, &serde_json::Value)> { + match self { + AgentMessage::Assistant { content, .. } => content + .iter() + .filter_map(|b| match b { + ContentBlock::ToolCall { + id, name, arguments, .. + } => Some((id.as_str(), name.as_str(), arguments)), + _ => None, + }) + .collect(), + _ => Vec::new(), + } + } + + /// The model id, for assistant messages. + pub fn model(&self) -> Option<&str> { + match self { + AgentMessage::Assistant { model, .. } => Some(model), + _ => None, + } + } +} + +/// Per-message token accounting. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct Usage { + /// Prompt tokens. + #[serde(default)] + pub input: u64, + /// Completion tokens. + #[serde(default)] + pub output: u64, + /// Prompt-cache read tokens. + #[serde(default, rename = "cacheRead")] + pub cache_read: u64, + /// Prompt-cache write tokens. + #[serde(default, rename = "cacheWrite")] + pub cache_write: u64, + /// Headline total. + #[serde(default, rename = "totalTokens")] + pub total_tokens: u64, + /// Per-class cost breakdown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cost: Option, +} + +/// Dollar cost breakdown accompanying [`Usage`]. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct CostBreakdown { + /// Input cost. + #[serde(default)] + pub input: f64, + /// Output cost. + #[serde(default)] + pub output: f64, + /// Cache-read cost. + #[serde(default, rename = "cacheRead")] + pub cache_read: f64, + /// Cache-write cost. + #[serde(default, rename = "cacheWrite")] + pub cache_write: f64, + /// Total cost. + #[serde(default)] + pub total: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn roundtrip(value: &T) -> T { + let s = serde_json::to_string(value).expect("serialize"); + serde_json::from_str(&s).expect("deserialize") + } + + #[test] + fn header_roundtrip() { + let j = r#"{"type":"session","version":3,"id":"s1","timestamp":"2026-06-30T12:00:00Z","cwd":"/p"}"#; + let e: Entry = serde_json::from_str(j).unwrap(); + let h = match e { + Entry::Session(h) => h, + _ => panic!("expected session header"), + }; + assert_eq!(h.version, 3); + assert_eq!(h.id, "s1"); + assert!(h.parent_session.is_none()); + let back: SessionHeader = roundtrip(&h); + assert_eq!(back.cwd, "/p"); + } + + #[test] + fn user_content_string_or_array() { + let s: AgentMessage = + serde_json::from_str(r#"{"role":"user","content":"hi","timestamp":1}"#).unwrap(); + assert_eq!(s.text(), "hi"); + let a: AgentMessage = serde_json::from_str( + r#"{"role":"user","content":[{"type":"text","text":"hi"}],"timestamp":1}"#, + ) + .unwrap(); + assert_eq!(a.text(), "hi"); + } + + #[test] + fn assistant_message_with_blocks_roundtrip() { + let j = json!({ + "type":"message","id":"e1","parentId":"e0","timestamp":"2026-06-30T12:00:05Z", + "message":{"role":"assistant","content":[ + {"type":"thinking","thinking":"hm","thinkingSignature":"sig"}, + {"type":"text","text":"hi"}, + {"type":"toolCall","id":"c1","name":"read_file","arguments":{"path":"x"}}], + "api":"anthropic-messages","provider":"anthropic","model":"claude-x", + "usage":{"input":1,"output":2,"cacheRead":0,"cacheWrite":0,"totalTokens":3}, + "stopReason":"toolUse","timestamp":1751284805000u64} + }); + let e: Entry = serde_json::from_value(j).unwrap(); + match &e { + Entry::Message { + message: AgentMessage::Assistant { content, usage, .. }, + .. + } => { + assert_eq!(content.len(), 3); + assert_eq!(usage.total_tokens, 3); + } + _ => panic!("wrong variant"), + } + let m = match &e { + Entry::Message { message, .. } => message, + _ => unreachable!(), + }; + assert_eq!(m.text(), "hi"); + assert_eq!(m.thinking().as_deref(), Some("hm")); + assert_eq!(m.tool_calls().len(), 1); + assert_eq!(m.tool_calls()[0].1, "read_file"); + // thinkingSignature survives via `extra` + let back: Entry = roundtrip(&e); + assert!(matches!(back, Entry::Message { .. })); + } + + #[test] + fn tool_result_roundtrip() { + let j = json!({ + "type":"message","id":"e2","parentId":"e1","timestamp":"t", + "message":{"role":"toolResult","toolCallId":"c1","toolName":"read_file", + "content":[{"type":"text","text":"file body"}],"isError":false,"timestamp":1} + }); + let e: Entry = serde_json::from_value(j).unwrap(); + match &e { + Entry::Message { + message: + AgentMessage::ToolResult { + tool_call_id, + is_error, + .. + }, + .. + } => { + assert_eq!(tool_call_id, "c1"); + assert!(!is_error); + } + _ => panic!("wrong variant"), + } + assert!(matches!(roundtrip(&e), Entry::Message { .. })); + } + + #[test] + fn leaf_and_compaction_roundtrip() { + let leaf: Entry = serde_json::from_str( + r#"{"type":"leaf","id":"l1","parentId":"e2","timestamp":"t","targetId":"e2"}"#, + ) + .unwrap(); + match &leaf { + Entry::Leaf { target_id, .. } => assert_eq!(target_id.as_deref(), Some("e2")), + _ => panic!("wrong variant"), + } + assert!(matches!(roundtrip(&leaf), Entry::Leaf { .. })); + + let comp: Entry = serde_json::from_value(json!({ + "type":"compaction","id":"k1","parentId":"e2","timestamp":"t", + "summary":"## Goal","firstKeptEntryId":"e1","tokensBefore":5400u64, + "details":{"readFiles":["x"],"modifiedFiles":[]},"fromHook":false + })) + .unwrap(); + match &comp { + Entry::Compaction { + first_kept_entry_id, + tokens_before, + .. + } => { + assert_eq!(first_kept_entry_id, "e1"); + assert_eq!(*tokens_before, 5400); + } + _ => panic!("wrong variant"), + } + assert!(matches!(roundtrip(&comp), Entry::Compaction { .. })); + } + + #[test] + fn session_info_and_label_roundtrip() { + let si: Entry = serde_json::from_str( + r#"{"type":"session_info","id":"s","parentId":null,"timestamp":"t","name":"My chat"}"#, + ) + .unwrap(); + match &si { + Entry::SessionInfo { name, .. } => assert_eq!(name.as_deref(), Some("My chat")), + _ => panic!("wrong variant"), + } + let lab: Entry = serde_json::from_str( + r#"{"type":"label","id":"l","parentId":"e1","timestamp":"t","targetId":"e1","label":"pin"}"#, + ) + .unwrap(); + match &lab { + Entry::Label { target_id, label, .. } => { + assert_eq!(target_id.as_deref(), Some("e1")); + assert_eq!(label.as_deref(), Some("pin")); + } + _ => panic!("wrong variant"), + } + } + + #[test] + fn append_mode_side_roundtrips() { + let e: Entry = serde_json::from_str( + r#"{"type":"message","id":"e","parentId":null,"timestamp":"t","appendMode":"side", + "message":{"role":"user","content":"x","timestamp":1}}"#, + ) + .unwrap(); + assert_eq!(e.base().unwrap().append_mode.as_deref(), Some("side")); + } + + #[test] + fn stop_reason_unknown_roundtrips() { + let sr: StopReason = serde_json::from_str("\"weird\"").unwrap(); + assert_eq!(sr, StopReason::Other("weird".into())); + let known: StopReason = serde_json::from_str("\"toolUse\"").unwrap(); + assert_eq!(known, StopReason::Known(KnownStopReason::ToolUse)); + } +} From 243a34aa3310f0d12303cd3a448f3e96b5ff776a Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:16:56 -0400 Subject: [PATCH 05/22] feat(openclaw): path + session-key resolution --- crates/toolpath-openclaw/src/paths.rs | 403 +++++++++++++++++++++++++- 1 file changed, 401 insertions(+), 2 deletions(-) diff --git a/crates/toolpath-openclaw/src/paths.rs b/crates/toolpath-openclaw/src/paths.rs index ceab5363..5bb8c02d 100644 --- a/crates/toolpath-openclaw/src/paths.rs +++ b/crates/toolpath-openclaw/src/paths.rs @@ -1,3 +1,402 @@ -//! On-disk layout + state-directory + session-id resolution. +//! On-disk layout resolution for OpenClaw's state directory. //! -//! Filled in Phase 2, Task 3. +//! OpenClaw persists sessions under a single state directory (default +//! `~/.openclaw`) as `agents//sessions/.jsonl`, with a +//! `sessions.json` index mapping routing keys to files. See +//! `docs/agents/formats/openclaw/directory-layout.md`. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; + +use crate::error::{OpenClawError, Result}; + +/// The default agent bucket when none is given. +pub const DEFAULT_AGENT_ID: &str = "main"; + +/// Normalize an agent id the way OpenClaw does: lowercase and path-sanitize. +pub fn normalize_agent_id(agent_id: &str) -> String { + let trimmed = agent_id.trim(); + if trimmed.is_empty() { + return DEFAULT_AGENT_ID.to_string(); + } + trimmed + .to_lowercase() + .chars() + .map(|c| match c { + '/' | '\\' | ':' | ' ' => '-', + other => other, + }) + .collect() +} + +/// Parsed components of an OpenClaw routing/session key +/// (`agent::::` and its variants). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ParsedKey { + /// The agent persona id. + pub agent_id: String, + /// Messaging channel (whatsapp/telegram/slack/…), if any. + pub channel: Option, + /// Peer kind (direct/dm/group/channel), if any. + pub peer_kind: Option, + /// Channel-native peer id (a person for DMs, a room for groups). + pub peer_id: Option, + /// Thread id, when the key is thread-scoped. + pub thread_id: Option, +} + +/// Parse an OpenClaw session key into its components. Lenient: unknown shapes +/// yield `None` parts rather than an error. +pub fn parse_session_key(key: &str) -> ParsedKey { + let segs: Vec<&str> = key.split(':').collect(); + let mut out = ParsedKey::default(); + if segs.first() != Some(&"agent") || segs.len() < 2 { + return out; + } + out.agent_id = segs[1].to_string(); + let mut rest: Vec<&str> = segs[2..].to_vec(); + + if let Some(i) = rest.iter().position(|s| *s == "thread") { + if i + 1 < rest.len() { + out.thread_id = Some(rest[i + 1].to_string()); + } + rest.truncate(i); + } + + if rest.is_empty() || rest == ["main"] { + return out; + } + if rest[0] == "direct" && rest.len() >= 2 { + out.peer_kind = Some("direct".to_string()); + out.peer_id = Some(rest[1].to_string()); + return out; + } + out.channel = Some(rest[0].to_string()); + if let Some(i) = rest.iter().position(|s| *s == "direct") { + out.peer_kind = Some("direct".to_string()); + if i + 1 < rest.len() { + out.peer_id = Some(rest[i + 1].to_string()); + } + } else if rest.len() >= 3 { + out.peer_kind = Some(rest[1].to_string()); + out.peer_id = Some(rest[2].to_string()); + } else if rest.len() == 2 { + out.peer_id = Some(rest[1].to_string()); + } + out +} + +/// One entry in `sessions.json` (keyed by routing key). +#[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IndexEntry { + /// The transcript session id (filename stem). + #[serde(default)] + pub session_id: Option, + /// The transcript filename (relative to the sessions dir, or absolute). + #[serde(default)] + pub session_file: Option, + /// Anything else OpenClaw stores per session. + #[serde(flatten)] + pub extra: HashMap, +} + +/// Parsed `sessions.json` index: routing key -> entry. +#[derive(Debug, Clone, Default)] +pub struct SessionsIndex(pub BTreeMap); + +impl SessionsIndex { + /// Load `sessions.json` from a sessions directory, if present and parseable. + pub fn load(sessions_dir: &Path) -> Option { + let path = sessions_dir.join("sessions.json"); + let bytes = std::fs::read(&path).ok()?; + let map: BTreeMap = serde_json::from_slice(&bytes).ok()?; + Some(SessionsIndex(map)) + } + + /// Find the routing key (and parse) for a given session id. + pub fn routing_key_for(&self, session_id: &str) -> Option<(String, ParsedKey)> { + self.0.iter().find_map(|(key, entry)| { + if entry.session_id.as_deref() == Some(session_id) { + Some((key.clone(), parse_session_key(key))) + } else { + None + } + }) + } + + /// The recorded session file for a session id, if the index names one. + pub fn session_file_for(&self, session_id: &str) -> Option { + self.0.values().find_map(|e| { + if e.session_id.as_deref() == Some(session_id) { + e.session_file.clone() + } else { + None + } + }) + } +} + +/// Resolves OpenClaw's on-disk layout. Construct with [`PathResolver::new`] +/// (reads the environment) or [`PathResolver::with_state_dir`] (explicit, used +/// in tests). +#[derive(Debug, Clone)] +pub struct PathResolver { + state_dir: PathBuf, +} + +impl Default for PathResolver { + fn default() -> Self { + Self::new() + } +} + +impl PathResolver { + /// Resolve the state directory from the process environment. + pub fn new() -> Self { + let state_dir = resolve_state_dir(&|k| std::env::var(k).ok(), os_home().as_deref()); + Self { state_dir } + } + + /// Use an explicit state directory (test seam / `--base` override). + pub fn with_state_dir(dir: impl Into) -> Self { + Self { + state_dir: dir.into(), + } + } + + /// The resolved state directory (e.g. `~/.openclaw`). + pub fn state_dir(&self) -> &Path { + &self.state_dir + } + + /// `/agents`. + pub fn agents_dir(&self) -> PathBuf { + self.state_dir.join("agents") + } + + /// `/agents//sessions`. + pub fn agent_sessions_dir(&self, agent_id: &str) -> PathBuf { + self.agents_dir() + .join(normalize_agent_id(agent_id)) + .join("sessions") + } + + /// List agent ids that have a `sessions/` directory on disk. + pub fn list_agent_ids(&self) -> std::io::Result> { + let mut ids = Vec::new(); + let dir = self.agents_dir(); + if !dir.exists() { + return Ok(ids); + } + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() + && let Some(name) = entry.file_name().to_str() + { + ids.push(name.to_string()); + } + } + ids.sort(); + Ok(ids) + } + + /// Resolve a session id to a concrete transcript file under an agent. + /// + /// Tries, in order: the `sessions.json` index (`sessionFile`/`sessionId`), + /// then `.jsonl`, then any `*.jsonl` whose stem is ``, ends with + /// `_` (forked/rotated), or starts with `-topic-`. + pub fn resolve_session_file(&self, agent_id: &str, session_id: &str) -> Result { + let dir = self.agent_sessions_dir(agent_id); + if !dir.exists() { + return Err(OpenClawError::agent_not_found(agent_id)); + } + + if let Some(index) = SessionsIndex::load(&dir) + && let Some(file) = index.session_file_for(session_id) + { + let p = PathBuf::from(&file); + let candidate = if p.is_absolute() { p } else { dir.join(p) }; + if candidate.exists() { + return Ok(candidate); + } + } + + let direct = dir.join(format!("{session_id}.jsonl")); + if direct.exists() { + return Ok(direct); + } + + let forked_suffix = format!("_{session_id}"); + let topic_prefix = format!("{session_id}-topic-"); + for entry in std::fs::read_dir(&dir)? { + let path = entry?.path(); + if path.extension().and_then(|e| e.to_str()) != Some("jsonl") { + continue; + } + let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(""); + if stem == session_id || stem.ends_with(&forked_suffix) || stem.starts_with(&topic_prefix) + { + return Ok(path); + } + } + + Err(OpenClawError::session_not_found(session_id)) + } +} + +/// The OS home directory, honoring `OPENCLAW_HOME` then `HOME`/`USERPROFILE`. +fn os_home() -> Option { + resolve_home(&|k| std::env::var(k).ok()) +} + +fn resolve_home(get: &dyn Fn(&str) -> Option) -> Option { + let pick = |v: Option| v.filter(|s| !s.trim().is_empty()).map(PathBuf::from); + pick(get("OPENCLAW_HOME")) + .or_else(|| pick(get("HOME"))) + .or_else(|| pick(get("USERPROFILE"))) +} + +/// Resolve the state directory: `OPENCLAW_STATE_DIR` -> existing `~/.openclaw` +/// -> existing legacy `~/.clawdbot` -> default `~/.openclaw`. +fn resolve_state_dir(get: &dyn Fn(&str) -> Option, os_home: Option<&Path>) -> PathBuf { + if let Some(override_dir) = get("OPENCLAW_STATE_DIR").filter(|s| !s.trim().is_empty()) { + return PathBuf::from(override_dir); + } + let home = resolve_home(get) + .or_else(|| os_home.map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(".")); + let new_dir = home.join(".openclaw"); + if new_dir.exists() { + return new_dir; + } + let legacy = home.join(".clawdbot"); + if legacy.exists() { + return legacy; + } + new_dir +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn agent_sessions_dir_layout() { + let tmp = tempfile::tempdir().unwrap(); + let r = PathResolver::with_state_dir(tmp.path()); + assert_eq!(r.state_dir(), tmp.path()); + assert_eq!( + r.agent_sessions_dir("main"), + tmp.path().join("agents/main/sessions") + ); + assert_eq!( + r.agent_sessions_dir("Work Bot"), + tmp.path().join("agents/work-bot/sessions") + ); + } + + #[test] + fn state_dir_env_override_wins() { + let get = |k: &str| match k { + "OPENCLAW_STATE_DIR" => Some("/custom/state".to_string()), + "HOME" => Some("/home/u".to_string()), + _ => None, + }; + assert_eq!( + resolve_state_dir(&get, None), + PathBuf::from("/custom/state") + ); + } + + #[test] + fn state_dir_defaults_to_openclaw_under_home() { + let get = |k: &str| match k { + "HOME" => Some("/home/u".to_string()), + _ => None, + }; + // Neither ~/.openclaw nor ~/.clawdbot exists under this fake home. + assert_eq!( + resolve_state_dir(&get, None), + PathBuf::from("/home/u/.openclaw") + ); + } + + #[test] + fn home_prefers_openclaw_home() { + let get = |k: &str| match k { + "OPENCLAW_HOME" => Some("/oc/home".to_string()), + "HOME" => Some("/home/u".to_string()), + _ => None, + }; + assert_eq!(resolve_home(&get), Some(PathBuf::from("/oc/home"))); + } + + #[test] + fn parses_dm_key() { + let dm = parse_session_key("agent:main:whatsapp:direct:15555550123"); + assert_eq!(dm.agent_id, "main"); + assert_eq!(dm.channel.as_deref(), Some("whatsapp")); + assert_eq!(dm.peer_kind.as_deref(), Some("direct")); + assert_eq!(dm.peer_id.as_deref(), Some("15555550123")); + } + + #[test] + fn parses_group_key() { + let grp = parse_session_key("agent:main:slack:group:T42"); + assert_eq!(grp.channel.as_deref(), Some("slack")); + assert_eq!(grp.peer_kind.as_deref(), Some("group")); + assert_eq!(grp.peer_id.as_deref(), Some("T42")); + } + + #[test] + fn parses_account_scoped_dm_and_thread() { + let k = parse_session_key("agent:main:telegram:acct1:direct:99:thread:7"); + assert_eq!(k.channel.as_deref(), Some("telegram")); + assert_eq!(k.peer_kind.as_deref(), Some("direct")); + assert_eq!(k.peer_id.as_deref(), Some("99")); + assert_eq!(k.thread_id.as_deref(), Some("7")); + } + + #[test] + fn parses_main_key() { + let main = parse_session_key("agent:main:main"); + assert_eq!(main.agent_id, "main"); + assert!(main.channel.is_none()); + assert!(main.peer_id.is_none()); + } + + #[test] + fn resolves_session_file_by_stem_and_index() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("abc-123.jsonl"), "{}").unwrap(); + std::fs::write( + dir.join("sessions.json"), + r#"{"agent:main:whatsapp:direct:55":{"sessionId":"abc-123","sessionFile":"abc-123.jsonl"}}"#, + ) + .unwrap(); + let r = PathResolver::with_state_dir(tmp.path()); + + // by index + let by_index = r.resolve_session_file("main", "abc-123").unwrap(); + assert_eq!(by_index, dir.join("abc-123.jsonl")); + + // routing key recovered + let idx = SessionsIndex::load(&dir).unwrap(); + let (key, parsed) = idx.routing_key_for("abc-123").unwrap(); + assert!(key.contains("whatsapp")); + assert_eq!(parsed.channel.as_deref(), Some("whatsapp")); + + // by stem (different file, no index entry) + std::fs::write(dir.join("loose-9.jsonl"), "{}").unwrap(); + assert_eq!( + r.resolve_session_file("main", "loose-9").unwrap(), + dir.join("loose-9.jsonl") + ); + + // missing → error + assert!(r.resolve_session_file("main", "nope").is_err()); + } +} From 5cf397b1fb0f6dd9ea5c59163b8af402ffd4431a Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:19:54 -0400 Subject: [PATCH 06/22] feat(openclaw): session reader + tree assembly + listing/metadata --- crates/toolpath-openclaw/src/io.rs | 250 ++++++++++++++++- crates/toolpath-openclaw/src/reader.rs | 262 +++++++++++++++++- .../tests/fixtures/dm_session.jsonl | 9 + .../tests/fixtures/sessions.json | 8 + 4 files changed, 523 insertions(+), 6 deletions(-) create mode 100644 crates/toolpath-openclaw/tests/fixtures/dm_session.jsonl create mode 100644 crates/toolpath-openclaw/tests/fixtures/sessions.json diff --git a/crates/toolpath-openclaw/src/io.rs b/crates/toolpath-openclaw/src/io.rs index d368ac08..5afcb088 100644 --- a/crates/toolpath-openclaw/src/io.rs +++ b/crates/toolpath-openclaw/src/io.rs @@ -1,3 +1,247 @@ -//! Lightweight session listing and metadata extraction. -//! -//! Filled in Phase 3, Task 5. +//! Lightweight session listing and metadata extraction (no full parse). + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::error::Result; +use crate::error::OpenClawError; +use crate::paths::{PathResolver, SessionsIndex}; + +/// Lightweight metadata for one session, gathered without a full parse. +#[derive(Debug, Clone)] +pub struct SessionMeta { + /// Session id (header `id`, or filename stem as a fallback). + pub id: String, + /// Session timestamp (header `timestamp`, or file mtime as a fallback). + pub timestamp: String, + /// Path to the transcript file. + pub file_path: PathBuf, + /// Number of non-header entries. + pub entry_count: usize, + /// First non-empty user message text, for topic display. + pub first_user_message: Option, + /// Working directory recorded in the header. + pub cwd: Option, + /// Routing key from `sessions.json`, if any. + pub session_key: Option, + /// The agent bucket this session was found under. + pub agent_id: String, +} + +/// List sessions under one agent, newest first. +pub fn list_sessions(resolver: &PathResolver, agent_id: &str) -> Result> { + let dir = resolver.agent_sessions_dir(agent_id); + if !dir.exists() { + return Err(OpenClawError::agent_not_found(agent_id)); + } + let index = SessionsIndex::load(&dir); + + let mut rows: Vec<(SessionMeta, SystemTime)> = Vec::new(); + for entry in std::fs::read_dir(&dir)? { + let path = entry?.path(); + if !is_transcript_file(&path) { + continue; + } + let stem = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(); + + let header = peek_header(&path); + let (id, timestamp, cwd) = match header { + Some((id, ts, cwd)) => (id, ts, cwd), + None => (stem, file_mtime_rfc3339(&path).unwrap_or_default(), None), + }; + let entry_count = count_nonempty_lines(&path)?.saturating_sub(1); + let first_user_message = extract_first_user_message(&path)?; + let session_key = index + .as_ref() + .and_then(|i| i.routing_key_for(&id).map(|(k, _)| k)); + let mtime = std::fs::metadata(&path) + .and_then(|m| m.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + + rows.push(( + SessionMeta { + id, + timestamp, + file_path: path, + entry_count, + first_user_message, + cwd, + session_key, + agent_id: agent_id.to_string(), + }, + mtime, + )); + } + + rows.sort_by(|a, b| b.0.timestamp.cmp(&a.0.timestamp).then_with(|| b.1.cmp(&a.1))); + Ok(rows.into_iter().map(|(m, _)| m).collect()) +} + +/// List sessions across every agent bucket, newest first. +pub fn list_all_sessions(resolver: &PathResolver) -> Result> { + let mut all = Vec::new(); + for agent_id in resolver.list_agent_ids()? { + if let Ok(rows) = list_sessions(resolver, &agent_id) { + all.extend(rows); + } + } + all.sort_by(|a, b| b.timestamp.cmp(&a.timestamp)); + Ok(all) +} + +/// True for a session transcript `*.jsonl`, excluding `*.trajectory.jsonl` +/// telemetry sidecars. +fn is_transcript_file(path: &Path) -> bool { + if path.extension().and_then(|e| e.to_str()) != Some("jsonl") { + return false; + } + let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(""); + !stem.ends_with(".trajectory") +} + +/// Read just the header line and pull `id` / `timestamp` / `cwd`. +fn peek_header(path: &Path) -> Option<(String, String, Option)> { + let content = std::fs::read_to_string(path).ok()?; + let line = content.lines().find(|l| !l.trim().is_empty())?; + let v: serde_json::Value = serde_json::from_str(line).ok()?; + let obj = v.as_object()?; + if obj.get("type").and_then(|t| t.as_str()) != Some("session") { + return None; + } + let id = obj.get("id")?.as_str()?.to_string(); + let timestamp = obj + .get("timestamp") + .and_then(|t| t.as_str()) + .unwrap_or_default() + .to_string(); + let cwd = obj + .get("cwd") + .and_then(|c| c.as_str()) + .map(str::to_string); + Some((id, timestamp, cwd)) +} + +fn count_nonempty_lines(path: &Path) -> Result { + let content = std::fs::read_to_string(path)?; + Ok(content.lines().filter(|l| !l.trim().is_empty()).count()) +} + +fn file_mtime_rfc3339(path: &Path) -> Option { + let mtime = std::fs::metadata(path).ok()?.modified().ok()?; + let dt: chrono::DateTime = mtime.into(); + Some(dt.to_rfc3339()) +} + +/// Extract the first non-empty user-message text (string or text blocks). +pub fn extract_first_user_message(path: &Path) -> Result> { + let content = std::fs::read_to_string(path)?; + for line in content.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let v: serde_json::Value = match serde_json::from_str(line) { + Ok(v) => v, + Err(_) => continue, + }; + let obj = match v.as_object() { + Some(o) => o, + None => continue, + }; + if obj.get("type").and_then(|t| t.as_str()) != Some("message") { + continue; + } + let msg = match obj.get("message").and_then(|m| m.as_object()) { + Some(m) => m, + None => continue, + }; + if msg.get("role").and_then(|r| r.as_str()) != Some("user") { + continue; + } + let text = match msg.get("content") { + Some(serde_json::Value::String(s)) => s.clone(), + Some(serde_json::Value::Array(blocks)) => blocks + .iter() + .filter_map(|b| { + let bo = b.as_object()?; + if bo.get("type").and_then(|t| t.as_str()) == Some("text") { + bo.get("text").and_then(|t| t.as_str()).map(str::to_string) + } else { + None + } + }) + .collect::>() + .join("\n"), + _ => continue, + }; + let trimmed = text.trim(); + if !trimmed.is_empty() { + return Ok(Some(trimmed.to_string())); + } + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a temp state dir with the DM fixture under agents/main/sessions. + fn temp_state() -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + let fixture = std::fs::read("tests/fixtures/dm_session.jsonl").unwrap(); + std::fs::write(dir.join("sess-abc.jsonl"), fixture).unwrap(); + std::fs::copy("tests/fixtures/sessions.json", dir.join("sessions.json")).unwrap(); + // a trajectory sidecar that must be ignored + std::fs::write(dir.join("sess-abc.trajectory.jsonl"), "{}\n").unwrap(); + tmp + } + + #[test] + fn lists_sessions_with_metadata() { + let tmp = temp_state(); + let resolver = PathResolver::with_state_dir(tmp.path()); + let rows = list_sessions(&resolver, "main").unwrap(); + assert_eq!(rows.len(), 1, "trajectory sidecar must be excluded"); + let m = &rows[0]; + assert_eq!(m.id, "sess-abc"); + assert_eq!(m.first_user_message.as_deref(), Some("Fix the bug in x.ts")); + assert_eq!(m.cwd.as_deref(), Some("/home/u/proj")); + assert_eq!( + m.session_key.as_deref(), + Some("agent:main:whatsapp:direct:15555550123") + ); + assert!(m.entry_count >= 6); + } + + #[test] + fn lists_all_sessions_across_agents() { + let tmp = temp_state(); + let resolver = PathResolver::with_state_dir(tmp.path()); + let rows = list_all_sessions(&resolver).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].agent_id, "main"); + } + + #[test] + fn first_user_message_from_string_content() { + let tmp = tempfile::tempdir().unwrap(); + let p = tmp.path().join("s.jsonl"); + std::fs::write( + &p, + "{\"type\":\"session\",\"version\":3,\"id\":\"x\",\"timestamp\":\"t\",\"cwd\":\"/\"}\n\ + {\"type\":\"message\",\"id\":\"m\",\"parentId\":null,\"timestamp\":\"t\",\"message\":{\"role\":\"user\",\"content\":\"hello there\",\"timestamp\":1}}\n", + ) + .unwrap(); + assert_eq!( + extract_first_user_message(&p).unwrap().as_deref(), + Some("hello there") + ); + } +} diff --git a/crates/toolpath-openclaw/src/reader.rs b/crates/toolpath-openclaw/src/reader.rs index 9e8b11dc..20fc436e 100644 --- a/crates/toolpath-openclaw/src/reader.rs +++ b/crates/toolpath-openclaw/src/reader.rs @@ -1,3 +1,259 @@ -//! Session reading: parse JSONL, assemble the entry tree, follow parents. -//! -//! Filled in Phase 3, Task 4. +//! Reading OpenClaw session JSONL: parse the header + entry tree, follow +//! `parentSession` chains, and resolve the visible-leaf thread. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use crate::error::{OpenClawError, Result}; +use crate::paths::{ParsedKey, SessionsIndex}; +use crate::types::{Entry, SessionHeader, SUPPORTED_VERSION}; + +/// A parsed OpenClaw session: its header, the entry tree, and (optionally) the +/// routing identity recovered from `sessions.json`. +#[derive(Debug, Clone)] +pub struct OpenClawSession { + /// The session header (line 1). + pub header: SessionHeader, + /// All entries after the header, in file order. + pub entries: Vec, + /// Path the session was read from. + pub file_path: PathBuf, + /// Parent session (when this session was forked via `parentSession`). + pub parent: Option>, + /// The routing key from `sessions.json`, if found. + pub session_key: Option, + /// Parsed components of [`Self::session_key`]. + pub parsed_key: Option, +} + +/// Read a single session file (does not follow `parentSession`). +pub fn read_session_from_file(path: &Path) -> Result { + let content = std::fs::read_to_string(path)?; + let mut lines = content.lines().filter(|l| !l.trim().is_empty()); + + let header_line = lines + .next() + .ok_or_else(|| OpenClawError::invalid_session_file(path, "empty file"))?; + let header = parse_header(header_line, path)?; + if header.version != SUPPORTED_VERSION { + return Err(OpenClawError::UnsupportedVersion(header.version)); + } + + let mut entries = Vec::new(); + for line in lines { + match serde_json::from_str::(line) { + Ok(Entry::Session(_)) => {} // stray header line; ignore + Ok(entry) => entries.push(entry), + Err(_) => { + // Tolerate unknown future entry types: if the line is valid JSON + // (just a `type` we don't model), skip it; otherwise it's + // genuinely malformed. + if serde_json::from_str::(line).is_err() { + return Err(OpenClawError::invalid_session_file( + path, + "malformed JSON entry line", + )); + } + } + } + } + + Ok(OpenClawSession { + header, + entries, + file_path: path.to_path_buf(), + parent: None, + session_key: None, + parsed_key: None, + }) +} + +/// Read a session and follow `parentSession` up to `max_depth` parents. +pub fn read_session_with_parent(path: &Path, max_depth: usize) -> Result { + let mut session = read_session_from_file(path)?; + if max_depth == 0 { + return Ok(session); + } + if let Some(parent_path) = session.header.parent_session.clone() { + match read_session_with_parent(Path::new(&parent_path), max_depth - 1) { + Ok(parent) => session.parent = Some(Box::new(parent)), + Err(e) => eprintln!("warning: failed to read parent session {parent_path}: {e}"), + } + } + Ok(session) +} + +fn parse_header(line: &str, path: &Path) -> Result { + match serde_json::from_str::(line) { + Ok(Entry::Session(h)) => Ok(h), + Ok(_) => Err(OpenClawError::malformed_header( + "first line is not a session header", + )), + Err(e) => Err(OpenClawError::invalid_session_file( + path, + format!("bad header line: {e}"), + )), + } +} + +impl OpenClawSession { + /// Recover the routing key (channel/peer) for this session from a + /// `sessions.json` index in the session file's directory, if present. + pub fn attach_routing_key(&mut self) { + if let Some(dir) = self.file_path.parent() + && let Some(index) = SessionsIndex::load(dir) + && let Some((key, parsed)) = index.routing_key_for(&self.header.id) + { + self.session_key = Some(key); + self.parsed_key = Some(parsed); + } + } + + /// The session id chain (oldest first), following `parentSession`. + pub fn session_id_chain(&self) -> Vec { + let mut ids = Vec::new(); + if let Some(parent) = &self.parent { + ids.extend(parent.session_id_chain()); + } + ids.push(self.header.id.clone()); + ids + } + + /// The entries on the live conversation, root-to-leaf. + /// + /// The visible head is the target of the last `leaf` entry; failing that, + /// the last non-leaf entry. From there we walk `parentId` to the root and + /// reverse, so the result is the live thread in chronological order. + pub fn main_thread(&self) -> Vec<&Entry> { + let mut by_id: HashMap<&str, &Entry> = HashMap::new(); + for e in &self.entries { + if let Some(b) = e.base() { + by_id.insert(b.id.as_str(), e); + } + } + + let leaf_target = self.entries.iter().rev().find_map(|e| match e { + Entry::Leaf { + target_id: Some(t), .. + } => Some(t.as_str()), + _ => None, + }); + let start = leaf_target.or_else(|| { + self.entries.iter().rev().find_map(|e| match e { + Entry::Leaf { .. } => None, + _ => e.base().map(|b| b.id.as_str()), + }) + }); + + let mut chain: Vec<&Entry> = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + let mut cur = start; + while let Some(id) = cur { + if !seen.insert(id) { + break; // cycle guard + } + match by_id.get(id) { + Some(e) => { + chain.push(*e); + cur = e.base().and_then(|b| b.parent_id.as_deref()); + } + None => break, + } + } + chain.reverse(); + chain + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::AgentMessage; + + fn fixture() -> OpenClawSession { + read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap() + } + + #[test] + fn reads_dm_fixture() { + let s = fixture(); + assert_eq!(s.header.version, 3); + assert_eq!(s.header.id, "sess-abc"); + assert_eq!(s.header.cwd, "/home/u/proj"); + // 7 entries (blank line tolerated; header excluded): 3 messages-user/asst, + // model_change, 2 tool results, leaf. + assert!(s.entries.len() >= 6); + assert!( + s.entries + .iter() + .any(|e| matches!(e, Entry::Message { .. })) + ); + } + + #[test] + fn blank_lines_tolerated() { + // The fixture contains a blank line between entries; if it weren't + // tolerated, parsing would error. + assert!(read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).is_ok()); + } + + #[test] + fn rejects_non_v3() { + let tmp = tempfile::tempdir().unwrap(); + let p = tmp.path().join("old.jsonl"); + std::fs::write( + &p, + r#"{"type":"session","version":2,"id":"x","timestamp":"t","cwd":"/"}"#, + ) + .unwrap(); + let err = read_session_from_file(&p).unwrap_err(); + assert!(matches!(err, OpenClawError::UnsupportedVersion(2))); + } + + #[test] + fn tolerates_unknown_entry_type() { + let tmp = tempfile::tempdir().unwrap(); + let p = tmp.path().join("s.jsonl"); + std::fs::write( + &p, + "{\"type\":\"session\",\"version\":3,\"id\":\"x\",\"timestamp\":\"t\",\"cwd\":\"/\"}\n\ + {\"type\":\"future_thing\",\"id\":\"f\",\"parentId\":null,\"timestamp\":\"t\"}\n\ + {\"type\":\"message\",\"id\":\"m\",\"parentId\":null,\"timestamp\":\"t\",\"message\":{\"role\":\"user\",\"content\":\"hi\",\"timestamp\":1}}\n", + ) + .unwrap(); + let s = read_session_from_file(&p).unwrap(); + // unknown skipped, message kept + assert_eq!(s.entries.len(), 1); + assert!(matches!(&s.entries[0], Entry::Message { .. })); + } + + #[test] + fn main_thread_follows_leaf_to_root() { + let s = fixture(); + let thread = s.main_thread(); + // root is the user message e10 + let first = thread.first().unwrap(); + assert_eq!(first.base().unwrap().id, "e10"); + assert!(matches!( + first, + Entry::Message { + message: AgentMessage::User { .. }, + .. + } + )); + // the last content entry is the edit tool result e15 (leaf target) + let last = thread.last().unwrap(); + assert_eq!(last.base().unwrap().id, "e15"); + // the read tool result is on the thread + assert!(thread.iter().any(|e| e.base().unwrap().id == "e13")); + } + + #[test] + fn attach_routing_key_recovers_channel() { + let mut s = fixture(); + s.attach_routing_key(); + let key = s.parsed_key.expect("routing key"); + assert_eq!(key.channel.as_deref(), Some("whatsapp")); + assert_eq!(key.peer_id.as_deref(), Some("15555550123")); + } +} diff --git a/crates/toolpath-openclaw/tests/fixtures/dm_session.jsonl b/crates/toolpath-openclaw/tests/fixtures/dm_session.jsonl new file mode 100644 index 00000000..883a34ee --- /dev/null +++ b/crates/toolpath-openclaw/tests/fixtures/dm_session.jsonl @@ -0,0 +1,9 @@ +{"type":"session","version":3,"id":"sess-abc","timestamp":"2026-06-30T12:00:00.000Z","cwd":"/home/u/proj"} +{"type":"message","id":"e10","parentId":null,"timestamp":"2026-06-30T12:00:01.000Z","message":{"role":"user","content":[{"type":"text","text":"Fix the bug in x.ts"}],"timestamp":1751284801000}} +{"type":"model_change","id":"e11","parentId":"e10","timestamp":"2026-06-30T12:00:01.500Z","provider":"anthropic","modelId":"claude-x"} +{"type":"message","id":"e12","parentId":"e11","timestamp":"2026-06-30T12:00:05.000Z","message":{"role":"assistant","content":[{"type":"thinking","thinking":"Let me read the file"},{"type":"text","text":"I'll read x.ts first."},{"type":"toolCall","id":"call_1","name":"read_file","arguments":{"path":"src/x.ts"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-x","usage":{"input":1200,"output":340,"cacheRead":0,"cacheWrite":0,"totalTokens":1540},"stopReason":"toolUse","timestamp":1751284805000}} +{"type":"message","id":"e13","parentId":"e12","timestamp":"2026-06-30T12:00:05.500Z","message":{"role":"toolResult","toolCallId":"call_1","toolName":"read_file","content":[{"type":"text","text":"file contents of x.ts"}],"isError":false,"timestamp":1751284805500}} + +{"type":"message","id":"e14","parentId":"e13","timestamp":"2026-06-30T12:00:08.000Z","message":{"role":"assistant","content":[{"type":"text","text":"Found it. Applying the fix."},{"type":"toolCall","id":"call_2","name":"edit_file","arguments":{"path":"src/x.ts","old":"buggy","new":"fixed"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-x","usage":{"input":1500,"output":120,"cacheRead":0,"cacheWrite":0,"totalTokens":1620},"stopReason":"toolUse","timestamp":1751284808000}} +{"type":"message","id":"e15","parentId":"e14","timestamp":"2026-06-30T12:00:08.500Z","message":{"role":"toolResult","toolCallId":"call_2","toolName":"edit_file","content":[{"type":"text","text":"edited 1 file"}],"isError":false,"timestamp":1751284808500}} +{"type":"leaf","id":"e16","parentId":"e15","timestamp":"2026-06-30T12:00:08.600Z","targetId":"e15"} diff --git a/crates/toolpath-openclaw/tests/fixtures/sessions.json b/crates/toolpath-openclaw/tests/fixtures/sessions.json new file mode 100644 index 00000000..4aff2e64 --- /dev/null +++ b/crates/toolpath-openclaw/tests/fixtures/sessions.json @@ -0,0 +1,8 @@ +{ + "agent:main:whatsapp:direct:15555550123": { + "sessionId": "sess-abc", + "sessionFile": "dm_session.jsonl", + "updatedAt": "2026-06-30T12:00:08.600Z", + "sessionStartedAt": "2026-06-30T12:00:00.000Z" + } +} From 507f03a21f02ed6a5fb7c073c2d03c13bcd58a27 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:24:20 -0400 Subject: [PATCH 07/22] feat(openclaw): session_to_view + tool/file/usage mapping + channel actor helpers --- crates/toolpath-openclaw/src/provider.rs | 562 ++++++++++++++++++++++- crates/toolpath-openclaw/src/types.rs | 14 +- 2 files changed, 564 insertions(+), 12 deletions(-) diff --git a/crates/toolpath-openclaw/src/provider.rs b/crates/toolpath-openclaw/src/provider.rs index 5a692a8d..5a91ed39 100644 --- a/crates/toolpath-openclaw/src/provider.rs +++ b/crates/toolpath-openclaw/src/provider.rs @@ -1,6 +1,564 @@ -//! `ConversationView` derivation (`session_to_view`) + tool classification. +//! Forward derivation: [`OpenClawSession`] -> [`ConversationView`]. //! -//! Filled in Phase 4, Task 6. +//! Mirrors the two-pass structure used by the other providers: pass 1 emits a +//! [`Turn`] per entry and records tool-call locations; pass 2 folds each +//! `toolResult` entry onto its matching invocation. The tree (`id`/`parentId`) +//! is preserved on `Turn.parent_id` and turned into a step DAG by +//! [`toolpath_convo::derive_path`]. + +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; +use std::collections::{HashMap, HashSet}; + +use toolpath_convo::{ + ConversationView, DelegatedWork, EnvironmentSnapshot, FileMutation, Role, SessionBase, + TokenUsage, ToolCategory, ToolInvocation, ToolResult, Turn, +}; + +use crate::paths::ParsedKey; +use crate::reader::OpenClawSession; +use crate::types::{AgentMessage, ContentBlock, Entry, StopReason, Usage}; /// Provider identifier written to `path.meta.source`. pub const PROVIDER_ID: &str = "openclaw"; + +/// Classify an OpenClaw tool name into a cross-harness [`ToolCategory`]. +/// +/// Tool names are free-form; the matching is case-insensitive and lenient. +pub fn classify_tool(name: &str) -> Option { + let lower = name.to_lowercase(); + if lower.contains("task") || lower.contains("agent") || lower.contains("subagent") { + return Some(ToolCategory::Delegation); + } + match lower.as_str() { + "read" | "read_file" | "readfile" | "cat" => Some(ToolCategory::FileRead), + "write" | "write_file" | "writefile" | "edit" | "edit_file" | "editfile" | "apply_patch" + | "str_replace" => Some(ToolCategory::FileWrite), + "bash" | "shell" | "run" | "exec" | "run_shell" | "run_command" => { + Some(ToolCategory::Shell) + } + "grep" | "glob" | "find" | "ls" | "list_directory" | "search" => { + Some(ToolCategory::FileSearch) + } + "webfetch" | "websearch" | "fetch" | "web_search" | "web_fetch" => { + Some(ToolCategory::Network) + } + _ => None, + } +} + +/// Reverse of [`classify_tool`]: pick an OpenClaw-native tool name for a +/// generic [`ToolCategory`], disambiguating by call args. Used by the +/// projector when projecting tool calls from foreign harnesses. The names are +/// best-effort for OpenClaw's vocabulary. +pub fn native_name(category: ToolCategory, args: &Value) -> Option<&'static str> { + match category { + ToolCategory::Shell => Some("bash"), + ToolCategory::FileRead => Some("read_file"), + ToolCategory::FileSearch => Some(if args.get("pattern").is_some() { + "grep" + } else { + "glob" + }), + ToolCategory::FileWrite => Some( + if args.get("old").is_some() + || args.get("old_string").is_some() + || args.get("edits").is_some() + { + "edit_file" + } else { + "write_file" + }, + ), + ToolCategory::Network => Some(if args.get("url").is_some() { + "web_fetch" + } else { + "web_search" + }), + ToolCategory::Delegation => Some("task"), + } +} + +fn extract_prompt(args: &Value) -> String { + for key in ["prompt", "input", "instructions", "task"] { + if let Some(s) = args.get(key).and_then(|v| v.as_str()) { + return s.to_string(); + } + } + args.to_string() +} + +fn extract_file_path(args: &Value) -> Option { + for key in ["file_path", "path", "filename", "file"] { + if let Some(s) = args.get(key).and_then(|v| v.as_str()) { + return Some(s.to_string()); + } + } + None +} + +/// `update` for in-place edits, `add` for whole-file writes. +fn write_operation(name: &str, args: &Value) -> String { + let lower = name.to_lowercase(); + if lower.contains("edit") + || lower.contains("patch") + || lower.contains("replace") + || args.get("old").is_some() + || args.get("old_string").is_some() + || args.get("edits").is_some() + { + "update".to_string() + } else { + "add".to_string() + } +} + +fn parse_ts(ts: &str) -> Option> { + DateTime::parse_from_rfc3339(ts) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn stop_reason_to_string(sr: &StopReason) -> String { + match serde_json::to_value(sr).ok() { + Some(Value::String(s)) => s, + _ => format!("{sr:?}").to_lowercase(), + } +} + +/// An all-zero `usage` decodes as "no usage recorded", not `Some(zeros)`. +fn usage_to_token_usage(usage: &Usage) -> Option { + if usage.input == 0 && usage.output == 0 && usage.cache_read == 0 && usage.cache_write == 0 { + return None; + } + Some(TokenUsage { + input_tokens: Some(usage.input as u32), + output_tokens: Some(usage.output as u32), + cache_read_tokens: (usage.cache_read > 0).then_some(usage.cache_read as u32), + cache_write_tokens: (usage.cache_write > 0).then_some(usage.cache_write as u32), + ..Default::default() + }) +} + +fn environment_for(session: &OpenClawSession) -> EnvironmentSnapshot { + EnvironmentSnapshot { + working_dir: Some(session.header.cwd.clone()), + vcs_branch: None, + vcs_revision: None, + } +} + +/// A channel-aware human actor string for this session, if its routing key is +/// known. DMs become `human:/`; groups become +/// `human:/group/`; a peer without a channel becomes +/// `human:`. Returns `None` for main/CLI sessions (the caller then +/// falls back to the default `human:user`). +pub fn user_actor_for(parsed: Option<&ParsedKey>) -> Option { + let key = parsed?; + let peer = key.peer_id.as_deref()?; + match (&key.channel, key.peer_kind.as_deref()) { + (Some(ch), Some("group")) | (Some(ch), Some("channel")) => { + Some(format!("human:{ch}/group/{peer}")) + } + (Some(ch), _) => Some(format!("human:{ch}/{peer}")), + (None, _) => Some(format!("human:{peer}")), + } +} + +/// The session kind implied by a routing key. +fn session_kind(parsed: Option<&ParsedKey>) -> &'static str { + match parsed { + Some(k) if matches!(k.peer_kind.as_deref(), Some("group") | Some("channel")) => "group", + Some(k) if k.peer_id.is_some() => "direct", + _ => "main", + } +} + +/// OpenClaw-specific metadata to stash on `path.meta.extra["openclaw"]`. +pub fn openclaw_meta_extra(session: &OpenClawSession) -> serde_json::Map { + let mut map = serde_json::Map::new(); + let parsed = session.parsed_key.as_ref(); + if let Some(key) = &session.session_key { + map.insert("sessionKey".into(), json!(key)); + } + if let Some(p) = parsed { + if !p.agent_id.is_empty() { + map.insert("agentId".into(), json!(p.agent_id)); + } + if let Some(c) = &p.channel { + map.insert("channel".into(), json!(c)); + } + if let Some(k) = &p.peer_kind { + map.insert("peerKind".into(), json!(k)); + } + if let Some(id) = &p.peer_id { + map.insert("peerId".into(), json!(id)); + } + if let Some(t) = &p.thread_id { + map.insert("threadId".into(), json!(t)); + } + } + map.insert("sessionKind".into(), json!(session_kind(parsed))); + map +} + +/// Convert an [`OpenClawSession`] into a provider-agnostic [`ConversationView`]. +pub fn session_to_view(session: &OpenClawSession) -> ConversationView { + let env = environment_for(session); + + let mut turns: Vec = Vec::new(); + let mut tool_call_locs: HashMap = HashMap::new(); + let mut delegation_locs: HashMap = HashMap::new(); + let mut tool_results: Vec<(String, String, bool)> = Vec::new(); + + let system_turn = |id: &str, parent: &Option, ts: &str, text: String| Turn { + id: id.to_string(), + parent_id: parent.clone(), + group_id: None, + role: Role::System, + timestamp: ts.to_string(), + text, + thinking: None, + tool_uses: vec![], + model: None, + stop_reason: None, + token_usage: None, + attributed_token_usage: None, + environment: Some(env.clone()), + delegations: vec![], + file_mutations: vec![], + }; + + for entry in &session.entries { + match entry { + Entry::Session(_) + | Entry::ModelChange { .. } + | Entry::ThinkingLevelChange { .. } + | Entry::Label { .. } + | Entry::SessionInfo { .. } + | Entry::Custom { .. } + | Entry::Leaf { .. } => { + // Rendering/metadata/pointer entries: no IR turn. + } + + Entry::Compaction { base, summary, .. } => { + turns.push(system_turn( + &base.id, + &base.parent_id, + &base.timestamp, + format!("Compacted (summary): {summary}"), + )); + } + Entry::BranchSummary { base, summary, .. } => { + turns.push(system_turn( + &base.id, + &base.parent_id, + &base.timestamp, + format!("Branch summary: {summary}"), + )); + } + Entry::CustomMessage { + base, + custom_type, + content, + .. + } => { + turns.push(Turn { + role: Role::Other(format!("custom:{custom_type}")), + text: content.text(), + ..system_turn(&base.id, &base.parent_id, &base.timestamp, String::new()) + }); + } + + Entry::Message { base, message, .. } => { + let text; + let mut thinking = None; + let mut tool_uses: Vec = Vec::new(); + let mut file_mutations: Vec = Vec::new(); + let mut model: Option = None; + let mut stop_reason_s: Option = None; + let mut token_usage: Option = None; + let mut delegations: Vec = Vec::new(); + let role: Role; + + match message { + AgentMessage::User { .. } => { + role = Role::User; + text = message.text(); + } + AgentMessage::Assistant { + content, + model: m, + usage, + stop_reason, + .. + } => { + role = Role::Assistant; + text = message.text(); + thinking = message.thinking(); + model = Some(m.clone()); + stop_reason_s = Some(stop_reason_to_string(stop_reason)); + token_usage = usage_to_token_usage(usage); + + let turn_idx = turns.len(); + for block in content { + let ContentBlock::ToolCall { + id, + name, + arguments, + .. + } = block + else { + continue; + }; + let category = classify_tool(name); + let tool_idx = tool_uses.len(); + tool_call_locs.insert(id.clone(), (turn_idx, tool_idx)); + + if category == Some(ToolCategory::Delegation) { + let deleg_idx = delegations.len(); + delegations.push(DelegatedWork { + agent_id: id.clone(), + prompt: extract_prompt(arguments), + turns: vec![], + result: None, + }); + delegation_locs.insert(id.clone(), (turn_idx, deleg_idx)); + } + + if category == Some(ToolCategory::FileWrite) + && let Some(path) = extract_file_path(arguments) + { + file_mutations.push(FileMutation { + path, + tool_id: Some(id.clone()), + operation: Some(write_operation(name, arguments)), + raw_diff: None, + before: None, + after: None, + rename_to: None, + }); + } + + tool_uses.push(ToolInvocation { + id: id.clone(), + name: name.clone(), + input: arguments.clone(), + result: None, + category, + }); + } + } + AgentMessage::ToolResult { + tool_call_id, + is_error, + .. + } => { + tool_results.push((tool_call_id.clone(), message.text(), *is_error)); + continue; + } + AgentMessage::BashExecution { + command, + output, + exit_code, + .. + } => { + role = Role::Other("bash".to_string()); + text = format!("$ {command}\n{output}"); + tool_uses.push(ToolInvocation { + id: base.id.clone(), + name: "bash".to_string(), + input: json!({ "command": command }), + result: Some(ToolResult { + content: output.clone(), + is_error: !matches!(exit_code, Some(0)), + }), + category: Some(ToolCategory::Shell), + }); + } + } + + turns.push(Turn { + id: base.id.clone(), + parent_id: base.parent_id.clone(), + group_id: None, + role, + timestamp: base.timestamp.clone(), + text, + thinking, + tool_uses, + model, + stop_reason: stop_reason_s, + token_usage, + attributed_token_usage: None, + environment: Some(env.clone()), + delegations, + file_mutations, + }); + } + } + } + + // Pass 2: fold tool results onto their invocations. + for (tool_call_id, content, is_error) in &tool_results { + if let Some((turn_idx, tool_idx)) = tool_call_locs.get(tool_call_id) + && let Some(inv) = turns + .get_mut(*turn_idx) + .and_then(|t| t.tool_uses.get_mut(*tool_idx)) + { + inv.result = Some(ToolResult { + content: content.clone(), + is_error: *is_error, + }); + } + if let Some((turn_idx, deleg_idx)) = delegation_locs.get(tool_call_id) + && let Some(d) = turns + .get_mut(*turn_idx) + .and_then(|t| t.delegations.get_mut(*deleg_idx)) + { + d.result = Some(content.clone()); + } + } + + // Aggregate token usage across assistant turns. + let mut have_usage = false; + let mut total = TokenUsage::default(); + for turn in &turns { + if let Some(u) = &turn.token_usage { + have_usage = true; + total.input_tokens = Some(total.input_tokens.unwrap_or(0) + u.input_tokens.unwrap_or(0)); + total.output_tokens = + Some(total.output_tokens.unwrap_or(0) + u.output_tokens.unwrap_or(0)); + if let Some(r) = u.cache_read_tokens { + total.cache_read_tokens = Some(total.cache_read_tokens.unwrap_or(0) + r); + } + if let Some(w) = u.cache_write_tokens { + total.cache_write_tokens = Some(total.cache_write_tokens.unwrap_or(0) + w); + } + } + } + let total_usage = have_usage.then_some(total); + + // files_changed: dedup-in-order from FileWrite tool inputs. + let mut files_changed: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for turn in &turns { + for inv in &turn.tool_uses { + if inv.category == Some(ToolCategory::FileWrite) + && let Some(p) = extract_file_path(&inv.input) + && seen.insert(p.clone()) + { + files_changed.push(p); + } + } + } + + let started_at = parse_ts(&session.header.timestamp); + let last_activity = turns.last().and_then(|t| parse_ts(&t.timestamp)); + let base = (!session.header.cwd.is_empty()).then(|| SessionBase { + working_dir: Some(session.header.cwd.clone()), + ..Default::default() + }); + + ConversationView { + id: session.header.id.clone(), + started_at, + last_activity, + turns, + total_usage, + provider_id: Some(PROVIDER_ID.to_string()), + files_changed, + session_ids: session.session_id_chain(), + events: vec![], + base, + ..Default::default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reader::read_session_from_file; + use std::path::Path; + + fn fixture_view() -> ConversationView { + let mut s = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + s.attach_routing_key(); + session_to_view(&s) + } + + #[test] + fn view_has_roles_tools_and_usage() { + let v = fixture_view(); + assert_eq!(v.provider_id.as_deref(), Some("openclaw")); + assert!(v.turns.iter().any(|t| t.role == Role::User)); + + let asst = v + .turns + .iter() + .find(|t| t.role == Role::Assistant && !t.tool_uses.is_empty()) + .unwrap(); + let read = &asst.tool_uses[0]; + assert_eq!(read.name, "read_file"); + assert_eq!(read.category, Some(ToolCategory::FileRead)); + assert!(read.result.is_some(), "tool result correlated"); + assert_eq!(read.result.as_ref().unwrap().content, "file contents of x.ts"); + assert!(asst.token_usage.is_some()); + } + + #[test] + fn edit_tool_emits_structural_file_mutation_without_raw() { + let v = fixture_view(); + let mutation = v + .turns + .iter() + .flat_map(|t| &t.file_mutations) + .find(|m| m.path == "src/x.ts") + .expect("file mutation for the edit"); + assert_eq!(mutation.operation.as_deref(), Some("update")); + assert!(mutation.raw_diff.is_none(), "no raw diff perspective"); + assert_eq!(mutation.tool_id.as_deref(), Some("call_2")); + assert_eq!(v.files_changed, vec!["src/x.ts"]); + } + + #[test] + fn total_usage_sums_assistant_turns() { + let v = fixture_view(); + let tu = v.total_usage.unwrap(); + // 1200 + 1500 input, 340 + 120 output + assert_eq!(tu.input_tokens, Some(2700)); + assert_eq!(tu.output_tokens, Some(460)); + } + + #[test] + fn user_actor_for_dm_and_group_and_main() { + assert_eq!( + user_actor_for(Some(&crate::paths::parse_session_key( + "agent:main:whatsapp:direct:155" + ))) + .as_deref(), + Some("human:whatsapp/155") + ); + assert_eq!( + user_actor_for(Some(&crate::paths::parse_session_key( + "agent:main:slack:group:T42" + ))) + .as_deref(), + Some("human:slack/group/T42") + ); + assert_eq!( + user_actor_for(Some(&crate::paths::parse_session_key("agent:main:main"))), + None + ); + } + + #[test] + fn meta_extra_carries_channel_and_kind() { + let mut s = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + s.attach_routing_key(); + let extra = openclaw_meta_extra(&s); + assert_eq!(extra.get("channel").and_then(|v| v.as_str()), Some("whatsapp")); + assert_eq!(extra.get("peerId").and_then(|v| v.as_str()), Some("15555550123")); + assert_eq!(extra.get("sessionKind").and_then(|v| v.as_str()), Some("direct")); + } +} diff --git a/crates/toolpath-openclaw/src/types.rs b/crates/toolpath-openclaw/src/types.rs index 0c4a6f7f..d519da2e 100644 --- a/crates/toolpath-openclaw/src/types.rs +++ b/crates/toolpath-openclaw/src/types.rs @@ -409,15 +409,9 @@ pub enum AgentMessage { api: String, /// Provider id. provider: String, - /// Requested model. + /// Requested model. (`responseModel`, when present, is preserved via + /// `extra`.) model: String, - /// Concrete served model when it differs. - #[serde( - rename = "responseModel", - default, - skip_serializing_if = "Option::is_none" - )] - response_model: Option, /// Token usage for this call. usage: Usage, /// Why the turn ended. @@ -559,8 +553,8 @@ pub struct Usage { #[serde(default, rename = "totalTokens")] pub total_tokens: u64, /// Per-class cost breakdown. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cost: Option, + #[serde(default)] + pub cost: CostBreakdown, } /// Dollar cost breakdown accompanying [`Usage`]. From d7382d03b058e686b08a5eeee715b0371ddce92c Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:25:32 -0400 Subject: [PATCH 08/22] feat(openclaw): derive_path/derive_graph with channel-aware actor + meta --- crates/toolpath-openclaw/src/derive.rs | 128 ++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 2 deletions(-) diff --git a/crates/toolpath-openclaw/src/derive.rs b/crates/toolpath-openclaw/src/derive.rs index cfc8e18b..3240bd87 100644 --- a/crates/toolpath-openclaw/src/derive.rs +++ b/crates/toolpath-openclaw/src/derive.rs @@ -1,3 +1,127 @@ -//! Thin `derive_path` / `derive_graph` wrappers over `toolpath_convo`. +//! Thin wrapper: [`OpenClawSession`] -> [`ConversationView`] -> +//! [`toolpath_convo::derive_path`]. //! -//! Filled in Phase 5, Task 7. +//! Beyond the shared derivation, this layer (1) sets a channel-aware +//! `user_actor` from the session's routing key and (2) stashes OpenClaw's +//! channel/peer/session-kind metadata on `path.meta.extra["openclaw"]`. + +use serde_json::Value; +use toolpath::v1::{Graph, GraphIdentity, GraphMeta, Path, PathOrRef}; + +pub use toolpath_convo::DeriveConfig; + +use crate::provider::{openclaw_meta_extra, session_to_view, user_actor_for}; +use crate::reader::OpenClawSession; + +/// Derive a Toolpath [`Path`] from a single OpenClaw session. +pub fn derive_path(session: &OpenClawSession, config: &DeriveConfig) -> Path { + let view = session_to_view(session); + + let mut cfg = config.clone(); + if cfg.user_actor.is_none() { + cfg.user_actor = user_actor_for(session.parsed_key.as_ref()); + } + + let mut path = toolpath_convo::derive_path(&view, &cfg); + + let extra = openclaw_meta_extra(session); + if !extra.is_empty() { + let meta = path.meta.get_or_insert_with(Default::default); + meta.extra + .insert("openclaw".to_string(), Value::Object(extra)); + } + + path +} + +/// Derive a Toolpath [`Graph`] from multiple OpenClaw sessions (one path each). +pub fn derive_graph( + sessions: &[OpenClawSession], + title: Option<&str>, + config: &DeriveConfig, +) -> Graph { + let id_suffix = sessions + .first() + .map(|s| s.header.id.chars().take(8).collect::()) + .unwrap_or_else(|| "empty".to_string()); + let graph_id = format!("graph-openclaw-{id_suffix}"); + + let paths: Vec = sessions + .iter() + .map(|s| PathOrRef::Path(Box::new(derive_path(s, config)))) + .collect(); + + let meta = title.map(|t| GraphMeta { + title: Some(t.to_string()), + ..Default::default() + }); + + Graph { + graph: GraphIdentity { id: graph_id }, + paths, + meta, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reader::read_session_from_file; + use std::path::Path as FsPath; + use toolpath::v1::PATH_KIND_AGENT_CODING_SESSION; + + fn fixture(with_key: bool) -> OpenClawSession { + let mut s = read_session_from_file(FsPath::new("tests/fixtures/dm_session.jsonl")).unwrap(); + if with_key { + s.attach_routing_key(); + } + s + } + + #[test] + fn derive_sets_channel_actor_kind_and_meta() { + let s = fixture(true); + let p = derive_path(&s, &DeriveConfig::default()); + + let user_step = p + .steps + .iter() + .find(|st| st.step.actor.starts_with("human:")) + .expect("a human step"); + assert_eq!(user_step.step.actor, "human:whatsapp/15555550123"); + + let meta = p.meta.as_ref().unwrap(); + assert_eq!(meta.source.as_deref(), Some("openclaw")); + assert_eq!(meta.kind.as_deref(), Some(PATH_KIND_AGENT_CODING_SESSION)); + assert_eq!(meta.extra["openclaw"]["channel"], "whatsapp"); + assert_eq!(meta.extra["openclaw"]["sessionKind"], "direct"); + } + + #[test] + fn derive_without_routing_key_defaults_to_human_user() { + let s = fixture(false); + let p = derive_path(&s, &DeriveConfig::default()); + let user_step = p + .steps + .iter() + .find(|st| st.step.actor.starts_with("human:")) + .expect("a human step"); + assert_eq!(user_step.step.actor, "human:user"); + // No routing key → no openclaw channel metadata, but sessionKind present. + let meta = p.meta.as_ref().unwrap(); + assert_eq!(meta.extra["openclaw"]["sessionKind"], "main"); + } + + #[test] + fn derive_graph_basics() { + let s = fixture(true); + let g = derive_graph(std::slice::from_ref(&s), Some("Release"), &DeriveConfig::default()); + assert_eq!(g.paths.len(), 1); + assert_eq!(g.meta.as_ref().and_then(|m| m.title.as_deref()), Some("Release")); + assert!(g.graph.id.starts_with("graph-openclaw-")); + + let empty = derive_graph(&[], None, &DeriveConfig::default()); + assert!(empty.paths.is_empty()); + assert_eq!(empty.graph.id, "graph-openclaw-empty"); + } +} From a9538ab6a7c2dc3a288ff89f1a7c168ccdbf54ea Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:27:43 -0400 Subject: [PATCH 09/22] feat(openclaw): OpenClawConvo manager + ConversationProvider + derive_project --- crates/toolpath-openclaw/src/derive.rs | 11 ++ crates/toolpath-openclaw/src/lib.rs | 145 +++++++++++++++++++++++ crates/toolpath-openclaw/src/provider.rs | 63 +++++++++- 3 files changed, 217 insertions(+), 2 deletions(-) diff --git a/crates/toolpath-openclaw/src/derive.rs b/crates/toolpath-openclaw/src/derive.rs index 3240bd87..68356cd4 100644 --- a/crates/toolpath-openclaw/src/derive.rs +++ b/crates/toolpath-openclaw/src/derive.rs @@ -63,6 +63,17 @@ pub fn derive_graph( } } +/// Derive a [`Graph`] from all sessions under one agent. +pub fn derive_project( + manager: &crate::OpenClawConvo, + agent_id: &str, + title: Option<&str>, + config: &DeriveConfig, +) -> crate::Result { + let sessions = manager.read_all_sessions(agent_id)?; + Ok(derive_graph(&sessions, title, config)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/toolpath-openclaw/src/lib.rs b/crates/toolpath-openclaw/src/lib.rs index 7d79acc9..41e4e104 100644 --- a/crates/toolpath-openclaw/src/lib.rs +++ b/crates/toolpath-openclaw/src/lib.rs @@ -10,4 +10,149 @@ pub mod provider; pub mod reader; pub mod types; +pub use derive::{derive_graph, derive_path, derive_project}; pub use error::{OpenClawError, Result}; +pub use io::SessionMeta; +pub use paths::{DEFAULT_AGENT_ID, ParsedKey, PathResolver}; +pub use provider::{PROVIDER_ID, session_to_view}; +pub use reader::OpenClawSession; +pub use toolpath_convo::DeriveConfig; +pub use types::{ + AgentMessage, ContentBlock, CostBreakdown, Entry, EntryBase, SessionHeader, Usage, +}; + +use toolpath_convo::ConversationView; + +/// How many `parentSession` links to follow when reading a session. +const MAX_PARENT_DEPTH: usize = 32; + +/// High-level interface for reading OpenClaw sessions from a state directory. +/// +/// OpenClaw buckets sessions by agent persona (`agents//sessions/`), +/// so the "project" axis used by [`toolpath_convo::ConversationProvider`] is +/// the **agent id** here (default [`DEFAULT_AGENT_ID`]). +#[derive(Debug, Clone, Default)] +pub struct OpenClawConvo { + resolver: PathResolver, +} + +impl OpenClawConvo { + /// Build a manager with the default resolver (`~/.openclaw`). + pub fn new() -> Self { + Self { + resolver: PathResolver::new(), + } + } + + /// Build a manager with a custom resolver (useful for tests / `--base`). + pub fn with_resolver(resolver: PathResolver) -> Self { + Self { resolver } + } + + /// Access the underlying resolver. + pub fn resolver(&self) -> &PathResolver { + &self.resolver + } + + /// Whether the OpenClaw `agents/` directory exists on disk. + pub fn exists(&self) -> bool { + self.resolver.agents_dir().exists() + } + + /// List agent ids that have a sessions directory. + pub fn list_agents(&self) -> Result> { + Ok(self.resolver.list_agent_ids()?) + } + + /// List session metadata for one agent, newest first. + pub fn list_sessions(&self, agent_id: &str) -> Result> { + io::list_sessions(&self.resolver, agent_id) + } + + /// List session metadata across every agent, newest first. + pub fn list_all_sessions(&self) -> Result> { + io::list_all_sessions(&self.resolver) + } + + /// Read a specific session by id under an agent, following `parentSession` + /// links and attaching the routing key from `sessions.json`. + pub fn read_session(&self, agent_id: &str, session_id: &str) -> Result { + let file = self.resolver.resolve_session_file(agent_id, session_id)?; + let mut session = reader::read_session_with_parent(&file, MAX_PARENT_DEPTH)?; + session.attach_routing_key(); + Ok(session) + } + + /// Read the most recently active session for an agent, if any. + pub fn most_recent_session(&self, agent_id: &str) -> Result> { + let mut metas = self.list_sessions(agent_id)?; + if let Some(meta) = metas.drain(..).next() { + Ok(Some(self.read_session(agent_id, &meta.id)?)) + } else { + Ok(None) + } + } + + /// Convert a session into a provider-agnostic [`ConversationView`]. + pub fn to_view(&self, session: &OpenClawSession) -> ConversationView { + provider::session_to_view(session) + } + + /// Read every session for an agent (skipping unreadable ones with a warning). + pub fn read_all_sessions(&self, agent_id: &str) -> Result> { + let metas = self.list_sessions(agent_id)?; + let mut sessions = Vec::new(); + for meta in metas { + match self.read_session(agent_id, &meta.id) { + Ok(s) => sessions.push(s), + Err(e) => eprintln!("Warning: failed to read OpenClaw session {}: {}", meta.id, e), + } + } + Ok(sessions) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a temp state dir with the DM fixture under agents/main/sessions. + fn temp_state() -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + let fixture = std::fs::read("tests/fixtures/dm_session.jsonl").unwrap(); + std::fs::write(dir.join("sess-abc.jsonl"), fixture).unwrap(); + std::fs::copy("tests/fixtures/sessions.json", dir.join("sessions.json")).unwrap(); + tmp + } + + #[test] + fn manager_reads_and_derives() { + let tmp = temp_state(); + let mgr = OpenClawConvo::with_resolver(PathResolver::with_state_dir(tmp.path())); + + assert_eq!(mgr.list_agents().unwrap(), vec!["main".to_string()]); + let sessions = mgr.list_sessions("main").unwrap(); + assert_eq!(sessions.len(), 1); + + let s = mgr.read_session("main", "sess-abc").unwrap(); + assert_eq!(s.parsed_key.as_ref().unwrap().channel.as_deref(), Some("whatsapp")); + + let path = derive_path(&s, &DeriveConfig::default()); + assert_eq!(path.meta.as_ref().unwrap().source.as_deref(), Some("openclaw")); + assert!( + path.steps + .iter() + .any(|st| st.step.actor == "human:whatsapp/15555550123") + ); + } + + #[test] + fn most_recent_session_works() { + let tmp = temp_state(); + let mgr = OpenClawConvo::with_resolver(PathResolver::with_state_dir(tmp.path())); + let s = mgr.most_recent_session("main").unwrap().unwrap(); + assert_eq!(s.header.id, "sess-abc"); + } +} diff --git a/crates/toolpath-openclaw/src/provider.rs b/crates/toolpath-openclaw/src/provider.rs index 5a91ed39..16c3e8e2 100644 --- a/crates/toolpath-openclaw/src/provider.rs +++ b/crates/toolpath-openclaw/src/provider.rs @@ -11,10 +11,12 @@ use serde_json::{Value, json}; use std::collections::{HashMap, HashSet}; use toolpath_convo::{ - ConversationView, DelegatedWork, EnvironmentSnapshot, FileMutation, Role, SessionBase, - TokenUsage, ToolCategory, ToolInvocation, ToolResult, Turn, + ConversationMeta, ConversationProvider, ConversationView, ConvoError, DelegatedWork, + EnvironmentSnapshot, FileMutation, Role, SessionBase, TokenUsage, ToolCategory, ToolInvocation, + ToolResult, Turn, }; +use crate::OpenClawConvo; use crate::paths::ParsedKey; use crate::reader::OpenClawSession; use crate::types::{AgentMessage, ContentBlock, Entry, StopReason, Usage}; @@ -475,6 +477,63 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { } } +// ── ConversationProvider impl ──────────────────────────────────────── + +fn to_convo_err(e: crate::error::OpenClawError) -> ConvoError { + ConvoError::Provider(e.to_string()) +} + +fn meta_to_conversation_meta(meta: crate::io::SessionMeta) -> ConversationMeta { + let ts = parse_ts(&meta.timestamp); + ConversationMeta { + id: meta.id, + started_at: ts, + last_activity: ts, + message_count: meta.entry_count, + file_path: Some(meta.file_path), + predecessor: None, + successor: None, + } +} + +/// The `project` argument is the OpenClaw **agent id** (default +/// [`crate::DEFAULT_AGENT_ID`]). +impl ConversationProvider for OpenClawConvo { + fn list_conversations(&self, project: &str) -> Result, ConvoError> { + let metas = self.list_sessions(project).map_err(to_convo_err)?; + Ok(metas.into_iter().map(|m| m.id).collect()) + } + + fn load_conversation( + &self, + project: &str, + conversation_id: &str, + ) -> Result { + let session = self + .read_session(project, conversation_id) + .map_err(to_convo_err)?; + Ok(session_to_view(&session)) + } + + fn load_metadata( + &self, + project: &str, + conversation_id: &str, + ) -> Result { + let metas = self.list_sessions(project).map_err(to_convo_err)?; + let meta = metas + .into_iter() + .find(|m| m.id == conversation_id) + .ok_or_else(|| ConvoError::Provider(format!("session not found: {conversation_id}")))?; + Ok(meta_to_conversation_meta(meta)) + } + + fn list_metadata(&self, project: &str) -> Result, ConvoError> { + let metas = self.list_sessions(project).map_err(to_convo_err)?; + Ok(metas.into_iter().map(meta_to_conversation_meta).collect()) + } +} + #[cfg(test)] mod tests { use super::*; From a31169881a3a5db1cdf63ee629d665683e21257d Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:44:25 -0400 Subject: [PATCH 10/22] feat(openclaw): projector + sessions.json inception + full round-trip test --- crates/toolpath-openclaw/src/paths.rs | 6 +- crates/toolpath-openclaw/src/project.rs | 626 +++++++++++++++++- .../tests/projection_roundtrip.rs | 81 +++ 3 files changed, 708 insertions(+), 5 deletions(-) create mode 100644 crates/toolpath-openclaw/tests/projection_roundtrip.rs diff --git a/crates/toolpath-openclaw/src/paths.rs b/crates/toolpath-openclaw/src/paths.rs index 5bb8c02d..6de78e28 100644 --- a/crates/toolpath-openclaw/src/paths.rs +++ b/crates/toolpath-openclaw/src/paths.rs @@ -87,14 +87,14 @@ pub fn parse_session_key(key: &str) -> ParsedKey { } /// One entry in `sessions.json` (keyed by routing key). -#[derive(Debug, Clone, Default, serde::Deserialize)] +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct IndexEntry { /// The transcript session id (filename stem). - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub session_id: Option, /// The transcript filename (relative to the sessions dir, or absolute). - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub session_file: Option, /// Anything else OpenClaw stores per session. #[serde(flatten)] diff --git a/crates/toolpath-openclaw/src/project.rs b/crates/toolpath-openclaw/src/project.rs index 57e3005d..2cac77cd 100644 --- a/crates/toolpath-openclaw/src/project.rs +++ b/crates/toolpath-openclaw/src/project.rs @@ -1,3 +1,625 @@ -//! Projector: `ConversationView` -> OpenClaw session JSONL (inception). +//! [`OpenClawProjector`] — maps a [`ConversationView`] back into an OpenClaw +//! [`OpenClawSession`], and writes it to disk as a resume-ready ("incepted") +//! session under `agents//sessions/` plus a `sessions.json` routing +//! entry so a running OpenClaw instance can pick it up. //! -//! Filled in Phase 6, Task 9. +//! This is the inverse of [`crate::provider::session_to_view`]. Provider- +//! specific extras are not carried on the IR, so the projector synthesizes +//! OpenClaw fields from typed IR fields and sensible defaults (api/provider +//! `anthropic`, stop reason `stop`). Foreign tool names are remapped through +//! [`crate::provider::native_name`]. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; + +use serde_json::{Value, json}; +use toolpath_convo::{ + ConversationProjector, ConversationView, Result, Role, ToolInvocation, Turn, +}; + +use crate::error::OpenClawError; +use crate::paths::{IndexEntry, SessionsIndex, normalize_agent_id}; +use crate::reader::OpenClawSession; +use crate::types::{ + AgentMessage, ContentBlock, CostBreakdown, Entry, EntryBase, KnownStopReason, MessageContent, + SessionHeader, StopReason, ToolResultContent, Usage, +}; + +/// Project a [`ConversationView`] into an OpenClaw [`OpenClawSession`] and +/// optionally write ("incept") it into a state directory. +#[derive(Debug, Clone, Default)] +pub struct OpenClawProjector { + /// Override the session header's `cwd` (else pulled from the first turn's + /// environment, falling back to `/`). + pub cwd: Option, + /// Default `api` for assistant turns from a non-OpenClaw source. + pub default_api: Option, + /// Default `provider` for assistant turns from a non-OpenClaw source. + pub default_provider: Option, + /// Agent bucket the incepted session belongs to (default `main`). + pub agent_id: Option, + /// Channel for the inception routing key (e.g. `whatsapp`). + pub channel: Option, + /// Peer id for the inception routing key. + pub peer_id: Option, +} + +impl OpenClawProjector { + /// A projector with all defaults. + pub fn new() -> Self { + Self::default() + } + + /// Override the header `cwd`. + pub fn with_cwd(mut self, cwd: impl Into) -> Self { + self.cwd = Some(cwd.into()); + self + } + + /// Default `api` for assistant turns lacking one. + pub fn with_default_api(mut self, api: impl Into) -> Self { + self.default_api = Some(api.into()); + self + } + + /// Default `provider` for assistant turns lacking one. + pub fn with_default_provider(mut self, provider: impl Into) -> Self { + self.default_provider = Some(provider.into()); + self + } + + /// The agent bucket for inception. + pub fn with_agent(mut self, agent_id: impl Into) -> Self { + self.agent_id = Some(agent_id.into()); + self + } + + /// The channel for the inception routing key. + pub fn with_channel(mut self, channel: impl Into) -> Self { + self.channel = Some(channel.into()); + self + } + + /// The peer id for the inception routing key. + pub fn with_peer(mut self, peer_id: impl Into) -> Self { + self.peer_id = Some(peer_id.into()); + self + } + + /// Configure channel/peer/agent from a Path's `meta.extra["openclaw"]` + /// object (as written by [`crate::derive::derive_path`]). + pub fn with_meta_extra(mut self, extra: &Value) -> Self { + if let Some(c) = extra.get("channel").and_then(Value::as_str) { + self.channel = Some(c.to_string()); + } + if let Some(p) = extra.get("peerId").and_then(Value::as_str) { + self.peer_id = Some(p.to_string()); + } + if let Some(a) = extra.get("agentId").and_then(Value::as_str) { + self.agent_id = Some(a.to_string()); + } + self + } + + /// The effective agent id (default `main`). + fn effective_agent(&self) -> String { + normalize_agent_id(self.agent_id.as_deref().unwrap_or(crate::DEFAULT_AGENT_ID)) + } + + /// Write a projected session to `/agents//sessions/` + /// and upsert a `sessions.json` routing entry. Returns the transcript path. + pub fn write_session( + &self, + session: &OpenClawSession, + state_dir: &Path, + ) -> crate::error::Result { + let agent_id = self.effective_agent(); + let dir = state_dir.join("agents").join(&agent_id).join("sessions"); + std::fs::create_dir_all(&dir)?; + + let file_name = format!("{}.jsonl", session.header.id); + let file = dir.join(&file_name); + + let mut out = String::new(); + // The projected session's first entry is the header; write entries + // verbatim, one JSON object per line. + for entry in &session.entries { + out.push_str(&serde_json::to_string(entry).map_err(OpenClawError::Json)?); + out.push('\n'); + } + write_private(&file, out.as_bytes())?; + + self.upsert_routing(&dir, &agent_id, &session.header.id, &file_name)?; + Ok(file) + } + + fn upsert_routing( + &self, + dir: &Path, + agent_id: &str, + session_id: &str, + file_name: &str, + ) -> crate::error::Result<()> { + let key = match (&self.channel, &self.peer_id) { + (Some(ch), Some(peer)) => format!("agent:{agent_id}:{ch}:direct:{peer}"), + _ => format!("agent:{agent_id}:main"), + }; + + let mut map: BTreeMap = SessionsIndex::load(dir).map(|i| i.0).unwrap_or_default(); + map.insert( + key, + IndexEntry { + session_id: Some(session_id.to_string()), + session_file: Some(file_name.to_string()), + extra: HashMap::new(), + }, + ); + let json = serde_json::to_vec_pretty(&map).map_err(OpenClawError::Json)?; + write_private(&dir.join("sessions.json"), &json)?; + Ok(()) + } +} + +impl ConversationProjector for OpenClawProjector { + type Output = OpenClawSession; + + fn project(&self, view: &ConversationView) -> Result { + Ok(project_view(self, view)) + } +} + +// ── Projection logic ───────────────────────────────────────────────── + +fn project_view(cfg: &OpenClawProjector, view: &ConversationView) -> OpenClawSession { + let cwd = cfg + .cwd + .clone() + .or_else(|| { + view.turns + .iter() + .find_map(|t| t.environment.as_ref()?.working_dir.clone()) + }) + .unwrap_or_else(|| "/".to_string()); + + let timestamp = view + .started_at + .map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)) + .or_else(|| view.turns.first().map(|t| t.timestamp.clone())) + .unwrap_or_default(); + + let header = SessionHeader { + version: 3, + id: view.id.clone(), + timestamp, + cwd, + parent_session: None, + extra: HashMap::new(), + }; + + let mut entries: Vec = vec![Entry::Session(header.clone())]; + for turn in &view.turns { + emit_turn_entries(cfg, turn, &mut entries); + } + + // Append a trailing `leaf` row pointing at the last content entry so the + // incepted session's visible head is well defined. + if let Some(last_id) = entries + .iter() + .rev() + .find_map(|e| e.base().map(|b| b.id.clone())) + { + entries.push(Entry::Leaf { + base: EntryBase { + id: format!("{last_id}-leaf"), + parent_id: Some(last_id.clone()), + timestamp: header.timestamp.clone(), + append_mode: None, + }, + target_id: Some(last_id), + append_parent_id: None, + extra: HashMap::new(), + }); + } + + OpenClawSession { + header, + entries, + file_path: PathBuf::new(), + parent: None, + session_key: None, + parsed_key: None, + } +} + +fn emit_turn_entries(cfg: &OpenClawProjector, turn: &Turn, entries: &mut Vec) { + match &turn.role { + Role::User => emit_user(turn, entries), + Role::Assistant => emit_assistant(cfg, turn, entries), + Role::System => emit_system(turn, entries), + Role::Other(other) => match other.as_str() { + "bash" => emit_bash(turn, entries), + o if o.starts_with("custom:") => { + emit_custom_message(turn, o.strip_prefix("custom:").unwrap_or(o), entries) + } + o => emit_custom_message(turn, o, entries), + }, + } +} + +fn emit_user(turn: &Turn, entries: &mut Vec) { + entries.push(Entry::Message { + base: base_for(turn), + message: AgentMessage::User { + content: MessageContent::Text(turn.text.clone()), + timestamp: ts_millis(&turn.timestamp), + extra: HashMap::new(), + }, + extra: HashMap::new(), + }); +} + +fn emit_assistant(cfg: &OpenClawProjector, turn: &Turn, entries: &mut Vec) { + let mut blocks: Vec = Vec::new(); + if let Some(t) = &turn.thinking + && !t.is_empty() + { + blocks.push(ContentBlock::Thinking { + thinking: t.clone(), + extra: HashMap::new(), + }); + } + if !turn.text.is_empty() { + blocks.push(ContentBlock::Text { + text: turn.text.clone(), + extra: HashMap::new(), + }); + } + for tu in &turn.tool_uses { + blocks.push(ContentBlock::ToolCall { + id: tu.id.clone(), + name: tool_native_name(tu), + arguments: tu.input.clone(), + extra: HashMap::new(), + }); + } + + let api = cfg + .default_api + .clone() + .unwrap_or_else(|| "anthropic-messages".to_string()); + let provider = cfg + .default_provider + .clone() + .unwrap_or_else(|| "anthropic".to_string()); + + entries.push(Entry::Message { + base: base_for(turn), + message: AgentMessage::Assistant { + content: blocks, + api, + provider, + model: turn.model.clone().unwrap_or_default(), + usage: build_usage(turn), + stop_reason: parse_stop_reason(turn.stop_reason.as_deref()), + error_message: None, + timestamp: ts_millis(&turn.timestamp), + extra: HashMap::new(), + }, + extra: HashMap::new(), + }); + + // One separate toolResult entry per invocation with a result. + let mut prev_id = turn.id.clone(); + let mut suffix = 0usize; + for tu in &turn.tool_uses { + let Some(result) = &tu.result else { continue }; + suffix += 1; + let tr_id = format!("{}-tr-{}", turn.id, suffix); + entries.push(Entry::Message { + base: EntryBase { + id: tr_id.clone(), + parent_id: Some(prev_id.clone()), + timestamp: turn.timestamp.clone(), + append_mode: None, + }, + message: AgentMessage::ToolResult { + tool_call_id: tu.id.clone(), + tool_name: tool_native_name(tu), + content: vec![ToolResultContent::Text { + text: result.content.clone(), + extra: HashMap::new(), + }], + details: None, + is_error: result.is_error, + timestamp: ts_millis(&turn.timestamp), + extra: HashMap::new(), + }, + extra: HashMap::new(), + }); + prev_id = tr_id; + } +} + +fn emit_system(turn: &Turn, entries: &mut Vec) { + if let Some(rest) = turn.text.strip_prefix("Compacted (summary): ") { + entries.push(Entry::Compaction { + base: base_for(turn), + summary: rest.to_string(), + first_kept_entry_id: String::new(), + tokens_before: 0, + details: None, + from_hook: None, + extra: HashMap::new(), + }); + } else if let Some(rest) = turn.text.strip_prefix("Branch summary: ") { + entries.push(Entry::BranchSummary { + base: base_for(turn), + from_id: "root".to_string(), + summary: rest.to_string(), + details: None, + from_hook: None, + extra: HashMap::new(), + }); + } else { + emit_custom_message(turn, "system", entries); + } +} + +fn emit_bash(turn: &Turn, entries: &mut Vec) { + let (command, output) = match turn.text.strip_prefix("$ ") { + Some(rest) => match rest.split_once('\n') { + Some((cmd, out)) => (cmd.to_string(), out.to_string()), + None => (rest.to_string(), String::new()), + }, + None => (String::new(), turn.text.clone()), + }; + entries.push(Entry::Message { + base: base_for(turn), + message: AgentMessage::BashExecution { + command, + output, + exit_code: Some(0), + timestamp: ts_millis(&turn.timestamp), + extra: HashMap::new(), + }, + extra: HashMap::new(), + }); +} + +fn emit_custom_message(turn: &Turn, custom_type: &str, entries: &mut Vec) { + entries.push(Entry::CustomMessage { + base: base_for(turn), + custom_type: custom_type.to_string(), + content: MessageContent::Text(turn.text.clone()), + display: true, + details: None, + extra: HashMap::new(), + }); +} + +// ── Helpers ────────────────────────────────────────────────────────── + +fn base_for(turn: &Turn) -> EntryBase { + EntryBase { + id: turn.id.clone(), + parent_id: turn.parent_id.clone(), + timestamp: turn.timestamp.clone(), + append_mode: None, + } +} + +fn ts_millis(rfc3339: &str) -> u64 { + chrono::DateTime::parse_from_rfc3339(rfc3339) + .map(|dt| dt.timestamp_millis().max(0) as u64) + .unwrap_or(0) +} + +fn build_usage(turn: &Turn) -> Usage { + let (input, output, cache_read, cache_write) = turn + .token_usage + .as_ref() + .map(|u| { + ( + u.input_tokens.unwrap_or(0) as u64, + u.output_tokens.unwrap_or(0) as u64, + u.cache_read_tokens.unwrap_or(0) as u64, + u.cache_write_tokens.unwrap_or(0) as u64, + ) + }) + .unwrap_or((0, 0, 0, 0)); + Usage { + input, + output, + cache_read, + cache_write, + total_tokens: input + output, + cost: CostBreakdown::default(), + } +} + +fn parse_stop_reason(turn_stop: Option<&str>) -> StopReason { + let s = turn_stop.unwrap_or("stop"); + serde_json::from_value::(json!(s)) + .unwrap_or(StopReason::Known(KnownStopReason::Stop)) +} + +/// Route a tool name through `native_name` when its category is known; else +/// pass through verbatim (OpenClaw's format accepts any tool name). +fn tool_native_name(tu: &ToolInvocation) -> String { + if let Some(cat) = tu.category + && let Some(remap) = crate::provider::native_name(cat, &tu.input) + { + return remap.to_string(); + } + tu.name.clone() +} + +fn write_private(path: &Path, bytes: &[u8]) -> crate::error::Result<()> { + std::fs::write(path, bytes)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::session_to_view; + use crate::reader::read_session_from_file; + use std::path::Path as FsPath; + use toolpath_convo::{TokenUsage, ToolCategory, ToolInvocation, ToolResult}; + + fn user_turn(id: &str, text: &str) -> Turn { + Turn { + id: id.into(), + parent_id: None, + group_id: None, + role: Role::User, + timestamp: "2026-06-30T10:00:00Z".into(), + text: text.into(), + thinking: None, + tool_uses: vec![], + model: None, + stop_reason: None, + token_usage: None, + attributed_token_usage: None, + environment: None, + delegations: vec![], + file_mutations: Vec::new(), + } + } + + fn view_with(turns: Vec) -> ConversationView { + ConversationView { + id: "sess-1".into(), + turns, + provider_id: Some("openclaw".into()), + ..Default::default() + } + } + + #[test] + fn empty_view_projects_header_only() { + let s = OpenClawProjector::default().project(&view_with(vec![])).unwrap(); + assert_eq!(s.header.id, "sess-1"); + assert_eq!(s.entries.len(), 1); + assert!(matches!(s.entries[0], Entry::Session(_))); + } + + #[test] + fn user_turn_becomes_user_message() { + let s = OpenClawProjector::default() + .project(&view_with(vec![user_turn("u1", "hello")])) + .unwrap(); + // header + user + trailing leaf + assert!(matches!(&s.entries[1], Entry::Message { message: AgentMessage::User { .. }, .. })); + assert!(matches!(s.entries.last().unwrap(), Entry::Leaf { .. })); + } + + #[test] + fn assistant_tool_call_and_result_split() { + let mut t = user_turn("a1", "Reading."); + t.role = Role::Assistant; + t.model = Some("claude-x".into()); + t.token_usage = Some(TokenUsage { + input_tokens: Some(10), + output_tokens: Some(5), + ..Default::default() + }); + t.tool_uses = vec![ToolInvocation { + id: "tc1".into(), + name: "Read".into(), + input: json!({"path": "x"}), + result: Some(ToolResult { + content: "body".into(), + is_error: false, + }), + category: Some(ToolCategory::FileRead), + }]; + let s = OpenClawProjector::default().project(&view_with(vec![t])).unwrap(); + // assistant: foreign "Read" → native "read_file" + match &s.entries[1] { + Entry::Message { message: AgentMessage::Assistant { content, usage, .. }, .. } => { + assert!(matches!(&content[1], ContentBlock::ToolCall { name, .. } if name == "read_file")); + assert_eq!(usage.total_tokens, 15); + } + other => panic!("expected assistant, got {other:?}"), + } + // separate toolResult linked by id + match &s.entries[2] { + Entry::Message { message: AgentMessage::ToolResult { tool_call_id, tool_name, .. }, .. } => { + assert_eq!(tool_call_id, "tc1"); + assert_eq!(tool_name, "read_file"); + } + other => panic!("expected toolResult, got {other:?}"), + } + } + + #[test] + fn roundtrip_preserves_messages_tools_usage() { + let src = read_session_from_file(FsPath::new("tests/fixtures/dm_session.jsonl")).unwrap(); + let view = session_to_view(&src); + let projected = OpenClawProjector::default().project(&view).unwrap(); + assert_eq!(projected.header.version, 3); + + // Every entry serializes to a single line and re-parses as an Entry. + for entry in &projected.entries { + let line = serde_json::to_string(entry).unwrap(); + assert!(!line.contains('\n')); + let _: Entry = serde_json::from_str(&line).unwrap(); + } + + // A toolCall and its matching toolResult both survive with the same id. + let mut call_ids = std::collections::HashSet::new(); + let mut result_ids = std::collections::HashSet::new(); + for e in &projected.entries { + if let Entry::Message { message, .. } = e { + match message { + AgentMessage::Assistant { content, .. } => { + for b in content { + if let ContentBlock::ToolCall { id, .. } = b { + call_ids.insert(id.clone()); + } + } + } + AgentMessage::ToolResult { tool_call_id, .. } => { + result_ids.insert(tool_call_id.clone()); + } + _ => {} + } + } + } + assert!(call_ids.contains("call_1") && result_ids.contains("call_1")); + assert!(call_ids.contains("call_2") && result_ids.contains("call_2")); + } + + #[test] + fn inception_writes_session_and_routing_entry() { + let src = read_session_from_file(FsPath::new("tests/fixtures/dm_session.jsonl")).unwrap(); + let view = session_to_view(&src); + let tmp = tempfile::tempdir().unwrap(); + let proj = OpenClawProjector::default() + .with_channel("whatsapp") + .with_peer("15555550123"); + let session = proj.project(&view).unwrap(); + let path = proj.write_session(&session, tmp.path()).unwrap(); + assert!(path.exists()); + assert_eq!( + path, + tmp.path().join("agents/main/sessions/sess-abc.jsonl") + ); + + // sessions.json got a whatsapp routing entry pointing at the file. + let idx = SessionsIndex::load(&tmp.path().join("agents/main/sessions")).unwrap(); + let (key, parsed) = idx.routing_key_for("sess-abc").unwrap(); + assert!(key.contains("whatsapp")); + assert_eq!(parsed.peer_id.as_deref(), Some("15555550123")); + + // The written transcript re-reads cleanly. + let reread = read_session_from_file(&path).unwrap(); + assert_eq!(reread.header.id, "sess-abc"); + } +} diff --git a/crates/toolpath-openclaw/tests/projection_roundtrip.rs b/crates/toolpath-openclaw/tests/projection_roundtrip.rs new file mode 100644 index 00000000..0ae17e0c --- /dev/null +++ b/crates/toolpath-openclaw/tests/projection_roundtrip.rs @@ -0,0 +1,81 @@ +//! Full projection round-trip contract: +//! +//! native fixture → `session_to_view` → `derive_path` → serialize+reparse +//! `Path` → `extract_conversation` → `project` → OpenClaw session. +//! +//! Asserts the chain preserves messages, roles, tool calls/results, and the +//! OpenClaw channel metadata. + +use std::collections::HashSet; +use std::path::Path; + +use toolpath_convo::{ConversationProjector, Role, extract_conversation}; +use toolpath_openclaw::project::OpenClawProjector; +use toolpath_openclaw::reader::read_session_from_file; +use toolpath_openclaw::types::{AgentMessage, ContentBlock, Entry}; +use toolpath_openclaw::{DeriveConfig, derive_path, session_to_view}; + +#[test] +fn full_roundtrip_preserves_conversation() { + // 1. native fixture → view + let mut src = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + src.attach_routing_key(); + let view1 = session_to_view(&src); + assert_eq!(view1.provider_id.as_deref(), Some("openclaw")); + + // 2. view → Path (channel-aware actor + meta), then serialize + reparse. + let path = derive_path(&src, &DeriveConfig::default()); + let json = serde_json::to_string(&path).unwrap(); + let path2: toolpath::v1::Path = serde_json::from_str(&json).unwrap(); + + // Channel metadata survives the JSON round-trip on the Path. + let meta = path2.meta.as_ref().unwrap(); + assert_eq!(meta.source.as_deref(), Some("openclaw")); + assert_eq!(meta.extra["openclaw"]["channel"], "whatsapp"); + assert!( + path2 + .steps + .iter() + .any(|s| s.step.actor == "human:whatsapp/15555550123") + ); + + // 3. Path → view → project → OpenClaw session. + let view2 = extract_conversation(&path2); + assert!(view2.turns.iter().any(|t| t.role == Role::User)); + assert!(view2.turns.iter().any(|t| t.role == Role::Assistant)); + + let projected = OpenClawProjector::default().project(&view2).unwrap(); + assert_eq!(projected.header.version, 3); + + // 4. Every entry serializes one-per-line and re-parses. + for entry in &projected.entries { + let line = serde_json::to_string(entry).unwrap(); + assert!(!line.contains('\n'), "entry has embedded newline: {line}"); + let _: Entry = serde_json::from_str(&line).unwrap(); + } + + // 5. Tool calls and their results survive with matching ids. + let mut calls = HashSet::new(); + let mut results = HashSet::new(); + for e in &projected.entries { + if let Entry::Message { message, .. } = e { + match message { + AgentMessage::Assistant { content, .. } => { + for b in content { + if let ContentBlock::ToolCall { id, .. } = b { + calls.insert(id.clone()); + } + } + } + AgentMessage::ToolResult { tool_call_id, .. } => { + results.insert(tool_call_id.clone()); + } + _ => {} + } + } + } + for id in ["call_1", "call_2"] { + assert!(calls.contains(id), "missing tool call {id}"); + assert!(results.contains(id), "missing tool result for {id}"); + } +} From 0aa98810c68d8917653403df49cbddc89e429cdd Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:49:25 -0400 Subject: [PATCH 11/22] feat(cli): path p import openclaw (--agent/--session/--all/--base + picker) --- crates/path-cli/Cargo.toml | 2 + crates/path-cli/src/cmd_import.rs | 215 ++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+) diff --git a/crates/path-cli/Cargo.toml b/crates/path-cli/Cargo.toml index 01eea174..4616eabc 100644 --- a/crates/path-cli/Cargo.toml +++ b/crates/path-cli/Cargo.toml @@ -45,6 +45,7 @@ toolpath-codex = { workspace = true } toolpath-opencode = { workspace = true } toolpath-cursor = { workspace = true } toolpath-pi = { workspace = true } +toolpath-openclaw = { workspace = true } toolpath-convo = { workspace = true } toolpath-github = { workspace = true } pathbase-client = { workspace = true } @@ -69,6 +70,7 @@ toolpath-claude = { workspace = true } toolpath-gemini = { workspace = true } toolpath-codex = { workspace = true } toolpath-pi = { workspace = true } +toolpath-openclaw = { workspace = true } [features] default = ["embedded-picker"] diff --git a/crates/path-cli/src/cmd_import.rs b/crates/path-cli/src/cmd_import.rs index 7dac6532..461a9e5b 100644 --- a/crates/path-cli/src/cmd_import.rs +++ b/crates/path-cli/src/cmd_import.rs @@ -157,6 +157,24 @@ pub enum ImportSource { #[arg(long)] base: Option, }, + /// Import from OpenClaw agent-session logs + Openclaw { + /// Agent bucket (default: `main`) + #[arg(short, long)] + agent: Option, + + /// Specific session id (default: most recent or interactive pick) + #[arg(short, long)] + session: Option, + + /// Process all sessions for the agent (emits a Graph) + #[arg(long)] + all: bool, + + /// Override the OpenClaw state directory (default: ~/.openclaw) + #[arg(long)] + base: Option, + }, /// Import from Pathbase (download a previously uploaded path) Pathbase { /// Full Pathbase URL or bare `//` triple @@ -268,6 +286,12 @@ fn derive(source: ImportSource) -> Result> { all, base, } => derive_pi(project, session, all, base), + ImportSource::Openclaw { + agent, + session, + all, + base, + } => derive_openclaw(agent, session, all, base), ImportSource::Pathbase { target, url } => derive_pathbase(target, url), } } @@ -1419,6 +1443,163 @@ pub(crate) fn derive_pi_session( Ok(DerivedDoc { cache_id, doc }) } +fn openclaw_manager(base: Option) -> toolpath_openclaw::OpenClawConvo { + match base { + Some(p) => toolpath_openclaw::OpenClawConvo::with_resolver( + toolpath_openclaw::PathResolver::with_state_dir(&p), + ), + None => toolpath_openclaw::OpenClawConvo::new(), + } +} + +fn derive_openclaw( + agent: Option, + session: Option, + all: bool, + base: Option, +) -> Result> { + let manager = openclaw_manager(base); + derive_openclaw_with_manager(&manager, agent, session, all) +} + +fn derive_openclaw_with_manager( + manager: &toolpath_openclaw::OpenClawConvo, + agent: Option, + session: Option, + all: bool, +) -> Result> { + let config = toolpath_openclaw::DeriveConfig::default(); + let default_agent = || toolpath_openclaw::DEFAULT_AGENT_ID.to_string(); + + let pairs: Vec<(String, String)> = match (agent, session, all) { + (a, Some(s), _) => vec![(a.unwrap_or_else(default_agent), s)], + (a, None, true) => { + let agent_id = a.unwrap_or_else(default_agent); + let sessions = manager + .read_all_sessions(&agent_id) + .map_err(|e| anyhow::anyhow!("{}", e))?; + if sessions.is_empty() { + anyhow::bail!("No OpenClaw sessions found for agent: {}", agent_id); + } + let doc = toolpath_openclaw::derive::derive_graph(&sessions, None, &config); + let cache_id = make_id("openclaw", &doc_inner_id(&doc)); + return Ok(vec![DerivedDoc { cache_id, doc }]); + } + (a, None, false) => { + #[cfg(not(target_os = "emscripten"))] + { + if let Some(picks) = pick_openclaw(manager, a.as_deref())? { + picks + } else { + let agent_id = a.unwrap_or_else(default_agent); + let s = manager + .most_recent_session(&agent_id) + .map_err(|e| anyhow::anyhow!("{}", e))? + .ok_or_else(|| { + anyhow::anyhow!("No OpenClaw sessions found for agent: {}", agent_id) + })?; + let doc = + Graph::from_path(toolpath_openclaw::derive::derive_path(&s, &config)); + let cache_id = make_id("openclaw", &doc_inner_id(&doc)); + return Ok(vec![DerivedDoc { cache_id, doc }]); + } + } + #[cfg(target_os = "emscripten")] + { + let agent_id = a.unwrap_or_else(default_agent); + let s = manager + .most_recent_session(&agent_id) + .map_err(|e| anyhow::anyhow!("{}", e))? + .ok_or_else(|| { + anyhow::anyhow!("No OpenClaw sessions found for agent: {}", agent_id) + })?; + let doc = Graph::from_path(toolpath_openclaw::derive::derive_path(&s, &config)); + let cache_id = make_id("openclaw", &doc_inner_id(&doc)); + return Ok(vec![DerivedDoc { cache_id, doc }]); + } + } + }; + + let mut docs: Vec = Vec::with_capacity(pairs.len()); + for (agent_id, session_id) in &pairs { + let s = manager + .read_session(agent_id, session_id) + .map_err(|e| anyhow::anyhow!("{}", e))?; + let doc = Graph::from_path(toolpath_openclaw::derive::derive_path(&s, &config)); + let cache_id = make_id("openclaw", &doc_inner_id(&doc)); + docs.push(DerivedDoc { cache_id, doc }); + } + Ok(docs) +} + +/// Derive a single OpenClaw session given an explicit agent + session. +/// Used by `path share` / `path p export`. +#[allow(dead_code)] +pub(crate) fn derive_openclaw_session( + agent: &str, + session: &str, + base: Option, +) -> Result { + let manager = openclaw_manager(base); + let config = toolpath_openclaw::DeriveConfig::default(); + let s = manager + .read_session(agent, session) + .map_err(|e| anyhow::anyhow!("{}", e))?; + let doc = Graph::from_path(toolpath_openclaw::derive::derive_path(&s, &config)); + let cache_id = make_id("openclaw", &doc_inner_id(&doc)); + Ok(DerivedDoc { cache_id, doc }) +} + +#[cfg(not(target_os = "emscripten"))] +fn pick_openclaw( + manager: &toolpath_openclaw::OpenClawConvo, + agent: Option<&str>, +) -> Result>> { + if !fuzzy::available() { + return Ok(None); + } + let metas = match agent { + Some(a) => manager.list_sessions(a).map_err(|e| anyhow::anyhow!("{}", e))?, + None => manager + .list_all_sessions() + .map_err(|e| anyhow::anyhow!("{}", e))?, + }; + if metas.is_empty() { + return Ok(None); + } + let lines: Vec = metas + .iter() + .map(|m| { + format!( + "{}\t{}\t{}", + tab_safe(&m.agent_id), + tab_safe(&m.id), + render_row( + None, + parse_rfc3339(&m.timestamp), + &count(m.entry_count, "entries"), + None, + m.first_user_message.as_deref().unwrap_or("(no prompt)"), + ), + ) + }) + .collect(); + let opts = fuzzy::PickOptions { + with_nth: "3", + prompt: "openclaw session> ", + preview: Some("{exe} show --ansi openclaw --agent {1} --session {2}"), + header: Some("pick an OpenClaw session (TAB = multi-select, Enter = confirm)"), + preview_window: "right:60%:wrap-word", + tiebreak: "index", + multi: true, + }; + let selected = match fuzzy::pick(&lines, &opts)? { + fuzzy::PickResult::Selected(v) => v, + fuzzy::PickResult::NoMatch | fuzzy::PickResult::Cancelled => Vec::new(), + }; + Ok(Some(parse_project_session(&selected))) +} + #[cfg(not(target_os = "emscripten"))] fn pick_pi_in_project( manager: &toolpath_pi::PiConvo, @@ -1724,6 +1905,40 @@ mod tests { ); } + #[test] + fn openclaw_import_derives_channel_aware_path() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("s1.jsonl"), + "{\"type\":\"session\",\"version\":3,\"id\":\"s1\",\"timestamp\":\"2026-06-30T12:00:00Z\",\"cwd\":\"/p\"}\n\ + {\"type\":\"message\",\"id\":\"e1\",\"parentId\":null,\"timestamp\":\"2026-06-30T12:00:01Z\",\"message\":{\"role\":\"user\",\"content\":\"hi\",\"timestamp\":1}}\n", + ) + .unwrap(); + std::fs::write( + dir.join("sessions.json"), + "{\"agent:main:whatsapp:direct:42\":{\"sessionId\":\"s1\",\"sessionFile\":\"s1.jsonl\"}}", + ) + .unwrap(); + + let manager = toolpath_openclaw::OpenClawConvo::with_resolver( + toolpath_openclaw::PathResolver::with_state_dir(tmp.path()), + ); + let docs = + derive_openclaw_with_manager(&manager, Some("main".into()), Some("s1".into()), false) + .unwrap(); + assert_eq!(docs.len(), 1); + assert!(docs[0].cache_id.starts_with("openclaw-")); + let path = docs[0].doc.single_path().unwrap(); + assert_eq!(path.meta.as_ref().unwrap().source.as_deref(), Some("openclaw")); + assert!( + path.steps + .iter() + .any(|s| s.step.actor == "human:whatsapp/42") + ); + } + #[test] fn parse_pathbase_ref_bare_triple_with_url_flag() { let target = format!("alex/pathstash/{UUID}"); From 21c38231f6e0ea7758a668621bc7279cbea17762 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 30 Jun 2026 16:52:53 -0400 Subject: [PATCH 12/22] feat(cli): path p list/show openclaw --- crates/path-cli/src/cmd_list.rs | 129 ++++++++++++++++++++++++++++++++ crates/path-cli/src/cmd_show.rs | 33 ++++++++ 2 files changed, 162 insertions(+) diff --git a/crates/path-cli/src/cmd_list.rs b/crates/path-cli/src/cmd_list.rs index 3baf63eb..c7174294 100644 --- a/crates/path-cli/src/cmd_list.rs +++ b/crates/path-cli/src/cmd_list.rs @@ -65,6 +65,17 @@ pub enum ListSource { #[arg(long)] base: Option, }, + /// List OpenClaw agents or sessions + Openclaw { + /// Agent bucket — if omitted, lists all agents (pretty) or all + /// sessions across agents (tsv/json). + #[arg(short, long)] + agent: Option, + + /// Override the OpenClaw state directory (default: ~/.openclaw) + #[arg(long)] + base: Option, + }, } /// Output format selector. When neither `--format` nor `--json` is set, the @@ -107,6 +118,7 @@ pub fn run(source: ListSource, format: Option, json_flag: bool) -> R ListSource::Opencode { project } => run_opencode(project, fmt), ListSource::Cursor { project } => run_cursor(project, fmt), ListSource::Pi { project, base } => run_pi(project, base, fmt), + ListSource::Openclaw { agent, base } => run_openclaw(agent, base, fmt), } } @@ -1066,6 +1078,123 @@ fn emit_pi_tsv(project: &str, m: &toolpath_pi::SessionMeta) { ); } +// ── OpenClaw ──────────────────────────────────────────────────────────────── + +fn run_openclaw(agent: Option, base: Option, fmt: ListFormat) -> Result<()> { + let manager = match base { + Some(p) => toolpath_openclaw::OpenClawConvo::with_resolver( + toolpath_openclaw::PathResolver::with_state_dir(&p), + ), + None => toolpath_openclaw::OpenClawConvo::new(), + }; + match (agent, fmt) { + (None, ListFormat::Pretty) => list_openclaw_agents(&manager), + (None, f) => list_openclaw_sessions_all(&manager, f), + (Some(a), f) => list_openclaw_sessions(&manager, &a, f), + } +} + +fn list_openclaw_agents(manager: &toolpath_openclaw::OpenClawConvo) -> Result<()> { + let agents = manager.list_agents().map_err(|e| anyhow::anyhow!("{}", e))?; + if agents.is_empty() { + println!( + "No OpenClaw agents found. State dir: {:?}", + manager.resolver().state_dir() + ); + } else { + println!("OpenClaw agents:"); + println!(); + for a in &agents { + println!(" {a}"); + } + } + Ok(()) +} + +fn list_openclaw_sessions( + manager: &toolpath_openclaw::OpenClawConvo, + agent: &str, + fmt: ListFormat, +) -> Result<()> { + let sessions = manager + .list_sessions(agent) + .map_err(|e| anyhow::anyhow!("{}", e))?; + match fmt { + ListFormat::Json => print_openclaw_json(&sessions)?, + ListFormat::Tsv => { + for m in &sessions { + emit_openclaw_tsv(m); + } + } + ListFormat::Pretty => { + if sessions.is_empty() { + println!("No sessions found for agent: {agent}"); + } else { + println!("Sessions for {agent}:"); + println!(); + for m in &sessions { + println!(" {}\t{}\t{} entries", m.id, m.timestamp, m.entry_count); + } + } + } + } + Ok(()) +} + +fn list_openclaw_sessions_all( + manager: &toolpath_openclaw::OpenClawConvo, + fmt: ListFormat, +) -> Result<()> { + let sessions = manager + .list_all_sessions() + .map_err(|e| anyhow::anyhow!("{}", e))?; + match fmt { + ListFormat::Json => print_openclaw_json(&sessions)?, + ListFormat::Tsv => { + for m in &sessions { + emit_openclaw_tsv(m); + } + } + ListFormat::Pretty => unreachable!("pretty falls back to agent listing"), + } + Ok(()) +} + +fn print_openclaw_json(sessions: &[toolpath_openclaw::SessionMeta]) -> Result<()> { + let items: Vec = sessions + .iter() + .map(|m| { + serde_json::json!({ + "id": m.id, + "agent": m.agent_id, + "timestamp": m.timestamp, + "entry_count": m.entry_count, + "file_path": m.file_path, + "cwd": m.cwd, + "session_key": m.session_key, + "first_user_message": m.first_user_message, + }) + }) + .collect(); + let output = serde_json::json!({ "source": "openclaw", "sessions": items }); + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +fn emit_openclaw_tsv(m: &toolpath_openclaw::SessionMeta) { + println!( + "{}\t{}\t{}\t{}\t{}", + sanitize_tsv(&m.agent_id), + sanitize_tsv(&m.id), + sanitize_tsv(&m.timestamp), + m.entry_count, + m.first_user_message + .as_deref() + .map(sanitize_tsv) + .unwrap_or_default(), + ); +} + // ── Helpers ───────────────────────────────────────────────────────────────── /// Replace tabs and newlines in a TSV field so the column structure stays diff --git a/crates/path-cli/src/cmd_show.rs b/crates/path-cli/src/cmd_show.rs index 128235ec..da0200bf 100644 --- a/crates/path-cli/src/cmd_show.rs +++ b/crates/path-cli/src/cmd_show.rs @@ -77,6 +77,20 @@ pub enum ShowSource { #[arg(long)] base: Option, }, + /// Show an OpenClaw session as a markdown summary + Openclaw { + /// Agent bucket (default: `main`) + #[arg(short, long)] + agent: Option, + + /// Session id + #[arg(short, long)] + session: String, + + /// Override the OpenClaw state directory (default: ~/.openclaw) + #[arg(long)] + base: Option, + }, } pub fn run(source: ShowSource, ansi: bool) -> Result<()> { @@ -173,5 +187,24 @@ fn derive_one(source: ShowSource) -> Result { let cfg = toolpath_pi::DeriveConfig::default(); Ok(toolpath_pi::derive::derive_path(&s, &cfg)) } + ShowSource::Openclaw { + agent, + session, + base, + } => { + let manager = match base { + Some(p) => toolpath_openclaw::OpenClawConvo::with_resolver( + toolpath_openclaw::PathResolver::with_state_dir(&p), + ), + None => toolpath_openclaw::OpenClawConvo::new(), + }; + let agent = + agent.unwrap_or_else(|| toolpath_openclaw::DEFAULT_AGENT_ID.to_string()); + let s = manager + .read_session(&agent, &session) + .map_err(|e| anyhow::anyhow!("{}", e))?; + let cfg = toolpath_openclaw::DeriveConfig::default(); + Ok(toolpath_openclaw::derive::derive_path(&s, &cfg)) + } } } From 36946bd16df57f08b962d1d47e9948579264afab Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 1 Jul 2026 09:47:54 -0400 Subject: [PATCH 13/22] feat(openclaw): wire toolpath-openclaw CLI (share/resume/export) + workspace metadata & docs --- CLAUDE.md | 9 +- Cargo.lock | 17 ++- README.md | 1 + crates/path-cli/src/cmd_export.rs | 112 ++++++++++++++++++ crates/path-cli/src/cmd_import.rs | 16 ++- crates/path-cli/src/cmd_resume.rs | 9 ++ crates/path-cli/src/cmd_share.rs | 87 ++++++++++++++ docs/agents/formats/openclaw/README.md | 9 +- .../formats/openclaw/format-changelog.md | 2 + scripts/release.sh | 2 +- site/_data/crates.json | 8 ++ site/pages/crates.md | 3 +- 12 files changed, 265 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 01c70e9f..e0260a89 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ crates/ toolpath-opencode/ # derive from opencode SQLite databases toolpath-cursor/ # derive from Cursor (IDE) state.vscdb bubble store toolpath-pi/ # derive from Pi (pi.dev) agent session logs + toolpath-openclaw/ # derive from OpenClaw agent-session logs toolpath-dot/ # Graphviz DOT rendering toolpath-md/ # Markdown rendering for LLM consumption path-cli/ # unified CLI (binary: path) @@ -47,6 +48,7 @@ path-cli (binary: path) ├── toolpath-opencode → toolpath, toolpath-convo ├── toolpath-cursor → toolpath, toolpath-convo ├── toolpath-pi → toolpath, toolpath-convo + ├── toolpath-openclaw → toolpath, toolpath-convo ├── toolpath-dot → toolpath └── toolpath-md → toolpath @@ -56,7 +58,7 @@ toolpath-cli (deprecated shim, binary: path) └── path-cli ``` -Cross-dependencies between satellite crates: `toolpath-claude → toolpath-convo`, `toolpath-gemini → toolpath-convo`, `toolpath-codex → toolpath-convo`, `toolpath-opencode → toolpath-convo`, `toolpath-cursor → toolpath-convo`, `toolpath-pi → toolpath-convo`. +Cross-dependencies between satellite crates: `toolpath-claude → toolpath-convo`, `toolpath-gemini → toolpath-convo`, `toolpath-codex → toolpath-convo`, `toolpath-opencode → toolpath-convo`, `toolpath-cursor → toolpath-convo`, `toolpath-pi → toolpath-convo`, `toolpath-openclaw → toolpath-convo`. The desktop GUI lives in the private [pathbase](https://github.com/empathic/pathbase) repo as `pathbase-app` — it consumes the toolpath crates via git or crates.io. @@ -91,6 +93,7 @@ cargo run -p path-cli -- p import codex --session cargo run -p path-cli -- p import opencode --session ses_ cargo run -p path-cli -- p import cursor --session # IDE composer from state.vscdb cargo run -p path-cli -- p import pi --project /path/to/project +cargo run -p path-cli -- p import openclaw --agent main --session # OpenClaw agent session cargo run -p path-cli -- p import pathbase cargo run -p path-cli -- p import claude --project . --no-cache | path p render md --input - @@ -185,8 +188,9 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has in - `toolpath-opencode`: 52 unit + 19 integration + 1 doc test (SQLite reader, JSON payload serde, provider assembly, snapshot-based derive, tool-input fallback for gitignored paths, reasoning breakdown) - `toolpath-cursor`: 78 unit + 8 integration round-trip + 1 real-DB sanity + 1 doc test (state.vscdb SQLite reader, bubble store + composer header parsing, content-addressed blob lookup, projector with full TOOL_TABLE coverage, JSONL transcript ingest in `examples/dump_fixture.rs`) - `toolpath-pi`: 133 unit + 26 integration + 5 doc tests (types, paths, error, reader, io, provider) +- `toolpath-openclaw`: 48 unit + 1 integration test (v3 schema serde, path/session-key resolution, reader + tree/visible-leaf assembly, listing, session_to_view mapping + channel-aware actor, derive, projector inception + sessions.json, full projection round-trip) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) -- `path-cli`: 294 unit + 65 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, `path resume` orchestration with injectable `ExecStrategy`). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. +- `path-cli`: 296 unit + 65 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, openclaw import + share aggregation, `path resume` orchestration with injectable `ExecStrategy`). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. - `toolpath-cli`: 0 tests (it's a one-line `path_cli::run()` shim crate that exists only so `cargo install toolpath-cli` keeps installing the `path` binary) Validate example documents: `for f in examples/*.json; do cargo run -p path-cli -- p validate --input "$f"; done` @@ -246,6 +250,7 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - Token accounting (kind v1.1.0): two keys on `conversation.append`/`Turn`, both optional. `token_usage` = "the total for a message" (on the group's final step; `Σ` over a path = session total). `attributed_token_usage` = "this step's own attributed spend", populated only where the source genuinely reports per-step spend (its own key, so the sum is unaffected; remainder = group total − Σ attributed, computed not stored). One provider message can span several steps (Claude writes one JSONL line per content block); `Turn.group_id` groups them. `toolpath-claude` fills `group_id` from `message.id` and takes the **field-wise-max** group total (line order not trusted). Claude's per-line `usage` is a cumulative *streaming snapshot* (Anthropic streaming API: `message_start` seeds output near 0, `message_delta` is cumulative), NOT a per-block cost — so Claude emits no `attributed_token_usage`; the projector re-expands the total onto every line. `toolpath-codex` differences the cumulative `total_token_usage` (dedup-safe: never sum `last_token_usage` — Codex re-emits it stale; openai/codex #14489), attributes each per-call delta to the step it follows, and derives the round total from those attributions. pi/opencode decode all-zero wire counters as `None`. Never stamp a cumulative counter, a repeated message total, or zero-filled placeholders onto a step; never derive attribution from Claude's streaming snapshots. - Token usage `breakdowns` (kind v1.1.0, additive): an optional third key on `TokenUsage` — a decomposition of a top-level class into named sub-classes, keyed by class (e.g. `"output"`), inner map sub-class → tokens (e.g. `breakdowns["output"]["reasoning"] = 243`). INFORMATIONAL ONLY: **never summed into any total** (the parent class already counts those tokens, so the session-total guarantee is untouched); invariant `Σ(inner) ≤ parent`; omitted when empty; rides both `token_usage` and `attributed_token_usage`. Per-provider reality: **Gemini** reports `thoughts` (reasoning) as an additive sibling that the derivation used to **drop** (under-counting output) — it's now folded into `output_tokens` *and* recorded as `breakdowns["output"]["reasoning"]`, with the projector un-folding it on the reverse path for a lossless round-trip (`Some(0)` preserved as a real Gemini-3 zero-reasoning signal). **OpenCode** folds `reasoning` into output and records the same breakdown. **Codex** differences `reasoning_output_tokens` (⊆ output, cumulative) into `breakdowns["output"]["reasoning"]` on both per-step `attributed_token_usage` and per-round `token_usage`. **Claude** records no breakdown (its JSONL `usage` doesn't itemize thinking tokens). - Pi provider: `toolpath-pi` reads Pi session JSONL from `~/.pi/agent/sessions/`. Sessions use a tree (id/parentId) in a single file, and may link to a parent file via `parentSession` in the header. The tree is preserved as a DAG in the derived `Path`. +- OpenClaw provider: `toolpath-openclaw` reads OpenClaw agent-session JSONL from `~/.openclaw/agents//sessions/.jsonl` (override the state dir with `OPENCLAW_STATE_DIR`/`OPENCLAW_HOME`; legacy `~/.clawdbot`). Format is **version 3**: a session header line + an `id`/`parentId` entry tree whose visible head is moved by `leaf` rows (same family as Pi). OpenClaw is **multi-channel** — the human isn't a transcript field; identity lives in the `sessions.json` routing key `agent::::`, which the provider recovers into a **channel-aware** `human:/` actor (via the additive `DeriveConfig.user_actor`, kind v1.x) and stashes (channel/peer/sessionKey/sessionKind) on `path.meta.extra["openclaw"]`. **No file diffs** are stored, so changes are structural/tool-input-only (no `raw` perspective). The projector (`project.rs`) **incepts** a `Path` back onto disk — writing the transcript plus a `sessions.json` routing entry so a running OpenClaw instance can pick it up (`path p export openclaw --project`). CLI is agent-keyed (`--agent`, default `main`), not project-keyed. See `docs/agents/formats/openclaw/` for the full format reference. - Codex provider: `toolpath-codex` reads Codex CLI rollout files from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl`. Sessions are date-bucketed (not project-keyed). File-change fidelity is excellent — Codex's `patch_apply_end` events carry either the unified diff (for updates) or the full file content (for adds), so the derived `Path` gets a real `raw` perspective on every file artifact. See `docs/agents/formats/codex.md` for the full format reference. - opencode provider: `toolpath-opencode` reads a SQLite database at `~/.local/share/opencode/opencode.db` (opened read-only). Each session's messages and 12 typed part variants (text, reasoning, tool, step-start/-finish, snapshot, patch, file, agent, subtask, retry, compaction) land as one step per message with tool invocations attached. File diffs come from a sibling bare git repo at `snapshot//[]/` via `git2` tree↔tree diffs — opencode respects the user's `.gitignore`, so changes under gitignored paths fall back to tool-input-derived structural changes with no `raw` perspective. Project id is the SHA of the repo's first root commit. See `docs/agents/formats/opencode.md` for the full format reference. - Cursor (IDE) provider: `toolpath-cursor` reads Cursor.app's global `state.vscdb` SQLite (opened read-only) at `~/Library/Application Support/Cursor/User/globalStorage/state.vscdb` (macOS; `~/.config/Cursor/...` on Linux). Composers, bubbles, and content-addressed file blobs are stored as key-prefixed rows in the `cursorDiskKV` table (`composerData:`, `bubbleId::`, `composer.content.`) plus a `composer.composerHeaders` index blob in `ItemTable`. The full tool-dispatch enum (53 entries, ids 0–63) is extracted from the workbench bundle into `TOOL_TABLE` for round-trip-correct numeric ids — projector-written composers load back into Cursor.app's UI with the right tool rendering. The cursor-agent CLI uses a different per-chat protobuf store at `~/.cursor/chats///store.db` that this crate does not yet parse — that's deferred to a future `toolpath-cursor-cli` companion. See `docs/agents/formats/cursor.md` for the full format reference. diff --git a/Cargo.lock b/Cargo.lock index 77e86508..63b5a2c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2341,6 +2341,7 @@ dependencies = [ "toolpath-git", "toolpath-github", "toolpath-md", + "toolpath-openclaw", "toolpath-opencode", "toolpath-pi", "uuid", @@ -3924,7 +3925,7 @@ dependencies = [ [[package]] name = "toolpath-convo" -version = "0.11.0" +version = "0.12.0" dependencies = [ "chrono", "jsonschema", @@ -4004,6 +4005,20 @@ dependencies = [ "toolpath", ] +[[package]] +name = "toolpath-openclaw" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "toolpath", + "toolpath-convo", +] + [[package]] name = "toolpath-opencode" version = "0.5.0" diff --git a/README.md b/README.md index b08e3c83..c22535a2 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ crates/ toolpath-opencode/ Derive from opencode SQLite databases toolpath-cursor/ Derive from Cursor (IDE) state.vscdb bubble store toolpath-pi/ Derive from Pi (pi.dev) agent sessions + toolpath-openclaw/ Derive from OpenClaw agent-session logs toolpath-dot/ Graphviz DOT visualization toolpath-md/ Markdown rendering for LLM consumption pathbase-client/ Progenitor-derived typed client for the Pathbase HTTP API diff --git a/crates/path-cli/src/cmd_export.rs b/crates/path-cli/src/cmd_export.rs index 9447f76e..0ab5e222 100644 --- a/crates/path-cli/src/cmd_export.rs +++ b/crates/path-cli/src/cmd_export.rs @@ -79,6 +79,24 @@ pub enum ExportTarget { #[arg(short, long, conflicts_with = "project")] output: Option, }, + /// Project ("incept") a toolpath document into an OpenClaw session + Openclaw { + /// Input: cache id (e.g. `claude-abc`) or path to a toolpath JSON file + #[arg(short, long)] + input: String, + + /// Working directory to record in the session header. With this flag, + /// the session is incepted into `~/.openclaw/agents//sessions/` + /// (plus a `sessions.json` routing entry) so a running OpenClaw + /// instance can pick it up. Defaults to cwd when neither --project nor + /// --output is given. + #[arg(short, long)] + project: Option, + + /// Output JSONL to this file. Mutually exclusive with --project. + #[arg(short, long, conflicts_with = "project")] + output: Option, + }, /// Project a toolpath document into a Codex CLI session Codex { /// Input: cache id (e.g. `claude-abc`) or path to a toolpath JSON file @@ -213,6 +231,11 @@ pub fn run(target: ExportTarget) -> Result<()> { project, output, } => run_pi(input, project, output), + ExportTarget::Openclaw { + input, + project, + output, + } => run_openclaw(input, project, output), ExportTarget::Codex { input, project, @@ -375,6 +398,95 @@ pub(crate) fn project_pi( Ok(session.header.id) } +/// Incept a Path into the real OpenClaw state dir (`~/.openclaw`), recording +/// `cwd` in the header. Returns the session id. Used by `path resume`. +pub(crate) fn project_openclaw( + path: &toolpath::v1::Path, + cwd: &std::path::Path, +) -> Result { + let (projector, session) = build_openclaw(path, &cwd.to_string_lossy())?; + let state_dir = toolpath_openclaw::PathResolver::new() + .state_dir() + .to_path_buf(); + projector + .write_session(&session, &state_dir) + .map_err(|e| anyhow::anyhow!("OpenClaw inception failed: {}", e))?; + Ok(session.header.id) +} + +/// Build an OpenClaw projector (channel/peer/agent recovered from +/// `meta.extra["openclaw"]`) plus the projected session. +fn build_openclaw( + path: &toolpath::v1::Path, + cwd: &str, +) -> Result<( + toolpath_openclaw::project::OpenClawProjector, + toolpath_openclaw::OpenClawSession, +)> { + use toolpath_convo::ConversationProjector; + let view = toolpath_convo::extract_conversation(path); + let mut projector = + toolpath_openclaw::project::OpenClawProjector::new().with_cwd(cwd.to_string()); + if let Some(extra) = path.meta.as_ref().and_then(|m| m.extra.get("openclaw")) { + projector = projector.with_meta_extra(extra); + } + let session = projector + .project(&view) + .map_err(|e| anyhow::anyhow!("Projection failed: {}", e))?; + if session.header.id.is_empty() { + anyhow::bail!("Projected session has no id"); + } + Ok((projector, session)) +} + +fn openclaw_session_to_jsonl(session: &toolpath_openclaw::OpenClawSession) -> Result { + let mut out = String::new(); + for entry in &session.entries { + out.push_str(&serde_json::to_string(entry)?); + out.push('\n'); + } + Ok(out) +} + +fn run_openclaw(input: String, project: Option, output: Option) -> Result<()> { + #[cfg(target_os = "emscripten")] + { + let _ = (input, project, output); + anyhow::bail!("'path p export openclaw' requires a native environment"); + } + #[cfg(not(target_os = "emscripten"))] + { + let path = load_path_doc(&input)?; + match (project, output) { + (Some(project_dir), None) => { + let id = project_openclaw(&path, &project_dir)?; + let state = toolpath_openclaw::PathResolver::new() + .state_dir() + .to_path_buf(); + eprintln!("Incepted OpenClaw session {id} into {}", state.display()); + eprintln!(); + eprintln!("A running OpenClaw instance routes it via sessions.json."); + } + (None, Some(out_path)) => { + let cwd = std::env::current_dir() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "/".to_string()); + let (_projector, session) = build_openclaw(&path, &cwd)?; + let jsonl = openclaw_session_to_jsonl(&session)?; + std::fs::write(&out_path, &jsonl) + .with_context(|| format!("write {}", out_path.display()))?; + eprintln!("Wrote {} bytes to {}", jsonl.len(), out_path.display()); + } + (None, None) => { + let (_projector, session) = build_openclaw(&path, "/")?; + print!("{}", openclaw_session_to_jsonl(&session)?); + } + (Some(_), Some(_)) => unreachable!("clap enforces conflicts_with"), + } + Ok(()) + } +} + fn run_claude(input: String, project: Option, output: Option) -> Result<()> { #[cfg(target_os = "emscripten")] { diff --git a/crates/path-cli/src/cmd_import.rs b/crates/path-cli/src/cmd_import.rs index 461a9e5b..61a7bf62 100644 --- a/crates/path-cli/src/cmd_import.rs +++ b/crates/path-cli/src/cmd_import.rs @@ -1533,8 +1533,6 @@ fn derive_openclaw_with_manager( } /// Derive a single OpenClaw session given an explicit agent + session. -/// Used by `path share` / `path p export`. -#[allow(dead_code)] pub(crate) fn derive_openclaw_session( agent: &str, session: &str, @@ -1550,6 +1548,20 @@ pub(crate) fn derive_openclaw_session( Ok(DerivedDoc { cache_id, doc }) } +/// Derive an OpenClaw session by id, discovering its agent bucket by scanning +/// all agents (used by `path share`, which has no `--agent` flag). +pub(crate) fn derive_openclaw_session_any(session: &str) -> Result { + let manager = openclaw_manager(None); + let agent = manager + .list_all_sessions() + .map_err(|e| anyhow::anyhow!("{}", e))? + .into_iter() + .find(|m| m.id == session) + .map(|m| m.agent_id) + .unwrap_or_else(|| toolpath_openclaw::DEFAULT_AGENT_ID.to_string()); + derive_openclaw_session(&agent, session, None) +} + #[cfg(not(target_os = "emscripten"))] fn pick_openclaw( manager: &toolpath_openclaw::OpenClawConvo, diff --git a/crates/path-cli/src/cmd_resume.rs b/crates/path-cli/src/cmd_resume.rs index 675dbd8c..c4218d86 100644 --- a/crates/path-cli/src/cmd_resume.rs +++ b/crates/path-cli/src/cmd_resume.rs @@ -134,6 +134,7 @@ pub(crate) fn infer_source_harness(path: &TPath) -> Option return Some(Harness::Opencode), "cursor" => return Some(Harness::Cursor), "pi" => return Some(Harness::Pi), + "openclaw" => return Some(Harness::Openclaw), _ => {} // fall through to actor sniffing } } @@ -434,6 +435,13 @@ pub(crate) fn argv_for(harness: crate::cmd_share::Harness, session_id: &str) -> vec![".".into()] } Harness::Pi => vec!["--session".into(), session_id.into()], + // OpenClaw has no "resume composer by id" flag; the session is + // incepted onto disk (sessions.json routes it) and the gateway picks + // it up, so we just launch `openclaw`. + Harness::Openclaw => { + let _ = session_id; + vec![] + } } } @@ -487,6 +495,7 @@ pub(crate) fn project_into_harness( Harness::Opencode => crate::cmd_export::project_opencode(path, cwd), Harness::Cursor => crate::cmd_export::project_cursor(path, cwd), Harness::Pi => crate::cmd_export::project_pi(path, cwd), + Harness::Openclaw => crate::cmd_export::project_openclaw(path, cwd), } } diff --git a/crates/path-cli/src/cmd_share.rs b/crates/path-cli/src/cmd_share.rs index fba314ec..34e8b7f7 100644 --- a/crates/path-cli/src/cmd_share.rs +++ b/crates/path-cli/src/cmd_share.rs @@ -19,6 +19,7 @@ pub enum HarnessArg { Opencode, Cursor, Pi, + Openclaw, } #[derive(Args, Debug)] @@ -74,6 +75,7 @@ pub(crate) enum Harness { Opencode, Cursor, Pi, + Openclaw, } impl Harness { @@ -85,6 +87,7 @@ impl Harness { Harness::Opencode => "opencode", Harness::Cursor => "cursor", Harness::Pi => "pi", + Harness::Openclaw => "openclaw", } } @@ -98,6 +101,7 @@ impl Harness { Harness::Opencode => "opencode", Harness::Cursor => "cursor ", Harness::Pi => "pi ", + Harness::Openclaw => "openclaw", } } @@ -117,6 +121,7 @@ impl Harness { HarnessArg::Opencode => Harness::Opencode, HarnessArg::Cursor => Harness::Cursor, HarnessArg::Pi => Harness::Pi, + HarnessArg::Openclaw => Harness::Openclaw, } } @@ -128,6 +133,7 @@ impl Harness { "opencode" => Some(Harness::Opencode), "cursor" => Some(Harness::Cursor), "pi" => Some(Harness::Pi), + "openclaw" => Some(Harness::Openclaw), _ => None, } } @@ -159,6 +165,7 @@ pub(crate) struct HarnessBundle { pub(crate) opencode: Option, pub(crate) cursor: Option, pub(crate) pi: Option, + pub(crate) openclaw: Option, } impl HarnessBundle { @@ -173,6 +180,7 @@ impl HarnessBundle { opencode: Some(toolpath_opencode::OpencodeConvo::new()), cursor: Some(toolpath_cursor::CursorConvo::new()), pi: Some(toolpath_pi::PiConvo::new()), + openclaw: Some(toolpath_openclaw::OpenClawConvo::new()), } } } @@ -226,6 +234,11 @@ pub(crate) fn gather_sessions( { collect_cursor(mgr, &canonical_cwd, canonical_project.as_deref(), &mut rows); } + if want(Harness::Openclaw) + && let Some(mgr) = &bundle.openclaw + { + collect_openclaw(mgr, &canonical_cwd, canonical_project.as_deref(), &mut rows); + } rows.sort_by(|a, b| { b.matches_cwd @@ -475,6 +488,51 @@ fn collect_opencode( } } +fn collect_openclaw( + mgr: &toolpath_openclaw::OpenClawConvo, + canonical_cwd: &std::path::Path, + project_filter: Option<&std::path::Path>, + out: &mut Vec, +) { + let metas = match mgr.list_all_sessions() { + Ok(m) if !m.is_empty() => m, + Ok(_) => return, + Err(e) => { + eprintln!("warning: openclaw aggregation failed: {e}"); + return; + } + }; + for m in metas { + let cwd_path = m.cwd.as_ref().map(std::path::PathBuf::from); + if let Some(filter) = project_filter { + match &cwd_path { + Some(c) if paths_match(c, filter) => {} + _ => continue, + } + } + let matches_cwd = cwd_path + .as_ref() + .map(|c| paths_match(c, canonical_cwd)) + .unwrap_or(false); + let last_activity = chrono::DateTime::parse_from_rfc3339(&m.timestamp) + .ok() + .map(|d| d.with_timezone(&Utc)); + out.push(SessionRow { + harness: Harness::Openclaw, + project: None, + cwd: m.cwd.clone(), + session_id: m.id, + title: m + .first_user_message + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "(no prompt)".to_string()), + last_activity, + message_count: m.entry_count, + matches_cwd, + }); + } +} + fn collect_cursor( mgr: &toolpath_cursor::CursorConvo, canonical_cwd: &std::path::Path, @@ -681,6 +739,7 @@ fn harness_to_arg(h: Harness) -> HarnessArg { Harness::Opencode => HarnessArg::Opencode, Harness::Cursor => HarnessArg::Cursor, Harness::Pi => HarnessArg::Pi, + Harness::Openclaw => HarnessArg::Openclaw, } } @@ -989,6 +1048,7 @@ fn derive_session( Harness::Codex => crate::cmd_import::derive_codex_session(session), Harness::Opencode => crate::cmd_import::derive_opencode_session(session, false), Harness::Cursor => crate::cmd_import::derive_cursor_session(session), + Harness::Openclaw => crate::cmd_import::derive_openclaw_session_any(session), } } @@ -1242,6 +1302,33 @@ mod tests { ); } + #[test] + fn collect_openclaw_produces_cwd_matched_row() { + let tmp = tempfile::tempdir().unwrap(); + let canonical = std::fs::canonicalize(tmp.path()).unwrap(); + let cwd = canonical.to_string_lossy().into_owned(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("s1.jsonl"), + format!( + "{{\"type\":\"session\",\"version\":3,\"id\":\"s1\",\"timestamp\":\"2026-06-30T12:00:00Z\",\"cwd\":\"{cwd}\"}}\n\ + {{\"type\":\"message\",\"id\":\"e1\",\"parentId\":null,\"timestamp\":\"2026-06-30T12:00:01Z\",\"message\":{{\"role\":\"user\",\"content\":\"hello\",\"timestamp\":1}}}}\n" + ), + ) + .unwrap(); + let mgr = toolpath_openclaw::OpenClawConvo::with_resolver( + toolpath_openclaw::PathResolver::with_state_dir(tmp.path()), + ); + let mut rows = Vec::new(); + collect_openclaw(&mgr, &canonical, None, &mut rows); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].harness, Harness::Openclaw); + assert_eq!(rows[0].session_id, "s1"); + assert_eq!(rows[0].title, "hello"); + assert!(rows[0].matches_cwd, "header cwd should match canonical cwd"); + } + #[test] fn parse_picker_row_roundtrips_keyed() { let row = SessionRow { diff --git a/docs/agents/formats/openclaw/README.md b/docs/agents/formats/openclaw/README.md index 656f0f3d..7d280ac5 100644 --- a/docs/agents/formats/openclaw/README.md +++ b/docs/agents/formats/openclaw/README.md @@ -65,9 +65,12 @@ samples or by our own parser: - **No installed OpenClaw / no sample file.** Every claim is derived from **upstream source code** at the pinned commit, not from observed bytes. -- **No `toolpath-openclaw` crate yet.** We can't corroborate with our own - tests. This is the same posture `opencode.md` was written in before its - crate existed. +- **A `toolpath-openclaw` crate now implements this reference** + (`crates/toolpath-openclaw`), so the envelope, entry types, message/content + shapes, token-usage mapping, and the projector round-trip are corroborated + by its tests — but still against **synthesized** fixtures, not a captured + real session. Claims here remain "from source" until a real OpenClaw + session confirms them. Consequences for how to read these docs: diff --git a/docs/agents/formats/openclaw/format-changelog.md b/docs/agents/formats/openclaw/format-changelog.md index e6a8844d..f64939a5 100644 --- a/docs/agents/formats/openclaw/format-changelog.md +++ b/docs/agents/formats/openclaw/format-changelog.md @@ -15,6 +15,8 @@ e.g. `2026.6.10`) is separate from this on-disk format version. covers. Established by reading upstream source at `openclaw/openclaw @ 68c533cfb339cbb8650832cb2a4bf38dba7022fa` (package `2026.6.10`); **not** yet confirmed against a first-hand on-disk sample. +Implemented by the `toolpath-openclaw` crate (forward + projector), whose +tests exercise the shapes below against synthesized fixtures. Salient v3 facts (all detailed elsewhere in this folder): diff --git a/scripts/release.sh b/scripts/release.sh index d1c3e356..819ccb16 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -36,7 +36,7 @@ set -euo pipefail # 3. path-cli (depends on all of the above) # 4. toolpath-cli (deprecated shim that depends on path-cli) -_all_crates=(toolpath pathbase-client toolpath-convo toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-opencode toolpath-cursor toolpath-pi path-cli toolpath-cli) +_all_crates=(toolpath pathbase-client toolpath-convo toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-opencode toolpath-cursor toolpath-pi toolpath-openclaw path-cli toolpath-cli) _execute=0 _auto_yes="" diff --git a/site/_data/crates.json b/site/_data/crates.json index a9e3efa9..507d37ec 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -71,6 +71,14 @@ "crate": "https://crates.io/crates/toolpath-pi", "role": "Reads Pi (pi.dev) coding-agent session JSONL from `~/.pi/agent/sessions/`, implements `ConversationProvider`, and derives Toolpath `Path` documents via `toolpath-convo`'s shared `derive_path`. Preserves Pi's in-file conversation tree (id/parentId) as a DAG and follows `parentSession` links across files." }, + { + "name": "toolpath-openclaw", + "version": "0.1.0", + "description": "Derive Toolpath provenance documents from OpenClaw agent-session logs", + "docs": "https://docs.rs/toolpath-openclaw", + "crate": "https://crates.io/crates/toolpath-openclaw", + "role": "Reads OpenClaw's multi-channel agent-session JSONL from `~/.openclaw/agents//sessions/`, implements `ConversationProvider`, and derives Toolpath `Path` documents (forward) plus a projector that incepts a `Path` back onto disk (reverse). Preserves the version-3 entry tree (id/parentId + visible-leaf pointers) as a DAG, recovers the channel/peer identity from the `sessions.json` routing key into a channel-aware `human:/` actor, and records file changes structurally (OpenClaw stores no diffs)." + }, { "name": "toolpath-cursor", "version": "0.2.0", diff --git a/site/pages/crates.md b/site/pages/crates.md index 8b5e8457..e43590d3 100644 --- a/site/pages/crates.md +++ b/site/pages/crates.md @@ -20,6 +20,7 @@ path-cli (binary: path) +-- toolpath-codex -> toolpath, toolpath-convo +-- toolpath-opencode -> toolpath, toolpath-convo +-- toolpath-pi -> toolpath, toolpath-convo + +-- toolpath-openclaw -> toolpath, toolpath-convo +-- toolpath-cursor -> toolpath, toolpath-convo +-- toolpath-dot -> toolpath +-- toolpath-md -> toolpath @@ -30,7 +31,7 @@ toolpath-cli (deprecated shim, binary: path) +-- path-cli ``` -Cross-dependencies between satellite crates: `toolpath-claude -> toolpath-convo`, `toolpath-gemini -> toolpath-convo`, `toolpath-codex -> toolpath-convo`, `toolpath-opencode -> toolpath-convo`, `toolpath-cursor -> toolpath-convo`, `toolpath-pi -> toolpath-convo`. +Cross-dependencies between satellite crates: `toolpath-claude -> toolpath-convo`, `toolpath-gemini -> toolpath-convo`, `toolpath-codex -> toolpath-convo`, `toolpath-opencode -> toolpath-convo`, `toolpath-cursor -> toolpath-convo`, `toolpath-pi -> toolpath-convo`, `toolpath-openclaw -> toolpath-convo`. {% for crate in crates %} From 914b1b621883458017c3825423ecb4b11293d3e9 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 7 Jul 2026 13:30:24 -0400 Subject: [PATCH 14/22] chore(scripts): add openclaw-docker.sh to launch OpenClaw in Docker for real-session capture Subcommand launcher (up/status/agent/sessions/import/logs/down/nuke) over the public openclaw/openclaw Docker Hub image. Starts the gateway with --bind lan + a generated token, volume-mounts the state dir so agents//sessions/ transcripts are readable on the host, and wires 'import' straight into 'path p import openclaw --base' to validate the toolpath-openclaw provider against real sessions. Verified: image pull, gateway /healthz, and state-dir mount write-through (agent turn needs ANTHROPIC_API_KEY). --- scripts/openclaw-docker.sh | 294 +++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 scripts/openclaw-docker.sh diff --git a/scripts/openclaw-docker.sh b/scripts/openclaw-docker.sh new file mode 100755 index 00000000..b3fd4632 --- /dev/null +++ b/scripts/openclaw-docker.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# Launch a local OpenClaw instance in Docker for capturing real agent +# sessions — used to validate the `toolpath-openclaw` provider against +# genuine on-disk transcripts instead of synthesized fixtures. +# +# Grounded in openclaw/openclaw @ 453f5968 (v2026.6.11): +# - Prebuilt image openclaw/openclaw:latest (Docker Hub — public). The +# ghcr.io/openclaw/openclaw mirror needs `docker login ghcr.io`; override +# with OPENCLAW_IMAGE if you prefer it. ENTRYPOINT tini, default CMD +# `node openclaw.mjs gateway` (== `node dist/index.js gateway`). +# - The gateway binds LOOPBACK by default, so `-p` alone is unreachable +# from the host; we override to `--bind lan` + a gateway token. +# - State (config, credentials, SQLite, and agents//sessions/) lives at +# /home/node/.openclaw. We volume-mount it so the host — and +# `path p import openclaw --base ` — can read the transcripts. +# +# Usage: +# scripts/openclaw-docker.sh up # pull + start gateway (onboards if ANTHROPIC_API_KEY is set) +# scripts/openclaw-docker.sh status # container + /healthz + paths +# scripts/openclaw-docker.sh agent "" # run one agent turn (writes a session transcript) +# scripts/openclaw-docker.sh sessions # list sessions + host session dir +# scripts/openclaw-docker.sh import [] # validate via toolpath-openclaw (list, or import+render one) +# scripts/openclaw-docker.sh logs # follow gateway logs +# scripts/openclaw-docker.sh down # stop + remove the container (keeps state dir) +# scripts/openclaw-docker.sh nuke # down + delete the host state dir +# +# Env (override as needed; UPPER = intended for the caller): +# ANTHROPIC_API_KEY Anthropic key; required for onboarding + `agent`. +# OPENCLAW_IMAGE default openclaw/openclaw:latest (Docker Hub) +# OPENCLAW_HOST_STATE_DIR default ~/.openclaw-docker (host mount for /home/node/.openclaw) +# OPENCLAW_HOST_PORT default 18789 +# OPENCLAW_MODEL default anthropic/claude-opus-4-8 +# OPENCLAW_AGENT_ID default main +# OPENCLAW_CONTAINER default openclaw-toolpath + +set -euo pipefail + +# ── Config ──────────────────────────────────────────────────────────────── + +_root="$(cd "$(dirname "$0")/.." && pwd)" +IMAGE="${OPENCLAW_IMAGE:-openclaw/openclaw:latest}" +STATE_DIR="${OPENCLAW_HOST_STATE_DIR:-${HOME}/.openclaw-docker}" +HOST_PORT="${OPENCLAW_HOST_PORT:-18789}" +MODEL="${OPENCLAW_MODEL:-anthropic/claude-opus-4-8}" +AGENT_ID="${OPENCLAW_AGENT_ID:-main}" +CONTAINER="${OPENCLAW_CONTAINER:-openclaw-toolpath}" + +_gw_port=18789 # in-container gateway port (fixed by the image) +_state_mount=/home/node/.openclaw # in-container state dir (fixed by the image) +_token_file="${STATE_DIR}/.gateway-token" + +# ── Helpers ─────────────────────────────────────────────────────────────── + +_log() { printf '\033[1;36m[openclaw]\033[0m %s\n' "$*" >&2; } +_warn() { printf '\033[1;33m[openclaw] WARN:\033[0m %s\n' "$*" >&2; } +_die() { printf '\033[1;31m[openclaw] ERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +_need_docker() { + command -v docker >/dev/null 2>&1 || _die "docker not found on PATH" + docker info >/dev/null 2>&1 || _die "docker daemon not reachable (is Docker running?)" +} + +_container_exists() { docker inspect "${CONTAINER}" >/dev/null 2>&1; } +_container_running() { + [[ "$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null)" == "true" ]] +} + +_healthy() { curl -fsS "http://127.0.0.1:${HOST_PORT}/healthz" >/dev/null 2>&1; } + +_load_or_make_token() { + if [[ -s "${_token_file}" ]]; then + cat "${_token_file}" + return 0 + fi + local _t + _t="$(openssl rand -hex 32)" + printf '%s' "${_t}" >"${_token_file}" + chmod 600 "${_token_file}" + printf '%s' "${_t}" +} + +_write_min_config() { + # Minimal model config. Grounded in docs/providers/anthropic.md, which + # pairs the env API key with agents.defaults.model.primary. + local _cfg="${STATE_DIR}/openclaw.json" + [[ -f "${_cfg}" ]] && return 0 + cat >"${_cfg}" <&2 || true + _die "gateway container exited during startup (logs above)" + fi + sleep 2 + done + docker logs --tail 40 "${CONTAINER}" >&2 || true + _die "gateway did not become healthy within ~120s (logs above)" +} + +# ── Subcommands ─────────────────────────────────────────────────────────── + +cmd_up() { + _need_docker + mkdir -p "${STATE_DIR}" + # On Linux the container runs as uid 1000 (node) and needs to write the + # bind mount. Docker Desktop (macOS/Windows) maps this automatically. + if [[ "$(uname -s)" == "Linux" ]]; then + chown -R 1000:1000 "${STATE_DIR}" 2>/dev/null \ + || _warn "could not chown ${STATE_DIR} to uid 1000 (may need sudo)" + fi + + if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then + _log "pulling ${IMAGE} (first run; may take a while)…" + docker pull "${IMAGE}" + fi + + if [[ -n "${ANTHROPIC_API_KEY:-}" ]]; then + _write_min_config + _onboard + else + _warn "ANTHROPIC_API_KEY unset — the gateway will start unconfigured;" + _warn "set it and re-run '$0 up' before '$0 agent'." + fi + + if _container_running; then + _log "already running (${CONTAINER})" + cmd_status + return 0 + fi + _container_exists && docker rm -f "${CONTAINER}" >/dev/null + + local _token + _token="$(_load_or_make_token)" + + local -a _args=( + run -d --name "${CONTAINER}" + -p "${HOST_PORT}:${_gw_port}" + -e "OPENCLAW_GATEWAY_TOKEN=${_token}" + -v "${STATE_DIR}:${_state_mount}" + --add-host host.docker.internal:host-gateway + --restart unless-stopped + ) + [[ -n "${ANTHROPIC_API_KEY:-}" ]] && _args+=(-e "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}") + + _log "starting gateway (${CONTAINER})…" + # --bind lan makes the published port reachable from the host; + # --allow-unconfigured lets the gateway start before/without full config. + docker "${_args[@]}" "${IMAGE}" \ + node dist/index.js gateway --bind lan --port "${_gw_port}" --allow-unconfigured \ + >/dev/null + + _wait_healthy + cmd_status + _log "Control UI: http://127.0.0.1:${HOST_PORT}/ (gateway token: ${_token_file})" + _log "Send a turn: $0 agent \"hello from docker\"" +} + +cmd_status() { + _need_docker + if _container_running; then + _log "container ${CONTAINER}: running" + else + _log "container ${CONTAINER}: not running" + fi + if _healthy; then + _log "healthz: OK (http://127.0.0.1:${HOST_PORT})" + else + _warn "healthz: not responding on :${HOST_PORT}" + fi + _log "state dir (host): ${STATE_DIR}" + _log "sessions dir: ${STATE_DIR}/agents/${AGENT_ID}/sessions" +} + +cmd_agent() { + _need_docker + [[ $# -ge 1 ]] || _die "usage: $0 agent \"\"" + _container_running || _die "gateway not running; run: $0 up" + [[ -n "${ANTHROPIC_API_KEY:-}" || -f "${STATE_DIR}/agents/${AGENT_ID}/agent/auth-profiles.json" ]] \ + || _warn "no auth profile / ANTHROPIC_API_KEY — this turn will likely fail auth" + local _msg="$*" + _log "running one agent turn (agent=${AGENT_ID}, model=${MODEL})…" + docker exec "${CONTAINER}" node dist/index.js agent \ + --agent "${AGENT_ID}" --model "${MODEL}" --message "${_msg}" + _log "transcript written under ${STATE_DIR}/agents/${AGENT_ID}/sessions" +} + +cmd_sessions() { + _need_docker + if _container_running; then + docker exec "${CONTAINER}" node dist/index.js sessions --agent "${AGENT_ID}" --json || true + fi + _log "host session files (${STATE_DIR}/agents/${AGENT_ID}/sessions):" + ls -1 "${STATE_DIR}/agents/${AGENT_ID}/sessions" 2>/dev/null || _warn " (none yet)" +} + +cmd_import() { + local _bin="${_root}/target/debug/path" + if [[ ! -x "${_bin}" ]]; then + _log "building the path binary…" + (cd "${_root}" && cargo build -q -p path-cli) + fi + _log "sessions visible to toolpath-openclaw (--base ${STATE_DIR}):" + "${_bin}" p list openclaw --base "${STATE_DIR}" --format tsv || _warn " (list failed / no sessions)" + if [[ $# -ge 1 ]]; then + _log "importing + rendering session $1…" + "${_bin}" p import openclaw --base "${STATE_DIR}" --agent "${AGENT_ID}" --session "$1" --no-cache \ + | "${_bin}" p render md --input - + else + _log "pass a from the list above to import + render it." + fi +} + +cmd_logs() { + _need_docker + docker logs -f "${CONTAINER}" +} + +cmd_down() { + _need_docker + if _container_exists; then + docker rm -f "${CONTAINER}" >/dev/null + _log "removed container ${CONTAINER} (state dir ${STATE_DIR} kept)" + else + _log "container ${CONTAINER} not present" + fi +} + +cmd_nuke() { + cmd_down + if [[ -n "${STATE_DIR}" && "${STATE_DIR}" != "/" && -d "${STATE_DIR}" ]]; then + rm -rf "${STATE_DIR}" + _log "deleted state dir ${STATE_DIR}" + fi +} + +# ── Dispatch ────────────────────────────────────────────────────────────── + +_cmd="${1:-}" +shift || true +case "${_cmd}" in + up) cmd_up "$@" ;; + status) cmd_status "$@" ;; + agent) cmd_agent "$@" ;; + sessions) cmd_sessions "$@" ;; + import) cmd_import "$@" ;; + logs) cmd_logs "$@" ;; + down) cmd_down "$@" ;; + nuke) cmd_nuke "$@" ;; + ""|-h|--help|help) + sed -n '2,40p' "$0" + ;; + *) + echo "unknown command: ${_cmd}" >&2 + echo "usage: $0 {up|status|agent |sessions|import []|logs|down|nuke}" >&2 + exit 64 + ;; +esac From 1c33b321815e42948b874b270d42765e299ce750 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 7 Jul 2026 13:46:29 -0400 Subject: [PATCH 15/22] fix(scripts): openclaw-docker onboarding --skip-health + embedded --local agent turns Onboarding runs before our gateway starts, so --skip-health avoids a spurious reachability failure. Agent turns use --local (embedded) to write the transcript deterministically without the gateway auth-token handshake. Verified end-to-end: real Anthropic turn captured a v3 session that toolpath-openclaw imports (correct source/kind/actors/token-usage/file-changes; real read/write tools classify) and re-exports cleanly. --- scripts/openclaw-docker.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/openclaw-docker.sh b/scripts/openclaw-docker.sh index b3fd4632..31e2f394 100755 --- a/scripts/openclaw-docker.sh +++ b/scripts/openclaw-docker.sh @@ -106,13 +106,16 @@ _onboard() { return 0 fi _log "running onboarding (writes the auth profile)…" + # --skip-health: onboarding runs before our gateway container starts, so + # its default gateway-reachability probe would otherwise fail; we only need + # it to write the config + auth profile. docker run --rm \ -e "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" \ -v "${STATE_DIR}:${_state_mount}" \ "${IMAGE}" node dist/index.js onboard \ --non-interactive --accept-risk --mode local \ --auth-choice apiKey --anthropic-api-key "${ANTHROPIC_API_KEY}" \ - --gateway-bind lan \ + --gateway-bind lan --skip-health \ || _warn "onboarding exited nonzero — the written openclaw.json may still suffice; check '$0 logs'" } @@ -211,13 +214,16 @@ cmd_status() { cmd_agent() { _need_docker [[ $# -ge 1 ]] || _die "usage: $0 agent \"\"" - _container_running || _die "gateway not running; run: $0 up" - [[ -n "${ANTHROPIC_API_KEY:-}" || -f "${STATE_DIR}/agents/${AGENT_ID}/agent/auth-profiles.json" ]] \ - || _warn "no auth profile / ANTHROPIC_API_KEY — this turn will likely fail auth" + _container_running || _die "gateway not running; run '$0 up' with ANTHROPIC_API_KEY set" local _msg="$*" _log "running one agent turn (agent=${AGENT_ID}, model=${MODEL})…" + # --local forces embedded execution: the turn runs in-process and still + # writes the transcript to the sessions dir, avoiding the gateway auth-token + # handshake (which the exec'd client and the gateway don't share). The + # container inherits ANTHROPIC_API_KEY from `up`, so auth uses that. If this + # errors on auth, re-run '$0 up' with ANTHROPIC_API_KEY set. docker exec "${CONTAINER}" node dist/index.js agent \ - --agent "${AGENT_ID}" --model "${MODEL}" --message "${_msg}" + --agent "${AGENT_ID}" --model "${MODEL}" --local --message "${_msg}" _log "transcript written under ${STATE_DIR}/agents/${AGENT_ID}/sessions" } From 0eba9aab7a78207eafd0d5cb7b61901e656e60a4 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Tue, 7 Jul 2026 17:19:34 -0400 Subject: [PATCH 16/22] feat(cli): inception+ resume for openclaw (adopt-if-running, else exec gateway) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit path resume --harness openclaw now treats projection as the resume: the incepted transcript + sessions.json routing entry are adopted by a live gateway without restart (verified live via probing-question test — a follow-up turn answered from the incepted context and appended to the same transcript). Resume probes OPENCLAW_GATEWAY_ADDR (default 127.0.0.1:18789); if listening it reports adoption and skips exec, otherwise execvp's 'openclaw gateway'. Adds 4 RecordingExec/probe tests (path-cli 296 -> 300 unit). --- CLAUDE.md | 4 +- crates/path-cli/src/cmd_resume.rs | 176 +++++++++++++++++++++++++++++- 2 files changed, 174 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e0260a89..cd5ca388 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -190,7 +190,7 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has in - `toolpath-pi`: 133 unit + 26 integration + 5 doc tests (types, paths, error, reader, io, provider) - `toolpath-openclaw`: 48 unit + 1 integration test (v3 schema serde, path/session-key resolution, reader + tree/visible-leaf assembly, listing, session_to_view mapping + channel-aware actor, derive, projector inception + sessions.json, full projection round-trip) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) -- `path-cli`: 296 unit + 65 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, openclaw import + share aggregation, `path resume` orchestration with injectable `ExecStrategy`). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. +- `path-cli`: 300 unit + 65 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, openclaw import + share aggregation + inception+ resume, `path resume` orchestration with injectable `ExecStrategy`). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. - `toolpath-cli`: 0 tests (it's a one-line `path_cli::run()` shim crate that exists only so `cargo install toolpath-cli` keeps installing the `path` binary) Validate example documents: `for f in examples/*.json; do cargo run -p path-cli -- p validate --input "$f"; done` @@ -258,4 +258,4 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - Interactive session selection: `path p import ` (claude / gemini / pi / codex / opencode) auto-launches a fuzzy picker when stdin and stderr are TTYs and no `--session` was given. Backend: external `fzf` if on `$PATH`, otherwise the embedded skim picker (default-feature `embedded-picker`, defined in `crates/path-cli/src/skim_picker.rs`). Multi-select (TAB) produces a `Graph` document; single-select produces a `Path`. The picker uses `path show --…` as its `--preview` command. When neither backend can run (no TTY, or `--no-default-features` AND no `fzf`), it falls back to most-recent (with `--project`) or prints the manual recipe (without). `path p list --format tsv` is the documented machine-readable surface — column 1 is the project (for claude/gemini/pi) or session id (for codex/opencode), and the trailing column carries `first_user_message` so consumers can fuzzy-match by topic. - Conversation metadata title field: `toolpath-claude::ConversationMetadata`, `toolpath-gemini::ConversationMetadata`, and `toolpath-pi::SessionMeta` all expose `first_user_message: Option` — the first non-empty user-prompt text. Populated cheaply during the metadata pass (single-pass for Claude/Gemini; one extra short read for Pi). Used by the picker UI but useful for any "list sessions by topic" surface. - `path share` is the one-shot equivalent of `path p import | path p export pathbase`. It probes installed agent harnesses (claude/gemini/codex/opencode/pi), aggregates their sessions into a single fzf picker, and ranks rows whose project (claude/gemini/pi) or recorded cwd (codex/opencode) canonicalizes to the current directory at the top. `--harness` narrows the picker to one provider; `--harness X --session Y` (and `--project P` for keyed providers) skips the picker entirely. Pathbase flags (`--url`, `--anon`, `--repo`, `--slug`, `--public`) match `path export pathbase`. By default the derived doc is written to the cache like `import` does; pass `--no-cache` to skip. -- `path resume ` is the inverse of `path share`. It accepts a Pathbase URL, an `owner/repo/slug` shorthand, a local toolpath JSON file, or a cache id; resolves it (caching URL fetches under `~/.toolpath/documents/` unless `--no-cache`); validates that the document is a single agent-bearing `Path`; then opens an `fzf` harness picker (skipped with `--harness X`). The picker pre-selects the source harness inferred from `path.meta.source` (`claude-code`/`gemini-cli`/`codex`/`opencode`/`pi`) when it's installed. After picking, `path resume` projects the session into the harness's on-disk layout under the chosen working directory (default: shell cwd; override with `-C, --cwd P`) and `execvp`'s the harness's resume command (`claude -r ` / `gemini --resume ` / `codex resume ` / `opencode --session ` / `pi --session `). On Windows it spawns and waits, propagating the exit code. The exec is mockable via `cmd_resume::ExecStrategy` — production uses `RealExec`; integration tests use `RecordingExec` to capture the recipe without launching a real harness. +- `path resume ` is the inverse of `path share`. It accepts a Pathbase URL, an `owner/repo/slug` shorthand, a local toolpath JSON file, or a cache id; resolves it (caching URL fetches under `~/.toolpath/documents/` unless `--no-cache`); validates that the document is a single agent-bearing `Path`; then opens an `fzf` harness picker (skipped with `--harness X`). The picker pre-selects the source harness inferred from `path.meta.source` (`claude-code`/`gemini-cli`/`codex`/`opencode`/`pi`/`openclaw`) when it's installed. After picking, `path resume` projects the session into the harness's on-disk layout under the chosen working directory (default: shell cwd; override with `-C, --cwd P`) and `execvp`'s the harness's resume command (`claude -r ` / `gemini --resume ` / `codex resume ` / `opencode --session ` / `pi --session `). On Windows it spawns and waits, propagating the exit code. **OpenClaw is "inception+"** — it has no resume-by-id command, so the projection (transcript + `sessions.json` routing entry) *is* the resume: a live gateway adopts the incepted session without restart and appends follow-up turns to the same transcript (verified live). `path resume … --harness openclaw` therefore probes the gateway address (`OPENCLAW_GATEWAY_ADDR`, default `127.0.0.1:18789`) and either reports the adoption (gateway running → no exec) or `execvp`'s `openclaw gateway` to start one. The exec is mockable via `cmd_resume::ExecStrategy` — production uses `RealExec`; integration tests use `RecordingExec` to capture the recipe without launching a real harness. diff --git a/crates/path-cli/src/cmd_resume.rs b/crates/path-cli/src/cmd_resume.rs index c4218d86..658215b5 100644 --- a/crates/path-cli/src/cmd_resume.rs +++ b/crates/path-cli/src/cmd_resume.rs @@ -114,6 +114,28 @@ pub fn run_with_strategy(args: ResumeArgs, exec: &dyn ExecStrategy) -> Result<() ); let session_id = project_into_harness(path, target, &cwd)?; + + // OpenClaw is "inception+": the projection above already wrote the + // transcript and pointed the agent's sessions.json routing key at it, + // and a live gateway adopts that without a restart. If one is already + // listening, exec'ing another `openclaw gateway` would just fail on + // the port bind — report the adoption and stop here instead. + if target == crate::cmd_share::Harness::Openclaw { + if openclaw_gateway_running() { + eprintln!( + "OpenClaw gateway already running at {} — incepted session {} adopted via sessions.json.", + openclaw_gateway_addr(), + session_id + ); + eprintln!("Continue it from the Control UI or with: openclaw agent --message \"…\""); + return Ok(()); + } + eprintln!( + "No OpenClaw gateway on {} — starting one (Docker users: scripts/openclaw-docker.sh up).", + openclaw_gateway_addr() + ); + } + let (binary, argv) = invocation_for(target, &session_id, &cwd); exec_harness(&binary, &argv, &cwd, exec) } @@ -435,16 +457,42 @@ pub(crate) fn argv_for(harness: crate::cmd_share::Harness, session_id: &str) -> vec![".".into()] } Harness::Pi => vec!["--session".into(), session_id.into()], - // OpenClaw has no "resume composer by id" flag; the session is - // incepted onto disk (sessions.json routes it) and the gateway picks - // it up, so we just launch `openclaw`. + // OpenClaw has no "resume session by id" flag. Resume is + // "inception+": the session was already incepted onto disk (the + // sessions.json routing entry points the agent's main key at it — + // verified live: a gateway adopts it without restart and appends + // follow-up turns to the same transcript). If no gateway is + // running, we kick one off in the foreground; the session id is + // routed via sessions.json, not argv. Harness::Openclaw => { let _ = session_id; - vec![] + vec!["gateway".into()] } } } +/// Where a local OpenClaw gateway would be listening. Overridable for +/// tests and non-default deployments via `OPENCLAW_GATEWAY_ADDR`. +fn openclaw_gateway_addr() -> String { + std::env::var("OPENCLAW_GATEWAY_ADDR").unwrap_or_else(|_| "127.0.0.1:18789".to_string()) +} + +/// True when something is already listening on the OpenClaw gateway +/// address. A plain TCP probe is enough — we only need to know whether +/// exec'ing a second `openclaw gateway` would collide with a live one +/// (which has already adopted the incepted session via sessions.json). +pub(crate) fn openclaw_gateway_running() -> bool { + use std::net::ToSocketAddrs; + let addr = openclaw_gateway_addr(); + let Ok(mut addrs) = addr.to_socket_addrs() else { + return false; + }; + let Some(sa) = addrs.next() else { + return false; + }; + std::net::TcpStream::connect_timeout(&sa, std::time::Duration::from_millis(400)).is_ok() +} + pub(crate) fn invocation_for( harness: crate::cmd_share::Harness, session_id: &str, @@ -647,6 +695,126 @@ mod tests { assert_eq!(cap.cwd, std::fs::canonicalize(cwd.path()).unwrap()); } + /// Scope `OPENCLAW_GATEWAY_ADDR` for a test (callers hold TEST_ENV_LOCK). + struct ScopedGatewayAddr { + prev: Option, + } + + impl ScopedGatewayAddr { + fn set(addr: &str) -> Self { + let prev = std::env::var_os("OPENCLAW_GATEWAY_ADDR"); + unsafe { std::env::set_var("OPENCLAW_GATEWAY_ADDR", addr) }; + Self { prev } + } + } + + impl Drop for ScopedGatewayAddr { + fn drop(&mut self) { + match &self.prev { + Some(v) => unsafe { std::env::set_var("OPENCLAW_GATEWAY_ADDR", v) }, + None => unsafe { std::env::remove_var("OPENCLAW_GATEWAY_ADDR") }, + } + } + } + + /// An address on 127.0.0.1 that nothing is listening on: bind an + /// ephemeral port, note it, and drop the listener. + fn closed_local_addr() -> String { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = l.local_addr().unwrap().to_string(); + drop(l); + addr + } + + fn openclaw_resume_args(cwd: &std::path::Path, doc_file: &std::path::Path) -> ResumeArgs { + ResumeArgs { + input: doc_file.to_string_lossy().to_string(), + cwd: Some(cwd.to_path_buf()), + harness: Some(HarnessArg::Openclaw), + no_cache: false, + force: false, + url: None, + } + } + + fn write_openclaw_resume_doc(dir: &std::path::Path) -> std::path::PathBuf { + let mut path = make_convo_path_for_resume("openclaw://resume-test-session"); + path.steps[0].step.actor = "agent:openclaw".to_string(); + let graph = toolpath::v1::Graph::from_path(path); + let doc_file = dir.join("doc.json"); + std::fs::write(&doc_file, graph.to_json().unwrap()).unwrap(); + doc_file + } + + #[test] + fn invocation_for_openclaw_launches_gateway() { + let cwd = std::path::PathBuf::from("/tmp"); + let (binary, argv) = invocation_for(Harness::Openclaw, "ignored-session-id", &cwd); + assert_eq!(binary, "openclaw"); + assert_eq!(argv, vec!["gateway".to_string()]); + } + + #[test] + fn openclaw_gateway_probe_detects_listener() { + let _env = crate::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let live = listener.local_addr().unwrap().to_string(); + { + let _addr = ScopedGatewayAddr::set(&live); + assert!(openclaw_gateway_running(), "listener should be detected"); + } + { + let _addr = ScopedGatewayAddr::set(&closed_local_addr()); + assert!(!openclaw_gateway_running(), "closed port must probe false"); + } + } + + #[test] + fn run_with_strategy_openclaw_skips_exec_when_gateway_running() { + let _env = crate::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + let _home = scoped_home_for_resume(); + let _path_guard = ScopedPathForResume::with_binaries(&["openclaw"]); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let _addr = ScopedGatewayAddr::set(&listener.local_addr().unwrap().to_string()); + + let cwd = tempfile::tempdir().unwrap(); + let doc_file = write_openclaw_resume_doc(cwd.path()); + + let recorder = RecordingExec::default(); + run_with_strategy(openclaw_resume_args(cwd.path(), &doc_file), &recorder).unwrap(); + + let cap = recorder.captured(); + assert!( + cap.binary.is_empty(), + "exec must be skipped when a gateway is already listening (got {:?})", + cap.binary + ); + } + + #[test] + fn run_with_strategy_openclaw_execs_gateway_when_not_running() { + let _env = crate::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + let _home = scoped_home_for_resume(); + let _path_guard = ScopedPathForResume::with_binaries(&["openclaw"]); + let _addr = ScopedGatewayAddr::set(&closed_local_addr()); + + let cwd = tempfile::tempdir().unwrap(); + let doc_file = write_openclaw_resume_doc(cwd.path()); + + let recorder = RecordingExec::default(); + run_with_strategy(openclaw_resume_args(cwd.path(), &doc_file), &recorder).unwrap(); + + let cap = recorder.captured(); + assert_eq!(cap.binary, "openclaw"); + assert_eq!(cap.args, vec!["gateway".to_string()]); + } + use crate::cmd_share::Harness; use toolpath::v1::{Graph, PathMeta, PathOrRef}; From 43d4dad3f434956ba729b2e7a0c3d652c778e742 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 10:11:15 -0400 Subject: [PATCH 17/22] fix(openclaw): sessions.json upsert merges raw JSON instead of clobbering The projector's routing upsert previously round-tripped the whole index through typed IndexEntry; any entry our types couldn't parse made the load fail and the rewrite silently dropped every other session's routing. Now it merges through raw serde_json::Value (foreign entries preserved verbatim), refuses to overwrite an unparseable file, and stamps sessionStartedAt/updatedAt (epoch ms) on the incepted entry so listings show sane freshness. --- crates/toolpath-openclaw/src/project.rs | 116 +++++++++++++++++++++--- 1 file changed, 104 insertions(+), 12 deletions(-) diff --git a/crates/toolpath-openclaw/src/project.rs b/crates/toolpath-openclaw/src/project.rs index 2cac77cd..f47bd5a1 100644 --- a/crates/toolpath-openclaw/src/project.rs +++ b/crates/toolpath-openclaw/src/project.rs @@ -9,7 +9,7 @@ //! `anthropic`, stop reason `stop`). Foreign tool names are remapped through //! [`crate::provider::native_name`]. -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use serde_json::{Value, json}; @@ -18,7 +18,7 @@ use toolpath_convo::{ }; use crate::error::OpenClawError; -use crate::paths::{IndexEntry, SessionsIndex, normalize_agent_id}; +use crate::paths::normalize_agent_id; use crate::reader::OpenClawSession; use crate::types::{ AgentMessage, ContentBlock, CostBreakdown, Entry, EntryBase, KnownStopReason, MessageContent, @@ -145,17 +145,46 @@ impl OpenClawProjector { _ => format!("agent:{agent_id}:main"), }; - let mut map: BTreeMap = SessionsIndex::load(dir).map(|i| i.0).unwrap_or_default(); - map.insert( + // Merge through raw JSON, never through our typed view: a real + // sessions.json carries ~19 fields per entry (and could carry shapes + // our IndexEntry can't parse). Deserializing the whole file into + // typed entries and rewriting it would silently drop or mangle every + // other session's routing. Read the object verbatim, replace only + // our key, and refuse to touch a file we can't parse at all. + let index_path = dir.join("sessions.json"); + let mut root: serde_json::Map = if index_path.exists() { + let bytes = std::fs::read(&index_path)?; + match serde_json::from_slice::(&bytes) { + Ok(Value::Object(map)) => map, + _ => { + return Err(OpenClawError::other(format!( + "refusing to overwrite unparseable sessions.json at {} \ + (fix or remove it, then retry)", + index_path.display() + ))); + } + } + } else { + serde_json::Map::new() + }; + + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + root.insert( key, - IndexEntry { - session_id: Some(session_id.to_string()), - session_file: Some(file_name.to_string()), - extra: HashMap::new(), - }, + json!({ + "sessionId": session_id, + "sessionFile": file_name, + "sessionStartedAt": now_ms, + "updatedAt": now_ms, + }), ); - let json = serde_json::to_vec_pretty(&map).map_err(OpenClawError::Json)?; - write_private(&dir.join("sessions.json"), &json)?; + + let json_bytes = + serde_json::to_vec_pretty(&Value::Object(root)).map_err(OpenClawError::Json)?; + write_private(&index_path, &json_bytes)?; Ok(()) } } @@ -596,6 +625,68 @@ mod tests { assert!(call_ids.contains("call_2") && result_ids.contains("call_2")); } + #[test] + fn inception_preserves_foreign_routing_entries_verbatim() { + let src = read_session_from_file(FsPath::new("tests/fixtures/dm_session.jsonl")).unwrap(); + let view = session_to_view(&src); + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + // A realistic pre-existing index: one rich entry (19-field style) and + // one entry whose value our typed IndexEntry can NOT parse (a bare + // string). Neither may be lost or altered by the upsert. + std::fs::write( + dir.join("sessions.json"), + r#"{ + "agent:main:telegram:direct:99": { + "sessionId": "other-1", "sessionFile": "/abs/other-1.jsonl", + "updatedAt": 1783458808875, "totalTokens": 1234, + "systemPromptReport": {"nested": true} + }, + "agent:main:weird": "just-a-string" + }"#, + ) + .unwrap(); + + let proj = OpenClawProjector::default(); + let session = proj.project(&view).unwrap(); + proj.write_session(&session, tmp.path()).unwrap(); + + let root: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.join("sessions.json")).unwrap()).unwrap(); + let obj = root.as_object().unwrap(); + // Foreign entries survive byte-for-byte semantically. + assert_eq!(obj["agent:main:weird"], serde_json::json!("just-a-string")); + let tg = &obj["agent:main:telegram:direct:99"]; + assert_eq!(tg["totalTokens"], 1234); + assert_eq!(tg["systemPromptReport"]["nested"], true); + // Our key was upserted alongside, with freshness timestamps. + let ours = &obj["agent:main:main"]; + assert_eq!(ours["sessionId"], "sess-abc"); + assert!(ours["updatedAt"].is_u64(), "updatedAt stamped (epoch ms)"); + assert!(ours["sessionStartedAt"].is_u64()); + } + + #[test] + fn inception_refuses_to_clobber_unparseable_sessions_json() { + let src = read_session_from_file(FsPath::new("tests/fixtures/dm_session.jsonl")).unwrap(); + let view = session_to_view(&src); + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("agents/main/sessions"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("sessions.json"), "{ not json at all").unwrap(); + + let proj = OpenClawProjector::default(); + let session = proj.project(&view).unwrap(); + let err = proj.write_session(&session, tmp.path()); + assert!(err.is_err(), "must error rather than overwrite a corrupt index"); + // The corrupt file is untouched. + assert_eq!( + std::fs::read_to_string(dir.join("sessions.json")).unwrap(), + "{ not json at all" + ); + } + #[test] fn inception_writes_session_and_routing_entry() { let src = read_session_from_file(FsPath::new("tests/fixtures/dm_session.jsonl")).unwrap(); @@ -613,7 +704,8 @@ mod tests { ); // sessions.json got a whatsapp routing entry pointing at the file. - let idx = SessionsIndex::load(&tmp.path().join("agents/main/sessions")).unwrap(); + let idx = + crate::paths::SessionsIndex::load(&tmp.path().join("agents/main/sessions")).unwrap(); let (key, parsed) = idx.routing_key_for("sess-abc").unwrap(); assert!(key.contains("whatsapp")); assert_eq!(parsed.peer_id.as_deref(), Some("15555550123")); From 4850eb6c859eb98d00ee5810e2bb638d6c500a42 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 10:25:37 -0400 Subject: [PATCH 18/22] feat(openclaw): real-session fixtures + spawn/subagent classification from observed data Commits real sessions captured from the official Docker image (v2026.6.11) as test-fixtures/openclaw/ (elicit main session, telegram-keyed DM, spawned child, sessions.json) with a real-fixture test suite. Real-data fixes: sessions_spawn classifies as Delegation (observed sub-agent dispatch tool); agent::subagent: keys map to sessionKind spawn-child with no fabricated channel actor. Telegram-keyed session verifies the channel-aware human:telegram/ actor on genuinely OpenClaw-written bytes. --- crates/toolpath-openclaw/src/provider.rs | 49 +- .../toolpath-openclaw/tests/real_fixture.rs | 136 ++ test-fixtures/openclaw/convo.jsonl | 29 + test-fixtures/openclaw/sessions.json | 1306 +++++++++++++++++ test-fixtures/openclaw/subagent.jsonl | 8 + test-fixtures/openclaw/telegram-dm.jsonl | 7 + 6 files changed, 1534 insertions(+), 1 deletion(-) create mode 100644 crates/toolpath-openclaw/tests/real_fixture.rs create mode 100644 test-fixtures/openclaw/convo.jsonl create mode 100644 test-fixtures/openclaw/sessions.json create mode 100644 test-fixtures/openclaw/subagent.jsonl create mode 100644 test-fixtures/openclaw/telegram-dm.jsonl diff --git a/crates/toolpath-openclaw/src/provider.rs b/crates/toolpath-openclaw/src/provider.rs index 16c3e8e2..4b39bb7f 100644 --- a/crates/toolpath-openclaw/src/provider.rs +++ b/crates/toolpath-openclaw/src/provider.rs @@ -29,7 +29,15 @@ pub const PROVIDER_ID: &str = "openclaw"; /// Tool names are free-form; the matching is case-insensitive and lenient. pub fn classify_tool(name: &str) -> Option { let lower = name.to_lowercase(); - if lower.contains("task") || lower.contains("agent") || lower.contains("subagent") { + // `sessions_spawn` is OpenClaw's sub-agent dispatch (observed in real + // sessions); `spawn` alone is the discriminating token. Session + // control-flow tools (`sessions_yield`, `sessions_send`, …) are not + // delegation and fall through unclassified. + if lower.contains("task") + || lower.contains("agent") + || lower.contains("subagent") + || lower.contains("spawn") + { return Some(ToolCategory::Delegation); } match lower.as_str() { @@ -157,6 +165,12 @@ fn environment_for(session: &OpenClawSession) -> EnvironmentSnapshot { /// falls back to the default `human:user`). pub fn user_actor_for(parsed: Option<&ParsedKey>) -> Option { let key = parsed?; + // Sub-agent runs (`agent::subagent:`, observed on disk after a + // `sessions_spawn`) have no human peer; the "subagent" segment is a key + // namespace, not a messaging channel. + if key.channel.as_deref() == Some("subagent") { + return None; + } let peer = key.peer_id.as_deref()?; match (&key.channel, key.peer_kind.as_deref()) { (Some(ch), Some("group")) | (Some(ch), Some("channel")) => { @@ -170,6 +184,7 @@ pub fn user_actor_for(parsed: Option<&ParsedKey>) -> Option { /// The session kind implied by a routing key. fn session_kind(parsed: Option<&ParsedKey>) -> &'static str { match parsed { + Some(k) if k.channel.as_deref() == Some("subagent") => "spawn-child", Some(k) if matches!(k.peer_kind.as_deref(), Some("group") | Some("channel")) => "group", Some(k) if k.peer_id.is_some() => "direct", _ => "main", @@ -589,6 +604,38 @@ mod tests { assert_eq!(tu.output_tokens, Some(460)); } + #[test] + fn classify_real_observed_tool_names() { + // Names observed in real sessions captured from the v2026.6.11 image. + assert_eq!(classify_tool("exec"), Some(ToolCategory::Shell)); + assert_eq!(classify_tool("read"), Some(ToolCategory::FileRead)); + assert_eq!(classify_tool("write"), Some(ToolCategory::FileWrite)); + assert_eq!(classify_tool("edit"), Some(ToolCategory::FileWrite)); + // Sub-agent spawn is OpenClaw's delegation tool. + assert_eq!(classify_tool("sessions_spawn"), Some(ToolCategory::Delegation)); + // Control-flow tools are not delegation (and not misclassified). + assert_eq!(classify_tool("sessions_yield"), None); + } + + #[test] + fn subagent_session_key_is_spawn_child_not_a_channel() { + // Real key shape observed on disk after a sessions_spawn delegation. + let key = crate::paths::parse_session_key( + "agent:main:subagent:e1b075db-1f72-4148-a846-bc5b9d4c2ede", + ); + // A sub-agent run has no human peer: no channel actor… + assert_eq!(user_actor_for(Some(&key)), None); + // …and its session kind is spawn-child, not direct. + let mut s = read_session_from_file(Path::new("tests/fixtures/dm_session.jsonl")).unwrap(); + s.session_key = Some("agent:main:subagent:e1b075db".into()); + s.parsed_key = Some(key); + let extra = openclaw_meta_extra(&s); + assert_eq!( + extra.get("sessionKind").and_then(|v| v.as_str()), + Some("spawn-child") + ); + } + #[test] fn user_actor_for_dm_and_group_and_main() { assert_eq!( diff --git a/crates/toolpath-openclaw/tests/real_fixture.rs b/crates/toolpath-openclaw/tests/real_fixture.rs new file mode 100644 index 00000000..74cb146b --- /dev/null +++ b/crates/toolpath-openclaw/tests/real_fixture.rs @@ -0,0 +1,136 @@ +//! Tests against REAL OpenClaw sessions captured from the official Docker +//! image (v2026.6.11) via `scripts/openclaw-docker.sh` — not synthesized +//! fixtures. Fixtures live at `test-fixtures/openclaw/` (workspace root): +//! +//! - `convo.jsonl` — a feature-elicit main session (shell/read/write/edit +//! tools, one errored result, a `sessions_spawn` delegation, custom entries) +//! - `telegram-dm.jsonl` — a telegram-keyed DM session +//! - `subagent.jsonl` — the spawned child session +//! - `sessions.json` — the real routing index tying them together + +use std::path::{Path, PathBuf}; + +use toolpath_convo::{Role, ToolCategory}; +use toolpath_openclaw::reader::read_session_from_file; +use toolpath_openclaw::{DeriveConfig, derive_path, session_to_view}; + +fn fixtures() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/openclaw") +} + +fn read_with_key(name: &str) -> toolpath_openclaw::OpenClawSession { + let mut s = read_session_from_file(&fixtures().join(name)).expect("fixture parses"); + s.attach_routing_key(); + s +} + +#[test] +fn real_main_session_parses_and_classifies() { + let s = read_with_key("convo.jsonl"); + assert_eq!(s.header.version, 3); + let v = session_to_view(&s); + assert_eq!(v.provider_id.as_deref(), Some("openclaw")); + + // Roles as observed: one user prompt, many assistant/toolResult rounds. + assert!(v.turns.iter().any(|t| t.role == Role::User)); + let n_assistant = v.turns.iter().filter(|t| t.role == Role::Assistant).count(); + assert!(n_assistant >= 5, "elicit run has many assistant turns"); + + // Real tool vocabulary classifies: exec (shell), read, write/edit, spawn. + let cats: Vec<(String, Option)> = v + .turns + .iter() + .flat_map(|t| &t.tool_uses) + .map(|tu| (tu.name.clone(), tu.category)) + .collect(); + let has = |name: &str, cat: ToolCategory| { + cats.iter() + .any(|(n, c)| n == name && *c == Some(cat)) + }; + assert!(has("exec", ToolCategory::Shell), "exec classifies as Shell: {cats:?}"); + assert!(has("read", ToolCategory::FileRead)); + assert!(has("write", ToolCategory::FileWrite)); + assert!(has("edit", ToolCategory::FileWrite)); + assert!( + has("sessions_spawn", ToolCategory::Delegation), + "sessions_spawn classifies as Delegation" + ); + + // The delegation surfaced as DelegatedWork. + assert!( + v.turns.iter().any(|t| !t.delegations.is_empty()), + "sessions_spawn produces a DelegatedWork" + ); + + // The elicit prompt's deliberate missing-file read produced an error. + assert!( + v.turns + .iter() + .flat_map(|t| &t.tool_uses) + .any(|tu| tu.result.as_ref().is_some_and(|r| r.is_error)), + "one errored tool result present" + ); + + // File changes recovered from write/edit tool inputs. + assert!(!v.files_changed.is_empty()); + + // Real per-message usage sums into a session total. + let total = v.total_usage.expect("usage recorded"); + assert!(total.output_tokens.unwrap_or(0) > 0); +} + +#[test] +fn real_telegram_session_gets_channel_actor() { + let s = read_with_key("telegram-dm.jsonl"); + assert_eq!( + s.session_key.as_deref(), + Some("agent:main:telegram:direct:15555550123") + ); + let path = derive_path(&s, &DeriveConfig::default()); + assert!( + path.steps + .iter() + .any(|st| st.step.actor == "human:telegram/15555550123"), + "channel-aware human actor derived from a real telegram-keyed session" + ); + let meta = path.meta.as_ref().unwrap(); + assert_eq!(meta.extra["openclaw"]["channel"], "telegram"); + assert_eq!(meta.extra["openclaw"]["sessionKind"], "direct"); +} + +#[test] +fn real_subagent_session_is_spawn_child_without_channel_actor() { + let s = read_with_key("subagent.jsonl"); + let key = s.session_key.as_deref().expect("routing key found"); + assert!(key.starts_with("agent:main:subagent:"), "got {key}"); + let path = derive_path(&s, &DeriveConfig::default()); + let meta = path.meta.as_ref().unwrap(); + assert_eq!(meta.extra["openclaw"]["sessionKind"], "spawn-child"); + // The sub-agent's user turns are the orchestrator's prompt, not a human + // channel peer — no channel actor may be fabricated. + assert!( + !path + .steps + .iter() + .any(|st| st.step.actor.starts_with("human:subagent")), + "no fake 'subagent' channel actor" + ); +} + +#[test] +fn real_sessions_roundtrip_through_projector() { + use toolpath_convo::ConversationProjector; + for name in ["convo.jsonl", "telegram-dm.jsonl", "subagent.jsonl"] { + let s = read_with_key(name); + let view = session_to_view(&s); + let projected = toolpath_openclaw::project::OpenClawProjector::default() + .project(&view) + .expect("project"); + assert_eq!(projected.header.version, 3); + for entry in &projected.entries { + let line = serde_json::to_string(entry).expect("serialize"); + let _: toolpath_openclaw::Entry = + serde_json::from_str(&line).unwrap_or_else(|e| panic!("{name}: {e}")); + } + } +} diff --git a/test-fixtures/openclaw/convo.jsonl b/test-fixtures/openclaw/convo.jsonl new file mode 100644 index 00000000..e222e813 --- /dev/null +++ b/test-fixtures/openclaw/convo.jsonl @@ -0,0 +1,29 @@ +{"type":"session","version":3,"id":"907f477b-0bba-4b63-9d04-481a03f2db9e","timestamp":"2026-07-08T14:14:21.962Z","cwd":"/home/node/.openclaw/workspace"} +{"type":"model_change","id":"8f646d90","parentId":null,"timestamp":"2026-07-08T14:14:21.967Z","provider":"anthropic","modelId":"claude-opus-4-8"} +{"type":"thinking_level_change","id":"3afe0c33","parentId":"8f646d90","timestamp":"2026-07-08T14:14:21.967Z","thinkingLevel":"off"} +{"type":"custom","customType":"model-snapshot","data":{"timestamp":1783520061998,"provider":"anthropic","modelApi":"anthropic-messages","modelId":"claude-opus-4-8"},"id":"83d3d8de","parentId":"3afe0c33","timestamp":"2026-07-08T14:14:21.998Z"} +{"type":"message","id":"adbadd05","parentId":"83d3d8de","timestamp":"2026-07-08T14:14:22.025Z","message":{"role":"user","content":[{"type":"text","text":"You're going to walk through a small set of tasks designed to exercise every common tool category. Stay inside the current working directory — do not touch anything outside it. Take the steps in order. Briefly think out loud before each tool call so the reasoning is captured. Tool-selection rule: if a step mentions a dedicated tool *and* that tool is available to you, use it; otherwise, use whatever shell-based equivalent you have (e.g. `ls`, `cat`, `sed`, `find`, `grep`). Don't skip steps because a dedicated tool is missing.\n\n1. List the files in the current directory (prefer a dedicated list/directory tool; `bash ls` is fine).\n2. Using a dedicated write/create-file tool if available (otherwise a shell heredoc), write `notes.md` with exactly these two lines:\n scratch — feature elicitation\n this file will be edited and searched.\n3. Read `notes.md` back using a dedicated read-file tool (preferred) or `bash cat`. Confirm both lines are present.\n4. Using a dedicated edit/replace/string-replace tool if available (otherwise `sed -i` or rewrite via the write tool), change the word \"scratch\" to \"fixture\" in `notes.md`.\n5. Find every file in the current directory whose name starts with \"note\". Prefer a dedicated glob/find-files tool; otherwise `bash find . -maxdepth 1 -name 'note*'`.\n6. Search the contents of files in the current directory for the word \"fixture\". Prefer a dedicated grep/content-search tool; otherwise `bash grep -r fixture .`. It should appear in `notes.md`.\n7. Read a file that does not exist: `does-not-exist.txt`. Use a read-file tool (or `cat does-not-exist.txt`) — the call MUST fail and the failure MUST be reported as a tool result, not silently swallowed. Acknowledge the error in one sentence and move on.\n8. Write a shell script `count.sh` that prints the numbers 1 through 5 on separate lines (one `echo` per line is fine). Run it with `bash count.sh` and confirm the output is exactly five lines.\n9. If you have a sub-agent / task-dispatch tool available (Claude `Task`, Gemini sub-agents, Codex sub-tasks, opencode subtask, etc.), dispatch a sub-agent with this exact instruction: \"Count the words in `notes.md` and report the number.\" Wait for the sub-agent's reply and quote its result in one sentence. If no such tool is available, skip this step and say so explicitly.\n10. Briefly summarize in 2–3 sentences what you did. Stop without asking for further work."}],"timestamp":1783520062023}} +{"type":"message","id":"99584eef","parentId":"adbadd05","timestamp":"2026-07-08T14:14:28.585Z","message":{"role":"assistant","content":[{"type":"text","text":"I'll walk through these tasks in order. Note: there's a pending BOOTSTRAP.md, but this is an explicit sequential tool-exercise task, so I'll complete it as requested and stay inside the working directory.\n\nStep 1: Listing the current directory. I have a dedicated `read` tool but no dedicated directory-lister, so I'll use `ls`."},{"type":"toolCall","id":"toolu_01J9Y1N7BpHYmjkYckBct86R","name":"exec","arguments":{"command":"ls -la"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":169,"cacheRead":24402,"cacheWrite":1668,"totalTokens":26241,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520062030,"responseId":"msg_011CcpjH6cZqMu5Me7rRt1Nc","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"f358691b","parentId":"99584eef","timestamp":"2026-07-08T14:14:28.862Z","message":{"role":"toolResult","toolCallId":"toolu_01J9Y1N7BpHYmjkYckBct86R","toolName":"exec","content":[{"type":"text","text":"total 40\ndrwx------ 11 node node 352 Jul 8 14:14 .\ndrwx------ 13 node node 416 Jul 8 14:14 ..\ndrwxr-xr-x 10 node node 320 Jul 8 14:14 .git\n-rw-r--r-- 1 node node 8715 Jul 8 14:14 AGENTS.md\n-rw-r--r-- 1 node node 1511 Jul 8 14:14 BOOTSTRAP.md\n-rw-r--r-- 1 node node 244 Jul 8 14:14 HEARTBEAT.md\n-rw-r--r-- 1 node node 696 Jul 8 14:14 IDENTITY.md\n-rw-r--r-- 1 node node 1806 Jul 8 14:14 SOUL.md\n-rw-r--r-- 1 node node 920 Jul 8 14:14 TOOLS.md\n-rw-r--r-- 1 node node 537 Jul 8 14:14 USER.md\n-rw------- 1 node node 70 Jul 8 14:14 openclaw-workspace-state.json"}],"details":{"status":"completed","exitCode":0,"durationMs":13,"aggregated":"total 40\ndrwx------ 11 node node 352 Jul 8 14:14 .\ndrwx------ 13 node node 416 Jul 8 14:14 ..\ndrwxr-xr-x 10 node node 320 Jul 8 14:14 .git\n-rw-r--r-- 1 node node 8715 Jul 8 14:14 AGENTS.md\n-rw-r--r-- 1 node node 1511 Jul 8 14:14 BOOTSTRAP.md\n-rw-r--r-- 1 node node 244 Jul 8 14:14 HEARTBEAT.md\n-rw-r--r-- 1 node node 696 Jul 8 14:14 IDENTITY.md\n-rw-r--r-- 1 node node 1806 Jul 8 14:14 SOUL.md\n-rw-r--r-- 1 node node 920 Jul 8 14:14 TOOLS.md\n-rw-r--r-- 1 node node 537 Jul 8 14:14 USER.md\n-rw------- 1 node node 70 Jul 8 14:14 openclaw-workspace-state.json","cwd":"/home/node/.openclaw/workspace"},"isError":false,"timestamp":1783520068860}} +{"type":"message","id":"b5bf4f7d","parentId":"f358691b","timestamp":"2026-07-08T14:14:31.919Z","message":{"role":"assistant","content":[{"type":"text","text":"Directory listed. Step 2: I have a dedicated `write` tool, so I'll use it to create `notes.md` with exactly the two specified lines."},{"type":"toolCall","id":"toolu_01Dya2A2bLj6AoZye3TYXEns","name":"write","arguments":{"path":"notes.md","content":"scratch — feature elicitation\nthis file will be edited and searched.\n"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":140,"cacheRead":26070,"cacheWrite":539,"totalTokens":26751,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520068875,"responseId":"msg_011CcpjHbUtcvqgLHpNR96QW","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"4c08eaaf","parentId":"b5bf4f7d","timestamp":"2026-07-08T14:14:31.932Z","message":{"role":"toolResult","toolCallId":"toolu_01Dya2A2bLj6AoZye3TYXEns","toolName":"write","content":[{"type":"text","text":"Successfully wrote 69 bytes to notes.md"}],"isError":false,"timestamp":1783520071931}} +{"type":"message","id":"9ecd2467","parentId":"4c08eaaf","timestamp":"2026-07-08T14:14:38.922Z","message":{"role":"assistant","content":[{"type":"text","text":"File written. Step 3: Reading `notes.md` back with the dedicated `read` tool to confirm both lines."},{"type":"toolCall","id":"toolu_015g69ogvvi6USd7EiLmd3zp","name":"read","arguments":{"path":"notes.md"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":86,"cacheRead":26609,"cacheWrite":163,"totalTokens":26860,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520071937,"responseId":"msg_011CcpjHuVN4puvqUXZ5Yxyx","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"d780c945","parentId":"9ecd2467","timestamp":"2026-07-08T14:14:38.934Z","message":{"role":"toolResult","toolCallId":"toolu_015g69ogvvi6USd7EiLmd3zp","toolName":"read","content":[{"type":"text","text":"scratch — feature elicitation\nthis file will be edited and searched.\n"}],"isError":false,"timestamp":1783520078933}} +{"type":"message","id":"31af52a1","parentId":"d780c945","timestamp":"2026-07-08T14:14:42.009Z","message":{"role":"assistant","content":[{"type":"text","text":"Both lines confirmed present. Step 4: Using the dedicated `edit` tool to replace \"scratch\" with \"fixture\"."},{"type":"toolCall","id":"toolu_01GuhDC5KaXE9kEtc1fgimz2","name":"edit","arguments":{"path":"notes.md","edits":[{"oldText":"scratch — feature elicitation","newText":"fixture — feature elicitation"}]}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":150,"cacheRead":26772,"cacheWrite":113,"totalTokens":27037,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520078938,"responseId":"msg_011CcpjJLNL9xDh6am3fcgCv","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"5236699e","parentId":"31af52a1","timestamp":"2026-07-08T14:14:42.023Z","message":{"role":"toolResult","toolCallId":"toolu_01GuhDC5KaXE9kEtc1fgimz2","toolName":"edit","content":[{"type":"text","text":"Successfully replaced 1 block(s) in notes.md."}],"details":{"diff":"-1 scratch — feature elicitation\n+1 fixture — feature elicitation\n 2 this file will be edited and searched.","patch":"--- notes.md\n+++ notes.md\n@@ -1,2 +1,2 @@\n-scratch — feature elicitation\n+fixture — feature elicitation\n this file will be edited and searched.\n","firstChangedLine":1},"isError":false,"timestamp":1783520082022}} +{"type":"message","id":"c72ddce5","parentId":"5236699e","timestamp":"2026-07-08T14:14:46.067Z","message":{"role":"assistant","content":[{"type":"text","text":"Edit applied. Step 5: Finding files starting with \"note\". I have no dedicated glob tool, so I'll use `find`. Step 6: Searching contents for \"fixture\" — no dedicated grep tool, so `grep`. These are independent, so I'll run both together."},{"type":"toolCall","id":"toolu_01UNZfvfHQrdHjujs9cEHL5d","name":"exec","arguments":{"command":"find . -maxdepth 1 -name 'note*'"}},{"type":"toolCall","id":"toolu_01LB8rA7gSUboxKoJnYwEQiK","name":"exec","arguments":{"command":"grep -rn fixture ."}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":203,"cacheRead":26885,"cacheWrite":183,"totalTokens":27273,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520082026,"responseId":"msg_011CcpjJahnkzZjtN6WMHwtg","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"c4585840","parentId":"c72ddce5","timestamp":"2026-07-08T14:14:46.168Z","message":{"role":"toolResult","toolCallId":"toolu_01UNZfvfHQrdHjujs9cEHL5d","toolName":"exec","content":[{"type":"text","text":"./notes.md"}],"details":{"status":"completed","exitCode":0,"durationMs":15,"aggregated":"./notes.md","cwd":"/home/node/.openclaw/workspace"},"isError":false,"timestamp":1783520086164}} +{"type":"message","id":"149d7e75","parentId":"c4585840","timestamp":"2026-07-08T14:14:46.179Z","message":{"role":"toolResult","toolCallId":"toolu_01LB8rA7gSUboxKoJnYwEQiK","toolName":"exec","content":[{"type":"text","text":"./notes.md:1:fixture — feature elicitation"}],"details":{"status":"completed","exitCode":0,"durationMs":49,"aggregated":"./notes.md:1:fixture — feature elicitation","cwd":"/home/node/.openclaw/workspace"},"isError":false,"timestamp":1783520086175}} +{"type":"message","id":"2a49e6b9","parentId":"149d7e75","timestamp":"2026-07-08T14:14:49.111Z","message":{"role":"assistant","content":[{"type":"text","text":"Step 5 found `./notes.md`; Step 6 confirms \"fixture\" appears in `notes.md` at line 1. \n\nStep 7: Reading a nonexistent file with the dedicated `read` tool — this must fail and be reported."},{"type":"toolCall","id":"toolu_01WHMcahUkPhkXih1ovtU8yq","name":"read","arguments":{"path":"does-not-exist.txt"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":133,"cacheRead":27068,"cacheWrite":280,"totalTokens":27483,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520086203,"responseId":"msg_011CcpjJsNtrad6Qrm3fx7Di","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"938383c6","parentId":"2a49e6b9","timestamp":"2026-07-08T14:14:49.139Z","message":{"role":"toolResult","toolCallId":"toolu_01WHMcahUkPhkXih1ovtU8yq","toolName":"read","content":[{"type":"text","text":"{\n \"status\": \"error\",\n \"tool\": \"read\",\n \"error\": \"ENOENT: no such file or directory, access '/home/node/.openclaw/workspace/does-not-exist.txt'\"\n}"}],"details":{"status":"error","tool":"read","error":"ENOENT: no such file or directory, access '/home/node/.openclaw/workspace/does-not-exist.txt'"},"isError":true,"timestamp":1783520089137}} +{"type":"message","id":"3c573256","parentId":"938383c6","timestamp":"2026-07-08T14:14:52.726Z","message":{"role":"assistant","content":[{"type":"text","text":"The read failed as expected with an ENOENT (file not found) error, correctly surfaced as a tool result — moving on.\n\nStep 8: Writing `count.sh` with the `write` tool, then running it with `bash`."},{"type":"toolCall","id":"toolu_014FR7ffqrLjNJf7FzkGpcU1","name":"write","arguments":{"path":"count.sh","content":"#!/usr/bin/env bash\necho 1\necho 2\necho 3\necho 4\necho 5\n"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":177,"cacheRead":27348,"cacheWrite":211,"totalTokens":27738,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520089153,"responseId":"msg_011CcpjK6FYiRis4NgRWFNzT","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"69ca8002","parentId":"3c573256","timestamp":"2026-07-08T14:14:52.736Z","message":{"role":"toolResult","toolCallId":"toolu_014FR7ffqrLjNJf7FzkGpcU1","toolName":"write","content":[{"type":"text","text":"Successfully wrote 55 bytes to count.sh"}],"isError":false,"timestamp":1783520092735}} +{"type":"message","id":"a4608665","parentId":"69ca8002","timestamp":"2026-07-08T14:14:55.441Z","message":{"role":"assistant","content":[{"type":"text","text":"Now running it and counting the lines to confirm exactly five."},{"type":"toolCall","id":"toolu_01C8VQvysdp1e6WfTeAgHobD","name":"exec","arguments":{"command":"bash count.sh; echo \"---line count:\"; bash count.sh | wc -l"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":92,"cacheRead":27559,"cacheWrite":200,"totalTokens":27853,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520092741,"responseId":"msg_011CcpjKMDTFWJuswUAf8GrA","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"c4d1682c","parentId":"a4608665","timestamp":"2026-07-08T14:14:55.468Z","message":{"role":"toolResult","toolCallId":"toolu_01C8VQvysdp1e6WfTeAgHobD","toolName":"exec","content":[{"type":"text","text":"1\n2\n3\n4\n5\n---line count:\n5"}],"details":{"status":"completed","exitCode":0,"durationMs":10,"aggregated":"1\n2\n3\n4\n5\n---line count:\n5","cwd":"/home/node/.openclaw/workspace"},"isError":false,"timestamp":1783520095468}} +{"type":"message","id":"5a55f955","parentId":"c4d1682c","timestamp":"2026-07-08T14:14:59.397Z","message":{"role":"assistant","content":[{"type":"text","text":"Confirmed: exactly five lines. \n\nStep 9: I have a sub-agent dispatch tool (`sessions_spawn`). I'll dispatch one with the exact instruction and wait for its reply."},{"type":"toolCall","id":"toolu_01XiJ4sjwcPKpyasPDVUaqwq","name":"sessions_spawn","arguments":{"task":"Count the words in `notes.md` and report the number.","mode":"run","cwd":"/home/node/.openclaw/workspace","label":"wordcount"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":217,"cacheRead":27759,"cacheWrite":113,"totalTokens":28091,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520095472,"responseId":"msg_011CcpjKZabsJqURFb4diCNb","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"4bbf6410","parentId":"5a55f955","timestamp":"2026-07-08T14:14:59.637Z","message":{"role":"toolResult","toolCallId":"toolu_01XiJ4sjwcPKpyasPDVUaqwq","toolName":"sessions_spawn","content":[{"type":"text","text":"{\n \"status\": \"accepted\",\n \"childSessionKey\": \"agent:main:subagent:e1b075db-1f72-4148-a846-bc5b9d4c2ede\",\n \"runId\": \"d0b8f1dc-9502-47a0-8b83-43fcddf8cf04\",\n \"mode\": \"run\",\n \"note\": \"Auto-announce is push-based. After spawning children, do NOT call sessions_list, sessions_history, exec sleep, or any polling tool. Track expected child session keys. Continue any independent work. If your final answer depends on child output, wait for runtime completion events to arrive as user messages and only answer after completion events for ALL required children arrive. If a child completion event arrives AFTER your final answer, reply ONLY with NO_REPLY.\",\n \"resolvedModel\": \"anthropic/claude-opus-4-8\",\n \"resolvedProvider\": \"anthropic\",\n \"modelApplied\": true\n}"}],"details":{"status":"accepted","childSessionKey":"agent:main:subagent:e1b075db-1f72-4148-a846-bc5b9d4c2ede","runId":"d0b8f1dc-9502-47a0-8b83-43fcddf8cf04","mode":"run","note":"Auto-announce is push-based. After spawning children, do NOT call sessions_list, sessions_history, exec sleep, or any polling tool. Track expected child session keys. Continue any independent work. If your final answer depends on child output, wait for runtime completion events to arrive as user messages and only answer after completion events for ALL required children arrive. If a child completion event arrives AFTER your final answer, reply ONLY with NO_REPLY.","resolvedModel":"anthropic/claude-opus-4-8","resolvedProvider":"anthropic","modelApplied":true},"isError":false,"timestamp":1783520099636}} +{"type":"message","id":"1598ddb3","parentId":"4bbf6410","timestamp":"2026-07-08T14:15:02.286Z","message":{"role":"assistant","content":[{"type":"text","text":"Sub-agent dispatched. I'll yield and wait for its completion event."},{"type":"toolCall","id":"toolu_01Wpd7YYEX9Ryw5sodY4wnGZ","name":"sessions_yield","arguments":{"message":"Waiting for the wordcount sub-agent to report the word count of notes.md."}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":101,"cacheRead":27872,"cacheWrite":532,"totalTokens":28507,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520099706,"responseId":"msg_011CcpjKt7a4PyV5zRrnZSGX","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"2b96b6a1","parentId":"1598ddb3","timestamp":"2026-07-08T14:15:02.295Z","message":{"role":"toolResult","toolCallId":"toolu_01Wpd7YYEX9Ryw5sodY4wnGZ","toolName":"sessions_yield","content":[{"type":"text","text":"{\n \"status\": \"yielded\",\n \"message\": \"Waiting for the wordcount sub-agent to report the word count of notes.md.\"\n}"}],"details":{"status":"yielded","message":"Waiting for the wordcount sub-agent to report the word count of notes.md."},"isError":false,"timestamp":1783520102294}} +{"type":"custom_message","customType":"openclaw.sessions_yield","content":"Waiting for the wordcount sub-agent to report the word count of notes.md.\n\n[Context: The previous turn ended intentionally via sessions_yield while waiting for a follow-up event.]","display":false,"details":{"source":"sessions_yield","message":"Waiting for the wordcount sub-agent to report the word count of notes.md."},"id":"aaf35670","parentId":"2b96b6a1","timestamp":"2026-07-08T14:15:02.303Z"} +{"type":"custom","customType":"openclaw:bootstrap-context:full","data":{"timestamp":1783520102307,"runId":"a4c763bc-ddfe-4c0a-8d3d-4135c8cdc805","sessionId":"907f477b-0bba-4b63-9d04-481a03f2db9e"},"id":"c5142936","parentId":"aaf35670","timestamp":"2026-07-08T14:15:02.307Z"} +{"type":"message","id":"735b14a0-85e2-44dd-af5b-4e3b6d45f194","parentId":"c5142936","timestamp":"2026-07-08T14:15:02.349Z","message":{"role":"assistant","content":[{"type":"text","text":"Sub-agent dispatched. I'll yield and wait for its completion event."}],"api":"cli","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":20,"output":1468,"cacheRead":268344,"cacheWrite":4002,"totalTokens":28507,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1783520102347}} diff --git a/test-fixtures/openclaw/sessions.json b/test-fixtures/openclaw/sessions.json new file mode 100644 index 00000000..17605ba3 --- /dev/null +++ b/test-fixtures/openclaw/sessions.json @@ -0,0 +1,1306 @@ +{ + "agent:main:main": { + "sessionId": "907f477b-0bba-4b63-9d04-481a03f2db9e", + "updatedAt": 1783520102351, + "sessionStartedAt": 1783520061124, + "skillsSnapshot": { + "skills": [ + { + "name": "browser-automation" + }, + { + "name": "canvas" + }, + { + "name": "clawhub" + }, + { + "name": "diagram-maker" + }, + { + "name": "healthcheck" + }, + { + "name": "meme-maker", + "requiredEnv": [] + }, + { + "name": "node-connect" + }, + { + "name": "node-inspect-debugger", + "requiredEnv": [] + }, + { + "name": "notion", + "primaryEnv": "NOTION_API_TOKEN", + "requiredEnv": [] + }, + { + "name": "python-debugpy", + "requiredEnv": [] + }, + { + "name": "skill-creator" + }, + { + "name": "spike" + }, + { + "name": "taskflow" + }, + { + "name": "taskflow-inbox-triage" + }, + { + "name": "weather" + } + ], + "version": 1783520060622, + "promptFormatVersion": 1, + "promptRef": { + "version": 1, + "algorithm": "sha256", + "hash": "e5fe28e69f48dbacedd00eaae618e2965ff91d2b87d9989e52c0446673d3ab47", + "bytes": 4874 + } + }, + "lastInteractionAt": 1783520102340, + "sessionFile": "/home/node/.openclaw/agents/main/sessions/907f477b-0bba-4b63-9d04-481a03f2db9e.jsonl", + "contextTokens": 1048576, + "modelProvider": "anthropic", + "model": "claude-opus-4-8", + "agentHarnessId": "openclaw", + "abortedLastRun": false, + "systemPromptReport": { + "source": "run", + "generatedAt": 1783520061957, + "sessionId": "907f477b-0bba-4b63-9d04-481a03f2db9e", + "sessionKey": "agent:main:main", + "provider": "anthropic", + "model": "claude-opus-4-8", + "workspaceDir": "/home/node/.openclaw/workspace", + "bootstrapMaxChars": 20000, + "bootstrapTotalMaxChars": 60000, + "bootstrapTruncation": { + "warningMode": "always", + "warningShown": false, + "truncatedFiles": 0, + "nearLimitFiles": 0, + "totalNearLimit": false + }, + "sandbox": { + "mode": "off", + "sandboxed": false + }, + "systemPrompt": { + "chars": 32101, + "hash": "c5391030f5e3a2ed081e1ec6f3dcc57b4bdbc3b5b3f7537e7f82281a3081a09b", + "projectContextChars": 14519, + "nonProjectContextChars": 17582 + }, + "injectedWorkspaceFiles": [ + { + "name": "AGENTS.md", + "path": "/home/node/.openclaw/workspace/AGENTS.md", + "missing": false, + "rawChars": 8654, + "injectedChars": 8654, + "truncated": false + }, + { + "name": "SOUL.md", + "path": "/home/node/.openclaw/workspace/SOUL.md", + "missing": false, + "rawChars": 1797, + "injectedChars": 1797, + "truncated": false + }, + { + "name": "TOOLS.md", + "path": "/home/node/.openclaw/workspace/TOOLS.md", + "missing": false, + "rawChars": 910, + "injectedChars": 910, + "truncated": false + }, + { + "name": "IDENTITY.md", + "path": "/home/node/.openclaw/workspace/IDENTITY.md", + "missing": false, + "rawChars": 693, + "injectedChars": 693, + "truncated": false + }, + { + "name": "USER.md", + "path": "/home/node/.openclaw/workspace/USER.md", + "missing": false, + "rawChars": 534, + "injectedChars": 534, + "truncated": false + }, + { + "name": "HEARTBEAT.md", + "path": "/home/node/.openclaw/workspace/HEARTBEAT.md", + "missing": false, + "rawChars": 243, + "injectedChars": 243, + "truncated": false + }, + { + "name": "BOOTSTRAP.md", + "path": "/home/node/.openclaw/workspace/BOOTSTRAP.md", + "missing": false, + "rawChars": 1510, + "injectedChars": 1510, + "truncated": false + } + ], + "skills": { + "promptChars": 4872, + "hash": "c96dc0d8eeefa7f8bea8fba0ffc088a2964387e4241ca56d78badc77d514f39d", + "entries": [ + { + "name": "browser-automation", + "blockChars": 371 + }, + { + "name": "canvas", + "blockChars": 289 + }, + { + "name": "clawhub", + "blockChars": 305 + }, + { + "name": "diagram-maker", + "blockChars": 278 + }, + { + "name": "healthcheck", + "blockChars": 290 + }, + { + "name": "meme-maker", + "blockChars": 263 + }, + { + "name": "node-connect", + "blockChars": 293 + }, + { + "name": "node-inspect-debugger", + "blockChars": 289 + }, + { + "name": "notion", + "blockChars": 284 + }, + { + "name": "python-debugpy", + "blockChars": 277 + }, + { + "name": "skill-creator", + "blockChars": 271 + }, + { + "name": "spike", + "blockChars": 263 + }, + { + "name": "taskflow", + "blockChars": 293 + }, + { + "name": "taskflow-inbox-triage", + "blockChars": 303 + }, + { + "name": "weather", + "blockChars": 301 + } + ] + }, + "tools": { + "listChars": 0, + "schemaChars": 20290, + "entries": [ + { + "name": "read", + "summaryChars": 298, + "summaryHash": "f8a1d7cf1ecfe02b1b5865c93387e6a5891e5079117b4fe18614b6f912521727", + "schemaChars": 304, + "schemaHash": "134f19bcabe3e29d63c5cebb38f1d2556759fd08adad6bc90a4b4d3cd1fb8441", + "propertiesCount": 3 + }, + { + "name": "edit", + "summaryChars": 326, + "summaryHash": "c498a85576e187d069e8e8ea4f70f81bddd57829600d385ef61c4359fd634c9e", + "schemaChars": 834, + "schemaHash": "457e8bb9f800b21be76c4fd8c0e3078bf07ee9bf93ac82f39944c171bd8f6d84", + "propertiesCount": 2 + }, + { + "name": "write", + "summaryChars": 127, + "summaryHash": "cdb1c46027b22d1f1b923b47bfe297821d9fe589a9f9d92994961948060a4d89", + "schemaChars": 225, + "schemaHash": "e98a2484f667cf7c22d76ca103bf2022bf9113dc63fe38b899e71c328cb1e833", + "propertiesCount": 2 + }, + { + "name": "apply_patch", + "summaryChars": 132, + "summaryHash": "432bf38688a4f011d2b765a441ac07e1fdfa9faf2051679adee04abcd9332d93", + "schemaChars": 153, + "schemaHash": "05521268363ff4f191472c90d69ef08a503ba9acf1d2febc50b3ee68db32ab65", + "propertiesCount": 1 + }, + { + "name": "exec", + "summaryChars": 539, + "summaryHash": "4825198988fbfa8ebf5c2a8fbe3946da0be639b1e3b737ceaa4364195bba3f04", + "schemaChars": 1283, + "schemaHash": "2608ea36cf51376859b7204cda1bf1a2507134eb542097b9e8dbc8af617f216b", + "propertiesCount": 12 + }, + { + "name": "process", + "summaryChars": 456, + "summaryHash": "55297e65d460e0e1e06e7fad4cedb75ffda8fac9fa8b10aa06d6cead3c5a76ea", + "schemaChars": 1074, + "schemaHash": "a2d30a25569f409d7e8539844c89a088044535388b24031d014577b96ed27c5c", + "propertiesCount": 12 + }, + { + "name": "cron", + "summaryChars": 3820, + "summaryHash": "d3bc0032e9c0ba71e5983ee4d06bd4b70ea5170dcb5d85fbff1bd71f01b070f4", + "schemaChars": 8705, + "schemaHash": "830efafdf26c1cd1c32452d2baea3b8fc0c7b7b7269d9ab0a503aded48f88ab1", + "propertiesCount": 15 + }, + { + "name": "get_goal", + "summaryChars": 71, + "summaryHash": "58dea7906d9e17e5f89a75fea24597ee068d8a56af824370dc27f8501374d6e3", + "schemaChars": 33, + "schemaHash": "8243f0af367f188a376f2c17b5eabe872a2f7a979813e0d4e2be6d594c2aa259", + "propertiesCount": 0 + }, + { + "name": "create_goal", + "summaryChars": 155, + "summaryHash": "f01e3a63d673a410ba55213ce6bd3fa2b0d2279e8e780c18aa0cb5aa991b29b3", + "schemaChars": 267, + "schemaHash": "27ee794432736471bd76390e236cc7d7e98f8469a02656e7a27109041eea0f8c", + "propertiesCount": 2 + }, + { + "name": "update_goal", + "summaryChars": 212, + "summaryHash": "285201690e5f2456b4cdcd252ff391709b84d69538df61bad694d3c0cfc19958", + "schemaChars": 207, + "schemaHash": "c37894492eedfad02f1e7fe1488900765bfe547f3b3f34cefbab4f7f57441c2c", + "propertiesCount": 2 + }, + { + "name": "skill_workshop", + "summaryChars": 171, + "summaryHash": "577c1d2b8f56bf3e9dd5e00463bbce8b8bf4cf201da9b21b588dcaa3bc6ed8cf", + "schemaChars": 2273, + "schemaHash": "15481d6889a320a46728ee32304f0a2b510c0bcf77d445052c1ef2fa5dd5368b", + "propertiesCount": 13 + }, + { + "name": "update_plan", + "summaryChars": 163, + "summaryHash": "ea18cc44224e849c68cdf0a4839c295c1b20daf2699e2fc2be4c4e1552327455", + "schemaChars": 489, + "schemaHash": "50cb08b8a29e72637eb6720e6f5f3786783824537574f05ee9ae7a5f4ecacc4b", + "propertiesCount": 2 + }, + { + "name": "sessions_list", + "summaryChars": 135, + "summaryHash": "c7e411ae02e3adff5504aaee48f02d262a8abb9152558a3172aae289e3b05faf", + "schemaChars": 435, + "schemaHash": "c29c6b82c36a2d95a88c8944f129929b2621ee2af6d17b2ab101d4d7e9633d89", + "propertiesCount": 9 + }, + { + "name": "sessions_history", + "summaryChars": 117, + "summaryHash": "f5d45b76f11a8ac63cc45d456fb6f675f0b1a2e191feb9671924cb5830058f03", + "schemaChars": 162, + "schemaHash": "ac33b8af2252262afe479236046f5a04bc018f1fcf24d1a8bf32743f0e867598", + "propertiesCount": 3 + }, + { + "name": "sessions_send", + "summaryChars": 282, + "summaryHash": "061e9ef3487676bb2a4fd4c0336bd973aaab5eea0725e7443c4cb17518cd1e78", + "schemaChars": 275, + "schemaHash": "f1fbaa412fe6b6aee102234c87c5aa140f39c9086f9bae6e1e9691309a602ffb", + "propertiesCount": 5 + }, + { + "name": "sessions_spawn", + "summaryChars": 578, + "summaryHash": "c75061234ba01a4eb29ca223c1c63a44935e9e555ef2ca280722ba9e844db86d", + "schemaChars": 1151, + "schemaHash": "9a91c7668463da8a66386260ce2ae97e7efe3bb93b9b6f03ac85102c354457e1", + "propertiesCount": 15 + }, + { + "name": "sessions_yield", + "summaryChars": 79, + "summaryHash": "f3782fc41c1522657e1d3494f486a2df998fed38a6b89dcf7f6934fb41424320", + "schemaChars": 60, + "schemaHash": "b1c4c753820bc3096d85ef7a266a14c9fbe12450536db58076b75bb61d14dbbb", + "propertiesCount": 1 + }, + { + "name": "subagents", + "summaryChars": 132, + "summaryHash": "1c2ffd38549c0444b118d43736fc4b875ea1aaabc446bbf8bc1fc4ee9180cca4", + "schemaChars": 122, + "schemaHash": "c4a241d2c3d46ba7985abfce4d3fcd63298ce81f89d49183c8f3d584c48d0c33", + "propertiesCount": 2 + }, + { + "name": "session_status", + "summaryChars": 278, + "summaryHash": "71177e543dc6235bad4b1a26761995681dcec27ab58dcb4f4f8a7fc9a7b2b5aa", + "schemaChars": 89, + "schemaHash": "d2995f3d69d126e98ac93eec3c2a9fe5556925bd7979a8e4b6b5a2bf62d329dd", + "propertiesCount": 2 + }, + { + "name": "web_search", + "summaryChars": 65, + "summaryHash": "b38e9b3c079beaa988d3c576303660b91a55c2e72f56351da04160529173273f", + "schemaChars": 991, + "schemaHash": "3879e368875d824b14e0e683f81408824e9a2a5850fa2f8514a68caebda54d18", + "propertiesCount": 12 + }, + { + "name": "web_fetch", + "summaryChars": 93, + "summaryHash": "8875afbb51bd63673e5e6836ca31eeda73914a464862665cd73b346e28d28fd5", + "schemaChars": 317, + "schemaHash": "d2d8ad58a1b84d849dd16142282aff6c5606f7b0e3d08405e1b740a6a79e6bcc", + "propertiesCount": 3 + }, + { + "name": "image", + "summaryChars": 169, + "summaryHash": "a46820ca8952445d3663a8908bfa5d3adddff5b53d18b9d8e9eed8f9e593b494", + "schemaChars": 350, + "schemaHash": "49bb766071c935edca8c2d0df47fc2109bbffe3d4ba32d0b19fe90ec873736ce", + "propertiesCount": 6 + }, + { + "name": "memory_search", + "summaryChars": 605, + "summaryHash": "0f918d28864d3682ae77731f149273997c84773592a466b18c7cd0121b599a05", + "schemaChars": 250, + "schemaHash": "088cd2a3d3a47b251429ff7c56844cdedd1c45c3af0fa8bfaf7aa795766976bf", + "propertiesCount": 4 + }, + { + "name": "memory_get", + "summaryChars": 239, + "summaryHash": "bff1141a3da31a345c550337417efc9ace4da2909004606c727c59ca1a294495", + "schemaChars": 241, + "schemaHash": "78f5d58460a82f9104deebad2f1873a8cde05d1ce7328b3ba4da6e994653e70a", + "propertiesCount": 4 + } + ] + }, + "currentTurn": { + "promptChars": 2420, + "runtimeContextChars": 0 + } + }, + "contextBudgetStatus": { + "schemaVersion": 1, + "source": "pre-prompt-estimate", + "updatedAt": 1783520062019, + "provider": "anthropic", + "model": "claude-opus-4-8", + "route": "fits", + "shouldCompact": false, + "estimatedPromptTokens": 10395, + "contextTokenBudget": 1048576, + "promptBudgetBeforeReserve": 1028576, + "reserveTokens": 20000, + "effectiveReserveTokens": 20000, + "remainingPromptBudgetTokens": 1018181, + "overflowTokens": 0, + "toolResultReducibleChars": 0, + "messageCount": 0, + "unwindowedMessageCount": 0, + "sessionId": "907f477b-0bba-4b63-9d04-481a03f2db9e" + }, + "inputTokens": 20, + "outputTokens": 1468, + "totalTokens": 28406, + "totalTokensFresh": true, + "cacheRead": 268344, + "cacheWrite": 4002 + }, + "agent:main:subagent:e1b075db-1f72-4148-a846-bc5b9d4c2ede": { + "spawnDepth": 1, + "subagentRole": "leaf", + "subagentControlScope": "none", + "inheritedToolAllow": [ + "read", + "edit", + "write", + "apply_patch", + "exec", + "process", + "cron", + "get_goal", + "create_goal", + "update_goal", + "skill_workshop", + "update_plan", + "sessions_list", + "sessions_history", + "sessions_send", + "sessions_spawn", + "sessions_yield", + "subagents", + "session_status", + "web_search", + "web_fetch", + "image", + "memory_search", + "memory_get" + ], + "thinkingLevel": "off", + "model": "claude-opus-4-8", + "modelProvider": "anthropic", + "sessionId": "cdff4998-f609-45e7-beeb-ecb980d44cab", + "updatedAt": 1783520106093, + "sessionStartedAt": 1783520099409, + "spawnedBy": "agent:main:main", + "spawnedWorkspaceDir": "/home/node/.openclaw/workspace", + "spawnedCwd": "/home/node/.openclaw/workspace", + "lastInteractionAt": 1783520106092, + "label": "wordcount", + "startedAt": 1783520099557, + "endedAt": 1783520106108, + "runtimeMs": 6044, + "status": "done", + "skillsSnapshot": { + "skills": [ + { + "name": "browser-automation" + }, + { + "name": "canvas" + }, + { + "name": "clawhub" + }, + { + "name": "diagram-maker" + }, + { + "name": "healthcheck" + }, + { + "name": "meme-maker", + "requiredEnv": [] + }, + { + "name": "node-connect" + }, + { + "name": "node-inspect-debugger", + "requiredEnv": [] + }, + { + "name": "notion", + "primaryEnv": "NOTION_API_TOKEN", + "requiredEnv": [] + }, + { + "name": "python-debugpy", + "requiredEnv": [] + }, + { + "name": "skill-creator" + }, + { + "name": "spike" + }, + { + "name": "taskflow" + }, + { + "name": "taskflow-inbox-triage" + }, + { + "name": "weather" + } + ], + "version": 1783520060622, + "promptFormatVersion": 1, + "promptRef": { + "version": 1, + "algorithm": "sha256", + "hash": "e5fe28e69f48dbacedd00eaae618e2965ff91d2b87d9989e52c0446673d3ab47", + "bytes": 4874 + } + }, + "sessionFile": "/home/node/.openclaw/agents/main/sessions/cdff4998-f609-45e7-beeb-ecb980d44cab.jsonl", + "contextTokens": 1048576, + "agentHarnessId": "openclaw", + "abortedLastRun": false, + "systemPromptReport": { + "source": "run", + "generatedAt": 1783520100243, + "sessionId": "cdff4998-f609-45e7-beeb-ecb980d44cab", + "sessionKey": "agent:main:subagent:e1b075db-1f72-4148-a846-bc5b9d4c2ede", + "provider": "anthropic", + "model": "claude-opus-4-8", + "workspaceDir": "/home/node/.openclaw/workspace", + "bootstrapMaxChars": 20000, + "bootstrapTotalMaxChars": 60000, + "bootstrapTruncation": { + "warningMode": "always", + "warningShown": false, + "truncatedFiles": 0, + "nearLimitFiles": 0, + "totalNearLimit": false + }, + "sandbox": { + "mode": "off", + "sandboxed": false + }, + "systemPrompt": { + "chars": 22993, + "hash": "d0e6ef0e5445c8c6daac705f2103aae9edb865d97504ee8c717b9f15dcea75d8", + "projectContextChars": 12334, + "nonProjectContextChars": 10659 + }, + "injectedWorkspaceFiles": [ + { + "name": "AGENTS.md", + "path": "/home/node/.openclaw/workspace/AGENTS.md", + "missing": false, + "rawChars": 8654, + "injectedChars": 8654, + "truncated": false + }, + { + "name": "TOOLS.md", + "path": "/home/node/.openclaw/workspace/TOOLS.md", + "missing": false, + "rawChars": 910, + "injectedChars": 910, + "truncated": false + } + ], + "skills": { + "promptChars": 4872, + "hash": "c96dc0d8eeefa7f8bea8fba0ffc088a2964387e4241ca56d78badc77d514f39d", + "entries": [ + { + "name": "browser-automation", + "blockChars": 371 + }, + { + "name": "canvas", + "blockChars": 289 + }, + { + "name": "clawhub", + "blockChars": 305 + }, + { + "name": "diagram-maker", + "blockChars": 278 + }, + { + "name": "healthcheck", + "blockChars": 290 + }, + { + "name": "meme-maker", + "blockChars": 263 + }, + { + "name": "node-connect", + "blockChars": 293 + }, + { + "name": "node-inspect-debugger", + "blockChars": 289 + }, + { + "name": "notion", + "blockChars": 284 + }, + { + "name": "python-debugpy", + "blockChars": 277 + }, + { + "name": "skill-creator", + "blockChars": 271 + }, + { + "name": "spike", + "blockChars": 263 + }, + { + "name": "taskflow", + "blockChars": 293 + }, + { + "name": "taskflow-inbox-triage", + "blockChars": 303 + }, + { + "name": "weather", + "blockChars": 301 + } + ] + }, + "tools": { + "listChars": 0, + "schemaChars": 9351, + "entries": [ + { + "name": "read", + "summaryChars": 298, + "summaryHash": "f8a1d7cf1ecfe02b1b5865c93387e6a5891e5079117b4fe18614b6f912521727", + "schemaChars": 304, + "schemaHash": "134f19bcabe3e29d63c5cebb38f1d2556759fd08adad6bc90a4b4d3cd1fb8441", + "propertiesCount": 3 + }, + { + "name": "edit", + "summaryChars": 326, + "summaryHash": "c498a85576e187d069e8e8ea4f70f81bddd57829600d385ef61c4359fd634c9e", + "schemaChars": 834, + "schemaHash": "457e8bb9f800b21be76c4fd8c0e3078bf07ee9bf93ac82f39944c171bd8f6d84", + "propertiesCount": 2 + }, + { + "name": "write", + "summaryChars": 127, + "summaryHash": "cdb1c46027b22d1f1b923b47bfe297821d9fe589a9f9d92994961948060a4d89", + "schemaChars": 225, + "schemaHash": "e98a2484f667cf7c22d76ca103bf2022bf9113dc63fe38b899e71c328cb1e833", + "propertiesCount": 2 + }, + { + "name": "apply_patch", + "summaryChars": 132, + "summaryHash": "432bf38688a4f011d2b765a441ac07e1fdfa9faf2051679adee04abcd9332d93", + "schemaChars": 153, + "schemaHash": "05521268363ff4f191472c90d69ef08a503ba9acf1d2febc50b3ee68db32ab65", + "propertiesCount": 1 + }, + { + "name": "exec", + "summaryChars": 446, + "summaryHash": "369f5e5e195c647ab1e3cfad282e6d25488d4b9fe269f1157e273a546ca80295", + "schemaChars": 1283, + "schemaHash": "2608ea36cf51376859b7204cda1bf1a2507134eb542097b9e8dbc8af617f216b", + "propertiesCount": 12 + }, + { + "name": "process", + "summaryChars": 362, + "summaryHash": "21d78afb643a24f211fe9aa86c748f165ca82f1dc21bbaf6e3e6d6d4d4411d86", + "schemaChars": 1074, + "schemaHash": "a2d30a25569f409d7e8539844c89a088044535388b24031d014577b96ed27c5c", + "propertiesCount": 12 + }, + { + "name": "get_goal", + "summaryChars": 71, + "summaryHash": "58dea7906d9e17e5f89a75fea24597ee068d8a56af824370dc27f8501374d6e3", + "schemaChars": 33, + "schemaHash": "8243f0af367f188a376f2c17b5eabe872a2f7a979813e0d4e2be6d594c2aa259", + "propertiesCount": 0 + }, + { + "name": "create_goal", + "summaryChars": 155, + "summaryHash": "f01e3a63d673a410ba55213ce6bd3fa2b0d2279e8e780c18aa0cb5aa991b29b3", + "schemaChars": 267, + "schemaHash": "27ee794432736471bd76390e236cc7d7e98f8469a02656e7a27109041eea0f8c", + "propertiesCount": 2 + }, + { + "name": "update_goal", + "summaryChars": 212, + "summaryHash": "285201690e5f2456b4cdcd252ff391709b84d69538df61bad694d3c0cfc19958", + "schemaChars": 207, + "schemaHash": "c37894492eedfad02f1e7fe1488900765bfe547f3b3f34cefbab4f7f57441c2c", + "propertiesCount": 2 + }, + { + "name": "skill_workshop", + "summaryChars": 171, + "summaryHash": "577c1d2b8f56bf3e9dd5e00463bbce8b8bf4cf201da9b21b588dcaa3bc6ed8cf", + "schemaChars": 2273, + "schemaHash": "15481d6889a320a46728ee32304f0a2b510c0bcf77d445052c1ef2fa5dd5368b", + "propertiesCount": 13 + }, + { + "name": "update_plan", + "summaryChars": 163, + "summaryHash": "ea18cc44224e849c68cdf0a4839c295c1b20daf2699e2fc2be4c4e1552327455", + "schemaChars": 489, + "schemaHash": "50cb08b8a29e72637eb6720e6f5f3786783824537574f05ee9ae7a5f4ecacc4b", + "propertiesCount": 2 + }, + { + "name": "sessions_yield", + "summaryChars": 79, + "summaryHash": "f3782fc41c1522657e1d3494f486a2df998fed38a6b89dcf7f6934fb41424320", + "schemaChars": 60, + "schemaHash": "b1c4c753820bc3096d85ef7a266a14c9fbe12450536db58076b75bb61d14dbbb", + "propertiesCount": 1 + }, + { + "name": "web_search", + "summaryChars": 65, + "summaryHash": "b38e9b3c079beaa988d3c576303660b91a55c2e72f56351da04160529173273f", + "schemaChars": 991, + "schemaHash": "3879e368875d824b14e0e683f81408824e9a2a5850fa2f8514a68caebda54d18", + "propertiesCount": 12 + }, + { + "name": "web_fetch", + "summaryChars": 93, + "summaryHash": "8875afbb51bd63673e5e6836ca31eeda73914a464862665cd73b346e28d28fd5", + "schemaChars": 317, + "schemaHash": "d2d8ad58a1b84d849dd16142282aff6c5606f7b0e3d08405e1b740a6a79e6bcc", + "propertiesCount": 3 + }, + { + "name": "image", + "summaryChars": 169, + "summaryHash": "a46820ca8952445d3663a8908bfa5d3adddff5b53d18b9d8e9eed8f9e593b494", + "schemaChars": 350, + "schemaHash": "49bb766071c935edca8c2d0df47fc2109bbffe3d4ba32d0b19fe90ec873736ce", + "propertiesCount": 6 + }, + { + "name": "memory_search", + "summaryChars": 605, + "summaryHash": "0f918d28864d3682ae77731f149273997c84773592a466b18c7cd0121b599a05", + "schemaChars": 250, + "schemaHash": "088cd2a3d3a47b251429ff7c56844cdedd1c45c3af0fa8bfaf7aa795766976bf", + "propertiesCount": 4 + }, + { + "name": "memory_get", + "summaryChars": 239, + "summaryHash": "bff1141a3da31a345c550337417efc9ace4da2909004606c727c59ca1a294495", + "schemaChars": 241, + "schemaHash": "78f5d58460a82f9104deebad2f1873a8cde05d1ce7328b3ba4da6e994653e70a", + "propertiesCount": 4 + } + ] + }, + "currentTurn": { + "promptChars": 251, + "runtimeContextChars": 0 + } + }, + "contextBudgetStatus": { + "schemaVersion": 1, + "source": "pre-prompt-estimate", + "updatedAt": 1783520100296, + "provider": "anthropic", + "model": "claude-opus-4-8", + "route": "fits", + "shouldCompact": false, + "estimatedPromptTokens": 7012, + "contextTokenBudget": 1048576, + "promptBudgetBeforeReserve": 1028576, + "reserveTokens": 20000, + "effectiveReserveTokens": 20000, + "remainingPromptBudgetTokens": 1021564, + "overflowTokens": 0, + "toolResultReducibleChars": 0, + "messageCount": 0, + "unwindowedMessageCount": 0, + "sessionId": "cdff4998-f609-45e7-beeb-ecb980d44cab" + }, + "inputTokens": 4, + "outputTokens": 84, + "totalTokens": 14385, + "totalTokensFresh": true, + "cacheRead": 14304, + "cacheWrite": 14383 + }, + "agent:main:telegram:direct:15555550123": { + "sessionId": "02f4cce4-d744-4e62-a34c-4e99292becd2", + "updatedAt": 1783520172252, + "sessionStartedAt": 1783520166884, + "skillsSnapshot": { + "skills": [ + { + "name": "browser-automation" + }, + { + "name": "canvas" + }, + { + "name": "clawhub" + }, + { + "name": "diagram-maker" + }, + { + "name": "healthcheck" + }, + { + "name": "meme-maker", + "requiredEnv": [] + }, + { + "name": "node-connect" + }, + { + "name": "node-inspect-debugger", + "requiredEnv": [] + }, + { + "name": "notion", + "primaryEnv": "NOTION_API_TOKEN", + "requiredEnv": [] + }, + { + "name": "python-debugpy", + "requiredEnv": [] + }, + { + "name": "skill-creator" + }, + { + "name": "spike" + }, + { + "name": "taskflow" + }, + { + "name": "taskflow-inbox-triage" + }, + { + "name": "weather" + } + ], + "version": 1783520166453, + "promptFormatVersion": 1, + "promptRef": { + "version": 1, + "algorithm": "sha256", + "hash": "e5fe28e69f48dbacedd00eaae618e2965ff91d2b87d9989e52c0446673d3ab47", + "bytes": 4874 + } + }, + "lastInteractionAt": 1783520172250, + "sessionFile": "/home/node/.openclaw/agents/main/sessions/02f4cce4-d744-4e62-a34c-4e99292becd2.jsonl", + "contextTokens": 1048576, + "modelProvider": "anthropic", + "model": "claude-opus-4-8", + "agentHarnessId": "openclaw", + "abortedLastRun": false, + "systemPromptReport": { + "source": "run", + "generatedAt": 1783520167610, + "sessionId": "02f4cce4-d744-4e62-a34c-4e99292becd2", + "sessionKey": "agent:main:telegram:direct:15555550123", + "provider": "anthropic", + "model": "claude-opus-4-8", + "workspaceDir": "/home/node/.openclaw/workspace", + "bootstrapMaxChars": 20000, + "bootstrapTotalMaxChars": 60000, + "bootstrapTruncation": { + "warningMode": "always", + "warningShown": false, + "truncatedFiles": 0, + "nearLimitFiles": 0, + "totalNearLimit": false + }, + "sandbox": { + "mode": "off", + "sandboxed": false + }, + "systemPrompt": { + "chars": 32124, + "hash": "6ee2f4344cb3d4ce6803167be49a3063bd76f142d045ef01574e32ed7a52f97e", + "projectContextChars": 14519, + "nonProjectContextChars": 17605 + }, + "injectedWorkspaceFiles": [ + { + "name": "AGENTS.md", + "path": "/home/node/.openclaw/workspace/AGENTS.md", + "missing": false, + "rawChars": 8654, + "injectedChars": 8654, + "truncated": false + }, + { + "name": "SOUL.md", + "path": "/home/node/.openclaw/workspace/SOUL.md", + "missing": false, + "rawChars": 1797, + "injectedChars": 1797, + "truncated": false + }, + { + "name": "TOOLS.md", + "path": "/home/node/.openclaw/workspace/TOOLS.md", + "missing": false, + "rawChars": 910, + "injectedChars": 910, + "truncated": false + }, + { + "name": "IDENTITY.md", + "path": "/home/node/.openclaw/workspace/IDENTITY.md", + "missing": false, + "rawChars": 693, + "injectedChars": 693, + "truncated": false + }, + { + "name": "USER.md", + "path": "/home/node/.openclaw/workspace/USER.md", + "missing": false, + "rawChars": 534, + "injectedChars": 534, + "truncated": false + }, + { + "name": "HEARTBEAT.md", + "path": "/home/node/.openclaw/workspace/HEARTBEAT.md", + "missing": false, + "rawChars": 243, + "injectedChars": 243, + "truncated": false + }, + { + "name": "BOOTSTRAP.md", + "path": "/home/node/.openclaw/workspace/BOOTSTRAP.md", + "missing": false, + "rawChars": 1510, + "injectedChars": 1510, + "truncated": false + } + ], + "skills": { + "promptChars": 4872, + "hash": "c96dc0d8eeefa7f8bea8fba0ffc088a2964387e4241ca56d78badc77d514f39d", + "entries": [ + { + "name": "browser-automation", + "blockChars": 371 + }, + { + "name": "canvas", + "blockChars": 289 + }, + { + "name": "clawhub", + "blockChars": 305 + }, + { + "name": "diagram-maker", + "blockChars": 278 + }, + { + "name": "healthcheck", + "blockChars": 290 + }, + { + "name": "meme-maker", + "blockChars": 263 + }, + { + "name": "node-connect", + "blockChars": 293 + }, + { + "name": "node-inspect-debugger", + "blockChars": 289 + }, + { + "name": "notion", + "blockChars": 284 + }, + { + "name": "python-debugpy", + "blockChars": 277 + }, + { + "name": "skill-creator", + "blockChars": 271 + }, + { + "name": "spike", + "blockChars": 263 + }, + { + "name": "taskflow", + "blockChars": 293 + }, + { + "name": "taskflow-inbox-triage", + "blockChars": 303 + }, + { + "name": "weather", + "blockChars": 301 + } + ] + }, + "tools": { + "listChars": 0, + "schemaChars": 20290, + "entries": [ + { + "name": "read", + "summaryChars": 298, + "summaryHash": "f8a1d7cf1ecfe02b1b5865c93387e6a5891e5079117b4fe18614b6f912521727", + "schemaChars": 304, + "schemaHash": "134f19bcabe3e29d63c5cebb38f1d2556759fd08adad6bc90a4b4d3cd1fb8441", + "propertiesCount": 3 + }, + { + "name": "edit", + "summaryChars": 326, + "summaryHash": "c498a85576e187d069e8e8ea4f70f81bddd57829600d385ef61c4359fd634c9e", + "schemaChars": 834, + "schemaHash": "457e8bb9f800b21be76c4fd8c0e3078bf07ee9bf93ac82f39944c171bd8f6d84", + "propertiesCount": 2 + }, + { + "name": "write", + "summaryChars": 127, + "summaryHash": "cdb1c46027b22d1f1b923b47bfe297821d9fe589a9f9d92994961948060a4d89", + "schemaChars": 225, + "schemaHash": "e98a2484f667cf7c22d76ca103bf2022bf9113dc63fe38b899e71c328cb1e833", + "propertiesCount": 2 + }, + { + "name": "apply_patch", + "summaryChars": 132, + "summaryHash": "432bf38688a4f011d2b765a441ac07e1fdfa9faf2051679adee04abcd9332d93", + "schemaChars": 153, + "schemaHash": "05521268363ff4f191472c90d69ef08a503ba9acf1d2febc50b3ee68db32ab65", + "propertiesCount": 1 + }, + { + "name": "exec", + "summaryChars": 539, + "summaryHash": "4825198988fbfa8ebf5c2a8fbe3946da0be639b1e3b737ceaa4364195bba3f04", + "schemaChars": 1283, + "schemaHash": "2608ea36cf51376859b7204cda1bf1a2507134eb542097b9e8dbc8af617f216b", + "propertiesCount": 12 + }, + { + "name": "process", + "summaryChars": 456, + "summaryHash": "55297e65d460e0e1e06e7fad4cedb75ffda8fac9fa8b10aa06d6cead3c5a76ea", + "schemaChars": 1074, + "schemaHash": "a2d30a25569f409d7e8539844c89a088044535388b24031d014577b96ed27c5c", + "propertiesCount": 12 + }, + { + "name": "cron", + "summaryChars": 3820, + "summaryHash": "d3bc0032e9c0ba71e5983ee4d06bd4b70ea5170dcb5d85fbff1bd71f01b070f4", + "schemaChars": 8705, + "schemaHash": "830efafdf26c1cd1c32452d2baea3b8fc0c7b7b7269d9ab0a503aded48f88ab1", + "propertiesCount": 15 + }, + { + "name": "get_goal", + "summaryChars": 71, + "summaryHash": "58dea7906d9e17e5f89a75fea24597ee068d8a56af824370dc27f8501374d6e3", + "schemaChars": 33, + "schemaHash": "8243f0af367f188a376f2c17b5eabe872a2f7a979813e0d4e2be6d594c2aa259", + "propertiesCount": 0 + }, + { + "name": "create_goal", + "summaryChars": 155, + "summaryHash": "f01e3a63d673a410ba55213ce6bd3fa2b0d2279e8e780c18aa0cb5aa991b29b3", + "schemaChars": 267, + "schemaHash": "27ee794432736471bd76390e236cc7d7e98f8469a02656e7a27109041eea0f8c", + "propertiesCount": 2 + }, + { + "name": "update_goal", + "summaryChars": 212, + "summaryHash": "285201690e5f2456b4cdcd252ff391709b84d69538df61bad694d3c0cfc19958", + "schemaChars": 207, + "schemaHash": "c37894492eedfad02f1e7fe1488900765bfe547f3b3f34cefbab4f7f57441c2c", + "propertiesCount": 2 + }, + { + "name": "skill_workshop", + "summaryChars": 171, + "summaryHash": "577c1d2b8f56bf3e9dd5e00463bbce8b8bf4cf201da9b21b588dcaa3bc6ed8cf", + "schemaChars": 2273, + "schemaHash": "15481d6889a320a46728ee32304f0a2b510c0bcf77d445052c1ef2fa5dd5368b", + "propertiesCount": 13 + }, + { + "name": "update_plan", + "summaryChars": 163, + "summaryHash": "ea18cc44224e849c68cdf0a4839c295c1b20daf2699e2fc2be4c4e1552327455", + "schemaChars": 489, + "schemaHash": "50cb08b8a29e72637eb6720e6f5f3786783824537574f05ee9ae7a5f4ecacc4b", + "propertiesCount": 2 + }, + { + "name": "sessions_list", + "summaryChars": 135, + "summaryHash": "c7e411ae02e3adff5504aaee48f02d262a8abb9152558a3172aae289e3b05faf", + "schemaChars": 435, + "schemaHash": "c29c6b82c36a2d95a88c8944f129929b2621ee2af6d17b2ab101d4d7e9633d89", + "propertiesCount": 9 + }, + { + "name": "sessions_history", + "summaryChars": 117, + "summaryHash": "f5d45b76f11a8ac63cc45d456fb6f675f0b1a2e191feb9671924cb5830058f03", + "schemaChars": 162, + "schemaHash": "ac33b8af2252262afe479236046f5a04bc018f1fcf24d1a8bf32743f0e867598", + "propertiesCount": 3 + }, + { + "name": "sessions_send", + "summaryChars": 282, + "summaryHash": "061e9ef3487676bb2a4fd4c0336bd973aaab5eea0725e7443c4cb17518cd1e78", + "schemaChars": 275, + "schemaHash": "f1fbaa412fe6b6aee102234c87c5aa140f39c9086f9bae6e1e9691309a602ffb", + "propertiesCount": 5 + }, + { + "name": "sessions_spawn", + "summaryChars": 578, + "summaryHash": "c75061234ba01a4eb29ca223c1c63a44935e9e555ef2ca280722ba9e844db86d", + "schemaChars": 1151, + "schemaHash": "9a91c7668463da8a66386260ce2ae97e7efe3bb93b9b6f03ac85102c354457e1", + "propertiesCount": 15 + }, + { + "name": "sessions_yield", + "summaryChars": 79, + "summaryHash": "f3782fc41c1522657e1d3494f486a2df998fed38a6b89dcf7f6934fb41424320", + "schemaChars": 60, + "schemaHash": "b1c4c753820bc3096d85ef7a266a14c9fbe12450536db58076b75bb61d14dbbb", + "propertiesCount": 1 + }, + { + "name": "subagents", + "summaryChars": 132, + "summaryHash": "1c2ffd38549c0444b118d43736fc4b875ea1aaabc446bbf8bc1fc4ee9180cca4", + "schemaChars": 122, + "schemaHash": "c4a241d2c3d46ba7985abfce4d3fcd63298ce81f89d49183c8f3d584c48d0c33", + "propertiesCount": 2 + }, + { + "name": "session_status", + "summaryChars": 278, + "summaryHash": "71177e543dc6235bad4b1a26761995681dcec27ab58dcb4f4f8a7fc9a7b2b5aa", + "schemaChars": 89, + "schemaHash": "d2995f3d69d126e98ac93eec3c2a9fe5556925bd7979a8e4b6b5a2bf62d329dd", + "propertiesCount": 2 + }, + { + "name": "web_search", + "summaryChars": 65, + "summaryHash": "b38e9b3c079beaa988d3c576303660b91a55c2e72f56351da04160529173273f", + "schemaChars": 991, + "schemaHash": "3879e368875d824b14e0e683f81408824e9a2a5850fa2f8514a68caebda54d18", + "propertiesCount": 12 + }, + { + "name": "web_fetch", + "summaryChars": 93, + "summaryHash": "8875afbb51bd63673e5e6836ca31eeda73914a464862665cd73b346e28d28fd5", + "schemaChars": 317, + "schemaHash": "d2d8ad58a1b84d849dd16142282aff6c5606f7b0e3d08405e1b740a6a79e6bcc", + "propertiesCount": 3 + }, + { + "name": "image", + "summaryChars": 169, + "summaryHash": "a46820ca8952445d3663a8908bfa5d3adddff5b53d18b9d8e9eed8f9e593b494", + "schemaChars": 350, + "schemaHash": "49bb766071c935edca8c2d0df47fc2109bbffe3d4ba32d0b19fe90ec873736ce", + "propertiesCount": 6 + }, + { + "name": "memory_search", + "summaryChars": 605, + "summaryHash": "0f918d28864d3682ae77731f149273997c84773592a466b18c7cd0121b599a05", + "schemaChars": 250, + "schemaHash": "088cd2a3d3a47b251429ff7c56844cdedd1c45c3af0fa8bfaf7aa795766976bf", + "propertiesCount": 4 + }, + { + "name": "memory_get", + "summaryChars": 239, + "summaryHash": "bff1141a3da31a345c550337417efc9ace4da2909004606c727c59ca1a294495", + "schemaChars": 241, + "schemaHash": "78f5d58460a82f9104deebad2f1873a8cde05d1ce7328b3ba4da6e994653e70a", + "propertiesCount": 4 + } + ] + }, + "currentTurn": { + "promptChars": 44, + "runtimeContextChars": 0 + } + }, + "contextBudgetStatus": { + "schemaVersion": 1, + "source": "pre-prompt-estimate", + "updatedAt": 1783520167682, + "provider": "anthropic", + "model": "claude-opus-4-8", + "route": "fits", + "shouldCompact": false, + "estimatedPromptTokens": 9688, + "contextTokenBudget": 1048576, + "promptBudgetBeforeReserve": 1028576, + "reserveTokens": 20000, + "effectiveReserveTokens": 20000, + "remainingPromptBudgetTokens": 1018888, + "overflowTokens": 0, + "toolResultReducibleChars": 0, + "messageCount": 0, + "unwindowedMessageCount": 0, + "sessionId": "02f4cce4-d744-4e62-a34c-4e99292becd2" + }, + "inputTokens": 2, + "outputTokens": 10, + "totalTokens": 25216, + "totalTokensFresh": true, + "cacheRead": 24402, + "cacheWrite": 812 + } +} \ No newline at end of file diff --git a/test-fixtures/openclaw/subagent.jsonl b/test-fixtures/openclaw/subagent.jsonl new file mode 100644 index 00000000..1fc34ac3 --- /dev/null +++ b/test-fixtures/openclaw/subagent.jsonl @@ -0,0 +1,8 @@ +{"type":"session","version":3,"id":"cdff4998-f609-45e7-beeb-ecb980d44cab","timestamp":"2026-07-08T14:15:00.247Z","cwd":"/home/node/.openclaw/workspace"} +{"type":"model_change","id":"dcb10950","parentId":null,"timestamp":"2026-07-08T14:15:00.257Z","provider":"anthropic","modelId":"claude-opus-4-8"} +{"type":"thinking_level_change","id":"7a9f51cb","parentId":"dcb10950","timestamp":"2026-07-08T14:15:00.257Z","thinkingLevel":"off"} +{"type":"custom","customType":"model-snapshot","data":{"timestamp":1783520100282,"provider":"anthropic","modelApi":"anthropic-messages","modelId":"claude-opus-4-8"},"id":"da709a26","parentId":"7a9f51cb","timestamp":"2026-07-08T14:15:00.282Z"} +{"type":"message","id":"f25395ef","parentId":"da709a26","timestamp":"2026-07-08T14:15:00.308Z","message":{"role":"user","content":[{"type":"text","text":"[Subagent Context] You are running as a subagent (depth 1/1). Results auto-announce to your requester; do not busy-poll for status.\n\n[Subagent Task]\n\nCount the words in `notes.md` and report the number.\n\nBegin. Execute the assigned task to completion."}],"timestamp":1783520100299}} +{"type":"message","id":"c84b7bd9","parentId":"f25395ef","timestamp":"2026-07-08T14:15:04.185Z","message":{"role":"assistant","content":[{"type":"text","text":"I'll count the words in `notes.md`."},{"type":"toolCall","id":"toolu_01RPGQ3ZPQPZaw7tRp6V9cKE","name":"exec","arguments":{"command":"wc -w notes.md"}}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":68,"cacheRead":0,"cacheWrite":14304,"totalTokens":14374,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1783520100313,"responseId":"msg_011CcpjKtyuftCVALUHE3gx8","responseModel":"claude-opus-4-8"}} +{"type":"message","id":"5c929d18","parentId":"c84b7bd9","timestamp":"2026-07-08T14:15:04.254Z","message":{"role":"toolResult","toolCallId":"toolu_01RPGQ3ZPQPZaw7tRp6V9cKE","toolName":"exec","content":[{"type":"text","text":"10 notes.md"}],"details":{"status":"completed","exitCode":0,"durationMs":23,"aggregated":"10 notes.md","cwd":"/home/node/.openclaw/workspace"},"isError":false,"timestamp":1783520104253}} +{"type":"message","id":"c217f5d2","parentId":"5c929d18","timestamp":"2026-07-08T14:15:06.043Z","message":{"role":"assistant","content":[{"type":"text","text":"`notes.md` contains **10 words**."}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":16,"cacheRead":14304,"cacheWrite":79,"totalTokens":14401,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1783520104260,"responseId":"msg_011CcpjLBZJratQCMCEQdL6e","responseModel":"claude-opus-4-8"}} diff --git a/test-fixtures/openclaw/telegram-dm.jsonl b/test-fixtures/openclaw/telegram-dm.jsonl new file mode 100644 index 00000000..8f3ad999 --- /dev/null +++ b/test-fixtures/openclaw/telegram-dm.jsonl @@ -0,0 +1,7 @@ +{"type":"session","version":3,"id":"02f4cce4-d744-4e62-a34c-4e99292becd2","timestamp":"2026-07-08T14:16:07.619Z","cwd":"/home/node/.openclaw/workspace"} +{"type":"model_change","id":"fbd8c826","parentId":null,"timestamp":"2026-07-08T14:16:07.628Z","provider":"anthropic","modelId":"claude-opus-4-8"} +{"type":"thinking_level_change","id":"5b9b82cf","parentId":"fbd8c826","timestamp":"2026-07-08T14:16:07.628Z","thinkingLevel":"off"} +{"type":"custom","customType":"model-snapshot","data":{"timestamp":1783520167656,"provider":"anthropic","modelApi":"anthropic-messages","modelId":"claude-opus-4-8"},"id":"f22a1dd0","parentId":"5b9b82cf","timestamp":"2026-07-08T14:16:07.657Z"} +{"type":"message","id":"841e2311","parentId":"f22a1dd0","timestamp":"2026-07-08T14:16:07.689Z","message":{"role":"user","content":[{"type":"text","text":"Reply with exactly: telegram channel test ok"}],"timestamp":1783520167686}} +{"type":"message","id":"8c800a8a","parentId":"841e2311","timestamp":"2026-07-08T14:16:12.213Z","message":{"role":"assistant","content":[{"type":"text","text":"telegram channel test ok"}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","usage":{"input":2,"output":10,"cacheRead":24402,"cacheWrite":812,"totalTokens":25226,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1783520167694,"responseId":"msg_011CcpjR1B7daoDeoAnfHRZj","responseModel":"claude-opus-4-8"}} +{"type":"custom","customType":"openclaw:bootstrap-context:full","data":{"timestamp":1783520172223,"runId":"bd12772c-b4c6-4a08-89de-4b81a6acb08e","sessionId":"02f4cce4-d744-4e62-a34c-4e99292becd2"},"id":"ac8989ee","parentId":"8c800a8a","timestamp":"2026-07-08T14:16:12.223Z"} From f9ed14be0d280ebe0a58855573274a25f5b9f3bf Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 10:25:37 -0400 Subject: [PATCH 19/22] test(cli): openclaw in the cross-harness matrix (all cells pass on the real fixture) --- crates/path-cli/tests/cross_harness_matrix.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/path-cli/tests/cross_harness_matrix.rs b/crates/path-cli/tests/cross_harness_matrix.rs index 34e55fee..f4d8ab60 100644 --- a/crates/path-cli/tests/cross_harness_matrix.rs +++ b/crates/path-cli/tests/cross_harness_matrix.rs @@ -162,6 +162,45 @@ impl Harness for PiHarness { } } +struct OpenclawHarness; +impl Harness for OpenclawHarness { + fn name(&self) -> &'static str { + "openclaw" + } + fn roundtrip(&self, view: &ConversationView) -> ConversationView { + let projector = toolpath_openclaw::project::OpenClawProjector::default(); + let session = projector.project(view).expect("openclaw project"); + toolpath_openclaw::session_to_view(&session) + } + fn load_fixture(&self) -> Option { + let path = fixtures_dir().join("openclaw/convo.jsonl"); + if !path.exists() { + return None; + } + let session = toolpath_openclaw::reader::read_session_from_file(&path) + .expect("openclaw fixture parse"); + Some(toolpath_openclaw::session_to_view(&session)) + } + fn schema_validates(&self, view: &ConversationView) -> Result<(), String> { + let projector = toolpath_openclaw::project::OpenClawProjector::default(); + let session = projector + .project(view) + .map_err(|e| format!("project: {}", e))?; + let mut lines: Vec = Vec::new(); + for entry in &session.entries { + lines.push(serde_json::to_string(entry).map_err(|e| format!("entry: {}", e))?); + } + let tmp = tempfile::Builder::new() + .suffix(".jsonl") + .tempfile() + .map_err(|e| format!("tempfile: {}", e))?; + std::fs::write(tmp.path(), lines.join("\n")).map_err(|e| format!("write: {}", e))?; + toolpath_openclaw::reader::read_session_from_file(tmp.path()) + .map_err(|e| format!("re-read: {}", e))?; + Ok(()) + } +} + struct GeminiHarness; impl Harness for GeminiHarness { fn name(&self) -> &'static str { @@ -983,6 +1022,7 @@ fn run_matrix(label: &str, sources: &[(String, ConversationView)]) { Box::new(ClaudeHarness), Box::new(CodexHarness), Box::new(PiHarness), + Box::new(OpenclawHarness), Box::new(GeminiHarness), Box::new(OpencodeHarness), Box::new(CursorHarness), @@ -1033,6 +1073,7 @@ fn all_harnesses() -> Vec> { Box::new(ClaudeHarness), Box::new(CodexHarness), Box::new(PiHarness), + Box::new(OpenclawHarness), Box::new(GeminiHarness), Box::new(OpencodeHarness), Box::new(CursorHarness), From 01a344b794dcca7ead17306c6391aca39c430a44 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 10:25:37 -0400 Subject: [PATCH 20/22] docs(openclaw): upgrade format reference to observed + record real-data findings First-hand observation pass (image v2026.6.11): two-code-layers question resolved, transcript mode corrected to 0644, skills-prompts/ dir, observed tool vocabulary, subagent keys, DM-scope default, sessions.json 19-field entries, inception adoption. CHANGELOG entry for toolpath-openclaw 0.1.0. --- CHANGELOG.md | 15 +++++ CLAUDE.md | 4 +- docs/agents/feature-elicit.md | 1 + docs/agents/formats/openclaw/README.md | 61 +++++++++++-------- .../formats/openclaw/directory-layout.md | 5 +- .../formats/openclaw/format-changelog.md | 29 +++++++++ docs/agents/formats/openclaw/known-issues.md | 24 +++++--- 7 files changed, 98 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00545273..a46cbf83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to the Toolpath workspace are documented here. +## toolpath-openclaw 0.1.0: new provider (derive + inception) — 2026-07-08 + +New crate `toolpath-openclaw`: reads OpenClaw's multi-channel agent-session +JSONL (`~/.openclaw/agents//sessions/`, format v3), derives `Path` +documents with channel-aware `human:/` actors recovered +from the `sessions.json` routing key, and projects/incepts a `Path` back +onto disk (transcript + routing entry) that a **running** OpenClaw gateway +adopts without restart — verified live. Wired through the `path` CLI +(`p import/list/show/export openclaw`, `share`, and "inception+" `resume` +that skips exec when a gateway is already listening). Validated against +real sessions captured from the official Docker image (v2026.6.11), +committed as `test-fixtures/openclaw/` and wired into the cross-harness +matrix (all cells pass). File changes are structural only — OpenClaw +stores no diffs. + ## toolpath-convo 0.12.0: session-level `user_actor` override — 2026-06-30 Adds `DeriveConfig.user_actor: Option` so a provider can set a diff --git a/CLAUDE.md b/CLAUDE.md index cd5ca388..d3f5b5bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -188,7 +188,7 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has in - `toolpath-opencode`: 52 unit + 19 integration + 1 doc test (SQLite reader, JSON payload serde, provider assembly, snapshot-based derive, tool-input fallback for gitignored paths, reasoning breakdown) - `toolpath-cursor`: 78 unit + 8 integration round-trip + 1 real-DB sanity + 1 doc test (state.vscdb SQLite reader, bubble store + composer header parsing, content-addressed blob lookup, projector with full TOOL_TABLE coverage, JSONL transcript ingest in `examples/dump_fixture.rs`) - `toolpath-pi`: 133 unit + 26 integration + 5 doc tests (types, paths, error, reader, io, provider) -- `toolpath-openclaw`: 48 unit + 1 integration test (v3 schema serde, path/session-key resolution, reader + tree/visible-leaf assembly, listing, session_to_view mapping + channel-aware actor, derive, projector inception + sessions.json, full projection round-trip) +- `toolpath-openclaw`: 52 unit + 5 integration tests (v3 schema serde, path/session-key resolution, reader + tree/visible-leaf assembly, listing, session_to_view mapping + channel-aware actor, derive, projector inception + raw-JSON sessions.json merge, full projection round-trip, and a real-session suite over the captured fixtures in `test-fixtures/openclaw/` — telegram channel actor, spawn-child kind, real tool vocabulary) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) - `path-cli`: 300 unit + 65 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, openclaw import + share aggregation + inception+ resume, `path resume` orchestration with injectable `ExecStrategy`). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. - `toolpath-cli`: 0 tests (it's a one-line `path_cli::run()` shim crate that exists only so `cargo install toolpath-cli` keeps installing the `path` binary) @@ -250,7 +250,7 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - Token accounting (kind v1.1.0): two keys on `conversation.append`/`Turn`, both optional. `token_usage` = "the total for a message" (on the group's final step; `Σ` over a path = session total). `attributed_token_usage` = "this step's own attributed spend", populated only where the source genuinely reports per-step spend (its own key, so the sum is unaffected; remainder = group total − Σ attributed, computed not stored). One provider message can span several steps (Claude writes one JSONL line per content block); `Turn.group_id` groups them. `toolpath-claude` fills `group_id` from `message.id` and takes the **field-wise-max** group total (line order not trusted). Claude's per-line `usage` is a cumulative *streaming snapshot* (Anthropic streaming API: `message_start` seeds output near 0, `message_delta` is cumulative), NOT a per-block cost — so Claude emits no `attributed_token_usage`; the projector re-expands the total onto every line. `toolpath-codex` differences the cumulative `total_token_usage` (dedup-safe: never sum `last_token_usage` — Codex re-emits it stale; openai/codex #14489), attributes each per-call delta to the step it follows, and derives the round total from those attributions. pi/opencode decode all-zero wire counters as `None`. Never stamp a cumulative counter, a repeated message total, or zero-filled placeholders onto a step; never derive attribution from Claude's streaming snapshots. - Token usage `breakdowns` (kind v1.1.0, additive): an optional third key on `TokenUsage` — a decomposition of a top-level class into named sub-classes, keyed by class (e.g. `"output"`), inner map sub-class → tokens (e.g. `breakdowns["output"]["reasoning"] = 243`). INFORMATIONAL ONLY: **never summed into any total** (the parent class already counts those tokens, so the session-total guarantee is untouched); invariant `Σ(inner) ≤ parent`; omitted when empty; rides both `token_usage` and `attributed_token_usage`. Per-provider reality: **Gemini** reports `thoughts` (reasoning) as an additive sibling that the derivation used to **drop** (under-counting output) — it's now folded into `output_tokens` *and* recorded as `breakdowns["output"]["reasoning"]`, with the projector un-folding it on the reverse path for a lossless round-trip (`Some(0)` preserved as a real Gemini-3 zero-reasoning signal). **OpenCode** folds `reasoning` into output and records the same breakdown. **Codex** differences `reasoning_output_tokens` (⊆ output, cumulative) into `breakdowns["output"]["reasoning"]` on both per-step `attributed_token_usage` and per-round `token_usage`. **Claude** records no breakdown (its JSONL `usage` doesn't itemize thinking tokens). - Pi provider: `toolpath-pi` reads Pi session JSONL from `~/.pi/agent/sessions/`. Sessions use a tree (id/parentId) in a single file, and may link to a parent file via `parentSession` in the header. The tree is preserved as a DAG in the derived `Path`. -- OpenClaw provider: `toolpath-openclaw` reads OpenClaw agent-session JSONL from `~/.openclaw/agents//sessions/.jsonl` (override the state dir with `OPENCLAW_STATE_DIR`/`OPENCLAW_HOME`; legacy `~/.clawdbot`). Format is **version 3**: a session header line + an `id`/`parentId` entry tree whose visible head is moved by `leaf` rows (same family as Pi). OpenClaw is **multi-channel** — the human isn't a transcript field; identity lives in the `sessions.json` routing key `agent::::`, which the provider recovers into a **channel-aware** `human:/` actor (via the additive `DeriveConfig.user_actor`, kind v1.x) and stashes (channel/peer/sessionKey/sessionKind) on `path.meta.extra["openclaw"]`. **No file diffs** are stored, so changes are structural/tool-input-only (no `raw` perspective). The projector (`project.rs`) **incepts** a `Path` back onto disk — writing the transcript plus a `sessions.json` routing entry so a running OpenClaw instance can pick it up (`path p export openclaw --project`). CLI is agent-keyed (`--agent`, default `main`), not project-keyed. See `docs/agents/formats/openclaw/` for the full format reference. +- OpenClaw provider: `toolpath-openclaw` reads OpenClaw agent-session JSONL from `~/.openclaw/agents//sessions/.jsonl` (override the state dir with `OPENCLAW_STATE_DIR`/`OPENCLAW_HOME`; legacy `~/.clawdbot`). Format is **version 3**: a session header line + an `id`/`parentId` entry tree whose visible head is moved by `leaf` rows (same family as Pi). Validated against real sessions captured from the official Docker image (`scripts/openclaw-docker.sh`; fixtures at `test-fixtures/openclaw/`, wired into the cross-harness matrix). Observed native tools: `exec`/`read`/`write`/`edit`/`sessions_spawn` (delegation); sub-agent runs get `agent::subagent:` keys → sessionKind `spawn-child`, no channel actor. OpenClaw is **multi-channel** — the human isn't a transcript field; identity lives in the `sessions.json` routing key `agent::::`, which the provider recovers into a **channel-aware** `human:/` actor (via the additive `DeriveConfig.user_actor`, kind v1.x) and stashes (channel/peer/sessionKey/sessionKind) on `path.meta.extra["openclaw"]`. **No file diffs** are stored, so changes are structural/tool-input-only (no `raw` perspective). The projector (`project.rs`) **incepts** a `Path` back onto disk — writing the transcript plus a `sessions.json` routing entry so a running OpenClaw instance can pick it up (`path p export openclaw --project`). CLI is agent-keyed (`--agent`, default `main`), not project-keyed. See `docs/agents/formats/openclaw/` for the full format reference. - Codex provider: `toolpath-codex` reads Codex CLI rollout files from `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl`. Sessions are date-bucketed (not project-keyed). File-change fidelity is excellent — Codex's `patch_apply_end` events carry either the unified diff (for updates) or the full file content (for adds), so the derived `Path` gets a real `raw` perspective on every file artifact. See `docs/agents/formats/codex.md` for the full format reference. - opencode provider: `toolpath-opencode` reads a SQLite database at `~/.local/share/opencode/opencode.db` (opened read-only). Each session's messages and 12 typed part variants (text, reasoning, tool, step-start/-finish, snapshot, patch, file, agent, subtask, retry, compaction) land as one step per message with tool invocations attached. File diffs come from a sibling bare git repo at `snapshot//[]/` via `git2` tree↔tree diffs — opencode respects the user's `.gitignore`, so changes under gitignored paths fall back to tool-input-derived structural changes with no `raw` perspective. Project id is the SHA of the repo's first root commit. See `docs/agents/formats/opencode.md` for the full format reference. - Cursor (IDE) provider: `toolpath-cursor` reads Cursor.app's global `state.vscdb` SQLite (opened read-only) at `~/Library/Application Support/Cursor/User/globalStorage/state.vscdb` (macOS; `~/.config/Cursor/...` on Linux). Composers, bubbles, and content-addressed file blobs are stored as key-prefixed rows in the `cursorDiskKV` table (`composerData:`, `bubbleId::`, `composer.content.`) plus a `composer.composerHeaders` index blob in `ItemTable`. The full tool-dispatch enum (53 entries, ids 0–63) is extracted from the workbench bundle into `TOOL_TABLE` for round-trip-correct numeric ids — projector-written composers load back into Cursor.app's UI with the right tool rendering. The cursor-agent CLI uses a different per-chat protobuf store at `~/.cursor/chats///store.db` that this crate does not yet parse — that's deferred to a future `toolpath-cursor-cli` companion. See `docs/agents/formats/cursor.md` for the full format reference. diff --git a/docs/agents/feature-elicit.md b/docs/agents/feature-elicit.md index 8a00b0ca..aa60f2ab 100644 --- a/docs/agents/feature-elicit.md +++ b/docs/agents/feature-elicit.md @@ -48,6 +48,7 @@ version uses different flags): | Gemini | `gemini -p ""` | `~/.gemini/tmp//chats/session-*.json` | | Pi | `pi -p ""` (varies; manual fallback documented below) | `~/.pi/agent/sessions/----/*.jsonl` | | Opencode | `opencode run ""` then `path p export opencode` | rows in `opencode.db`, exported to JSON | +| OpenClaw | `scripts/openclaw-docker.sh up` then `agent "$(cat docs/agents/feature-elicit.prompt.txt)"` | `~/.openclaw-docker/agents/main/sessions/.jsonl` | Cursor is **not** in the automated list. The IDE has no scriptable chat-panel entry point, and the `cursor-agent` CLI writes its real diff --git a/docs/agents/formats/openclaw/README.md b/docs/agents/formats/openclaw/README.md index 7d280ac5..b6484e5c 100644 --- a/docs/agents/formats/openclaw/README.md +++ b/docs/agents/formats/openclaw/README.md @@ -1,10 +1,12 @@ # OpenClaw on-disk format -> **Reference revision:** 2026-06-30 -> **Tracks:** OpenClaw package `2026.6.10`, session format **version 3** +> **Reference revision:** 2026-07-08 +> **Tracks:** OpenClaw package `2026.6.10`–`2026.6.11`, session format **version 3** > **Sourced from:** upstream code at -> `openclaw/openclaw @ 68c533cfb339cbb8650832cb2a4bf38dba7022fa` — **no -> first-hand on-disk sample yet** (see [Sourcing](#sourcing-and-confidence)). +> `openclaw/openclaw @ 68c533cfb339cbb8650832cb2a4bf38dba7022fa`, now +> corroborated by **first-hand sessions** captured from the official +> `openclaw/openclaw` Docker image (v`2026.6.11`) via +> `scripts/openclaw-docker.sh` (see [Sourcing](#sourcing-and-confidence)). > > When you change anything in this directory, bump the revision date here > and add a note to [format-changelog.md](format-changelog.md). @@ -59,32 +61,37 @@ prefer a concrete example, start with the **walkthrough** (#10). ## Sourcing and confidence -Unlike the other references in this directory (claude-code, codex, cursor, -gemini, opencode, pi), this one is **not** backed by first-hand on-disk -samples or by our own parser: - -- **No installed OpenClaw / no sample file.** Every claim is derived from - **upstream source code** at the pinned commit, not from observed bytes. -- **A `toolpath-openclaw` crate now implements this reference** - (`crates/toolpath-openclaw`), so the envelope, entry types, message/content - shapes, token-usage mapping, and the projector round-trip are corroborated - by its tests — but still against **synthesized** fixtures, not a captured - real session. Claims here remain "from source" until a real OpenClaw - session confirms them. +Originally compiled purely from upstream source; now backed by both a +parser and first-hand samples: + +- **First-hand sessions observed** (2026-07-07/08): real transcripts were + captured from the official Docker image (v`2026.6.11`) via + `scripts/openclaw-docker.sh` and read back with `toolpath-openclaw`. The + header, entry types (`session`/`message`/`model_change`/ + `thinking_level_change`/`custom`), roles (`user`/`assistant`/`toolResult`), + content blocks, `stopReason`, and the `Usage` shape (including a real + prompt-cache `cacheWrite`) all matched this reference exactly. Observed + native tool names: bare `read` / `write`. Inception was verified live: a + running gateway adopts a projected transcript + `sessions.json` entry + without restart and appends follow-up turns to the same file. +- **The `toolpath-openclaw` crate implements this reference** + (`crates/toolpath-openclaw`) with tests over both synthesized fixtures and + the captured real sessions. Consequences for how to read these docs: -- Field tables here mean "**from source**" (the producer's type), not - "observed in the wild." Where a serialized form could not be confirmed - from the code, the doc says so explicitly. -- The transcript is touched by **two code layers** (agent-core harness - storage and the gateway session manager) with slightly different type - names for the same JSON; we use the agent-core names and flag the - reconciliation in - [known-issues.md](known-issues.md#two-code-layers-for-one-format). - -When the crate is built, re-verify against a real session, upgrade claims -from "from source" to "observed," and record it in the changelog. +- "**Observed**" now means seen in a real captured session; "**from + source**" still marks claims only read from upstream code (branching + `leaf`/`appendMode` mechanics, `compaction`/`branch_summary` on disk, + `parentSession` chaining, and group-channel sessions have not yet been + observed in the wild). +- The **two-code-layers question is resolved**: transcripts written by the + running gateway parse cleanly against the agent-core shapes documented + here — the two layers serialize the same JSON + (see [known-issues.md](known-issues.md#two-code-layers-for-one-format)). + +When a new OpenClaw release changes behavior, re-verify against a fresh +capture and record it in the changelog. ## Conventions diff --git a/docs/agents/formats/openclaw/directory-layout.md b/docs/agents/formats/openclaw/directory-layout.md index 06a46065..95ce109c 100644 --- a/docs/agents/formats/openclaw/directory-layout.md +++ b/docs/agents/formats/openclaw/directory-layout.md @@ -59,6 +59,7 @@ find sessions missing, check whether a profile is in play. │ └── openclaw-agent.sqlite # per-agent DB (caches, RAG index) — NOT content └── sessions/ # ← canonical transcripts live here ├── sessions.json # index: sessionKey -> { sessionId, sessionFile, … } + ├── skills-prompts/ # per-session skill prompt snapshots (observed) ├── .jsonl # canonical session transcript ├── -topic-.jsonl # topic-scoped transcript ├── _.jsonl # forked / rotated transcript @@ -125,8 +126,8 @@ of the live store. | Path | Mode | |---|---| -| `…/sessions/.jsonl` | `0600` (append opens `a+`, `0o600`; `src/config/sessions/transcript-jsonl.ts:98`) | -| `…/sessions/sessions.json` | `0600` (atomic write; dir `0o777 & ~umask`) | +| `…/sessions/.jsonl` | **`0644` observed** on real state dirs (the source's append path opens `a+` `0o600` — `src/config/sessions/transcript-jsonl.ts:98` — but the file as created by the running gateway is world-readable) | +| `…/sessions/sessions.json` | `0600` (atomic write; dir `0o777 & ~umask`) — confirmed observed | | `…/sessions/.trajectory.jsonl` | `0600` | | `~/.openclaw/state/openclaw.sqlite` | dir `0700`, file `0600` | | `~/.openclaw/agents//agent/openclaw-agent.sqlite` | dir `0700`, file `0600` | diff --git a/docs/agents/formats/openclaw/format-changelog.md b/docs/agents/formats/openclaw/format-changelog.md index f64939a5..c7c87b53 100644 --- a/docs/agents/formats/openclaw/format-changelog.md +++ b/docs/agents/formats/openclaw/format-changelog.md @@ -9,6 +9,35 @@ The format self-identifies via the header `version` field hard-rejects anything other than `3`. The OpenClaw package version (CalVer, e.g. `2026.6.10`) is separate from this on-disk format version. +## 2026-07-08: first-hand observation pass (image v2026.6.11) + +Real sessions were captured from the official `openclaw/openclaw` Docker +image via `scripts/openclaw-docker.sh` and are committed as fixtures at +`test-fixtures/openclaw/`. Confirmed against real bytes: the v3 header, +entry types `session`/`message`/`model_change`/`thinking_level_change`/ +`custom`/`custom_message`, roles `user`/`assistant`/`toolResult`, content +blocks, `stopReason`, and the full `Usage` shape (including prompt-cache +`cacheWrite`). Newly observed facts: + +- **Native tool names:** `exec`, `read`, `write`, `edit`, + `sessions_spawn` (sub-agent dispatch), `sessions_yield`. +- **Sub-agent routing keys:** `agent::subagent:` appears in + `sessions.json` after a `sessions_spawn`, with the child transcript as a + sibling file. +- **DM scope default:** `agent --to ` routes into + `agent::main` (the default DM scope collapses DMs into the main + session); a channel-scoped key requires an explicit `--session-key`. +- **`sessions.json` entries carry ~19 fields** (`updatedAt`, + `sessionStartedAt`, token counters, `systemPromptReport`, …) and + `sessionFile` is stored **absolute**. +- **Transcript file mode is `0644`** (docs previously said `0600`; see + known-issues). +- **Inception adoption:** a running gateway adopts a transcript + + `sessions.json` entry written by an external tool without restart, and + appends follow-up turns to the same file. +- `leaf` rows are **not** written on every append — a linear real session + contained none (the visible head falls back to the last entry). + ## Format version 3 **Status as of this reference:** the only format version this doc set diff --git a/docs/agents/formats/openclaw/known-issues.md b/docs/agents/formats/openclaw/known-issues.md index 7b789d6c..1b3abb27 100644 --- a/docs/agents/formats/openclaw/known-issues.md +++ b/docs/agents/formats/openclaw/known-issues.md @@ -86,19 +86,23 @@ Old logs predating entry ids are migrated with synthetic `parentId` chains (`migrateLegacySessionEntries`, `src/trajectory/export.ts:946-989`). Ids in such files may be derived, not original. -## Two code layers for one format +## Two code layers for one format (resolved) The transcript is produced/consumed by both the agent-core harness storage (`packages/agent-core/src/harness/session/jsonl-storage.ts`, types in -`harness/types.ts`: `SessionTreeEntry`, `MessageEntry`, `LeafEntry`, …) and -the gateway session manager (`src/agents/sessions/session-manager.ts`, -writer `src/config/sessions/transcript-jsonl.ts`: `SessionEntry`, -`SessionMessageEntry`, `CompactionEntry`, `FileEntry`). These appear to be -two views of the **same** on-disk JSONL, but they use different type names. -This reference uses the agent-core names. **Unconfirmed:** that the two -serialize byte-identically in every case — verify against a real file (and -against whichever layer actually writes the user's transcript) when the -`toolpath-openclaw` crate is built. +`harness/types.ts`) and the gateway session manager +(`src/agents/sessions/session-manager.ts`). **Resolved by observation +(2026-07-07, image v2026.6.11):** transcripts written by the running +gateway parse cleanly against the agent-core shapes documented here — the +two layers serialize the same JSON. This reference keeps the agent-core +names. + +## Transcript file mode is 0644, not 0600 (observed) + +We originally documented transcripts as `0600` from reading the gateway's +append path. Observed on a real state dir: the **transcript** `.jsonl` is +created `0644`, while `sessions.json` and the `.trajectory.jsonl` sidecar +are `0600`. Treat transcript permissions as world-readable by default. ## Daemon profile can relocate the whole store From 5141b8564d5234ba3bc98f60f0a85f730ae8fb85 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 13:15:20 -0400 Subject: [PATCH 21/22] feat(convo): typed round-trip fidelity fields (signatures, response_model, markers) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Turn.thinking_signature/text_signature (opaque reasoning/text replay signatures), Turn.response_model (served-vs-requested model), Turn.marker (ConversationMarker::Compaction/BranchSummary with first_kept_id/tokens_before/file lists/from_hook), and ToolInvocation.thought_signature/execution_mode — all typed, optional, serde-defaulted, and wired through derive_path/extract_conversation so they survive the Path JSON hop. No provider-namespaced extras. New conversation.append keys are valid under the permissive kind v1.1.0 schema. Mechanical None-stubs across existing provider literals. Bumps toolpath-convo 0.12.0 -> 0.13.0. --- CHANGELOG.md | 27 ++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- crates/toolpath-claude/src/project.rs | 11 +++ crates/toolpath-claude/src/provider.rs | 6 ++ crates/toolpath-codex/src/project.rs | 5 + crates/toolpath-codex/src/provider.rs | 3 + crates/toolpath-convo/Cargo.toml | 2 +- crates/toolpath-convo/src/derive.rs | 42 ++++++++ crates/toolpath-convo/src/extract.rs | 32 +++++++ crates/toolpath-convo/src/lib.rs | 95 +++++++++++++++++++ crates/toolpath-convo/src/project.rs | 6 ++ .../toolpath-cursor/examples/dump_fixture.rs | 2 + crates/toolpath-cursor/src/project.rs | 11 +++ crates/toolpath-cursor/src/provider.rs | 3 + .../tests/projection_roundtrip.rs | 3 + crates/toolpath-gemini/src/project.rs | 5 + crates/toolpath-gemini/src/provider.rs | 2 + crates/toolpath-opencode/src/project.rs | 5 + crates/toolpath-opencode/src/provider.rs | 3 + crates/toolpath-pi/src/project.rs | 4 + crates/toolpath-pi/src/provider.rs | 7 ++ site/_data/crates.json | 2 +- 23 files changed, 276 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a46cbf83..8b629c39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,33 @@ All notable changes to the Toolpath workspace are documented here. +## toolpath-convo 0.13.0: typed round-trip fidelity fields — 2026-07-08 + +Adds typed IR homes for cross-provider replay-fidelity data that providers +previously had to drop (no provider-namespaced extras exist on the IR by +design): `Turn.thinking_signature` / `Turn.text_signature` (opaque +reasoning/text replay signatures — Anthropic thinking `signature`, +OpenClaw `thinkingSignature`/`textSignature`), `Turn.response_model` +(concrete served model under alias/auto routing), `Turn.marker` +(`ConversationMarker::Compaction` / `BranchSummary` with +`first_kept_id`/`tokens_before`/file lists/`from_hook` so structural +boundaries survive projection), and +`ToolInvocation.thought_signature`/`execution_mode`. All optional, +serde-defaulted, written/read by the shared `derive_path` / +`extract_conversation` (new `conversation.append` keys are valid under the +permissive kind v1.1.0 schema; a future kind revision will document them). + +`toolpath-openclaw` uses all of them: signatures, `responseId` +(→ `Turn.group_id`, the per-response accounting id), `responseModel`, +and compaction/branch markers now survive session → Path → session +round-trips byte-faithfully; `totalTokens` re-emits with the observed +`input+output+cacheRead+cacheWrite` convention. Also fixes a real +double-counting bug found in captured data: OpenClaw's final assembled +reply after a `sessions_yield` carries run-cumulative usage with no +`responseId`; the provider now detects that aggregate row (exact +field-wise sum match, native files only) and refuses to stamp it onto a +step, keeping session totals single-counted. + ## toolpath-openclaw 0.1.0: new provider (derive + inception) — 2026-07-08 New crate `toolpath-openclaw`: reads OpenClaw's multi-channel agent-session diff --git a/Cargo.lock b/Cargo.lock index 63b5a2c0..28bf2d9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3925,7 +3925,7 @@ dependencies = [ [[package]] name = "toolpath-convo" -version = "0.12.0" +version = "0.13.0" dependencies = [ "chrono", "jsonschema", diff --git a/Cargo.toml b/Cargo.toml index 593ce60d..db204908 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ license = "Apache-2.0" [workspace.dependencies] toolpath = { version = "0.7.0", path = "crates/toolpath" } -toolpath-convo = { version = "0.12.0", path = "crates/toolpath-convo" } +toolpath-convo = { version = "0.13.0", path = "crates/toolpath-convo" } toolpath-git = { version = "0.6.0", path = "crates/toolpath-git" } toolpath-claude = { version = "0.12.0", path = "crates/toolpath-claude", default-features = false } toolpath-gemini = { version = "0.6.0", path = "crates/toolpath-gemini", default-features = false } diff --git a/crates/toolpath-claude/src/project.rs b/crates/toolpath-claude/src/project.rs index 85969d7d..1919e337 100644 --- a/crates/toolpath-claude/src/project.rs +++ b/crates/toolpath-claude/src/project.rs @@ -1033,6 +1033,7 @@ mod tests { fn user_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.to_string(), parent_id: None, group_id: None, @@ -1053,6 +1054,7 @@ mod tests { fn assistant_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.to_string(), parent_id: None, group_id: None, @@ -1226,6 +1228,7 @@ mod tests { let mut turn = assistant_turn("a1", "I'll read the file."); turn.thinking = Some("Hmm, need to read the file first.".to_string()); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Read".to_string(), input: serde_json::json!({"file_path": "src/main.rs"}), @@ -1294,6 +1297,7 @@ mod tests { fn test_tool_results_emitted_as_separate_user_entries() { let mut turn = assistant_turn("a1", "Reading file."); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Read".to_string(), input: serde_json::json!({"file_path": "src/main.rs"}), @@ -1345,6 +1349,7 @@ mod tests { fn test_no_tool_result_entry_when_no_results() { let mut turn = assistant_turn("a1", "Reading..."); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Read".to_string(), input: serde_json::json!({}), @@ -1446,6 +1451,7 @@ mod tests { fn test_assistant_no_text_only_tool_use_produces_parts() { let mut turn = assistant_turn("a1", ""); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Bash".to_string(), input: serde_json::json!({"command": "ls"}), @@ -1474,6 +1480,7 @@ mod tests { let mut turn = assistant_turn("a1", "Reading two files."); turn.tool_uses = vec![ ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Read".to_string(), input: serde_json::json!({}), @@ -1484,6 +1491,7 @@ mod tests { category: None, }, ToolInvocation { + thought_signature: None, execution_mode: None, id: "t2".to_string(), name: "Read".to_string(), input: serde_json::json!({}), @@ -1550,6 +1558,7 @@ mod tests { let mut turn = assistant_turn("a1", "Using tools."); turn.tool_uses = vec![ ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Read".to_string(), input: serde_json::json!({}), @@ -1560,6 +1569,7 @@ mod tests { category: None, }, ToolInvocation { + thought_signature: None, execution_mode: None, id: "t2".to_string(), name: "Write".to_string(), input: serde_json::json!({}), @@ -1601,6 +1611,7 @@ mod tests { vcs_revision: None, }); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".to_string(), name: "Read".to_string(), input: serde_json::json!({}), diff --git a/crates/toolpath-claude/src/provider.rs b/crates/toolpath-claude/src/provider.rs index 1896d1e3..02bffd2f 100644 --- a/crates/toolpath-claude/src/provider.rs +++ b/crates/toolpath-claude/src/provider.rs @@ -89,6 +89,7 @@ fn message_to_turn(entry: &ConversationEntry, msg: &Message) -> Turn { let result = find_tool_result_in_parts(msg, tu.id); let category = tool_category(tu.name); ToolInvocation { + thought_signature: None, execution_mode: None, id: tu.id.to_string(), name: tu.name.to_string(), input: tu.input.clone(), @@ -121,6 +122,7 @@ fn message_to_turn(entry: &ConversationEntry, msg: &Message) -> Turn { let delegations = extract_delegations(&tool_uses); Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: entry.uuid.clone(), parent_id: entry.parent_uuid.clone(), // The API message ID (`msg_…`). Claude Code writes one JSONL line @@ -852,6 +854,7 @@ mod tests { fn message_turn_stub(id: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -1425,6 +1428,7 @@ mod tests { fn test_merge_tool_results_by_id() { // Verify that merge matches by tool_use_id, not position let mut turns = vec![Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t1".into(), parent_id: None, group_id: None, @@ -1434,6 +1438,7 @@ mod tests { thinking: None, tool_uses: vec![ ToolInvocation { + thought_signature: None, execution_mode: None, id: "tool-a".into(), name: "Read".into(), input: serde_json::json!({}), @@ -1441,6 +1446,7 @@ mod tests { category: Some(ToolCategory::FileRead), }, ToolInvocation { + thought_signature: None, execution_mode: None, id: "tool-b".into(), name: "Write".into(), input: serde_json::json!({}), diff --git a/crates/toolpath-codex/src/project.rs b/crates/toolpath-codex/src/project.rs index 2ec3dd51..84bf6a6b 100644 --- a/crates/toolpath-codex/src/project.rs +++ b/crates/toolpath-codex/src/project.rs @@ -699,6 +699,7 @@ mod tests { fn user_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -719,6 +720,7 @@ mod tests { fn assistant_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -824,6 +826,7 @@ mod tests { fn assistant_turn_with_function_call_and_output() { let mut t = assistant_turn("a1", "Let me check."); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "call_001".into(), name: "exec_command".into(), input: json!({"cmd": "pwd"}), @@ -884,6 +887,7 @@ mod tests { // category routes it through `native_name(Shell, _)`. let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "call_x".into(), name: "Bash".into(), input: json!({"command": "ls"}), @@ -911,6 +915,7 @@ mod tests { "*** Begin Patch\n*** Add File: hello.rs\n+fn main(){}\n*** End Patch".to_string(); let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "call_p".into(), name: "apply_patch".into(), input: Value::String(patch_body.clone()), diff --git a/crates/toolpath-codex/src/provider.rs b/crates/toolpath-codex/src/provider.rs index 00bcdf4c..1390043f 100644 --- a/crates/toolpath-codex/src/provider.rs +++ b/crates/toolpath-codex/src/provider.rs @@ -498,6 +498,7 @@ impl<'a> Builder<'a> { fn attach_tool_call(&mut self, timestamp: &str, call_id: String, name: String, input: Value) { let category = tool_category(&name); let invocation = ToolInvocation { + thought_signature: None, execution_mode: None, id: call_id.clone(), name, input, @@ -807,6 +808,7 @@ fn message_to_turn( }); Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: msg.id.clone().unwrap_or_default(), parent_id: None, group_id: None, @@ -835,6 +837,7 @@ fn synthetic_assistant_turn( model: Option<&str>, ) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: format!("synth-{}", timestamp), parent_id: None, group_id: None, diff --git a/crates/toolpath-convo/Cargo.toml b/crates/toolpath-convo/Cargo.toml index 6ec4f1d0..5e04cec0 100644 --- a/crates/toolpath-convo/Cargo.toml +++ b/crates/toolpath-convo/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toolpath-convo" -version = "0.12.0" +version = "0.13.0" edition.workspace = true license.workspace = true repository = "https://github.com/empathic/toolpath" diff --git a/crates/toolpath-convo/src/derive.rs b/crates/toolpath-convo/src/derive.rs index ca53202f..f731fd75 100644 --- a/crates/toolpath-convo/src/derive.rs +++ b/crates/toolpath-convo/src/derive.rs @@ -162,6 +162,30 @@ pub fn derive_path(view: &ConversationView, config: &DeriveConfig) -> Path { ); } + if let Some(sig) = &turn.text_signature { + extra.insert( + "text_signature".to_string(), + serde_json::Value::String(sig.clone()), + ); + } + if let Some(sig) = &turn.thinking_signature { + extra.insert( + "thinking_signature".to_string(), + serde_json::Value::String(sig.clone()), + ); + } + if let Some(rm) = &turn.response_model { + extra.insert( + "response_model".to_string(), + serde_json::Value::String(rm.clone()), + ); + } + if let Some(marker) = &turn.marker + && let Ok(v) = serde_json::to_value(marker) + { + extra.insert("marker".to_string(), v); + } + if config.include_tool_uses && !turn.tool_uses.is_empty() { let arr: Vec = turn .tool_uses @@ -178,6 +202,18 @@ pub fn derive_path(view: &ConversationView, config: &DeriveConfig) -> Path { { obj.as_object_mut().unwrap().insert("result".to_string(), v); } + if let Some(sig) = &t.thought_signature { + obj.as_object_mut().unwrap().insert( + "thought_signature".to_string(), + serde_json::Value::String(sig.clone()), + ); + } + if let Some(mode) = &t.execution_mode { + obj.as_object_mut().unwrap().insert( + "execution_mode".to_string(), + serde_json::Value::String(mode.clone()), + ); + } obj }) .collect(); @@ -684,6 +720,7 @@ mod tests { fn base_turn(id: &str, role: Role) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.to_string(), parent_id: None, group_id: None, @@ -946,6 +983,7 @@ mod tests { vcs_revision: None, }); assistant.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "call-1".into(), name: "write_file".into(), input: serde_json::json!({ "file_path": "a.rs", "content": "fn main() {}" }), @@ -1016,6 +1054,7 @@ mod tests { fn fw_tool(name: &str, id: &str, input: serde_json::Value) -> ToolInvocation { ToolInvocation { + thought_signature: None, execution_mode: None, id: id.to_string(), name: name.to_string(), input, @@ -1086,6 +1125,7 @@ mod tests { fn test_tool_use_non_filewrite_ignored() { let mut turn = base_turn("t1", Role::Assistant); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tu1".into(), name: "Read".into(), input: serde_json::json!({"file_path": "x.rs"}), @@ -1280,6 +1320,7 @@ mod tests { fn test_tool_uses_included_when_enabled() { let mut turn = base_turn("t1", Role::Assistant); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tu1".into(), name: "Read".into(), input: serde_json::json!({}), @@ -1299,6 +1340,7 @@ mod tests { fn test_tool_uses_omitted_when_disabled() { let mut turn = base_turn("t1", Role::Assistant); turn.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tu1".into(), name: "Read".into(), input: serde_json::json!({}), diff --git a/crates/toolpath-convo/src/extract.rs b/crates/toolpath-convo/src/extract.rs index c1d83800..e16f312e 100644 --- a/crates/toolpath-convo/src/extract.rs +++ b/crates/toolpath-convo/src/extract.rs @@ -315,6 +315,22 @@ fn build_turn(step: &Step, extra: &HashMap) -> Turn { .get("attributed_token_usage") .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let text_signature = extra + .get("text_signature") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let thinking_signature = extra + .get("thinking_signature") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let response_model = extra + .get("response_model") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let marker = extra + .get("marker") + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + Turn { id: step.step.id.clone(), parent_id, @@ -322,10 +338,14 @@ fn build_turn(step: &Step, extra: &HashMap) -> Turn { role, timestamp: step.step.timestamp.clone(), text, + text_signature, thinking, + thinking_signature, tool_uses, model, + response_model, stop_reason, + marker, token_usage, attributed_token_usage, environment, @@ -379,12 +399,22 @@ fn build_inline_tool_uses(extra: &HashMap) -> Vec(v.clone()).ok()); + let thought_signature = obj + .get("thought_signature") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let execution_mode = obj + .get("execution_mode") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); Some(ToolInvocation { id, name, input, result, category, + thought_signature, + execution_mode, }) }) .collect() @@ -474,6 +504,8 @@ fn build_tool_invocation(extra: &HashMap) -> ToolInvo input, result, category, + thought_signature: None, + execution_mode: None, } } diff --git a/crates/toolpath-convo/src/lib.rs b/crates/toolpath-convo/src/lib.rs index dcf3c3e2..34a5ca7c 100644 --- a/crates/toolpath-convo/src/lib.rs +++ b/crates/toolpath-convo/src/lib.rs @@ -233,6 +233,16 @@ pub struct ToolInvocation { /// crate; `None` for unrecognized tools. #[serde(default, skip_serializing_if = "Option::is_none")] pub category: Option, + /// Opaque provider signature tying this call to the reasoning context + /// that produced it (Google/Gemini-style thought signatures; OpenClaw's + /// `toolCall.thoughtSignature`). Required by some providers to replay + /// the conversation; carried verbatim, never interpreted. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thought_signature: Option, + /// Provider scheduling hint for this call (e.g. `"sequential"` / + /// `"parallel"`). Informational; carried verbatim. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub execution_mode: Option, } /// The result of a tool invocation. @@ -272,18 +282,50 @@ pub struct Turn { /// The visible text content (already collapsed from provider-specific formats). pub text: String, + /// Opaque provider signature attached to the turn's visible text (e.g. + /// OpenClaw's `textSignature`). Carried verbatim for replay fidelity. + /// Granularity matches `text`: when a provider message has several text + /// blocks (already collapsed into one string), only the first block's + /// signature is carried. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_signature: Option, + /// Internal reasoning (chain-of-thought, thinking blocks). pub thinking: Option, + /// Opaque provider signature attached to the turn's reasoning (Anthropic + /// thinking-block `signature`, OpenClaw's `thinkingSignature`, OpenAI + /// reasoning item ids). Some provider APIs reject a replayed + /// conversation whose thinking blocks lack their signature, so this is + /// load-bearing for resume/inception, not cosmetic. Granularity matches + /// `thinking` (collapsed): the first block's signature is carried. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_signature: Option, + /// Tool invocations in this turn. pub tool_uses: Vec, /// Model identifier (e.g. "claude-opus-4-6", "gpt-4o"). pub model: Option, + /// Concrete model that actually served this turn when it differs from + /// the requested `model` — alias/auto routing (OpenRouter `auto`, + /// provider-side aliases). `None` when identical or unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub response_model: Option, + /// Why the turn ended (e.g. "end_turn", "tool_use", "max_tokens"). pub stop_reason: Option, + /// Structural conversation boundary this turn represents, when it is a + /// compaction or branch-summary marker rather than an ordinary message. + /// Carries the fields providers persist on such boundaries (Claude's + /// `compact_boundary`, Pi/OpenClaw `compaction` / `branch_summary` + /// entries) so they survive a round-trip instead of being flattened + /// into summary text. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub marker: Option, + /// Token usage for this turn. When this turn belongs to a `group_id` /// group, this is the **whole message's total**, carried on the /// group's final turn only (it always means "the total for a @@ -321,6 +363,52 @@ pub struct Turn { pub file_mutations: Vec, } +/// A structural conversation boundary carried on the turn that represents +/// it. These are cross-harness concepts: Claude marks compactions with +/// `compact_boundary` entries, Pi and OpenClaw persist `compaction` and +/// `branch_summary` entries with the fields below. The summary text itself +/// stays in `Turn.text`; the marker carries the structured fields so a +/// projector can re-emit the native entry losslessly. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ConversationMarker { + /// A context-compaction boundary: history before `first_kept_id` was + /// replaced by the summary in `Turn.text`. + Compaction { + /// Entry/step id of the first entry retained verbatim. + #[serde(default, skip_serializing_if = "Option::is_none")] + first_kept_id: Option, + /// Estimated context tokens before compaction. + #[serde(default, skip_serializing_if = "Option::is_none")] + tokens_before: Option, + /// Files the summarized span had read. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + read_files: Vec, + /// Files the summarized span had modified. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + modified_files: Vec, + /// True when produced by an app hook rather than the built-in + /// summarizer (Pi/OpenClaw `fromHook`). + #[serde(default, skip_serializing_if = "Option::is_none")] + from_hook: Option, + }, + /// A summary of an abandoned branch (Pi/OpenClaw `branch_summary`). + BranchSummary { + /// Entry/step id of the abandoned branch's source leaf. + #[serde(default, skip_serializing_if = "Option::is_none")] + from_id: Option, + /// Files the abandoned branch had read. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + read_files: Vec, + /// Files the abandoned branch had modified. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + modified_files: Vec, + /// True when produced by an app hook. + #[serde(default, skip_serializing_if = "Option::is_none")] + from_hook: Option, + }, +} + /// A complete conversation from any provider. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ConversationView { @@ -575,6 +663,7 @@ mod tests { last_activity: None, turns: vec![ Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t1".into(), parent_id: None, group_id: None, @@ -592,6 +681,7 @@ mod tests { file_mutations: Vec::new(), }, Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t2".into(), parent_id: Some("t1".into()), group_id: None, @@ -600,6 +690,7 @@ mod tests { text: "I'll fix that for you.".into(), thinking: Some("The bug is in the token validation".into()), tool_uses: vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tool-1".into(), name: "Read".into(), input: serde_json::json!({"file": "src/login.rs"}), @@ -624,6 +715,7 @@ mod tests { file_mutations: Vec::new(), }, Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t3".into(), parent_id: Some("t2".into()), group_id: None, @@ -911,6 +1003,7 @@ mod tests { #[test] fn test_tool_category_serde() { let ti = ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".into(), name: "Bash".into(), input: serde_json::json!({"command": "ls"}), @@ -926,6 +1019,7 @@ mod tests { #[test] fn test_tool_category_none_skipped() { let ti = ToolInvocation { + thought_signature: None, execution_mode: None, id: "t1".into(), name: "CustomTool".into(), input: serde_json::json!({}), @@ -964,6 +1058,7 @@ mod tests { #[test] fn test_turn_with_environment_and_delegations() { let turn = Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t1".into(), parent_id: None, group_id: None, diff --git a/crates/toolpath-convo/src/project.rs b/crates/toolpath-convo/src/project.rs index 3a0a0511..5560c761 100644 --- a/crates/toolpath-convo/src/project.rs +++ b/crates/toolpath-convo/src/project.rs @@ -155,6 +155,7 @@ mod tests { fn make_turn(id: &str, role: Role, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -337,6 +338,7 @@ mod tests { last_activity: None, events: vec![], turns: vec![Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t1".into(), parent_id: None, group_id: None, @@ -346,6 +348,7 @@ mod tests { thinking: None, tool_uses: vec![ ToolInvocation { + thought_signature: None, execution_mode: None, id: "u1".into(), name: "Read".into(), input: serde_json::json!({"file": "src/main.rs"}), @@ -356,6 +359,7 @@ mod tests { category: None, }, ToolInvocation { + thought_signature: None, execution_mode: None, id: "u2".into(), name: "Bash".into(), input: serde_json::json!({"command": "cargo test"}), @@ -413,6 +417,7 @@ mod tests { events: vec![], turns: vec![ Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t1".into(), parent_id: None, group_id: None, @@ -436,6 +441,7 @@ mod tests { file_mutations: Vec::new(), }, Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "t2".into(), parent_id: Some("t1".into()), group_id: None, diff --git a/crates/toolpath-cursor/examples/dump_fixture.rs b/crates/toolpath-cursor/examples/dump_fixture.rs index e6fba08e..65e5754c 100644 --- a/crates/toolpath-cursor/examples/dump_fixture.rs +++ b/crates/toolpath-cursor/examples/dump_fixture.rs @@ -250,6 +250,7 @@ fn view_from_jsonl( // it's the lossy view); leave `result: None` and let // the projector emit a `running` status downstream. ToolInvocation { + thought_signature: None, execution_mode: None, id: format!("tool_{tool_counter:04}"), // Pass the agent-side name through; the projector // remaps to native via category. @@ -265,6 +266,7 @@ fn view_from_jsonl( .collect(); turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: turn_id.clone(), parent_id: prev_id.clone(), group_id: None, diff --git a/crates/toolpath-cursor/src/project.rs b/crates/toolpath-cursor/src/project.rs index 34bd0c76..8413e752 100644 --- a/crates/toolpath-cursor/src/project.rs +++ b/crates/toolpath-cursor/src/project.rs @@ -922,6 +922,7 @@ mod tests { fn user_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -946,6 +947,7 @@ mod tests { fn assistant_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -1052,6 +1054,7 @@ mod tests { fn shell_tool_use_round_trips_via_native_id() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "Bash".into(), input: json!({"command": "ls"}), @@ -1079,6 +1082,7 @@ mod tests { fn file_write_tool_registers_blobs_and_emits_result() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc2".into(), name: "edit_file_v2".into(), input: json!({"relativeWorkspacePath": "/proj/x.rs"}), @@ -1124,6 +1128,7 @@ mod tests { fn errored_tool_emits_null_result_and_status_error() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc3".into(), name: "edit_file_v2".into(), input: json!({"relativeWorkspacePath": "/proj/x.rs"}), @@ -1143,6 +1148,7 @@ mod tests { fn no_result_yet_emits_completed_status_not_running() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc4".into(), name: "read_file_v2".into(), input: json!({"target_file": "/proj/x.rs"}), @@ -1227,6 +1233,7 @@ mod tests { use crate::types::{TOOL_TABLE, TOOL_UNSPECIFIED}; for &(id, name) in TOOL_TABLE { let tu = ToolInvocation { + thought_signature: None, execution_mode: None, id: "test".into(), name: name.to_string(), input: json!({}), @@ -1258,6 +1265,7 @@ mod tests { ]; for (cat, want_id, want_name) in cases { let tu = ToolInvocation { + thought_signature: None, execution_mode: None, id: "test".into(), name: "Foreign_Name_Not_In_Cursor".into(), input: json!({}), @@ -1273,6 +1281,7 @@ mod tests { #[test] fn foreign_name_without_category_falls_back_to_unspecified() { let tu = ToolInvocation { + thought_signature: None, execution_mode: None, id: "test".into(), name: "weird_thing_v9".into(), input: json!({}), @@ -1331,6 +1340,7 @@ mod tests { fn edit_tool_with_raw_diff_emits_content_blobs() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "edit_file_v2".into(), input: json!({"relativeWorkspacePath": "/proj/x.rs"}), @@ -1369,6 +1379,7 @@ mod tests { fn edit_tool_with_full_content_prefers_blob_over_reconstruction() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "edit_file_v2".into(), input: json!({"relativeWorkspacePath": "/proj/x.rs"}), diff --git a/crates/toolpath-cursor/src/provider.rs b/crates/toolpath-cursor/src/provider.rs index 02746ab1..df6b1ef1 100644 --- a/crates/toolpath-cursor/src/provider.rs +++ b/crates/toolpath-cursor/src/provider.rs @@ -387,6 +387,7 @@ impl<'a> Builder<'a> { fn user_turn(&self, bubble: &Bubble, parent: Option<&str>) -> Turn { let environment = self.environment(); Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: bubble.bubble_id.clone(), parent_id: parent.map(str::to_string), group_id: None, @@ -478,6 +479,7 @@ impl<'a> Builder<'a> { .or_else(|| self.session.data.default_model().map(str::to_string)); Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: bubble.bubble_id.clone(), parent_id: parent.map(str::to_string), group_id: None, @@ -544,6 +546,7 @@ impl<'a> Builder<'a> { _ => None, }; ToolInvocation { + thought_signature: None, execution_mode: None, id: tf.tool_call_id.clone(), name: tf.name.clone(), input, diff --git a/crates/toolpath-cursor/tests/projection_roundtrip.rs b/crates/toolpath-cursor/tests/projection_roundtrip.rs index 16c88777..2bcd7a75 100644 --- a/crates/toolpath-cursor/tests/projection_roundtrip.rs +++ b/crates/toolpath-cursor/tests/projection_roundtrip.rs @@ -227,6 +227,7 @@ fn projector_accepts_foreign_view_shape() { }), turns: vec![ Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "uA".into(), parent_id: None, group_id: None, @@ -247,6 +248,7 @@ fn projector_accepts_foreign_view_shape() { file_mutations: vec![], }, Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "aA".into(), parent_id: Some("uA".into()), group_id: None, @@ -255,6 +257,7 @@ fn projector_accepts_foreign_view_shape() { text: "done".into(), thinking: Some("planning...".into()), tool_uses: vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "claude_edit".into(), name: "Edit".into(), input: json!({"file_path": "/foreign/main.rs", "old_string": "old", "new_string": "new"}), diff --git a/crates/toolpath-gemini/src/project.rs b/crates/toolpath-gemini/src/project.rs index 39bde557..c8ec9917 100644 --- a/crates/toolpath-gemini/src/project.rs +++ b/crates/toolpath-gemini/src/project.rs @@ -594,6 +594,7 @@ mod tests { fn user_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -614,6 +615,7 @@ mod tests { fn assistant_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -733,6 +735,7 @@ mod tests { fn test_tool_call_with_success_result_wraps_into_function_response() { let mut t = assistant_turn("a1", "Reading."); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "read_file".into(), input: serde_json::json!({"path": "src/main.rs"}), @@ -763,6 +766,7 @@ mod tests { fn test_tool_call_with_error_result_sets_error_status() { let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "run_shell_command".into(), input: serde_json::json!({"command": "nope"}), @@ -845,6 +849,7 @@ mod tests { ..Default::default() }); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "read_file".into(), input: serde_json::json!({"path": "src/a.rs"}), diff --git a/crates/toolpath-gemini/src/provider.rs b/crates/toolpath-gemini/src/provider.rs index f39da516..1a35586b 100644 --- a/crates/toolpath-gemini/src/provider.rs +++ b/crates/toolpath-gemini/src/provider.rs @@ -112,6 +112,7 @@ fn message_to_turn(msg: &GeminiMessage, working_dir: Option<&str>) -> Turn { }); Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: msg.id.clone(), parent_id: None, group_id: None, @@ -302,6 +303,7 @@ fn tool_call_to_invocation(call: &ToolCall) -> ToolInvocation { }) }; ToolInvocation { + thought_signature: None, execution_mode: None, id: call.id.clone(), name: call.name.clone(), input: call.args.clone(), diff --git a/crates/toolpath-opencode/src/project.rs b/crates/toolpath-opencode/src/project.rs index 0f34ae45..0aad6c15 100644 --- a/crates/toolpath-opencode/src/project.rs +++ b/crates/toolpath-opencode/src/project.rs @@ -752,6 +752,7 @@ mod tests { fn user_turn(text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "u1".into(), parent_id: None, group_id: None, @@ -772,6 +773,7 @@ mod tests { fn assistant_turn(text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: "a1".into(), parent_id: None, group_id: None, @@ -844,6 +846,7 @@ mod tests { fn tool_call_lands_as_tool_part() { let mut t = assistant_turn(""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "call_x".into(), name: "Bash".into(), input: json!({"command": "ls"}), @@ -882,6 +885,7 @@ mod tests { fn errored_tool_use_produces_error_state() { let mut t = assistant_turn(""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "c".into(), name: "bash".into(), input: json!({"command": "false"}), @@ -909,6 +913,7 @@ mod tests { fn foreign_tool_name_remaps_via_category() { let mut t = assistant_turn(""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "c".into(), name: "Edit".into(), input: json!({"file_path": "x.rs", "old_string": "a", "new_string": "b"}), diff --git a/crates/toolpath-opencode/src/provider.rs b/crates/toolpath-opencode/src/provider.rs index 32158689..d3d9e449 100644 --- a/crates/toolpath-opencode/src/provider.rs +++ b/crates/toolpath-opencode/src/provider.rs @@ -284,6 +284,7 @@ impl<'a> Builder<'a> { }); self.turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: msg.id.clone(), parent_id: None, group_id: None, @@ -449,6 +450,7 @@ impl<'a> Builder<'a> { let file_mutations = self.compute_turn_mutations(&snapshots, &tool_uses); self.turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: msg.id.clone(), parent_id: if a.parent_id.is_empty() { None @@ -589,6 +591,7 @@ fn to_invocation( } ToolInvocation { + thought_signature: None, execution_mode: None, id: tp.call_id.clone(), name: tp.tool.clone(), input, diff --git a/crates/toolpath-pi/src/project.rs b/crates/toolpath-pi/src/project.rs index a82b15d1..7b4a3252 100644 --- a/crates/toolpath-pi/src/project.rs +++ b/crates/toolpath-pi/src/project.rs @@ -758,6 +758,7 @@ mod tests { fn user_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -778,6 +779,7 @@ mod tests { fn assistant_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -852,6 +854,7 @@ mod tests { fn test_assistant_turn_with_tool_call_and_result() { let mut t = assistant_turn("a1", "I'll read it."); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "read".into(), input: serde_json::json!({"path": "x.rs"}), @@ -911,6 +914,7 @@ mod tests { // routes it through `native_name(Shell, _)`. let mut t = assistant_turn("a1", ""); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "Bash".into(), input: serde_json::json!({"command": "ls"}), diff --git a/crates/toolpath-pi/src/provider.rs b/crates/toolpath-pi/src/provider.rs index 6f091c2d..1edb72c2 100644 --- a/crates/toolpath-pi/src/provider.rs +++ b/crates/toolpath-pi/src/provider.rs @@ -244,6 +244,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { Entry::Compaction { base, summary, .. } => { turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: base.id.clone(), parent_id: base.parent_id.clone(), group_id: None, @@ -264,6 +265,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { Entry::BranchSummary { base, summary, .. } => { turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: base.id.clone(), parent_id: base.parent_id.clone(), group_id: None, @@ -284,6 +286,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { Entry::Custom { base, .. } => { turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: base.id.clone(), parent_id: base.parent_id.clone(), group_id: None, @@ -309,6 +312,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { .. } => { turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: base.id.clone(), parent_id: base.parent_id.clone(), group_id: None, @@ -380,6 +384,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { delegation_locs.insert(id.clone(), (turn_idx, deleg_idx)); } tool_uses.push(ToolInvocation { + thought_signature: None, execution_mode: None, id: id.clone(), name: name.clone(), input: arguments.clone(), @@ -422,6 +427,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { text = format!("$ {}\n{}", command, out_trunc); // Synthetic ToolInvocation representing the bash run itself. tool_uses.push(ToolInvocation { + thought_signature: None, execution_mode: None, id: base.id.clone(), name: "bash".to_string(), input: json!({ "command": command }), @@ -449,6 +455,7 @@ pub fn session_to_view(session: &PiSession) -> ConversationView { } turns.push(Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: base.id.clone(), parent_id: base.parent_id.clone(), group_id: None, diff --git a/site/_data/crates.json b/site/_data/crates.json index 507d37ec..2033cfbd 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -9,7 +9,7 @@ }, { "name": "toolpath-convo", - "version": "0.12.0", + "version": "0.13.0", "description": "Provider-agnostic conversation types, traits, and Toolpath-Path derivation", "docs": "https://docs.rs/toolpath-convo", "crate": "https://crates.io/crates/toolpath-convo", From fe335a2a7f0e10459b99a1e877e2f951a00ce5c3 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Wed, 8 Jul 2026 13:15:32 -0400 Subject: [PATCH 22/22] feat(openclaw): byte-faithful round-trips + cumulative-usage aggregate fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forward path populates the new typed IR fields (thinkingSignature/textSignature/thoughtSignature/executionMode, responseId -> group_id, responseModel, compaction/branch markers); projector re-emits them all, plus the observed totalTokens convention (input+output+cacheRead+cacheWrite). Fixes a real double-count found in captured data: the final assembled reply of a multi-call run (post sessions_yield, no responseId) carries run-cumulative usage — now detected (exact field-wise-sum match, native files only) and never stamped onto a step; real-fixture test pins session output at the single-counted 1468. New fidelity_roundtrip integration test covers the full session -> Path JSON -> session chain. Docs record the aggregate row, totalTokens formula, and responseId observations. --- CLAUDE.md | 6 +- crates/toolpath-openclaw/src/project.rs | 109 ++++++++++- crates/toolpath-openclaw/src/provider.rs | 185 ++++++++++++++++-- .../tests/fidelity_roundtrip.rs | 174 ++++++++++++++++ .../tests/fixtures/fidelity_session.jsonl | 7 + .../toolpath-openclaw/tests/real_fixture.rs | 22 ++- .../formats/openclaw/format-changelog.md | 5 + docs/agents/formats/openclaw/known-issues.md | 17 +- docs/agents/formats/openclaw/usage.md | 22 ++- 9 files changed, 505 insertions(+), 42 deletions(-) create mode 100644 crates/toolpath-openclaw/tests/fidelity_roundtrip.rs create mode 100644 crates/toolpath-openclaw/tests/fixtures/fidelity_session.jsonl diff --git a/CLAUDE.md b/CLAUDE.md index d3f5b5bf..9c00aa94 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -179,7 +179,7 @@ the server publishes that operation. Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has integration tests in `tests/`. Per-crate counts: - `toolpath`: 69 unit + 11 doc tests (serde roundtrip, builders, query) -- `toolpath-convo`: 118 unit + 4 doc tests (types, enrichment, display, ConversationView -> Path derivation, message-group usage accounting, breakdowns) +- `toolpath-convo`: 120 unit + 4 doc tests (types, enrichment, display, ConversationView -> Path derivation, message-group usage accounting, breakdowns, typed fidelity fields: signatures/response_model/markers) - `toolpath-git`: 33 unit + 3 doc tests (derive, branch detection, diffstat) - `toolpath-github`: 32 unit + 3 doc tests (mapping, DAG construction, fixtures) - `toolpath-claude`: 229 unit + 18 integration + 6 doc tests (path resolution, conversation reading, query, chaining, watcher, derive, metadata first-user-message, group_id grouping + once-per-message usage totals) @@ -188,7 +188,7 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has in - `toolpath-opencode`: 52 unit + 19 integration + 1 doc test (SQLite reader, JSON payload serde, provider assembly, snapshot-based derive, tool-input fallback for gitignored paths, reasoning breakdown) - `toolpath-cursor`: 78 unit + 8 integration round-trip + 1 real-DB sanity + 1 doc test (state.vscdb SQLite reader, bubble store + composer header parsing, content-addressed blob lookup, projector with full TOOL_TABLE coverage, JSONL transcript ingest in `examples/dump_fixture.rs`) - `toolpath-pi`: 133 unit + 26 integration + 5 doc tests (types, paths, error, reader, io, provider) -- `toolpath-openclaw`: 52 unit + 5 integration tests (v3 schema serde, path/session-key resolution, reader + tree/visible-leaf assembly, listing, session_to_view mapping + channel-aware actor, derive, projector inception + raw-JSON sessions.json merge, full projection round-trip, and a real-session suite over the captured fixtures in `test-fixtures/openclaw/` — telegram channel actor, spawn-child kind, real tool vocabulary) +- `toolpath-openclaw`: 52 unit + 7 integration tests (v3 schema serde, path/session-key resolution, reader + tree/visible-leaf assembly, listing, session_to_view mapping + channel-aware actor, derive, projector inception + raw-JSON sessions.json merge, full projection round-trip, and a real-session suite over the captured fixtures in `test-fixtures/openclaw/` — telegram channel actor, spawn-child kind, real tool vocabulary, fidelity round-trip of signatures/responseId/responseModel/markers, cumulative-usage aggregate detection) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) - `path-cli`: 300 unit + 65 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, openclaw import + share aggregation + inception+ resume, `path resume` orchestration with injectable `ExecStrategy`). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. - `toolpath-cli`: 0 tests (it's a one-line `path_cli::run()` shim crate that exists only so `cargo install toolpath-cli` keeps installing the `path` binary) @@ -247,7 +247,7 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - Provider-specific extras convention: `Turn.extra` and `WatcherEvent::Progress.data` use provider-namespaced keys (e.g. `extra["claude"]`, `extra["gemini"]`). `toolpath-claude` populates `Turn.extra["claude"]` from `ConversationEntry.extra`; `toolpath-gemini` populates `Turn.extra["gemini"]` with the full `tokens` struct, per-thought metadata, and tool-call status. This lets trait-only consumers access provider metadata without importing provider types. - Shared derivation: `toolpath-convo` provides a provider-agnostic `ConversationView → Path` mapping via `toolpath_convo::derive_path`. New conversation providers should build on it rather than re-implementing the mapping. - Path kinds: `toolpath::v1::PathMeta.kind` is an optional URI naming a hosted kind spec; URIs are immutable and semver-versioned. The only one defined so far is `https://toolpath.net/kinds/agent-coding-session/v1.1.0` (constant `toolpath::v1::PATH_KIND_AGENT_CODING_SESSION`; `…_V1_0_0` names the superseded URI); every conversation → `Path` derivation sets it via the shared `toolpath_convo::derive_path` or each provider crate's own. Carried through the JSONL form via `PathOpen.meta` and `PathMeta` patch lines. Spec sources live in `site/kinds///{index.md,schema.json}` (schema.json is a symlink into `crates/path-cli/kinds/`, which `path p validate` bundles — both versions) and publish under `https://toolpath.net/kinds/`; the registry index is `site/kinds/index.md`. RFC: "Document Kind". JSON Schema: `$defs/pathMeta`. -- Token accounting (kind v1.1.0): two keys on `conversation.append`/`Turn`, both optional. `token_usage` = "the total for a message" (on the group's final step; `Σ` over a path = session total). `attributed_token_usage` = "this step's own attributed spend", populated only where the source genuinely reports per-step spend (its own key, so the sum is unaffected; remainder = group total − Σ attributed, computed not stored). One provider message can span several steps (Claude writes one JSONL line per content block); `Turn.group_id` groups them. `toolpath-claude` fills `group_id` from `message.id` and takes the **field-wise-max** group total (line order not trusted). Claude's per-line `usage` is a cumulative *streaming snapshot* (Anthropic streaming API: `message_start` seeds output near 0, `message_delta` is cumulative), NOT a per-block cost — so Claude emits no `attributed_token_usage`; the projector re-expands the total onto every line. `toolpath-codex` differences the cumulative `total_token_usage` (dedup-safe: never sum `last_token_usage` — Codex re-emits it stale; openai/codex #14489), attributes each per-call delta to the step it follows, and derives the round total from those attributions. pi/opencode decode all-zero wire counters as `None`. Never stamp a cumulative counter, a repeated message total, or zero-filled placeholders onto a step; never derive attribution from Claude's streaming snapshots. +- Token accounting (kind v1.1.0): two keys on `conversation.append`/`Turn`, both optional. `token_usage` = "the total for a message" (on the group's final step; `Σ` over a path = session total). `attributed_token_usage` = "this step's own attributed spend", populated only where the source genuinely reports per-step spend (its own key, so the sum is unaffected; remainder = group total − Σ attributed, computed not stored). One provider message can span several steps (Claude writes one JSONL line per content block); `Turn.group_id` groups them. `toolpath-claude` fills `group_id` from `message.id` and takes the **field-wise-max** group total (line order not trusted). Claude's per-line `usage` is a cumulative *streaming snapshot* (Anthropic streaming API: `message_start` seeds output near 0, `message_delta` is cumulative), NOT a per-block cost — so Claude emits no `attributed_token_usage`; the projector re-expands the total onto every line. `toolpath-codex` differences the cumulative `total_token_usage` (dedup-safe: never sum `last_token_usage` — Codex re-emits it stale; openai/codex #14489), attributes each per-call delta to the step it follows, and derives the round total from those attributions. pi/opencode decode all-zero wire counters as `None`. **OpenClaw** stamps per-call usage keyed by `responseId` (→ `Turn.group_id`); the final assembled reply of a multi-call run has no `responseId` and carries run-cumulative usage — `toolpath-openclaw` detects it (exact field-wise-sum match, native files only) and drops it; re-emitted `totalTokens` = `input+output+cacheRead+cacheWrite` (observed convention). Never stamp a cumulative counter, a repeated message total, or zero-filled placeholders onto a step; never derive attribution from Claude's streaming snapshots. - Token usage `breakdowns` (kind v1.1.0, additive): an optional third key on `TokenUsage` — a decomposition of a top-level class into named sub-classes, keyed by class (e.g. `"output"`), inner map sub-class → tokens (e.g. `breakdowns["output"]["reasoning"] = 243`). INFORMATIONAL ONLY: **never summed into any total** (the parent class already counts those tokens, so the session-total guarantee is untouched); invariant `Σ(inner) ≤ parent`; omitted when empty; rides both `token_usage` and `attributed_token_usage`. Per-provider reality: **Gemini** reports `thoughts` (reasoning) as an additive sibling that the derivation used to **drop** (under-counting output) — it's now folded into `output_tokens` *and* recorded as `breakdowns["output"]["reasoning"]`, with the projector un-folding it on the reverse path for a lossless round-trip (`Some(0)` preserved as a real Gemini-3 zero-reasoning signal). **OpenCode** folds `reasoning` into output and records the same breakdown. **Codex** differences `reasoning_output_tokens` (⊆ output, cumulative) into `breakdowns["output"]["reasoning"]` on both per-step `attributed_token_usage` and per-round `token_usage`. **Claude** records no breakdown (its JSONL `usage` doesn't itemize thinking tokens). - Pi provider: `toolpath-pi` reads Pi session JSONL from `~/.pi/agent/sessions/`. Sessions use a tree (id/parentId) in a single file, and may link to a parent file via `parentSession` in the header. The tree is preserved as a DAG in the derived `Path`. - OpenClaw provider: `toolpath-openclaw` reads OpenClaw agent-session JSONL from `~/.openclaw/agents//sessions/.jsonl` (override the state dir with `OPENCLAW_STATE_DIR`/`OPENCLAW_HOME`; legacy `~/.clawdbot`). Format is **version 3**: a session header line + an `id`/`parentId` entry tree whose visible head is moved by `leaf` rows (same family as Pi). Validated against real sessions captured from the official Docker image (`scripts/openclaw-docker.sh`; fixtures at `test-fixtures/openclaw/`, wired into the cross-harness matrix). Observed native tools: `exec`/`read`/`write`/`edit`/`sessions_spawn` (delegation); sub-agent runs get `agent::subagent:` keys → sessionKind `spawn-child`, no channel actor. OpenClaw is **multi-channel** — the human isn't a transcript field; identity lives in the `sessions.json` routing key `agent::::`, which the provider recovers into a **channel-aware** `human:/` actor (via the additive `DeriveConfig.user_actor`, kind v1.x) and stashes (channel/peer/sessionKey/sessionKind) on `path.meta.extra["openclaw"]`. **No file diffs** are stored, so changes are structural/tool-input-only (no `raw` perspective). The projector (`project.rs`) **incepts** a `Path` back onto disk — writing the transcript plus a `sessions.json` routing entry so a running OpenClaw instance can pick it up (`path p export openclaw --project`). CLI is agent-keyed (`--agent`, default `main`), not project-keyed. See `docs/agents/formats/openclaw/` for the full format reference. diff --git a/crates/toolpath-openclaw/src/project.rs b/crates/toolpath-openclaw/src/project.rs index f47bd5a1..9f864fa5 100644 --- a/crates/toolpath-openclaw/src/project.rs +++ b/crates/toolpath-openclaw/src/project.rs @@ -3,11 +3,14 @@ //! session under `agents//sessions/` plus a `sessions.json` routing //! entry so a running OpenClaw instance can pick it up. //! -//! This is the inverse of [`crate::provider::session_to_view`]. Provider- -//! specific extras are not carried on the IR, so the projector synthesizes -//! OpenClaw fields from typed IR fields and sensible defaults (api/provider -//! `anthropic`, stop reason `stop`). Foreign tool names are remapped through -//! [`crate::provider::native_name`]. +//! This is the inverse of [`crate::provider::session_to_view`]. Replay- +//! fidelity data flows through TYPED IR fields — `Turn.thinking_signature` / +//! `text_signature`, `Turn.group_id` (`responseId`), `Turn.response_model`, +//! `Turn.marker` (compaction / branch-summary boundaries), and +//! `ToolInvocation.thought_signature` / `execution_mode` — no provider- +//! namespaced extras exist on the IR. Remaining gaps are synthesized from +//! sensible defaults (api/provider `anthropic`, stop reason `stop`). +//! Foreign tool names are remapped through [`crate::provider::native_name`]. use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -288,27 +291,39 @@ fn emit_user(turn: &Turn, entries: &mut Vec) { } fn emit_assistant(cfg: &OpenClawProjector, turn: &Turn, entries: &mut Vec) { + let sig_extra = |key: &str, sig: &Option| -> HashMap { + let mut m = HashMap::new(); + if let Some(s) = sig { + m.insert(key.to_string(), Value::String(s.clone())); + } + m + }; + let mut blocks: Vec = Vec::new(); if let Some(t) = &turn.thinking && !t.is_empty() { blocks.push(ContentBlock::Thinking { thinking: t.clone(), - extra: HashMap::new(), + extra: sig_extra("thinkingSignature", &turn.thinking_signature), }); } if !turn.text.is_empty() { blocks.push(ContentBlock::Text { text: turn.text.clone(), - extra: HashMap::new(), + extra: sig_extra("textSignature", &turn.text_signature), }); } for tu in &turn.tool_uses { + let mut extra = sig_extra("thoughtSignature", &tu.thought_signature); + if let Some(mode) = &tu.execution_mode { + extra.insert("executionMode".to_string(), Value::String(mode.clone())); + } blocks.push(ContentBlock::ToolCall { id: tu.id.clone(), name: tool_native_name(tu), arguments: tu.input.clone(), - extra: HashMap::new(), + extra, }); } @@ -321,6 +336,16 @@ fn emit_assistant(cfg: &OpenClawProjector, turn: &Turn, entries: &mut Vec .clone() .unwrap_or_else(|| "anthropic".to_string()); + // `group_id` is the per-response accounting id (OpenClaw's + // `responseId`); `response_model` the concrete served model. + let mut msg_extra: HashMap = HashMap::new(); + if let Some(rid) = &turn.group_id { + msg_extra.insert("responseId".to_string(), Value::String(rid.clone())); + } + if let Some(rm) = &turn.response_model { + msg_extra.insert("responseModel".to_string(), Value::String(rm.clone())); + } + entries.push(Entry::Message { base: base_for(turn), message: AgentMessage::Assistant { @@ -332,7 +357,7 @@ fn emit_assistant(cfg: &OpenClawProjector, turn: &Turn, entries: &mut Vec stop_reason: parse_stop_reason(turn.stop_reason.as_deref()), error_message: None, timestamp: ts_millis(&turn.timestamp), - extra: HashMap::new(), + extra: msg_extra, }, extra: HashMap::new(), }); @@ -370,6 +395,56 @@ fn emit_assistant(cfg: &OpenClawProjector, turn: &Turn, entries: &mut Vec } fn emit_system(turn: &Turn, entries: &mut Vec) { + // Typed marker first (lossless); fall back to text-prefix parsing for + // views from sources that only carry the flattened summary text. + match &turn.marker { + Some(toolpath_convo::ConversationMarker::Compaction { + first_kept_id, + tokens_before, + read_files, + modified_files, + from_hook, + }) => { + let summary = turn + .text + .strip_prefix("Compacted (summary): ") + .unwrap_or(&turn.text) + .to_string(); + entries.push(Entry::Compaction { + base: base_for(turn), + summary, + first_kept_entry_id: first_kept_id.clone().unwrap_or_default(), + tokens_before: tokens_before.unwrap_or(0), + details: detail_files_value(read_files, modified_files), + from_hook: *from_hook, + extra: HashMap::new(), + }); + return; + } + Some(toolpath_convo::ConversationMarker::BranchSummary { + from_id, + read_files, + modified_files, + from_hook, + }) => { + let summary = turn + .text + .strip_prefix("Branch summary: ") + .unwrap_or(&turn.text) + .to_string(); + entries.push(Entry::BranchSummary { + base: base_for(turn), + from_id: from_id.clone().unwrap_or_else(|| "root".to_string()), + summary, + details: detail_files_value(read_files, modified_files), + from_hook: *from_hook, + extra: HashMap::new(), + }); + return; + } + None => {} + } + if let Some(rest) = turn.text.strip_prefix("Compacted (summary): ") { entries.push(Entry::Compaction { base: base_for(turn), @@ -394,6 +469,16 @@ fn emit_system(turn: &Turn, entries: &mut Vec) { } } +/// Re-encode summary `details` (`{ readFiles, modifiedFiles }`), omitting +/// the object entirely when both lists are empty (matching native output, +/// which omits `details` rather than writing empty arrays). +fn detail_files_value(read_files: &[String], modified_files: &[String]) -> Option { + if read_files.is_empty() && modified_files.is_empty() { + return None; + } + Some(json!({ "readFiles": read_files, "modifiedFiles": modified_files })) +} + fn emit_bash(turn: &Turn, entries: &mut Vec) { let (command, output) = match turn.text.strip_prefix("$ ") { Some(rest) => match rest.split_once('\n') { @@ -461,7 +546,9 @@ fn build_usage(turn: &Turn) -> Usage { output, cache_read, cache_write, - total_tokens: input + output, + // Observed convention (11/11 per-call rows in real v2026.6.11 + // sessions): totalTokens = input + output + cacheRead + cacheWrite. + total_tokens: input + output + cache_read + cache_write, cost: CostBreakdown::default(), } } @@ -503,6 +590,7 @@ mod tests { fn user_turn(id: &str, text: &str) -> Turn { Turn { + text_signature: None, thinking_signature: None, response_model: None, marker: None, id: id.into(), parent_id: None, group_id: None, @@ -559,6 +647,7 @@ mod tests { ..Default::default() }); t.tool_uses = vec![ToolInvocation { + thought_signature: None, execution_mode: None, id: "tc1".into(), name: "Read".into(), input: json!({"path": "x"}), diff --git a/crates/toolpath-openclaw/src/provider.rs b/crates/toolpath-openclaw/src/provider.rs index 4b39bb7f..bdb97300 100644 --- a/crates/toolpath-openclaw/src/provider.rs +++ b/crates/toolpath-openclaw/src/provider.rs @@ -11,9 +11,9 @@ use serde_json::{Value, json}; use std::collections::{HashMap, HashSet}; use toolpath_convo::{ - ConversationMeta, ConversationProvider, ConversationView, ConvoError, DelegatedWork, - EnvironmentSnapshot, FileMutation, Role, SessionBase, TokenUsage, ToolCategory, ToolInvocation, - ToolResult, Turn, + ConversationMarker, ConversationMeta, ConversationProvider, ConversationView, ConvoError, + DelegatedWork, EnvironmentSnapshot, FileMutation, Role, SessionBase, TokenUsage, ToolCategory, + ToolInvocation, ToolResult, Turn, }; use crate::OpenClawConvo; @@ -150,6 +150,44 @@ fn usage_to_token_usage(usage: &Usage) -> Option { }) } +/// Pull a string field out of a flattened forward-compat `extra` map. +fn extra_str(extra: &std::collections::HashMap, key: &str) -> Option { + extra.get(key).and_then(|v| v.as_str()).map(str::to_string) +} + +/// The signature riding on the FIRST content block of `block_type` — the +/// same collapsed granularity the IR uses for `text` / `thinking` bodies. +fn first_block_signature( + content: &[ContentBlock], + block_type: &str, + sig_key: &str, +) -> Option { + content.iter().find_map(|b| { + let extra = match (block_type, b) { + ("text", ContentBlock::Text { extra, .. }) => extra, + ("thinking", ContentBlock::Thinking { extra, .. }) => extra, + _ => return None, + }; + extra.get(sig_key).and_then(|v| v.as_str()).map(str::to_string) + }) +} + +/// Decode a summary entry's `details` (`{ readFiles, modifiedFiles }`). +fn summary_detail_files(details: Option<&Value>) -> (Vec, Vec) { + let list = |key: &str| -> Vec { + details + .and_then(|d| d.get(key)) + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() + }; + (list("readFiles"), list("modifiedFiles")) +} + fn environment_for(session: &OpenClawSession) -> EnvironmentSnapshot { EnvironmentSnapshot { working_dir: Some(session.header.cwd.clone()), @@ -227,6 +265,9 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { let mut tool_call_locs: HashMap = HashMap::new(); let mut delegation_locs: HashMap = HashMap::new(); let mut tool_results: Vec<(String, String, bool)> = Vec::new(); + // (turn index, wire usage, has responseId) per usage-bearing assistant + // message, for run-cumulative aggregate detection below. + let mut assistant_usage_rows: Vec<(usize, Usage, bool)> = Vec::new(); let system_turn = |id: &str, parent: &Option, ts: &str, text: String| Turn { id: id.to_string(), @@ -235,10 +276,14 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { role: Role::System, timestamp: ts.to_string(), text, + text_signature: None, thinking: None, + thinking_signature: None, tool_uses: vec![], model: None, + response_model: None, stop_reason: None, + marker: None, token_usage: None, attributed_token_usage: None, environment: Some(env.clone()), @@ -258,21 +303,56 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { // Rendering/metadata/pointer entries: no IR turn. } - Entry::Compaction { base, summary, .. } => { - turns.push(system_turn( - &base.id, - &base.parent_id, - &base.timestamp, - format!("Compacted (summary): {summary}"), - )); + Entry::Compaction { + base, + summary, + first_kept_entry_id, + tokens_before, + details, + from_hook, + .. + } => { + let (read_files, modified_files) = summary_detail_files(details.as_ref()); + turns.push(Turn { + marker: Some(ConversationMarker::Compaction { + first_kept_id: (!first_kept_entry_id.is_empty()) + .then(|| first_kept_entry_id.clone()), + tokens_before: Some(*tokens_before), + read_files, + modified_files, + from_hook: *from_hook, + }), + ..system_turn( + &base.id, + &base.parent_id, + &base.timestamp, + format!("Compacted (summary): {summary}"), + ) + }); } - Entry::BranchSummary { base, summary, .. } => { - turns.push(system_turn( - &base.id, - &base.parent_id, - &base.timestamp, - format!("Branch summary: {summary}"), - )); + Entry::BranchSummary { + base, + summary, + from_id, + details, + from_hook, + .. + } => { + let (read_files, modified_files) = summary_detail_files(details.as_ref()); + turns.push(Turn { + marker: Some(ConversationMarker::BranchSummary { + from_id: (!from_id.is_empty()).then(|| from_id.clone()), + read_files, + modified_files, + from_hook: *from_hook, + }), + ..system_turn( + &base.id, + &base.parent_id, + &base.timestamp, + format!("Branch summary: {summary}"), + ) + }); } Entry::CustomMessage { base, @@ -289,10 +369,14 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { Entry::Message { base, message, .. } => { let text; + let mut text_signature = None; let mut thinking = None; + let mut thinking_signature = None; let mut tool_uses: Vec = Vec::new(); let mut file_mutations: Vec = Vec::new(); let mut model: Option = None; + let mut response_model: Option = None; + let mut group_id: Option = None; let mut stop_reason_s: Option = None; let mut token_usage: Option = None; let mut delegations: Vec = Vec::new(); @@ -308,6 +392,7 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { model: m, usage, stop_reason, + extra, .. } => { role = Role::Assistant; @@ -316,6 +401,22 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { model = Some(m.clone()); stop_reason_s = Some(stop_reason_to_string(stop_reason)); token_usage = usage_to_token_usage(usage); + // `responseId` is the provider's per-response message id — + // the same identifier class Claude carries in `group_id` + // (Anthropic message.id). One OpenClaw assistant message = + // one API response = one accounting unit. + group_id = extra_str(extra, "responseId"); + response_model = extra_str(extra, "responseModel"); + text_signature = first_block_signature(content, "text", "textSignature"); + thinking_signature = + first_block_signature(content, "thinking", "thinkingSignature"); + if token_usage.is_some() { + assistant_usage_rows.push(( + turns.len(), + usage.clone(), + group_id.is_some(), + )); + } let turn_idx = turns.len(); for block in content { @@ -323,7 +424,7 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { id, name, arguments, - .. + extra: block_extra, } = block else { continue; @@ -363,6 +464,14 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { input: arguments.clone(), result: None, category, + thought_signature: block_extra + .get("thoughtSignature") + .and_then(|v| v.as_str()) + .map(str::to_string), + execution_mode: block_extra + .get("executionMode") + .and_then(|v| v.as_str()) + .map(str::to_string), }); } } @@ -391,6 +500,8 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { is_error: !matches!(exit_code, Some(0)), }), category: Some(ToolCategory::Shell), + thought_signature: None, + execution_mode: None, }); } } @@ -398,14 +509,18 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { turns.push(Turn { id: base.id.clone(), parent_id: base.parent_id.clone(), - group_id: None, + group_id, role, timestamp: base.timestamp.clone(), text, + text_signature, thinking, + thinking_signature, tool_uses, model, + response_model, stop_reason: stop_reason_s, + marker: None, token_usage, attributed_token_usage: None, environment: Some(env.clone()), @@ -416,6 +531,38 @@ pub fn session_to_view(session: &OpenClawSession) -> ConversationView { } } + // Run-cumulative aggregate detection. OpenClaw's final assembled reply + // of a multi-call run (observed after a `sessions_yield` re-context) is + // written WITHOUT a `responseId` and with `usage` equal to the + // field-wise SUM of every prior per-call usage — the run accumulator's + // totals, not this step's spend. Stamping it would double-count the + // session (the repo's token-accounting law: never stamp a cumulative + // counter or repeated total onto a step). Detection is deliberately + // conservative: only in sessions that stamp `responseId` on real API + // responses (native modern files), only for rows lacking one, and only + // on an exact four-field sum match against the accepted per-call rows. + let has_response_ids = assistant_usage_rows.iter().any(|(_, _, has_rid)| *has_rid); + if has_response_ids { + let mut sum = (0u64, 0u64, 0u64, 0u64); + let mut priors = 0usize; + for (turn_idx, usage, has_rid) in &assistant_usage_rows { + let row = (usage.input, usage.output, usage.cache_read, usage.cache_write); + if !*has_rid && priors > 0 && row == sum { + if let Some(t) = turns.get_mut(*turn_idx) { + t.token_usage = None; + } + continue; + } + sum = ( + sum.0 + row.0, + sum.1 + row.1, + sum.2 + row.2, + sum.3 + row.3, + ); + priors += 1; + } + } + // Pass 2: fold tool results onto their invocations. for (tool_call_id, content, is_error) in &tool_results { if let Some((turn_idx, tool_idx)) = tool_call_locs.get(tool_call_id) diff --git a/crates/toolpath-openclaw/tests/fidelity_roundtrip.rs b/crates/toolpath-openclaw/tests/fidelity_roundtrip.rs new file mode 100644 index 00000000..ac392742 --- /dev/null +++ b/crates/toolpath-openclaw/tests/fidelity_roundtrip.rs @@ -0,0 +1,174 @@ +//! Round-trip fidelity contract for the fields that used to be dropped: +//! thinking/text/thought signatures, `executionMode`, `responseId`, +//! `responseModel`, compaction and branch-summary structured fields, and +//! the `totalTokens` convention. All flow through TYPED IR fields +//! (`Turn.thinking_signature`, `Turn.marker`, `ToolInvocation.thought_signature`, +//! `Turn.group_id`, …) and the full Path JSON hop — no provider-namespaced +//! extras stash exists on the IR. + +use std::path::Path; + +use toolpath_convo::{ConversationMarker, ConversationProjector, extract_conversation}; +use toolpath_openclaw::project::OpenClawProjector; +use toolpath_openclaw::reader::read_session_from_file; +use toolpath_openclaw::types::{AgentMessage, ContentBlock, Entry}; +use toolpath_openclaw::{DeriveConfig, derive_path, session_to_view}; + +fn fixture() -> toolpath_openclaw::OpenClawSession { + read_session_from_file(Path::new("tests/fixtures/fidelity_session.jsonl")).unwrap() +} + +/// The full chain: session → view → Path → serialize+reparse JSON → view → +/// project → session. Every fidelity field must survive to the final wire. +#[test] +fn fidelity_fields_survive_the_full_chain() { + let src = fixture(); + + // Forward: typed IR fields are populated. + let view1 = session_to_view(&src); + let asst = view1 + .turns + .iter() + .find(|t| t.id == "f2") + .expect("assistant turn"); + assert_eq!(asst.thinking_signature.as_deref(), Some("sig-think-AAAA")); + assert_eq!(asst.text_signature.as_deref(), Some("sig-text-BBBB")); + assert_eq!(asst.group_id.as_deref(), Some("msg_01FIDELITY")); + assert_eq!( + asst.response_model.as_deref(), + Some("claude-opus-4-8-20260115") + ); + let call = &asst.tool_uses[0]; + assert_eq!(call.thought_signature.as_deref(), Some("sig-thought-CCCC")); + assert_eq!(call.execution_mode.as_deref(), Some("sequential")); + + let compaction = view1.turns.iter().find(|t| t.id == "f4").unwrap(); + assert_eq!( + compaction.marker, + Some(ConversationMarker::Compaction { + first_kept_id: Some("f2".into()), + tokens_before: Some(54321), + read_files: vec!["src/a.rs".into()], + modified_files: vec!["src/a.rs".into()], + from_hook: Some(true), + }) + ); + let branch = view1.turns.iter().find(|t| t.id == "f5").unwrap(); + assert_eq!( + branch.marker, + Some(ConversationMarker::BranchSummary { + from_id: Some("f3".into()), + read_files: vec!["src/b.rs".into()], + modified_files: vec![], + from_hook: Some(false), + }) + ); + + // Through the Path JSON hop. + let path_doc = derive_path(&src, &DeriveConfig::default()); + let json = serde_json::to_string(&path_doc).unwrap(); + let path_doc2: toolpath::v1::Path = serde_json::from_str(&json).unwrap(); + let view2 = extract_conversation(&path_doc2); + + let asst2 = view2.turns.iter().find(|t| t.id == "f2").unwrap(); + assert_eq!(asst2.thinking_signature, asst.thinking_signature); + assert_eq!(asst2.text_signature, asst.text_signature); + assert_eq!(asst2.group_id, asst.group_id); + assert_eq!(asst2.response_model, asst.response_model); + assert_eq!( + asst2.tool_uses[0].thought_signature.as_deref(), + Some("sig-thought-CCCC") + ); + assert_eq!(asst2.tool_uses[0].execution_mode.as_deref(), Some("sequential")); + assert_eq!( + view2.turns.iter().find(|t| t.id == "f4").unwrap().marker, + compaction.marker + ); + assert_eq!( + view2.turns.iter().find(|t| t.id == "f5").unwrap().marker, + branch.marker + ); + + // Reverse: the projected wire carries the native fields again. + let projected = OpenClawProjector::default().project(&view2).unwrap(); + let mut saw_assistant = false; + let mut saw_compaction = false; + let mut saw_branch = false; + for entry in &projected.entries { + match entry { + Entry::Message { + message: + AgentMessage::Assistant { + content, + usage, + extra, + .. + }, + .. + } => { + saw_assistant = true; + assert_eq!( + extra.get("responseId").and_then(|v| v.as_str()), + Some("msg_01FIDELITY") + ); + assert_eq!( + extra.get("responseModel").and_then(|v| v.as_str()), + Some("claude-opus-4-8-20260115") + ); + for b in content { + match b { + ContentBlock::Thinking { extra, .. } => assert_eq!( + extra.get("thinkingSignature").and_then(|v| v.as_str()), + Some("sig-think-AAAA") + ), + ContentBlock::Text { extra, .. } => assert_eq!( + extra.get("textSignature").and_then(|v| v.as_str()), + Some("sig-text-BBBB") + ), + ContentBlock::ToolCall { extra, .. } => { + assert_eq!( + extra.get("thoughtSignature").and_then(|v| v.as_str()), + Some("sig-thought-CCCC") + ); + assert_eq!( + extra.get("executionMode").and_then(|v| v.as_str()), + Some("sequential") + ); + } + _ => {} + } + } + // totalTokens convention: input+output+cacheRead+cacheWrite. + assert_eq!(usage.total_tokens, 10 + 20 + 30 + 40); + } + Entry::Compaction { + first_kept_entry_id, + tokens_before, + details, + from_hook, + .. + } => { + saw_compaction = true; + assert_eq!(first_kept_entry_id, "f2"); + assert_eq!(*tokens_before, 54321); + assert_eq!(*from_hook, Some(true)); + let d = details.as_ref().expect("details re-emitted"); + assert_eq!(d["readFiles"][0], "src/a.rs"); + assert_eq!(d["modifiedFiles"][0], "src/a.rs"); + } + Entry::BranchSummary { + from_id, + details, + from_hook, + .. + } => { + saw_branch = true; + assert_eq!(from_id, "f3"); + assert_eq!(*from_hook, Some(false)); + assert_eq!(details.as_ref().unwrap()["readFiles"][0], "src/b.rs"); + } + _ => {} + } + } + assert!(saw_assistant && saw_compaction && saw_branch); +} diff --git a/crates/toolpath-openclaw/tests/fixtures/fidelity_session.jsonl b/crates/toolpath-openclaw/tests/fixtures/fidelity_session.jsonl new file mode 100644 index 00000000..d02cb725 --- /dev/null +++ b/crates/toolpath-openclaw/tests/fixtures/fidelity_session.jsonl @@ -0,0 +1,7 @@ +{"type":"session","version":3,"id":"fidelity-1","timestamp":"2026-07-08T09:00:00.000Z","cwd":"/home/u/proj"} +{"type":"message","id":"f1","parentId":null,"timestamp":"2026-07-08T09:00:01.000Z","message":{"role":"user","content":"Think hard, then patch the file.","timestamp":1751964001000}} +{"type":"message","id":"f2","parentId":"f1","timestamp":"2026-07-08T09:00:05.000Z","message":{"role":"assistant","content":[{"type":"thinking","thinking":"Let me reason about this.","thinkingSignature":"sig-think-AAAA"},{"type":"text","text":"Patching now.","textSignature":"sig-text-BBBB"},{"type":"toolCall","id":"fc1","name":"edit","arguments":{"path":"src/a.rs","old":"x","new":"y"},"thoughtSignature":"sig-thought-CCCC","executionMode":"sequential"}],"api":"anthropic-messages","provider":"anthropic","model":"claude-opus-4-8","responseModel":"claude-opus-4-8-20260115","responseId":"msg_01FIDELITY","usage":{"input":10,"output":20,"cacheRead":30,"cacheWrite":40,"totalTokens":100},"stopReason":"toolUse","timestamp":1751964005000}} +{"type":"message","id":"f3","parentId":"f2","timestamp":"2026-07-08T09:00:06.000Z","message":{"role":"toolResult","toolCallId":"fc1","toolName":"edit","content":[{"type":"text","text":"edited"}],"isError":false,"timestamp":1751964006000}} +{"type":"compaction","id":"f4","parentId":"f3","timestamp":"2026-07-08T09:10:00.000Z","summary":"Earlier work summarized.","firstKeptEntryId":"f2","tokensBefore":54321,"details":{"readFiles":["src/a.rs"],"modifiedFiles":["src/a.rs"]},"fromHook":true} +{"type":"branch_summary","id":"f5","parentId":"f4","timestamp":"2026-07-08T09:11:00.000Z","fromId":"f3","summary":"Abandoned experiment.","details":{"readFiles":["src/b.rs"],"modifiedFiles":[]},"fromHook":false} +{"type":"leaf","id":"f6","parentId":"f5","timestamp":"2026-07-08T09:11:01.000Z","targetId":"f5"} diff --git a/crates/toolpath-openclaw/tests/real_fixture.rs b/crates/toolpath-openclaw/tests/real_fixture.rs index 74cb146b..f67e8fbe 100644 --- a/crates/toolpath-openclaw/tests/real_fixture.rs +++ b/crates/toolpath-openclaw/tests/real_fixture.rs @@ -74,9 +74,27 @@ fn real_main_session_parses_and_classifies() { // File changes recovered from write/edit tool inputs. assert!(!v.files_changed.is_empty()); - // Real per-message usage sums into a session total. + // Real per-message usage sums into a session total — SINGLE-counted. + // This session's final assembled reply (post `sessions_yield`, no + // `responseId`) carries the run-cumulative usage: out=1468 = the exact + // sum of the 10 per-call outputs. Stamping it would double the session + // to 2936; the provider must detect and drop that aggregate row. let total = v.total_usage.expect("usage recorded"); - assert!(total.output_tokens.unwrap_or(0) > 0); + assert_eq!( + total.output_tokens, + Some(1468), + "session output must be the sum of per-call rows, not doubled by the aggregate row" + ); + assert_eq!(total.input_tokens, Some(20)); + + // responseId / responseModel from real API responses land on typed IR + // fields (group_id / response_model). + let with_rid = v + .turns + .iter() + .filter(|t| t.role == Role::Assistant && t.group_id.is_some()) + .count(); + assert_eq!(with_rid, 10, "the 10 real API responses carry responseId"); } #[test] diff --git a/docs/agents/formats/openclaw/format-changelog.md b/docs/agents/formats/openclaw/format-changelog.md index c7c87b53..b2e93262 100644 --- a/docs/agents/formats/openclaw/format-changelog.md +++ b/docs/agents/formats/openclaw/format-changelog.md @@ -35,6 +35,11 @@ blocks, `stopReason`, and the full `Usage` shape (including prompt-cache - **Inception adoption:** a running gateway adopts a transcript + `sessions.json` entry written by an external tool without restart, and appends follow-up turns to the same file. +- **`totalTokens` = `input + output + cacheRead + cacheWrite`** on + per-call rows (11/11 observed). +- **Real API responses carry `responseId` (+ often `responseModel`)**; + the run's final assembled reply after a `sessions_yield` has neither and + carries **run-cumulative usage** (see usage.md — double-count hazard). - `leaf` rows are **not** written on every append — a linear real session contained none (the visible head falls back to the last entry). diff --git a/docs/agents/formats/openclaw/known-issues.md b/docs/agents/formats/openclaw/known-issues.md index 1b3abb27..e52c6b37 100644 --- a/docs/agents/formats/openclaw/known-issues.md +++ b/docs/agents/formats/openclaw/known-issues.md @@ -1,10 +1,9 @@ # Known issues and gotchas -Format-level quirks, ambiguities, and things to defend against. Because we -have **no first-hand on-disk sample** yet (see -[README.md §Sourcing](README.md#sourcing-and-confidence)), some of these are -"from the producer type, not confirmed against a real file" — flagged where -so. +Format-level quirks, ambiguities, and things to defend against. Entries +marked **(observed)** were confirmed against real captured sessions (see +[README.md §Sourcing](README.md#sourcing-and-confidence)); the rest are +from the producer types and not yet seen in the wild. ## Two timestamp encodings per line @@ -37,6 +36,14 @@ on disk (`stripStaleAssistantUsageBeforeLatestCompaction`). Summing transcript usage without accounting for compaction boundaries undercounts the session total. See [usage.md](usage.md#compaction-zeroes-stale-usage). +## The final reply of a multi-call run carries cumulative usage (observed) + +After a `sessions_yield` re-context, the run's final assembled assistant +message has **no `responseId`** and its `usage` is the field-wise **sum of +all prior per-call usages** — run-accumulator totals, not that step's +spend. Naive summing double-counts the session. See +[usage.md](usage.md#the-run-cumulative-aggregate-row-observed). + ## No reasoning token count on disk The persisted `Usage` has input/output/cache/total only. Reasoning/thinking diff --git a/docs/agents/formats/openclaw/usage.md b/docs/agents/formats/openclaw/usage.md index f0af5630..bb23423d 100644 --- a/docs/agents/formats/openclaw/usage.md +++ b/docs/agents/formats/openclaw/usage.md @@ -35,12 +35,28 @@ Key properties: ### `totalTokens` / `prompt_tokens` conventions -`Usage.total` is `totalTokens`. OpenClaw's notion of `prompt_tokens` is -`input + cacheRead` (cacheWrite excluded). For toolpath, prefer summing the -raw `input`/`output`/`cacheRead`/`cacheWrite` independently rather than +`Usage.total` is `totalTokens`. **Observed** (11/11 per-call rows in real +v2026.6.11 sessions): `totalTokens = input + output + cacheRead + +cacheWrite`. OpenClaw's notion of `prompt_tokens` is `input + cacheRead` +(cacheWrite excluded). For toolpath, prefer summing the raw +`input`/`output`/`cacheRead`/`cacheWrite` independently rather than trusting a single headline number, the way `toolpath-pi` does — it stays correct regardless of convention drift. +### The run-cumulative aggregate row (observed) + +One row per multi-call run violates "per-step delta": after a +`sessions_yield` re-context, the run's **final assembled reply** is written +as an assistant message **without a `responseId`** whose `usage` equals the +**field-wise sum of every prior per-call usage** in the run (the runtime +accumulator's totals). Its `totalTokens`, confusingly, repeats the previous +row's context total rather than the sum. Summing transcript usage naively +therefore **double-counts** such sessions. Every real API response carries +a `responseId`; the aggregate row is the one that doesn't. A reader must +detect this row (`toolpath-openclaw` drops its usage on an exact +field-wise-sum match in sessions that stamp `responseId`s) — never stamp it +onto a step. + > **Do not confuse with `SessionEntry.totalTokens`.** A separate > `deriveSessionTotalTokens` (`src/agents/usage.ts`) produces a > prompt/context-size snapshot that **excludes output tokens**. That is a