[telemetry] 🤖 feat: measure advisor cache usage and savings

docs/reference/telemetry.mdx

@@ -8,7 +8,7 @@ Xum collects anonymous usage telemetry to help improve the product.
 ## Privacy policy
 
 - **No personal information**: Xum does not collect usernames, project names, file paths, or code content.
-- **Random IDs only**: Only randomly generated workspace IDs are sent.
+- **Random IDs only**: Workspace, parent-turn, and advisor-call IDs contain no user content.
 - **No hashing**: Hashing is vulnerable to rainbow table attacks.
 - **Transparent payload**: See exactly what is sent in [`src/common/telemetry/payload.ts`](https://github.com/coder/mux/blob/main/src/common/telemetry/payload.ts).
 
@@ -28,6 +28,28 @@ All telemetry events include basic system information:
 - **Message sending**: When messages are sent (model, mode, message length rounded to base-2)
 - **Errors**: Error types and context (no sensitive data)
 
+### Advisor completion
+
+The backend sends advisor_call_completed after each admitted advisor call, including errors and cancellations.
+Calls rejected by the usage limit do not send this event.
+
+The event includes:
+
+- Random workspace, parent-turn, and advisor-call IDs, plus the built-in provider route.
+- A catalog model name. Unresolved custom model names become unknown.
+- The outcome, call index, and time since the previous call within the current tool instance.
+- The duration and time to the first text or reasoning token.
+- Input, uncached input, cache-read, cache-write, and output token counts.
+- Explicit Anthropic cache marker counts and their requested TTL.
+- Estimated input cost, cache-write premium, cache-read savings, and net cache savings.
+
+Token counts, times, and costs use base-2 rounding. Unknown measurements use null, not zero.
+Cost estimates use catalog rates for known Anthropic models with one known TTL and complete token counts.
+They are not invoice charges. Mixed TTLs and unsupported pricing produce unknown costs.
+Net savings subtract the write premium from read savings before rounding. Negative values indicate an estimated loss.
+Request markers do not prove that the provider accepts or reuses a cache.
+The event contains no prompt content, questions, transcript hashes, cache keys, or endpoint URLs.
+
 ### What Xum does _not_ track
 
 - Your messages or code

src/common/orpc/schemas/telemetry.ts

@@ -187,6 +187,34 @@ const MCPOAuthFlowFailedPropertiesSchema = z.object({
   error_category: TelemetryMCPOAuthFlowErrorCategorySchema,
 });
 
+export const AdvisorCallCompletedPropertiesSchema = z.object({
+  call_id: z.string(),
+  parent_turn_id: z.string().optional(),
+  provider_route: z.string().nullable(),
+  workspaceId: z.string().optional(),
+  model: z.string(),
+  outcome: z.enum(["success", "error", "cancelled"]),
+  call_index: z.number(),
+  // This gap covers the current tool instance, not the whole workspace history.
+  previous_call_gap_ms_b2: z.number().nullable(),
+  duration_ms_b2: z.number(),
+  time_to_first_token_ms_b2: z.number().nullable(),
+  usage_available: z.boolean(),
+  input_tokens_b2: z.number().nullable(),
+  uncached_input_tokens_b2: z.number().nullable(),
+  cache_read_tokens_b2: z.number().nullable(),
+  cache_write_tokens_b2: z.number().nullable(),
+  output_tokens_b2: z.number().nullable(),
+  // Request markers do not prove that the provider accepts or reuses the cache.
+  cache_marker_count: z.number(),
+  cache_ttl: z.enum(["5m", "1h", "mixed", "unknown"]),
+  // Estimates use catalog rates, not invoice charges. Null means unknown.
+  input_cost_usd_b2: z.number().nullable(),
+  cache_write_premium_usd_b2: z.number().nullable(),
+  cache_read_savings_usd_b2: z.number().nullable(),
+  cache_net_savings_usd_b2: z.number().nullable(),
+});
+
 const StreamCompletedPropertiesSchema = z.object({
   model: z.string(),
   wasInterrupted: z.boolean(),
@@ -228,6 +256,10 @@ const ExperimentOverriddenPropertiesSchema = z.object({
 
 // Union of all telemetry events
 export const TelemetryEventSchema = z.discriminatedUnion("event", [
+  z.object({
+    event: z.literal("advisor_call_completed"),
+    properties: AdvisorCallCompletedPropertiesSchema,
+  }),
   z.object({
     event: z.literal("app_started"),
     properties: AppStartedPropertiesSchema,

src/common/telemetry/payload.ts

@@ -20,6 +20,8 @@
  * code only needs to provide event-specific properties.
  */
 
+import type { z } from "zod";
+import type { AdvisorCallCompletedPropertiesSchema } from "@/common/orpc/schemas/telemetry";
 import type { RuntimeMode } from "@/common/types/runtime";
 
 /**
@@ -243,9 +245,9 @@ export interface StreamTimingInvalidPayload {
   reason: string;
 }
 
-/**
- * Stream completion event - tracks when AI responses finish
- */
+/** Advisor cache measurements use rounded values, not prompt content. */
+export type AdvisorCallCompletedPayload = z.infer<typeof AdvisorCallCompletedPropertiesSchema>;
+
 export interface StreamCompletedPayload {
   /** Model used for generation */
   model: string;
@@ -359,6 +361,7 @@ export interface ExperimentOverriddenPayload {
  * Frontend sends these; backend adds BaseTelemetryProperties before forwarding to PostHog
  */
 export type TelemetryEventPayload =
+  | { event: "advisor_call_completed"; properties: AdvisorCallCompletedPayload }
   | { event: "app_started"; properties: AppStartedPayload }
   | { event: "workspace_created"; properties: WorkspaceCreatedPayload }
   | { event: "workspace_switched"; properties: WorkspaceSwitchedPayload }

src/common/utils/tools/tools.ts

@@ -1,3 +1,4 @@
+import type { AdvisorCallCompletedPayload } from "@/common/telemetry/payload";
 import type { HistoryService } from "@/node/services/historyService";
 import { createSessionHistoryTool } from "@/node/services/tools/session_history";
 import { createNewContextTool } from "@/node/services/tools/new_context";
@@ -355,6 +356,8 @@ export interface ToolConfiguration {
   };
   /** Runtime bundle for the advisor tool (present only when advisor is eligible for this stream). */
   advisorRuntime?: {
+    /** Report cache economics without sending transcript content. */
+    reportTelemetry?: (event: AdvisorCallCompletedPayload) => void;
     /** The advisor model string (e.g. "anthropic:claude-sonnet-4-20250514") */
     advisorModelString: string;
     /** Optional reasoning/thinking level metadata for the advisor request. */
@@ -374,7 +377,10 @@ export interface ToolConfiguration {
      * Coder identities retain their actual instance and scoped aliases; option
      * construction resolves their wire from this snapshot, never live config.
      */
-    createModel: (modelString: string) => Promise<{
+    createModel: (
+      modelString: string,
+      onAnthropicRequest?: (requestBody: unknown) => void
+    ) => Promise<{
       model: LanguageModel;
       metadataModel?: string;
       optionsModelString: string;

src/node/services/agentSkills/builtInSkillContent.generated.ts

@@ -7821,7 +7821,7 @@ export const BUILTIN_SKILL_FILES: Record<string, Record<string, string>> = {
       "## Privacy policy",
       "",
       "- **No personal information**: Xum does not collect usernames, project names, file paths, or code content.",
-      "- **Random IDs only**: Only randomly generated workspace IDs are sent.",
+      "- **Random IDs only**: Workspace, parent-turn, and advisor-call IDs contain no user content.",
       "- **No hashing**: Hashing is vulnerable to rainbow table attacks.",
       "- **Transparent payload**: See exactly what is sent in [`src/common/telemetry/payload.ts`](https://github.com/coder/mux/blob/main/src/common/telemetry/payload.ts).",
       "",
@@ -7841,6 +7841,28 @@ export const BUILTIN_SKILL_FILES: Record<string, Record<string, string>> = {
       "- **Message sending**: When messages are sent (model, mode, message length rounded to base-2)",
       "- **Errors**: Error types and context (no sensitive data)",
       "",
+      "### Advisor completion",
+      "",
+      "The backend sends advisor_call_completed after each admitted advisor call, including errors and cancellations.",
+      "Calls rejected by the usage limit do not send this event.",
+      "",
+      "The event includes:",
+      "",
+      "- Random workspace, parent-turn, and advisor-call IDs, plus the built-in provider route.",
+      "- A catalog model name. Unresolved custom model names become unknown.",
+      "- The outcome, call index, and time since the previous call within the current tool instance.",
+      "- The duration and time to the first text or reasoning token.",
+      "- Input, uncached input, cache-read, cache-write, and output token counts.",
+      "- Explicit Anthropic cache marker counts and their requested TTL.",
+      "- Estimated input cost, cache-write premium, cache-read savings, and net cache savings.",
+      "",
+      "Token counts, times, and costs use base-2 rounding. Unknown measurements use null, not zero.",
+      "Cost estimates use catalog rates for known Anthropic models with one known TTL and complete token counts.",
+      "They are not invoice charges. Mixed TTLs and unsupported pricing produce unknown costs.",
+      "Net savings subtract the write premium from read savings before rounding. Negative values indicate an estimated loss.",
+      "Request markers do not prove that the provider accepts or reuses a cache.",
+      "The event contains no prompt content, questions, transcript hashes, cache keys, or endpoint URLs.",
+      "",
       "### What Xum does _not_ track",
       "",
       "- Your messages or code",

src/node/services/providerModelFactory.test.ts

@@ -39,6 +39,7 @@ import { CodexOauthService } from "./codexOauthService";
 import type { CoderOauthService } from "./coderOauthService";
 import { PolicyService } from "./policyService";
 import { ProviderService } from "./providerService";
+import { advisorWireCachePolicy } from "./tools/advisorTelemetry";
 
 const LOCAL_VLLM_BASE_URL = "http://localhost:8000/v1";
 const LOCAL_VLLM_MODEL = "qwen3-coder";
@@ -2494,11 +2495,93 @@ describe("wrapFetchWithXAIServiceTier", () => {
 // Effort "xhigh" and thinking.display flow through the SDK directly as of
 // @ai-sdk/anthropic 4.0.11 (see buildProviderOptions), so the wrapper must NOT
 // rewrite reasoning fields — it only normalizes cache_control.
+describe("Anthropic cache request observation", () => {
+  it.each([
+    { ttl: undefined, marked: false, count: 1, expectedTtl: "5m" },
+    { ttl: "1h", marked: false, count: 1, expectedTtl: "1h" },
+    { ttl: "1h", marked: true, count: 2, expectedTtl: "1h" },
+  ] as const)(
+    "observes pinned model wire markers: %j",
+    async ({ ttl, marked, count, expectedTtl }) => {
+      await withTempConfig(async (config, factory, _oauth, store) => {
+        store.saveProvidersConfig({ anthropic: { apiKey: "test-key", cacheTtl: ttl } });
+        await saveRoutePriority(config, ["direct"]);
+        const observed: unknown[] = [];
+        const { calls, fakeFetch } = createCapturingFetch();
+        const fetchSpy = spyOn(globalThis, "fetch").mockImplementation(fakeFetch);
+        try {
+          const result = await factory.createModelWithPinnedOptions(
+            "anthropic:claude-sonnet-4-20250514",
+            {
+              onAnthropicRequest: (body) => observed.push(body),
+            }
+          );
+          if (!result.success) throw new Error(result.error.type);
+          // The capture fetch has no model response. Only request serialization matters here.
+          await generateText({
+            model: result.data.model,
+            maxRetries: 0,
+            messages: [
+              ...(marked
+                ? [
+                    {
+                      role: "assistant" as const,
+                      content: "Earlier advice",
+                      providerOptions: {
+                        anthropic: { cacheControl: { type: "ephemeral", ttl: "1h" } },
+                      },
+                    },
+                  ]
+                : []),
+              { role: "user", content: "hello" },
+            ],
+          }).catch((error: unknown) => {
+            if (calls.length === 0) throw error;
+          });
+          expect(observed).toHaveLength(1);
+          expect(calls).toHaveLength(1);
+          expect(observed[0]).toEqual(parseSentBody(calls[0]));
+          expect(advisorWireCachePolicy(observed[0])).toEqual({
+            cache_marker_count: count,
+            cache_ttl: expectedTtl,
+          });
+        } finally {
+          fetchSpy.mockRestore();
+        }
+      });
+    }
+  );
+
+  it("keeps wire injection when the observer throws", async () => {
+    const { calls, fakeFetch } = createCapturingFetch();
+    const wrapped = wrapFetchWithAnthropicCacheControl(fakeFetch, "1h", {
+      onRequest: () => {
+        throw new Error("observer failed");
+      },
+    });
+    await wrapped("https://api.anthropic.com/v1/messages", {
+      method: "POST",
+      body: JSON.stringify({
+        messages: [{ role: "user", content: [{ type: "text", text: "hello" }] }],
+      }),
+    });
+    expect(calls).toHaveLength(1);
+    expect(advisorWireCachePolicy(parseSentBody(calls[0]))).toEqual({
+      cache_marker_count: 1,
+      cache_ttl: "1h",
+    });
+  });
+});
+
 describe("wrapFetchWithAnthropicCacheControl — ZDR stripping", () => {
   it("strips existing cache markers when injection is disabled", async () => {
     const { calls, fakeFetch } = createCapturingFetch();
+    let observed: unknown;
     const wrapped = wrapFetchWithAnthropicCacheControl(fakeFetch, undefined, {
       injectCacheControl: false,
+      onRequest: (body) => {
+        observed = body;
+      },
     });
 
     // Markers the request pipeline can serialize before the wrapper runs:
@@ -2523,6 +2606,10 @@ describe("wrapFetchWithAnthropicCacheControl — ZDR stripping", () => {
     const sent = JSON.stringify(parseSentBody(calls[0]));
     expect(sent).not.toContain("cache_control");
     expect(sent).not.toContain("cacheControl");
+    expect(advisorWireCachePolicy(observed)).toEqual({
+      cache_marker_count: 0,
+      cache_ttl: "unknown",
+    });
   });
 
   it("keeps markers when injection is enabled", async () => {