From 3b8f64828226555754bdb779863bb528e5ae8726 Mon Sep 17 00:00:00 2001 From: Tyler Wells Date: Fri, 17 Jul 2026 11:40:52 -0500 Subject: [PATCH] anthropicprovider: opt-in prompt caching via per-content cache_control Adds Anthropic prompt caching as an explicit, per-content-block primitive rather than an agent-wide toggle, matching how the .NET Anthropic SDK models cache_control (a property of the individual content, not a mode of the client). The caller decides exactly which blocks become cache breakpoints. content := anthropicprovider.WithCacheControl( message.NewTextContent(bigSystemContext), anthropicprovider.WithTTL(anthropicprovider.Ephemeral1h), ) The marker is stored in the content's ContentHeader.AdditionalProperties under a package-private key -- the same hook .NET's AIContent uses for provider metadata -- and read back in buildMessageParam, which transfers it onto whatever Anthropic block the content becomes (text, image, document, tool-use, tool-result). A content whose block type has no cache_control field on the wire (thinking / redacted_thinking) is a clean no-op, never a panic. Caching is opt-in by design: a cache WRITE costs more than a plain input token, so it is a net loss on a short single-turn prompt and nothing turns it on unless the caller asks. cache_control is applied to the block via reflection over the SDK's ContentBlockParamUnion rather than a hand-written variant switch: the union has 17 variants today (every server-tool result among them), and reflection makes the SDK's own type definitions the source of truth so a new variant is handled the day it appears rather than silently left uncached. A canary test asserts the "exactly one non-nil pointer field = the variant" invariant across all variants. TTL supports the ephemeral 5-minute (default) and 1-hour tiers via WithTTL(Ephemeral5m|Ephemeral1h). The TTL type is the SDK's own anthropic.CacheControlEphemeralTTL, consistent with this package already handing raw SDK types across its public surface (MessageNewParams), rather than wrapping it in a parallel type callers would have to convert. Anthropic permits at most 4 cache_control breakpoints per request; the caller is responsible for staying within that budget. This primitive does not dedupe or cap, and higher-level automatic placement is intentionally left to a future change. MessageNewParams remains the low-level escape hatch for breakpoints on tools or the system prompt. --- provider/anthropicprovider/agent.go | 26 +- provider/anthropicprovider/caching.go | 165 ++++++++++ .../caching_internal_test.go | 289 ++++++++++++++++++ 3 files changed, 476 insertions(+), 4 deletions(-) create mode 100644 provider/anthropicprovider/caching.go create mode 100644 provider/anthropicprovider/caching_internal_test.go diff --git a/provider/anthropicprovider/agent.go b/provider/anthropicprovider/agent.go index 476e0391..7cf4936f 100644 --- a/provider/anthropicprovider/agent.go +++ b/provider/anthropicprovider/agent.go @@ -434,9 +434,14 @@ func buildMessageParam(msg *message.Message) (anthropic.MessageParam, error) { var content []anthropic.ContentBlockParamUnion for _, c := range msg.Contents { + var ( + block anthropic.ContentBlockParamUnion + built bool + ) switch c := c.(type) { case *message.TextContent: - content = append(content, anthropic.NewTextBlock(c.Text)) + block = anthropic.NewTextBlock(c.Text) + built = true case *message.FunctionCallContent: // Parse the JSON string arguments into a map for Anthropic SDK var args map[string]any @@ -445,7 +450,8 @@ func buildMessageParam(msg *message.Message) (anthropic.MessageParam, error) { return anthropic.MessageParam{}, fmt.Errorf("failed to unmarshal tool arguments: %w", err) } } - content = append(content, anthropic.NewToolUseBlock(c.CallID, args, c.Name)) + block = anthropic.NewToolUseBlock(c.CallID, args, c.Name) + built = true case *message.FunctionResultContent: resStr := "" switch r := c.Result.(type) { @@ -464,16 +470,28 @@ func buildMessageParam(msg *message.Message) (anthropic.MessageParam, error) { resStr = string(jsonBytes) } } - content = append(content, anthropic.NewToolResultBlock(c.CallID, resStr, c.Error != nil)) + block = anthropic.NewToolResultBlock(c.CallID, resStr, c.Error != nil) + built = true case *message.DataContent: if c.TopLevelMediaType() == "image" { mediaType := c.MediaType if mediaType == "" { mediaType = "image/jpeg" } - content = append(content, anthropic.NewImageBlockBase64(mediaType, c.Data)) + block = anthropic.NewImageBlockBase64(mediaType, c.Data) + built = true } } + if !built { + continue + } + // Opt-in cache_control: if the source content was marked via WithCacheControl, + // carry that breakpoint onto the block just built for it. A content with no marker + // is left untouched, so caching never turns itself on silently. This runs for every + // block type -- text, image, document, tool-use, tool-result -- because the marker + // lives on the message.Content, not on any particular Anthropic wire type. + applyContentCacheControl(c, &block) + content = append(content, block) } switch msg.Role { diff --git a/provider/anthropicprovider/caching.go b/provider/anthropicprovider/caching.go new file mode 100644 index 00000000..78380989 --- /dev/null +++ b/provider/anthropicprovider/caching.go @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft. All rights reserved. + +package anthropicprovider + +import ( + "reflect" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/microsoft/agent-framework-go/message" +) + +// Ephemeral5m and Ephemeral1h are the cache lifetimes Anthropic supports for a +// cache_control breakpoint. They are the anthropic-sdk-go values, surfaced here as +// named constants so callers do not have to reach for the SDK's verbose identifiers. +// +// Their type is the SDK's own anthropic.CacheControlEphemeralTTL, deliberately: this +// package already hands raw SDK types across its public surface (see MessageNewParams), +// so a provider-specific caching primitive stays consistent by doing the same rather +// than wrapping the SDK enum in a parallel type the caller would have to convert. +const ( + // Ephemeral5m caches for 5 minutes. This is also the default when no TTL is set. + Ephemeral5m = anthropic.CacheControlEphemeralTTLTTL5m + // Ephemeral1h caches for 1 hour. + Ephemeral1h = anthropic.CacheControlEphemeralTTLTTL1h +) + +// CacheControl describes an Anthropic cache_control breakpoint on a single content block. +type CacheControl struct { + // TTL is the cache lifetime. The zero value ("") means the Anthropic default of + // 5-minute ephemeral caching; Ephemeral1h selects the 1-hour tier. + TTL anthropic.CacheControlEphemeralTTL +} + +// CacheControlOption configures the CacheControl attached by WithCacheControl. +type CacheControlOption func(*CacheControl) + +// WithTTL sets the cache lifetime for the breakpoint. Passing the zero value leaves +// Anthropic's default (5 minutes) in effect. +func WithTTL(ttl anthropic.CacheControlEphemeralTTL) CacheControlOption { + return func(cc *CacheControl) { cc.TTL = ttl } +} + +// cacheControlKey is the package-private key under which a cache_control marker is stored +// in a content's AdditionalProperties bag. Unexported and package-qualified so it cannot +// collide with a key a caller sets for their own purposes. +const cacheControlKey = "anthropicprovider.cacheControl" + +// WithCacheControl attaches an Anthropic cache_control breakpoint to a single +// message.Content, in place, and returns it for chaining. It is the Go analogue of the +// .NET Anthropic SDK's per-AIContent cache-control extension: caching is expressed on the +// individual content block, not toggled agent-wide, so the caller decides exactly which +// blocks become cache breakpoints. +// +// The marker is stored in the content's ContentHeader.AdditionalProperties under a +// package-private key and read back in buildMessageParam, which transfers it onto whatever +// Anthropic block the content becomes: text, image, document, tool-use, or tool-result. A +// content whose block type has no cache_control field on the wire (the thinking and +// redacted_thinking variants) is a clean no-op rather than an error. +// +// Opt-in, deliberately. A cache WRITE costs more than a plain input token, so caching is a +// net loss for a short single-turn prompt; nothing here turns it on unless the caller asks. +// +// Breakpoint budget: Anthropic permits at most 4 cache_control breakpoints per request +// (the tool definitions, the system prompt, and message blocks all draw from the same +// budget). The caller is responsible for staying within it; this function does not dedupe +// or cap, and a request with too many breakpoints is rejected by the API. Higher-level +// automatic placement is intentionally left to a future change. +// +// For full control over the raw request, including breakpoints on tools or the system +// prompt, use MessageNewParams as the low-level escape hatch. +func WithCacheControl(c message.Content, opts ...CacheControlOption) message.Content { + var cc CacheControl + for _, opt := range opts { + opt(&cc) + } + markCacheControl(c, cc) + return c +} + +// markCacheControl writes the cache_control marker into a content's AdditionalProperties. +// +// Every message.Content implementation embeds message.ContentHeader by value, so each one +// already carries an AdditionalProperties bag -- the same hook .NET's AIContent uses for +// provider metadata. ContentHeader.Header() only hands back a COPY, though, so the field +// has to be reached through the content's own pointer to mutate the original. Reflection +// does that generically: it finds the promoted AdditionalProperties field on any content +// type, present and future, without a per-type switch that would silently miss a type the +// day it is added. +func markCacheControl(c message.Content, cc CacheControl) { + v := reflect.ValueOf(c) + if v.Kind() != reflect.Pointer || v.IsNil() { + return + } + f := v.Elem().FieldByName("AdditionalProperties") + if !f.IsValid() || !f.CanSet() || f.Kind() != reflect.Map { + return + } + if f.IsNil() { + f.Set(reflect.MakeMap(f.Type())) + } + f.SetMapIndex(reflect.ValueOf(cacheControlKey), reflect.ValueOf(cc).Convert(f.Type().Elem())) +} + +// cacheControlOf reads back a marker set by WithCacheControl. Reading uses the public +// Header() copy -- no reflection needed, because reading a value does not require mutating +// the original. +func cacheControlOf(c message.Content) (CacheControl, bool) { + props := c.Header().AdditionalProperties + if props == nil { + return CacheControl{}, false + } + cc, ok := props[cacheControlKey].(CacheControl) + return cc, ok +} + +// applyContentCacheControl transfers a WithCacheControl marker from a message.Content onto +// the Anthropic block that was built from it. No marker means no change (opt-in). If the +// block type cannot carry a breakpoint, setCacheControl reports false and this is a no-op +// rather than a panic: degraded (that block simply uncached) beats a crash. +func applyContentCacheControl(c message.Content, block *anthropic.ContentBlockParamUnion) { + cc, ok := cacheControlOf(c) + if !ok { + return + } + setCacheControl(block, cc.TTL) +} + +var cacheControlType = reflect.TypeOf(anthropic.NewCacheControlEphemeralParam()) + +// setCacheControl marks a content block as a cache breakpoint with the given TTL, +// reporting whether it could. False means the populated variant has no cache_control field +// (thinking and redacted_thinking are the only two), so the block cannot carry one. +// +// This reflects over the union rather than switching on its variants, and that is a +// deliberate trade. ContentBlockParamUnion has 17 variants today; a hand-written switch +// covering the obvious five leaves the other twelve -- every server-tool result, which is +// to say the BIGGEST blocks in a modern agent's context -- silently uncached, and rots the +// moment the SDK adds an eighteenth. Reflection makes the SDK's own type definitions the +// source of truth: a block is cacheable exactly when its param struct carries the field. +// New variants are then handled the day they appear, correctly, with no edit here. +// +// The cost is one reflect walk over a 17-field struct per request. Next to a network call +// to an LLM, that is not a cost. +// +// An empty TTL leaves CacheControlEphemeralParam.TTL at its zero value, which the SDK +// omits from the wire, so Anthropic applies its 5-minute default. +func setCacheControl(b *anthropic.ContentBlockParamUnion, ttl anthropic.CacheControlEphemeralTTL) bool { + v := reflect.ValueOf(b).Elem() + for i := range v.NumField() { + f := v.Field(i) + if f.Kind() != reflect.Pointer || f.IsNil() { + continue + } + // Exactly one variant is non-nil, so this is the block's real type. + cc := f.Elem().FieldByName("CacheControl") + if !cc.IsValid() || !cc.CanSet() || cc.Type() != cacheControlType { + return false // e.g. thinking: no cache_control field exists on the wire type. + } + param := anthropic.NewCacheControlEphemeralParam() + param.TTL = ttl + cc.Set(reflect.ValueOf(param)) + return true + } + return false // empty union: nothing populated. +} diff --git a/provider/anthropicprovider/caching_internal_test.go b/provider/anthropicprovider/caching_internal_test.go new file mode 100644 index 00000000..e4e26929 --- /dev/null +++ b/provider/anthropicprovider/caching_internal_test.go @@ -0,0 +1,289 @@ +// Copyright (c) Microsoft. All rights reserved. + +package anthropicprovider + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/microsoft/agent-framework-go/agent" + "github.com/microsoft/agent-framework-go/message" +) + +// marshalled returns the request as it would go on the wire. The assertions below check +// the JSON rather than SDK struct internals: cache_control is a wire-format contract with +// Anthropic, and that is the thing that has to be right. +func marshalled(t *testing.T, a *client, msgs []*message.Message) map[string]any { + t.Helper() + params, err := a.buildMessageParams(msgs, []agent.Option{agent.WithInstructions("be terse")}) + if err != nil { + t.Fatalf("buildMessageParams: %v", err) + } + raw, err := json.Marshal(params) + if err != nil { + t.Fatalf("marshal params: %v", err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("unmarshal params: %v", err) + } + return out +} + +// lastMessageBlocks returns the content blocks of the final request message as decoded +// JSON, so tests can assert on the wire shape rather than SDK internals. +func lastMessageBlocks(t *testing.T, out map[string]any) []any { + t.Helper() + msgs, _ := out["messages"].([]any) + if len(msgs) == 0 { + t.Fatal("expected at least one message") + } + blocks, _ := msgs[len(msgs)-1].(map[string]any)["content"].([]any) + if len(blocks) == 0 { + t.Fatal("expected content blocks on the last message") + } + return blocks +} + +// Opt-in, not default. An unmarked content must produce no cache_control anywhere. +// +// A cache WRITE costs more than a plain input token, so caching must never turn itself on: +// that would quietly raise the bill for the short single-turn agent whose author is least +// likely to be measuring. +func TestCacheControl_AbsentWhenUnmarked(t *testing.T) { + a := &client{config: AgentConfig{Model: "m"}} + raw, _ := json.Marshal(mustParams(t, a, []*message.Message{message.NewText("hi")})) + + if strings.Contains(string(raw), "cache_control") { + t.Fatalf("cache_control present on an unmarked request:\n%s", raw) + } +} + +// A text content marked with WithCacheControl must produce a cache_control breakpoint on +// exactly the block built from it, and default to the 5-minute tier (no ttl on the wire). +func TestCacheControl_MarksTextBlock(t *testing.T) { + a := &client{config: AgentConfig{Model: "m"}} + + msg := &message.Message{ + Role: message.RoleUser, + Contents: []message.Content{ + WithCacheControl(&message.TextContent{Text: "cache me"}), + }, + } + out := marshalled(t, a, []*message.Message{msg}) + + blocks := lastMessageBlocks(t, out) + last, _ := blocks[len(blocks)-1].(map[string]any) + cc, ok := last["cache_control"].(map[string]any) + if !ok { + t.Fatalf("no cache_control on the marked text block: %v", last) + } + if cc["type"] != "ephemeral" { + t.Fatalf("cache_control type = %v, want ephemeral", cc["type"]) + } + // Default TTL is omitted from the wire; Anthropic reads that as its 5-minute default. + if _, present := cc["ttl"]; present { + t.Fatalf("default breakpoint should carry no ttl, got %v", cc["ttl"]) + } +} + +// TTL flows through: an explicit 1-hour marker must land ttl=1h on the wire. +func TestCacheControl_TTLFlowsThrough(t *testing.T) { + a := &client{config: AgentConfig{Model: "m"}} + + msg := &message.Message{ + Role: message.RoleUser, + Contents: []message.Content{ + WithCacheControl(&message.TextContent{Text: "cache me for an hour"}, WithTTL(Ephemeral1h)), + }, + } + out := marshalled(t, a, []*message.Message{msg}) + + blocks := lastMessageBlocks(t, out) + cc, _ := blocks[len(blocks)-1].(map[string]any)["cache_control"].(map[string]any) + if cc["ttl"] != "1h" { + t.Fatalf("cache_control ttl = %v, want 1h", cc["ttl"]) + } + + // And the explicit 5-minute tier is a real value too, distinct from unset default. + msg5m := &message.Message{ + Role: message.RoleUser, + Contents: []message.Content{ + WithCacheControl(&message.TextContent{Text: "cache me for five"}, WithTTL(Ephemeral5m)), + }, + } + out5m := marshalled(t, a, []*message.Message{msg5m}) + blocks5m := lastMessageBlocks(t, out5m) + cc5m, _ := blocks5m[len(blocks5m)-1].(map[string]any)["cache_control"].(map[string]any) + if cc5m["ttl"] != "5m" { + t.Fatalf("explicit 5m ttl = %v, want 5m", cc5m["ttl"]) + } +} + +// Every block type qmuntal listed must carry the marker: text, image, tool-use, +// tool-result. If any one silently dropped it, the expensive half of an agent's context +// (tool results, images) could go uncached while the caller believed they had opted in. +func TestCacheControl_MarksEveryBlockType(t *testing.T) { + a := &client{config: AgentConfig{Model: "m"}} + + cases := []struct { + name string + msg *message.Message + wantType string + }{ + { + name: "text", + msg: &message.Message{Role: message.RoleUser, Contents: []message.Content{ + WithCacheControl(&message.TextContent{Text: "hi"}), + }}, + wantType: "text", + }, + { + name: "image", + msg: &message.Message{Role: message.RoleUser, Contents: []message.Content{ + WithCacheControl(&message.DataContent{MediaType: "image/png", Data: "aGVsbG8="}), + }}, + wantType: "image", + }, + { + name: "tool_use", + msg: &message.Message{Role: message.RoleAssistant, Contents: []message.Content{ + WithCacheControl(&message.FunctionCallContent{CallID: "call-1", Name: "search", Arguments: `{"q":"x"}`}), + }}, + wantType: "tool_use", + }, + { + name: "tool_result", + msg: &message.Message{Role: message.RoleTool, Contents: []message.Content{ + WithCacheControl(&message.FunctionResultContent{CallID: "call-1", Result: "a large tool result"}), + }}, + wantType: "tool_result", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out := marshalled(t, a, []*message.Message{tc.msg}) + blocks := lastMessageBlocks(t, out) + last, _ := blocks[len(blocks)-1].(map[string]any) + if last["type"] != tc.wantType { + t.Fatalf("block type = %v, want %v", last["type"], tc.wantType) + } + if _, ok := last["cache_control"]; !ok { + t.Fatalf("no cache_control on the marked %s block: %v", tc.wantType, last) + } + }) + } +} + +// Marking a block that has no cache_control field on the wire is a clean no-op. +// +// thinking and redacted_thinking are the only two content variants with no cache_control +// field, so a breakpoint cannot sit on them. setCacheControl must report false and leave +// the block untouched rather than panic -- degraded (that block uncached) beats a crash. +func TestSetCacheControl_NoFieldBlockIsCleanNoop(t *testing.T) { + thinking := anthropic.NewThinkingBlock("sig", "let me think") + if got := setCacheControl(&thinking, ""); got != false { + t.Fatalf("setCacheControl on a thinking block = %v, want false", got) + } + // The block must be unchanged and still marshal cleanly (no panic, no cache_control). + raw, err := json.Marshal(thinking) + if err != nil { + t.Fatalf("marshal thinking block: %v", err) + } + if strings.Contains(string(raw), "cache_control") { + t.Fatalf("thinking block gained a cache_control it cannot carry:\n%s", raw) + } +} + +// The invariant, guarded against a moving SDK. +// +// Every ContentBlockParamUnion variant whose param struct HAS a cache_control field must +// actually receive a breakpoint, and every variant that lacks one must report false. +// This drives itself off the union's own reflected shape, so when the SDK adds an +// 18th variant this test covers it the day it lands -- and if anyone ever replaces the +// reflective setCacheControl with a hand-written switch, it fails for whatever they +// forgot. That is the point: a switch covering the obvious five would silently skip twelve +// cacheable variants, including every server-tool result. +// +// It also proves the invariant this whole design rests on: for each variant exactly one +// pointer field of the union is non-nil, which is what lets setCacheControl treat "the +// first non-nil field" as the block's real type. +func TestSetCacheControl_CoversEveryCacheableVariant(t *testing.T) { + union := reflect.TypeOf(anthropic.ContentBlockParamUnion{}) + + var cacheable, skipped int + for i := range union.NumField() { + field := union.Field(i) + if field.Type.Kind() != reflect.Pointer { + continue // the embedded paramUnion marker + } + + // Build a union with just this variant populated. + block := anthropic.ContentBlockParamUnion{} + variant := reflect.New(field.Type.Elem()) + reflect.ValueOf(&block).Elem().Field(i).Set(variant) + + // Exactly one non-nil pointer = the variant. Guard that invariant directly. + if n := nonNilPointerFields(block); n != 1 { + t.Fatalf("%s: union has %d non-nil pointer fields, want exactly 1", field.Name, n) + } + + _, wantsCache := field.Type.Elem().FieldByName("CacheControl") + got := setCacheControl(&block, Ephemeral1h) + + if got != wantsCache { + t.Errorf("%s: setCacheControl = %v, but the wire type %s a cache_control field", + field.Name, got, map[bool]string{true: "HAS", false: "has no"}[wantsCache]) + continue + } + if wantsCache { + cacheable++ + cc := variant.Elem().FieldByName("CacheControl") + ephemeral := cc.Interface().(anthropic.CacheControlEphemeralParam) + if ephemeral.Type != "ephemeral" { + t.Errorf("%s: reported success but wrote no ephemeral breakpoint", field.Name) + } + // TTL must flow through the reflective setter, not get dropped. + if ephemeral.TTL != Ephemeral1h { + t.Errorf("%s: ttl = %q, want %q", field.Name, ephemeral.TTL, Ephemeral1h) + } + } else { + skipped++ + } + } + + // Guard the guard: if the union were ever read as empty, every assertion above would + // vacuously pass and this test would prove nothing. + if cacheable < 10 || skipped != 2 { + t.Fatalf("union shape looks wrong: %d cacheable, %d non-cacheable "+ + "(expected 10+ cacheable and exactly 2 non-cacheable: thinking, redacted_thinking)", + cacheable, skipped) + } +} + +// nonNilPointerFields counts the non-nil pointer fields of a ContentBlockParamUnion, the +// "exactly one variant is populated" invariant setCacheControl relies on. +func nonNilPointerFields(block anthropic.ContentBlockParamUnion) int { + v := reflect.ValueOf(block) + var n int + for i := range v.NumField() { + f := v.Field(i) + if f.Kind() == reflect.Pointer && !f.IsNil() { + n++ + } + } + return n +} + +func mustParams(t *testing.T, a *client, msgs []*message.Message) any { + t.Helper() + params, err := a.buildMessageParams(msgs, []agent.Option{agent.WithInstructions("be terse")}) + if err != nil { + t.Fatalf("buildMessageParams: %v", err) + } + return params +}