Skip to content

Add extended prompt cache TTL setup option #6515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Sanskarzz wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add extended prompt cache TTL setup option #6515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 32 additions & 8 deletions cmd/thv/app/llm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,15 @@ func addLLMConnectionFlags(cmd *cobra.Command, opts *llm.SetOptions) {
// field unchanged (nil pointer = "not provided"). Shared by "config set" and
// "setup" so both commands treat these flags identically.
func applyChangedLLMFlags(
cmd *cobra.Command, opts *llm.SetOptions, tlsSkipVerify, bedrockCompat, enable1M bool, models []string,
cmd *cobra.Command, opts *llm.SetOptions,
tlsSkipVerify, extendedTTLCache, bedrockCompat, enable1M bool, models []string,
) {
if cmd.Flags().Changed("tls-skip-verify") {
opts.TLSSkipVerify = &tlsSkipVerify
}
if cmd.Flags().Changed("extended-ttl-cache") {
opts.ExtendedTTLCache = &extendedTTLCache
}
if cmd.Flags().Changed("bedrock-compat") {
opts.BedrockCompat = &bedrockCompat
}
Expand All @@ -111,11 +115,12 @@ func applyChangedLLMFlags(

func newConfigSetCommand() *cobra.Command {
var (
opts llm.SetOptions
tlsSkipVerify bool
bedrockCompat bool
enable1M bool
models []string
opts llm.SetOptions
tlsSkipVerify bool
extendedTTLCache bool
bedrockCompat bool
enable1M bool
models []string
)

cmd := &cobra.Command{
Expand All @@ -130,7 +135,7 @@ Example:
--client-id my-client-id`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
applyChangedLLMFlags(cmd, &opts, tlsSkipVerify, bedrockCompat, enable1M, models)
applyChangedLLMFlags(cmd, &opts, tlsSkipVerify, extendedTTLCache, bedrockCompat, enable1M, models)
return config.UpdateConfig(func(c *config.Config) error {
return c.LLM.SetFields(opts)
})
Expand All @@ -140,6 +145,9 @@ Example:
addLLMConnectionFlags(cmd, &opts)
cmd.Flags().BoolVar(&tlsSkipVerify, "tls-skip-verify", false,
"Skip TLS certificate verification for the upstream gateway (local dev only; use --tls-skip-verify=false to clear)")
cmd.Flags().BoolVar(&extendedTTLCache, "extended-ttl-cache", false,
"Persist the one-hour prompt-cache lifetime for clients that support it. Applied by \"thv llm setup\". "+
"Use --extended-ttl-cache=false to clear.")
cmd.Flags().BoolVar(&bedrockCompat, "bedrock-compat", false,
"Persist Bedrock compatibility for Claude Code (CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 + per-tier "+
"Bedrock model IDs). Applied by \"thv llm setup\". Use --bedrock-compat=false to clear.")
Expand Down Expand Up @@ -263,6 +271,7 @@ func newLLMSetupCommand() *cobra.Command {
var (
opts llm.SetOptions
tlsSkipVerify bool
extendedTTLCache bool
bedrockCompat bool
enable1M bool
targetClient string
Expand Down Expand Up @@ -318,7 +327,7 @@ Re-running is idempotent and uses the cached token (no browser prompt).
Run "thv llm teardown" to revert all changes.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
applyChangedLLMFlags(cmd, &opts, tlsSkipVerify, bedrockCompat, enable1M, models)
applyChangedLLMFlags(cmd, &opts, tlsSkipVerify, extendedTTLCache, bedrockCompat, enable1M, models)
cm, err := client.NewClientManager()
if err != nil {
return fmt.Errorf("initializing client manager: %w", err)
Expand All @@ -340,6 +349,9 @@ Run "thv llm teardown" to revert all changes.`,
"For direct-mode tools (Claude Code, Gemini CLI) this sets NODE_TLS_REJECT_UNAUTHORIZED=0, "+
"disabling TLS for ALL of that tool's outbound connections. "+
"For proxy-mode tools only the proxy-to-gateway connection is affected.")
cmd.Flags().BoolVar(&extendedTTLCache, "extended-ttl-cache", false,
"Request the one-hour prompt-cache lifetime from each client that supports it. Persisted, so a later plain "+
"\"thv llm setup\" re-applies it; clear with --extended-ttl-cache=false.")
cmd.Flags().StringVar(&anthropicPathPrefix, "anthropic-path-prefix", "",
"Path prefix appended to the gateway URL when writing ANTHROPIC_BASE_URL for direct-mode tools "+
"(e.g. /anthropic). When omitted, the gateway is probed automatically.")
Expand Down Expand Up @@ -480,6 +492,18 @@ func (a *clientManagerAdapter) LLMGatewayModeFor(clientType string) string {
return a.cm.LLMGatewayModeFor(client.ClientApp(clientType))
}

func (a *clientManagerAdapter) SupportsExtendedTTLCache(clientType string) bool {
return a.cm.SupportsExtendedTTLCache(client.ClientApp(clientType))
}

func (a *clientManagerAdapter) ExtendedTTLCacheConflict(clientType string) (string, error) {
return a.cm.ExtendedTTLCacheConflict(client.ClientApp(clientType))
}

func (a *clientManagerAdapter) ClearExtendedTTLCache(clientType, configPath string) error {
return a.cm.ClearExtendedTTLCache(client.ClientApp(clientType), configPath)
}

func (a *clientManagerAdapter) IsManaged(clientType string) bool {
return a.cm.IsManaged(client.ClientApp(clientType))
}
Expand Down
23 changes: 23 additions & 0 deletions cmd/thv/app/llm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
Expand Down Expand Up @@ -47,6 +48,28 @@ func llmProvider(t *testing.T, llmCfg llm.Config) config.Provider {
// Use it in tests that don't exercise the authentication path.
var noopLogin llm.LoginFunc = func(context.Context, *llm.Config) error { return nil }

func TestConfigSetCommand_ExtendedTTLCacheFlagWiring(t *testing.T) {
t.Parallel()

for _, enabled := range []bool{true, false} {
t.Run(fmt.Sprintf("enabled=%t", enabled), func(t *testing.T) {
t.Parallel()
cmd := newConfigSetCommand()
flag := cmd.Flags().Lookup("extended-ttl-cache")
require.NotNil(t, flag)
require.NoError(t, cmd.Flags().Set("extended-ttl-cache", fmt.Sprintf("%t", enabled)))

parsed, err := cmd.Flags().GetBool("extended-ttl-cache")
require.NoError(t, err)
var opts llm.SetOptions
applyChangedLLMFlags(cmd, &opts, false, parsed, false, false, nil)

require.NotNil(t, opts.ExtendedTTLCache)
assert.Equal(t, enabled, *opts.ExtendedTTLCache)
})
}
}

// errOnUpdateProvider wraps a base Provider but returns a fixed error from
// UpdateConfig. Used to inject deterministic failures without relying on
// filesystem permission tricks that are unreliable on Windows.
Expand Down
1 change: 1 addition & 0 deletions docs/cli/thv_llm_config_set.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions docs/cli/thv_llm_setup.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 21 additions & 10 deletions pkg/client/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,11 @@ const (
// - ValueField names which ApplyConfig field to write. Valid values:
// "GatewayURL", "AnthropicBaseURL", "ProxyBaseURL", "ProxyOrigin",
// "TokenHelperCommand", "PlaceholderAPIKey", "ClaudeCodeHelperTTLMillis",
// "NodeTLSRejectUnauthorized", "BedrockDisableExperimentalBetas",
// "BedrockHaikuModel", "BedrockOpusModel", "BedrockSonnetModel". An
// unrecognised ValueField is a programming error and causes
// ConfigureLLMGateway to return an error.
// "NodeTLSRejectUnauthorized", "ExtendedTTLCache",
// "ExtendedTTLCacheLegacy",
// "BedrockDisableExperimentalBetas", "BedrockHaikuModel",
// "BedrockOpusModel", "BedrockSonnetModel". An unrecognised ValueField is
// a programming error and causes ConfigureLLMGateway to return an error.
// - Literal is written verbatim into the settings key (e.g. a fixed auth
// type string). Use Literal instead of ValueField for constant values so
// that typos in ValueField are caught as errors rather than silently
Expand All @@ -184,8 +185,9 @@ type LLMGatewayKeySpec struct {
JSONPointer string // RFC 6901 path
// ValueField: "GatewayURL" | "AnthropicBaseURL" | "ProxyBaseURL" | "ProxyOrigin" |
// "TokenHelperCommand" | "PlaceholderAPIKey" | "ClaudeCodeHelperTTLMillis" |
// "NodeTLSRejectUnauthorized" | "BedrockDisableExperimentalBetas" |
// "BedrockHaikuModel" | "BedrockOpusModel" | "BedrockSonnetModel"
// "NodeTLSRejectUnauthorized" | "ExtendedTTLCache" | "ExtendedTTLCacheLegacy" |
// "BedrockDisableExperimentalBetas" | "BedrockHaikuModel" |
// "BedrockOpusModel" | "BedrockSonnetModel"
ValueField string
Literal string // constant value written verbatim; mutually exclusive with ValueField
ClearWhenEmpty bool // remove the key when the resolved value is empty (ignored for Literal)
Expand Down Expand Up @@ -253,6 +255,8 @@ type clientAppConfig struct {
// LLMGatewayMode identifies the gateway integration strategy (direct token
// helper, proxy, credential helper, or Codex auth), or "" when unsupported.
LLMGatewayMode string
// SupportsExtendedTTLCache declares support for the conditional TTL keys.
SupportsExtendedTTLCache bool
// LLMBinaryName is the executable name looked up via exec.LookPath to
// confirm the tool is actually installed (not just a leftover config
// directory). Leave empty for tools that are not on $PATH (e.g. macOS
Expand Down Expand Up @@ -535,10 +539,11 @@ var supportedClientIntegrations = []clientAppConfig{
PluginsGlobalPath: []string{".claude", "plugins"},
PluginsProjectPath: []string{".claude", "plugins"},
// LLM gateway: patches ~/.claude/settings.json (different from the MCP .claude.json)
LLMGatewayMode: llmgateway.ModeDirect,
LLMBinaryName: "claude",
LLMSettingsFile: "settings.json",
LLMSettingsRelPath: []string{".claude"},
LLMGatewayMode: llmgateway.ModeDirect,
SupportsExtendedTTLCache: true,
LLMBinaryName: "claude",
LLMSettingsFile: "settings.json",
LLMSettingsRelPath: []string{".claude"},
LLMGatewayKeys: []LLMGatewayKeySpec{
{JSONPointer: "/apiKeyHelper", ValueField: "TokenHelperCommand"},
{JSONPointer: "/env/ANTHROPIC_BASE_URL", ValueField: "AnthropicBaseURL"},
Expand All @@ -549,6 +554,12 @@ var supportedClientIntegrations = []clientAppConfig{
// NODE_TLS_REJECT_UNAUTHORIZED is only written when --tls-skip-verify is set.
// ClearWhenEmpty ensures it is removed when the flag is later cleared.
{JSONPointer: "/env/NODE_TLS_REJECT_UNAUTHORIZED", ValueField: "NodeTLSRejectUnauthorized", ClearWhenEmpty: true},
// Current Claude Code versions expose separate controls for the main
// conversation and auxiliary requests. ENABLE_PROMPT_CACHING_1H is the
// fallback for versions that predate those per-bucket settings.
{JSONPointer: "/promptCacheTtl", ValueField: "ExtendedTTLCache", ClearWhenEmpty: true},
{JSONPointer: "/subagentPromptCacheTtl", ValueField: "ExtendedTTLCache", ClearWhenEmpty: true},
{JSONPointer: "/env/ENABLE_PROMPT_CACHING_1H", ValueField: "ExtendedTTLCacheLegacy", ClearWhenEmpty: true},
// Bedrock-compat keys (written only with --bedrock-compat). Bedrock rejects
// Claude Code's experimental anthropic-beta headers, so betas are disabled;
// the per-tier model IDs pin Bedrock inference-profile IDs. All use
Expand Down
Loading
Loading