feat(tok): add opt-in live secret verification and exact token counting

internal/core/exact_counter.go

@@ -0,0 +1,130 @@
+package core
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// exactCountClient is used for opt-in exact-count API calls. It is a var
+// (not embedded per-call) so tests can override it, and is only ever
+// touched by CountTokensExact — never by EstimateTokens/EstimateTokensPrecise,
+// which remain fully offline and unchanged.
+var exactCountClient = &http.Client{Timeout: 10 * time.Second}
+
+// anthropicCountTokensURL and googleCountTokensURL are vars (not consts) so
+// tests can point them at a local mock server.
+var (
+	anthropicCountTokensURL = "https://api.anthropic.com/v1/messages/count_tokens"
+	googleCountTokensURL    = "https://generativelanguage.googleapis.com/v1beta/models/%s:countTokens"
+)
+
+// CountTokensExact returns the exact token count for text under the real
+// tokenizer of the model's provider (Anthropic or Google), by calling that
+// provider's live count-tokens API. This exists because neither Anthropic
+// nor Google publish an offline BPE tokenizer library — EstimateTokens and
+// friends only approximate Claude/Gemini counts via OpenAI's cl100k_base
+// encoding (see ClaudeBase/GeminiBase in the tok package).
+//
+// apiKey is required and is never read from the environment implicitly:
+// this function makes a real network call using the given text and key, and
+// is never invoked by any other function in this package — callers must opt
+// in explicitly. model must have a "claude" or "gemini" prefix (case
+// insensitive); any other prefix returns an error rather than silently
+// falling back to an approximation.
+func CountTokensExact(ctx context.Context, apiKey, model, text string) (int, error) {
+	if apiKey == "" {
+		return 0, fmt.Errorf("tok: CountTokensExact: apiKey is required")
+	}
+	lower := strings.ToLower(strings.TrimSpace(model))
+	switch {
+	case strings.HasPrefix(lower, "claude"):
+		return countTokensAnthropic(ctx, apiKey, model, text)
+	case strings.HasPrefix(lower, "gemini"):
+		return countTokensGoogle(ctx, apiKey, model, text)
+	default:
+		return 0, fmt.Errorf("tok: CountTokensExact: unsupported model %q (must be a claude* or gemini* model)", model)
+	}
+}
+
+func countTokensAnthropic(ctx context.Context, apiKey, model, text string) (int, error) {
+	body, err := json.Marshal(map[string]any{
+		"model": model,
+		"messages": []map[string]string{
+			{"role": "user", "content": text},
+		},
+	})
+	if err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: marshal request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, anthropicCountTokensURL, bytes.NewReader(body))
+	if err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("x-api-key", apiKey)
+	req.Header.Set("anthropic-version", "2023-06-01")
+
+	resp, err := exactCountClient.Do(req)
+	if err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: request failed: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		data, _ := io.ReadAll(resp.Body)
+		return 0, fmt.Errorf("tok: CountTokensExact: anthropic API returned %d: %s", resp.StatusCode, string(data))
+	}
+
+	var out struct {
+		InputTokens int `json:"input_tokens"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: decode response: %w", err)
+	}
+	return out.InputTokens, nil
+}
+
+func countTokensGoogle(ctx context.Context, apiKey, model, text string) (int, error) {
+	body, err := json.Marshal(map[string]any{
+		"contents": []map[string]any{
+			{"parts": []map[string]string{{"text": text}}},
+		},
+	})
+	if err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: marshal request: %w", err)
+	}
+
+	endpoint := fmt.Sprintf(googleCountTokensURL, url.PathEscape(model)) + "?key=" + url.QueryEscape(apiKey)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
+	if err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := exactCountClient.Do(req)
+	if err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: request failed: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		data, _ := io.ReadAll(resp.Body)
+		return 0, fmt.Errorf("tok: CountTokensExact: google API returned %d: %s", resp.StatusCode, string(data))
+	}
+
+	var out struct {
+		TotalTokens int `json:"totalTokens"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return 0, fmt.Errorf("tok: CountTokensExact: decode response: %w", err)
+	}
+	return out.TotalTokens, nil
+}

internal/core/exact_counter_test.go

@@ -0,0 +1,106 @@
+package core
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestCountTokensExact_RequiresAPIKey(t *testing.T) {
+	if _, err := CountTokensExact(context.Background(), "", "claude-3-5-sonnet", "hello"); err == nil {
+		t.Fatal("expected an error when apiKey is empty, got nil")
+	}
+}
+
+func TestCountTokensExact_UnsupportedModel(t *testing.T) {
+	if _, err := CountTokensExact(context.Background(), "sk-fake", "gpt-4o", "hello"); err == nil {
+		t.Fatal("expected an error for a non-claude/gemini model, got nil")
+	}
+}
+
+func TestCountTokensExact_Anthropic(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("x-api-key"); got != "sk-ant-fake" {
+			t.Errorf("x-api-key = %q, want %q", got, "sk-ant-fake")
+		}
+		if got := r.Header.Get("anthropic-version"); got == "" {
+			t.Error("anthropic-version header missing")
+		}
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatalf("decode request body: %v", err)
+		}
+		if body["model"] != "claude-3-5-sonnet-20241022" {
+			t.Errorf("model = %v, want claude-3-5-sonnet-20241022", body["model"])
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]int{"input_tokens": 7})
+	}))
+	defer srv.Close()
+
+	orig := anthropicCountTokensURL
+	anthropicCountTokensURL = srv.URL
+	defer func() { anthropicCountTokensURL = orig }()
+
+	got, err := CountTokensExact(context.Background(), "sk-ant-fake", "claude-3-5-sonnet-20241022", "hello world")
+	if err != nil {
+		t.Fatalf("CountTokensExact() error: %v", err)
+	}
+	if got != 7 {
+		t.Errorf("CountTokensExact() = %d, want 7", got)
+	}
+}
+
+func TestCountTokensExact_AnthropicError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusUnauthorized)
+		_, _ = w.Write([]byte(`{"error": "invalid api key"}`))
+	}))
+	defer srv.Close()
+
+	orig := anthropicCountTokensURL
+	anthropicCountTokensURL = srv.URL
+	defer func() { anthropicCountTokensURL = orig }()
+
+	if _, err := CountTokensExact(context.Background(), "bad-key", "claude-3-5-sonnet", "hi"); err == nil {
+		t.Fatal("expected an error for a 401 response, got nil")
+	} else if !strings.Contains(err.Error(), "401") {
+		t.Errorf("error = %v, want it to mention status 401", err)
+	}
+}
+
+func TestCountTokensExact_Google(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if got := r.URL.Query().Get("key"); got != "goog-fake" {
+			t.Errorf("key query param = %q, want %q", got, "goog-fake")
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]int{"totalTokens": 5})
+	}))
+	defer srv.Close()
+
+	orig := googleCountTokensURL
+	googleCountTokensURL = srv.URL + "/%s"
+	defer func() { googleCountTokensURL = orig }()
+
+	got, err := CountTokensExact(context.Background(), "goog-fake", "gemini-2.5-pro", "hello world")
+	if err != nil {
+		t.Fatalf("CountTokensExact() error: %v", err)
+	}
+	if got != 5 {
+		t.Errorf("CountTokensExact() = %d, want 5", got)
+	}
+}
+
+func TestCountTokensExact_NeverCalledByEstimateTokens(t *testing.T) {
+	// EstimateTokens and friends must remain fully offline. This test
+	// doesn't hit the network by construction (no http.Client involved),
+	// which is the point: there is no code path from EstimateTokens into
+	// CountTokensExact or exactCountClient.
+	if got := EstimateTokens("hello world"); got <= 0 {
+		t.Errorf("EstimateTokens() = %d, want > 0", got)
+	}
+}

internal/secrets/secrets.go

@@ -1,7 +1,9 @@
 package secrets
 
 import (
+	"context"
 	"math"
+	"net/http"
 	"regexp"
 	"sort"
 	"strings"
@@ -15,11 +17,22 @@ type SecretMatch struct {
 	Masked   string
 	StartPos int
 	EndPos   int
+
+	// Verified reports whether a live-credential check confirmed this
+	// secret is an active credential. nil means verification was not
+	// attempted — this is always the case for DetectSecrets, which never
+	// makes network calls. A non-nil value means DetectSecretsVerified
+	// attempted a live check for this match's type; verification errors
+	// (network failures, timeouts, etc.) also leave this nil rather than
+	// reporting a false negative.
+	Verified *bool
 }
 
 type secretPattern struct {
 	Type    string
 	Pattern *regexp.Regexp
+	// Verify is optional; nil means no live verifier exists for this type.
+	Verify Verifier
 }
 
 // SecretDetector detects and redacts secrets from text using compiled regex patterns.
@@ -93,7 +106,7 @@ func compilePatterns() []secretPattern {
 		if err != nil {
 			continue
 		}
-		patterns = append(patterns, secretPattern{Type: r.Type, Pattern: compiled})
+		patterns = append(patterns, secretPattern{Type: r.Type, Pattern: compiled, Verify: patternVerifiers[r.Type]})
 	}
 	return patterns
 }
@@ -138,6 +151,59 @@ func (sd *SecretDetector) DetectSecrets(text string) []SecretMatch {
 	return matches
 }
 
+// DetectSecretsVerified is like DetectSecrets, but additionally makes a live
+// network call per detected secret (where a verifier exists for its type)
+// to confirm whether the credential is actually an active one — following
+// TruffleHog's live-verification model. This is the only function in this
+// package that makes network calls; DetectSecrets and RedactSecrets never
+// do, and remain unchanged whether or not this function is ever called.
+//
+// client may be nil, in which case a default 5-second-timeout client is
+// used. AWS Access Keys are only verified when a paired AWS Secret Key was
+// also matched in the same text — an access key ID alone can't be signed,
+// so without a pair the match's Verified field stays nil, identical to
+// DetectSecrets.
+func (sd *SecretDetector) DetectSecretsVerified(ctx context.Context, client *http.Client, text string) []SecretMatch {
+	if client == nil {
+		client = defaultVerifyClient
+	}
+	matches := sd.DetectSecrets(text)
+
+	var awsSecretKey string
+	for _, m := range matches {
+		if m.Type == "AWS Secret Key" {
+			awsSecretKey = m.Value
+			break
+		}
+	}
+
+	for i := range matches {
+		m := &matches[i]
+		if m.Type == "AWS Access Key" {
+			if awsSecretKey == "" {
+				continue
+			}
+			ok, err := verifyAWSKeyPair(ctx, client, m.Value, awsSecretKey)
+			if err != nil {
+				continue
+			}
+			m.Verified = &ok
+			continue
+		}
+
+		verify := patternVerifiers[m.Type]
+		if verify == nil {
+			continue
+		}
+		ok, err := verify(ctx, client, m.Value)
+		if err != nil {
+			continue
+		}
+		m.Verified = &ok
+	}
+	return matches
+}
+
 // RedactSecrets replaces detected secrets with [REDACTED] markers.
 func (sd *SecretDetector) RedactSecrets(text string) string {
 	matches := sd.DetectSecrets(text)