From 26cdf6ebb19c4d267a35f497dd3edeb7b173abd3 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Thu, 20 Aug 2026 11:45:12 +0200 Subject: [PATCH 1/3] feat(llamacpp): surface model context window and modalities at /v1/models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama-server reports the context it is running with and its multimodal support, but the plain OpenAI decode dropped both, leaving local GGUFs with no metadata at all — they are not in the model catalog either. Context resolves strongest-first: meta.n_ctx (per-slot, per-model), then /props for builds predating it, then n_ctx_train as an upper-bound fallback. The trained context is deliberately last: llama-server's --ctx-size default sits far below it, so advertising it would overstate the real limit. Modes stay unset so the registry's ID heuristic keeps classifying local embedding and reranking models. --- docs/advanced/model-metadata.mdx | 8 +- docs/providers/llamacpp.mdx | 42 +++- internal/providers/llamacpp/llamacpp.go | 5 - internal/providers/llamacpp/models.go | 166 +++++++++++++++ internal/providers/llamacpp/models_test.go | 237 +++++++++++++++++++++ 5 files changed, 441 insertions(+), 17 deletions(-) create mode 100644 internal/providers/llamacpp/models.go create mode 100644 internal/providers/llamacpp/models_test.go diff --git a/docs/advanced/model-metadata.mdx b/docs/advanced/model-metadata.mdx index d21b0b1d9..a5dcaada5 100644 --- a/docs/advanced/model-metadata.mdx +++ b/docs/advanced/model-metadata.mdx @@ -39,9 +39,11 @@ flowchart LR 4. **Provider discovery** — some providers report capabilities in their own model listings, and GoModel keeps them for the models it discovers there: Gemini's `supportedGenerationMethods`, Cohere's per-model `endpoints` and - context length, OpenRouter's architecture modalities and context length, and - Ollama's `/api/show` capabilities. Models declared via configured model - lists skip this step. + context length, OpenRouter's architecture modalities and context length, + Chutes' context length, max output, capabilities, and pricing, Ollama's + `/api/show` capabilities, and llama.cpp's context window and modalities (see + [llama.cpp](/providers/llamacpp)). Models declared via configured model lists + skip this step. 5. **ID heuristic** — a last-resort name check for models that end up with no modes at all (typical for llama.cpp and LM Studio): IDs containing `embed` or matching well-known embedding families (`bge`, `e5`, `gte`, `minilm`) become diff --git a/docs/providers/llamacpp.mdx b/docs/providers/llamacpp.mdx index f8e61e485..6c197c838 100644 --- a/docs/providers/llamacpp.mdx +++ b/docs/providers/llamacpp.mdx @@ -75,17 +75,41 @@ curl -s http://localhost:8080/v1/embeddings \ -d '{"model": "llamacpp/nomic-embed-text-v1.5", "input": "hello"}' ``` +## Model metadata + +Local GGUFs are not in the upstream model catalog, so GoModel reads what it can +from llama-server itself and reports it on `GET /v1/models`: + +- **Context window** — the per-slot `n_ctx` llama-server reports for the model, + which is the context it is *running* with (`--ctx-size`, divided across + `--parallel` slots) rather than the one the GGUF was trained for. Recent + builds report it as `meta.n_ctx` in the listing itself; older ones are asked + for it via `/props`. If neither answers — LM Studio, or a proxy that hides + `/props` — the model's trained `n_ctx_train` is used instead, which is only an + upper bound and may exceed what the server will accept. +- **Modalities** — a multimodal server (started with `--mmproj`) reports + `vision`, `video`, and `audio` on `/props`; the supported ones become model + capabilities. + +`/props` describes the one model the server has loaded, so it is only consulted +for a single-entry listing. In [router +mode](https://github.com/ggml-org/llama.cpp/tree/master/tools/server), where one +process serves several models, each model carries its own `meta.n_ctx` and needs +no `/props` at all. + +Both are defaults, not decisions: anything you declare under the provider's +model metadata wins — see [Model metadata](/advanced/model-metadata). + ## Model classification -llama-server's `/v1/models` listing carries no capability metadata, so GoModel -classifies its models by ID: names containing `embed` or matching well-known -embedding families (`bge`, `e5`, `gte`, `minilm`) are categorized as embedding -models, and names containing `rerank` as reranking models — namespaced IDs are -checked by their final path segment. For anything that stays unclassified, -declare `modes` (and context window or pricing) under the provider's model -metadata — see [Model metadata](/advanced/model-metadata). Categories only -affect dashboard grouping and failover suggestions; `/v1/embeddings` routes to -any model the provider serves regardless of category. +llama-server reports nothing that separates a chat model from an embedding one, +so GoModel classifies its models by ID: names containing `embed` or matching +well-known embedding families (`bge`, `e5`, `gte`, `minilm`) are categorized as +embedding models, and names containing `rerank` as reranking models — namespaced +IDs are checked by their final path segment. For anything that stays +unclassified, declare `modes` under the provider's model metadata. Categories +only affect dashboard grouping and failover suggestions; `/v1/embeddings` routes +to any model the provider serves regardless of category. ## Beyond chat and embeddings diff --git a/internal/providers/llamacpp/llamacpp.go b/internal/providers/llamacpp/llamacpp.go index 256aadcdc..e0ede6768 100644 --- a/internal/providers/llamacpp/llamacpp.go +++ b/internal/providers/llamacpp/llamacpp.go @@ -108,11 +108,6 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque return p.compatible.StreamChatCompletion(ctx, req) } -// ListModels retrieves the list of available models from llama-server. -func (p *Provider) ListModels(ctx context.Context) (*core.ModelsResponse, error) { - return p.compatible.ListModels(ctx) -} - // Responses sends a Responses API request to llama-server. func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error) { return p.compatible.Responses(ctx, req) diff --git a/internal/providers/llamacpp/models.go b/internal/providers/llamacpp/models.go new file mode 100644 index 000000000..831451dd4 --- /dev/null +++ b/internal/providers/llamacpp/models.go @@ -0,0 +1,166 @@ +package llamacpp + +import ( + "context" + "net/http" + "strings" + + "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" +) + +// modelsResponse mirrors llama-server's /v1/models payload. It restates the +// OpenAI-compatible fields core.Model already carries because the response also +// holds a "meta" object describing the loaded GGUF, which the plain OpenAI +// shape drops — decoding it here keeps the llama.cpp-specific field in this +// package instead of in core. +type modelsResponse struct { + Object string `json:"object"` + Data []modelEntry `json:"data"` +} + +type modelEntry struct { + ID string `json:"id"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + Created int64 `json:"created"` + Meta *modelMeta `json:"meta"` +} + +// modelMeta is llama-server's description of the GGUF behind a model entry. +// n_ctx is the context the server is running with and n_ctx_train the one the +// model was trained for; the rest of the object (n_embd, n_params, size, +// n_vocab, vocab_type, ftype) stays undecoded rather than leaking llama.cpp +// internals into the OpenAI-compatible model shape. +type modelMeta struct { + NCtx int `json:"n_ctx"` + NCtxTrain int `json:"n_ctx_train"` +} + +// serverProps is the subset of llama-server's /props response we surface. +// default_generation_settings.n_ctx is the per-slot context the server was +// actually started with — the limit a request is measured against. +type serverProps struct { + DefaultGenerationSettings struct { + NCtx int `json:"n_ctx"` + } `json:"default_generation_settings"` + Modalities map[string]bool `json:"modalities"` +} + +// ListModels retrieves the list of available models from llama-server, keeping +// the context window and modalities the server reports for the model it has +// loaded so clients can size requests without a model-registry entry. +func (p *Provider) ListModels(ctx context.Context) (*core.ModelsResponse, error) { + var raw modelsResponse + if err := p.compatible.Do(ctx, llmclient.Request{ + Method: http.MethodGet, + Endpoint: "/models", + }, &raw); err != nil { + return nil, err + } + return raw.toCore(p.fetchServerProps(ctx, len(raw.Data))), nil +} + +// fetchServerProps reads the running server's own state, or nil when it is +// unavailable or cannot be attributed. /props describes the single model +// llama-server has loaded, so it is only meaningful for a single-entry listing; +// llama.cpp's router mode serves several models from one process, and each of +// those already carries its own meta.n_ctx. Failures are ignored — /props is +// enrichment, and servers that do not implement it (LM Studio answers 200 with +// an error body) must still list their models. +func (p *Provider) fetchServerProps(ctx context.Context, modelCount int) *serverProps { + if modelCount != 1 { + return nil + } + var props serverProps + if err := p.rootClient.Do(ctx, llmclient.Request{ + Method: http.MethodGet, + Endpoint: "/props", + }, &props); err != nil { + return nil + } + return &props +} + +func (r *modelsResponse) toCore(props *serverProps) *core.ModelsResponse { + object := strings.TrimSpace(r.Object) + if object == "" { + object = "list" + } + resp := &core.ModelsResponse{Object: object, Data: make([]core.Model, 0, len(r.Data))} + for _, entry := range r.Data { + resp.Data = append(resp.Data, entry.toCore(props)) + } + return resp +} + +func (e modelEntry) toCore(props *serverProps) core.Model { + object := strings.TrimSpace(e.Object) + if object == "" { + object = "model" + } + model := core.Model{ + ID: strings.TrimSpace(e.ID), + Object: object, + OwnedBy: strings.TrimSpace(e.OwnedBy), + Created: e.Created, + } + + // Modes stay unset on purpose. llama-server reports nothing that separates a + // chat model from an embedding one, so leaving them empty keeps the + // registry's ID heuristic free to classify local GGUFs. + metadata := core.ModelMetadata{} + if contextWindow := e.contextWindow(props); contextWindow > 0 { + metadata.ContextWindow = &contextWindow + } + if props != nil { + metadata.Capabilities = modalityCapabilities(props.Modalities) + } + if metadata.ContextWindow == nil && len(metadata.Capabilities) == 0 { + // Nothing was reported; leaving Metadata nil keeps the model catalog and + // the ID heuristic free to supply everything. + return model + } + model.Metadata = &metadata + return model +} + +// contextWindow resolves the limit a request is actually measured against, +// strongest source first: +// +// - meta.n_ctx — the per-slot context the server is running with, reported +// per model, so it stays correct when one process serves several. +// - /props — the same number, for builds whose listing predates meta.n_ctx. +// - meta.n_ctx_train — the GGUF's trained ceiling. Only an upper bound: +// llama-server's --ctx-size default sits far below it for most models, so +// it is a last resort rather than the headline figure. +func (e modelEntry) contextWindow(props *serverProps) int { + if e.Meta != nil && e.Meta.NCtx > 0 { + return e.Meta.NCtx + } + if props != nil && props.DefaultGenerationSettings.NCtx > 0 { + return props.DefaultGenerationSettings.NCtx + } + if e.Meta != nil { + return e.Meta.NCtxTrain + } + return 0 +} + +// modalityCapabilities maps llama-server's multimodal flags onto GoModel +// capability keys. Unsupported modalities are omitted rather than recorded as +// false, so a later metadata layer can still claim them. +func modalityCapabilities(modalities map[string]bool) map[string]bool { + capabilities := make(map[string]bool, len(modalities)) + for modality, supported := range modalities { + name := strings.ToLower(strings.TrimSpace(modality)) + if !supported || name == "" { + continue + } + capabilities[name] = true + } + if len(capabilities) == 0 { + return nil + } + return capabilities +} diff --git a/internal/providers/llamacpp/models_test.go b/internal/providers/llamacpp/models_test.go new file mode 100644 index 000000000..b69572142 --- /dev/null +++ b/internal/providers/llamacpp/models_test.go @@ -0,0 +1,237 @@ +package llamacpp + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/enterpilot/gomodel/internal/llmclient" +) + +// legacyListing is the /v1/models payload of builds whose meta object predates +// n_ctx, leaving only the GGUF's trained context. +const legacyListing = `{ + "object":"list", + "data":[{ + "id":"Meta-Llama-3.1-8B-Instruct", + "object":"model", + "created":1735142223, + "owned_by":"llamacpp", + "meta":{"vocab_type":2,"n_vocab":128256,"n_ctx_train":131072,"n_embd":4096,"n_params":8030261312,"size":4912898304} + }] +}` + +func TestListModels_SurfacesServerReportedMetadata(t *testing.T) { + // realListing is a verbatim llama-server b10470 payload (gemma-3-270m-it + // started with -c 2048), whose meta carries the running context as n_ctx. + const realListing = `{ + "object":"list", + "data":[{ + "id":"gemma-3-270m-it", + "object":"model", + "created":1787218858, + "owned_by":"llamacpp", + "meta":{"vocab_type":1,"n_vocab":262144,"n_ctx":2048,"n_ctx_train":32768,"n_embd":640,"n_params":268098176,"size":285018624,"ftype":"Q8_0"} + }] + }` + + tests := []struct { + name string + listing string + propsStatus int + props string + wantContextWindow int + wantModelID string + wantCapabilities map[string]bool + wantPropsFetched bool + }{ + { + // meta.n_ctx is per-model, so it must win even over a /props that + // disagrees — in router mode /props cannot be attributed at all. + name: "listing n_ctx wins over props and trained context", + listing: realListing, + propsStatus: http.StatusOK, + props: `{"default_generation_settings":{"n_ctx":9999},"modalities":{"vision":false,"video":false,"audio":false}}`, + wantContextWindow: 2048, + wantModelID: "gemma-3-270m-it", + wantPropsFetched: true, + }, + { + name: "props runtime context wins over trained context on older builds", + listing: legacyListing, + propsStatus: http.StatusOK, + props: `{"default_generation_settings":{"n_ctx":8192},"total_slots":1,"modalities":{"vision":false}}`, + wantContextWindow: 8192, + wantPropsFetched: true, + }, + { + name: "trained context is the fallback when props is unavailable", + listing: legacyListing, + propsStatus: http.StatusNotFound, + props: `{"error":"not found"}`, + wantContextWindow: 131072, + wantPropsFetched: true, + }, + { + // LM Studio answers /props with 200 and an error body rather than a + // 404, so a decodable-but-empty payload must not be mistaken for a + // server reporting a zero context. + name: "props answered with an unrelated 200 payload", + listing: legacyListing, + propsStatus: http.StatusOK, + props: `{"error":"Unexpected endpoint or method. (GET /props)"}`, + wantContextWindow: 131072, + wantPropsFetched: true, + }, + { + name: "supported modalities become capabilities", + listing: legacyListing, + propsStatus: http.StatusOK, + props: `{"default_generation_settings":{"n_ctx":4096},"modalities":{"vision":true,"video":true,"audio":false}}`, + wantContextWindow: 4096, + wantCapabilities: map[string]bool{"vision": true, "video": true}, + wantPropsFetched: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + propsFetched := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(tt.listing)) + case "/props": + propsFetched = true + w.WriteHeader(tt.propsStatus) + _, _ = w.Write([]byte(tt.props)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", server.Client(), llmclient.Hooks{}) + + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() error = %v", err) + } + if len(resp.Data) != 1 { + t.Fatalf("len(resp.Data) = %d, want 1", len(resp.Data)) + } + if propsFetched != tt.wantPropsFetched { + t.Fatalf("props fetched = %v, want %v", propsFetched, tt.wantPropsFetched) + } + + model := resp.Data[0] + wantID := tt.wantModelID + if wantID == "" { + wantID = "Meta-Llama-3.1-8B-Instruct" + } + if model.ID != wantID { + t.Fatalf("model.ID = %q, want %q", model.ID, wantID) + } + if model.Metadata == nil { + t.Fatalf("model.Metadata = nil, want context window %d", tt.wantContextWindow) + } + if model.Metadata.ContextWindow == nil || *model.Metadata.ContextWindow != tt.wantContextWindow { + t.Fatalf("context window = %v, want %d", model.Metadata.ContextWindow, tt.wantContextWindow) + } + if len(model.Metadata.Capabilities) != len(tt.wantCapabilities) { + t.Fatalf("capabilities = %v, want %v", model.Metadata.Capabilities, tt.wantCapabilities) + } + for name, want := range tt.wantCapabilities { + if model.Metadata.Capabilities[name] != want { + t.Fatalf("capability %q = %v, want %v", name, model.Metadata.Capabilities[name], want) + } + } + // Modes must stay empty so the registry's ID heuristic can still + // classify local embedding and reranking GGUFs. + if len(model.Metadata.Modes) != 0 { + t.Fatalf("modes = %v, want none", model.Metadata.Modes) + } + }) + } +} + +func TestListModels_RouterModeKeepsPerModelContextAndSkipsProps(t *testing.T) { + propsFetched := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(`{ + "object":"list", + "data":[ + {"id":"gemma-3-4b","object":"model","meta":{"n_ctx":8192,"n_ctx_train":131072}}, + {"id":"qwen3-8b","object":"model","meta":{"n_ctx":32768,"n_ctx_train":262144}} + ] + }`)) + case "/props": + propsFetched = true + _, _ = w.Write([]byte(`{"default_generation_settings":{"n_ctx":512}}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", server.Client(), llmclient.Hooks{}) + + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() error = %v", err) + } + if propsFetched { + t.Fatal("props was fetched for a multi-model listing; it describes a single loaded model") + } + + want := map[string]int{"gemma-3-4b": 8192, "qwen3-8b": 32768} + for _, model := range resp.Data { + if model.Metadata == nil || model.Metadata.ContextWindow == nil { + t.Fatalf("model %q lost its context window", model.ID) + } + if got := *model.Metadata.ContextWindow; got != want[model.ID] { + t.Fatalf("model %q context window = %d, want %d", model.ID, got, want[model.ID]) + } + } +} + +func TestListModels_LeavesMetadataUnsetWhenServerReportsNothing(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + // LM Studio and other plain OpenAI-compatible servers omit "meta". + _, _ = w.Write([]byte(`{"data":[{"id":"local-model"}]}`)) + case "/props": + _, _ = w.Write([]byte(`{"default_generation_settings":{"n_ctx":0}}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + provider := NewWithHTTPClient("", server.URL+"/v1", server.Client(), llmclient.Hooks{}) + + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() error = %v", err) + } + if len(resp.Data) != 1 { + t.Fatalf("len(resp.Data) = %d, want 1", len(resp.Data)) + } + if resp.Object != "list" { + t.Fatalf("resp.Object = %q, want list", resp.Object) + } + if resp.Data[0].Object != "model" { + t.Fatalf("model.Object = %q, want model", resp.Data[0].Object) + } + if resp.Data[0].Metadata != nil { + t.Fatalf("model.Metadata = %+v, want nil so lower metadata layers still apply", resp.Data[0].Metadata) + } +} From e947e095a1f276c6aecab2decbd544e60decbb01 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Thu, 20 Aug 2026 12:08:11 +0200 Subject: [PATCH 2/3] fix(modeldata): merge catalog metadata under provider-reported metadata Enrich replaced a model's metadata wholesale whenever the catalog knew its ID, discarding what the provider reported about its own deployment. The catalog lookup falls back to a provider-agnostic ID match, so a local llama.cpp alias colliding with a catalog entry lost its real context window and gained one the server would reject. The catalog is now the base and the provider's report the override, merged field-wise, so catalog-only fields (display names, pricing, rankings) still land while the provider wins on what it actually knows. Merging onto the model's own previous output would pin stale catalog values across refreshes, so ModelInfo keeps the provider's pristine report in Discovered and every pass recomputes from it. All ModelInfo construction now goes through newModelInfo so that value cannot be missed. --- docs/advanced/model-metadata.mdx | 29 ++++---- internal/modeldata/enricher.go | 26 +++++-- internal/modeldata/enricher_test.go | 65 ++++++++++++++++++ internal/providers/configured_models.go | 7 +- internal/providers/registry.go | 19 ++++++ internal/providers/registry_cache.go | 19 +++--- internal/providers/registry_init.go | 7 +- internal/providers/registry_metadata.go | 9 +++ .../registry_metadata_override_test.go | 68 +++++++++++++++++++ 9 files changed, 207 insertions(+), 42 deletions(-) diff --git a/docs/advanced/model-metadata.mdx b/docs/advanced/model-metadata.mdx index a5dcaada5..51645b38b 100644 --- a/docs/advanced/model-metadata.mdx +++ b/docs/advanced/model-metadata.mdx @@ -13,8 +13,8 @@ strongest first: ```mermaid flowchart LR A[Pricing overrides] -->|pricing fields only| B[config.yaml metadata] - B -->|declared fields win| C[ai-model-list catalog] - C -->|when it knows the model| D[Provider discovery] + B -->|declared fields win| C[Provider discovery] + C -->|fields the provider reports| D[ai-model-list catalog] D -.->|modes still missing| E[ID heuristic] ``` @@ -28,22 +28,25 @@ flowchart LR fields win field-by-field over everything below; omitted fields inherit. This is the escape hatch for local models: declaring `modes: [embedding]` also derives the model's category. -3. **The model catalog** — the +3. **Provider discovery** — some providers report metadata in their own model + listings, and what a provider says about its own deployment wins over the + catalog field by field: Gemini's `supportedGenerationMethods`, Cohere's + per-model `endpoints` and context length, OpenRouter's architecture + modalities and context length, Chutes' context length, max output, + capabilities, and pricing, Ollama's `/api/show` capabilities, and llama.cpp's + context window and modalities (see [llama.cpp](/providers/llamacpp)). This + matters most for self-hosted servers, where the running process is the only + source that can know the real context window. Models declared via configured + model lists skip this step. +4. **The model catalog** — the [`ai-model-list`](https://github.com/ENTERPILOT/ai-model-list) registry, fetched from `MODEL_LIST_URL` (default: the registry's `models.min.json` on GitHub) at startup and on every catalog refresh. It supplies the rich defaults — pricing, context windows, capabilities, modes — for most hosted models, matching IDs directly, through aliases, and with release-date - suffixes stripped. Wrong or missing data is best fixed by contributing to the - registry; use an override for an immediate fix. -4. **Provider discovery** — some providers report capabilities in their own - model listings, and GoModel keeps them for the models it discovers there: - Gemini's `supportedGenerationMethods`, Cohere's per-model `endpoints` and - context length, OpenRouter's architecture modalities and context length, - Chutes' context length, max output, capabilities, and pricing, Ollama's - `/api/show` capabilities, and llama.cpp's context window and modalities (see - [llama.cpp](/providers/llamacpp)). Models declared via configured model lists - skip this step. + suffixes stripped, and it fills in every field the provider above did not + report. Wrong or missing data is best fixed by contributing to the registry; + use an override for an immediate fix. 5. **ID heuristic** — a last-resort name check for models that end up with no modes at all (typical for llama.cpp and LM Studio): IDs containing `embed` or matching well-known embedding families (`bge`, `e5`, `gte`, `minilm`) become diff --git a/internal/modeldata/enricher.go b/internal/modeldata/enricher.go index 65dc68860..4454c8d4a 100644 --- a/internal/modeldata/enricher.go +++ b/internal/modeldata/enricher.go @@ -14,6 +14,11 @@ type ModelInfoAccessor interface { GetProviderType(modelID string) string // SetMetadata sets the metadata for a model ID. SetMetadata(modelID string, meta *core.ModelMetadata) + // DiscoveredMetadata returns the metadata the provider itself reported for + // a model, or nil when it reported none. It must keep returning the same + // value across enrichment passes: Enrich merges onto it rather than onto + // its own previous result, which is what keeps repeated passes idempotent. + DiscoveredMetadata(modelID string) *core.ModelMetadata } // EnrichStats summarizes one metadata enrichment pass. @@ -22,8 +27,15 @@ type EnrichStats struct { Total int } -// Enrich iterates all models accessible via the accessor and attaches resolved -// metadata from the model list. Models not found in the list are left unchanged. +// Enrich iterates all models accessible via the accessor and merges resolved +// catalog metadata into each one. Models the catalog does not know are left +// unchanged. +// +// The catalog is the base and the provider's own report the override, field by +// field: a running provider describes its actual deployment (a local server's +// real context window, an API's live capability flags) better than a static +// registry can, while the catalog still supplies everything the provider never +// reports — display names, pricing, rankings. func Enrich(accessor ModelInfoAccessor, list *ModelList) EnrichStats { if list == nil || accessor == nil { return EnrichStats{} @@ -34,11 +46,13 @@ func Enrich(accessor ModelInfoAccessor, list *ModelList) EnrichStats { for _, modelID := range ids { providerType := accessor.GetProviderType(modelID) - meta := Resolve(list, providerType, modelID) - if meta != nil { - accessor.SetMetadata(modelID, meta) - enriched++ + catalog := Resolve(list, providerType, modelID) + if catalog == nil { + // Nothing in the catalog: whatever the provider discovered stands. + continue } + accessor.SetMetadata(modelID, MergeMetadata(catalog, accessor.DiscoveredMetadata(modelID))) + enriched++ } return EnrichStats{ diff --git a/internal/modeldata/enricher_test.go b/internal/modeldata/enricher_test.go index 1d07d41b4..9cec79bf4 100644 --- a/internal/modeldata/enricher_test.go +++ b/internal/modeldata/enricher_test.go @@ -11,12 +11,14 @@ type mockAccessor struct { ids []string providerTypes map[string]string metadata map[string]*core.ModelMetadata + discovered map[string]*core.ModelMetadata } func newMockAccessor(models map[string]string) *mockAccessor { a := &mockAccessor{ providerTypes: models, metadata: make(map[string]*core.ModelMetadata), + discovered: make(map[string]*core.ModelMetadata), } for id := range models { a.ids = append(a.ids, id) @@ -30,6 +32,10 @@ func (a *mockAccessor) SetMetadata(modelID string, meta *core.ModelMetadata) { a.metadata[modelID] = meta } +func (a *mockAccessor) DiscoveredMetadata(modelID string) *core.ModelMetadata { + return a.discovered[modelID] +} + func TestEnrich_MatchedAndUnmatched(t *testing.T) { list := &ModelList{ Models: map[string]ModelEntry{ @@ -165,3 +171,62 @@ func TestEnrich_ProviderModelOverride(t *testing.T) { t.Errorf("ContextWindow = %d, want 64000 (azure override)", *meta.ContextWindow) } } + +func TestEnrich_ProviderDiscoveryWinsFieldWiseOverCatalog(t *testing.T) { + accessor := newMockAccessor(map[string]string{"gemma-3-4b-it": "llamacpp"}) + // What a local server reported about the model it is actually running. + accessor.discovered["gemma-3-4b-it"] = &core.ModelMetadata{ + ContextWindow: new(4096), + Capabilities: map[string]bool{"vision": true}, + } + // A catalog entry that knows the bare ID, with no llamacpp entry at all. + list := &ModelList{Models: map[string]ModelEntry{ + "gemma-3-4b-it": { + DisplayName: "Gemma 3 4B IT", + ContextWindow: new(131072), + Capabilities: map[string]bool{"tools": true}, + }, + }} + + Enrich(accessor, list) + + got := accessor.metadata["gemma-3-4b-it"] + if got == nil { + t.Fatal("metadata = nil, want merged metadata") + } + if got.ContextWindow == nil || *got.ContextWindow != 4096 { + t.Fatalf("context window = %v, want the running server's 4096", got.ContextWindow) + } + if !got.Capabilities["vision"] { + t.Fatal("discovered capability vision was dropped") + } + // Fields the provider never reports still come from the catalog. + if !got.Capabilities["tools"] { + t.Fatal("catalog capability tools was dropped") + } + if got.DisplayName != "Gemma 3 4B IT" { + t.Fatalf("display name = %q, want the catalog's", got.DisplayName) + } +} + +func TestEnrich_RepeatedPassesTrackCatalogUpdates(t *testing.T) { + accessor := newMockAccessor(map[string]string{"gpt-4o": "openai"}) + list := &ModelList{Models: map[string]ModelEntry{ + "gpt-4o": {DisplayName: "GPT-4o", ContextWindow: new(128000)}, + }} + + Enrich(accessor, list) + if got := accessor.metadata["gpt-4o"]; got.ContextWindow == nil || *got.ContextWindow != 128000 { + t.Fatalf("first pass context window = %v, want 128000", got.ContextWindow) + } + + // A later catalog refresh corrects the value. Because Enrich merges onto the + // provider's pristine report rather than onto its own previous output, the + // new value must win instead of being pinned by the stale one. + list.Models["gpt-4o"] = ModelEntry{DisplayName: "GPT-4o", ContextWindow: new(200000)} + Enrich(accessor, list) + + if got := accessor.metadata["gpt-4o"]; got.ContextWindow == nil || *got.ContextWindow != 200000 { + t.Fatalf("second pass context window = %v, want the refreshed 200000", got.ContextWindow) + } +} diff --git a/internal/providers/configured_models.go b/internal/providers/configured_models.go index 8bb2723f6..1740f2785 100644 --- a/internal/providers/configured_models.go +++ b/internal/providers/configured_models.go @@ -160,12 +160,7 @@ func modelInfoMapFromResponse(resp *core.ModelsResponse, provider core.Provider, continue } model.ID = modelID - out[modelID] = &ModelInfo{ - Model: model, - Provider: provider, - ProviderName: providerName, - ProviderType: providerType, - } + out[modelID] = newModelInfo(model, provider, providerName, providerType) } return out } diff --git a/internal/providers/registry.go b/internal/providers/registry.go index 48215ea0a..6507db5ae 100644 --- a/internal/providers/registry.go +++ b/internal/providers/registry.go @@ -23,6 +23,25 @@ type ModelInfo struct { Provider core.Provider ProviderName string ProviderType string + // Discovered is the metadata the provider itself reported for this model, + // held apart from Model.Metadata because enrichment rewrites that field on + // every catalog refresh. Keeping the pristine value lets each refresh + // recompute the merge from the same inputs rather than layering onto its own + // previous output, which would otherwise pin stale catalog data in place. + Discovered *core.ModelMetadata +} + +// newModelInfo registers a model together with the metadata its provider +// reported for it. Always build ModelInfo through this so Discovered is +// captured before enrichment has a chance to overwrite Model.Metadata. +func newModelInfo(model core.Model, provider core.Provider, providerName, providerType string) *ModelInfo { + return &ModelInfo{ + Model: model, + Provider: provider, + ProviderName: providerName, + ProviderType: providerType, + Discovered: model.Metadata.Clone(), + } } // ModelRegistry manages the mapping of models to their providers. diff --git a/internal/providers/registry_cache.go b/internal/providers/registry_cache.go index b12f29696..c83fadee6 100644 --- a/internal/providers/registry_cache.go +++ b/internal/providers/registry_cache.go @@ -69,17 +69,14 @@ func (r *ModelRegistry) LoadFromCache(ctx context.Context) (int, error) { } providerModels := make(map[string]*ModelInfo, len(cachedProv.Models)) for _, cached := range cachedProv.Models { - info := &ModelInfo{ - Model: core.Model{ - ID: cached.ID, - Object: "model", - OwnedBy: cachedProv.OwnedBy, - Created: cached.Created, - }, - Provider: provider, - ProviderName: providerName, - ProviderType: providerType, - } + // The cache stores no metadata, so nothing was discovered yet; the + // next live refresh supplies it. + info := newModelInfo(core.Model{ + ID: cached.ID, + Object: "model", + OwnedBy: cachedProv.OwnedBy, + Created: cached.Created, + }, provider, providerName, providerType) providerModels[cached.ID] = info if _, exists := newModels[cached.ID]; !exists { newModels[cached.ID] = info diff --git a/internal/providers/registry_init.go b/internal/providers/registry_init.go index 570eeae95..6bfba85c8 100644 --- a/internal/providers/registry_init.go +++ b/internal/providers/registry_init.go @@ -260,12 +260,7 @@ func (r *ModelRegistry) fetchAllProviderModels( } for _, model := range resp.Data { - info := &ModelInfo{ - Model: model, - Provider: provider, - ProviderName: providerName, - ProviderType: providerTypes[provider], - } + info := newModelInfo(model, provider, providerName, providerTypes[provider]) out.modelsByProvider[providerName][model.ID] = info if _, exists := out.models[model.ID]; exists { diff --git a/internal/providers/registry_metadata.go b/internal/providers/registry_metadata.go index 5cfd730e1..8e2139db1 100644 --- a/internal/providers/registry_metadata.go +++ b/internal/providers/registry_metadata.go @@ -487,3 +487,12 @@ func (a *registryAccessor) SetMetadata(modelID string, meta *core.ModelMetadata) info.Model.Metadata = meta } } + +// DiscoveredMetadata returns the metadata the provider reported for a model at +// registration, which Enrich merges on top of the catalog's. +func (a *registryAccessor) DiscoveredMetadata(modelID string) *core.ModelMetadata { + if info, ok := a.models[modelID]; ok { + return info.Discovered + } + return nil +} diff --git a/internal/providers/registry_metadata_override_test.go b/internal/providers/registry_metadata_override_test.go index 358d4cfe6..5aaaca764 100644 --- a/internal/providers/registry_metadata_override_test.go +++ b/internal/providers/registry_metadata_override_test.go @@ -455,3 +455,71 @@ func TestSetProviderMetadataOverrides_DeepClonesExternalInput(t *testing.T) { t.Errorf("stored Pricing mutated via caller: %+v", stored.Pricing) } } + +// TestEnrichModels_KeepsProviderDiscoveredMetadataOverCatalog covers the local +// server case: a llama.cpp alias that collides with a catalog ID must not lose +// the context window the running server actually reported. +func TestEnrichModels_KeepsProviderDiscoveredMetadataOverCatalog(t *testing.T) { + registry := NewModelRegistry() + + runtimeContext := 4096 + mock := ®istryMockProvider{ + name: "llamacpp", + modelsResponse: &core.ModelsResponse{ + Object: "list", + Data: []core.Model{ + { + ID: "gemma-3-4b-it", + Object: "model", + OwnedBy: "llamacpp", + Metadata: &core.ModelMetadata{ + ContextWindow: &runtimeContext, + Capabilities: map[string]bool{"vision": true}, + }, + }, + }, + }, + } + registry.RegisterProviderWithType(mock, "llamacpp") + if err := registry.Initialize(context.Background()); err != nil { + t.Fatalf("Initialize() error = %v", err) + } + + // The catalog knows the bare model ID and has no llamacpp entry at all. + raw := []byte(`{ + "version": 1, + "updated_at": "2025-01-01T00:00:00Z", + "models": { + "gemma-3-4b-it": { + "display_name": "Gemma 3 4B IT", + "modes": ["chat"], + "context_window": 131072 + } + } + }`) + list, err := modeldata.Parse(raw) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + registry.SetModelList(list, raw) + registry.EnrichModels() + + info := registry.GetModel("gemma-3-4b-it") + if info == nil || info.Model.Metadata == nil { + t.Fatal("expected the model to stay registered with metadata") + } + meta := info.Model.Metadata + if meta.ContextWindow == nil || *meta.ContextWindow != runtimeContext { + t.Fatalf("ContextWindow = %v, want the server-reported %d", meta.ContextWindow, runtimeContext) + } + if !meta.Capabilities["vision"] { + t.Fatal("discovered vision capability was dropped by enrichment") + } + // The catalog still fills in what the server never reports. + if meta.DisplayName != "Gemma 3 4B IT" { + t.Fatalf("DisplayName = %q, want the catalog's", meta.DisplayName) + } + if len(meta.Modes) != 1 || meta.Modes[0] != "chat" { + t.Fatalf("Modes = %v, want [chat] from the catalog", meta.Modes) + } +} From c2e8c2c1462b9a91c144edd44a6872e520b80b97 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Thu, 20 Aug 2026 12:28:16 +0200 Subject: [PATCH 3/3] fix(llamacpp): isolate optional /props call from the native-route budget /props enrichment shared rootClient with native passthrough, so a server answering it with a retryable status spent that client's retry budget and tripped its circuit breaker: six discovery cycles against a 503 produced 20 upstream attempts and then rejected /health locally. It now uses a dedicated client with no retries and no breaker, bounded by a short timeout so a non-answering server cannot stall discovery. Also drops unrecognized /props modalities instead of publishing them as capabilities, and resets a model to its provider-reported metadata when a catalog refresh stops matching it, so catalog-only fields do not linger. --- internal/modeldata/enricher.go | 8 ++- internal/modeldata/enricher_test.go | 42 ++++++++++-- internal/providers/llamacpp/llamacpp.go | 28 ++++++++ internal/providers/llamacpp/models.go | 25 +++++-- internal/providers/llamacpp/models_test.go | 80 +++++++++++++++++++++- 5 files changed, 167 insertions(+), 16 deletions(-) diff --git a/internal/modeldata/enricher.go b/internal/modeldata/enricher.go index 4454c8d4a..55118ad90 100644 --- a/internal/modeldata/enricher.go +++ b/internal/modeldata/enricher.go @@ -46,12 +46,16 @@ func Enrich(accessor ModelInfoAccessor, list *ModelList) EnrichStats { for _, modelID := range ids { providerType := accessor.GetProviderType(modelID) + discovered := accessor.DiscoveredMetadata(modelID) catalog := Resolve(list, providerType, modelID) if catalog == nil { - // Nothing in the catalog: whatever the provider discovered stands. + // The catalog does not know this model. Fall back to the provider's + // own report rather than leaving whatever a previous pass merged in: + // a refresh that drops an entry must drop its fields too. + accessor.SetMetadata(modelID, discovered.Clone()) continue } - accessor.SetMetadata(modelID, MergeMetadata(catalog, accessor.DiscoveredMetadata(modelID))) + accessor.SetMetadata(modelID, MergeMetadata(catalog, discovered)) enriched++ } diff --git a/internal/modeldata/enricher_test.go b/internal/modeldata/enricher_test.go index 9cec79bf4..c6a913f1a 100644 --- a/internal/modeldata/enricher_test.go +++ b/internal/modeldata/enricher_test.go @@ -70,14 +70,13 @@ func TestEnrich_MatchedAndUnmatched(t *testing.T) { } } - // unknown-model should NOT be enriched - if _, ok := accessor.metadata["unknown-model"]; ok { - t.Error("expected unknown-model to NOT be enriched") + // Models the catalog does not know keep only what their provider reported, + // which here is nothing. + if meta := accessor.metadata["unknown-model"]; meta != nil { + t.Errorf("expected unknown-model to carry no catalog metadata, got %+v", meta) } - - // custom-finetune should NOT be enriched - if _, ok := accessor.metadata["custom-finetune"]; ok { - t.Error("expected custom-finetune to NOT be enriched") + if meta := accessor.metadata["custom-finetune"]; meta != nil { + t.Errorf("expected custom-finetune to carry no catalog metadata, got %+v", meta) } } @@ -230,3 +229,32 @@ func TestEnrich_RepeatedPassesTrackCatalogUpdates(t *testing.T) { t.Fatalf("second pass context window = %v, want the refreshed 200000", got.ContextWindow) } } + +func TestEnrich_DropsCatalogFieldsWhenEntryDisappears(t *testing.T) { + accessor := newMockAccessor(map[string]string{"gemma-3-4b-it": "llamacpp"}) + accessor.discovered["gemma-3-4b-it"] = &core.ModelMetadata{ContextWindow: new(4096)} + list := &ModelList{Models: map[string]ModelEntry{ + "gemma-3-4b-it": {DisplayName: "Gemma 3 4B IT", ContextWindow: new(131072)}, + }} + + Enrich(accessor, list) + if got := accessor.metadata["gemma-3-4b-it"]; got.DisplayName != "Gemma 3 4B IT" { + t.Fatalf("first pass display name = %q, want the catalog's", got.DisplayName) + } + + // The catalog drops the entry on a later refresh; its fields must go with + // it, leaving only what the provider itself reported. + delete(list.Models, "gemma-3-4b-it") + Enrich(accessor, list) + + got := accessor.metadata["gemma-3-4b-it"] + if got == nil { + t.Fatal("metadata = nil, want the provider's own report to survive") + } + if got.DisplayName != "" { + t.Fatalf("display name = %q, want it dropped with the catalog entry", got.DisplayName) + } + if got.ContextWindow == nil || *got.ContextWindow != 4096 { + t.Fatalf("context window = %v, want the provider's 4096", got.ContextWindow) + } +} diff --git a/internal/providers/llamacpp/llamacpp.go b/internal/providers/llamacpp/llamacpp.go index e0ede6768..a6f6c9be4 100644 --- a/internal/providers/llamacpp/llamacpp.go +++ b/internal/providers/llamacpp/llamacpp.go @@ -33,6 +33,12 @@ var Registration = providers.Registration{ type Provider struct { compatible *openai.CompatibleProvider rootClient *llmclient.Client + // propsClient issues the optional /props metadata call. It is deliberately + // separate from rootClient: /props is best-effort enrichment, so it must not + // spend the retry budget or trip the circuit breaker that native passthrough + // routes share. A server that fails /props with a retryable status would + // otherwise take /health, /rerank and /tokenize down with it. + propsClient *llmclient.Client } var ( @@ -63,9 +69,23 @@ func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Prov }, func(req *http.Request) { setHeaders(req, keys.NextForContext(req.Context())) }), + propsClient: newPropsClient(baseURL, opts.Hooks, func(req *http.Request) { + setHeaders(req, keys.NextForContext(req.Context())) + }), } } +// newPropsClient builds the client used for optional /props enrichment: no +// retries and no circuit breaker, so a failing /props costs one request and +// leaves the shared native-route budget untouched. +func newPropsClient(baseURL string, hooks llmclient.Hooks, setHeader llmclient.HeaderSetter) *llmclient.Client { + return llmclient.New(llmclient.Config{ + ProviderName: "llamacpp", + BaseURL: passthroughBaseURL(baseURL), + Hooks: hooks, + }, setHeader) +} + // NewWithHTTPClient creates a new llama.cpp provider with a custom HTTP client. // If httpClient is nil, http.DefaultClient is used. func NewWithHTTPClient(apiKey string, baseURL string, httpClient *http.Client, hooks llmclient.Hooks) *Provider { @@ -81,6 +101,13 @@ func NewWithHTTPClient(apiKey string, baseURL string, httpClient *http.Client, h rootClient: llmclient.NewWithHTTPClient(httpClient, rootClientCfg, func(req *http.Request) { setHeaders(req, apiKey) }), + propsClient: llmclient.NewWithHTTPClient(httpClient, llmclient.Config{ + ProviderName: "llamacpp", + BaseURL: passthroughBaseURL(resolvedBaseURL), + Hooks: hooks, + }, func(req *http.Request) { + setHeaders(req, apiKey) + }), } } @@ -88,6 +115,7 @@ func NewWithHTTPClient(apiKey string, baseURL string, httpClient *http.Client, h func (p *Provider) SetBaseURL(url string) { p.compatible.SetBaseURL(url) p.rootClient.SetBaseURL(passthroughBaseURL(url)) + p.propsClient.SetBaseURL(passthroughBaseURL(url)) } func setHeaders(req *http.Request, apiKey string) { diff --git a/internal/providers/llamacpp/models.go b/internal/providers/llamacpp/models.go index 831451dd4..30c1d0b5c 100644 --- a/internal/providers/llamacpp/models.go +++ b/internal/providers/llamacpp/models.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "strings" + "time" "github.com/enterpilot/gomodel/internal/core" "github.com/enterpilot/gomodel/internal/llmclient" @@ -40,6 +41,9 @@ type modelMeta struct { // serverProps is the subset of llama-server's /props response we surface. // default_generation_settings.n_ctx is the per-slot context the server was // actually started with — the limit a request is measured against. +// propsTimeout bounds the optional /props enrichment call. +const propsTimeout = 5 * time.Second + type serverProps struct { DefaultGenerationSettings struct { NCtx int `json:"n_ctx"` @@ -72,8 +76,13 @@ func (p *Provider) fetchServerProps(ctx context.Context, modelCount int) *server if modelCount != 1 { return nil } + // Bounded so a server that accepts the connection but never answers cannot + // stall discovery on an optional call. + ctx, cancel := context.WithTimeout(ctx, propsTimeout) + defer cancel() + var props serverProps - if err := p.rootClient.Do(ctx, llmclient.Request{ + if err := p.propsClient.Do(ctx, llmclient.Request{ Method: http.MethodGet, Endpoint: "/props", }, &props); err != nil { @@ -148,16 +157,22 @@ func (e modelEntry) contextWindow(props *serverProps) int { } // modalityCapabilities maps llama-server's multimodal flags onto GoModel -// capability keys. Unsupported modalities are omitted rather than recorded as -// false, so a later metadata layer can still claim them. +// capability keys. Only the modalities that have an established capability name +// are published — an unrecognized key llama.cpp adds later would otherwise +// become a public capability nobody can interpret. Unsupported modalities are +// omitted rather than recorded as false, so a later metadata layer can still +// claim them. func modalityCapabilities(modalities map[string]bool) map[string]bool { capabilities := make(map[string]bool, len(modalities)) for modality, supported := range modalities { name := strings.ToLower(strings.TrimSpace(modality)) - if !supported || name == "" { + if !supported { continue } - capabilities[name] = true + switch name { + case "vision", "video", "audio": + capabilities[name] = true + } } if len(capabilities) == 0 { return nil diff --git a/internal/providers/llamacpp/models_test.go b/internal/providers/llamacpp/models_test.go index b69572142..9289f7acc 100644 --- a/internal/providers/llamacpp/models_test.go +++ b/internal/providers/llamacpp/models_test.go @@ -5,8 +5,12 @@ import ( "net/http" "net/http/httptest" "testing" + "time" + "github.com/enterpilot/gomodel/config" + "github.com/enterpilot/gomodel/internal/core" "github.com/enterpilot/gomodel/internal/llmclient" + "github.com/enterpilot/gomodel/internal/providers" ) // legacyListing is the /v1/models payload of builds whose meta object predates @@ -85,10 +89,21 @@ func TestListModels_SurfacesServerReportedMetadata(t *testing.T) { wantPropsFetched: true, }, { - name: "supported modalities become capabilities", + // A truncated or non-JSON body must not be read as a zero context. + name: "props answered with malformed json", listing: legacyListing, propsStatus: http.StatusOK, - props: `{"default_generation_settings":{"n_ctx":4096},"modalities":{"vision":true,"video":true,"audio":false}}`, + props: `{"default_generation_settings":{"n_ctx":`, + wantContextWindow: 131072, + wantPropsFetched: true, + }, + { + name: "supported modalities become capabilities", + listing: legacyListing, + propsStatus: http.StatusOK, + // "telepathy" stands in for a modality a future llama.cpp adds: it + // must not become a public capability on its own. + props: `{"default_generation_settings":{"n_ctx":4096},"modalities":{"vision":true,"video":true,"audio":false,"telepathy":true}}`, wantContextWindow: 4096, wantCapabilities: map[string]bool{"vision": true, "video": true}, wantPropsFetched: true, @@ -235,3 +250,64 @@ func TestListModels_LeavesMetadataUnsetWhenServerReportsNothing(t *testing.T) { t.Fatalf("model.Metadata = %+v, want nil so lower metadata layers still apply", resp.Data[0].Metadata) } } + +// TestListModels_FailingPropsLeavesNativeRoutesUsable pins the isolation of the +// optional /props call: it must not retry against the shared native-route +// budget, nor trip the circuit breaker those routes depend on. Sharing +// rootClient here cost four attempts per listing and locked /health out +// entirely after six discovery cycles. +func TestListModels_FailingPropsLeavesNativeRoutesUsable(t *testing.T) { + var propsAttempts, healthUpstream int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(`{"object":"list","data":[{"id":"m","object":"model","meta":{"n_ctx_train":8192}}]}`)) + case "/props": + propsAttempts++ + w.WriteHeader(http.StatusServiceUnavailable) // retryable status + _, _ = w.Write([]byte(`{"error":"unavailable"}`)) + case "/health": + healthUpstream++ + _, _ = w.Write([]byte(`{"status":"ok"}`)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + retry := config.DefaultRetryConfig() + retry.InitialBackoff = time.Millisecond // the attempt count is what matters + provider := New(providers.ProviderConfig{BaseURL: server.URL + "/v1"}, providers.ProviderOptions{ + Resilience: config.ResilienceConfig{ + Retry: retry, + CircuitBreaker: config.DefaultCircuitBreakerConfig(), + }, + }).(*Provider) + + const listings = 6 + for i := range listings { + resp, err := provider.ListModels(context.Background()) + if err != nil { + t.Fatalf("ListModels() #%d error = %v", i, err) + } + // The listing still succeeds on meta.n_ctx_train despite /props failing. + if resp.Data[0].Metadata == nil || *resp.Data[0].Metadata.ContextWindow != 8192 { + t.Fatalf("listing #%d lost its fallback context window", i) + } + } + if propsAttempts != listings { + t.Fatalf("props attempts = %d, want %d (one per listing, no retries)", propsAttempts, listings) + } + + if _, err := provider.Passthrough(context.Background(), &core.PassthroughRequest{ + Method: http.MethodGet, + Endpoint: "health", + Headers: http.Header{}, + }); err != nil { + t.Fatalf("native /health rejected after failing /props calls: %v", err) + } + if healthUpstream != 1 { + t.Fatalf("health upstream hits = %d, want 1", healthUpstream) + } +}